util/cbfstool: Add zstd support

This adds zstd support to cbfstool. The code is taken from zstd-1.5.7
with modifications:
- renaming bits.h to zstd_bits.h to avoid conflicts with coreboot's
  bits.h used on riscv
- renaming compiler.h to zstd_compiler.h to avoid conflicts with
  coreboot's compiler.h
- Dropped all streaming API functions
- Dropped multithreaded support, since it's now unused
- Dropped local DDict support

zstd offers similar compression ratios to LZMA, but a vastly fast
decompress speed. Typically zstd results in slightly larger binaries
than LZMA. Whether zstd should then be preferred over LZMA depends on
a few things:
- Caching: When loading from memory mapped boot devices, zstd will read
  the boot medium multiple times, while LZMA will not. If the memory
  mapped boot medium is not cached zstd results in much slower
  decompression.
- Boot medium speed: Often, but not always LZMA results in smaller
  binaries. If the boot medium is the bottleneck, than loading smaller
  binaries might actually be faster. On a fast boot medium (high spi
  freq, using quad/dual io), the performance benefits from zstd might be
  more substantial
- zstd decompression code has a much larger footprint than LZMA. If the
  stage (postcar) is loaded in uncached memory the size increase might
  slow things down.
  On QEMU Q35 postcar .text section size doubled, while heap section
  has growen by 50%.
- zstd uses a lot of .bss (CTX is about 32KiB large). This might not be
  available in some environments.

Orignal commit from 2022 was using zstd-1.5.2. Updated to zstd-1.5.7.

Change-Id: I34508268f8767008ef25cb9e466d201345881232
Signed-off-by: Arthur Heymans <arthur@aheymans.xyz>
Signed-off-by: Patrick Rudolph <patrick.rudolph@9elements.com>
Reviewed-on: https://review.coreboot.org/c/coreboot/+/69753
Reviewed-by: Julius Werner <jwerner@chromium.org>
Reviewed-by: Matt DeVillier <matt.devillier@gmail.com>
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>

util/cbfstool/Makefile.mk

@@ -12,6 +12,34 @@ compressionobj += lzma.o
 compressionobj += LzFind.o
 compressionobj += LzmaDec.o
 compressionobj += LzmaEnc.o
+# ZSTD
+# common
+compressionobj += debug.o
+compressionobj += entropy_common.o
+compressionobj += error_private.o
+compressionobj += fse_decompress.o
+compressionobj += zstd_common.o
+# compress
+compressionobj += fse_compress.o
+compressionobj += hist.o
+compressionobj += huf_compress.o
+compressionobj += zstd_compress.o
+compressionobj += zstd_compress_literals.o
+compressionobj += zstd_compress_sequences.o
+compressionobj += zstd_compress_superblock.o
+compressionobj += zstd_double_fast.o
+compressionobj += zstd_fast.o
+compressionobj += zstd_lazy.o
+compressionobj += zstd_ldm.o
+compressionobj += zstd_preSplit.o
+compressionobj += zstd_opt.o
+# xxhash.o Already included for LZ4
+# decompress
+compressionobj += huf_decompress.o
+compressionobj += zstd_ddict.o
+compressionobj += zstd_decompress_block.o
+compressionobj += zstd_decompress.o
+#compressionobj += huf_decomppress_amd64.o
 
 cbfsobj :=
 cbfsobj += cbfstool.o
@@ -132,6 +160,7 @@ TOOLCPPFLAGS += -I$(VBOOT_SOURCE)/host/lib/include
 # have right now.
 TOOLCPPFLAGS += -I$(top)/src
 TOOLCPPFLAGS += -I$(top)/src/vendorcode/intel/edk2/uefi_2.4/MdePkg/Include
+TOOLCPPFLAGS += -I$(top)/src/commonlib/bsd/zstd/
 
 TOOLLDFLAGS ?=
 
@@ -147,6 +176,7 @@ TOOLCFLAGS+=-std=c11
 endif
 
 LZ4CFLAGS ?= -Wno-strict-prototypes
+ZSTDCPPFLAGS ?= -DZSTD_DISABLE_ASM=1
 
 VBOOT_HOSTLIB = $(VBOOT_HOST_BUILD)/libvboot_host.a
 
@@ -189,6 +219,10 @@ $(objutil)/cbfstool/%.o: $(top)/src/commonlib/bsd/%.c
 	printf "    HOSTCC     $(subst $(objutil)/,,$(@))\n"
 	$(HOSTCC) $(TOOLCPPFLAGS) $(TOOLCFLAGS) $(HOSTCFLAGS) -c -o $@ $<
 
+$(objutil)/cbfstool/%.o: $(top)/src/commonlib/bsd/zstd/*/%.c
+	printf "    HOSTCC     $(subst $(objutil)/,,$(@))\n"
+	$(HOSTCC) $(TOOLCPPFLAGS) $(TOOLCFLAGS) $(HOSTCFLAGS) $(ZSTDCPPFLAGS) -c -o $@ $<
+
 $(objutil)/cbfstool/%.o: $(top)/util/cbfstool/lz4/lib/%.c
 	printf "    HOSTCC     $(subst $(objutil)/,,$(@))\n"
 	$(HOSTCC) $(TOOLCPPFLAGS) $(TOOLCFLAGS) $(HOSTCFLAGS) $(LZ4CFLAGS) -c -o $@ $<

util/cbfstool/cbfs.h

@@ -25,6 +25,7 @@ static const struct typedesc_t types_cbfs_compression[] = {
 	{CBFS_COMPRESS_NONE, "none"},
 	{CBFS_COMPRESS_LZMA, "LZMA"},
 	{CBFS_COMPRESS_LZ4, "LZ4"},
+	{CBFS_COMPRESS_ZSTD, "ZSTD"},
 	{0, NULL},
 };
 

util/cbfstool/compress.c

@@ -6,9 +6,11 @@
 #include <stdlib.h>
 #include "common.h"
 #include "lz4/lib/lz4frame.h"
+#include <zstd.h>
 #include <commonlib/bsd/compression.h>
 
-static int lz4_compress(char *in, int in_len, char *out, int *out_len)
+static int
+lz4_compress(char *in, int in_len, char *out, int *out_len)
 {
 	LZ4F_preferences_t prefs = {
 		.compressionLevel = 20,
@@ -53,6 +55,52 @@ static int lzma_decompress(char *in, int in_len, char *out, unused int out_len,
 {
 	return do_lzma_uncompress(out, out_len, in, in_len, actual_size);
 }
+
+static int zstd_compress(char *in, int in_len, char *out, int *out_len)
+{
+	size_t worst_size = ZSTD_compressBound(in_len);
+	size_t ret;
+
+	void *bounce = malloc(worst_size);
+	if (!bounce)
+		return -1;
+	ret = ZSTD_compress(bounce, worst_size, in, in_len, ZSTD_maxCLevel());
+	if (ZSTD_isError(ret)) {
+		ERROR("ZSTD_compress (v%s) returned %zu (%s)\n",
+		      ZSTD_versionString(), ret, ZSTD_getErrorName(ret));
+		free(bounce);
+		return -2;
+	}
+	if (ret >= (size_t)in_len) {
+		free(bounce);
+		return -3;
+	}
+	*out_len = (int)ret;
+
+	memcpy(out, bounce, *out_len);
+	free(bounce);
+	return 0;
+}
+
+static int zstd_decompress(char *in, int in_len, char *out, int out_len,
+			   size_t *actual_size)
+{
+	ZSTD_DCtx* dctx = ZSTD_createDCtx();
+	size_t ret = ZSTD_decompressDCtx(dctx, out, out_len, in, in_len);
+	ZSTD_freeDCtx(dctx);
+
+	if (ret == 0)
+		return -1;
+	if (ZSTD_isError(ret)) {
+		ERROR("ZSTD_decompress (v%s) returned %zu (%s)\n",
+		      ZSTD_versionString(), ret, ZSTD_getErrorName(ret));
+		return -2;
+	}
+	if (actual_size != NULL)
+		*actual_size = ret;
+	return 0;
+}
+
 static int none_compress(char *in, int in_len, char *out, int *out_len)
 {
 	memcpy(out, in, in_len);
@@ -82,6 +130,9 @@ comp_func_ptr compression_function(enum cbfs_compression algo)
 	case CBFS_COMPRESS_LZ4:
 		compress = lz4_compress;
 		break;
+	case CBFS_COMPRESS_ZSTD:
+		compress = zstd_compress;
+		break;
 	default:
 		ERROR("Unknown compression algorithm %d!\n", algo);
 		return NULL;
@@ -102,6 +153,9 @@ decomp_func_ptr decompression_function(enum cbfs_compression algo)
 	case CBFS_COMPRESS_LZ4:
 		decompress = lz4_decompress;
 		break;
+	case CBFS_COMPRESS_ZSTD:
+		decompress = zstd_decompress;
+		break;
 	default:
 		ERROR("Unknown compression algorithm %d!\n", algo);
 		return NULL;