svn commit: r350760 - in head: lib/libzstd sys/conf sys/contrib/zstd sys/contrib/zstd/contrib/largeNbDicts sys/contrib/zstd/contrib/seekable_format sys/contrib/zstd/contrib/seekable_format/examples...
Conrad Meyer
cem at FreeBSD.org
Thu Aug 8 16:54:26 UTC 2019
Author: cem
Date: Thu Aug 8 16:54:22 2019
New Revision: 350760
URL: https://svnweb.freebsd.org/changeset/base/350760
Log:
Update to Zstandard 1.4.2
The full release notes for 1.4.1 (skipped) and 1.4.2 can be found on Github:
https://github.com/facebook/zstd/releases/tag/v1.4.1
https://github.com/facebook/zstd/releases/tag/v1.4.2
These are mostly minor updates; 1.4.1 purportedly brings something like 7%
faster decompression speed.
Relnotes: yes
Added:
head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression_mem.c
- copied unchanged from r350755, vendor/zstd/dist/contrib/seekable_format/examples/seekable_decompression_mem.c
head/sys/contrib/zstd/lib/compress/zstd_compress_literals.c
- copied unchanged from r350755, vendor/zstd/dist/lib/compress/zstd_compress_literals.c
head/sys/contrib/zstd/lib/compress/zstd_compress_literals.h
- copied unchanged from r350755, vendor/zstd/dist/lib/compress/zstd_compress_literals.h
head/sys/contrib/zstd/lib/compress/zstd_compress_sequences.c
- copied unchanged from r350755, vendor/zstd/dist/lib/compress/zstd_compress_sequences.c
head/sys/contrib/zstd/lib/compress/zstd_compress_sequences.h
- copied unchanged from r350755, vendor/zstd/dist/lib/compress/zstd_compress_sequences.h
head/sys/contrib/zstd/tests/bigdict.c
- copied unchanged from r350755, vendor/zstd/dist/tests/bigdict.c
head/sys/contrib/zstd/tests/fuzz/simple_compress.c
- copied unchanged from r350755, vendor/zstd/dist/tests/fuzz/simple_compress.c
head/sys/contrib/zstd/tests/fuzz/zstd_frame_info.c
- copied unchanged from r350755, vendor/zstd/dist/tests/fuzz/zstd_frame_info.c
Deleted:
head/sys/contrib/zstd/tests/fuzz/default.options
Modified:
head/lib/libzstd/Makefile
head/sys/conf/files
head/sys/conf/kern.pre.mk
head/sys/contrib/zstd/CHANGELOG
head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c
head/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore
head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile
head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c
head/sys/contrib/zstd/doc/zstd_compression_format.md
head/sys/contrib/zstd/doc/zstd_manual.html
head/sys/contrib/zstd/examples/Makefile
head/sys/contrib/zstd/lib/Makefile
head/sys/contrib/zstd/lib/common/compiler.h
head/sys/contrib/zstd/lib/common/zstd_internal.h
head/sys/contrib/zstd/lib/compress/zstd_compress.c
head/sys/contrib/zstd/lib/compress/zstd_compress_internal.h
head/sys/contrib/zstd/lib/compress/zstd_double_fast.c
head/sys/contrib/zstd/lib/compress/zstd_fast.c
head/sys/contrib/zstd/lib/compress/zstd_lazy.c
head/sys/contrib/zstd/lib/compress/zstd_ldm.c
head/sys/contrib/zstd/lib/compress/zstd_opt.c
head/sys/contrib/zstd/lib/compress/zstdmt_compress.c
head/sys/contrib/zstd/lib/compress/zstdmt_compress.h
head/sys/contrib/zstd/lib/decompress/zstd_decompress.c
head/sys/contrib/zstd/lib/decompress/zstd_decompress_block.c
head/sys/contrib/zstd/lib/dictBuilder/cover.c
head/sys/contrib/zstd/lib/dictBuilder/cover.h
head/sys/contrib/zstd/lib/dictBuilder/fastcover.c
head/sys/contrib/zstd/lib/dictBuilder/zdict.c
head/sys/contrib/zstd/lib/dictBuilder/zdict.h
head/sys/contrib/zstd/lib/legacy/zstd_legacy.h
head/sys/contrib/zstd/lib/legacy/zstd_v01.c
head/sys/contrib/zstd/lib/legacy/zstd_v02.c
head/sys/contrib/zstd/lib/legacy/zstd_v03.c
head/sys/contrib/zstd/lib/legacy/zstd_v04.c
head/sys/contrib/zstd/lib/legacy/zstd_v05.c
head/sys/contrib/zstd/lib/legacy/zstd_v06.c
head/sys/contrib/zstd/lib/legacy/zstd_v07.c
head/sys/contrib/zstd/lib/zstd.h
head/sys/contrib/zstd/programs/.gitignore
head/sys/contrib/zstd/programs/README.md
head/sys/contrib/zstd/programs/benchfn.c
head/sys/contrib/zstd/programs/fileio.c
head/sys/contrib/zstd/programs/fileio.h
head/sys/contrib/zstd/programs/util.c
head/sys/contrib/zstd/programs/zstd.1
head/sys/contrib/zstd/programs/zstd.1.md
head/sys/contrib/zstd/programs/zstdcli.c
head/sys/contrib/zstd/programs/zstdgrep
head/sys/contrib/zstd/programs/zstdgrep.1
head/sys/contrib/zstd/programs/zstdless.1
head/sys/contrib/zstd/tests/.gitignore
head/sys/contrib/zstd/tests/Makefile
head/sys/contrib/zstd/tests/decodecorpus.c
head/sys/contrib/zstd/tests/fullbench.c
head/sys/contrib/zstd/tests/fuzz/Makefile
head/sys/contrib/zstd/tests/fuzz/dictionary_decompress.c
head/sys/contrib/zstd/tests/fuzz/fuzz.py
head/sys/contrib/zstd/tests/fuzz/simple_decompress.c
head/sys/contrib/zstd/tests/fuzzer.c
head/sys/contrib/zstd/tests/paramgrill.c
head/sys/contrib/zstd/tests/playTests.sh
head/sys/contrib/zstd/tests/poolTests.c
head/sys/contrib/zstd/tests/regression/results.csv
head/sys/contrib/zstd/tests/zstreamtest.c
head/sys/contrib/zstd/zlibWrapper/.gitignore
Directory Properties:
head/sys/contrib/zstd/ (props changed)
Modified: head/lib/libzstd/Makefile
==============================================================================
--- head/lib/libzstd/Makefile Thu Aug 8 16:51:49 2019 (r350759)
+++ head/lib/libzstd/Makefile Thu Aug 8 16:54:22 2019 (r350760)
@@ -11,6 +11,8 @@ SRCS= entropy_common.c \
fse_compress.c \
huf_compress.c \
zstd_compress.c \
+ zstd_compress_literals.c \
+ zstd_compress_sequences.c \
zstdmt_compress.c \
huf_decompress.c \
zstd_ddict.c \
@@ -43,7 +45,20 @@ ZSTDDIR= ${SRCTOP}/sys/contrib/zstd
${ZSTDDIR}/lib/decompress ${ZSTDDIR}/lib/deprecated \
${ZSTDDIR}/lib/dictBuilder ${ZSTDDIR}/lib
+.include <bsd.compiler.mk>
+
+# https://github.com/facebook/zstd/commit/812e8f2a [zstd 1.4.1]
+# "Note that [GCC] autovectorization still does not do a good job on the
+# optimized version, so it's turned off via attribute and flag. I found
+# that neither attribute nor command-line flag were entirely successful in
+# turning off vectorization, which is why there were both."
+.if ${COMPILER_TYPE} == "gcc"
+CFLAGS.zstd_decompress_block.c+= -fno-tree-vectorize
+.endif
+
# Work around for LLVM bug 35023, https://bugs.llvm.org/show_bug.cgi?id=35023
+# Fixed in https://reviews.llvm.org/rL349935 ; not sure if we have that version
+# in our LLVM or not yet.
.if ${MACHINE_ARCH:Marm*} != ""
CFLAGS.zstd_compress.c+= -O0
.endif
Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/conf/files Thu Aug 8 16:54:22 2019 (r350760)
@@ -648,6 +648,8 @@ contrib/zstd/lib/common/entropy_common.c optional zstd
contrib/zstd/lib/common/error_private.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/common/xxhash.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/compress/zstd_compress.c optional zstdio compile-with ${ZSTD_C}
+contrib/zstd/lib/compress/zstd_compress_literals.c optional zstdio compile-with ${ZSTD_C}
+contrib/zstd/lib/compress/zstd_compress_sequences.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/compress/fse_compress.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/compress/hist.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/compress/huf_compress.c optional zstdio compile-with ${ZSTD_C}
@@ -658,7 +660,9 @@ contrib/zstd/lib/compress/zstd_ldm.c optional zstdio
contrib/zstd/lib/compress/zstd_opt.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/decompress/zstd_ddict.c optional zstdio compile-with ${ZSTD_C}
contrib/zstd/lib/decompress/zstd_decompress.c optional zstdio compile-with ${ZSTD_C}
-contrib/zstd/lib/decompress/zstd_decompress_block.c optional zstdio compile-with ${ZSTD_C}
+# See comment in sys/conf/kern.pre.mk
+contrib/zstd/lib/decompress/zstd_decompress_block.c optional zstdio \
+ compile-with "${ZSTD_C} ${ZSTD_DECOMPRESS_BLOCK_FLAGS}"
contrib/zstd/lib/decompress/huf_decompress.c optional zstdio compile-with ${ZSTD_C}
# Blake 2
contrib/libb2/blake2b-ref.c optional crypto | ipsec | ipsec_support \
Modified: head/sys/conf/kern.pre.mk
==============================================================================
--- head/sys/conf/kern.pre.mk Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/conf/kern.pre.mk Thu Aug 8 16:54:22 2019 (r350760)
@@ -175,6 +175,14 @@ NORMAL_FWO= ${LD} -b binary --no-warn-mismatch -d -war
# for ZSTD in the kernel (include zstd/lib/freebsd before other CFLAGS)
ZSTD_C= ${CC} -c -DZSTD_HEAPMODE=1 -I$S/contrib/zstd/lib/freebsd ${CFLAGS} -I$S/contrib/zstd/lib -I$S/contrib/zstd/lib/common ${WERROR} -Wno-inline -Wno-missing-prototypes ${PROF} -U__BMI__ ${.IMPSRC}
+# https://github.com/facebook/zstd/commit/812e8f2a [zstd 1.4.1]
+# "Note that [GCC] autovectorization still does not do a good job on the
+# optimized version, so it's turned off via attribute and flag. I found
+# that neither attribute nor command-line flag were entirely successful in
+# turning off vectorization, which is why there were both."
+.if ${COMPILER_TYPE} == "gcc"
+ZSTD_DECOMPRESS_BLOCK_FLAGS= -fno-tree-vectorize
+.endif
# Common for dtrace / zfs
CDDL_CFLAGS= -DFREEBSD_NAMECACHE -nostdinc -I$S/cddl/compat/opensolaris -I$S/cddl/contrib/opensolaris/uts/common -I$S -I$S/cddl/contrib/opensolaris/common ${CFLAGS} -Wno-unknown-pragmas -Wno-missing-prototypes -Wno-undef -Wno-strict-prototypes -Wno-cast-qual -Wno-parentheses -Wno-redundant-decls -Wno-missing-braces -Wno-uninitialized -Wno-unused -Wno-inline -Wno-switch -Wno-pointer-arith -Wno-unknown-pragmas
Modified: head/sys/contrib/zstd/CHANGELOG
==============================================================================
--- head/sys/contrib/zstd/CHANGELOG Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/CHANGELOG Thu Aug 8 16:54:22 2019 (r350760)
@@ -1,3 +1,39 @@
+v1.4.2
+bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696)
+bug: Fix seekable decompression in-memory API by @iburinoc (#1695)
+misc: Validate blocks are smaller than size limit by @vivekmg (#1685)
+misc: Restructure source files by @ephiepark (#1679)
+
+v1.4.1
+bug: Fix data corruption in niche use cases by @terrelln (#1659)
+bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595)
+bug: Fix out of bounds read by @terrelln (#1590)
+perf: Improve decode speed by ~7% @mgrice (#1668)
+perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681)
+perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658)
+perf: Improve compression ratio for small windowLog by @cyan4973 (#1624)
+perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635)
+api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656)
+cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640)
+cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631)
+cli: Restrict read permissions on destination files by @chungy (#1644)
+cli: zstdgrep: handle -f flag by @felixhandte (#1618)
+cli: zstdcat: follow symlinks by @vejnar (#1604)
+doc: Remove extra size limit on compressed blocks by @felixhandte (#1689)
+doc: Fix typo by @yk-tanigawa (#1633)
+doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629)
+build: CMake: support building with LZ4 @leeyoung624 (#1626)
+build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647)
+build: CMake: respect existing uninstall target by @j301scott (#1619)
+build: Make: skip multithread tests when built without support by @michaelforney (#1620)
+build: Make: Fix examples/ test target by @sjnam (#1603)
+build: Meson: rename options out of deprecated namespace by @lzutao (#1665)
+build: Meson: fix build by @lzutao (#1602)
+build: Visual Studio: don't export symbols in static lib by @scharan (#1650)
+build: Visual Studio: fix linking by @absotively (#1639)
+build: Fix MinGW-W64 build by @myzhang1029 (#1600)
+misc: Expand decodecorpus coverage by @ephiepark (#1664)
+
v1.4.0
perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
api: Move the advanced API, including all functions in the staging section, to the stable section
Modified: head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c
==============================================================================
--- head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c Thu Aug 8 16:54:22 2019 (r350760)
@@ -559,7 +559,7 @@ static int benchMem(slice_collection_t dstBlocks,
CONTROL(BMK_isSuccessful_runOutcome(outcome));
BMK_runTime_t const result = BMK_extract_runTime(outcome);
- U64 const dTime_ns = result.nanoSecPerRun;
+ double const dTime_ns = result.nanoSecPerRun;
double const dTime_sec = (double)dTime_ns / 1000000000;
size_t const srcSize = result.sumOfReturn;
double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB);
Modified: head/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore
==============================================================================
--- head/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore Thu Aug 8 16:54:22 2019 (r350760)
@@ -1,4 +1,5 @@
seekable_compression
seekable_decompression
+seekable_decompression_mem
parallel_processing
parallel_compression
Modified: head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile
==============================================================================
--- head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile Thu Aug 8 16:54:22 2019 (r350760)
@@ -24,7 +24,8 @@ SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_dec
default: all
-all: seekable_compression seekable_decompression parallel_processing
+all: seekable_compression seekable_decompression seekable_decompression_mem \
+ parallel_processing
$(ZSTDLIB):
make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME)
@@ -35,6 +36,9 @@ seekable_compression : seekable_compression.c $(SEEKAB
seekable_decompression : seekable_decompression.c $(SEEKABLE_OBJS)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+seekable_decompression_mem : seekable_decompression_mem.c $(SEEKABLE_OBJS)
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
parallel_processing : parallel_processing.c $(SEEKABLE_OBJS)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread
@@ -44,5 +48,6 @@ parallel_compression : parallel_compression.c $(SEEKAB
clean:
@rm -f core *.o tmp* result* *.zst \
seekable_compression seekable_decompression \
+ seekable_decompression_mem \
parallel_processing parallel_compression
@echo Cleaning completed
Copied: head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression_mem.c (from r350755, vendor/zstd/dist/contrib/seekable_format/examples/seekable_decompression_mem.c)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression_mem.c Thu Aug 8 16:54:22 2019 (r350760, copy of r350755, vendor/zstd/dist/contrib/seekable_format/examples/seekable_decompression_mem.c)
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+
+#include <stdlib.h> // malloc, exit
+#include <stdio.h> // fprintf, perror, feof
+#include <string.h> // strerror
+#include <errno.h> // errno
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h> // presumes zstd library is installed
+#include <zstd_errors.h>
+
+#include "zstd_seekable.h"
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+#define MAX_FILE_SIZE (8 * 1024 * 1024)
+
+static void* malloc_orDie(size_t size)
+{
+ void* const buff = malloc(size);
+ if (buff) return buff;
+ /* error */
+ perror("malloc");
+ exit(1);
+}
+
+static void* realloc_orDie(void* ptr, size_t size)
+{
+ ptr = realloc(ptr, size);
+ if (ptr) return ptr;
+ /* error */
+ perror("realloc");
+ exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+ FILE* const inFile = fopen(filename, instruction);
+ if (inFile) return inFile;
+ /* error */
+ perror(filename);
+ exit(3);
+}
+
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+ size_t const readSize = fread(buffer, 1, sizeToRead, file);
+ if (readSize == sizeToRead) return readSize; /* good */
+ if (feof(file)) return readSize; /* good, reached end of file */
+ /* error */
+ perror("fread");
+ exit(4);
+}
+
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+ size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+ if (writtenSize == sizeToWrite) return sizeToWrite; /* good */
+ /* error */
+ perror("fwrite");
+ exit(5);
+}
+
+static size_t fclose_orDie(FILE* file)
+{
+ if (!fclose(file)) return 0;
+ /* error */
+ perror("fclose");
+ exit(6);
+}
+
+static void fseek_orDie(FILE* file, long int offset, int origin) {
+ if (!fseek(file, offset, origin)) {
+ if (!fflush(file)) return;
+ }
+ /* error */
+ perror("fseek");
+ exit(7);
+}
+
+
+static void decompressFile_orDie(const char* fname, off_t startOffset, off_t endOffset)
+{
+ FILE* const fin = fopen_orDie(fname, "rb");
+ FILE* const fout = stdout;
+ // Just for demo purposes, assume file is <= MAX_FILE_SIZE
+ void* const buffIn = malloc_orDie(MAX_FILE_SIZE);
+ size_t const inSize = fread_orDie(buffIn, MAX_FILE_SIZE, fin);
+ size_t const buffOutSize = ZSTD_DStreamOutSize(); /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+ void* const buffOut = malloc_orDie(buffOutSize);
+
+ ZSTD_seekable* const seekable = ZSTD_seekable_create();
+ if (seekable==NULL) { fprintf(stderr, "ZSTD_seekable_create() error \n"); exit(10); }
+
+ size_t const initResult = ZSTD_seekable_initBuff(seekable, buffIn, inSize);
+ if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+
+ while (startOffset < endOffset) {
+ size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset);
+
+ if (ZSTD_isError(result)) {
+ fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n",
+ ZSTD_getErrorName(result));
+ exit(12);
+ }
+ fwrite_orDie(buffOut, result, fout);
+ startOffset += result;
+ }
+
+ ZSTD_seekable_free(seekable);
+ fclose_orDie(fin);
+ fclose_orDie(fout);
+ free(buffIn);
+ free(buffOut);
+}
+
+
+int main(int argc, const char** argv)
+{
+ const char* const exeName = argv[0];
+
+ if (argc!=4) {
+ fprintf(stderr, "wrong arguments\n");
+ fprintf(stderr, "usage:\n");
+ fprintf(stderr, "%s FILE START END\n", exeName);
+ return 1;
+ }
+
+ {
+ const char* const inFilename = argv[1];
+ off_t const startOffset = atoll(argv[2]);
+ off_t const endOffset = atoll(argv[3]);
+ decompressFile_orDie(inFilename, startOffset, endOffset);
+ }
+
+ return 0;
+}
Modified: head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c
==============================================================================
--- head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c Thu Aug 8 16:54:22 2019 (r350760)
@@ -106,7 +106,7 @@ typedef struct {
static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n)
{
buffWrapper_t* buff = (buffWrapper_t*) opaque;
- if (buff->size + n > buff->pos) return -1;
+ if (buff->pos + n > buff->size) return -1;
memcpy(buffer, (const BYTE*)buff->ptr + buff->pos, n);
buff->pos += n;
return 0;
@@ -124,7 +124,7 @@ static int ZSTD_seekable_seek_buff(void* opaque, long
newOffset = (unsigned long long)buff->pos + offset;
break;
case SEEK_END:
- newOffset = (unsigned long long)buff->size - offset;
+ newOffset = (unsigned long long)buff->size + offset;
break;
default:
assert(0); /* not possible */
Modified: head/sys/contrib/zstd/doc/zstd_compression_format.md
==============================================================================
--- head/sys/contrib/zstd/doc/zstd_compression_format.md Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/doc/zstd_compression_format.md Thu Aug 8 16:54:22 2019 (r350760)
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
### Version
-0.3.1 (25/10/18)
+0.3.2 (17/07/19)
Introduction
@@ -390,9 +390,7 @@ A block can contain any number of bytes (even zero), u
- Window_Size
- 128 KB
-A `Compressed_Block` has the extra restriction that `Block_Size` is always
-strictly less than the decompressed size.
-If this condition cannot be respected,
+If this condition cannot be respected when generating a `Compressed_Block`,
the block must be sent uncompressed instead (`Raw_Block`).
@@ -1655,6 +1653,7 @@ or at least provide a meaningful error code explaining
Version changes
---------------
+- 0.3.2 : remove additional block size restriction on compressed blocks
- 0.3.1 : minor clarification regarding offset history update rules
- 0.3.0 : minor edits to match RFC8478
- 0.2.9 : clarifications for huffman weights direct representation, by Ulrich Kunitz
Modified: head/sys/contrib/zstd/doc/zstd_manual.html
==============================================================================
--- head/sys/contrib/zstd/doc/zstd_manual.html Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/doc/zstd_manual.html Thu Aug 8 16:54:22 2019 (r350760)
@@ -1,46 +1,36 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.4.0 Manual</title>
+<title>zstd 1.4.2 Manual</title>
</head>
<body>
-<h1>zstd 1.4.0 Manual</h1>
+<h1>zstd 1.4.2 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
<li><a href="#Chapter1">Introduction</a></li>
<li><a href="#Chapter2">Version</a></li>
-<li><a href="#Chapter3">Default constant</a></li>
-<li><a href="#Chapter4">Constants</a></li>
-<li><a href="#Chapter5">Simple API</a></li>
-<li><a href="#Chapter6">Explicit context</a></li>
-<li><a href="#Chapter7">Advanced compression API</a></li>
-<li><a href="#Chapter8">Advanced decompression API</a></li>
-<li><a href="#Chapter9">Streaming</a></li>
-<li><a href="#Chapter10">Streaming compression - HowTo</a></li>
-<li><a href="#Chapter11">This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and</a></li>
-<li><a href="#Chapter12">Equivalent to:</a></li>
-<li><a href="#Chapter13">Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).</a></li>
-<li><a href="#Chapter14">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).</a></li>
-<li><a href="#Chapter15">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).</a></li>
-<li><a href="#Chapter16">Streaming decompression - HowTo</a></li>
-<li><a href="#Chapter17">Simple dictionary API</a></li>
-<li><a href="#Chapter18">Bulk processing dictionary API</a></li>
-<li><a href="#Chapter19">Dictionary helper functions</a></li>
-<li><a href="#Chapter20">Advanced dictionary and prefix API</a></li>
-<li><a href="#Chapter21">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
-<li><a href="#Chapter22">experimental API (static linking only)</a></li>
-<li><a href="#Chapter23">Frame size functions</a></li>
-<li><a href="#Chapter24">ZSTD_decompressBound() :</a></li>
-<li><a href="#Chapter25">Memory management</a></li>
-<li><a href="#Chapter26">Advanced compression functions</a></li>
-<li><a href="#Chapter27">Advanced decompression functions</a></li>
-<li><a href="#Chapter28">Advanced streaming functions</a></li>
-<li><a href="#Chapter29">Buffer-less and synchronous inner streaming functions</a></li>
-<li><a href="#Chapter30">Buffer-less streaming compression (synchronous mode)</a></li>
-<li><a href="#Chapter31">Buffer-less streaming decompression (synchronous mode)</a></li>
-<li><a href="#Chapter32">ZSTD_getFrameHeader() :</a></li>
-<li><a href="#Chapter33">Block level API</a></li>
+<li><a href="#Chapter3">Simple API</a></li>
+<li><a href="#Chapter4">Explicit context</a></li>
+<li><a href="#Chapter5">Advanced compression API</a></li>
+<li><a href="#Chapter6">Advanced decompression API</a></li>
+<li><a href="#Chapter7">Streaming</a></li>
+<li><a href="#Chapter8">Streaming compression - HowTo</a></li>
+<li><a href="#Chapter9">Streaming decompression - HowTo</a></li>
+<li><a href="#Chapter10">Simple dictionary API</a></li>
+<li><a href="#Chapter11">Bulk processing dictionary API</a></li>
+<li><a href="#Chapter12">Dictionary helper functions</a></li>
+<li><a href="#Chapter13">Advanced dictionary and prefix API</a></li>
+<li><a href="#Chapter14">experimental API (static linking only)</a></li>
+<li><a href="#Chapter15">Frame size functions</a></li>
+<li><a href="#Chapter16">Memory management</a></li>
+<li><a href="#Chapter17">Advanced compression functions</a></li>
+<li><a href="#Chapter18">Advanced decompression functions</a></li>
+<li><a href="#Chapter19">Advanced streaming functions</a></li>
+<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li>
+<li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li>
+<li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li>
+<li><a href="#Chapter23">Block level API</a></li>
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -78,12 +68,8 @@
<pre><b>unsigned ZSTD_versionNumber(void); </b>/**< to check runtime library version */<b>
</b></pre><BR>
-<a name="Chapter3"></a><h2>Default constant</h2><pre></pre>
+<a name="Chapter3"></a><h2>Simple API</h2><pre></pre>
-<a name="Chapter4"></a><h2>Constants</h2><pre></pre>
-
-<a name="Chapter5"></a><h2>Simple API</h2><pre></pre>
-
<pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
@@ -152,12 +138,17 @@ const char* ZSTD_getErrorName(size_t code); </b>/*
int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b>
int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
</pre></b><BR>
-<a name="Chapter6"></a><h2>Explicit context</h2><pre></pre>
+<a name="Chapter4"></a><h2>Explicit context</h2><pre></pre>
<h3>Compression context</h3><pre> When compressing many times,
- it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+ it is recommended to allocate a context just once,
+ and re-use it for each successive compression operation.
This will make workload friendlier for system's memory.
- Use one context per thread for parallel execution in multi-threaded environments.
+ Note : re-using context is just a speed / resource optimization.
+ It doesn't change the compression ratio, which remains identical.
+ Note 2 : In multi-threaded environments,
+ use one different context per thread for parallel execution.
+
</pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx;
ZSTD_CCtx* ZSTD_createCCtx(void);
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
@@ -189,7 +180,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</p></pre><BR>
-<a name="Chapter7"></a><h2>Advanced compression API</h2><pre></pre>
+<a name="Chapter5"></a><h2>Advanced compression API</h2><pre></pre>
<pre><b>typedef enum { ZSTD_fast=1,
ZSTD_dfast=2,
@@ -332,6 +323,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* ZSTD_c_forceMaxWindow
* ZSTD_c_forceAttachDict
* ZSTD_c_literalCompressionMode
+ * ZSTD_c_targetCBlockSize
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change.
@@ -341,6 +333,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_c_experimentalParam3=1000,
ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002,
+ ZSTD_c_experimentalParam6=1003,
} ZSTD_cParameter;
</b></pre><BR>
<pre><b>typedef struct {
@@ -424,7 +417,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</p></pre><BR>
-<a name="Chapter8"></a><h2>Advanced decompression API</h2><pre></pre>
+<a name="Chapter6"></a><h2>Advanced decompression API</h2><pre></pre>
<pre><b>typedef enum {
@@ -472,7 +465,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</p></pre><BR>
-<a name="Chapter9"></a><h2>Streaming</h2><pre></pre>
+<a name="Chapter7"></a><h2>Streaming</h2><pre></pre>
<pre><b>typedef struct ZSTD_inBuffer_s {
const void* src; </b>/**< start of input buffer */<b>
@@ -486,7 +479,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
size_t pos; </b>/**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */<b>
} ZSTD_outBuffer;
</b></pre><BR>
-<a name="Chapter10"></a><h2>Streaming compression - HowTo</h2><pre>
+<a name="Chapter8"></a><h2>Streaming compression - HowTo</h2><pre>
A ZSTD_CStream object is required to track streaming operation.
Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
@@ -592,31 +585,28 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
<pre><b>size_t ZSTD_CStreamInSize(void); </b>/**< recommended size for input buffer */<b>
</b></pre><BR>
-<pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b>
+<pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */<b>
</b></pre><BR>
-<a name="Chapter11"></a><h2>This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and</h2><pre> ZSTD_compressStream2(). It is redundent, but is still fully supported.
- Advanced parameters and dictionary compression can only be used through the
- new API.
-<BR></pre>
-
-<a name="Chapter12"></a><h2>Equivalent to:</h2><pre>
+<pre><b>size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+</b>/*!<b>
+ * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
+ * NOTE: The return value is different. ZSTD_compressStream() returns a hint for
+ * the next read size (if non-zero and not an error). ZSTD_compressStream2()
+ * returns the minimum nb of bytes left to flush (if non-zero and not an error).
+ */
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+</b>/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */<b>
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+</b>/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */<b>
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+</b><p>
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
-<BR></pre>
+</p></pre><BR>
-<a name="Chapter13"></a><h2>Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).</h2><pre> NOTE: The return value is different. ZSTD_compressStream() returns a hint for
- the next read size (if non-zero and not an error). ZSTD_compressStream2()
- returns the number of bytes left to flush (if non-zero and not an error).
-
-<BR></pre>
-
-<a name="Chapter14"></a><h2>Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).</h2><pre></pre>
-
-<a name="Chapter15"></a><h2>Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).</h2><pre></pre>
-
-<a name="Chapter16"></a><h2>Streaming decompression - HowTo</h2><pre>
+<a name="Chapter9"></a><h2>Streaming decompression - HowTo</h2><pre>
A ZSTD_DStream object is required to track streaming operations.
Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
ZSTD_DStream objects can be re-used multiple times.
@@ -647,14 +637,12 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
<h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void);
size_t ZSTD_freeDStream(ZSTD_DStream* zds);
</pre></b><BR>
-<h3>Streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_initDStream(ZSTD_DStream* zds);
-size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-</pre></b><BR>
+<h3>Streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR>
<pre><b>size_t ZSTD_DStreamInSize(void); </b>/*!< recommended size for input buffer */<b>
</b></pre><BR>
<pre><b>size_t ZSTD_DStreamOutSize(void); </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b>
</b></pre><BR>
-<a name="Chapter17"></a><h2>Simple dictionary API</h2><pre></pre>
+<a name="Chapter10"></a><h2>Simple dictionary API</h2><pre></pre>
<pre><b>size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
void* dst, size_t dstCapacity,
@@ -680,7 +668,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_o
Note : When `dict == NULL || dictSize < 8` no dictionary is used.
</p></pre><BR>
-<a name="Chapter18"></a><h2>Bulk processing dictionary API</h2><pre></pre>
+<a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre>
<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
int compressionLevel);
@@ -723,7 +711,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_o
Recommended when same dictionary is used multiple times.
</p></pre><BR>
-<a name="Chapter19"></a><h2>Dictionary helper functions</h2><pre></pre>
+<a name="Chapter12"></a><h2>Dictionary helper functions</h2><pre></pre>
<pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
</b><p> Provides the dictID stored within dictionary.
@@ -749,7 +737,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_o
When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code.
</p></pre><BR>
-<a name="Chapter20"></a><h2>Advanced dictionary and prefix API</h2><pre>
+<a name="Chapter13"></a><h2>Advanced dictionary and prefix API</h2><pre>
This API allows dictionaries to be used with ZSTD_compress2(),
ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and
only reset with the context is reset with ZSTD_reset_parameters or
@@ -867,15 +855,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
Note that object memory usage can evolve (increase or decrease) over time.
</p></pre><BR>
-<a name="Chapter21"></a><h2>ADVANCED AND EXPERIMENTAL FUNCTIONS</h2><pre>
- The definitions in the following section are considered experimental.
- They are provided for advanced scenarios.
- They should never be used with a dynamic library, as prototypes may change in the future.
- Use them only in association with static linking.
-
-<BR></pre>
-
-<a name="Chapter22"></a><h2>experimental API (static linking only)</h2><pre>
+<a name="Chapter14"></a><h2>experimental API (static linking only)</h2><pre>
The following symbols and constants
are not planned to join "stable API" status in the near future.
They can still change in future versions.
@@ -973,7 +953,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
ZSTD_lcm_uncompressed = 2, </b>/**< Always emit uncompressed literals. */<b>
} ZSTD_literalCompressionMode_e;
</b></pre><BR>
-<a name="Chapter23"></a><h2>Frame size functions</h2><pre></pre>
+<a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre>
<pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
</b><p> `src` should point to the start of a series of ZSTD encoded and/or skippable frames
@@ -998,7 +978,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
however it does mean that all frame data must be present and valid.
</p></pre><BR>
-<a name="Chapter24"></a><h2>ZSTD_decompressBound() :</h2><pre> `src` should point to the start of a series of ZSTD encoded and/or skippable frames
+<pre><b>unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
+</b><p> `src` should point to the start of a series of ZSTD encoded and/or skippable frames
`srcSize` must be the _exact_ size of this series
(i.e. there should be a frame boundary at `src + srcSize`)
@return : - upper-bound for the decompressed size of all data in all successive frames
@@ -1010,7 +991,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
upper-bound = # blocks * min(128 KB, Window_Size)
-<BR></pre>
+</p></pre><BR>
<pre><b>size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
</b><p> srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
@@ -1018,7 +999,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
or an error code (if srcSize is too small)
</p></pre><BR>
-<a name="Chapter25"></a><h2>Memory management</h2><pre></pre>
+<a name="Chapter16"></a><h2>Memory management</h2><pre></pre>
<pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel);
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -1098,7 +1079,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL,
</p></pre><BR>
-<a name="Chapter26"></a><h2>Advanced compression functions</h2><pre></pre>
+<a name="Chapter17"></a><h2>Advanced compression functions</h2><pre></pre>
<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
</b><p> Create a digested dictionary for compression
@@ -1243,7 +1224,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</p></pre><BR>
-<a name="Chapter27"></a><h2>Advanced decompression functions</h2><pre></pre>
+<a name="Chapter18"></a><h2>Advanced decompression functions</h2><pre></pre>
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -1305,7 +1286,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
</p></pre><BR>
-<a name="Chapter28"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API.
+<a name="Chapter19"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API.
Once Advanced API reaches "stable" status,
redundant functions will be deprecated, and then at some point removed.
<BR></pre>
@@ -1407,18 +1388,41 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStre
</p></pre><BR>
-<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: no dictionary will be used if dict == NULL or dictSize < 8 */<b>
-size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b>
-size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
+<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre></b>/**<b>
+ * This function is deprecated, and is equivalent to:
+ *
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ * ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
+ *
+ * note: no dictionary will be used if dict == NULL or dictSize < 8
+ */
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+</b>/**<b>
+ * This function is deprecated, and is equivalent to:
+ *
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ * ZSTD_DCtx_refDDict(zds, ddict);
+ *
+ * note : ddict is referenced, it must outlive decompression session
+ */
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
+</b>/**<b>
+ * This function is deprecated, and is equivalent to:
+ *
+ * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
+ *
+ * re-use decompression parameters from previous init; saves dictionary loading
+ */
+size_t ZSTD_resetDStream(ZSTD_DStream* zds);
</pre></b><BR>
-<a name="Chapter29"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
+<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
But it's also a complex one, with several restrictions, documented below.
Prefer normal streaming API for an easier experience.
<BR></pre>
-<a name="Chapter30"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
+<a name="Chapter21"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -1454,7 +1458,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx,
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
</pre></b><BR>
-<a name="Chapter31"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
+<a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times.
@@ -1536,23 +1540,21 @@ typedef struct {
unsigned checksumFlag;
} ZSTD_frameHeader;
</pre></b><BR>
-<a name="Chapter32"></a><h2>ZSTD_getFrameHeader() :</h2><pre> decode Frame Header, or requires larger `srcSize`.
+<pre><b>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b>
+</b>/*! ZSTD_getFrameHeader_advanced() :<b>
+ * same as ZSTD_getFrameHeader(),
+ * with added capability to select a format (like ZSTD_f_zstd1_magicless) */
+size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
+</b><p> decode Frame Header, or requires larger `srcSize`.
@return : 0, `zfhPtr` is correctly filled,
>0, `srcSize` is too small, value is wanted `srcSize` amount,
or an error code, which can be tested using ZSTD_isError()
-<BR></pre>
-
-<pre><b>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b>
-</b></pre><BR>
-<pre><b>size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
-size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b>
-</b><p> same as ZSTD_getFrameHeader(),
- with added capability to select a format (like ZSTD_f_zstd1_magicless)
</p></pre><BR>
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
</b></pre><BR>
-<a name="Chapter33"></a><h2>Block level API</h2><pre></pre>
+<a name="Chapter23"></a><h2>Block level API</h2><pre></pre>
<pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
User will have to take in charge required information to regenerate data, such as compressed and content sizes.
Modified: head/sys/contrib/zstd/examples/Makefile
==============================================================================
--- head/sys/contrib/zstd/examples/Makefile Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/examples/Makefile Thu Aug 8 16:54:22 2019 (r350760)
@@ -77,7 +77,6 @@ test: all
@echo -- Edge cases detection
! ./streaming_decompression tmp # invalid input, must fail
! ./simple_decompression tmp # invalid input, must fail
- ! ./simple_decompression tmp.zst # unknown input size, must fail
touch tmpNull # create 0-size file
./simple_compression tmpNull
./simple_decompression tmpNull.zst # 0-size frame : must work
Modified: head/sys/contrib/zstd/lib/Makefile
==============================================================================
--- head/sys/contrib/zstd/lib/Makefile Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/lib/Makefile Thu Aug 8 16:54:22 2019 (r350760)
@@ -17,6 +17,7 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
LIBVER := $(shell echo $(LIBVER_SCRIPT))
VERSION?= $(LIBVER)
+CCVER := $(shell $(CC) --version)
CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
ifeq ($(OS),Windows_NT) # MinGW assumed
@@ -44,6 +45,10 @@ ZSTDDECOMP_FILES := $(sort $(wildcard decompress/*.c))
ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c))
ZDEPR_FILES := $(sort $(wildcard deprecated/*.c))
ZSTD_FILES := $(ZSTDCOMMON_FILES)
+
+ifeq ($(findstring GCC,$(CCVER)),GCC)
+decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize
+endif
ZSTD_LEGACY_SUPPORT ?= 5
ZSTD_LIB_COMPRESSION ?= 1
Modified: head/sys/contrib/zstd/lib/common/compiler.h
==============================================================================
--- head/sys/contrib/zstd/lib/common/compiler.h Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/lib/common/compiler.h Thu Aug 8 16:54:22 2019 (r350760)
@@ -127,6 +127,13 @@
} \
}
+/* vectorization */
+#if !defined(__clang__) && defined(__GNUC__)
+# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
+#else
+# define DONT_VECTORIZE
+#endif
+
/* disable warnings */
#ifdef _MSC_VER /* Visual Studio */
# include <intrin.h> /* For Visual 2005 */
Modified: head/sys/contrib/zstd/lib/common/zstd_internal.h
==============================================================================
--- head/sys/contrib/zstd/lib/common/zstd_internal.h Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/lib/common/zstd_internal.h Thu Aug 8 16:54:22 2019 (r350760)
@@ -34,7 +34,6 @@
#endif
#include "xxhash.h" /* XXH_reset, update, digest */
-
#if defined (__cplusplus)
extern "C" {
#endif
@@ -193,19 +192,72 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG
* Shared functions to include for inlining
*********************************************/
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
+
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
+static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); }
+#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
+#define WILDCOPY_OVERLENGTH 8
+#define VECLEN 16
+
+typedef enum {
+ ZSTD_no_overlap,
+ ZSTD_overlap_src_before_dst,
+ /* ZSTD_overlap_dst_before_src, */
+} ZSTD_overlap_e;
+
/*! ZSTD_wildcopy() :
* custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */
-#define WILDCOPY_OVERLENGTH 8
-MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
+MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE
+void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype)
{
+ ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
- do
- COPY8(op, ip)
- while (op < oend);
+
+ assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8));
+ if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) {
+ do
+ COPY8(op, ip)
+ while (op < oend);
+ }
+ else {
+ if ((length & 8) == 0)
+ COPY8(op, ip);
+ do {
+ COPY16(op, ip);
+ }
+ while (op < oend);
+ }
+}
+
+/*! ZSTD_wildcopy_16min() :
+ * same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */
+MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE
+void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype)
+{
+ ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + length;
+
+ assert(length >= 8);
+ assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8));
+
+ if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) {
+ do
+ COPY8(op, ip)
+ while (op < oend);
+ }
+ else {
+ if ((length & 8) == 0)
+ COPY8(op, ip);
+ do {
+ COPY16(op, ip);
+ }
+ while (op < oend);
+ }
}
MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
Modified: head/sys/contrib/zstd/lib/compress/zstd_compress.c
==============================================================================
--- head/sys/contrib/zstd/lib/compress/zstd_compress.c Thu Aug 8 16:51:49 2019 (r350759)
+++ head/sys/contrib/zstd/lib/compress/zstd_compress.c Thu Aug 8 16:54:22 2019 (r350760)
@@ -21,6 +21,8 @@
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
#include "zstd_compress_internal.h"
+#include "zstd_compress_sequences.h"
+#include "zstd_compress_literals.h"
#include "zstd_fast.h"
#include "zstd_double_fast.h"
#include "zstd_lazy.h"
@@ -385,6 +387,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter para
bounds.upperBound = ZSTD_lcm_uncompressed;
return bounds;
+ case ZSTD_c_targetCBlockSize:
+ bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN;
+ bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX;
+ return bounds;
+
default:
{ ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 };
return boundError;
@@ -392,18 +399,6 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter para
}
}
-/* ZSTD_cParam_withinBounds:
- * @return 1 if value is within cParam bounds,
- * 0 otherwise */
-static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
-{
- ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam);
- if (ZSTD_isError(bounds.error)) return 0;
- if (value < bounds.lowerBound) return 0;
- if (value > bounds.upperBound) return 0;
- return 1;
-}
-
/* ZSTD_cParam_clampBounds:
* Clamps the value into the bounded range.
*/
@@ -452,6 +447,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter par
case ZSTD_c_ldmHashRateLog:
case ZSTD_c_forceAttachDict:
case ZSTD_c_literalCompressionMode:
+ case ZSTD_c_targetCBlockSize:
default:
return 0;
}
@@ -497,6 +493,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cP
case ZSTD_c_ldmHashLog:
case ZSTD_c_ldmMinMatch:
case ZSTD_c_ldmBucketSizeLog:
+ case ZSTD_c_targetCBlockSize:
break;
default: RETURN_ERROR(parameter_unsupported);
@@ -671,6 +668,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params*
CCtxParams->ldmParams.hashRateLog = value;
return CCtxParams->ldmParams.hashRateLog;
+ case ZSTD_c_targetCBlockSize :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_targetCBlockSize, value);
+ CCtxParams->targetCBlockSize = value;
+ return CCtxParams->targetCBlockSize;
+
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
}
@@ -692,13 +695,13 @@ size_t ZSTD_CCtxParams_getParameter(
*value = CCtxParams->compressionLevel;
break;
case ZSTD_c_windowLog :
- *value = CCtxParams->cParams.windowLog;
+ *value = (int)CCtxParams->cParams.windowLog;
break;
case ZSTD_c_hashLog :
- *value = CCtxParams->cParams.hashLog;
+ *value = (int)CCtxParams->cParams.hashLog;
break;
case ZSTD_c_chainLog :
- *value = CCtxParams->cParams.chainLog;
+ *value = (int)CCtxParams->cParams.chainLog;
break;
case ZSTD_c_searchLog :
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list