svn commit: r361426 - in head: lib/libzstd sys/conf sys/contrib/zstd sys/contrib/zstd/contrib sys/contrib/zstd/contrib/docker sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilde...

Sat May 23 21:23:49 UTC 2020

Author: cem
Date: Sat May 23 21:23:46 2020
New Revision: 361426
URL: https://svnweb.freebsd.org/changeset/base/361426

Log:
  Update to Zstandard 1.4.5
  
  As usual, the full release notes are found on Github:
  
    https://github.com/facebook/zstd/releases/tag/v1.4.5
  
  Notable changes include:
  
  * Improved decompress performance on amd64 and arm (5-10%
    and 15-50%, respectively).
  * '--patch-from' zstd(1) CLI option, which provides something like a very fast
    version of bspatch(1) with slightly worse compression.  See release notes.
  
  In this update, I dropped the 3-year old -O0 workaround for an LLVM ARM bug;
  the bug was fixed in LLVM SVN in 2017, but we didn't remove this workaround
  from our tree until now.
  
  MFC after:	I won't, but feel free
  Relnotes:	yes

Added:
  head/sys/contrib/zstd/lib/compress/zstd_compress_superblock.c
     - copied unchanged from r361425, vendor/zstd/dist/lib/compress/zstd_compress_superblock.c
  head/sys/contrib/zstd/lib/compress/zstd_compress_superblock.h
     - copied unchanged from r361425, vendor/zstd/dist/lib/compress/zstd_compress_superblock.h
Deleted:
  head/sys/contrib/zstd/contrib/cleanTabs
  head/sys/contrib/zstd/contrib/docker/Dockerfile
  head/sys/contrib/zstd/contrib/docker/README.md
  head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile
  head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md
  head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c
  head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/dictBuilder.h
  head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh
  head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/Makefile
  head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/README.md
  head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/fastCover.c
  head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/fastCover.h
  head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/main.c
  head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/test.sh
  head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/Makefile
  head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/README.md
  head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/io.c
  head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/io.h
  head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/main.c
  head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/random.c
  head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/random.h
  head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/test.sh
  head/sys/contrib/zstd/contrib/gen_html/Makefile
  head/sys/contrib/zstd/contrib/gen_html/README.md
  head/sys/contrib/zstd/contrib/gen_html/gen-zstd-manual.sh
  head/sys/contrib/zstd/contrib/gen_html/gen_html.cpp
  head/sys/contrib/zstd/contrib/largeNbDicts/Makefile
  head/sys/contrib/zstd/contrib/largeNbDicts/README.md
  head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c
  head/sys/contrib/zstd/contrib/premake/premake4.lua
  head/sys/contrib/zstd/contrib/premake/zstd.lua
  head/sys/contrib/zstd/contrib/pzstd/BUCK
  head/sys/contrib/zstd/contrib/pzstd/ErrorHolder.h
  head/sys/contrib/zstd/contrib/pzstd/Logging.h
  head/sys/contrib/zstd/contrib/pzstd/Makefile
  head/sys/contrib/zstd/contrib/pzstd/Options.cpp
  head/sys/contrib/zstd/contrib/pzstd/Options.h
  head/sys/contrib/zstd/contrib/pzstd/Pzstd.cpp
  head/sys/contrib/zstd/contrib/pzstd/Pzstd.h
  head/sys/contrib/zstd/contrib/pzstd/README.md
  head/sys/contrib/zstd/contrib/pzstd/SkippableFrame.cpp
  head/sys/contrib/zstd/contrib/pzstd/SkippableFrame.h
  head/sys/contrib/zstd/contrib/pzstd/images/Cspeed.png
  head/sys/contrib/zstd/contrib/pzstd/images/Dspeed.png
  head/sys/contrib/zstd/contrib/pzstd/main.cpp
  head/sys/contrib/zstd/contrib/pzstd/test/BUCK
  head/sys/contrib/zstd/contrib/pzstd/test/OptionsTest.cpp
  head/sys/contrib/zstd/contrib/pzstd/test/PzstdTest.cpp
  head/sys/contrib/zstd/contrib/pzstd/test/RoundTrip.h
  head/sys/contrib/zstd/contrib/pzstd/test/RoundTripTest.cpp
  head/sys/contrib/zstd/contrib/pzstd/utils/BUCK
  head/sys/contrib/zstd/contrib/pzstd/utils/Buffer.h
  head/sys/contrib/zstd/contrib/pzstd/utils/FileSystem.h
  head/sys/contrib/zstd/contrib/pzstd/utils/Likely.h
  head/sys/contrib/zstd/contrib/pzstd/utils/Range.h
  head/sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h
  head/sys/contrib/zstd/contrib/pzstd/utils/ScopeGuard.h
  head/sys/contrib/zstd/contrib/pzstd/utils/ThreadPool.h
  head/sys/contrib/zstd/contrib/pzstd/utils/WorkQueue.h
  head/sys/contrib/zstd/contrib/pzstd/utils/test/BUCK
  head/sys/contrib/zstd/contrib/pzstd/utils/test/BufferTest.cpp
  head/sys/contrib/zstd/contrib/pzstd/utils/test/RangeTest.cpp
  head/sys/contrib/zstd/contrib/pzstd/utils/test/ResourcePoolTest.cpp
  head/sys/contrib/zstd/contrib/pzstd/utils/test/ScopeGuardTest.cpp
  head/sys/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp
  head/sys/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp
  head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile
  head/sys/contrib/zstd/contrib/seekable_format/examples/parallel_compression.c
  head/sys/contrib/zstd/contrib/seekable_format/examples/parallel_processing.c
  head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_compression.c
  head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression.c
  head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression_mem.c
  head/sys/contrib/zstd/contrib/seekable_format/zstd_seekable.h
  head/sys/contrib/zstd/contrib/seekable_format/zstd_seekable_compression_format.md
  head/sys/contrib/zstd/contrib/seekable_format/zstdseek_compress.c
  head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c
  head/sys/contrib/zstd/contrib/snap/snapcraft.yaml
  head/sys/contrib/zstd/tests/Makefile
  head/sys/contrib/zstd/tests/README.md
  head/sys/contrib/zstd/tests/bigdict.c
  head/sys/contrib/zstd/tests/checkTag.c
  head/sys/contrib/zstd/tests/datagencli.c
  head/sys/contrib/zstd/tests/decodecorpus.c
  head/sys/contrib/zstd/tests/fullbench.c
  head/sys/contrib/zstd/tests/fuzz/Makefile
  head/sys/contrib/zstd/tests/fuzz/README.md
  head/sys/contrib/zstd/tests/fuzz/block_decompress.c
  head/sys/contrib/zstd/tests/fuzz/block_round_trip.c
  head/sys/contrib/zstd/tests/fuzz/dictionary_decompress.c
  head/sys/contrib/zstd/tests/fuzz/dictionary_loader.c
  head/sys/contrib/zstd/tests/fuzz/dictionary_round_trip.c
  head/sys/contrib/zstd/tests/fuzz/fuzz.h
  head/sys/contrib/zstd/tests/fuzz/fuzz.py
  head/sys/contrib/zstd/tests/fuzz/fuzz_data_producer.c
  head/sys/contrib/zstd/tests/fuzz/fuzz_data_producer.h
  head/sys/contrib/zstd/tests/fuzz/fuzz_helpers.h
  head/sys/contrib/zstd/tests/fuzz/regression_driver.c
  head/sys/contrib/zstd/tests/fuzz/simple_compress.c
  head/sys/contrib/zstd/tests/fuzz/simple_decompress.c
  head/sys/contrib/zstd/tests/fuzz/simple_round_trip.c
  head/sys/contrib/zstd/tests/fuzz/stream_decompress.c
  head/sys/contrib/zstd/tests/fuzz/stream_round_trip.c
  head/sys/contrib/zstd/tests/fuzz/zstd_frame_info.c
  head/sys/contrib/zstd/tests/fuzz/zstd_helpers.c
  head/sys/contrib/zstd/tests/fuzz/zstd_helpers.h
  head/sys/contrib/zstd/tests/fuzzer.c
  head/sys/contrib/zstd/tests/golden-compression/huffman-compressed-larger
  head/sys/contrib/zstd/tests/golden-decompression/rle-first-block.zst
  head/sys/contrib/zstd/tests/gzip/Makefile
  head/sys/contrib/zstd/tests/gzip/gzip-env.sh
  head/sys/contrib/zstd/tests/gzip/helin-segv.sh
  head/sys/contrib/zstd/tests/gzip/help-version.sh
  head/sys/contrib/zstd/tests/gzip/hufts-segv.gz
  head/sys/contrib/zstd/tests/gzip/hufts.sh
  head/sys/contrib/zstd/tests/gzip/init.cfg
  head/sys/contrib/zstd/tests/gzip/init.sh
  head/sys/contrib/zstd/tests/gzip/keep.sh
  head/sys/contrib/zstd/tests/gzip/list.sh
  head/sys/contrib/zstd/tests/gzip/memcpy-abuse.sh
  head/sys/contrib/zstd/tests/gzip/mixed.sh
  head/sys/contrib/zstd/tests/gzip/null-suffix-clobber.sh
  head/sys/contrib/zstd/tests/gzip/stdin.sh
  head/sys/contrib/zstd/tests/gzip/test-driver.sh
  head/sys/contrib/zstd/tests/gzip/trailing-nul.sh
  head/sys/contrib/zstd/tests/gzip/unpack-invalid.sh
  head/sys/contrib/zstd/tests/gzip/z-suffix.sh
  head/sys/contrib/zstd/tests/gzip/zdiff.sh
  head/sys/contrib/zstd/tests/gzip/zgrep-context.sh
  head/sys/contrib/zstd/tests/gzip/zgrep-f.sh
  head/sys/contrib/zstd/tests/gzip/zgrep-signal.sh
  head/sys/contrib/zstd/tests/gzip/znew-k.sh
  head/sys/contrib/zstd/tests/invalidDictionaries.c
  head/sys/contrib/zstd/tests/legacy.c
  head/sys/contrib/zstd/tests/libzstd_partial_builds.sh
  head/sys/contrib/zstd/tests/longmatch.c
  head/sys/contrib/zstd/tests/paramgrill.c
  head/sys/contrib/zstd/tests/playTests.sh
  head/sys/contrib/zstd/tests/poolTests.c
  head/sys/contrib/zstd/tests/rateLimiter.py
  head/sys/contrib/zstd/tests/regression/Makefile
  head/sys/contrib/zstd/tests/regression/config.c
  head/sys/contrib/zstd/tests/regression/config.h
  head/sys/contrib/zstd/tests/regression/data.c
  head/sys/contrib/zstd/tests/regression/data.h
  head/sys/contrib/zstd/tests/regression/levels.h
  head/sys/contrib/zstd/tests/regression/method.c
  head/sys/contrib/zstd/tests/regression/method.h
  head/sys/contrib/zstd/tests/regression/result.c
  head/sys/contrib/zstd/tests/regression/result.h
  head/sys/contrib/zstd/tests/regression/results.csv
  head/sys/contrib/zstd/tests/regression/test.c
  head/sys/contrib/zstd/tests/roundTripCrash.c
  head/sys/contrib/zstd/tests/seqgen.c
  head/sys/contrib/zstd/tests/seqgen.h
  head/sys/contrib/zstd/tests/symbols.c
  head/sys/contrib/zstd/tests/test-zstd-speed.py
  head/sys/contrib/zstd/tests/test-zstd-versions.py
  head/sys/contrib/zstd/tests/zbufftest.c
  head/sys/contrib/zstd/tests/zstreamtest.c
Modified:
  head/lib/libzstd/Makefile
  head/sys/conf/files
  head/sys/contrib/zstd/CHANGELOG
  head/sys/contrib/zstd/CONTRIBUTING.md
  head/sys/contrib/zstd/Makefile
  head/sys/contrib/zstd/README.md
  head/sys/contrib/zstd/TESTING.md
  head/sys/contrib/zstd/appveyor.yml
  head/sys/contrib/zstd/doc/educational_decoder/Makefile
  head/sys/contrib/zstd/doc/educational_decoder/README.md
  head/sys/contrib/zstd/doc/educational_decoder/harness.c
  head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c
  head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.h
  head/sys/contrib/zstd/doc/zstd_compression_format.md
  head/sys/contrib/zstd/doc/zstd_manual.html
  head/sys/contrib/zstd/examples/Makefile
  head/sys/contrib/zstd/examples/common.h
  head/sys/contrib/zstd/examples/dictionary_compression.c
  head/sys/contrib/zstd/examples/dictionary_decompression.c
  head/sys/contrib/zstd/examples/multiple_simple_compression.c
  head/sys/contrib/zstd/examples/multiple_streaming_compression.c
  head/sys/contrib/zstd/examples/simple_compression.c
  head/sys/contrib/zstd/examples/simple_decompression.c
  head/sys/contrib/zstd/examples/streaming_compression.c
  head/sys/contrib/zstd/examples/streaming_decompression.c
  head/sys/contrib/zstd/examples/streaming_memory_usage.c
  head/sys/contrib/zstd/lib/Makefile
  head/sys/contrib/zstd/lib/README.md
  head/sys/contrib/zstd/lib/common/bitstream.h
  head/sys/contrib/zstd/lib/common/compiler.h
  head/sys/contrib/zstd/lib/common/cpu.h
  head/sys/contrib/zstd/lib/common/debug.c
  head/sys/contrib/zstd/lib/common/debug.h
  head/sys/contrib/zstd/lib/common/entropy_common.c
  head/sys/contrib/zstd/lib/common/error_private.c
  head/sys/contrib/zstd/lib/common/error_private.h
  head/sys/contrib/zstd/lib/common/fse.h
  head/sys/contrib/zstd/lib/common/fse_decompress.c
  head/sys/contrib/zstd/lib/common/huf.h
  head/sys/contrib/zstd/lib/common/mem.h
  head/sys/contrib/zstd/lib/common/pool.c
  head/sys/contrib/zstd/lib/common/pool.h
  head/sys/contrib/zstd/lib/common/threading.c
  head/sys/contrib/zstd/lib/common/threading.h
  head/sys/contrib/zstd/lib/common/xxhash.c
  head/sys/contrib/zstd/lib/common/xxhash.h
  head/sys/contrib/zstd/lib/common/zstd_common.c
  head/sys/contrib/zstd/lib/common/zstd_errors.h
  head/sys/contrib/zstd/lib/common/zstd_internal.h
  head/sys/contrib/zstd/lib/compress/fse_compress.c
  head/sys/contrib/zstd/lib/compress/hist.c
  head/sys/contrib/zstd/lib/compress/hist.h
  head/sys/contrib/zstd/lib/compress/huf_compress.c
  head/sys/contrib/zstd/lib/compress/zstd_compress.c
  head/sys/contrib/zstd/lib/compress/zstd_compress_internal.h
  head/sys/contrib/zstd/lib/compress/zstd_compress_literals.c
  head/sys/contrib/zstd/lib/compress/zstd_compress_literals.h
  head/sys/contrib/zstd/lib/compress/zstd_compress_sequences.c
  head/sys/contrib/zstd/lib/compress/zstd_compress_sequences.h
  head/sys/contrib/zstd/lib/compress/zstd_cwksp.h
  head/sys/contrib/zstd/lib/compress/zstd_double_fast.c
  head/sys/contrib/zstd/lib/compress/zstd_double_fast.h
  head/sys/contrib/zstd/lib/compress/zstd_fast.c
  head/sys/contrib/zstd/lib/compress/zstd_fast.h
  head/sys/contrib/zstd/lib/compress/zstd_lazy.c
  head/sys/contrib/zstd/lib/compress/zstd_lazy.h
  head/sys/contrib/zstd/lib/compress/zstd_ldm.c
  head/sys/contrib/zstd/lib/compress/zstd_ldm.h
  head/sys/contrib/zstd/lib/compress/zstd_opt.c
  head/sys/contrib/zstd/lib/compress/zstd_opt.h
  head/sys/contrib/zstd/lib/compress/zstdmt_compress.c
  head/sys/contrib/zstd/lib/compress/zstdmt_compress.h
  head/sys/contrib/zstd/lib/decompress/huf_decompress.c
  head/sys/contrib/zstd/lib/decompress/zstd_ddict.c
  head/sys/contrib/zstd/lib/decompress/zstd_ddict.h
  head/sys/contrib/zstd/lib/decompress/zstd_decompress.c
  head/sys/contrib/zstd/lib/decompress/zstd_decompress_block.c
  head/sys/contrib/zstd/lib/decompress/zstd_decompress_block.h
  head/sys/contrib/zstd/lib/decompress/zstd_decompress_internal.h
  head/sys/contrib/zstd/lib/deprecated/zbuff.h
  head/sys/contrib/zstd/lib/deprecated/zbuff_common.c
  head/sys/contrib/zstd/lib/deprecated/zbuff_compress.c
  head/sys/contrib/zstd/lib/deprecated/zbuff_decompress.c
  head/sys/contrib/zstd/lib/dictBuilder/cover.c
  head/sys/contrib/zstd/lib/dictBuilder/cover.h
  head/sys/contrib/zstd/lib/dictBuilder/fastcover.c
  head/sys/contrib/zstd/lib/dictBuilder/zdict.c
  head/sys/contrib/zstd/lib/dictBuilder/zdict.h
  head/sys/contrib/zstd/lib/legacy/zstd_legacy.h
  head/sys/contrib/zstd/lib/legacy/zstd_v01.c
  head/sys/contrib/zstd/lib/legacy/zstd_v01.h
  head/sys/contrib/zstd/lib/legacy/zstd_v02.c
  head/sys/contrib/zstd/lib/legacy/zstd_v02.h
  head/sys/contrib/zstd/lib/legacy/zstd_v03.c
  head/sys/contrib/zstd/lib/legacy/zstd_v03.h
  head/sys/contrib/zstd/lib/legacy/zstd_v04.c
  head/sys/contrib/zstd/lib/legacy/zstd_v04.h
  head/sys/contrib/zstd/lib/legacy/zstd_v05.c
  head/sys/contrib/zstd/lib/legacy/zstd_v05.h
  head/sys/contrib/zstd/lib/legacy/zstd_v06.c
  head/sys/contrib/zstd/lib/legacy/zstd_v06.h
  head/sys/contrib/zstd/lib/legacy/zstd_v07.c
  head/sys/contrib/zstd/lib/legacy/zstd_v07.h
  head/sys/contrib/zstd/lib/libzstd.pc.in
  head/sys/contrib/zstd/lib/zstd.h
  head/sys/contrib/zstd/programs/Makefile
  head/sys/contrib/zstd/programs/README.md
  head/sys/contrib/zstd/programs/benchfn.c
  head/sys/contrib/zstd/programs/benchfn.h
  head/sys/contrib/zstd/programs/benchzstd.c
  head/sys/contrib/zstd/programs/benchzstd.h
  head/sys/contrib/zstd/programs/datagen.c
  head/sys/contrib/zstd/programs/datagen.h
  head/sys/contrib/zstd/programs/dibio.c
  head/sys/contrib/zstd/programs/dibio.h
  head/sys/contrib/zstd/programs/fileio.c
  head/sys/contrib/zstd/programs/fileio.h
  head/sys/contrib/zstd/programs/platform.h
  head/sys/contrib/zstd/programs/timefn.c
  head/sys/contrib/zstd/programs/timefn.h
  head/sys/contrib/zstd/programs/util.c
  head/sys/contrib/zstd/programs/util.h
  head/sys/contrib/zstd/programs/zstd.1
  head/sys/contrib/zstd/programs/zstd.1.md
  head/sys/contrib/zstd/programs/zstdcli.c
  head/sys/contrib/zstd/programs/zstdgrep
  head/sys/contrib/zstd/programs/zstdgrep.1
  head/sys/contrib/zstd/programs/zstdless.1
  head/sys/contrib/zstd/zlibWrapper/Makefile
  head/sys/contrib/zstd/zlibWrapper/examples/zwrapbench.c
  head/sys/contrib/zstd/zlibWrapper/gzcompatibility.h
  head/sys/contrib/zstd/zlibWrapper/zstd_zlibwrapper.c
  head/sys/contrib/zstd/zlibWrapper/zstd_zlibwrapper.h
Directory Properties:
  head/sys/contrib/zstd/   (props changed)

Modified: head/lib/libzstd/Makefile
==============================================================================

--- head/lib/libzstd/Makefile	Sat May 23 20:39:36 2020	(r361425)
+++ head/lib/libzstd/Makefile	Sat May 23 21:23:46 2020	(r361426)
@@ -13,6 +13,7 @@ SRCS=	entropy_common.c \
 	zstd_compress.c \
 	zstd_compress_literals.c \
 	zstd_compress_sequences.c \
+	zstd_compress_superblock.c \
 	zstdmt_compress.c \
 	huf_decompress.c \
 	zstd_ddict.c \
@@ -54,13 +55,6 @@ ZSTDDIR=	${SRCTOP}/sys/contrib/zstd
 # turning off vectorization, which is why there were both."
 .if ${COMPILER_TYPE} == "gcc"
 CFLAGS.zstd_decompress_block.c+=	-fno-tree-vectorize
-.endif
-
-# Work around for LLVM bug 35023, https://bugs.llvm.org/show_bug.cgi?id=35023
-# Fixed in https://reviews.llvm.org/rL349935 ; not sure if we have that version
-# in our LLVM or not yet.
-.if ${MACHINE_ARCH:Marm*} != ""
-CFLAGS.zstd_compress.c+= -O0
 .endif
 
 # Work around lack of __bswap[sd]i2() intrinsics on riscv.  Probably should be

Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/conf/files	Sat May 23 21:23:46 2020	(r361426)
@@ -659,6 +659,7 @@ contrib/zstd/lib/common/xxhash.c		optional zstdio comp
 contrib/zstd/lib/compress/zstd_compress.c	optional zstdio compile-with ${ZSTD_C}
 contrib/zstd/lib/compress/zstd_compress_literals.c	optional zstdio compile-with ${ZSTD_C}
 contrib/zstd/lib/compress/zstd_compress_sequences.c	optional zstdio compile-with ${ZSTD_C}
+contrib/zstd/lib/compress/zstd_compress_superblock.c	optional zstdio compile-with ${ZSTD_C}
 contrib/zstd/lib/compress/fse_compress.c	optional zstdio compile-with ${ZSTD_C}
 contrib/zstd/lib/compress/hist.c		optional zstdio compile-with ${ZSTD_C}
 contrib/zstd/lib/compress/huf_compress.c	optional zstdio compile-with ${ZSTD_C}

Modified: head/sys/contrib/zstd/CHANGELOG
==============================================================================
--- head/sys/contrib/zstd/CHANGELOG	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/CHANGELOG	Sat May 23 21:23:46 2020	(r361426)
@@ -1,3 +1,29 @@
+v1.4.5
+fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln
+perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln
+perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta)
+perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh)
+perf: Small level 1 compression speed gains (depending on compiler)
+cli : New --patch-from command, create and apply patches from files, by @bimbashreshta
+cli : New --filelist= : Provide a list of files to operate upon from a file
+cli : -b -d command can now benchmark decompression on multiple files
+cli : New --no-content-size command
+cli : New --show-default-cparams information command
+api : ZDICT_finalizeDictionary() is promoted to stable (#2111)
+api : new experimental parameter ZSTD_d_stableOutBuffer (#2094)
+build: Generate a single-file libzstd library (#2065, by @cwoffenden)
+build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte)
+build: zstd now compiles cleanly under -pedantic (#2099)
+build: zstd now compiles with make-4.3
+build: Support mingw cross-compilation from Linux, by @Ericson2314
+build: Meson multi-thread build fix on windows
+build: Some misc icc fixes backed by new ci test on travis
+misc: bitflip analyzer tool, by @felixhandte
+misc: Extend largeNbDicts benchmark to compression
+misc: Edit-distance match finder in contrib/
+doc : Improved beginner CONTRIBUTING.md docs
+doc : New issue templates for zstd
+
 v1.4.4
 perf: Improved decompression speed, by > 10%, by @terrelln
 perf: Better compression speed when re-using a context, by @felixhandte
@@ -14,7 +40,8 @@ cli: commands --stream-size=# and --size-hint=#, by @n
 cli: command --exclude-compressed, by @shashank0791
 cli: faster `-t` test mode
 cli: improved some error messages, by @vangyzen
-cli: rare deadlock condition within dictionary builder, by @terrelln
+cli: fix command `-D dictionary` on Windows, reported by @artyompetrov
+cli: fix rare deadlock condition within dictionary builder, by @terrelln
 build: single-file decoder with emscripten compilation script, by @cwoffenden
 build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive
 build: fixed deprecation warning for certain gcc version, reported by @jasonma163

Modified: head/sys/contrib/zstd/CONTRIBUTING.md
==============================================================================
--- head/sys/contrib/zstd/CONTRIBUTING.md	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/CONTRIBUTING.md	Sat May 23 21:23:46 2020	(r361426)
@@ -26,6 +26,356 @@ to do this once to work on any of Facebook's open sour
 
 Complete your CLA here: <https://code.facebook.com/cla>
 
+## Workflow
+Zstd uses a branch-based workflow for making changes to the codebase. Typically, zstd
+will use a new branch per sizable topic. For smaller changes, it is okay to lump multiple
+related changes into a branch.
+
+Our contribution process works in three main stages:
+1. Local development
+    * Update:
+        * Checkout your fork of zstd if you have not already
+        ```
+        git checkout https://github.com/<username>/zstd
+        cd zstd
+        ```
+        * Update your local dev branch
+        ```
+        git pull https://github.com/facebook/zstd dev
+        git push origin dev
+        ```
+    * Topic and development:
+        * Make a new branch on your fork about the topic you're developing for
+        ```
+        # branch names should be consise but sufficiently informative
+        git checkout -b <branch-name>
+        git push origin <branch-name>
+        ```
+        * Make commits and push
+        ```
+        # make some changes =
+        git add -u && git commit -m <message>
+        git push origin <branch-name>
+        ```
+        * Note: run local tests to ensure that your changes didn't break existing functionality
+            * Quick check
+            ```
+            make shortest
+            ```
+            * Longer check
+            ```
+            make test
+            ```
+2. Code Review and CI tests
+    * Ensure CI tests pass:
+        * Before sharing anything to the community, make sure that all CI tests pass on your local fork.
+        See our section on setting up your CI environment for more information on how to do this.
+        * Ensure that static analysis passes on your development machine. See the Static Analysis section
+        below to see how to do this.
+    * Create a pull request:
+        * When you are ready to share you changes to the community, create a pull request from your branch
+        to facebook:dev. You can do this very easily by clicking 'Create Pull Request' on your fork's home
+        page.
+        * From there, select the branch where you made changes as your source branch and facebook:dev
+        as the destination.
+        * Examine the diff presented between the two branches to make sure there is nothing unexpected.
+    * Write a good pull request description:
+        * While there is no strict template that our contributors follow, we would like them to
+        sufficiently summarize and motivate the changes they are proposing. We recommend all pull requests,
+        at least indirectly, address the following points.
+            * Is this pull request important and why?
+            * Is it addressing an issue? If so, what issue? (provide links for convenience please)
+            * Is this a new feature? If so, why is it useful and/or necessary?
+            * Are there background references and documents that reviewers should be aware of to properly assess this change?
+        * Note: make sure to point out any design and architectural decisions that you made and the rationale behind them.
+        * Note: if you have been working with a specific user and would like them to review your work, make sure you mention them using (@<username>)
+    * Submit the pull request and iterate with feedback.
+3. Merge and Release
+    * Getting approval:
+        * You will have to iterate on your changes with feedback from other collaborators to reach a point
+        where your pull request can be safely merged.
+        * To avoid too many comments on style and convention, make sure that you have a
+        look at our style section below before creating a pull request.
+        * Eventually, someone from the zstd team will approve your pull request and not long after merge it into
+        the dev branch.
+    * Housekeeping:
+        * Most PRs are linked with one or more Github issues. If this is the case for your PR, make sure
+        the corresponding issue is mentioned. If your change 'fixes' or completely addresses the
+        issue at hand, then please indicate this by requesting that an issue be closed by commenting.
+        * Just because your changes have been merged does not mean the topic or larger issue is complete. Remember
+        that the change must make it to an official zstd release for it to be meaningful. We recommend
+        that contributers track the activity on their pull request and corresponding issue(s) page(s) until
+        their change makes it to the next release of zstd. Users will often discover bugs in your code or
+        suggest ways to refine and improve your initial changes even after the pull request is merged.
+
+## Static Analysis
+Static analysis is a process for examining the correctness or validity of a program without actually
+executing it. It usually helps us find many simple bugs. Zstd uses clang's `scan-build` tool for
+static analysis. You can install it by following the instructions for your OS on https://clang-analyzer.llvm.org/scan-build.
+
+Once installed, you can ensure that our static analysis tests pass on your local development machine
+by running:
+```
+make staticAnalyze
+```
+
+In general, you can use `scan-build` to static analyze any build script. For example, to static analyze
+just `contrib/largeNbDicts` and nothing else, you can run:
+
+```
+scan-build make -C contrib/largeNbDicts largeNbDicts
+```
+
+## Performance
+Performance is extremely important for zstd and we only merge pull requests whose performance
+landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented.
+This high bar for performance means that every PR which has the potential to
+impact performance takes a very long time for us to properly review. That being said, we
+always welcome contributions to improve performance (or worsen performance for the trade-off of
+something else). Please keep the following in mind before submitting a performance related PR:
+
+1. Zstd isn't as old as gzip but it has been around for time now and its evolution is
+very well documented via past Github issues and pull requests. It may be the case that your
+particular performance optimization has already been considered in the past. Please take some
+time to search through old issues and pull requests using keywords specific to your
+would-be PR. Of course, just because a topic has already been discussed (and perhaps rejected
+on some grounds) in the past, doesn't mean it isn't worth bringing up again. But even in that case,
+it will be helpful for you to have context from that topic's history before contributing.
+2. The distinction between noise and actual performance gains can unfortunately be very subtle
+especially when microbenchmarking extremely small wins or losses. The only remedy to getting
+something subtle merged is extensive benchmarking. You will be doing us a great favor if you
+take the time to run extensive, long-duration, and potentially cross-(os, platform, process, etc)
+benchmarks on your end before submitting a PR. Of course, you will not be able to benchmark
+your changes on every single processor and os out there (and neither will we) but do that best
+you can:) We've adding some things to think about when benchmarking below in the Benchmarking
+Performance section which might be helpful for you.
+3. Optimizing performance for a certain OS, processor vendor, compiler, or network system is a perfectly
+legitimate thing to do as long as it does not harm the overall performance health of Zstd.
+This is a hard balance to strike but please keep in mind other aspects of Zstd when
+submitting changes that are clang-specific, windows-specific, etc.
+
+## Benchmarking Performance
+Performance microbenchmarking is a tricky subject but also essential for Zstd. We value empirical
+testing over theoretical speculation. This guide it not perfect but for most scenarios, it
+is a good place to start.
+
+### Stability
+Unfortunately, the most important aspect in being able to benchmark reliably is to have a stable
+benchmarking machine. A virtual machine, a machine with shared resources, or your laptop
+will typically not be stable enough to obtain reliable benchmark results. If you can get your
+hands on a desktop, this is usually a better scenario.
+
+Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to
+do a little more work to ensure that you are in fact measuring the changes you've made not and
+noise. Here are some things you can do to make your benchmarks more stable:
+
+1. The most simple thing you can do to drastically improve the stability of your benchmark is
+to run it multiple times and then aggregate the results of those runs. As a general rule of
+thumb, the smaller the change you are trying to measure, the more samples of benchmark runs
+you will have to aggregate over to get reliable results. Here are some additional things to keep in
+mind when running multiple trials:
+    * How you aggregate your samples are important. You might be tempted to use the mean of your
+    results. While this is certainly going to be a more stable number than a raw single sample
+    benchmark number, you might have more luck by taking the median. The mean is not robust to
+    outliers whereas the median is. Better still, you could simply take the fastest speed your
+    benchmark achieved on each run since that is likely the fastest your process will be
+    capable of running your code. In our experience, this (aggregating by just taking the sample
+    with the fastest running time) has been the most stable approach.
+    * The more samples you have, the more stable your benchmarks should be. You can verify
+    your improved stability by looking at the size of your confidence intervals as you
+    increase your sample count. These should get smaller and smaller. Eventually hopefully
+    smaller than the performance win you are expecting.
+    * Most processors will take some time to get `hot` when running anything. The observations
+    you collect during that time period will very different from the true performance number. Having
+    a very large number of sample will help alleviate this problem slightly but you can also
+    address is directly by simply not including the first `n` iterations of your benchmark in
+    your aggregations. You can determine `n` by simply looking at the results from each iteration
+    and then hand picking a good threshold after which the variance in results seems to stabilize.
+2. You cannot really get reliable benchmarks if your host machine is simultaneously running
+another cpu/memory-intensive application in the background. If you are running benchmarks on your
+personal laptop for instance, you should close all applications (including your code editor and
+browser) before running your benchmarks. You might also have invisible background applications
+running. You can see what these are by looking at either Activity Monitor on Mac or Task Manager
+on Windows. You will get more stable benchmark results of you end those processes as well.
+    * If you have multiple cores, you can even run your benchmark on a reserved core to prevent
+    pollution from other OS and user processes. There are a number of ways to do this depending
+    on your OS:
+        * On linux boxes, you have use https://github.com/lpechacek/cpuset.
+        * On Windows, you can "Set Processor Affinity" using https://www.thewindowsclub.com/processor-affinity-windows
+        * On Mac, you can try to use their dedicated affinity API https://developer.apple.com/library/archive/releasenotes/Performance/RN-AffinityAPI/#//apple_ref/doc/uid/TP40006635-CH1-DontLinkElementID_2
+3. To benchmark, you will likely end up writing a separate c/c++ program that will link libzstd.
+Dynamically linking your library will introduce some added variation (not a large amount but
+definitely some). Statically linking libzstd will be more stable. Static libraries should
+be enabled by default when building zstd.
+4. Use a profiler with a good high resolution timer. See the section below on profiling for
+details on this.
+5. Disable frequency scaling, turbo boost and address space randomization (this will vary by OS)
+6. Try to avoid storage. On some systems you can use tmpfs. Putting the program, inputs and outputs on
+tmpfs avoids touching a real storage system, which can have a pretty big variability.
+
+Also check our LLVM's guide on benchmarking here: https://llvm.org/docs/Benchmarking.html
+
+### Zstd benchmark
+The fastest signal you can get regarding your performance changes is via the in-build zstd cli
+bench option. You can run Zstd as you typically would for your scenario using some set of options
+and then additionally also specify the `-b#` option. Doing this will run our benchmarking pipeline
+for that options you have just provided. If you want to look at the internals of how this
+benchmarking script works, you can check out programs/benchzstd.c
+
+For example: say you have made a change that you believe improves the speed of zstd level 1. The
+very first thing you should use to asses whether you actually achieved any sort of improvement
+is `zstd -b`. You might try to do something like this. Note: you can use the `-i` option to
+specify a running time for your benchmark in seconds (default is 3 seconds).
+Usually, the longer the running time, the more stable your results will be.
+
+```
+$ git checkout <commit-before-your-change>
+$ make && cp zstd zstd-old
+$ git checkout <commit-after-your-change>
+$ make && cp zstd zstd-new
+$ zstd-old -i5 -b1 <your-test-data>
+ 1<your-test-data>         :      8990 ->      3992 (2.252), 302.6 MB/s , 626.4 MB/s
+$ zstd-new -i5 -b1 <your-test-data>
+ 1<your-test-data>         :      8990 ->      3992 (2.252), 302.8 MB/s , 628.4 MB/s
+```
+
+Unless your performance win is large enough to be visible despite the intrinsic noise
+on your computer, benchzstd alone will likely not be enough to validate the impact of your
+changes. For example, the results of the example above indicate that effectively nothing
+changed but there could be a small <3% improvement that the noise on the host machine
+obscured. So unless you see a large performance win (10-15% consistently) using just
+this method of evaluation will not be sufficient.
+
+### Profiling
+There are a number of great profilers out there. We're going to briefly mention how you can
+profile your code using `instruments` on mac, `perf` on linux and `visual studio profiler`
+on windows.
+
+Say you have an idea for a change that you think will provide some good performance gains
+for level 1 compression on Zstd. Typically this means, you have identified a section of
+code that you think can be made to run faster.
+
+The first thing you will want to do is make sure that the piece of code is actually taking up
+a notable amount of time to run. It is usually not worth optimzing something which accounts for less than
+0.0001% of the total running time. Luckily, there are tools to help with this.
+Profilers will let you see how much time your code spends inside a particular function.
+If your target code snippit is only part of a function, it might be worth trying to
+isolate that snippit by moving it to its own function (this is usually not necessary but
+might be).
+
+Most profilers (including the profilers dicusssed below) will generate a call graph of
+functions for you. Your goal will be to find your function of interest in this call grapch
+and then inspect the time spent inside of it. You might also want to to look at the
+annotated assembly which most profilers will provide you with.
+
+#### Instruments
+We will once again consider the scenario where you think you've identified a piece of code
+whose performance can be improved upon. Follow these steps to profile your code using
+Instruments.
+
+1. Open Instruments
+2. Select `Time Profiler` from the list of standard templates
+3. Close all other applications except for your instruments window and your terminal
+4. Run your benchmarking script from your terminal window
+    * You will want a benchmark that runs for at least a few seconds (5 seconds will
+    usually be long enough). This way the profiler will have something to work with
+    and you will have ample time to attach your profiler to this process:)
+    * I will just use benchzstd as my bencharmking script for this example:
+```
+$ zstd -b1 -i5 <my-data> # this will run for 5 seconds
+```
+5. Once you run your benchmarking script, switch back over to instruments and attach your
+process to the time profiler. You can do this by:
+    * Clicking on the `All Processes` drop down in the top left of the toolbar.
+    * Selecting your process from the dropdown. In my case, it is just going to be labled
+    `zstd`
+    * Hitting the bright red record circle button on the top left of the toolbar
+6. You profiler will now start collecting metrics from your bencharking script. Once
+you think you have collected enough samples (usually this is the case after 3 seconds of
+recording), stop your profiler.
+7. Make sure that in toolbar of the bottom window, `profile` is selected.
+8. You should be able to see your call graph.
+    * If you don't see the call graph or an incomplete call graph, make sure you have compiled
+    zstd and your benchmarking scripg using debug flags. On mac and linux, this just means
+    you will have to supply the `-g` flag alone with your build script. You might also
+    have to provide the `-fno-omit-frame-pointer` flag
+9. Dig down the graph to find your function call and then inspect it by double clicking
+the list item. You will be able to see the annotated source code and the assembly side by
+side.
+
+#### Perf
+
+This wiki has a pretty detailed tutorial on getting started working with perf so we'll
+leave you to check that out of you're getting started:
+
+https://perf.wiki.kernel.org/index.php/Tutorial
+
+Some general notes on perf:
+* Use `perf stat -r # <bench-program>` to quickly get some relevant timing and
+counter statistics. Perf uses a high resolution timer and this is likely one
+of the first things your team will run when assessing your PR.
+* Perf has a long list of hardware counters that can be viewed with `perf --list`.
+When measuring optimizations, something worth trying is to make sure the handware
+counters you expect to be impacted by your change are in fact being so. For example,
+if you expect the L1 cache misses to decrease with your change, you can look at the
+counter `L1-dcache-load-misses`
+* Perf hardware counters will not work on a virtual machine.
+
+#### Visual Studio
+
+TODO
+
+
+## Setting up continuous integration (CI) on your fork
+Zstd uses a number of different continuous integration (CI) tools to ensure that new changes
+are well tested before they make it to an official release. Specifically, we use the platforms
+travis-ci, circle-ci, and appveyor.
+
+Changes cannot be merged into the main dev branch unless they pass all of our CI tests.
+The easiest way to run these CI tests on your own before submitting a PR to our dev branch
+is to configure your personal fork of zstd with each of the CI platforms. Below, you'll find
+instructions for doing this.
+
+### travis-ci
+Follow these steps to link travis-ci with your github fork of zstd
+
+1. Make sure you are logged into your github account
+2. Go to https://travis-ci.org/
+3. Click 'Sign in with Github' on the top right
+4. Click 'Authorize travis-ci'
+5. Click 'Activate all repositories using Github Apps'
+6. Select 'Only select repositories' and select your fork of zstd from the drop down
+7. Click 'Approve and Install'
+8. Click 'Sign in with Github' again. This time, it will be for travis-pro (which will let you view your tests on the web dashboard)
+9. Click 'Authorize travis-pro'
+10. You should have travis set up on your fork now.
+
+### circle-ci
+TODO
+
+### appveyor
+Follow these steps to link circle-ci with your girhub fork of zstd
+
+1. Make sure you are logged into your github account
+2. Go to https://www.appveyor.com/
+3. Click 'Sign in' on the top right
+4. Select 'Github' on the left panel
+5. Click 'Authorize appveyor'
+6. You might be asked to select which repositories you want to give appveyor permission to. Select your fork of zstd if you're prompted
+7. You should have appveyor set up on your fork now.
+
+### General notes on CI
+CI tests run every time a pull request (PR) is created or updated. The exact tests
+that get run will depend on the destination branch you specify. Some tests take
+longer to run than others. Currently, our CI is set up to run a short
+series of tests when creating a PR to the dev branch and a longer series of tests
+when creating a PR to the master branch. You can look in the configuration files
+of the respective CI platform for more information on what gets run when.
+
+Most people will just want to create a PR with the destination set to their local dev
+branch of zstd. You can then find the status of the tests on the PR's page. You can also
+re-run tests and cancel running tests from the PR page or from the respective CI's dashboard.
+
 ## Issues
 We use GitHub issues to track public bugs. Please ensure your description is
 clear and has sufficient instructions to be able to reproduce the issue.
@@ -34,7 +384,7 @@ Facebook has a [bounty program](https://www.facebook.c
 disclosure of security bugs. In those cases, please go through the process
 outlined on that page and do not file a public issue.
 
-## Coding Style  
+## Coding Style
 * 4 spaces for indentation rather than tabs
 
 ## License

Modified: head/sys/contrib/zstd/Makefile
==============================================================================
--- head/sys/contrib/zstd/Makefile	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/Makefile	Sat May 23 21:23:46 2020	(r361426)
@@ -1,10 +1,11 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
 # LICENSE file in the root directory of this source tree) and the GPLv2 (found
 # in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
 PRGDIR   = programs
@@ -17,7 +18,16 @@ FUZZDIR  = $(TESTDIR)/fuzz
 # Define nul output
 VOID = /dev/null
 
-ifneq (,$(filter Windows%,$(OS)))
+# When cross-compiling from linux to windows, you might
+# need to specify this as "Windows." Fedora build fails
+# without it.
+#
+# Note: mingw-w64 build from linux to windows does not
+# fail on other tested distros (ubuntu, debian) even
+# without manually specifying the TARGET_SYSTEM.
+TARGET_SYSTEM ?= $(OS)
+
+ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
 EXT =.exe
 else
 EXT =
@@ -35,7 +45,7 @@ allmost: allzstd zlibwrapper
 
 # skip zwrapper, can't build that on alternate architectures without the proper zlib installed
 .PHONY: allzstd
-allzstd: lib
+allzstd: lib-all
 	$(MAKE) -C $(PRGDIR) all
 	$(MAKE) -C $(TESTDIR) all
 
@@ -45,7 +55,7 @@ all32:
 	$(MAKE) -C $(TESTDIR) all32
 
 .PHONY: lib lib-release libzstd.a
-lib lib-release :
+lib lib-release lib-all :
 	@$(MAKE) -C $(ZSTDDIR) $@
 
 .PHONY: zstd zstd-release
@@ -80,6 +90,13 @@ shortest:
 .PHONY: check
 check: shortest
 
+.PHONY: automated_benchmarking
+automated_benchmarking:
+	$(MAKE) -C $(TESTDIR) $@
+
+.PHONY: benchmarking
+benchmarking: automated_benchmarking
+
 ## examples: build all examples in `/examples` directory
 .PHONY: examples
 examples: lib
@@ -101,7 +118,8 @@ contrib: lib
 	$(MAKE) -C contrib/pzstd all
 	$(MAKE) -C contrib/seekable_format/examples all
 	$(MAKE) -C contrib/largeNbDicts all
-	cd contrib/single_file_decoder/ ; ./build_test.sh
+	cd contrib/single_file_libs/ ; ./build_decoder_test.sh
+	cd contrib/single_file_libs/ ; ./build_library_test.sh
 
 .PHONY: cleanTabs
 cleanTabs:
@@ -337,7 +355,7 @@ endif
 
 ifneq (,$(filter MSYS%,$(shell uname)))
 HOST_OS = MSYS
-CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
+CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
 endif
 
 
@@ -349,11 +367,15 @@ cmakebuild:
 	cmake --version
 	$(RM) -r $(BUILDIR)/cmake/build
 	mkdir $(BUILDIR)/cmake/build
-	cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
+	cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) ..
+	$(MAKE) -C $(BUILDIR)/cmake/build -j4;
+	$(MAKE) -C $(BUILDIR)/cmake/build install;
+	$(MAKE) -C $(BUILDIR)/cmake/build uninstall;
+	cd $(BUILDIR)/cmake/build; ctest -V -L Medium
 
-c90build: clean
+c89build: clean
 	$(CC) -v
-	CFLAGS="-std=c90 -Werror" $(MAKE) allmost  # will fail, due to missing support for `long long`
+	CFLAGS="-std=c89 -Werror" $(MAKE) allmost  # will fail, due to missing support for `long long`
 
 gnu90build: clean
 	$(CC) -v

Modified: head/sys/contrib/zstd/README.md
==============================================================================
--- head/sys/contrib/zstd/README.md	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/README.md	Sat May 23 21:23:46 2020	(r361426)
@@ -31,10 +31,10 @@ a list of known ports and bindings is provided on [Zst
 ## Benchmarks
 
 For reference, several fast compression algorithms were tested and compared
-on a server running Arch Linux (`Linux version 5.0.5-arch1-1`),
+on a server running Arch Linux (`Linux version 5.5.11-arch1-1`),
 with a Core i9-9900K CPU @ 5.0GHz,
 using [lzbench], an open-source in-memory benchmark by @inikep
-compiled with [gcc] 8.2.1,
+compiled with [gcc] 9.3.0,
 on the [Silesia compression corpus].
 
 [lzbench]: https://github.com/inikep/lzbench
@@ -43,18 +43,26 @@ on the [Silesia compression corpus].
 
 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| **zstd 1.4.0 -1**       | 2.884 |   530 MB/s |  1360 MB/s |
-| zlib 1.2.11 -1          | 2.743 |   110 MB/s |   440 MB/s |
-| brotli 1.0.7 -0         | 2.701 |   430 MB/s |   470 MB/s |
-| quicklz 1.5.0 -1        | 2.238 |   600 MB/s |   800 MB/s |
-| lzo1x 2.09 -1           | 2.106 |   680 MB/s |   950 MB/s |
-| lz4 1.8.3               | 2.101 |   800 MB/s |  4220 MB/s |
-| snappy 1.1.4            | 2.073 |   580 MB/s |  2020 MB/s |
-| lzf 3.6 -1              | 2.077 |   440 MB/s |   930 MB/s |
+| **zstd 1.4.5 -1**       | 2.884 |   500 MB/s |  1660 MB/s |
+| zlib 1.2.11 -1          | 2.743 |    90 MB/s |   400 MB/s |
+| brotli 1.0.7 -0         | 2.703 |   400 MB/s |   450 MB/s |
+| **zstd 1.4.5 --fast=1** | 2.434 |   570 MB/s |  2200 MB/s |
+| **zstd 1.4.5 --fast=3** | 2.312 |   640 MB/s |  2300 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   560 MB/s |   710 MB/s |
+| **zstd 1.4.5 --fast=5** | 2.178 |   700 MB/s |  2420 MB/s |
+| lzo1x 2.10 -1           | 2.106 |   690 MB/s |   820 MB/s |
+| lz4 1.9.2               | 2.101 |   740 MB/s |  4530 MB/s |
+| **zstd 1.4.5 --fast=7** | 2.096 |   750 MB/s |  2480 MB/s |
+| lzf 3.6 -1              | 2.077 |   410 MB/s |   860 MB/s |
+| snappy 1.1.8            | 2.073 |   560 MB/s |  1790 MB/s |
 
 [zlib]: http://www.zlib.net/
 [LZ4]: http://www.lz4.org/
 
+The negative compression levels, specified with `--fast=#`,
+offer faster compression and decompression speed in exchange for some loss in
+compression ratio compared to level 1, as seen in the table above.
+
 Zstd can also offer stronger compression ratios at the cost of compression speed.
 Speed vs Compression trade-off is configurable by small increments.
 Decompression speed is preserved and remains roughly the same at all settings,
@@ -142,6 +150,18 @@ You can also take a look at [`.travis.yml`](.travis.ym
 example about how Meson is used to build this project.
 
 Note that default build type is **release**.
+
+### VCPKG
+You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager:
+
+    git clone https://github.com/Microsoft/vcpkg.git
+    cd vcpkg
+    ./bootstrap-vcpkg.sh
+    ./vcpkg integrate install
+    ./vcpkg install zstd
+
+The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors.
+If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
 
 ### Visual Studio (Windows)
 

Modified: head/sys/contrib/zstd/TESTING.md
==============================================================================
--- head/sys/contrib/zstd/TESTING.md	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/TESTING.md	Sat May 23 21:23:46 2020	(r361426)
@@ -11,7 +11,7 @@ They consist of the following tests:
 - Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64)
 - Compilation on various versions of gcc, clang, and g++
 - `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests)
-- Small tests (`tests/legacy.c`, `tests/longmatch.c`, `tests/symbols.c`) on x64_64
+- Small tests (`tests/legacy.c`, `tests/longmatch.c`) on x64_64
 
 Medium Tests
 ------------
@@ -19,7 +19,7 @@ Medium tests run on every commit and pull request to `
 They consist of the following tests:
 - The following tests run with UBsan and Asan on x86_64 and x86, as well as with
   Msan on x86_64
-  - `tests/playTests.sh --test-long-data`
+  - `tests/playTests.sh --test-large-data`
   - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
 - `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
 - Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)

Modified: head/sys/contrib/zstd/appveyor.yml
==============================================================================
--- head/sys/contrib/zstd/appveyor.yml	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/appveyor.yml	Sat May 23 21:23:46 2020	(r361426)
@@ -14,7 +14,7 @@
     - COMPILER: "gcc"
       HOST:     "mingw"
       PLATFORM: "x64"
-      SCRIPT:   "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-lib"
+      SCRIPT:   "make allzstd MOREFLAGS=-static && make -C tests fullbench-lib"
       ARTIFACT: "true"
       BUILD:    "true"
     - COMPILER: "gcc"
@@ -169,7 +169,8 @@
   - SET "FUZZERTEST=-T30s"
   - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
       CD tests &&
-      SET ZSTD=./zstd.exe &&
+      SET ZSTD_BIN=./zstd.exe&&
+      SET DATAGEN_BIN=./datagen.exe&&
       sh -e playTests.sh --test-large-data &&
       fullbench.exe -i1 &&
       fullbench.exe -i1 -P0 &&
@@ -188,6 +189,9 @@
   environment:
     matrix:
     - COMPILER: "gcc"
+      HOST:     "cygwin"
+      PLATFORM: "x64"
+    - COMPILER: "gcc"
       HOST:     "mingw"
       PLATFORM: "x64"
       SCRIPT:   "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2"
@@ -220,6 +224,14 @@
   install:
   - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
   - SET PATH_ORIGINAL=%PATH%
+  - if [%HOST%]==[cygwin] (
+      ECHO Installing Cygwin Packages &&
+      C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^
+        gcc-g++,^
+        gcc,^
+        cmake,^
+        make
+    )
   - if [%HOST%]==[mingw] (
       SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
       SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
@@ -232,6 +244,17 @@
 
   build_script:
   - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
+  - if [%HOST%]==[cygwin] (
+      set CHERE_INVOKING=yes &&
+      set CC=%COMPILER% &&
+      C:\cygwin64\bin\bash --login -c "
+        set -e;
+        cd build/cmake;
+        CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_ZSTREAM_FLAGS=-T30s .;
+        make -j4;
+        ctest -V -L Medium;
+      "
+    )
   - if [%HOST%]==[mingw] (
       ( if [%PLATFORM%]==[x64] (
         SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"

Modified: head/sys/contrib/zstd/doc/educational_decoder/Makefile
==============================================================================
--- head/sys/contrib/zstd/doc/educational_decoder/Makefile	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/doc/educational_decoder/Makefile	Sat May 23 21:23:46 2020	(r361426)
@@ -1,10 +1,11 @@
 # ################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
 # LICENSE file in the root directory of this source tree) and the GPLv2 (found
 # in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
 ZSTD ?= zstd   # note: requires zstd installation on local system
@@ -36,7 +37,7 @@ harness: $(HARNESS_FILES)
 	$(CC) $(FLAGS) $^ -o $@
 
 clean:
-	@$(RM) harness
+	@$(RM) harness *.o
 	@$(RM) -rf harness.dSYM  # MacOS specific
 
 test: harness
@@ -59,4 +60,3 @@ test: harness
 	@./harness tmp.zst tmp dictionary
 	@$(DIFF) -s tmp README.md
 	@$(RM) tmp* dictionary
-	@$(MAKE) clean

Modified: head/sys/contrib/zstd/doc/educational_decoder/README.md
==============================================================================
--- head/sys/contrib/zstd/doc/educational_decoder/README.md	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/doc/educational_decoder/README.md	Sat May 23 21:23:46 2020	(r361426)
@@ -13,6 +13,13 @@ It also contains implementations of Huffman and FSE ta
 [Zstandard format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
 [format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
 
+While the library's primary objective is code clarity,
+it also happens to compile into a small object file.
+The object file can be made even smaller by removing error messages,
+using the macro directive `ZDEC_NO_MESSAGE` at compilation time.
+This can be reduced even further by foregoing dictionary support,
+by defining `ZDEC_NO_DICTIONARY`.
+
 `harness.c` provides a simple test harness around the decoder:
 
     harness <input-file> <output-file> [dictionary]

Modified: head/sys/contrib/zstd/doc/educational_decoder/harness.c
==============================================================================
--- head/sys/contrib/zstd/doc/educational_decoder/harness.c	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/doc/educational_decoder/harness.c	Sat May 23 21:23:46 2020	(r361426)
@@ -1,10 +1,11 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) 2017-2020, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
  */
 
 #include <stdio.h>
@@ -21,108 +22,98 @@ typedef unsigned char u8;
 // Protect against allocating too much memory for output
 #define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024)
 
-static size_t read_file(const char *path, u8 **ptr)
+// Error message then exit
+#define ERR_OUT(...) { fprintf(stderr, __VA_ARGS__); exit(1); }
+
+
+typedef struct {
+    u8* address;
+    size_t size;
+} buffer_s;
+
+static void freeBuffer(buffer_s b) { free(b.address); }
+
+static buffer_s read_file(const char *path)
 {
     FILE* const f = fopen(path, "rb");
-    if (!f) {
-        fprintf(stderr, "failed to open file %s \n", path);
-        exit(1);
-    }
+    if (!f) ERR_OUT("failed to open file %s \n", path);
 
     fseek(f, 0L, SEEK_END);
     size_t const size = (size_t)ftell(f);
     rewind(f);
 
-    *ptr = malloc(size);
-    if (!ptr) {
-        fprintf(stderr, "failed to allocate memory to hold %s \n", path);
-        exit(1);
-    }
+    void* const ptr = malloc(size);
+    if (!ptr) ERR_OUT("failed to allocate memory to hold %s \n", path);
 
-    size_t const read = fread(*ptr, 1, size, f);
-    if (read != size) {  /* must read everything in one pass */
-        fprintf(stderr, "error while reading file %s \n", path);
-        exit(1);
-    }
+    size_t const read = fread(ptr, 1, size, f);
+    if (read != size) ERR_OUT("error while reading file %s \n", path);
 
     fclose(f);
-
-    return read;
+    buffer_s const b = { ptr, size };
+    return b;
 }
 
-static void write_file(const char *path, const u8 *ptr, size_t size)
+static void write_file(const char* path, const u8* ptr, size_t size)
 {
     FILE* const f = fopen(path, "wb");
-    if (!f) {
-        fprintf(stderr, "failed to open file %s \n", path);
-        exit(1);
-    }
+    if (!f) ERR_OUT("failed to open file %s \n", path);
 
     size_t written = 0;
     while (written < size) {
         written += fwrite(ptr+written, 1, size, f);
-        if (ferror(f)) {
-            fprintf(stderr, "error while writing file %s\n", path);
-            exit(1);
-    }   }
+        if (ferror(f)) ERR_OUT("error while writing file %s\n", path);
+    }
 
     fclose(f);
 }
 
 int main(int argc, char **argv)
 {
-    if (argc < 3) {
-        fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary] \n",
-                argv[0]);
+    if (argc < 3)
+        ERR_OUT("usage: %s <file.zst> <out_path> [dictionary] \n", argv[0]);
 
-        return 1;
-    }
+    buffer_s const input = read_file(argv[1]);
 
-    u8* input;
-    size_t const input_size = read_file(argv[1], &input);
-
-    u8* dict = NULL;
-    size_t dict_size = 0;
+    buffer_s dict = { NULL, 0 };
     if (argc >= 4) {
-        dict_size = read_file(argv[3], &dict);
+        dict = read_file(argv[3]);
     }
 
-    size_t out_capacity = ZSTD_get_decompressed_size(input, input_size);
+    size_t out_capacity = ZSTD_get_decompressed_size(input.address, input.size);
     if (out_capacity == (size_t)-1) {
-        out_capacity = MAX_COMPRESSION_RATIO * input_size;
+        out_capacity = MAX_COMPRESSION_RATIO * input.size;
         fprintf(stderr, "WARNING: Compressed data does not contain "
                         "decompressed size, going to assume the compression "
                         "ratio is at most %d (decompressed size of at most "
                         "%u) \n",
                 MAX_COMPRESSION_RATIO, (unsigned)out_capacity);
     }
-    if (out_capacity > MAX_OUTPUT_SIZE) {
-        fprintf(stderr,
-                "Required output size too large for this implementation \n");
-        return 1;
-    }
+    if (out_capacity > MAX_OUTPUT_SIZE)
+        ERR_OUT("Required output size too large for this implementation \n");
 
     u8* const output = malloc(out_capacity);
-    if (!output) {
-        fprintf(stderr, "failed to allocate memory \n");
-        return 1;
-    }
+    if (!output) ERR_OUT("failed to allocate memory \n");
 
     dictionary_t* const parsed_dict = create_dictionary();
-    if (dict) {
-        parse_dictionary(parsed_dict, dict, dict_size);
+    if (dict.size) {
+#if defined (ZDEC_NO_DICTIONARY)
+        printf("dict.size = %zu \n", dict.size);
+        ERR_OUT("no dictionary support \n");
+#else
+        parse_dictionary(parsed_dict, dict.address, dict.size);
+#endif
     }
     size_t const decompressed_size =
         ZSTD_decompress_with_dict(output, out_capacity,
-                                  input, input_size,
+                                  input.address, input.size,
                                   parsed_dict);
 
     free_dictionary(parsed_dict);
 
     write_file(argv[2], output, decompressed_size);
 
-    free(input);
+    freeBuffer(input);
+    freeBuffer(dict);
     free(output);
-    free(dict);
     return 0;
 }

Modified: head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c
==============================================================================
--- head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c	Sat May 23 20:39:36 2020	(r361425)
+++ head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c	Sat May 23 21:23:46 2020	(r361426)
@@ -1,34 +1,52 @@
 /*
- * Copyright (c) 2017-present, Facebook, Inc.
+ * Copyright (c) 2017-2020, Facebook, Inc.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
  */
 
 /// Zstandard educational decoder implementation
 /// See https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
 
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
+#include <stdint.h>   // uint8_t, etc.
+#include <stdlib.h>   // malloc, free, exit
+#include <stdio.h>    // fprintf
+#include <string.h>   // memset, memcpy
 #include "zstd_decompress.h"
 
-/******* UTILITY MACROS AND TYPES *********************************************/
-// Max block size decompressed size is 128 KB and literal blocks can't be
-// larger than their block
-#define MAX_LITERALS_SIZE ((size_t)128 * 1024)
 
+/******* IMPORTANT CONSTANTS *********************************************/
+
+// Zstandard frame
+// "Magic_Number
+// 4 Bytes, little-endian format. Value : 0xFD2FB528"
+#define ZSTD_MAGIC_NUMBER 0xFD2FB528U
+
+// The size of `Block_Content` is limited by `Block_Maximum_Size`,
+#define ZSTD_BLOCK_SIZE_MAX ((size_t)128 * 1024)
+
+// literal blocks can't be larger than their block
+#define MAX_LITERALS_SIZE ZSTD_BLOCK_SIZE_MAX
+
+
+/******* UTILITY MACROS AND TYPES *********************************************/
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 
+#if defined(ZDEC_NO_MESSAGE)
+#define MESSAGE(...)
+#else
+#define MESSAGE(...)  fprintf(stderr, "" __VA_ARGS__)
+#endif
+
 /// This decoder calls exit(1) when it encounters an error, however a production
 /// library should propagate error codes
 #define ERROR(s)                                                               \
     do {                                                                       \
-        fprintf(stderr, "Error: %s\n", s);                                     \
+        MESSAGE("Error: %s\n", s);                                     \
         exit(1);                                                               \
     } while (0)
 #define INP_SIZE()                                                             \
@@ -39,12 +57,12 @@
 #define BAD_ALLOC() ERROR("Memory allocation error")
 #define IMPOSSIBLE() ERROR("An impossibility has occurred")
 
-typedef uint8_t u8;
+typedef uint8_t  u8;
 typedef uint16_t u16;
 typedef uint32_t u32;
 typedef uint64_t u64;
 
-typedef int8_t i8;
+typedef int8_t  i8;
 typedef int16_t i16;
 typedef int32_t i32;
 typedef int64_t i64;
@@ -176,10 +194,6 @@ static void HUF_init_dtable_usingweights(HUF_dtable *c
 
 /// Free the malloc'ed parts of a decoding table
 static void HUF_free_dtable(HUF_dtable *const dtable);
-
-/// Deep copy a decoding table, so that it can be used and free'd without
-/// impacting the source table.
-static void HUF_copy_dtable(HUF_dtable *const dst, const HUF_dtable *const src);

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
