svn commit: r346364 - in head: sys/contrib/zstd sys/contrib/zstd/contrib/docker sys/contrib/zstd/contrib/experimental_dict_builders sys/contrib/zstd/contrib/largeNbDicts sys/contrib/zstd/contrib/pr...
Conrad Meyer
cem at FreeBSD.org
Tue Sep 3 14:07:59 UTC 2019
Author: cem
Date: Fri Apr 19 02:54:13 2019
New Revision: 346364
URL: https://svnweb.freebsd.org/changeset/base/346364
Log:
Update to Zstandard 1.4.0
The full release notes can be found on Github:
https://github.com/facebook/zstd/releases/tag/v1.4.0
Relnotes: yes
Added:
head/sys/contrib/zstd/CHANGELOG
- copied unchanged from r346362, vendor/zstd/dist/CHANGELOG
head/sys/contrib/zstd/contrib/docker/
- copied from r346362, vendor/zstd/dist/contrib/docker/
head/sys/contrib/zstd/contrib/experimental_dict_builders/
- copied from r346362, vendor/zstd/dist/contrib/experimental_dict_builders/
head/sys/contrib/zstd/contrib/largeNbDicts/
- copied from r346362, vendor/zstd/dist/contrib/largeNbDicts/
head/sys/contrib/zstd/contrib/premake/
- copied from r346362, vendor/zstd/dist/contrib/premake/
head/sys/contrib/zstd/contrib/snap/
- copied from r346362, vendor/zstd/dist/contrib/snap/
head/sys/contrib/zstd/examples/
- copied from r346362, vendor/zstd/dist/examples/
head/sys/contrib/zstd/programs/timefn.c
- copied unchanged from r346362, vendor/zstd/dist/programs/timefn.c
head/sys/contrib/zstd/programs/timefn.h
- copied unchanged from r346362, vendor/zstd/dist/programs/timefn.h
head/sys/contrib/zstd/tests/fuzz/dictionary_decompress.c
- copied unchanged from r346362, vendor/zstd/dist/tests/fuzz/dictionary_decompress.c
head/sys/contrib/zstd/tests/fuzz/dictionary_round_trip.c
- copied unchanged from r346362, vendor/zstd/dist/tests/fuzz/dictionary_round_trip.c
Modified:
head/sys/contrib/zstd/Makefile
head/sys/contrib/zstd/README.md
head/sys/contrib/zstd/contrib/pzstd/Makefile
head/sys/contrib/zstd/contrib/pzstd/Pzstd.cpp
head/sys/contrib/zstd/contrib/pzstd/utils/Range.h
head/sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h
head/sys/contrib/zstd/doc/README.md
head/sys/contrib/zstd/doc/educational_decoder/Makefile
head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c
head/sys/contrib/zstd/doc/zstd_manual.html
head/sys/contrib/zstd/lib/Makefile
head/sys/contrib/zstd/lib/README.md
head/sys/contrib/zstd/lib/common/compiler.h
head/sys/contrib/zstd/lib/common/fse.h
head/sys/contrib/zstd/lib/common/threading.c
head/sys/contrib/zstd/lib/common/xxhash.c
head/sys/contrib/zstd/lib/common/zstd_internal.h
head/sys/contrib/zstd/lib/compress/fse_compress.c
head/sys/contrib/zstd/lib/compress/zstd_compress.c
head/sys/contrib/zstd/lib/compress/zstd_compress_internal.h
head/sys/contrib/zstd/lib/compress/zstd_fast.c
head/sys/contrib/zstd/lib/compress/zstd_lazy.h
head/sys/contrib/zstd/lib/compress/zstd_ldm.c
head/sys/contrib/zstd/lib/compress/zstd_opt.c
head/sys/contrib/zstd/lib/compress/zstdmt_compress.c
head/sys/contrib/zstd/lib/compress/zstdmt_compress.h
head/sys/contrib/zstd/lib/decompress/zstd_ddict.c
head/sys/contrib/zstd/lib/decompress/zstd_decompress.c
head/sys/contrib/zstd/lib/decompress/zstd_decompress_block.c
head/sys/contrib/zstd/lib/decompress/zstd_decompress_internal.h
head/sys/contrib/zstd/lib/dictBuilder/cover.c
head/sys/contrib/zstd/lib/dictBuilder/cover.h
head/sys/contrib/zstd/lib/dictBuilder/fastcover.c
head/sys/contrib/zstd/lib/dictBuilder/zdict.h
head/sys/contrib/zstd/lib/legacy/zstd_legacy.h
head/sys/contrib/zstd/lib/legacy/zstd_v01.c
head/sys/contrib/zstd/lib/legacy/zstd_v01.h
head/sys/contrib/zstd/lib/legacy/zstd_v02.c
head/sys/contrib/zstd/lib/legacy/zstd_v02.h
head/sys/contrib/zstd/lib/legacy/zstd_v03.c
head/sys/contrib/zstd/lib/legacy/zstd_v03.h
head/sys/contrib/zstd/lib/legacy/zstd_v04.c
head/sys/contrib/zstd/lib/legacy/zstd_v04.h
head/sys/contrib/zstd/lib/legacy/zstd_v05.c
head/sys/contrib/zstd/lib/legacy/zstd_v05.h
head/sys/contrib/zstd/lib/legacy/zstd_v06.c
head/sys/contrib/zstd/lib/legacy/zstd_v06.h
head/sys/contrib/zstd/lib/legacy/zstd_v07.c
head/sys/contrib/zstd/lib/legacy/zstd_v07.h
head/sys/contrib/zstd/lib/zstd.h
head/sys/contrib/zstd/programs/Makefile
head/sys/contrib/zstd/programs/benchfn.c
head/sys/contrib/zstd/programs/benchfn.h
head/sys/contrib/zstd/programs/benchzstd.c
head/sys/contrib/zstd/programs/benchzstd.h
head/sys/contrib/zstd/programs/dibio.c
head/sys/contrib/zstd/programs/fileio.c
head/sys/contrib/zstd/programs/fileio.h
head/sys/contrib/zstd/programs/platform.h
head/sys/contrib/zstd/programs/util.c
head/sys/contrib/zstd/programs/util.h
head/sys/contrib/zstd/programs/zstd.1
head/sys/contrib/zstd/programs/zstd.1.md
head/sys/contrib/zstd/programs/zstdcli.c
head/sys/contrib/zstd/programs/zstdgrep
head/sys/contrib/zstd/tests/Makefile
head/sys/contrib/zstd/tests/README.md
head/sys/contrib/zstd/tests/decodecorpus.c
head/sys/contrib/zstd/tests/fullbench.c
head/sys/contrib/zstd/tests/fuzz/Makefile
head/sys/contrib/zstd/tests/fuzz/README.md
head/sys/contrib/zstd/tests/fuzz/fuzz.py
head/sys/contrib/zstd/tests/fuzz/fuzz_helpers.h
head/sys/contrib/zstd/tests/fuzz/regression_driver.c
head/sys/contrib/zstd/tests/fuzz/simple_round_trip.c
head/sys/contrib/zstd/tests/fuzz/stream_decompress.c
head/sys/contrib/zstd/tests/fuzz/stream_round_trip.c
head/sys/contrib/zstd/tests/fuzz/zstd_helpers.c
head/sys/contrib/zstd/tests/fuzz/zstd_helpers.h
head/sys/contrib/zstd/tests/fuzzer.c
head/sys/contrib/zstd/tests/legacy.c
head/sys/contrib/zstd/tests/paramgrill.c
head/sys/contrib/zstd/tests/playTests.sh
head/sys/contrib/zstd/tests/poolTests.c
head/sys/contrib/zstd/tests/regression/config.c
head/sys/contrib/zstd/tests/regression/method.c
head/sys/contrib/zstd/tests/regression/results.csv
head/sys/contrib/zstd/tests/regression/test.c
head/sys/contrib/zstd/tests/roundTripCrash.c
head/sys/contrib/zstd/tests/symbols.c
head/sys/contrib/zstd/tests/zbufftest.c
head/sys/contrib/zstd/tests/zstreamtest.c
head/sys/contrib/zstd/zlibWrapper/Makefile
head/sys/contrib/zstd/zlibWrapper/README.md
head/sys/contrib/zstd/zlibWrapper/examples/fitblk.c
head/sys/contrib/zstd/zlibWrapper/examples/fitblk_original.c
head/sys/contrib/zstd/zlibWrapper/examples/zwrapbench.c
head/sys/contrib/zstd/zlibWrapper/gzguts.h
head/usr.bin/zstd/Makefile
Directory Properties:
head/sys/contrib/zstd/ (props changed)
Copied: head/sys/contrib/zstd/CHANGELOG (from r346362, vendor/zstd/dist/CHANGELOG)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/contrib/zstd/CHANGELOG Fri Apr 19 02:54:13 2019 (r346364, copy of r346362, vendor/zstd/dist/CHANGELOG)
@@ -0,0 +1,461 @@
+v1.4.0
+perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln
+api: Move the advanced API, including all functions in the staging section, to the stable section
+api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress
+api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter
+api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter
+api: Don't export ZSTDMT functions from the shared library by default
+api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT
+api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao
+api: Fix ZSTD_decompressDCtx() corner cases with a dictionary
+api: Move ZSTD_getDictID_*() functions to the stable section
+api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln
+api: Allow compression parameters to be set when a dictionary is used
+api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called
+api: Fix ZSTD_estimateCStreamSize_usingCCtxParams()
+api: Setting ZSTD_d_maxWindowLog to 0 means use the default
+cli: Ensure that a dictionary is not used to compress itself by @shakeelrao
+cli: Add --[no-]compress-literals flag to enable or disable literal compression
+doc: Update the examples to use the advanced API
+doc: Explain how to transition from old streaming functions to the advanced API in the header
+build: Improve the Windows release packages
+build: Improve CMake build by @hjmjohnson
+build: Build fixes for FreeBSD by @lwhsu
+build: Remove redundant warnings by @thatsafunnyname
+build: Fix tests on OpenBSD by @bket
+build: Extend fuzzer build system to work with the new clang engine
+build: CMake now creates the libzstd.so.1 symlink
+build: Improve Menson build by @lzutao
+misc: Fix symbolic link detection on FreeBSD
+misc: Use physical core count for -T0 on FreeBSD by @cemeyer
+misc: Fix zstd --list on truncated files by @kostmo
+misc: Improve logging in debug mode by @felixhandte
+misc: Add CirrusCI tests by @lwhsu
+misc: Optimize dictionary memory usage in corner cases
+misc: Improve the dictionary builder on small or homogeneous data
+misc: Fix spelling across the repo by @jsoref
+
+v1.3.8
+perf: better decompression speed on large files (+7%) and cold dictionaries (+15%)
+perf: slightly better compression ratio at high compression modes
+api : finalized advanced API, last stage before "stable" status
+api : new --rsyncable mode, by @terrelln
+api : support decompression of empty frames into NULL (used to be an error) (#1385)
+build: new set of macros to build a minimal size decoder, by @felixhandte
+build: fix compilation on MIPS32, reported by @clbr (#1441)
+build: fix compilation with multiple -arch flags, by @ryandesign
+build: highly upgraded meson build, by @lzutao
+build: improved buck support, by @obelisk
+build: fix cmake script : can create debug build, by @pitrou
+build: Makefile : grep works on both colored consoles and systems without color support
+build: fixed zstd-pgo, by @bmwiedemann
+cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423)
+cli : --no-progress flag, preserving final summary (#1371), by @terrelln
+cli : ensure destination file is not source file (#1422)
+cli : clearer error messages, especially when input file not present
+doc : clarified zstd_compression_format.md, by @ulikunitz
+misc: fixed zstdgrep, returns 1 on failure, by @lzutao
+misc: NEWS renamed as CHANGELOG, in accordance with fboss
+
+v1.3.7
+perf: slightly better decompression speed on clang (depending on hardware target)
+fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
+build: no longer build backtrace by default in release mode; restrict further automatic mode
+build: control backtrace support through build macro BACKTRACE
+misc: added man pages for zstdless and zstdgrep, by @samrussell
+
+v1.3.6
+perf: much faster dictionary builder, by @jenniferliu
+perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
+perf: faster dictionary decompression when using a very large number of dictionaries simultaneously
+cli : fix : does no longer overwrite destination when source does not exist (#1082)
+cli : new command --adapt, for automatic compression level adaptation
+api : fix : block api can be streamed with > 4 GB, reported by @catid
+api : reduced ZSTD_DDict size by 2 KB
+api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel().
+build: support Haiku target, by @korli
+build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT.
+doc : zstd_compression_format.md updated to match wording in IETF RFC 8478
+misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
+
+v1.3.5
+perf: much faster dictionary compression, by @felixhandte
+perf: small quality improvement for dictionary generation, by @terrelln
+perf: slightly improved high compression levels (notably level 19)
+mem : automatic memory release for long duration contexts
+cli : fix : overlapLog can be manually set
+cli : fix : decoding invalid lz4 frames
+api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln
+api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln
+build: select custom libzstd scope through control macros, by @GeorgeLu97
+build: OpenBSD patch, by @bket
+build: make and make all are compatible with -j
+doc : clarify zstd_compression_format.md, updated for IETF RFC process
+misc: pzstd compatible with reproducible compilation, by @lamby
+
+v1.3.4
+perf: faster speed (especially decoding speed) on recent cpus (haswell+)
+perf: much better performance associating --long with multi-threading, by @terrelln
+perf: better compression at levels 13-15
+cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior)
+cli : smoother status report in multi-threading mode
+cli : added command --fast=#, for faster compression modes
+cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb)
+api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode
+api : compression levels can be negative, for even more speed
+api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime
+api : ZSTDMT can accept new compression parameters during compression
+api : implemented all advanced dictionary decompression prototypes
+build: improved meson recipe, by Shawn Landden (@shawnl)
+build: VS2017 scripts, by @HaydnTrigg
+misc: all /contrib projects fixed
+misc: added /contrib/docker script by @gyscos
+
+v1.3.3
+perf: faster zstd_opt strategy (levels 16-19)
+fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen
+cli : fix : content size written in header by default
+cli : fix : improved LZ4 format support, by @felixhandte
+cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file
+api : fix : support large skippable frames, by @terrelln
+api : fix : streaming interface was adding a useless 3-bytes null block to small frames
+api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown"
+build: fix : compilation under rhel6 and centos6, reported by @pixelb
+build: added `check` target
+
+v1.3.2
+new : long range mode, using --long command, by Stella Lau (@stellamplau)
+new : ability to generate and decode magicless frames (#591)
+changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode
+fix : multi-threading compression works with custom allocators
+fix : ZSTD_sizeof_CStream() was over-evaluating memory usage
+fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22)
+fix : 32-bits build can now decode large offsets (levels 21+)
+cli : added LZ4 frame support by default, by Felix Handte (@felixhandte)
+cli : improved --list output
+cli : new : can split input file for dictionary training, using command -B#
+cli : new : clean operation artefact on Ctrl-C interruption
+cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851)
+cli : fix : write file size in header in multiple-files mode
+api : added macro ZSTD_COMPRESSBOUND() for static allocation
+api : experimental : new advanced decompression API
+api : fix : sizeof_CCtx() used to over-estimate
+build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819)
+build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818)
+example : added streaming_memory_usage
+license : changed /examples license to BSD + GPLv2
+license : fix a few header files to reflect new license (#825)
+
+v1.3.1
+New license : BSD + GPLv2
+perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk)
+perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760)
+cli : improved and fixed --list command, by @ib (#772)
+cli : command -vV to list supported formats, by @ib (#771)
+build : fixed binary variants, reported by @svenha (#788)
+build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718)
+API exp : breaking change : ZSTD_getframeHeader() provides more information
+API exp : breaking change : pinned down values of error codes
+doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz)
+new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74)
+new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau)
+updated : contrib/linux-kernel, by Nick Terrell (@terrelln)
+
+v1.3.0
+cli : new : `--list` command, by Paul Cruz
+cli : changed : xz/lzma support enabled by default
+cli : changed : `-t *` continue processing list after a decompression error
+API : added : ZSTD_versionString()
+API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell
+API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
+API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
+API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
+API exp : clarified memory estimation / measurement functions.
+API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
+tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz
+new : contrib/seekable_format, demo and API, by Sean Purcell
+changed : contrib/linux-kernel, updated version and license, by Nick Terrell
+
+v1.2.0
+cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
+cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
+cli : new : zstdmt symlink hardwired to `zstd -T0`
+cli : new : command --threads=# (#671)
+cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
+cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
+cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
+cli : fix : does not output compressed data on console
+cli : fix : ignore symbolic links unless --force specified,
+API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
+API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
+API : improved: ZSTDMT_compressCCtx() reduced memory usage
+API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
+API : fix : src size stored in frame header is controlled at end of frame
+API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
+API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
+build: improved cmake script, by @Majlen
+build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
+tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
+new : contrib/linux-kernel version, by Nick Terrell
+
+v1.1.4
+cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
+cli : new : advanced benchmark command --priority=rt
+cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
+cli : fix : --rm remains silent when input is stdin
+cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
+speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
+memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
+arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
+API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
+build : new: meson build system in contrib/meson, by Dima Krasner
+build : improved cmake script, by @Majlen
+build : added -Wformat-security flag, as recommended by Padraig Brady
+doc : new : educational decoder, by Sean Purcell
+
+v1.1.3
+cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
+cli : new : experimental target `make zstdmt`, with multi-threading support
+cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
+cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
+cli : fix zstdless on Mac OS-X, by Andrew Janke
+cli : fix #232 "compress non-files"
+dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
+API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
+API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
+API : new : ZDICT_finalizeDictionary()
+API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
+API : fix : all symbols properly exposed in libzstd, by Nick Terrell
+build : support for Solaris target, by Przemyslaw Skibinski
+doc : clarified specification, by Sean Purcell
+
+v1.1.2
+API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
+API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
+API : zbuff : changed : prototypes now generate deprecation warnings
+lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
+lib : changed : only public ZSTD_ symbols are now exposed
+lib : changed : reduced usage of stack memory
+lib : fixed : several corner case bugs, by Nick Terrell
+cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
+cli : new : preserve file attributes
+cli : new : added zstdless and zstdgrep tools
+cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
+cli : fixed : zstdcat
+zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
+install : better compatibility with FreeBSD, by Dimitry Andric
+source tree : changed : zbuff source files moved to lib/deprecated
+
+v1.1.1
+New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
+New : doc/zstd_manual.html, by Przemyslaw Skibinski
+Improved : slightly better compression ratio at --ultra levels (>= 20)
+Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
+Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
+Added : example/multiple_streaming_compression.c
+Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
+Updated man page
+Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
+
+v1.1.0
+New : contrib/pzstd, parallel version of zstd, by Nick Terrell
+added : NetBSD install target (#338)
+Improved : speed for batches of small files
+Improved : speed of zlib wrapper, by Przemyslaw Skibinski
+Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
+Fixed : CLI -d output to stdout by default when input is stdin (#322)
+Fixed : CLI correctly detects console on Mac OS-X
+Fixed : CLI supports recursive mode `-r` on Mac OS-X
+Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
+Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
+Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
+Fixed : zstd-pgo, reported by octoploid (#329)
+
+v1.0.0
+Change Licensing, all project is now BSD, Copyright Facebook
+Small decompression speed improvement
+API : Streaming API supports legacy format
+API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter()
+CLI supports legacy formats v0.4+
+Fixed : compression fails on certain huge files, reported by Jesse McGrew
+Enhanced documentation, by Przemyslaw Skibinski
+
+v0.8.1
+New streaming API
+Changed : --ultra now enables levels beyond 19
+Changed : -i# now selects benchmark time in second
+Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
+Fixed : speed regression on specific patterns (#272)
+Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
+Fixed : ICC compilation, by Przemyslaw Skibinski
+
+v0.8.0
+Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
+New : Build on FreeBSD and DragonFly, thanks to JrMarino
+Changed : modified API : ZSTD_compressEnd()
+Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
+Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
+Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
+Fixed : checksum correctly checked in single-pass mode
+Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
+Modified : minor compression level adaptations
+Updated : compression format specification to v0.2.0
+changed : zstd.h moved to /lib directory
+
+v0.7.5
+Transition version, supporting decoding of v0.8.x
+
+v0.7.4
+Added : homebrew for Mac, by Daniel Cade
+Added : more examples
+Fixed : segfault when using small dictionaries, reported by Felix Handte
+Modified : default compression level for CLI is now 3
+Updated : specification, to v0.1.1
+
+v0.7.3
+New : compression format specification
+New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
+New : `ZSTD_getDecompressedSize()`
+New : OpenBSD target, by Juan Francisco Cantero Hurtado
+New : `examples` directory
+fixed : dictBuilder using HC levels, reported by Bartosz Taudul
+fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
+fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
+modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
+modified : legacy functions no longer need magic number
+
+v0.7.2
+fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
+fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
+fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
+
+v0.7.1
+fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
+fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
+fixed : corruption issue, reported by cj
+modified : checksum enabled by default in command line mode
+
+v0.7.0
+New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
+New : Command `--rm`, to remove source file after successful de/compression
+New : Visual build scripts, by Christophe Chevalier
+New : Support for Sparse File-systems (do not use space for zero-filled sectors)
+New : Frame checksum support
+New : Support pass-through mode (when using `-df`)
+API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
+API : create dictionary files from custom content, by Giuseppe Ottaviano
+API : support for custom malloc/free functions
+New : controllable Dictionary ID
+New : Support for skippable frames
+
+v0.6.1
+New : zlib wrapper API, thanks to Przemyslaw Skibinski
+New : Ability to compile compressor / decompressor separately
+Changed : new lib directory structure
+Fixed : Legacy codec v0.5 compatible with dictionary decompression
+Fixed : Decoder corruption error (#173)
+Fixed : null-string roundtrip (#176)
+New : benchmark mode can select directory as input
+Experimental : midipix support, VMS support
+
+v0.6.0
+Stronger high compression modes, thanks to Przemyslaw Skibinski
+API : ZSTD_getFrameParams() provides size of decompressed content
+New : highest compression modes require `--ultra` command to fully unleash their capacity
+Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
+
+v0.5.1
+New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
+Changed : Dictionary builder integrated into libzstd and zstd cli
+Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
+Fix : high compression modes for big-endian platforms
+New : zstd cli : `-t` | `--test` command
+
+v0.5.0
+New : dictionary builder utility
+Changed : streaming & dictionary API
+Improved : better compression of small data
+
+v0.4.7
+Improved : small compression speed improvement in HC mode
+Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
+fix : bt search bug
+
+v0.4.6
+fix : fast compression mode on Windows
+New : cmake configuration file, thanks to Artyom Dymchenko
+Improved : high compression mode on repetitive data
+New : block-level API
+New : ZSTD_duplicateCCtx()
+
+v0.4.5
+new : -m/--multiple : compress/decompress multiple files
+
+v0.4.4
+Fixed : high compression modes for Windows 32 bits
+new : external dictionary API extended to buffered mode and accessible through command line
+new : windows DLL project, thanks to Christophe Chevalier
+
+v0.4.3 :
+new : external dictionary API
+new : zstd-frugal
+
+v0.4.2 :
+Generic minor improvements for small blocks
+Fixed : big-endian compatibility, by Peter Harris (#85)
+
+v0.4.1
+Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
+removed `zstd.c`
+
+v0.4.0
+Command line utility compatible with high compression levels
+Removed zstdhc => merged into zstd
+Added : ZBUFF API (see zstd_buffered.h)
+Rolling buffer support
+
+v0.3.6
+small blocks params
+
+v0.3.5
+minor generic compression improvements
+
+v0.3.4
+Faster fast cLevels
+
+v0.3.3
+Small compression ratio improvement
+
+v0.3.2
+Fixed Visual Studio
+
+v0.3.1 :
+Small compression ratio improvement
+
+v0.3
+HC mode : compression levels 2-26
+
+v0.2.2
+Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
+
+v0.2.1
+Fix : Read errors, advanced fuzzer tests, by Hanno Böck
+
+v0.2.0
+**Breaking format change**
+Faster decompression speed
+Can still decode v0.1 format
+
+v0.1.3
+fix uninitialization warning, reported by Evan Nemerson
+
+v0.1.2
+frame concatenation support
+
+v0.1.1
+fix compression bug
+detects write-flush errors
+
+v0.1.0
+first release
Modified: head/sys/contrib/zstd/Makefile
==============================================================================
--- head/sys/contrib/zstd/Makefile Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/Makefile Fri Apr 19 02:54:13 2019 (r346364)
@@ -156,7 +156,7 @@ list:
done \
} | column -t -s $$'\t'
-.PHONY: install clangtest armtest usan asan uasan
+.PHONY: install armtest usan asan uasan
install:
@$(MAKE) -C $(ZSTDDIR) $@
@$(MAKE) -C $(PRGDIR) $@
@@ -188,7 +188,7 @@ gcc7build: clean
.PHONY: clangbuild
clangbuild: clean
clang -v
- CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+ CXX=clang++ CC=clang CFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation" $(MAKE) all
m32build: clean
gcc -v
@@ -231,10 +231,6 @@ gcc5test: clean
gcc6test: clean
gcc-6 -v
$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
-
-clangtest: clean
- clang -v
- $(MAKE) all CXX=clang++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
armtest: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
Modified: head/sys/contrib/zstd/README.md
==============================================================================
--- head/sys/contrib/zstd/README.md Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/README.md Fri Apr 19 02:54:13 2019 (r346364)
@@ -14,6 +14,7 @@ a list of known ports and bindings is provided on [Zst
[![Build Status][travisDevBadge]][travisLink]
[![Build status][AppveyorDevBadge]][AppveyorLink]
[![Build status][CircleDevBadge]][CircleLink]
+[![Build status][CirrusDevBadge]][CirrusLink]
[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
[travisLink]: https://travis-ci.org/facebook/zstd
@@ -21,14 +22,16 @@ a list of known ports and bindings is provided on [Zst
[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
[CircleLink]: https://circleci.com/gh/facebook/zstd
+[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev
+[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd
## Benchmarks
For reference, several fast compression algorithms were tested and compared
-on a server running Linux Debian (`Linux version 4.14.0-3-amd64`),
-with a Core i7-6700K CPU @ 4.0GHz,
+on a server running Arch Linux (`Linux version 5.0.5-arch1-1`),
+with a Core i9-9900K CPU @ 5.0GHz,
using [lzbench], an open-source in-memory benchmark by @inikep
-compiled with [gcc] 7.3.0,
+compiled with [gcc] 8.2.1,
on the [Silesia compression corpus].
[lzbench]: https://github.com/inikep/lzbench
@@ -37,14 +40,14 @@ on the [Silesia compression corpus].
| Compressor name | Ratio | Compression| Decompress.|
| --------------- | ------| -----------| ---------- |
-| **zstd 1.3.4 -1** | 2.877 | 470 MB/s | 1380 MB/s |
-| zlib 1.2.11 -1 | 2.743 | 110 MB/s | 400 MB/s |
-| brotli 1.0.2 -0 | 2.701 | 410 MB/s | 430 MB/s |
-| quicklz 1.5.0 -1 | 2.238 | 550 MB/s | 710 MB/s |
-| lzo1x 2.09 -1 | 2.108 | 650 MB/s | 830 MB/s |
-| lz4 1.8.1 | 2.101 | 750 MB/s | 3700 MB/s |
-| snappy 1.1.4 | 2.091 | 530 MB/s | 1800 MB/s |
-| lzf 3.6 -1 | 2.077 | 400 MB/s | 860 MB/s |
+| **zstd 1.4.0 -1** | 2.884 | 530 MB/s | 1360 MB/s |
+| zlib 1.2.11 -1 | 2.743 | 110 MB/s | 440 MB/s |
+| brotli 1.0.7 -0 | 2.701 | 430 MB/s | 470 MB/s |
+| quicklz 1.5.0 -1 | 2.238 | 600 MB/s | 800 MB/s |
+| lzo1x 2.09 -1 | 2.106 | 680 MB/s | 950 MB/s |
+| lz4 1.8.3 | 2.101 | 800 MB/s | 4220 MB/s |
+| snappy 1.1.4 | 2.073 | 580 MB/s | 2020 MB/s |
+| lzf 3.6 -1 | 2.077 | 440 MB/s | 930 MB/s |
[zlib]: http://www.zlib.net/
[LZ4]: http://www.lz4.org/
Modified: head/sys/contrib/zstd/contrib/pzstd/Makefile
==============================================================================
--- head/sys/contrib/zstd/contrib/pzstd/Makefile Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/contrib/pzstd/Makefile Fri Apr 19 02:54:13 2019 (r346364)
@@ -190,13 +190,15 @@ $(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a
# Rules to build the tests
-test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \
+test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o \
+ $(PROGDIR)/util.o Options.o \
Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
$(LD_COMMAND)
test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
test/%Test$(EXT): LIBS += -lgtest -lgtest_main
-test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o Options.o Pzstd.o \
+test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o \
+ $(PROGDIR)/util.o Options.o Pzstd.o \
SkippableFrame.o $(ZSTDDIR)/libzstd.a
$(LD_COMMAND)
Modified: head/sys/contrib/zstd/contrib/pzstd/Pzstd.cpp
==============================================================================
--- head/sys/contrib/zstd/contrib/pzstd/Pzstd.cpp Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/contrib/pzstd/Pzstd.cpp Fri Apr 19 02:54:13 2019 (r346364)
@@ -55,7 +55,7 @@ static std::uint64_t handleOneInput(const Options &opt
SharedState& state) {
auto inputSize = fileSizeOrZero(inputFile);
// WorkQueue outlives ThreadPool so in the case of error we are certain
- // we don't accidently try to call push() on it after it is destroyed
+ // we don't accidentally try to call push() on it after it is destroyed
WorkQueue<std::shared_ptr<BufferWorkQueue>> outs{options.numThreads + 1};
std::uint64_t bytesRead;
std::uint64_t bytesWritten;
Modified: head/sys/contrib/zstd/contrib/pzstd/utils/Range.h
==============================================================================
--- head/sys/contrib/zstd/contrib/pzstd/utils/Range.h Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/contrib/pzstd/utils/Range.h Fri Apr 19 02:54:13 2019 (r346364)
@@ -9,7 +9,7 @@
/**
* A subset of `folly/Range.h`.
- * All code copied verbatiam modulo formatting
+ * All code copied verbatim modulo formatting
*/
#pragma once
Modified: head/sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h
==============================================================================
--- head/sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h Fri Apr 19 02:54:13 2019 (r346364)
@@ -54,7 +54,7 @@ class ResourcePool {
/**
* @returns A unique pointer to a resource. The resource is null iff
- * there are no avaiable resources and `factory()` returns null.
+ * there are no available resources and `factory()` returns null.
*/
UniquePtr get() {
std::lock_guard<std::mutex> lock(mutex_);
Modified: head/sys/contrib/zstd/doc/README.md
==============================================================================
--- head/sys/contrib/zstd/doc/README.md Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/doc/README.md Fri Apr 19 02:54:13 2019 (r346364)
@@ -12,8 +12,8 @@ __`zstd_compression_format.md`__ : This document defin
Compliant decoders must adhere to this document,
and compliant encoders must generate data that follows it.
-Should you look for ressources to develop your own port of Zstandard algorithm,
-you may find the following ressources useful :
+Should you look for resources to develop your own port of Zstandard algorithm,
+you may find the following resources useful :
__`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder,
compliant with the Zstandard compression format.
Modified: head/sys/contrib/zstd/doc/educational_decoder/Makefile
==============================================================================
--- head/sys/contrib/zstd/doc/educational_decoder/Makefile Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/doc/educational_decoder/Makefile Fri Apr 19 02:54:13 2019 (r346364)
@@ -7,7 +7,7 @@ CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDD
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
- -Wstrict-prototypes -Wundef -Wformat-security \
+ -Wstrict-prototypes -Wundef \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls
CFLAGS += $(DEBUGFLAGS)
Modified: head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c
==============================================================================
--- head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c Fri Apr 19 02:54:13 2019 (r346364)
@@ -358,7 +358,7 @@ static u32 copy_literals(const size_t seq, istream_t *
ostream_t *const out);
// Given an offset code from a sequence command (either an actual offset value
-// or an index for previous offset), computes the correct offset and udpates
+// or an index for previous offset), computes the correct offset and updates
// the offset history
static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist);
Modified: head/sys/contrib/zstd/doc/zstd_manual.html
==============================================================================
--- head/sys/contrib/zstd/doc/zstd_manual.html Fri Apr 19 02:28:04 2019 (r346363)
+++ head/sys/contrib/zstd/doc/zstd_manual.html Fri Apr 19 02:54:13 2019 (r346364)
@@ -1,37 +1,46 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.3.8 Manual</title>
+<title>zstd 1.4.0 Manual</title>
</head>
<body>
-<h1>zstd 1.3.8 Manual</h1>
+<h1>zstd 1.4.0 Manual</h1>
<hr>
<a name="Contents"></a><h2>Contents</h2>
<ol>
<li><a href="#Chapter1">Introduction</a></li>
<li><a href="#Chapter2">Version</a></li>
<li><a href="#Chapter3">Default constant</a></li>
-<li><a href="#Chapter4">Simple API</a></li>
-<li><a href="#Chapter5">Explicit context</a></li>
-<li><a href="#Chapter6">Simple dictionary API</a></li>
-<li><a href="#Chapter7">Bulk processing dictionary API</a></li>
-<li><a href="#Chapter8">Streaming</a></li>
-<li><a href="#Chapter9">Streaming compression - HowTo</a></li>
-<li><a href="#Chapter10">Streaming decompression - HowTo</a></li>
-<li><a href="#Chapter11">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
-<li><a href="#Chapter12">Candidate API for promotion to stable status</a></li>
-<li><a href="#Chapter13">Advanced compression API</a></li>
-<li><a href="#Chapter14">experimental API (static linking only)</a></li>
-<li><a href="#Chapter15">Frame size functions</a></li>
-<li><a href="#Chapter16">Memory management</a></li>
-<li><a href="#Chapter17">Advanced compression functions</a></li>
-<li><a href="#Chapter18">Advanced decompression functions</a></li>
-<li><a href="#Chapter19">Advanced streaming functions</a></li>
-<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li>
-<li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li>
-<li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li>
-<li><a href="#Chapter23">ZSTD_getFrameHeader() :</a></li>
-<li><a href="#Chapter24">Block level API</a></li>
+<li><a href="#Chapter4">Constants</a></li>
+<li><a href="#Chapter5">Simple API</a></li>
+<li><a href="#Chapter6">Explicit context</a></li>
+<li><a href="#Chapter7">Advanced compression API</a></li>
+<li><a href="#Chapter8">Advanced decompression API</a></li>
+<li><a href="#Chapter9">Streaming</a></li>
+<li><a href="#Chapter10">Streaming compression - HowTo</a></li>
+<li><a href="#Chapter11">This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and</a></li>
+<li><a href="#Chapter12">Equivalent to:</a></li>
+<li><a href="#Chapter13">Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).</a></li>
+<li><a href="#Chapter14">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).</a></li>
+<li><a href="#Chapter15">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).</a></li>
+<li><a href="#Chapter16">Streaming decompression - HowTo</a></li>
+<li><a href="#Chapter17">Simple dictionary API</a></li>
+<li><a href="#Chapter18">Bulk processing dictionary API</a></li>
+<li><a href="#Chapter19">Dictionary helper functions</a></li>
+<li><a href="#Chapter20">Advanced dictionary and prefix API</a></li>
+<li><a href="#Chapter21">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
+<li><a href="#Chapter22">experimental API (static linking only)</a></li>
+<li><a href="#Chapter23">Frame size functions</a></li>
+<li><a href="#Chapter24">ZSTD_decompressBound() :</a></li>
+<li><a href="#Chapter25">Memory management</a></li>
+<li><a href="#Chapter26">Advanced compression functions</a></li>
+<li><a href="#Chapter27">Advanced decompression functions</a></li>
+<li><a href="#Chapter28">Advanced streaming functions</a></li>
+<li><a href="#Chapter29">Buffer-less and synchronous inner streaming functions</a></li>
+<li><a href="#Chapter30">Buffer-less streaming compression (synchronous mode)</a></li>
+<li><a href="#Chapter31">Buffer-less streaming decompression (synchronous mode)</a></li>
+<li><a href="#Chapter32">ZSTD_getFrameHeader() :</a></li>
+<li><a href="#Chapter33">Block level API</a></li>
</ol>
<hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -71,8 +80,10 @@
</b></pre><BR>
<a name="Chapter3"></a><h2>Default constant</h2><pre></pre>
-<a name="Chapter4"></a><h2>Simple API</h2><pre></pre>
+<a name="Chapter4"></a><h2>Constants</h2><pre></pre>
+<a name="Chapter5"></a><h2>Simple API</h2><pre></pre>
+
<pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
@@ -126,13 +137,22 @@ unsigned long long ZSTD_getFrameContentSize(const void
@return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise.
</p></pre><BR>
+<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+</b><p> `src` should point to the start of a ZSTD frame or skippable frame.
+ `srcSize` must be >= first frame size
+ @return : the compressed size of the first frame starting at `src`,
+ suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
+ or an error code if input is invalid
+</p></pre><BR>
+
<h3>Helper functions</h3><pre></pre><b><pre>#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b>
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
+int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b>
int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
</pre></b><BR>
-<a name="Chapter5"></a><h2>Explicit context</h2><pre></pre>
+<a name="Chapter6"></a><h2>Explicit context</h2><pre></pre>
<h3>Compression context</h3><pre> When compressing many times,
it is recommended to allocate a context just once, and re-use it for each successive compression operation.
@@ -169,229 +189,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
</p></pre><BR>
-<a name="Chapter6"></a><h2>Simple dictionary API</h2><pre></pre>
+<a name="Chapter7"></a><h2>Advanced compression API</h2><pre></pre>
-<pre><b>size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize,
- int compressionLevel);
-</b><p> Compression at an explicit compression level using a Dictionary.
- A dictionary can be any arbitrary data segment (also called a prefix),
- or a buffer with specified information (see dictBuilder/zdict.h).
- Note : This function loads the dictionary, resulting in significant startup delay.
- It's intended for a dictionary used only once.
- Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used.
-</p></pre><BR>
-
-<pre><b>size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize);
-</b><p> Decompression using a known Dictionary.
- Dictionary must be identical to the one used during compression.
- Note : This function loads the dictionary, resulting in significant startup delay.
- It's intended for a dictionary used only once.
- Note : When `dict == NULL || dictSize < 8` no dictionary is used.
-</p></pre><BR>
-
-<a name="Chapter7"></a><h2>Bulk processing dictionary API</h2><pre></pre>
-
-<pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
- int compressionLevel);
-</b><p> When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once.
- ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost.
- ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
- `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict.
- Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content.
- Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data.
-</p></pre><BR>
-
-<pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
-</b><p> Function frees memory allocated by ZSTD_createCDict().
-</p></pre><BR>
-
-<pre><b>size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_CDict* cdict);
-</b><p> Compression using a digested Dictionary.
- Recommended when same dictionary is used multiple times.
- Note : compression level is _decided at dictionary creation time_,
- and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no)
-</p></pre><BR>
-
-<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
-</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
- dictBuffer can be released after DDict creation, as its content is copied inside DDict.
-</p></pre><BR>
-
-<pre><b>size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
-</b><p> Function frees memory allocated with ZSTD_createDDict()
-</p></pre><BR>
-
-<pre><b>size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTD_DDict* ddict);
-</b><p> Decompression using a digested Dictionary.
- Recommended when same dictionary is used multiple times.
-</p></pre><BR>
-
-<a name="Chapter8"></a><h2>Streaming</h2><pre></pre>
-
-<pre><b>typedef struct ZSTD_inBuffer_s {
- const void* src; </b>/**< start of input buffer */<b>
- size_t size; </b>/**< size of input buffer */<b>
- size_t pos; </b>/**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */<b>
-} ZSTD_inBuffer;
-</b></pre><BR>
-<pre><b>typedef struct ZSTD_outBuffer_s {
- void* dst; </b>/**< start of output buffer */<b>
- size_t size; </b>/**< size of output buffer */<b>
- size_t pos; </b>/**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */<b>
-} ZSTD_outBuffer;
-</b></pre><BR>
-<a name="Chapter9"></a><h2>Streaming compression - HowTo</h2><pre>
- A ZSTD_CStream object is required to track streaming operation.
- Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
- ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
- It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
-
- For parallel execution, use one separate ZSTD_CStream per thread.
-
- note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
-
- Parameters are sticky : when starting a new compression on the same context,
- it will re-use the same sticky parameters as previous compression session.
- When in doubt, it's recommended to fully initialize the context before usage.
- Use ZSTD_initCStream() to set the parameter to a selected compression level.
- Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters.
-
- Use ZSTD_compressStream() as many times as necessary to consume input stream.
- The function will automatically update both `pos` fields within `input` and `output`.
- Note that the function may not consume the entire input,
- for example, because the output buffer is already full,
- in which case `input.pos < input.size`.
- The caller must check if input has been entirely consumed.
- If not, the caller must make some room to receive more compressed data,
- and then present again remaining input data.
- @return : a size hint, preferred nb of bytes to use as input for next function call
- or an error code, which can be tested using ZSTD_isError().
- Note 1 : it's just a hint, to help latency a little, any value will work fine.
- Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
-
- At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
- using ZSTD_flushStream(). `output->pos` will be updated.
- Note that, if `output->size` is too small, a single invocation of ZSTD_flushStream() might not be enough (return code > 0).
- In which case, make some room to receive more compressed data, and call again ZSTD_flushStream().
- @return : 0 if internal buffers are entirely flushed,
- >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
- or an error code, which can be tested using ZSTD_isError().
-
- ZSTD_endStream() instructs to finish a frame.
- It will perform a flush and write frame epilogue.
- The epilogue is required for decoders to consider a frame completed.
- flush() operation is the same, and follows same rules as ZSTD_flushStream().
- @return : 0 if frame fully completed and fully flushed,
- >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
- or an error code, which can be tested using ZSTD_isError().
-
-
-<BR></pre>
-
-<pre><b>typedef ZSTD_CCtx ZSTD_CStream; </b>/**< CCtx and CStream are now effectively same object (>= v1.3.0) */<b>
-</b></pre><BR>
-<h3>ZSTD_CStream management functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream(void);
-size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
-</pre></b><BR>
-<h3>Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
-size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-</pre></b><BR>
-<pre><b>size_t ZSTD_CStreamInSize(void); </b>/**< recommended size for input buffer */<b>
-</b></pre><BR>
-<pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b>
-</b></pre><BR>
-<a name="Chapter10"></a><h2>Streaming decompression - HowTo</h2><pre>
- A ZSTD_DStream object is required to track streaming operations.
- Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
- ZSTD_DStream objects can be re-used multiple times.
-
- Use ZSTD_initDStream() to start a new decompression operation.
- @return : recommended first input size
- Alternatively, use advanced API to set specific properties.
-
- Use ZSTD_decompressStream() repetitively to consume your input.
- The function will update both `pos` fields.
- If `input.pos < input.size`, some input has not been consumed.
- It's up to the caller to present again remaining data.
- The function tries to flush all data decoded immediately, respecting output buffer size.
- If `output.pos < output.size`, decoder has flushed everything it could.
- But if `output.pos == output.size`, there might be some data left within internal buffers.,
- In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
- Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
- @return : 0 when a frame is completely decoded and fully flushed,
- or an error code, which can be tested using ZSTD_isError(),
- or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
- the return value is a suggested next input size (just a hint for better latency)
- that will never request more than the remaining frame size.
-
-<BR></pre>
-
-<pre><b>typedef ZSTD_DCtx ZSTD_DStream; </b>/**< DCtx and DStream are now effectively same object (>= v1.3.0) */<b>
-</b></pre><BR>
-<h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void);
-size_t ZSTD_freeDStream(ZSTD_DStream* zds);
-</pre></b><BR>
-<h3>Streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_initDStream(ZSTD_DStream* zds);
-size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-</pre></b><BR>
-<pre><b>size_t ZSTD_DStreamInSize(void); </b>/*!< recommended size for input buffer */<b>
-</b></pre><BR>
-<pre><b>size_t ZSTD_DStreamOutSize(void); </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b>
-</b></pre><BR>
-<a name="Chapter11"></a><h2>ADVANCED AND EXPERIMENTAL FUNCTIONS</h2><pre>
- The definitions in the following section are considered experimental.
- They are provided for advanced scenarios.
- They should never be used with a dynamic library, as prototypes may change in the future.
- Use them only in association with static linking.
-
-<BR></pre>
-
-<a name="Chapter12"></a><h2>Candidate API for promotion to stable status</h2><pre>
- The following symbols and constants form the "staging area" :
- they are considered to join "stable API" by v1.4.0.
- The proposal is written so that it can be made stable "as is",
- though it's still possible to suggest improvements.
- Staging is in fact last chance for changes,
- the API is locked once reaching "stable" status.
-
-<BR></pre>
-
-<pre><b>int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b>
-</b></pre><BR>
-<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
-</b><p> `src` should point to the start of a ZSTD frame or skippable frame.
- `srcSize` must be >= first frame size
- @return : the compressed size of the first frame starting at `src`,
- suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
- or an error code if input is invalid
-</p></pre><BR>
-
-<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
-size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
-size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
-size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
-</b><p> These functions give the _current_ memory usage of selected object.
- Note that object memory usage can evolve (increase or decrease) over time.
-</p></pre><BR>
-
-<a name="Chapter13"></a><h2>Advanced compression API</h2><pre></pre>
-
<pre><b>typedef enum { ZSTD_fast=1,
ZSTD_dfast=2,
ZSTD_greedy=3,
@@ -407,7 +206,10 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
</b></pre><BR>
<pre><b>typedef enum {
- </b>/* compression parameters */<b>
+ </b>/* compression parameters<b>
+ * Note: When compressing with a ZSTD_CDict these parameters are superseded
+ * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict()
+ * for more info (superseded-by-cdict). */
ZSTD_c_compressionLevel=100, </b>/* Update all compression parameters according to pre-defined cLevel table<b>
* Default level is ZSTD_CLEVEL_DEFAULT==3.
* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
@@ -529,6 +331,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* ZSTD_c_format
* ZSTD_c_forceMaxWindow
* ZSTD_c_forceAttachDict
+ * ZSTD_c_literalCompressionMode
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change.
@@ -536,7 +339,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
ZSTD_c_experimentalParam1=500,
ZSTD_c_experimentalParam2=10,
ZSTD_c_experimentalParam3=1000,
- ZSTD_c_experimentalParam4=1001
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list