svn commit: r317877 - in head: contrib/zstd contrib/zstd/contrib contrib/zstd/contrib/pzstd contrib/zstd/contrib/pzstd/utils/test contrib/zstd/doc contrib/zstd/doc/educational_decoder contrib/zstd/...
Baptiste Daroussin
bapt at FreeBSD.org
Sat May 6 10:26:43 UTC 2017
Author: bapt
Date: Sat May 6 10:26:40 2017
New Revision: 317877
URL: https://svnweb.freebsd.org/changeset/base/317877
Log:
Import Zstandard 1.2.0
Among new things it is now threaded by default, use zstd -T# to chose the
number of threads not that -T0 will automatically determine the number of
threads based on the number of CPU online.
Added:
head/contrib/zstd/contrib/cleanTabs
- copied unchanged from r317876, vendor/zstd/dist/contrib/cleanTabs
head/contrib/zstd/programs/zstd.1.md
- copied unchanged from r317876, vendor/zstd/dist/programs/zstd.1.md
Deleted:
head/contrib/zstd/.gitignore
head/contrib/zstd/.travis.yml
Modified:
head/contrib/zstd/Makefile
head/contrib/zstd/NEWS
head/contrib/zstd/README.md
head/contrib/zstd/appveyor.yml
head/contrib/zstd/circle.yml
head/contrib/zstd/contrib/pzstd/Options.cpp
head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp
head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp
head/contrib/zstd/doc/educational_decoder/zstd_decompress.c
head/contrib/zstd/doc/images/Cspeed4.png
head/contrib/zstd/doc/images/Dspeed4.png
head/contrib/zstd/doc/images/dict-cr.png
head/contrib/zstd/doc/images/dict-cs.png
head/contrib/zstd/doc/images/dict-ds.png
head/contrib/zstd/doc/zstd_compression_format.md
head/contrib/zstd/doc/zstd_manual.html
head/contrib/zstd/examples/simple_compression.c
head/contrib/zstd/examples/streaming_compression.c
head/contrib/zstd/examples/streaming_decompression.c
head/contrib/zstd/lib/Makefile
head/contrib/zstd/lib/README.md
head/contrib/zstd/lib/common/bitstream.h
head/contrib/zstd/lib/common/error_private.c
head/contrib/zstd/lib/common/fse.h
head/contrib/zstd/lib/common/huf.h
head/contrib/zstd/lib/common/mem.h
head/contrib/zstd/lib/common/zstd_errors.h
head/contrib/zstd/lib/common/zstd_internal.h
head/contrib/zstd/lib/compress/fse_compress.c
head/contrib/zstd/lib/compress/zstd_compress.c
head/contrib/zstd/lib/compress/zstd_opt.h
head/contrib/zstd/lib/compress/zstdmt_compress.c
head/contrib/zstd/lib/decompress/zstd_decompress.c
head/contrib/zstd/lib/dictBuilder/cover.c
head/contrib/zstd/lib/dictBuilder/zdict.c
head/contrib/zstd/lib/dictBuilder/zdict.h
head/contrib/zstd/lib/legacy/zstd_v01.c
head/contrib/zstd/lib/legacy/zstd_v02.c
head/contrib/zstd/lib/legacy/zstd_v03.c
head/contrib/zstd/lib/legacy/zstd_v04.c
head/contrib/zstd/lib/legacy/zstd_v05.c
head/contrib/zstd/lib/legacy/zstd_v06.c
head/contrib/zstd/lib/zstd.h
head/contrib/zstd/programs/Makefile
head/contrib/zstd/programs/README.md
head/contrib/zstd/programs/bench.c
head/contrib/zstd/programs/dibio.c
head/contrib/zstd/programs/fileio.c
head/contrib/zstd/programs/fileio.h
head/contrib/zstd/programs/platform.h
head/contrib/zstd/programs/util.h
head/contrib/zstd/programs/zstd.1
head/contrib/zstd/programs/zstdcli.c
head/contrib/zstd/tests/Makefile
head/contrib/zstd/tests/decodecorpus.c
head/contrib/zstd/tests/fullbench.c
head/contrib/zstd/tests/fuzzer.c
head/contrib/zstd/tests/paramgrill.c
head/contrib/zstd/tests/playTests.sh
head/contrib/zstd/tests/test-zstd-speed.py
head/contrib/zstd/tests/zbufftest.c
head/contrib/zstd/tests/zstreamtest.c
head/contrib/zstd/zlibWrapper/examples/zwrapbench.c
head/usr.bin/zstd/Makefile
Directory Properties:
head/contrib/zstd/ (props changed)
Modified: head/contrib/zstd/Makefile
==============================================================================
--- head/contrib/zstd/Makefile Sat May 6 10:18:45 2017 (r317876)
+++ head/contrib/zstd/Makefile Sat May 6 10:26:40 2017 (r317877)
@@ -90,6 +90,10 @@ examples:
manual:
$(MAKE) -C contrib/gen_html $@
+.PHONY: cleanTabs
+cleanTabs:
+ cd contrib; ./cleanTabs
+
.PHONY: clean
clean:
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
@@ -105,9 +109,15 @@ clean:
# make install is validated only for Linux, OSX, Hurd and some BSD targets
#------------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
+
HOST_OS = POSIX
-.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
+.PHONY: list
+list:
+ @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+
+.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan
install:
@$(MAKE) -C $(ZSTDDIR) $@
@$(MAKE) -C $(PRGDIR) $@
@@ -152,16 +162,16 @@ ppc64build: clean
CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch
armfuzz: clean
- CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+ CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
aarch64fuzz: clean
- CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+ CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
ppcfuzz: clean
- CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+ CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
ppc64fuzz: clean
- CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest
+ CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
gpptest: clean
CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
@@ -180,19 +190,19 @@ clangtest: clean
armtest: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
- $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+ $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
aarch64test:
$(MAKE) -C $(TESTDIR) datagen # use native, faster
- $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+ $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
ppctest: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
- $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
+ $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests
ppc64test: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
- $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
+ $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests
arm-ppc-compilation:
$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
@@ -263,7 +273,7 @@ endif
ifneq (,$(filter MSYS%,$(shell uname)))
HOST_OS = MSYS
-CMAKE_PARAMS = -G"MSYS Makefiles"
+CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
endif
@@ -275,7 +285,7 @@ cmakebuild:
cmake --version
$(RM) -r $(BUILDIR)/cmake/build
mkdir $(BUILDIR)/cmake/build
- cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
+ cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
c90build: clean
gcc -v
Modified: head/contrib/zstd/NEWS
==============================================================================
--- head/contrib/zstd/NEWS Sat May 6 10:18:45 2017 (r317876)
+++ head/contrib/zstd/NEWS Sat May 6 10:26:40 2017 (r317877)
@@ -1,3 +1,25 @@
+v1.2.0
+cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
+cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
+cli : new : zstdmt symlink hardwired to `zstd -T0`
+cli : new : command --threads=# (#671)
+cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
+cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
+cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
+cli : fix : does not output compressed data on console
+cli : fix : ignore symbolic links unless --force specified,
+API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
+API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
+API : improved: ZSTDMT_compressCCtx() reduced memory usage
+API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
+API : fix : src size stored in frame header is controlled at end of frame
+API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
+API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
+build: improved cmake script, by @Majlen
+build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
+tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
+new : contrib/linux-kernel version, by Nick Terrell
+
v1.1.4
cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
cli : new : advanced benchmark command --priority=rt
@@ -5,13 +27,13 @@ cli : fix : write on sparse-enabled file
cli : fix : --rm remains silent when input is stdin
cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
-memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
-arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
+arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
-build: new: meson build system in contrib/meson, by Dima Krasner
-build: improved cmake script, by @Majlen
-build: added -Wformat-security flag, as recommended by Padraig Brady
+build : new: meson build system in contrib/meson, by Dima Krasner
+build : improved cmake script, by @Majlen
+build : added -Wformat-security flag, as recommended by Padraig Brady
doc : new : educational decoder, by Sean Purcell
v1.1.3
Modified: head/contrib/zstd/README.md
==============================================================================
--- head/contrib/zstd/README.md Sat May 6 10:18:45 2017 (r317876)
+++ head/contrib/zstd/README.md Sat May 6 10:26:40 2017 (r317877)
@@ -12,13 +12,13 @@ you can consult a list of known ports on
|dev | [](https://travis-ci.org/facebook/zstd) |
As a reference, several fast compression algorithms were tested and compared
-on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`),
+on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),
with a Core i7-6700K CPU @ 4.0GHz,
-using [lzbench v1.6], an open-source in-memory benchmark by @inikep
+using [lzbench], an open-source in-memory benchmark by @inikep
compiled with GCC 6.3.0,
on the [Silesia compression corpus].
-[lzbench v1.6]: https://github.com/inikep/lzbench
+[lzbench]: https://github.com/inikep/lzbench
[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
| Compressor name | Ratio | Compression| Decompress.|
@@ -38,7 +38,12 @@ on the [Silesia compression corpus].
Zstd can also offer stronger compression ratios at the cost of compression speed.
Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
-The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.2.1, on the [Silesia compression corpus].
+The following tests were run
+on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)
+with a Core i7-6700K CPU @ 4.0GHz,
+using [lzbench], an open-source in-memory benchmark by @inikep
+compiled with GCC 6.3.0,
+on the [Silesia compression corpus].
Compression Speed vs Ratio | Decompression Speed
---------------------------|--------------------
Modified: head/contrib/zstd/appveyor.yml
==============================================================================
--- head/contrib/zstd/appveyor.yml Sat May 6 10:18:45 2017 (r317876)
+++ head/contrib/zstd/appveyor.yml Sat May 6 10:26:40 2017 (r317877)
@@ -1,66 +1,103 @@
-version: 1.0.{build}
-environment:
- matrix:
- - COMPILER: "gcc"
- PLATFORM: "mingw64"
- MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"'
- - COMPILER: "gcc"
- PLATFORM: "mingw32"
- MAKE_PARAMS: '"make -C tests test-zstd test-fullbench test-fuzzer test-invalidDictionaries"'
- - COMPILER: "gcc"
- PLATFORM: "clang"
- MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""'
- - COMPILER: "visual"
- CONFIGURATION: "Debug"
- PLATFORM: "x64"
- - COMPILER: "visual"
- CONFIGURATION: "Debug"
- PLATFORM: "Win32"
- - COMPILER: "visual"
- CONFIGURATION: "Release"
- PLATFORM: "x64"
- - COMPILER: "visual"
- CONFIGURATION: "Release"
- PLATFORM: "Win32"
+-
+ version: 1.0.{build}
+ branches:
+ only:
+ - dev
+ - master
+ environment:
+ matrix:
+ - COMPILER: "gcc"
+ HOST: "mingw"
+ PLATFORM: "x64"
+ SCRIPT: "make allarch && make -C tests test-symbols fullbench-dll fullbench-lib"
+ ARTIFACT: "true"
+ BUILD: "true"
+ - COMPILER: "gcc"
+ HOST: "mingw"
+ PLATFORM: "x86"
+ SCRIPT: "make allarch"
+ ARTIFACT: "true"
+ BUILD: "true"
+ - COMPILER: "clang"
+ HOST: "mingw"
+ PLATFORM: "x64"
+ SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch"
+ BUILD: "true"
+
+ - COMPILER: "gcc"
+ HOST: "mingw"
+ PLATFORM: "x64"
+ SCRIPT: ""
+ TEST: "cmake"
+
+ - COMPILER: "gcc"
+ HOST: "mingw"
+ PLATFORM: "x64"
+ SCRIPT: ""
+ TEST: "pzstd"
+
+ - COMPILER: "visual"
+ HOST: "visual"
+ PLATFORM: "x64"
+ CONFIGURATION: "Debug"
+ - COMPILER: "visual"
+ HOST: "visual"
+ PLATFORM: "Win32"
+ CONFIGURATION: "Debug"
+ - COMPILER: "visual"
+ HOST: "visual"
+ PLATFORM: "x64"
+ CONFIGURATION: "Release"
+ - COMPILER: "visual"
+ HOST: "visual"
+ PLATFORM: "Win32"
+ CONFIGURATION: "Release"
-install:
+ install:
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
- - MKDIR bin
- - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
- - if [%COMPILER%]==[gcc] (
- SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
- SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
- COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe &&
- COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe
- ) else (
- IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;")
+ - SET PATH_ORIGINAL=%PATH%
+ - if [%HOST%]==[mingw] (
+ SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
+ SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
+ COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
+ COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
+ )
+ - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
+ SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
+ )
+
+ build_script:
+ - if [%HOST%]==[mingw] (
+ ( if [%PLATFORM%]==[x64] (
+ SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
+ ) else if [%PLATFORM%]==[x86] (
+ SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
+ ) )
)
-
-build_script:
- - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
- - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL%
- - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
- - if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
- - if [%COMPILER%]==[gcc] (
- ECHO *** &&
- ECHO *** Building %PLATFORM% &&
- ECHO *** &&
+ - if [%HOST%]==[mingw] if [%BUILD%]==[true] (
make -v &&
- cc -v &&
- ECHO %MAKE_PARAMS% &&
- sh -c %MAKE_PARAMS%
- )
- - if [%PLATFORM%]==[clang] COPY tests\fuzzer.exe tests\fuzzer_clang.exe
- - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] (
- COPY programs\zstd.exe bin\zstd.exe &&
- appveyor PushArtifact bin\zstd.exe
- )
- - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw32] (
- COPY programs\zstd.exe bin\zstd32.exe &&
- appveyor PushArtifact bin\zstd32.exe
+ sh -c "%COMPILER% -v" &&
+ ECHO Building zlib to static link &&
+ SET "CC=%COMPILER%" &&
+ sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib" &&
+ sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
+ ECHO Building zstd &&
+ SET "CPPFLAGS=-I../../zlib" &&
+ SET "LDFLAGS=../../zlib/libz.a" &&
+ sh -c "%SCRIPT%" &&
+ ( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
+ lib\dll\example\build_package.bat &&
+ make -C programs DEBUGFLAGS= clean zstd &&
+ cp programs\zstd.exe zstd_%PLATFORM%.exe &&
+ appveyor PushArtifact zstd_%PLATFORM%.exe &&
+ cp programs\zstd.exe bin\zstd.exe &&
+ make -C programs DEBUGFLAGS= clean zstdmt &&
+ cp programs\zstd.exe bin\zstdmt.exe &&
+ cd bin\ && 7z a -tzip zstd-win-release-%PLATFORM%.zip * &&
+ appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
+ )
)
- - if [%COMPILER%]==[gcc] make clean
- - if [%COMPILER%]==[visual] (
+ - if [%HOST%]==[visual] (
ECHO *** &&
ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
ECHO *** &&
@@ -111,29 +148,26 @@ build_script:
COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
)
-test_script:
+ test_script:
- ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
- - SET FUZZERTEST=-T1mn
- - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[clang] (
- tests\fuzzer_clang.exe %FUZZERTEST% &&
- ECHO *** &&
- ECHO *** Building cmake for %PLATFORM% &&
- ECHO *** &&
+ - SET "CC=gcc"
+ - SET "CXX=g++"
+ - if [%TEST%]==[cmake] (
mkdir build\cmake\build &&
cd build\cmake\build &&
cmake -G "Visual Studio 14 2015 Win64" .. &&
cd ..\..\.. &&
- make clean &&
- ECHO *** &&
- ECHO *** Building pzstd for %PLATFORM% &&
- ECHO *** &&
+ make clean
+ )
+ - if [%TEST%]==[pzstd] (
make -C contrib\pzstd googletest-mingw64 &&
make -C contrib\pzstd pzstd.exe &&
make -C contrib\pzstd tests &&
make -C contrib\pzstd check &&
make -C contrib\pzstd clean
)
- - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
+ - SET "FUZZERTEST=-T30s"
+ - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
CD tests &&
SET ZSTD=./zstd.exe &&
sh -e playTests.sh --test-large-data &&
@@ -146,33 +180,76 @@ test_script:
fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
)
-branches:
- only:
- - dev
- - master
-
-artifacts:
- - path: bin\zstd.exe
- - path: bin\zstd32.exe
-
-deploy:
-- provider: GitHub
- auth_token:
- secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
- artifact: bin\zstd.exe
- force_update: true
- on:
- branch: autobuild
- COMPILER: gcc
- PLATFORM: "mingw64"
- appveyor_repo_tag: true
-- provider: GitHub
- auth_token:
- secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4
- artifact: bin\zstd32.exe
- force_update: true
- on:
- branch: autobuild
- COMPILER: gcc
- PLATFORM: "mingw32"
- appveyor_repo_tag: true
+-
+ version: 1.0.{build}
+ environment:
+ matrix:
+ - COMPILER: "gcc"
+ HOST: "mingw"
+ PLATFORM: "x64"
+ SCRIPT: "make allarch"
+ - COMPILER: "gcc"
+ HOST: "mingw"
+ PLATFORM: "x86"
+ SCRIPT: "make allarch"
+ - COMPILER: "clang"
+ HOST: "mingw"
+ PLATFORM: "x64"
+ SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch"
+
+ - COMPILER: "visual"
+ HOST: "visual"
+ PLATFORM: "x64"
+ CONFIGURATION: "Debug"
+ - COMPILER: "visual"
+ HOST: "visual"
+ PLATFORM: "Win32"
+ CONFIGURATION: "Debug"
+ - COMPILER: "visual"
+ HOST: "visual"
+ PLATFORM: "x64"
+ CONFIGURATION: "Release"
+ - COMPILER: "visual"
+ HOST: "visual"
+ PLATFORM: "Win32"
+ CONFIGURATION: "Release"
+
+ install:
+ - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
+ - SET PATH_ORIGINAL=%PATH%
+ - if [%HOST%]==[mingw] (
+ SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
+ SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
+ COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
+ COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
+ )
+ - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
+ SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
+ )
+
+ build_script:
+ - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION%
+ - if [%HOST%]==[mingw] (
+ ( if [%PLATFORM%]==[x64] (
+ SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
+ ) else if [%PLATFORM%]==[x86] (
+ SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
+ ) ) &&
+ make -v &&
+ sh -c "%COMPILER% -v" &&
+ set "CC=%COMPILER%" &&
+ sh -c "%SCRIPT%"
+ )
+ - if [%HOST%]==[visual] (
+ ECHO *** &&
+ ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
+ ECHO *** &&
+ msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+ DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+ MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+ msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+ DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+ MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+ COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+ COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
+ )
Modified: head/contrib/zstd/circle.yml
==============================================================================
--- head/contrib/zstd/circle.yml Sat May 6 10:18:45 2017 (r317876)
+++ head/contrib/zstd/circle.yml Sat May 6 10:26:40 2017 (r317877)
@@ -3,7 +3,7 @@ dependencies:
- sudo dpkg --add-architecture i386
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
- sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6
+ - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 zlib1g-dev liblzma-dev
- sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
test:
Copied: head/contrib/zstd/contrib/cleanTabs (from r317876, vendor/zstd/dist/contrib/cleanTabs)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/contrib/zstd/contrib/cleanTabs Sat May 6 10:26:40 2017 (r317877, copy of r317876, vendor/zstd/dist/contrib/cleanTabs)
@@ -0,0 +1,2 @@
+#!/bin/sh
+sed -i '' $'s/\t/ /g' ../lib/**/*.{h,c} ../programs/*.{h,c} ../tests/*.c ./**/*.{h,cpp} ../examples/*.c ../zlibWrapper/*.{h,c}
Modified: head/contrib/zstd/contrib/pzstd/Options.cpp
==============================================================================
--- head/contrib/zstd/contrib/pzstd/Options.cpp Sat May 6 10:18:45 2017 (r317876)
+++ head/contrib/zstd/contrib/pzstd/Options.cpp Sat May 6 10:26:40 2017 (r317877)
@@ -91,7 +91,7 @@ void usage() {
std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);
std::fprintf(stderr, " -d, --decompress : decompression\n");
std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n");
- std::fprintf(stderr, " -f, --force : overwrite output without prompting\n");
+ std::fprintf(stderr, " -f, --force : overwrite output without prompting, (de)compress links\n");
std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n");
std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n");
std::fprintf(stderr, " -h, --help : display help and exit\n");
@@ -121,6 +121,7 @@ Options::Status Options::parse(int argc,
bool recursive = false;
bool ultra = false;
bool forceStdout = false;
+ bool followLinks = false;
// Local copy of input files, which are pointers into argv.
std::vector<const char *> localInputFiles;
for (int i = 1; i < argc; ++i) {
@@ -255,6 +256,7 @@ Options::Status Options::parse(int argc,
case 'f':
overwrite = true;
forceStdout = true;
+ followLinks = true;
break;
case 't':
test = true;
@@ -328,13 +330,29 @@ Options::Status Options::parse(int argc,
}
}
+ g_utilDisplayLevel = verbosity;
+ // Remove local input files that are symbolic links
+ if (!followLinks) {
+ std::remove_if(localInputFiles.begin(), localInputFiles.end(),
+ [&](const char *path) {
+ bool isLink = UTIL_isLink(path);
+ if (isLink && verbosity >= 2) {
+ std::fprintf(
+ stderr,
+ "Warning : %s is symbolic link, ignoring\n",
+ path);
+ }
+ return isLink;
+ });
+ }
+
// Translate input files/directories into files to (de)compress
if (recursive) {
char *scratchBuffer = nullptr;
unsigned numFiles = 0;
const char **files =
UTIL_createFileList(localInputFiles.data(), localInputFiles.size(),
- &scratchBuffer, &numFiles);
+ &scratchBuffer, &numFiles, followLinks);
if (files == nullptr) {
std::fprintf(stderr, "Error traversing directories\n");
return Status::Failure;
Modified: head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp
==============================================================================
--- head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp Sat May 6 10:18:45 2017 (r317876)
+++ head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp Sat May 6 10:26:40 2017 (r317877)
@@ -10,6 +10,7 @@
#include <gtest/gtest.h>
#include <atomic>
+#include <iostream>
#include <thread>
#include <vector>
@@ -34,16 +35,19 @@ TEST(ThreadPool, AllJobsFinished) {
std::atomic<unsigned> numFinished{0};
std::atomic<bool> start{false};
{
+ std::cerr << "Creating executor" << std::endl;
ThreadPool executor(5);
for (int i = 0; i < 10; ++i) {
executor.add([ &numFinished, &start ] {
while (!start.load()) {
- // spin
+ std::this_thread::yield();
}
++numFinished;
});
}
+ std::cerr << "Starting" << std::endl;
start.store(true);
+ std::cerr << "Finishing" << std::endl;
}
EXPECT_EQ(10, numFinished.load());
}
Modified: head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp
==============================================================================
--- head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp Sat May 6 10:18:45 2017 (r317876)
+++ head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp Sat May 6 10:26:40 2017 (r317877)
@@ -10,6 +10,7 @@
#include "utils/WorkQueue.h"
#include <gtest/gtest.h>
+#include <iostream>
#include <memory>
#include <mutex>
#include <thread>
@@ -201,11 +202,13 @@ TEST(WorkQueue, BoundedSizeMPMC) {
WorkQueue<int> queue(10);
std::vector<int> results(200, -1);
std::mutex mutex;
+ std::cerr << "Creating popperThreads" << std::endl;
std::vector<std::thread> popperThreads;
for (int i = 0; i < 4; ++i) {
popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
}
+ std::cerr << "Creating pusherThreads" << std::endl;
std::vector<std::thread> pusherThreads;
for (int i = 0; i < 2; ++i) {
auto min = i * 100;
@@ -218,15 +221,19 @@ TEST(WorkQueue, BoundedSizeMPMC) {
});
}
+ std::cerr << "Joining pusherThreads" << std::endl;
for (auto& thread : pusherThreads) {
thread.join();
}
+ std::cerr << "Finishing queue" << std::endl;
queue.finish();
+ std::cerr << "Joining popperThreads" << std::endl;
for (auto& thread : popperThreads) {
thread.join();
}
+ std::cerr << "Inspecting results" << std::endl;
for (int i = 0; i < 200; ++i) {
EXPECT_EQ(i, results[i]);
}
Modified: head/contrib/zstd/doc/educational_decoder/zstd_decompress.c
==============================================================================
--- head/contrib/zstd/doc/educational_decoder/zstd_decompress.c Sat May 6 10:18:45 2017 (r317876)
+++ head/contrib/zstd/doc/educational_decoder/zstd_decompress.c Sat May 6 10:26:40 2017 (r317877)
@@ -27,16 +27,19 @@ size_t ZSTD_decompress_with_dict(void *c
/// Get the decompressed size of an input stream so memory can be allocated in
/// advance
+/// Returns -1 if the size can't be determined
size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len);
/******* UTILITY MACROS AND TYPES *********************************************/
-// Max block size decompressed size is 128 KB and literal blocks must be smaller
-// than that
+// Max block size decompressed size is 128 KB and literal blocks can't be
+// larger than their block
#define MAX_LITERALS_SIZE ((size_t)128 * 1024)
#define MAX(a, b) ((a) > (b) ? (a) : (b))
#define MIN(a, b) ((a) < (b) ? (a) : (b))
+/// This decoder calls exit(1) when it encounters an error, however a production
+/// library should propagate error codes
#define ERROR(s) \
do { \
fprintf(stderr, "Error: %s\n", s); \
@@ -67,29 +70,31 @@ typedef int64_t i64;
/// decompression functions.
/*** IO STREAM OPERATIONS *************/
-/// These structs are the interface for IO, and do bounds checking on all
-/// operations. They should be used opaquely to ensure safety.
-/// Output is always done byte-by-byte
+/// ostream_t/istream_t are used to wrap the pointers/length data passed into
+/// ZSTD_decompress, so that all IO operations are safely bounds checked
+/// They are written/read forward, and reads are treated as little-endian
+/// They should be used opaquely to ensure safety
typedef struct {
u8 *ptr;
size_t len;
} ostream_t;
-/// Input often reads a few bits at a time, so maintain an internal offset
typedef struct {
const u8 *ptr;
- int bit_offset;
size_t len;
+
+ // Input often reads a few bits at a time, so maintain an internal offset
+ int bit_offset;
} istream_t;
/// The following two functions are the only ones that allow the istream to be
/// non-byte aligned
/// Reads `num` bits from a bitstream, and updates the internal offset
-static inline u64 IO_read_bits(istream_t *const in, const int num);
-/// Rewinds the stream by `num` bits
-static inline void IO_rewind_bits(istream_t *const in, const int num);
+static inline u64 IO_read_bits(istream_t *const in, const int num_bits);
+/// Backs-up the stream by `num` bits so they can be read again
+static inline void IO_rewind_bits(istream_t *const in, const int num_bits);
/// If the remaining bits in a byte will be unused, advance to the end of the
/// byte
static inline void IO_align_stream(istream_t *const in);
@@ -101,30 +106,31 @@ static inline void IO_write_byte(ostream
/// be byte aligned.
static inline size_t IO_istream_len(const istream_t *const in);
-/// Returns a pointer where `len` bytes can be read, and advances the internal
-/// state. The stream must be byte aligned.
+/// Advances the stream by `len` bytes, and returns a pointer to the chunk that
+/// was skipped. The stream must be byte aligned.
static inline const u8 *IO_read_bytes(istream_t *const in, size_t len);
-/// Returns a pointer where `len` bytes can be written, and advances the internal
-/// state. The stream must be byte aligned.
+/// Advances the stream by `len` bytes, and returns a pointer to the chunk that
+/// was skipped so it can be written to.
static inline u8 *IO_write_bytes(ostream_t *const out, size_t len);
/// Advance the inner state by `len` bytes. The stream must be byte aligned.
static inline void IO_advance_input(istream_t *const in, size_t len);
-/// Returns an `ostream_t` constructed from the given pointer and length
+/// Returns an `ostream_t` constructed from the given pointer and length.
static inline ostream_t IO_make_ostream(u8 *out, size_t len);
-/// Returns an `istream_t` constructed from the given pointer and length
+/// Returns an `istream_t` constructed from the given pointer and length.
static inline istream_t IO_make_istream(const u8 *in, size_t len);
-/// Returns an `istream_t` with the same base as `in`, and length `len`
-/// Then, advance `in` to account for the consumed bytes
-/// `in` must be byte aligned
+/// Returns an `istream_t` with the same base as `in`, and length `len`.
+/// Then, advance `in` to account for the consumed bytes.
+/// `in` must be byte aligned.
static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len);
/*** END IO STREAM OPERATIONS *********/
/*** BITSTREAM OPERATIONS *************/
-/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits
-static inline u64 read_bits_LE(const u8 *src, const int num,
+/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits,
+/// and return them interpreted as a little-endian unsigned integer.
+static inline u64 read_bits_LE(const u8 *src, const int num_bits,
const size_t offset);
/// Read bits from the end of a HUF or FSE bitstream. `offset` is in bits, so
@@ -136,9 +142,8 @@ static inline u64 STREAM_read_bits(const
/*** END BITSTREAM OPERATIONS *********/
/*** BIT COUNTING OPERATIONS **********/
-/// Returns `x`, where `2^x` is the largest power of 2 less than or equal to
-/// `num`, or `-1` if `num == 0`.
-static inline int log2inf(const u64 num);
+/// Returns the index of the highest set bit in `num`, or `-1` if `num == 0`
+static inline int highest_set_bit(const u64 num);
/*** END BIT COUNTING OPERATIONS ******/
/*** HUFFMAN PRIMITIVES ***************/
@@ -384,8 +389,8 @@ size_t ZSTD_decompress_with_dict(void *c
parse_dictionary(&parsed_dict, (const u8 *)dict, dict_len);
}
- istream_t in = {(const u8 *)src, 0, src_len};
- ostream_t out = {(u8 *)dst, dst_len};
+ istream_t in = IO_make_istream(src, src_len);
+ ostream_t out = IO_make_ostream(dst, dst_len);
// "A content compressed by Zstandard is transformed into a Zstandard frame.
// Multiple frames can be appended into a single file or stream. A frame is
@@ -633,6 +638,7 @@ static void frame_context_apply_dict(fra
FSE_copy_dtable(&ctx->of_dtable, &dict->of_dtable);
FSE_copy_dtable(&ctx->ml_dtable, &dict->ml_dtable);
+ // Copy the repeated offsets
memcpy(ctx->previous_offsets, dict->previous_offsets,
sizeof(ctx->previous_offsets));
}
@@ -668,7 +674,7 @@ static void decompress_data(frame_contex
// number of bytes to read and copy."
const u8 *const read_ptr = IO_read_bytes(in, block_len);
u8 *const write_ptr = IO_write_bytes(out, block_len);
- //
+
// Copy the raw data into the output
memcpy(write_ptr, read_ptr, block_len);
@@ -682,7 +688,7 @@ static void decompress_data(frame_contex
const u8 *const read_ptr = IO_read_bytes(in, 1);
u8 *const write_ptr = IO_write_bytes(out, block_len);
- // Copy `block_len` copies of `streams->src[0]` to the output
+ // Copy `block_len` copies of `read_ptr[0]` to the output
memset(write_ptr, read_ptr[0], block_len);
ctx->current_total_output += block_len;
@@ -751,7 +757,7 @@ static size_t decode_literals_compressed
u8 **const literals,
const int block_type,
const int size_format);
-static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable);
+static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in);
static void fse_decode_hufweights(ostream_t *weights, istream_t *const in,
int *const num_symbs);
@@ -894,12 +900,12 @@ static size_t decode_literals_compressed
istream_t huf_stream = IO_make_sub_istream(in, compressed_size);
if (block_type == 2) {
- // Decode provided Huffman table
+ // Decode the provided Huffman table
// "This section is only present when Literals_Block_Type type is
// Compressed_Literals_Block (2)."
HUF_free_dtable(&ctx->literals_dtable);
- decode_huf_table(&huf_stream, &ctx->literals_dtable);
+ decode_huf_table(&ctx->literals_dtable, &huf_stream);
} else {
// If the previous Huffman table is being repeated, ensure it exists
if (!ctx->literals_dtable.symbols) {
@@ -922,13 +928,13 @@ static size_t decode_literals_compressed
}
// Decode the Huffman table description
-static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable) {
- const u8 header = IO_read_bits(in, 8);
-
+static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in) {
// "All literal values from zero (included) to last present one (excluded)
// are represented by Weight with values from 0 to Max_Number_of_Bits."
// "This is a single byte value (0-255), which describes how to decode the list of weights."
+ const u8 header = IO_read_bits(in, 8);
+
u8 weights[HUF_MAX_SYMBS];
memset(weights, 0, sizeof(weights));
@@ -997,7 +1003,7 @@ typedef struct {
u16 ll_state;
u16 of_state;
u16 ml_state;
-} sequence_state_t;
+} sequence_states_t;
/// Different modes to signal to decode_seq_tables what to do
typedef enum {
@@ -1052,10 +1058,10 @@ static void decompress_sequences(frame_c
istream_t *const in,
sequence_command_t *const sequences,
const size_t num_sequences);
-static sequence_command_t decode_sequence(sequence_state_t *const state,
+static sequence_command_t decode_sequence(sequence_states_t *const state,
const u8 *const src,
i64 *const offset);
-static void decode_seq_table(istream_t *const in, FSE_dtable *const table,
+static void decode_seq_table(FSE_dtable *const table, istream_t *const in,
const seq_part_t type, const seq_mode_t mode);
static size_t decode_sequences(frame_context_t *const ctx, istream_t *in,
@@ -1131,34 +1137,33 @@ static void decompress_sequences(frame_c
// Offsets
// Match Lengths"
// Update the tables we have stored in the context
- decode_seq_table(in, &ctx->ll_dtable, seq_literal_length,
+ decode_seq_table(&ctx->ll_dtable, in, seq_literal_length,
(compression_modes >> 6) & 3);
- decode_seq_table(in, &ctx->of_dtable, seq_offset,
+ decode_seq_table(&ctx->of_dtable, in, seq_offset,
(compression_modes >> 4) & 3);
- decode_seq_table(in, &ctx->ml_dtable, seq_match_length,
+ decode_seq_table(&ctx->ml_dtable, in, seq_match_length,
(compression_modes >> 2) & 3);
- // Check to make sure none of the tables are uninitialized
- if (!ctx->ll_dtable.symbols || !ctx->of_dtable.symbols ||
- !ctx->ml_dtable.symbols) {
- CORRUPTION();
- }
- sequence_state_t state;
- // Copy the context's tables into the local state
- memcpy(&state.ll_table, &ctx->ll_dtable, sizeof(FSE_dtable));
- memcpy(&state.of_table, &ctx->of_dtable, sizeof(FSE_dtable));
- memcpy(&state.ml_table, &ctx->ml_dtable, sizeof(FSE_dtable));
+ sequence_states_t states;
+
+ // Initialize the decoding tables
+ {
+ states.ll_table = ctx->ll_dtable;
+ states.of_table = ctx->of_dtable;
+ states.ml_table = ctx->ml_dtable;
+ }
- size_t len = IO_istream_len(in);
+ const size_t len = IO_istream_len(in);
const u8 *const src = IO_read_bytes(in, len);
// "After writing the last bit containing information, the compressor writes
// a single 1-bit and then fills the byte with 0-7 0 bits of padding."
- const int padding = 8 - log2inf(src[len - 1]);
- i64 offset = len * 8 - padding;
+ const int padding = 8 - highest_set_bit(src[len - 1]);
+ // The offset starts at the end because FSE streams are read backwards
+ i64 bit_offset = len * 8 - padding;
// "The bitstream starts with initial state values, each using the required
// number of bits in their respective accuracy, decoded previously from
@@ -1166,24 +1171,22 @@ static void decompress_sequences(frame_c
//
// It starts by Literals_Length_State, followed by Offset_State, and finally
// Match_Length_State."
- FSE_init_state(&state.ll_table, &state.ll_state, src, &offset);
- FSE_init_state(&state.of_table, &state.of_state, src, &offset);
- FSE_init_state(&state.ml_table, &state.ml_state, src, &offset);
+ FSE_init_state(&states.ll_table, &states.ll_state, src, &bit_offset);
+ FSE_init_state(&states.of_table, &states.of_state, src, &bit_offset);
+ FSE_init_state(&states.ml_table, &states.ml_state, src, &bit_offset);
for (size_t i = 0; i < num_sequences; i++) {
// Decode sequences one by one
- sequences[i] = decode_sequence(&state, src, &offset);
+ sequences[i] = decode_sequence(&states, src, &bit_offset);
}
- if (offset != 0) {
+ if (bit_offset != 0) {
CORRUPTION();
}
-
- // Don't free tables so they can be used in the next block
}
// Decode a single sequence and update the state
-static sequence_command_t decode_sequence(sequence_state_t *const state,
+static sequence_command_t decode_sequence(sequence_states_t *const states,
const u8 *const src,
i64 *const offset) {
// "Each symbol is a code in its own context, which specifies Baseline and
@@ -1191,9 +1194,9 @@ static sequence_command_t decode_sequenc
// additional bits in the same bitstream."
// Decode symbols, but don't update states
- const u8 of_code = FSE_peek_symbol(&state->of_table, state->of_state);
- const u8 ll_code = FSE_peek_symbol(&state->ll_table, state->ll_state);
- const u8 ml_code = FSE_peek_symbol(&state->ml_table, state->ml_state);
+ const u8 of_code = FSE_peek_symbol(&states->of_table, states->of_state);
+ const u8 ll_code = FSE_peek_symbol(&states->ll_table, states->ll_state);
+ const u8 ml_code = FSE_peek_symbol(&states->ml_table, states->ml_state);
// Offset doesn't need a max value as it's not decoded using a table
if (ll_code > SEQ_MAX_CODES[seq_literal_length] ||
@@ -1221,17 +1224,18 @@ static sequence_command_t decode_sequenc
// then Offset_State."
// If the stream is complete don't read bits to update state
if (*offset != 0) {
- FSE_update_state(&state->ll_table, &state->ll_state, src, offset);
- FSE_update_state(&state->ml_table, &state->ml_state, src, offset);
- FSE_update_state(&state->of_table, &state->of_state, src, offset);
+ FSE_update_state(&states->ll_table, &states->ll_state, src, offset);
+ FSE_update_state(&states->ml_table, &states->ml_state, src, offset);
+ FSE_update_state(&states->of_table, &states->of_state, src, offset);
}
return seq;
}
/// Given a sequence part and table mode, decode the FSE distribution
-static void decode_seq_table(istream_t *const in, FSE_dtable *const table,
- const seq_part_t type, const seq_mode_t mode) {
+/// Errors if the mode is `seq_repeat` without a pre-existing table in `table`
+static void decode_seq_table(FSE_dtable *const table, istream_t *const in,
+ const seq_part_t type, const seq_mode_t mode) {
// Constant arrays indexed by seq_part_t
const i16 *const default_distributions[] = {SEQ_LITERAL_LENGTH_DEFAULT_DIST,
SEQ_OFFSET_DEFAULT_DIST,
@@ -1272,12 +1276,17 @@ static void decode_seq_table(istream_t *
// "Repeat_Mode : re-use distribution table from previous compressed
// block."
// Nothing to do here, table will be unchanged
+ if (!table->symbols) {
+ // This mode is invalid if we don't already have a table
+ CORRUPTION();
+ }
break;
default:
// Impossible, as mode is from 0-3
IMPOSSIBLE();
break;
}
+
}
/******* END SEQUENCE DECODING ************************************************/
@@ -1296,6 +1305,8 @@ static void execute_sequences(frame_cont
const sequence_command_t seq = sequences[i];
{
+ // If the sequence asks for more literals than are left, the
+ // sequence must be corrupted
if (seq.literal_length > IO_istream_len(&litstream)) {
CORRUPTION();
}
@@ -1336,7 +1347,8 @@ static void execute_sequences(frame_cont
// as per the exception listed above
offset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1;
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list