svn commit: r320985 - in vendor/zstd/1.3.0: . build contrib contrib/pzstd contrib/pzstd/utils/test doc doc/educational_decoder doc/images examples lib lib/common lib/compress lib/decompress lib/dic...

Baptiste Daroussin bapt at FreeBSD.org
Fri Jul 14 14:52:46 UTC 2017 

Author: bapt
Date: Fri Jul 14 14:52:44 2017
New Revision: 320985
URL: https://svnweb.freebsd.org/changeset/base/320985

Log:
  Tag import of zstd 1.3.0

Added:
  vendor/zstd/1.3.0/
     - copied from r316975, vendor/zstd/dist/
  vendor/zstd/1.3.0/contrib/cleanTabs
     - copied unchanged from r317875, vendor/zstd/dist/contrib/cleanTabs
  vendor/zstd/1.3.0/programs/zstd.1.md
     - copied unchanged from r320983, vendor/zstd/dist/programs/zstd.1.md
Replaced:
  vendor/zstd/1.3.0/Makefile
     - copied unchanged from r320983, vendor/zstd/dist/Makefile
  vendor/zstd/1.3.0/NEWS
     - copied unchanged from r320983, vendor/zstd/dist/NEWS
  vendor/zstd/1.3.0/README.md
     - copied unchanged from r320983, vendor/zstd/dist/README.md
  vendor/zstd/1.3.0/appveyor.yml
     - copied unchanged from r320983, vendor/zstd/dist/appveyor.yml
  vendor/zstd/1.3.0/circle.yml
     - copied unchanged from r317875, vendor/zstd/dist/circle.yml
  vendor/zstd/1.3.0/contrib/pzstd/Options.cpp
     - copied unchanged from r317875, vendor/zstd/dist/contrib/pzstd/Options.cpp
  vendor/zstd/1.3.0/contrib/pzstd/Pzstd.cpp
     - copied unchanged from r320983, vendor/zstd/dist/contrib/pzstd/Pzstd.cpp
  vendor/zstd/1.3.0/contrib/pzstd/utils/test/ThreadPoolTest.cpp
     - copied unchanged from r317875, vendor/zstd/dist/contrib/pzstd/utils/test/ThreadPoolTest.cpp
  vendor/zstd/1.3.0/contrib/pzstd/utils/test/WorkQueueTest.cpp
     - copied unchanged from r317875, vendor/zstd/dist/contrib/pzstd/utils/test/WorkQueueTest.cpp
  vendor/zstd/1.3.0/doc/educational_decoder/zstd_decompress.c
     - copied unchanged from r317875, vendor/zstd/dist/doc/educational_decoder/zstd_decompress.c
  vendor/zstd/1.3.0/doc/images/Cspeed4.png
     - copied unchanged from r317875, vendor/zstd/dist/doc/images/Cspeed4.png
  vendor/zstd/1.3.0/doc/images/Dspeed4.png
     - copied unchanged from r317875, vendor/zstd/dist/doc/images/Dspeed4.png
  vendor/zstd/1.3.0/doc/images/dict-cr.png
     - copied unchanged from r317875, vendor/zstd/dist/doc/images/dict-cr.png
  vendor/zstd/1.3.0/doc/images/dict-cs.png
     - copied unchanged from r317875, vendor/zstd/dist/doc/images/dict-cs.png
  vendor/zstd/1.3.0/doc/images/dict-ds.png
     - copied unchanged from r317875, vendor/zstd/dist/doc/images/dict-ds.png
  vendor/zstd/1.3.0/doc/zstd_compression_format.md
     - copied unchanged from r317875, vendor/zstd/dist/doc/zstd_compression_format.md
  vendor/zstd/1.3.0/doc/zstd_manual.html
     - copied unchanged from r320983, vendor/zstd/dist/doc/zstd_manual.html
  vendor/zstd/1.3.0/examples/Makefile
     - copied unchanged from r320983, vendor/zstd/dist/examples/Makefile
  vendor/zstd/1.3.0/examples/simple_compression.c
     - copied unchanged from r317875, vendor/zstd/dist/examples/simple_compression.c
  vendor/zstd/1.3.0/examples/streaming_compression.c
     - copied unchanged from r317875, vendor/zstd/dist/examples/streaming_compression.c
  vendor/zstd/1.3.0/examples/streaming_decompression.c
     - copied unchanged from r317875, vendor/zstd/dist/examples/streaming_decompression.c
  vendor/zstd/1.3.0/lib/Makefile
     - copied unchanged from r320983, vendor/zstd/dist/lib/Makefile
  vendor/zstd/1.3.0/lib/README.md
     - copied unchanged from r317875, vendor/zstd/dist/lib/README.md
  vendor/zstd/1.3.0/lib/common/bitstream.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/bitstream.h
  vendor/zstd/1.3.0/lib/common/error_private.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/error_private.c
  vendor/zstd/1.3.0/lib/common/fse.h
     - copied unchanged from r317875, vendor/zstd/dist/lib/common/fse.h
  vendor/zstd/1.3.0/lib/common/huf.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/huf.h
  vendor/zstd/1.3.0/lib/common/mem.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/mem.h
  vendor/zstd/1.3.0/lib/common/pool.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/pool.c
  vendor/zstd/1.3.0/lib/common/pool.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/pool.h
  vendor/zstd/1.3.0/lib/common/threading.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/threading.c
  vendor/zstd/1.3.0/lib/common/zstd_common.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/zstd_common.c
  vendor/zstd/1.3.0/lib/common/zstd_errors.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/zstd_errors.h
  vendor/zstd/1.3.0/lib/common/zstd_internal.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/common/zstd_internal.h
  vendor/zstd/1.3.0/lib/compress/fse_compress.c
     - copied unchanged from r317875, vendor/zstd/dist/lib/compress/fse_compress.c
  vendor/zstd/1.3.0/lib/compress/huf_compress.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/compress/huf_compress.c
  vendor/zstd/1.3.0/lib/compress/zstd_compress.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/compress/zstd_compress.c
  vendor/zstd/1.3.0/lib/compress/zstd_opt.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/compress/zstd_opt.h
  vendor/zstd/1.3.0/lib/compress/zstdmt_compress.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/compress/zstdmt_compress.c
  vendor/zstd/1.3.0/lib/compress/zstdmt_compress.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/compress/zstdmt_compress.h
  vendor/zstd/1.3.0/lib/decompress/huf_decompress.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/decompress/huf_decompress.c
  vendor/zstd/1.3.0/lib/decompress/zstd_decompress.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/decompress/zstd_decompress.c
  vendor/zstd/1.3.0/lib/dictBuilder/cover.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/dictBuilder/cover.c
  vendor/zstd/1.3.0/lib/dictBuilder/zdict.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/dictBuilder/zdict.c
  vendor/zstd/1.3.0/lib/dictBuilder/zdict.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/dictBuilder/zdict.h
  vendor/zstd/1.3.0/lib/legacy/zstd_v01.c
     - copied unchanged from r317875, vendor/zstd/dist/lib/legacy/zstd_v01.c
  vendor/zstd/1.3.0/lib/legacy/zstd_v02.c
     - copied unchanged from r317875, vendor/zstd/dist/lib/legacy/zstd_v02.c
  vendor/zstd/1.3.0/lib/legacy/zstd_v03.c
     - copied unchanged from r317875, vendor/zstd/dist/lib/legacy/zstd_v03.c
  vendor/zstd/1.3.0/lib/legacy/zstd_v04.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/legacy/zstd_v04.c
  vendor/zstd/1.3.0/lib/legacy/zstd_v05.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/legacy/zstd_v05.c
  vendor/zstd/1.3.0/lib/legacy/zstd_v06.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/legacy/zstd_v06.c
  vendor/zstd/1.3.0/lib/legacy/zstd_v07.c
     - copied unchanged from r320983, vendor/zstd/dist/lib/legacy/zstd_v07.c
  vendor/zstd/1.3.0/lib/zstd.h
     - copied unchanged from r320983, vendor/zstd/dist/lib/zstd.h
  vendor/zstd/1.3.0/programs/Makefile
     - copied unchanged from r320983, vendor/zstd/dist/programs/Makefile
  vendor/zstd/1.3.0/programs/README.md
     - copied unchanged from r320983, vendor/zstd/dist/programs/README.md
  vendor/zstd/1.3.0/programs/bench.c
     - copied unchanged from r320983, vendor/zstd/dist/programs/bench.c
  vendor/zstd/1.3.0/programs/dibio.c
     - copied unchanged from r320983, vendor/zstd/dist/programs/dibio.c
  vendor/zstd/1.3.0/programs/dibio.h
     - copied unchanged from r320983, vendor/zstd/dist/programs/dibio.h
  vendor/zstd/1.3.0/programs/fileio.c
     - copied unchanged from r320983, vendor/zstd/dist/programs/fileio.c
  vendor/zstd/1.3.0/programs/fileio.h
     - copied unchanged from r320983, vendor/zstd/dist/programs/fileio.h
  vendor/zstd/1.3.0/programs/platform.h
     - copied unchanged from r317875, vendor/zstd/dist/programs/platform.h
  vendor/zstd/1.3.0/programs/util.h
     - copied unchanged from r320983, vendor/zstd/dist/programs/util.h
  vendor/zstd/1.3.0/programs/zstd.1
     - copied unchanged from r320983, vendor/zstd/dist/programs/zstd.1
  vendor/zstd/1.3.0/programs/zstdcli.c
     - copied unchanged from r320983, vendor/zstd/dist/programs/zstdcli.c
  vendor/zstd/1.3.0/tests/Makefile
     - copied unchanged from r320983, vendor/zstd/dist/tests/Makefile
  vendor/zstd/1.3.0/tests/datagencli.c
     - copied unchanged from r320983, vendor/zstd/dist/tests/datagencli.c
  vendor/zstd/1.3.0/tests/decodecorpus.c
     - copied unchanged from r320983, vendor/zstd/dist/tests/decodecorpus.c
  vendor/zstd/1.3.0/tests/fullbench.c
     - copied unchanged from r320983, vendor/zstd/dist/tests/fullbench.c
  vendor/zstd/1.3.0/tests/fuzzer.c
     - copied unchanged from r320983, vendor/zstd/dist/tests/fuzzer.c
  vendor/zstd/1.3.0/tests/paramgrill.c
     - copied unchanged from r320983, vendor/zstd/dist/tests/paramgrill.c
  vendor/zstd/1.3.0/tests/playTests.sh
     - copied unchanged from r320983, vendor/zstd/dist/tests/playTests.sh
  vendor/zstd/1.3.0/tests/roundTripCrash.c
     - copied unchanged from r320983, vendor/zstd/dist/tests/roundTripCrash.c
  vendor/zstd/1.3.0/tests/symbols.c
     - copied unchanged from r320983, vendor/zstd/dist/tests/symbols.c
  vendor/zstd/1.3.0/tests/test-zstd-speed.py
     - copied unchanged from r317875, vendor/zstd/dist/tests/test-zstd-speed.py
  vendor/zstd/1.3.0/tests/zbufftest.c
     - copied unchanged from r317875, vendor/zstd/dist/tests/zbufftest.c
  vendor/zstd/1.3.0/tests/zstreamtest.c
     - copied unchanged from r320983, vendor/zstd/dist/tests/zstreamtest.c
  vendor/zstd/1.3.0/zlibWrapper/Makefile
     - copied unchanged from r320983, vendor/zstd/dist/zlibWrapper/Makefile
  vendor/zstd/1.3.0/zlibWrapper/examples/zwrapbench.c
     - copied unchanged from r320983, vendor/zstd/dist/zlibWrapper/examples/zwrapbench.c
  vendor/zstd/1.3.0/zlibWrapper/zstd_zlibwrapper.c
     - copied unchanged from r320983, vendor/zstd/dist/zlibWrapper/zstd_zlibwrapper.c
Deleted:
  vendor/zstd/1.3.0/.gitignore
  vendor/zstd/1.3.0/.travis.yml
  vendor/zstd/1.3.0/build/
  vendor/zstd/1.3.0/lib/dll/
  vendor/zstd/1.3.0/programs/windres/

Copied: vendor/zstd/1.3.0/Makefile (from r320983, vendor/zstd/dist/Makefile)
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ vendor/zstd/1.3.0/Makefile	Fri Jul 14 14:52:44 2017	(r320985, copy of r320983, vendor/zstd/dist/Makefile)
@@ -0,0 +1,332 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+PRGDIR   = programs
+ZSTDDIR  = lib
+BUILDIR  = build
+ZWRAPDIR = zlibWrapper
+TESTDIR  = tests
+
+# Define nul output
+VOID = /dev/null
+
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+.PHONY: default
+default: lib-release zstd-release
+
+.PHONY: all
+all: | allmost examples manual
+
+.PHONY: allmost
+allmost:
+	$(MAKE) -C $(ZSTDDIR) all
+	$(MAKE) -C $(PRGDIR) all
+	$(MAKE) -C $(TESTDIR) all
+	$(MAKE) -C $(ZWRAPDIR) all
+
+#skip zwrapper, can't build that on alternate architectures without the proper zlib installed
+.PHONY: allarch
+allarch:
+	$(MAKE) -C $(ZSTDDIR) all
+	$(MAKE) -C $(PRGDIR) all
+	$(MAKE) -C $(TESTDIR) all
+
+.PHONY: all32
+all32:
+	$(MAKE) -C $(PRGDIR) zstd32
+	$(MAKE) -C $(TESTDIR) all32
+
+.PHONY: lib
+lib:
+	@$(MAKE) -C $(ZSTDDIR) $@
+
+.PHONY: lib-release
+lib-release:
+	@$(MAKE) -C $(ZSTDDIR)
+
+.PHONY: zstd
+zstd:
+	@$(MAKE) -C $(PRGDIR) $@
+	cp $(PRGDIR)/zstd$(EXT) .
+
+.PHONY: zstd-release
+zstd-release:
+	@$(MAKE) -C $(PRGDIR)
+	cp $(PRGDIR)/zstd$(EXT) .
+
+.PHONY: zstdmt
+zstdmt:
+	@$(MAKE) -C $(PRGDIR) $@
+	cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT)
+
+.PHONY: zlibwrapper
+zlibwrapper:
+	$(MAKE) -C $(ZWRAPDIR) test
+
+.PHONY: shortest
+shortest:
+	$(MAKE) -C $(TESTDIR) $@
+
+.PHONY: test
+test:
+	$(MAKE) -C $(TESTDIR) $@
+
+.PHONY: examples
+examples:
+	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
+
+.PHONY: manual
+manual:
+	$(MAKE) -C contrib/gen_html $@
+
+.PHONY: cleanTabs
+cleanTabs:
+	cd contrib; ./cleanTabs
+
+.PHONY: clean
+clean:
+	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
+	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
+	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
+	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
+	@$(MAKE) -C examples/ $@ > $(VOID)
+	@$(MAKE) -C contrib/gen_html $@ > $(VOID)
+	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
+	@echo Cleaning completed
+
+#------------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, Hurd and some BSD targets
+#------------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD MSYS_NT))
+
+HOST_OS = POSIX
+CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON
+
+.PHONY: list
+list:
+	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
+
+.PHONY: install clangtest gpptest armtest usan asan uasan
+install:
+	@$(MAKE) -C $(ZSTDDIR) $@
+	@$(MAKE) -C $(PRGDIR) $@
+
+.PHONY: uninstall
+uninstall:
+	@$(MAKE) -C $(ZSTDDIR) $@
+	@$(MAKE) -C $(PRGDIR) $@
+
+.PHONY: travis-install
+travis-install:
+	$(MAKE) install PREFIX=~/install_test_dir
+
+.PHONY: gppbuild
+gppbuild: clean
+	g++ -v
+	CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+
+.PHONY: gcc5build
+gcc5build: clean
+	gcc-5 -v
+	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"
+
+.PHONY: gcc6build
+gcc6build: clean
+	gcc-6 -v
+	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
+
+.PHONY: clangbuild
+clangbuild: clean
+	clang -v
+	CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+
+m32build: clean
+	gcc -v
+	$(MAKE) all32
+
+armbuild: clean
+	CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allarch
+
+aarch64build: clean
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allarch
+
+ppcbuild: clean
+	CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allarch
+
+ppc64build: clean
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch
+
+armfuzz: clean
+	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
+
+aarch64fuzz: clean
+	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
+
+ppcfuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
+
+ppc64fuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
+
+gpptest: clean
+	CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+
+gcc5test: clean
+	gcc-5 -v
+	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
+
+gcc6test: clean
+	gcc-6 -v
+	$(MAKE) all CC=gcc-6 MOREFLAGS="-Werror"
+
+clangtest: clean
+	clang -v
+	$(MAKE) all CXX=clang-++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+
+armtest: clean
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
+
+aarch64test:
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests
+
+ppctest: clean
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests
+
+ppc64test: clean
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests
+
+arm-ppc-compilation:
+	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
+
+# run UBsan with -fsanitize-recover=signed-integer-overflow
+# due to a bug in UBsan when doing pointer subtraction
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
+
+usan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined"
+
+asan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
+
+asan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $*
+
+msan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" HAVE_LZMA=0   # datagen.c fails this test for no obvious reason
+
+msan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $*
+
+asan32: clean
+	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
+
+uasan: clean
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined"
+
+uasan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*
+
+tsan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests
+
+apt-install:
+	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
+
+apt-add-repo:
+	sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+	sudo apt-get update -y -qq
+
+ppcinstall:
+	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
+
+arminstall:
+	APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
+
+valgrindinstall:
+	APT_PACKAGES="valgrind" $(MAKE) apt-install
+
+libc6install:
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install
+
+gcc6install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install
+
+gpp6install: apt-add-repo
+	APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
+
+clang38install:
+	APT_PACKAGES="clang-3.8" $(MAKE) apt-install
+
+endif
+
+
+ifneq (,$(filter MSYS%,$(shell uname)))
+HOST_OS = MSYS
+CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON
+endif
+
+
+#------------------------------------------------------------------------
+#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
+#------------------------------------------------------------------------
+ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+cmakebuild:
+	cmake --version
+	$(RM) -r $(BUILDIR)/cmake/build
+	mkdir $(BUILDIR)/cmake/build
+	cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
+
+c90build: clean
+	gcc -v
+	CFLAGS="-std=c90" $(MAKE) allmost  # will fail, due to missing support for `long long`
+
+gnu90build: clean
+	gcc -v
+	CFLAGS="-std=gnu90" $(MAKE) allmost
+
+c99build: clean
+	gcc -v
+	CFLAGS="-std=c99" $(MAKE) allmost
+
+gnu99build: clean
+	gcc -v
+	CFLAGS="-std=gnu99" $(MAKE) allmost
+
+c11build: clean
+	gcc -v
+	CFLAGS="-std=c11" $(MAKE) allmost
+
+bmix64build: clean
+	gcc -v
+	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test
+
+bmix32build: clean
+	gcc -v
+	CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test
+
+bmi32build: clean
+	gcc -v
+	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
+
+staticAnalyze: clean
+	gcc -v
+	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
+endif

Copied: vendor/zstd/1.3.0/NEWS (from r320983, vendor/zstd/dist/NEWS)
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ vendor/zstd/1.3.0/NEWS	Fri Jul 14 14:52:44 2017	(r320985, copy of r320983, vendor/zstd/dist/NEWS)
@@ -0,0 +1,298 @@
+v1.3.0
+cli : new : `--list` command, by Paul Cruz
+cli : changed : xz/lzma support enabled by default
+cli : changed : `-t *` continue processing list after a decompression error
+API : added : ZSTD_versionString()
+API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell
+API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter()
+API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx()
+API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700)
+API exp : clarified memory estimation / measurement functions.
+API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1
+tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz
+new : contrib/seekable_format, demo and API, by Sean Purcell
+changed : contrib/linux-kernel, updated version and license, by Nick Terrell
+
+v1.2.0
+cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable)
+cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell
+cli : new : zstdmt symlink hardwired to `zstd -T0`
+cli : new : command --threads=# (#671)
+cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell
+cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters
+cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell
+cli : fix : does not output compressed data on console
+cli : fix : ignore symbolic links unless --force specified,
+API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument
+API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters.
+API : improved: ZSTDMT_compressCCtx() reduced memory usage
+API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634)
+API : fix : src size stored in frame header is controlled at end of frame
+API : fix : enforced consistent rules for pledgedSrcSize==0 (#641)
+API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate
+build: improved cmake script, by @Majlen
+build: enabled Multi-threading support for *BSD, by Baptiste Daroussin
+tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target.
+new : contrib/linux-kernel version, by Nick Terrell
+
+v1.1.4
+cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
+cli : new : advanced benchmark command --priority=rt
+cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
+cli : fix : --rm remains silent when input is stdin
+cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
+speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
+memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
+arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
+API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
+build : new: meson build system in contrib/meson, by Dima Krasner
+build : improved cmake script, by @Majlen
+build : added -Wformat-security flag, as recommended by Padraig Brady
+doc : new : educational decoder, by Sean Purcell
+
+v1.1.3
+cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`)
+cli : new : experimental target `make zstdmt`, with multi-threading support
+cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano.
+cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski
+cli : fix zstdless on Mac OS-X, by Andrew Janke
+cli : fix #232 "compress non-files"
+dictBuilder : improved dictionary generation quality, thanks to Nick Terrell
+API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental)
+API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul
+API : new : ZDICT_finalizeDictionary()
+API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511)
+API : fix : all symbols properly exposed in libzstd, by Nick Terrell
+build : support for Solaris target, by Przemyslaw Skibinski
+doc : clarified specification, by Sean Purcell
+
+v1.1.2
+API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init
+API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize()
+API : zbuff : changed : prototypes now generate deprecation warnings
+lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
+lib : changed : only public ZSTD_ symbols are now exposed
+lib : changed : reduced usage  of stack memory
+lib : fixed : several corner case bugs, by Nick Terrell
+cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
+cli : new : preserve file attributes
+cli : new : added zstdless and zstdgrep tools
+cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
+cli : fixed : zstdcat
+zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
+install : better compatibility with FreeBSD, by Dimitry Andric
+source tree : changed : zbuff source files moved to lib/deprecated
+
+v1.1.1
+New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
+New : doc/zstd_manual.html, by Przemyslaw Skibinski
+Improved : slightly better compression ratio at --ultra levels (>= 20)
+Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report
+Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section)
+Added : example/multiple_streaming_compression.c
+Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h)
+Updated man page
+Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
+
+v1.1.0
+New : contrib/pzstd, parallel version of zstd, by Nick Terrell
+added : NetBSD install target (#338)
+Improved : speed for batches of small files
+Improved : speed of zlib wrapper, by Przemyslaw Skibinski
+Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
+Fixed : CLI -d output to stdout by default when input is stdin (#322)
+Fixed : CLI correctly detects console on Mac OS-X
+Fixed : CLI supports recursive mode `-r` on Mac OS-X
+Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski
+Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
+Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365)
+Fixed : zstd-pgo, reported by octoploid (#329)
+
+v1.0.0
+Change Licensing, all project is now BSD, Copyright Facebook
+Small decompression speed improvement
+API : Streaming API supports legacy format
+API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
+CLI supports legacy formats v0.4+
+Fixed : compression fails on certain huge files, reported by Jesse McGrew
+Enhanced documentation, by Przemyslaw Skibinski
+
+v0.8.1
+New streaming API
+Changed : --ultra now enables levels beyond 19
+Changed : -i# now selects benchmark time in second
+Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
+Fixed : speed regression on specific patterns (#272)
+Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291)
+Fixed : ICC compilation, by Przemyslaw Skibinski
+
+v0.8.0
+Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
+New : Build on FreeBSD and DragonFly, thanks to JrMarino
+Changed : modified API : ZSTD_compressEnd()
+Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist
+Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers
+Fixed : large dictionaries (> 384 KB), reported by Ilona Papava
+Fixed : checksum correctly checked in single-pass mode
+Fixed : combined --test amd --rm, reported by Andreas M. Nilsson
+Modified : minor compression level adaptations
+Updated : compression format specification to v0.2.0
+changed : zstd.h moved to /lib directory
+
+v0.7.5
+Transition version, supporting decoding of v0.8.x
+
+v0.7.4
+Added : homebrew for Mac, by Daniel Cade
+Added : more examples
+Fixed : segfault when using small dictionaries, reported by Felix Handte
+Modified : default compression level for CLI is now 3
+Updated : specification, to v0.1.1
+
+v0.7.3
+New : compression format specification
+New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
+New : `ZSTD_getDecompressedSize()`
+New : OpenBSD target, by Juan Francisco Cantero Hurtado
+New : `examples` directory
+fixed : dictBuilder using HC levels, reported by Bartosz Taudul
+fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
+fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
+modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
+modified : legacy functions no longer need magic number
+
+v0.7.2
+fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
+fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
+fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
+
+v0.7.1
+fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier
+fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв
+fixed : corruption issue, reported by cj
+modified : checksum enabled by default in command line mode
+
+v0.7.0
+New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
+New : Command `--rm`, to remove source file after successful de/compression
+New : Visual build scripts, by Christophe Chevalier
+New : Support for Sparse File-systems (do not use space for zero-filled sectors)
+New : Frame checksum support
+New : Support pass-through mode (when using `-df`)
+API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()`
+API : create dictionary files from custom content, by Giuseppe Ottaviano
+API : support for custom malloc/free functions
+New : controllable Dictionary ID
+New : Support for skippable frames
+
+v0.6.1
+New : zlib wrapper API, thanks to Przemyslaw Skibinski
+New : Ability to compile compressor / decompressor separately
+Changed : new lib directory structure
+Fixed : Legacy codec v0.5 compatible with dictionary decompression
+Fixed : Decoder corruption error (#173)
+Fixed : null-string roundtrip (#176)
+New : benchmark mode can select directory as input
+Experimental : midipix support, VMS support
+
+v0.6.0
+Stronger high compression modes, thanks to Przemyslaw Skibinski
+API : ZSTD_getFrameParams() provides size of decompressed content
+New : highest compression modes require `--ultra` command to fully unleash their capacity
+Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner
+
+v0.5.1
+New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski
+Changed : Dictionary builder integrated into libzstd and zstd cli
+Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`.
+Fix : high compression modes for big-endian platforms
+New : zstd cli : `-t` | `--test` command
+
+v0.5.0
+New : dictionary builder utility
+Changed : streaming & dictionary API
+Improved : better compression of small data
+
+v0.4.7
+Improved : small compression speed improvement in HC mode
+Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
+fix : bt search bug
+
+v0.4.6
+fix : fast compression mode on Windows
+New : cmake configuration file, thanks to Artyom Dymchenko
+Improved : high compression mode on repetitive data
+New : block-level API
+New : ZSTD_duplicateCCtx()
+
+v0.4.5
+new : -m/--multiple : compress/decompress multiple files
+
+v0.4.4
+Fixed : high compression modes for Windows 32 bits
+new : external dictionary API extended to buffered mode and accessible through command line
+new : windows DLL project, thanks to Christophe Chevalier
+
+v0.4.3 :
+new : external dictionary API
+new : zstd-frugal
+
+v0.4.2 :
+Generic minor improvements for small blocks
+Fixed : big-endian compatibility, by Peter Harris (#85)
+
+v0.4.1
+Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben)
+removed `zstd.c`
+
+v0.4.0
+Command line utility compatible with high compression levels
+Removed zstdhc => merged into zstd
+Added : ZBUFF API (see zstd_buffered.h)
+Rolling buffer support
+
+v0.3.6
+small blocks params
+
+v0.3.5
+minor generic compression improvements
+
+v0.3.4
+Faster fast cLevels
+
+v0.3.3
+Small compression ratio improvement
+
+v0.3.2
+Fixed Visual Studio
+
+v0.3.1 :
+Small compression ratio improvement
+
+v0.3
+HC mode : compression levels 2-26
+
+v0.2.2
+Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
+
+v0.2.1
+Fix : Read errors, advanced fuzzer tests, by Hanno Böck
+
+v0.2.0
+**Breaking format change**
+Faster decompression speed
+Can still decode v0.1 format
+
+v0.1.3
+fix uninitialization warning, reported by Evan Nemerson
+
+v0.1.2
+frame concatenation support
+
+v0.1.1
+fix compression bug
+detects write-flush errors
+
+v0.1.0
+first release

Copied: vendor/zstd/1.3.0/README.md (from r320983, vendor/zstd/dist/README.md)
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ vendor/zstd/1.3.0/README.md	Fri Jul 14 14:52:44 2017	(r320985, copy of r320983, vendor/zstd/dist/README.md)
@@ -0,0 +1,153 @@
+ __Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,
+ targeting real-time compression scenarios at zlib-level and better compression ratios.
+
+It is provided as an open-source BSD-licensed **C** library,
+and a command line utility producing and decoding `.zst` and `.gz` files.
+For other programming languages,
+you can consult a list of known ports on [Zstandard homepage](http://www.zstd.net/#other-languages).
+
+| dev branch status |
+|-------------------|
+| [![Build Status][travisDevBadge]][travisLink]   [![Build status][AppveyorDevBadge]][AppveyorLink]   [![Build status][CircleDevBadge]][CircleLink]
+
+[travisDevBadge]: https://travis-ci.org/facebook/zstd.svg?branch=dev "Continuous Integration test suite"
+[travisLink]: https://travis-ci.org/facebook/zstd
+[AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/xt38wbdxjk5mrbem/branch/dev?svg=true "Windows test suite"
+[AppveyorLink]: https://ci.appveyor.com/project/YannCollet/zstd-p0yf0
+[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite"
+[CircleLink]: https://circleci.com/gh/facebook/zstd
+
+
+As a reference, several fast compression algorithms were tested and compared
+on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),
+with a Core i7-6700K CPU @ 4.0GHz,
+using [lzbench], an open-source in-memory benchmark by @inikep
+compiled with GCC 6.3.0,
+on the [Silesia compression corpus].
+
+[lzbench]: https://github.com/inikep/lzbench
+[Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
+
+| Compressor name         | Ratio | Compression| Decompress.|
+| ---------------         | ------| -----------| ---------- |
+| **zstd 1.1.3 -1**       | 2.877 |   430 MB/s |  1110 MB/s |
+| zlib 1.2.8 -1           | 2.743 |   110 MB/s |   400 MB/s |
+| brotli 0.5.2 -0         | 2.708 |   400 MB/s |   430 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   550 MB/s |   710 MB/s |
+| lzo1x 2.09 -1           | 2.108 |   650 MB/s |   830 MB/s |
+| lz4 1.7.5               | 2.101 |   720 MB/s |  3600 MB/s |
+| snappy 1.1.3            | 2.091 |   500 MB/s |  1650 MB/s |
+| lzf 3.6 -1              | 2.077 |   400 MB/s |   860 MB/s |
+
+[zlib]:http://www.zlib.net/
+[LZ4]: http://www.lz4.org/
+
+Zstd can also offer stronger compression ratios at the cost of compression speed.
+Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.
+
+The following tests were run
+on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)
+with a Core i7-6700K CPU @ 4.0GHz,
+using [lzbench], an open-source in-memory benchmark by @inikep
+compiled with GCC 6.3.0,
+on the [Silesia compression corpus].
+
+Compression Speed vs Ratio | Decompression Speed
+---------------------------|--------------------
+![Compression Speed vs Ratio](doc/images/Cspeed4.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/Dspeed4.png "Decompression Speed")
+
+Several algorithms can produce higher compression ratios, but at slower speeds, falling outside of the graph.
+For a larger picture including very slow modes, [click on this link](doc/images/DCspeed5.png) .
+
+
+### The case for Small Data compression
+
+Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives.
+
+The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon.
+
+To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data.
+Training Zstandard is achieved by providing it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression.
+Using this dictionary, the compression ratio achievable on small data improves dramatically.
+
+The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users).
+It consists of roughly 10K records weighing about 1KB each.
+
+Compression Ratio | Compression Speed | Decompression Speed
+------------------|-------------------|--------------------
+![Compression Ratio](doc/images/dict-cr.png "Compression Ratio") | ![Compression Speed](doc/images/dict-cs.png "Compression Speed") | ![Decompression Speed](doc/images/dict-ds.png "Decompression Speed")
+
+
+These compression gains are achieved while simultaneously providing _faster_ compression and decompression speeds.
+
+Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no _universal dictionary_).
+Hence, deploying one dictionary per type of data will provide the greatest benefits.
+Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file.
+
+#### Dictionary compression How To :
+
+1) Create the dictionary
+
+`zstd --train FullPathToTrainingSet/* -o dictionaryName`
+
+2) Compress with dictionary
+
+`zstd -D dictionaryName FILE`
+
+3) Decompress with dictionary
+
+`zstd -D dictionaryName --decompress FILE.zst`
+
+
+### Build
+
+Once you have the repository cloned, there are multiple ways provided to build Zstandard.
+
+#### Makefile
+
+If your system is compatible with a standard `make` (or `gmake`) binary generator,
+you can simply run it at the root directory.
+It will generate `zstd` within root directory.
+
+Other available options include :
+- `make install` : create and install zstd binary, library and man page
+- `make test` : create and run `zstd` and test tools on local platform
+
+#### cmake
+
+A `cmake` project generator is provided within `build/cmake`.
+It can generate Makefiles or other build scripts
+to create `zstd` binary, and `libzstd` dynamic and static libraries.
+
+#### Meson
+
+A Meson project is provided within `contrib/meson`.
+
+#### Visual Studio (Windows)
+
+Going into `build` directory, you will find additional possibilities :
+- Projects for Visual Studio 2005, 2008 and 2010
+  + VS2010 project is compatible with VS2012, VS2013 and VS2015
+- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`,
+  which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution.
+
+
+### Status
+
+Zstandard is currently deployed within Facebook. It is used daily to compress and decompress very large amounts of data in multiple formats and use cases.
+Zstandard is considered safe for production environments.
+
+### License
+
+Zstandard is [BSD-licensed](LICENSE). We also provide an [additional patent grant](PATENTS).
+
+### Contributing
+
+The "dev" branch is the one where all contributions will be merged before reaching "master".
+If you plan to propose a patch, please commit into the "dev" branch or its own feature branch.
+Direct commit to "master" are not permitted.
+For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
+
+### Miscellaneous
+
+Zstd entropy stage is provided by [Huff0 and FSE, from Finite State Entropy library](https://github.com/Cyan4973/FiniteStateEntropy).

Copied: vendor/zstd/1.3.0/appveyor.yml (from r320983, vendor/zstd/dist/appveyor.yml)
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ vendor/zstd/1.3.0/appveyor.yml	Fri Jul 14 14:52:44 2017	(r320985, copy of r320983, vendor/zstd/dist/appveyor.yml)
@@ -0,0 +1,240 @@
+-
+  version: 1.0.{build}
+  branches:
+    only:
+    - dev
+    - master
+  environment:
+    matrix:
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "make allarch && make -C tests test-symbols fullbench-dll fullbench-lib"
+      ARTIFACT: "true"
+      BUILD:    "true"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x86"
+      SCRIPT:   "make allarch"
+      ARTIFACT: "true"
+      BUILD:    "true"
+    - COMPILER: "clang"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch"
+      BUILD:    "true"
+
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   ""
+      TEST:     "cmake"
+
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Release"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
+
+  install:
+  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - SET PATH_ORIGINAL=%PATH%
+  - if [%HOST%]==[mingw] (
+      SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" &&
+      SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe &&
+      COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe
+    )
+  - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] (
+      SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;"
+    )
+
+  build_script:
+  - if [%HOST%]==[mingw] (
+      ( if [%PLATFORM%]==[x64] (
+        SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%"
+      ) else if [%PLATFORM%]==[x86] (
+        SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%"
+      ) )
+    )
+  - if [%HOST%]==[mingw] if [%BUILD%]==[true] (
+      make -v &&
+      sh -c "%COMPILER% -v" &&
+      ECHO Building zlib to static link &&
+      SET "CC=%COMPILER%" &&
+      sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib" &&
+      sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
+      ECHO Building zstd &&
+      SET "CPPFLAGS=-I../../zlib" &&
+      SET "LDFLAGS=../../zlib/libz.a" &&
+      sh -c "%SCRIPT%" &&
+      ( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true]
+          lib\dll\example\build_package.bat &&
+          make -C programs DEBUGFLAGS= clean zstd &&
+          cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
+          appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
+          cp zstd.exe ..\bin\zstd.exe &&
+          cd ..\bin\ && 7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * &&
+          appveyor PushArtifact zstd-win-release-%PLATFORM%.zip
+      )
+    )
+  - if [%HOST%]==[visual] (
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% &&
+      ECHO *** &&
+      msbuild "build\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
+      COPY build\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
+      ECHO *** &&
+      ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
+      ECHO *** &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
+      DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe &&
+      MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
+    )
+
+  test_script:
+  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - SET "CC=gcc"
+  - SET "CXX=g++"
+  - if [%TEST%]==[cmake] (
+      mkdir build\cmake\build &&
+      cd build\cmake\build &&
+      cmake -G "Visual Studio 14 2015 Win64" .. &&
+      cd ..\..\.. &&
+      make clean
+    )
+  - SET "FUZZERTEST=-T30s"
+  - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] (
+      CD tests &&
+      SET ZSTD=./zstd.exe &&
+      sh -e playTests.sh --test-large-data &&
+      fullbench.exe -i1 &&
+      fullbench.exe -i1 -P0 &&
+      fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
+    )
+
+-
+  version: 1.0.{build}
+  environment:
+    matrix:
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "make allarch"
+    - COMPILER: "gcc"
+      HOST:     "mingw"
+      PLATFORM: "x86"
+      SCRIPT:   "make allarch"
+    - COMPILER: "clang"
+      HOST:     "mingw"
+      PLATFORM: "x64"
+      SCRIPT:   "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch"
+
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Debug"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "x64"
+      CONFIGURATION: "Release"
+    - COMPILER: "visual"
+      HOST:     "visual"
+      PLATFORM: "Win32"
+      CONFIGURATION: "Release"
+
+  install:
+  - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - SET PATH_ORIGINAL=%PATH%
+  - if [%HOST%]==[mingw] (

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***

More information about the svn-src-all mailing list