` header for the first line
+- comments of type `/*=` and `/**=` mean: use a `` header and show also all functions until first empty line
+- comments of type `/*X` where `X` is different from above-mentioned are ignored
+
+Moreover:
+- `ZSTDLIB_API` is removed to improve readability
+- `typedef` are detected and included even if uncommented
+- comments of type `/**<` and `/*!<` are detected and only function declaration is highlighted (bold)
+
+
+#### Usage
+
+The program requires 3 parameters:
+```
+gen_html [zstd_version] [input_file] [output_html]
+```
+
+To compile program and generate zstd manual we have used:
+```
+make
+./gen_html.exe 1.1.1 ../../lib/zstd.h zstd_manual.html
+```
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/gen_html/gen-zstd-manual.sh b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/gen_html/gen-zstd-manual.sh
new file mode 100755
index 0000000..57a8b6e
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/gen_html/gen-zstd-manual.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+LIBVER_MAJOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
+LIBVER_MINOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
+LIBVER_PATCH_SCRIPT=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
+LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT
+
+echo ZSTD_VERSION=$LIBVER_SCRIPT
+./gen_html $LIBVER_SCRIPT ../../lib/zstd.h ./zstd_manual.html
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/gen_html/gen_html.cpp b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/gen_html/gen_html.cpp
new file mode 100644
index 0000000..90d5b21
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/gen_html/gen_html.cpp
@@ -0,0 +1,224 @@
+/*
+ * Copyright (c) 2016-present, Przemyslaw Skibinski, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+#include
+#include
+#include
+#include
+using namespace std;
+
+
+/* trim string at the beginning and at the end */
+void trim(string& s, string characters)
+{
+ size_t p = s.find_first_not_of(characters);
+ s.erase(0, p);
+
+ p = s.find_last_not_of(characters);
+ if (string::npos != p)
+ s.erase(p+1);
+}
+
+
+/* trim C++ style comments */
+void trim_comments(string &s)
+{
+ size_t spos, epos;
+
+ spos = s.find("/*");
+ epos = s.find("*/");
+ s = s.substr(spos+3, epos-(spos+3));
+}
+
+
+/* get lines until a given terminator */
+vector get_lines(vector& input, int& linenum, string terminator)
+{
+ vector out;
+ string line;
+ size_t epos;
+
+ while ((size_t)linenum < input.size()) {
+ line = input[linenum];
+
+ if (terminator.empty() && line.empty()) { linenum--; break; }
+
+ epos = line.find(terminator);
+ if (!terminator.empty() && epos!=string::npos) {
+ out.push_back(line);
+ break;
+ }
+ out.push_back(line);
+ linenum++;
+ }
+ return out;
+}
+
+
+/* print line with ZSTDLIB_API removed and C++ comments not bold */
+void print_line(stringstream &sout, string line)
+{
+ size_t spos;
+
+ if (line.substr(0,12) == "ZSTDLIB_API ") line = line.substr(12);
+ spos = line.find("/*");
+ if (spos!=string::npos) {
+ sout << line.substr(0, spos);
+ sout << "" << line.substr(spos) << "" << endl;
+ } else {
+ // fprintf(stderr, "lines=%s\n", line.c_str());
+ sout << line << endl;
+ }
+}
+
+
+int main(int argc, char *argv[]) {
+ char exclam;
+ int linenum, chapter = 1;
+ vector input, lines, comments, chapters;
+ string line, version;
+ size_t spos, l;
+ stringstream sout;
+ ifstream istream;
+ ofstream ostream;
+
+ if (argc < 4) {
+ cout << "usage: " << argv[0] << " [zstd_version] [input_file] [output_html]" << endl;
+ return 1;
+ }
+
+ version = "zstd " + string(argv[1]) + " Manual";
+
+ istream.open(argv[2], ifstream::in);
+ if (!istream.is_open()) {
+ cout << "Error opening file " << argv[2] << endl;
+ return 1;
+ }
+
+ ostream.open(argv[3], ifstream::out);
+ if (!ostream.is_open()) {
+ cout << "Error opening file " << argv[3] << endl;
+ return 1;
+ }
+
+ while (getline(istream, line)) {
+ input.push_back(line);
+ }
+
+ for (linenum=0; (size_t)linenum < input.size(); linenum++) {
+ line = input[linenum];
+
+ /* typedefs are detected and included even if uncommented */
+ if (line.substr(0,7) == "typedef" && line.find("{")!=string::npos) {
+ lines = get_lines(input, linenum, "}");
+ sout << "";
+ for (l=0; l
" << endl;
+ continue;
+ }
+
+ /* comments of type /**< and /*!< are detected and only function declaration is highlighted (bold) */
+ if ((line.find("/**<")!=string::npos || line.find("/*!<")!=string::npos) && line.find("*/")!=string::npos) {
+ sout << "";
+ print_line(sout, line);
+ sout << "
" << endl;
+ continue;
+ }
+
+ spos = line.find("/**=");
+ if (spos==string::npos) {
+ spos = line.find("/*!");
+ if (spos==string::npos)
+ spos = line.find("/**");
+ if (spos==string::npos)
+ spos = line.find("/*-");
+ if (spos==string::npos)
+ spos = line.find("/*=");
+ if (spos==string::npos)
+ continue;
+ exclam = line[spos+2];
+ }
+ else exclam = '=';
+
+ comments = get_lines(input, linenum, "*/");
+ if (!comments.empty()) comments[0] = line.substr(spos+3);
+ if (!comments.empty()) comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/"));
+ for (l=0; l";
+ for (l=0; l";
+ for (l=0; l
" << endl << endl;
+ } else if (exclam == '=') { /* comments of type /*= and /**= mean: use a header and show also all functions until first empty line */
+ trim(comments[0], " ");
+ sout << "" << comments[0] << "
";
+ for (l=1; l";
+ lines = get_lines(input, ++linenum, "");
+ for (l=0; l
" << endl;
+ } else { /* comments of type /** and /*- mean: this is a comment; use a header for the first line */
+ if (comments.empty()) continue;
+
+ trim(comments[0], " ");
+ sout << "" << comments[0] << "
";
+ chapters.push_back(comments[0]);
+ chapter++;
+
+ for (l=1; l 1)
+ sout << "
" << endl << endl;
+ else
+ sout << "
" << endl << endl;
+ }
+ }
+
+ ostream << "\n\n\n" << version << " \n\n" << endl;
+ ostream << "" << version << "
\n";
+
+ ostream << "
\nContents
\n\n";
+ for (size_t i=0; i" << chapters[i].c_str() << "\n";
+ ostream << "
\n
\n";
+
+ ostream << sout.str();
+ ostream << "" << endl << "" << endl;
+
+ return 0;
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/.gitignore b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/.gitignore
new file mode 100644
index 0000000..e77c4e4
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/.gitignore
@@ -0,0 +1,2 @@
+# build artifacts
+largeNbDicts
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/Makefile b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/Makefile
new file mode 100644
index 0000000..4c055b0
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/Makefile
@@ -0,0 +1,58 @@
+# ################################################################
+# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR)
+
+CFLAGS ?= -O3
+CFLAGS += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+ -Wstrict-aliasing=1 -Wswitch-enum \
+ -Wstrict-prototypes -Wundef -Wpointer-arith \
+ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+ -Wredundant-decls
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+
+
+default: largeNbDicts
+
+all : largeNbDicts
+
+largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+ $(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+benchfn.o: $(PROGDIR)/benchfn.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+timefn.o: $(PROGDIR)/timefn.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+datagen.o: $(PROGDIR)/datagen.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+util.o: $(PROGDIR)/util.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+xxhash.o : $(LIBDIR)/common/xxhash.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+clean:
+ $(RM) *.o
+ $(MAKE) -C $(LIBDIR) clean > /dev/null
+ $(RM) largeNbDicts
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/README.md b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/README.md
new file mode 100644
index 0000000..f29bcdf
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/README.md
@@ -0,0 +1,25 @@
+largeNbDicts
+=====================
+
+`largeNbDicts` is a benchmark test tool
+dedicated to the specific scenario of
+dictionary decompression using a very large number of dictionaries.
+When dictionaries are constantly changing, they are always "cold",
+suffering from increased latency due to cache misses.
+
+The tool is created in a bid to investigate performance for this scenario,
+and experiment mitigation techniques.
+
+Command line :
+```
+largeNbDicts [Options] filename(s)
+
+Options :
+-r : recursively load all files in subdirectories (default: off)
+-B# : split input into blocks of size # (default: no split)
+-# : use compression level # (default: 3)
+-D # : use # as a dictionary (default: create one)
+-i# : nb benchmark rounds (default: 6)
+--nbDicts=# : set nb of dictionaries to # (default: one per block)
+-h : help (this text)
+```
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/largeNbDicts.c b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/largeNbDicts.c
new file mode 100644
index 0000000..ddbb3e0
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/largeNbDicts.c
@@ -0,0 +1,998 @@
+/*
+ * Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* largeNbDicts
+ * This is a benchmark test tool
+ * dedicated to the specific case of dictionary decompression
+ * using a very large nb of dictionaries
+ * thus suffering latency from lots of cache misses.
+ * It's created in a bid to investigate performance and find optimizations. */
+
+
+/*--- Dependencies ---*/
+
+#include /* size_t */
+#include /* malloc, free, abort */
+#include /* fprintf */
+#include /* UINT_MAX */
+#include /* assert */
+
+#include "util.h"
+#include "benchfn.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#include "zdict.h"
+
+
+/*--- Constants --- */
+
+#define KB *(1<<10)
+#define MB *(1<<20)
+
+#define BLOCKSIZE_DEFAULT 0 /* no slicing into blocks */
+#define DICTSIZE (4 KB)
+#define CLEVEL_DEFAULT 3
+
+#define BENCH_TIME_DEFAULT_S 6
+#define RUN_TIME_DEFAULT_MS 1000
+#define BENCH_TIME_DEFAULT_MS (BENCH_TIME_DEFAULT_S * RUN_TIME_DEFAULT_MS)
+
+#define DISPLAY_LEVEL_DEFAULT 3
+
+#define BENCH_SIZE_MAX (1200 MB)
+
+
+/*--- Macros ---*/
+
+#define CONTROL(c) { if (!(c)) abort(); }
+#undef MIN
+#define MIN(a,b) ((a) < (b) ? (a) : (b))
+
+
+/*--- Display Macros ---*/
+
+#define DISPLAY(...) fprintf(stdout, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
+static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
+
+
+/*--- buffer_t ---*/
+
+typedef struct {
+ void* ptr;
+ size_t size;
+ size_t capacity;
+} buffer_t;
+
+static const buffer_t kBuffNull = { NULL, 0, 0 };
+
+/* @return : kBuffNull if any error */
+static buffer_t createBuffer(size_t capacity)
+{
+ assert(capacity > 0);
+ void* const ptr = malloc(capacity);
+ if (ptr==NULL) return kBuffNull;
+
+ buffer_t buffer;
+ buffer.ptr = ptr;
+ buffer.capacity = capacity;
+ buffer.size = 0;
+ return buffer;
+}
+
+static void freeBuffer(buffer_t buff)
+{
+ free(buff.ptr);
+}
+
+
+static void fillBuffer_fromHandle(buffer_t* buff, FILE* f)
+{
+ size_t const readSize = fread(buff->ptr, 1, buff->capacity, f);
+ buff->size = readSize;
+}
+
+
+/* @return : kBuffNull if any error */
+static buffer_t createBuffer_fromFile(const char* fileName)
+{
+ U64 const fileSize = UTIL_getFileSize(fileName);
+ size_t const bufferSize = (size_t) fileSize;
+
+ if (fileSize == UTIL_FILESIZE_UNKNOWN) return kBuffNull;
+ assert((U64)bufferSize == fileSize); /* check overflow */
+
+ { FILE* const f = fopen(fileName, "rb");
+ if (f == NULL) return kBuffNull;
+
+ buffer_t buff = createBuffer(bufferSize);
+ CONTROL(buff.ptr != NULL);
+
+ fillBuffer_fromHandle(&buff, f);
+ CONTROL(buff.size == buff.capacity);
+
+ fclose(f); /* do nothing specific if fclose() fails */
+ return buff;
+ }
+}
+
+
+/* @return : kBuffNull if any error */
+static buffer_t
+createDictionaryBuffer(const char* dictionaryName,
+ const void* srcBuffer,
+ const size_t* srcBlockSizes, size_t nbBlocks,
+ size_t requestedDictSize)
+{
+ if (dictionaryName) {
+ DISPLAYLEVEL(3, "loading dictionary %s \n", dictionaryName);
+ return createBuffer_fromFile(dictionaryName); /* note : result might be kBuffNull */
+
+ } else {
+
+ DISPLAYLEVEL(3, "creating dictionary, of target size %u bytes \n",
+ (unsigned)requestedDictSize);
+ void* const dictBuffer = malloc(requestedDictSize);
+ CONTROL(dictBuffer != NULL);
+
+ assert(nbBlocks <= UINT_MAX);
+ size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, requestedDictSize,
+ srcBuffer,
+ srcBlockSizes, (unsigned)nbBlocks);
+ CONTROL(!ZSTD_isError(dictSize));
+
+ buffer_t result;
+ result.ptr = dictBuffer;
+ result.capacity = requestedDictSize;
+ result.size = dictSize;
+ return result;
+ }
+}
+
+static ZSTD_CDict* createCDictForDedicatedDictSearch(const void* dict, size_t dictSize, int compressionLevel)
+{
+ ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
+ ZSTD_CCtxParams_init(params, compressionLevel);
+ ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableDedicatedDictSearch, 1);
+ ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, compressionLevel);
+
+ ZSTD_CDict* cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, params, ZSTD_defaultCMem);
+
+ ZSTD_freeCCtxParams(params);
+ return cdict;
+}
+
+/*! BMK_loadFiles() :
+ * Loads `buffer`, with content from files listed within `fileNamesTable`.
+ * Fills `buffer` entirely.
+ * @return : 0 on success, !=0 on error */
+static int loadFiles(void* buffer, size_t bufferSize,
+ size_t* fileSizes,
+ const char* const * fileNamesTable, unsigned nbFiles)
+{
+ size_t pos = 0, totalSize = 0;
+
+ for (unsigned n=0; n 0);
+ void* const srcBuffer = malloc(loadedSize);
+ assert(srcBuffer != NULL);
+
+ assert(nbFiles > 0);
+ size_t* const fileSizes = (size_t*)calloc(nbFiles, sizeof(*fileSizes));
+ assert(fileSizes != NULL);
+
+ /* Load input buffer */
+ int const errorCode = loadFiles(srcBuffer, loadedSize,
+ fileSizes,
+ fileNamesTable, nbFiles);
+ assert(errorCode == 0);
+
+ void** sliceTable = (void**)malloc(nbFiles * sizeof(*sliceTable));
+ assert(sliceTable != NULL);
+
+ char* const ptr = (char*)srcBuffer;
+ size_t pos = 0;
+ unsigned fileNb = 0;
+ for ( ; (pos < loadedSize) && (fileNb < nbFiles); fileNb++) {
+ sliceTable[fileNb] = ptr + pos;
+ pos += fileSizes[fileNb];
+ }
+ assert(pos == loadedSize);
+ assert(fileNb == nbFiles);
+
+
+ buffer_t buffer;
+ buffer.ptr = srcBuffer;
+ buffer.capacity = loadedSize;
+ buffer.size = loadedSize;
+
+ slice_collection_t slices;
+ slices.slicePtrs = sliceTable;
+ slices.capacities = fileSizes;
+ slices.nbSlices = nbFiles;
+
+ buffer_collection_t bc;
+ bc.buffer = buffer;
+ bc.slices = slices;
+ return bc;
+}
+
+
+
+
+/*--- ddict_collection_t ---*/
+
+typedef struct {
+ ZSTD_DDict** ddicts;
+ size_t nbDDict;
+} ddict_collection_t;
+
+typedef struct {
+ ZSTD_CDict** cdicts;
+ size_t nbCDict;
+} cdict_collection_t;
+
+static const cdict_collection_t kNullCDictCollection = { NULL, 0 };
+
+static void freeCDictCollection(cdict_collection_t cdictc)
+{
+ for (size_t dictNb=0; dictNb < cdictc.nbCDict; dictNb++) {
+ ZSTD_freeCDict(cdictc.cdicts[dictNb]);
+ }
+ free(cdictc.cdicts);
+}
+
+/* returns .buffers=NULL if operation fails */
+static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel, int dedicatedDictSearch)
+{
+ ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*));
+ if (cdicts==NULL) return kNullCDictCollection;
+ for (size_t dictNb=0; dictNb < nbCDict; dictNb++) {
+ cdicts[dictNb] = dedicatedDictSearch ?
+ createCDictForDedicatedDictSearch(dictBuffer, dictSize, cLevel) :
+ ZSTD_createCDict(dictBuffer, dictSize, cLevel);
+ CONTROL(cdicts[dictNb] != NULL);
+ }
+ cdict_collection_t cdictc;
+ cdictc.cdicts = cdicts;
+ cdictc.nbCDict = nbCDict;
+ return cdictc;
+}
+
+static const ddict_collection_t kNullDDictCollection = { NULL, 0 };
+
+static void freeDDictCollection(ddict_collection_t ddictc)
+{
+ for (size_t dictNb=0; dictNb < ddictc.nbDDict; dictNb++) {
+ ZSTD_freeDDict(ddictc.ddicts[dictNb]);
+ }
+ free(ddictc.ddicts);
+}
+
+/* returns .buffers=NULL if operation fails */
+static ddict_collection_t createDDictCollection(const void* dictBuffer, size_t dictSize, size_t nbDDict)
+{
+ ZSTD_DDict** const ddicts = malloc(nbDDict * sizeof(ZSTD_DDict*));
+ assert(ddicts != NULL);
+ if (ddicts==NULL) return kNullDDictCollection;
+ for (size_t dictNb=0; dictNb < nbDDict; dictNb++) {
+ ddicts[dictNb] = ZSTD_createDDict(dictBuffer, dictSize);
+ assert(ddicts[dictNb] != NULL);
+ }
+ ddict_collection_t ddictc;
+ ddictc.ddicts = ddicts;
+ ddictc.nbDDict = nbDDict;
+ return ddictc;
+}
+
+
+/* mess with addresses, so that linear scanning dictionaries != linear address scanning */
+void shuffleCDictionaries(cdict_collection_t dicts)
+{
+ size_t const nbDicts = dicts.nbCDict;
+ for (size_t r=0; rcctx,
+ dst, srcSize,
+ src, srcSize,
+ ci->dictionaries.cdicts[ci->dictNb]);
+
+ ci->dictNb = ci->dictNb + 1;
+ if (ci->dictNb >= ci->nbDicts) ci->dictNb = 0;
+
+ return srcSize;
+}
+
+/* benched function */
+size_t decompress(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* payload)
+{
+ decompressInstructions* const di = (decompressInstructions*) payload;
+
+ size_t const result = ZSTD_decompress_usingDDict(di->dctx,
+ dst, dstCapacity,
+ src, srcSize,
+ di->dictionaries.ddicts[di->dictNb]);
+
+ di->dictNb = di->dictNb + 1;
+ if (di->dictNb >= di->nbDicts) di->dictNb = 0;
+
+ return result;
+}
+
+
+static int benchMem(slice_collection_t dstBlocks,
+ slice_collection_t srcBlocks,
+ ddict_collection_t ddictionaries,
+ cdict_collection_t cdictionaries,
+ unsigned nbRounds, int benchCompression)
+{
+ assert(dstBlocks.nbSlices == srcBlocks.nbSlices);
+
+ unsigned const ms_per_round = RUN_TIME_DEFAULT_MS;
+ unsigned const total_time_ms = nbRounds * ms_per_round;
+
+ double bestSpeed = 0.;
+
+ BMK_timedFnState_t* const benchState =
+ BMK_createTimedFnState(total_time_ms, ms_per_round);
+
+ decompressInstructions di = createDecompressInstructions(ddictionaries);
+ compressInstructions ci = createCompressInstructions(cdictionaries);
+ void* payload = benchCompression ? (void*)&ci : (void*)&di;
+ BMK_benchParams_t const bp = {
+ .benchFn = benchCompression ? compress : decompress,
+ .benchPayload = payload,
+ .initFn = NULL,
+ .initPayload = NULL,
+ .errorFn = ZSTD_isError,
+ .blockCount = dstBlocks.nbSlices,
+ .srcBuffers = (const void* const*) srcBlocks.slicePtrs,
+ .srcSizes = srcBlocks.capacities,
+ .dstBuffers = dstBlocks.slicePtrs,
+ .dstCapacities = dstBlocks.capacities,
+ .blockResults = NULL
+ };
+
+ for (;;) {
+ BMK_runOutcome_t const outcome = BMK_benchTimedFn(benchState, bp);
+ CONTROL(BMK_isSuccessful_runOutcome(outcome));
+
+ BMK_runTime_t const result = BMK_extract_runTime(outcome);
+ double const dTime_ns = result.nanoSecPerRun;
+ double const dTime_sec = (double)dTime_ns / 1000000000;
+ size_t const srcSize = result.sumOfReturn;
+ double const speed_MBps = (double)srcSize / dTime_sec / (1 MB);
+ if (speed_MBps > bestSpeed) bestSpeed = speed_MBps;
+ if (benchCompression)
+ DISPLAY("Compression Speed : %.1f MB/s \r", bestSpeed);
+ else
+ DISPLAY("Decompression Speed : %.1f MB/s \r", bestSpeed);
+
+ fflush(stdout);
+ if (BMK_isCompleted_TimedFn(benchState)) break;
+ }
+ DISPLAY("\n");
+
+ freeDecompressInstructions(di);
+ freeCompressInstructions(ci);
+ BMK_freeTimedFnState(benchState);
+
+ return 0; /* success */
+}
+
+
+/*! bench() :
+ * fileName : file to load for benchmarking purpose
+ * dictionary : optional (can be NULL), file to load as dictionary,
+ * if none provided : will be calculated on the fly by the program.
+ * @return : 0 is success, 1+ otherwise */
+int bench(const char** fileNameTable, unsigned nbFiles,
+ const char* dictionary,
+ size_t blockSize, int clevel,
+ unsigned nbDictMax, unsigned nbBlocks,
+ unsigned nbRounds, int benchCompression,
+ int dedicatedDictSearch)
+{
+ int result = 0;
+
+ DISPLAYLEVEL(3, "loading %u files... \n", nbFiles);
+ buffer_collection_t const srcs = createBufferCollection_fromFiles(fileNameTable, nbFiles);
+ CONTROL(srcs.buffer.ptr != NULL);
+ buffer_t srcBuffer = srcs.buffer;
+ size_t const srcSize = srcBuffer.size;
+ DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n",
+ (double)srcSize / (1 MB));
+
+ slice_collection_t const srcSlices = splitSlices(srcs.slices, blockSize, nbBlocks);
+ nbBlocks = (unsigned)(srcSlices.nbSlices);
+ DISPLAYLEVEL(3, "split input into %u blocks ", nbBlocks);
+ if (blockSize)
+ DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize);
+ DISPLAYLEVEL(3, "\n");
+ size_t const totalSrcSlicesSize = sliceCollection_totalCapacity(srcSlices);
+
+
+ size_t* const dstCapacities = malloc(nbBlocks * sizeof(*dstCapacities));
+ CONTROL(dstCapacities != NULL);
+ size_t dstBufferCapacity = 0;
+ for (size_t bnb=0; bnb='0') && (**stringPtr <='9')) {
+ unsigned const max = (((unsigned)(-1)) / 10) - 1;
+ assert(result <= max); /* check overflow */
+ result *= 10, result += (unsigned)**stringPtr - '0', (*stringPtr)++ ;
+ }
+ if ((**stringPtr=='K') || (**stringPtr=='M')) {
+ unsigned const maxK = ((unsigned)(-1)) >> 10;
+ assert(result <= maxK); /* check overflow */
+ result <<= 10;
+ if (**stringPtr=='M') {
+ assert(result <= maxK); /* check overflow */
+ result <<= 10;
+ }
+ (*stringPtr)++; /* skip `K` or `M` */
+ if (**stringPtr=='i') (*stringPtr)++;
+ if (**stringPtr=='B') (*stringPtr)++;
+ }
+ return result;
+}
+
+/** longCommandWArg() :
+ * check if *stringPtr is the same as longCommand.
+ * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
+ * @return 0 and doesn't modify *stringPtr otherwise.
+ */
+static int longCommandWArg(const char** stringPtr, const char* longCommand)
+{
+ size_t const comSize = strlen(longCommand);
+ int const result = !strncmp(*stringPtr, longCommand, comSize);
+ if (result) *stringPtr += comSize;
+ return result;
+}
+
+
+int usage(const char* exeName)
+{
+ DISPLAY (" \n");
+ DISPLAY (" %s [Options] filename(s) \n", exeName);
+ DISPLAY (" \n");
+ DISPLAY ("Options : \n");
+ DISPLAY ("-z : benchmark compression (default) \n");
+ DISPLAY ("-d : benchmark decompression \n");
+ DISPLAY ("-r : recursively load all files in subdirectories (default: off) \n");
+ DISPLAY ("-B# : split input into blocks of size # (default: no split) \n");
+ DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT);
+ DISPLAY ("-D # : use # as a dictionary (default: create one) \n");
+ DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S);
+ DISPLAY ("--nbBlocks=#: use # blocks for bench (default: one per file) \n");
+ DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n");
+ DISPLAY ("-h : help (this text) \n");
+ return 0;
+}
+
+int bad_usage(const char* exeName)
+{
+ DISPLAY (" bad usage : \n");
+ usage(exeName);
+ return 1;
+}
+
+int main (int argc, const char** argv)
+{
+ int recursiveMode = 0;
+ int benchCompression = 1;
+ int dedicatedDictSearch = 0;
+ unsigned nbRounds = BENCH_TIME_DEFAULT_S;
+ const char* const exeName = argv[0];
+
+ if (argc < 2) return bad_usage(exeName);
+
+ const char** nameTable = (const char**)malloc((size_t)argc * sizeof(const char*));
+ assert(nameTable != NULL);
+ unsigned nameIdx = 0;
+
+ const char* dictionary = NULL;
+ int cLevel = CLEVEL_DEFAULT;
+ size_t blockSize = BLOCKSIZE_DEFAULT;
+ unsigned nbDicts = 0; /* determine nbDicts automatically: 1 dictionary per block */
+ unsigned nbBlocks = 0; /* determine nbBlocks automatically, from source and blockSize */
+
+ for (int argNb = 1; argNb < argc ; argNb++) {
+ const char* argument = argv[argNb];
+ if (!strcmp(argument, "-h")) { free(nameTable); return usage(exeName); }
+ if (!strcmp(argument, "-d")) { benchCompression = 0; continue; }
+ if (!strcmp(argument, "-z")) { benchCompression = 1; continue; }
+ if (!strcmp(argument, "-r")) { recursiveMode = 1; continue; }
+ if (!strcmp(argument, "-D")) { argNb++; assert(argNb < argc); dictionary = argv[argNb]; continue; }
+ if (longCommandWArg(&argument, "-i")) { nbRounds = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--dictionary=")) { dictionary = argument; continue; }
+ if (longCommandWArg(&argument, "-B")) { blockSize = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--blockSize=")) { blockSize = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--nbBlocks=")) { nbBlocks = readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--clevel=")) { cLevel = (int)readU32FromChar(&argument); continue; }
+ if (longCommandWArg(&argument, "--dedicated-dict-search")) { dedicatedDictSearch = 1; continue; }
+ if (longCommandWArg(&argument, "-")) { cLevel = (int)readU32FromChar(&argument); continue; }
+ /* anything that's not a command is a filename */
+ nameTable[nameIdx++] = argument;
+ }
+
+ FileNamesTable* filenameTable;
+
+ if (recursiveMode) {
+#ifndef UTIL_HAS_CREATEFILELIST
+ assert(0); /* missing capability, do not run */
+#endif
+ filenameTable = UTIL_createExpandedFNT(nameTable, nameIdx, 1 /* follow_links */);
+ } else {
+ filenameTable = UTIL_assembleFileNamesTable(nameTable, nameIdx, NULL);
+ nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */
+ }
+
+ int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression, dedicatedDictSearch);
+
+ UTIL_freeFileNamesTable(filenameTable);
+ free(nameTable);
+
+ return result;
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/.gitignore b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/.gitignore
new file mode 100644
index 0000000..d8dfeef
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/.gitignore
@@ -0,0 +1,4 @@
+!lib/zstd
+!lib/zstd/*
+*.o
+*.a
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/Makefile b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/Makefile
new file mode 100644
index 0000000..c391df7
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/Makefile
@@ -0,0 +1,95 @@
+# ################################################################
+# Copyright (c) Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+.PHONY: libzstd
+libzstd:
+ rm -rf linux
+ mkdir -p linux
+ mkdir -p linux/include/linux
+ mkdir -p linux/lib/zstd
+ ../freestanding_lib/freestanding.py \
+ --source-lib ../../lib \
+ --output-lib linux/lib/zstd \
+ --xxhash '' \
+ --xxh64-state 'struct xxh64_state' \
+ --xxh64-prefix 'xxh64' \
+ --rewrite-include '=' \
+ --rewrite-include '=' \
+ --rewrite-include '"\.\./zstd.h"=' \
+ --rewrite-include '"(\.\./)?zstd_errors.h"=' \
+ --sed 's,/\*\*\*,/* *,g' \
+ --sed 's,/\*\*,/*,g' \
+ -DZSTD_NO_INTRINSICS \
+ -DZSTD_NO_UNUSED_FUNCTIONS \
+ -DZSTD_LEGACY_SUPPORT=0 \
+ -DZSTD_STATIC_LINKING_ONLY \
+ -DFSE_STATIC_LINKING_ONLY \
+ -DHUF_STATIC_LINKING_ONLY \
+ -DXXH_STATIC_LINKING_ONLY \
+ -DMEM_FORCE_MEMORY_ACCESS=0 \
+ -D__GNUC__ \
+ -DSTATIC_BMI2=0 \
+ -DZSTD_ADDRESS_SANITIZER=0 \
+ -DZSTD_MEMORY_SANITIZER=0 \
+ -DZSTD_COMPRESS_HEAPMODE=1 \
+ -UZSTD_NO_INLINE \
+ -UNO_PREFETCH \
+ -U__cplusplus \
+ -UZSTD_DLL_EXPORT \
+ -UZSTD_DLL_IMPORT \
+ -U__ICCARM__ \
+ -UZSTD_MULTITHREAD \
+ -U_MSC_VER \
+ -U_WIN32 \
+ -RZSTDLIB_VISIBILITY= \
+ -RZSTDERRORLIB_VISIBILITY= \
+ -DZSTD_HAVE_WEAK_SYMBOLS=0 \
+ -DZSTD_TRACE=0 \
+ -DZSTD_NO_TRACE
+ mv linux/lib/zstd/zstd.h linux/include/linux/zstd_lib.h
+ mv linux/lib/zstd/zstd_errors.h linux/include/linux/
+ cp linux_zstd.h linux/include/linux/zstd.h
+ cp zstd_compress_module.c linux/lib/zstd
+ cp zstd_decompress_module.c linux/lib/zstd
+ cp decompress_sources.h linux/lib/zstd
+ cp linux.mk linux/lib/zstd/Makefile
+
+LINUX ?= $(HOME)/repos/linux
+
+.PHONY: import
+import: libzstd
+ rm -f $(LINUX)/include/linux/zstd.h
+ rm -f $(LINUX)/include/linux/zstd_errors.h
+ rm -rf $(LINUX)/lib/zstd
+ cp linux/include/linux/zstd.h $(LINUX)/include/linux
+ cp linux/include/linux/zstd_lib.h $(LINUX)/include/linux
+ cp linux/include/linux/zstd_errors.h $(LINUX)/include/linux
+ cp -r linux/lib/zstd $(LINUX)/lib
+
+import-upstream:
+ rm -rf $(LINUX)/lib/zstd
+ mkdir $(LINUX)/lib/zstd
+ cp ../../lib/zstd.h $(LINUX)/include/linux/zstd_lib.h
+ cp -r ../../lib/common $(LINUX)/lib/zstd
+ cp -r ../../lib/compress $(LINUX)/lib/zstd
+ cp -r ../../lib/decompress $(LINUX)/lib/zstd
+ mv $(LINUX)/lib/zstd/zstd_errors.h $(LINUX)/include/linux
+ rm $(LINUX)/lib/zstd/common/threading.*
+ rm $(LINUX)/lib/zstd/common/pool.*
+ rm $(LINUX)/lib/zstd/common/xxhash.*
+ rm $(LINUX)/lib/zstd/compress/zstdmt_*
+
+.PHONY: test
+test: libzstd
+ $(MAKE) -C test run-test CFLAGS="-O3 $(CFLAGS)" -j
+
+.PHONY: clean
+clean:
+ $(RM) -rf linux
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/README.md b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/README.md
new file mode 100644
index 0000000..bfa070d
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/README.md
@@ -0,0 +1,14 @@
+# Zstd in the Linux Kernel
+
+This directory contains the scripts needed to transform upstream zstd into the version imported into the kernel. All the transforms are automated and tested by our continuous integration.
+
+## Upgrading Zstd in the Linux Kernel
+
+1. `cd` into this directory.
+2. Run `make libzstd` and read the output. Make sure that all the diffs printed and changes made by the script are correct.
+3. Run `make test` and ensure that it passes.
+4. Import zstd into the Linux Kernel `make import LINUX=/path/to/linux/repo`
+5. Inspect the diff for sanity.
+6. Check the Linux Kernel history for zstd. If any patches were made to the kernel version of zstd, but not to upstream zstd, then port them upstream if necessary.
+7. Test the diff. Benchmark if necessary. Make sure to test multiple architectures: At least x86, i386, and arm.
+8. Submit the patch to the LKML.
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/btrfs-benchmark.sh b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/btrfs-benchmark.sh
new file mode 100755
index 0000000..5e28da9
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/btrfs-benchmark.sh
@@ -0,0 +1,104 @@
+# !/bin/sh
+set -e
+
+# Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
+# The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and
+# 16 GB of RAM and an SSD.
+
+# silesia is a directory that can be downloaded from
+# http://mattmahoney.net/dc/silesia.html
+# ls -l silesia/
+# total 203M
+# -rwxr-xr-x 1 terrelln 9.8M Apr 12 2002 dickens
+# -rwxr-xr-x 1 terrelln 49M May 31 2002 mozilla
+# -rwxr-xr-x 1 terrelln 9.6M Mar 20 2003 mr
+# -rwxr-xr-x 1 terrelln 32M Apr 2 2002 nci
+# -rwxr-xr-x 1 terrelln 5.9M Jul 4 2002 ooffice
+# -rwxr-xr-x 1 terrelln 9.7M Apr 11 2002 osdb
+# -rwxr-xr-x 1 terrelln 6.4M Apr 2 2002 reymont
+# -rwxr-xr-x 1 terrelln 21M Mar 25 2002 samba
+# -rwxr-xr-x 1 terrelln 7.0M Mar 24 2002 sao
+# -rwxr-xr-x 1 terrelln 40M Mar 25 2002 webster
+# -rwxr-xr-x 1 terrelln 8.1M Apr 4 2002 x-ray
+# -rwxr-xr-x 1 terrelln 5.1M Nov 30 2000 xml
+
+# $HOME is on a ext4 filesystem
+BENCHMARK_DIR="$HOME/silesia/"
+N=10
+
+# Normalize the environment
+sudo umount /mnt/btrfs 2> /dev/null > /dev/null || true
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+sudo rm -rf /mnt/btrfs/*
+sync
+sudo umount /mnt/btrfs
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+
+# Run the benchmark
+echo "Compression"
+time sh -c "for i in \$(seq $N); do sudo cp -r $BENCHMARK_DIR /mnt/btrfs/\$i; done; sync"
+
+echo "Approximate compression ratio"
+printf "%d / %d\n" \
+ $(df /mnt/btrfs --output=used -B 1 | tail -n 1) \
+ $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);
+
+# Unmount and remount to avoid any caching
+sudo umount /mnt/btrfs
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+
+echo "Decompression"
+time sudo tar -c /mnt/btrfs 2> /dev/null | wc -c > /dev/null
+
+sudo rm -rf /mnt/btrfs/*
+sudo umount /mnt/btrfs
+
+# Run for each of -o compress-force={none, lzo, zlib, zstd} 5 times and take the
+# min time and ratio.
+# Ran zstd with compression levels {1, 3, 6, 9, 12, 15}.
+# Original size: 2119415342 B (using du /mnt/btrfs)
+
+# none
+# compress: 4.205 s
+# decompress: 3.090 s
+# ratio: 0.99
+
+# lzo
+# compress: 5.328 s
+# decompress: 4.793 s
+# ratio: 1.66
+
+# zlib
+# compress: 32.588 s
+# decompress: 8.791 s
+# ratio : 2.58
+
+# zstd 1
+# compress: 8.147 s
+# decompress: 5.527 s
+# ratio : 2.57
+
+# zstd 3
+# compress: 12.207 s
+# decompress: 5.195 s
+# ratio : 2.71
+
+# zstd 6
+# compress: 30.253 s
+# decompress: 5.324 s
+# ratio : 2.87
+
+# zstd 9
+# compress: 49.659 s
+# decompress: 5.220 s
+# ratio : 2.92
+
+# zstd 12
+# compress: 99.245 s
+# decompress: 5.193 s
+# ratio : 2.93
+
+# zstd 15
+# compress: 196.997 s
+# decompress: 5.992 s
+# ratio : 3.01
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/btrfs-extract-benchmark.sh b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/btrfs-extract-benchmark.sh
new file mode 100755
index 0000000..69721d0
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/btrfs-extract-benchmark.sh
@@ -0,0 +1,99 @@
+# !/bin/sh
+set -e
+
+# Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
+# The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and
+# 16 GB of RAM and an SSD.
+
+# silesia is a directory that can be downloaded from
+# http://mattmahoney.net/dc/silesia.html
+# ls -l silesia/
+# total 203M
+# -rwxr-xr-x 1 terrelln 9.8M Apr 12 2002 dickens
+# -rwxr-xr-x 1 terrelln 49M May 31 2002 mozilla
+# -rwxr-xr-x 1 terrelln 9.6M Mar 20 2003 mr
+# -rwxr-xr-x 1 terrelln 32M Apr 2 2002 nci
+# -rwxr-xr-x 1 terrelln 5.9M Jul 4 2002 ooffice
+# -rwxr-xr-x 1 terrelln 9.7M Apr 11 2002 osdb
+# -rwxr-xr-x 1 terrelln 6.4M Apr 2 2002 reymont
+# -rwxr-xr-x 1 terrelln 21M Mar 25 2002 samba
+# -rwxr-xr-x 1 terrelln 7.0M Mar 24 2002 sao
+# -rwxr-xr-x 1 terrelln 40M Mar 25 2002 webster
+# -rwxr-xr-x 1 terrelln 8.1M Apr 4 2002 x-ray
+# -rwxr-xr-x 1 terrelln 5.1M Nov 30 2000 xml
+
+# $HOME is on a ext4 filesystem
+BENCHMARK_FILE="linux-4.11.6.tar"
+BENCHMARK_DIR="$HOME/$BENCHMARK_FILE"
+
+# Normalize the environment
+sudo umount /mnt/btrfs 2> /dev/null > /dev/null || true
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+sudo rm -rf /mnt/btrfs/*
+sync
+sudo umount /mnt/btrfs
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+
+# Run the benchmark
+echo "Copy"
+time sh -c "sudo cp -r $BENCHMARK_DIR /mnt/btrfs/$BENCHMARK_FILE && sync"
+
+echo "Approximate tarred compression ratio"
+printf "%d / %d\n" \
+ $(df /mnt/btrfs --output=used -B 1 | tail -n 1) \
+ $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);
+
+# Unmount and remount to avoid any caching
+sudo umount /mnt/btrfs
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+
+echo "Extract"
+time sh -c "sudo tar -C /mnt/btrfs -xf /mnt/btrfs/$BENCHMARK_FILE && sync"
+
+# Remove the tarball, leaving only the extracted data
+sudo rm /mnt/btrfs/$BENCHMARK_FILE
+# Unmount and remount to avoid any caching
+sudo umount /mnt/btrfs
+sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
+
+echo "Approximate extracted compression ratio"
+printf "%d / %d\n" \
+ $(df /mnt/btrfs --output=used -B 1 | tail -n 1) \
+ $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);
+
+echo "Read"
+time sudo tar -c /mnt/btrfs 2> /dev/null | wc -c > /dev/null
+
+sudo rm -rf /mnt/btrfs/*
+sudo umount /mnt/btrfs
+
+# Run for each of -o compress-force={none, lzo, zlib, zstd} 5 times and take the
+# min time and ratio.
+
+# none
+# copy: 0.981 s
+# extract: 5.501 s
+# read: 8.807 s
+# tarball ratio: 0.97
+# extracted ratio: 0.78
+
+# lzo
+# copy: 1.631 s
+# extract: 8.458 s
+# read: 8.585 s
+# tarball ratio: 2.06
+# extracted ratio: 1.38
+
+# zlib
+# copy: 7.750 s
+# extract: 21.544 s
+# read: 11.744 s
+# tarball ratio : 3.40
+# extracted ratio: 1.86
+
+# zstd 1
+# copy: 2.579 s
+# extract: 11.479 s
+# read: 9.389 s
+# tarball ratio : 3.57
+# extracted ratio: 1.85
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/decompress_sources.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/decompress_sources.h
new file mode 100644
index 0000000..f35bef0
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/decompress_sources.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*
+ * This file includes every .c file needed for decompression.
+ * It is used by lib/decompress_unzstd.c to include the decompression
+ * source into the translation-unit, so it can be used for kernel
+ * decompression.
+ */
+
+#include "common/debug.c"
+#include "common/entropy_common.c"
+#include "common/error_private.c"
+#include "common/fse_decompress.c"
+#include "common/zstd_common.c"
+#include "decompress/huf_decompress.c"
+#include "decompress/zstd_ddict.c"
+#include "decompress/zstd_decompress.c"
+#include "decompress/zstd_decompress_block.c"
+#include "zstd_decompress_module.c"
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/linux.mk b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/linux.mk
new file mode 100644
index 0000000..19485e3
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/linux.mk
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# ################################################################
+# Copyright (c) Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o
+obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o
+
+ccflags-y += -O3
+
+zstd_compress-y := \
+ zstd_compress_module.o \
+ common/debug.o \
+ common/entropy_common.o \
+ common/error_private.o \
+ common/fse_decompress.o \
+ common/zstd_common.o \
+ compress/fse_compress.o \
+ compress/hist.o \
+ compress/huf_compress.o \
+ compress/zstd_compress.o \
+ compress/zstd_compress_literals.o \
+ compress/zstd_compress_sequences.o \
+ compress/zstd_compress_superblock.o \
+ compress/zstd_double_fast.o \
+ compress/zstd_fast.o \
+ compress/zstd_lazy.o \
+ compress/zstd_ldm.o \
+ compress/zstd_opt.o \
+
+zstd_decompress-y := \
+ zstd_decompress_module.o \
+ common/debug.o \
+ common/entropy_common.o \
+ common/error_private.o \
+ common/fse_decompress.o \
+ common/zstd_common.o \
+ decompress/huf_decompress.o \
+ decompress/zstd_ddict.o \
+ decompress/zstd_decompress.o \
+ decompress/zstd_decompress_block.o \
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/linux_zstd.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/linux_zstd.h
new file mode 100644
index 0000000..446ecab
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/linux_zstd.h
@@ -0,0 +1,447 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of https://github.com/facebook/zstd) and
+ * the GPLv2 (found in the COPYING file in the root directory of
+ * https://github.com/facebook/zstd). You may select, at your option, one of the
+ * above-listed licenses.
+ */
+
+#ifndef LINUX_ZSTD_H
+#define LINUX_ZSTD_H
+
+/**
+ * This is a kernel-style API that wraps the upstream zstd API, which cannot be
+ * used directly because the symbols aren't exported. It exposes the minimal
+ * functionality which is currently required by users of zstd in the kernel.
+ * Expose extra functions from lib/zstd/zstd.h as needed.
+ */
+
+/* ====== Dependency ====== */
+#include
+#include
+#include
+
+/* ====== Helper Functions ====== */
+/**
+ * zstd_compress_bound() - maximum compressed size in worst case scenario
+ * @src_size: The size of the data to compress.
+ *
+ * Return: The maximum compressed size in the worst case scenario.
+ */
+size_t zstd_compress_bound(size_t src_size);
+
+/**
+ * zstd_is_error() - tells if a size_t function result is an error code
+ * @code: The function result to check for error.
+ *
+ * Return: Non-zero iff the code is an error.
+ */
+unsigned int zstd_is_error(size_t code);
+
+/**
+ * enum zstd_error_code - zstd error codes
+ */
+typedef ZSTD_ErrorCode zstd_error_code;
+
+/**
+ * zstd_get_error_code() - translates an error function result to an error code
+ * @code: The function result for which zstd_is_error(code) is true.
+ *
+ * Return: A unique error code for this error.
+ */
+zstd_error_code zstd_get_error_code(size_t code);
+
+/**
+ * zstd_get_error_name() - translates an error function result to a string
+ * @code: The function result for which zstd_is_error(code) is true.
+ *
+ * Return: An error string corresponding to the error code.
+ */
+const char *zstd_get_error_name(size_t code);
+
+/**
+ * zstd_min_clevel() - minimum allowed compression level
+ *
+ * Return: The minimum allowed compression level.
+ */
+int zstd_min_clevel(void);
+
+/**
+ * zstd_max_clevel() - maximum allowed compression level
+ *
+ * Return: The maximum allowed compression level.
+ */
+int zstd_max_clevel(void);
+
+/* ====== Parameter Selection ====== */
+
+/**
+ * enum zstd_strategy - zstd compression search strategy
+ *
+ * From faster to stronger. See zstd_lib.h.
+ */
+typedef ZSTD_strategy zstd_strategy;
+
+/**
+ * struct zstd_compression_parameters - zstd compression parameters
+ * @windowLog: Log of the largest match distance. Larger means more
+ * compression, and more memory needed during decompression.
+ * @chainLog: Fully searched segment. Larger means more compression,
+ * slower, and more memory (useless for fast).
+ * @hashLog: Dispatch table. Larger means more compression,
+ * slower, and more memory.
+ * @searchLog: Number of searches. Larger means more compression and slower.
+ * @searchLength: Match length searched. Larger means faster decompression,
+ * sometimes less compression.
+ * @targetLength: Acceptable match size for optimal parser (only). Larger means
+ * more compression, and slower.
+ * @strategy: The zstd compression strategy.
+ *
+ * See zstd_lib.h.
+ */
+typedef ZSTD_compressionParameters zstd_compression_parameters;
+
+/**
+ * struct zstd_frame_parameters - zstd frame parameters
+ * @contentSizeFlag: Controls whether content size will be present in the
+ * frame header (when known).
+ * @checksumFlag: Controls whether a 32-bit checksum is generated at the
+ * end of the frame for error detection.
+ * @noDictIDFlag: Controls whether dictID will be saved into the frame
+ * header when using dictionary compression.
+ *
+ * The default value is all fields set to 0. See zstd_lib.h.
+ */
+typedef ZSTD_frameParameters zstd_frame_parameters;
+
+/**
+ * struct zstd_parameters - zstd parameters
+ * @cParams: The compression parameters.
+ * @fParams: The frame parameters.
+ */
+typedef ZSTD_parameters zstd_parameters;
+
+/**
+ * zstd_get_params() - returns zstd_parameters for selected level
+ * @level: The compression level
+ * @estimated_src_size: The estimated source size to compress or 0
+ * if unknown.
+ *
+ * Return: The selected zstd_parameters.
+ */
+zstd_parameters zstd_get_params(int level,
+ unsigned long long estimated_src_size);
+
+/* ====== Single-pass Compression ====== */
+
+typedef ZSTD_CCtx zstd_cctx;
+
+/**
+ * zstd_cctx_workspace_bound() - max memory needed to initialize a zstd_cctx
+ * @parameters: The compression parameters to be used.
+ *
+ * If multiple compression parameters might be used, the caller must call
+ * zstd_cctx_workspace_bound() for each set of parameters and use the maximum
+ * size.
+ *
+ * Return: A lower bound on the size of the workspace that is passed to
+ * zstd_init_cctx().
+ */
+size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *parameters);
+
+/**
+ * zstd_init_cctx() - initialize a zstd compression context
+ * @workspace: The workspace to emplace the context into. It must outlive
+ * the returned context.
+ * @workspace_size: The size of workspace. Use zstd_cctx_workspace_bound() to
+ * determine how large the workspace must be.
+ *
+ * Return: A zstd compression context or NULL on error.
+ */
+zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size);
+
+/**
+ * zstd_compress_cctx() - compress src into dst with the initialized parameters
+ * @cctx: The context. Must have been initialized with zstd_init_cctx().
+ * @dst: The buffer to compress src into.
+ * @dst_capacity: The size of the destination buffer. May be any size, but
+ * ZSTD_compressBound(srcSize) is guaranteed to be large enough.
+ * @src: The data to compress.
+ * @src_size: The size of the data to compress.
+ * @parameters: The compression parameters to be used.
+ *
+ * Return: The compressed size or an error, which can be checked using
+ * zstd_is_error().
+ */
+size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
+ const void *src, size_t src_size, const zstd_parameters *parameters);
+
+/* ====== Single-pass Decompression ====== */
+
+typedef ZSTD_DCtx zstd_dctx;
+
+/**
+ * zstd_dctx_workspace_bound() - max memory needed to initialize a zstd_dctx
+ *
+ * Return: A lower bound on the size of the workspace that is passed to
+ * zstd_init_dctx().
+ */
+size_t zstd_dctx_workspace_bound(void);
+
+/**
+ * zstd_init_dctx() - initialize a zstd decompression context
+ * @workspace: The workspace to emplace the context into. It must outlive
+ * the returned context.
+ * @workspace_size: The size of workspace. Use zstd_dctx_workspace_bound() to
+ * determine how large the workspace must be.
+ *
+ * Return: A zstd decompression context or NULL on error.
+ */
+zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size);
+
+/**
+ * zstd_decompress_dctx() - decompress zstd compressed src into dst
+ * @dctx: The decompression context.
+ * @dst: The buffer to decompress src into.
+ * @dst_capacity: The size of the destination buffer. Must be at least as large
+ * as the decompressed size. If the caller cannot upper bound the
+ * decompressed size, then it's better to use the streaming API.
+ * @src: The zstd compressed data to decompress. Multiple concatenated
+ * frames and skippable frames are allowed.
+ * @src_size: The exact size of the data to decompress.
+ *
+ * Return: The decompressed size or an error, which can be checked using
+ * zstd_is_error().
+ */
+size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
+ const void *src, size_t src_size);
+
+/* ====== Streaming Buffers ====== */
+
+/**
+ * struct zstd_in_buffer - input buffer for streaming
+ * @src: Start of the input buffer.
+ * @size: Size of the input buffer.
+ * @pos: Position where reading stopped. Will be updated.
+ * Necessarily 0 <= pos <= size.
+ *
+ * See zstd_lib.h.
+ */
+typedef ZSTD_inBuffer zstd_in_buffer;
+
+/**
+ * struct zstd_out_buffer - output buffer for streaming
+ * @dst: Start of the output buffer.
+ * @size: Size of the output buffer.
+ * @pos: Position where writing stopped. Will be updated.
+ * Necessarily 0 <= pos <= size.
+ *
+ * See zstd_lib.h.
+ */
+typedef ZSTD_outBuffer zstd_out_buffer;
+
+/* ====== Streaming Compression ====== */
+
+typedef ZSTD_CStream zstd_cstream;
+
+/**
+ * zstd_cstream_workspace_bound() - memory needed to initialize a zstd_cstream
+ * @cparams: The compression parameters to be used for compression.
+ *
+ * Return: A lower bound on the size of the workspace that is passed to
+ * zstd_init_cstream().
+ */
+size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams);
+
+/**
+ * zstd_init_cstream() - initialize a zstd streaming compression context
+ * @parameters The zstd parameters to use for compression.
+ * @pledged_src_size: If params.fParams.contentSizeFlag == 1 then the caller
+ * must pass the source size (zero means empty source).
+ * Otherwise, the caller may optionally pass the source
+ * size, or zero if unknown.
+ * @workspace: The workspace to emplace the context into. It must outlive
+ * the returned context.
+ * @workspace_size: The size of workspace.
+ * Use zstd_cstream_workspace_bound(params->cparams) to
+ * determine how large the workspace must be.
+ *
+ * Return: The zstd streaming compression context or NULL on error.
+ */
+zstd_cstream *zstd_init_cstream(const zstd_parameters *parameters,
+ unsigned long long pledged_src_size, void *workspace, size_t workspace_size);
+
+/**
+ * zstd_reset_cstream() - reset the context using parameters from creation
+ * @cstream: The zstd streaming compression context to reset.
+ * @pledged_src_size: Optionally the source size, or zero if unknown.
+ *
+ * Resets the context using the parameters from creation. Skips dictionary
+ * loading, since it can be reused. If `pledged_src_size` is non-zero the frame
+ * content size is always written into the frame header.
+ *
+ * Return: Zero or an error, which can be checked using
+ * zstd_is_error().
+ */
+size_t zstd_reset_cstream(zstd_cstream *cstream,
+ unsigned long long pledged_src_size);
+
+/**
+ * zstd_compress_stream() - streaming compress some of input into output
+ * @cstream: The zstd streaming compression context.
+ * @output: Destination buffer. `output->pos` is updated to indicate how much
+ * compressed data was written.
+ * @input: Source buffer. `input->pos` is updated to indicate how much data
+ * was read. Note that it may not consume the entire input, in which
+ * case `input->pos < input->size`, and it's up to the caller to
+ * present remaining data again.
+ *
+ * The `input` and `output` buffers may be any size. Guaranteed to make some
+ * forward progress if `input` and `output` are not empty.
+ *
+ * Return: A hint for the number of bytes to use as the input for the next
+ * function call or an error, which can be checked using
+ * zstd_is_error().
+ */
+size_t zstd_compress_stream(zstd_cstream *cstream, zstd_out_buffer *output,
+ zstd_in_buffer *input);
+
+/**
+ * zstd_flush_stream() - flush internal buffers into output
+ * @cstream: The zstd streaming compression context.
+ * @output: Destination buffer. `output->pos` is updated to indicate how much
+ * compressed data was written.
+ *
+ * zstd_flush_stream() must be called until it returns 0, meaning all the data
+ * has been flushed. Since zstd_flush_stream() causes a block to be ended,
+ * calling it too often will degrade the compression ratio.
+ *
+ * Return: The number of bytes still present within internal buffers or an
+ * error, which can be checked using zstd_is_error().
+ */
+size_t zstd_flush_stream(zstd_cstream *cstream, zstd_out_buffer *output);
+
+/**
+ * zstd_end_stream() - flush internal buffers into output and end the frame
+ * @cstream: The zstd streaming compression context.
+ * @output: Destination buffer. `output->pos` is updated to indicate how much
+ * compressed data was written.
+ *
+ * zstd_end_stream() must be called until it returns 0, meaning all the data has
+ * been flushed and the frame epilogue has been written.
+ *
+ * Return: The number of bytes still present within internal buffers or an
+ * error, which can be checked using zstd_is_error().
+ */
+size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output);
+
+/* ====== Streaming Decompression ====== */
+
+typedef ZSTD_DStream zstd_dstream;
+
+/**
+ * zstd_dstream_workspace_bound() - memory needed to initialize a zstd_dstream
+ * @max_window_size: The maximum window size allowed for compressed frames.
+ *
+ * Return: A lower bound on the size of the workspace that is passed
+ * to zstd_init_dstream().
+ */
+size_t zstd_dstream_workspace_bound(size_t max_window_size);
+
+/**
+ * zstd_init_dstream() - initialize a zstd streaming decompression context
+ * @max_window_size: The maximum window size allowed for compressed frames.
+ * @workspace: The workspace to emplace the context into. It must outlive
+ * the returned context.
+ * @workspaceSize: The size of workspace.
+ * Use zstd_dstream_workspace_bound(max_window_size) to
+ * determine how large the workspace must be.
+ *
+ * Return: The zstd streaming decompression context.
+ */
+zstd_dstream *zstd_init_dstream(size_t max_window_size, void *workspace,
+ size_t workspace_size);
+
+/**
+ * zstd_reset_dstream() - reset the context using parameters from creation
+ * @dstream: The zstd streaming decompression context to reset.
+ *
+ * Resets the context using the parameters from creation. Skips dictionary
+ * loading, since it can be reused.
+ *
+ * Return: Zero or an error, which can be checked using zstd_is_error().
+ */
+size_t zstd_reset_dstream(zstd_dstream *dstream);
+
+/**
+ * zstd_decompress_stream() - streaming decompress some of input into output
+ * @dstream: The zstd streaming decompression context.
+ * @output: Destination buffer. `output.pos` is updated to indicate how much
+ * decompressed data was written.
+ * @input: Source buffer. `input.pos` is updated to indicate how much data was
+ * read. Note that it may not consume the entire input, in which case
+ * `input.pos < input.size`, and it's up to the caller to present
+ * remaining data again.
+ *
+ * The `input` and `output` buffers may be any size. Guaranteed to make some
+ * forward progress if `input` and `output` are not empty.
+ * zstd_decompress_stream() will not consume the last byte of the frame until
+ * the entire frame is flushed.
+ *
+ * Return: Returns 0 iff a frame is completely decoded and fully flushed.
+ * Otherwise returns a hint for the number of bytes to use as the
+ * input for the next function call or an error, which can be checked
+ * using zstd_is_error(). The size hint will never load more than the
+ * frame.
+ */
+size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
+ zstd_in_buffer *input);
+
+/* ====== Frame Inspection Functions ====== */
+
+/**
+ * zstd_find_frame_compressed_size() - returns the size of a compressed frame
+ * @src: Source buffer. It should point to the start of a zstd encoded
+ * frame or a skippable frame.
+ * @src_size: The size of the source buffer. It must be at least as large as the
+ * size of the frame.
+ *
+ * Return: The compressed size of the frame pointed to by `src` or an error,
+ * which can be check with zstd_is_error().
+ * Suitable to pass to ZSTD_decompress() or similar functions.
+ */
+size_t zstd_find_frame_compressed_size(const void *src, size_t src_size);
+
+/**
+ * struct zstd_frame_params - zstd frame parameters stored in the frame header
+ * @frameContentSize: The frame content size, or ZSTD_CONTENTSIZE_UNKNOWN if not
+ * present.
+ * @windowSize: The window size, or 0 if the frame is a skippable frame.
+ * @blockSizeMax: The maximum block size.
+ * @frameType: The frame type (zstd or skippable)
+ * @headerSize: The size of the frame header.
+ * @dictID: The dictionary id, or 0 if not present.
+ * @checksumFlag: Whether a checksum was used.
+ *
+ * See zstd_lib.h.
+ */
+typedef ZSTD_frameHeader zstd_frame_header;
+
+/**
+ * zstd_get_frame_header() - extracts parameters from a zstd or skippable frame
+ * @params: On success the frame parameters are written here.
+ * @src: The source buffer. It must point to a zstd or skippable frame.
+ * @src_size: The size of the source buffer.
+ *
+ * Return: 0 on success. If more data is required it returns how many bytes
+ * must be provided to make forward progress. Otherwise it returns
+ * an error, which can be checked using zstd_is_error().
+ */
+size_t zstd_get_frame_header(zstd_frame_header *params, const void *src,
+ size_t src_size);
+
+#endif /* LINUX_ZSTD_H */
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/mem.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/mem.h
new file mode 100644
index 0000000..4b5db57
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/mem.h
@@ -0,0 +1,259 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef MEM_H_MODULE
+#define MEM_H_MODULE
+
+/*-****************************************
+* Dependencies
+******************************************/
+#include /* get_unaligned, put_unaligned* */
+#include /* inline */
+#include /* swab32, swab64 */
+#include /* size_t, ptrdiff_t */
+#include "debug.h" /* DEBUG_STATIC_ASSERT */
+
+/*-****************************************
+* Compiler specifics
+******************************************/
+#define MEM_STATIC static inline
+
+/*-**************************************************************
+* Basic Types
+*****************************************************************/
+typedef uint8_t BYTE;
+typedef uint16_t U16;
+typedef int16_t S16;
+typedef uint32_t U32;
+typedef int32_t S32;
+typedef uint64_t U64;
+typedef int64_t S64;
+
+/*-**************************************************************
+* Memory I/O API
+*****************************************************************/
+/*=== Static platform detection ===*/
+MEM_STATIC unsigned MEM_32bits(void);
+MEM_STATIC unsigned MEM_64bits(void);
+MEM_STATIC unsigned MEM_isLittleEndian(void);
+
+/*=== Native unaligned read/write ===*/
+MEM_STATIC U16 MEM_read16(const void* memPtr);
+MEM_STATIC U32 MEM_read32(const void* memPtr);
+MEM_STATIC U64 MEM_read64(const void* memPtr);
+MEM_STATIC size_t MEM_readST(const void* memPtr);
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value);
+MEM_STATIC void MEM_write32(void* memPtr, U32 value);
+MEM_STATIC void MEM_write64(void* memPtr, U64 value);
+
+/*=== Little endian unaligned read/write ===*/
+MEM_STATIC U16 MEM_readLE16(const void* memPtr);
+MEM_STATIC U32 MEM_readLE24(const void* memPtr);
+MEM_STATIC U32 MEM_readLE32(const void* memPtr);
+MEM_STATIC U64 MEM_readLE64(const void* memPtr);
+MEM_STATIC size_t MEM_readLEST(const void* memPtr);
+
+MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val);
+MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val);
+MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32);
+MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64);
+MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val);
+
+/*=== Big endian unaligned read/write ===*/
+MEM_STATIC U32 MEM_readBE32(const void* memPtr);
+MEM_STATIC U64 MEM_readBE64(const void* memPtr);
+MEM_STATIC size_t MEM_readBEST(const void* memPtr);
+
+MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32);
+MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64);
+MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val);
+
+/*=== Byteswap ===*/
+MEM_STATIC U32 MEM_swap32(U32 in);
+MEM_STATIC U64 MEM_swap64(U64 in);
+MEM_STATIC size_t MEM_swapST(size_t in);
+
+/*-**************************************************************
+* Memory I/O Implementation
+*****************************************************************/
+MEM_STATIC unsigned MEM_32bits(void)
+{
+ return sizeof(size_t) == 4;
+}
+
+MEM_STATIC unsigned MEM_64bits(void)
+{
+ return sizeof(size_t) == 8;
+}
+
+#if defined(__LITTLE_ENDIAN)
+#define MEM_LITTLE_ENDIAN 1
+#else
+#define MEM_LITTLE_ENDIAN 0
+#endif
+
+MEM_STATIC unsigned MEM_isLittleEndian(void)
+{
+ return MEM_LITTLE_ENDIAN;
+}
+
+MEM_STATIC U16 MEM_read16(const void *memPtr)
+{
+ return get_unaligned((const U16 *)memPtr);
+}
+
+MEM_STATIC U32 MEM_read32(const void *memPtr)
+{
+ return get_unaligned((const U32 *)memPtr);
+}
+
+MEM_STATIC U64 MEM_read64(const void *memPtr)
+{
+ return get_unaligned((const U64 *)memPtr);
+}
+
+MEM_STATIC size_t MEM_readST(const void *memPtr)
+{
+ return get_unaligned((const size_t *)memPtr);
+}
+
+MEM_STATIC void MEM_write16(void *memPtr, U16 value)
+{
+ put_unaligned(value, (U16 *)memPtr);
+}
+
+MEM_STATIC void MEM_write32(void *memPtr, U32 value)
+{
+ put_unaligned(value, (U32 *)memPtr);
+}
+
+MEM_STATIC void MEM_write64(void *memPtr, U64 value)
+{
+ put_unaligned(value, (U64 *)memPtr);
+}
+
+/*=== Little endian r/w ===*/
+
+MEM_STATIC U16 MEM_readLE16(const void *memPtr)
+{
+ return get_unaligned_le16(memPtr);
+}
+
+MEM_STATIC void MEM_writeLE16(void *memPtr, U16 val)
+{
+ put_unaligned_le16(val, memPtr);
+}
+
+MEM_STATIC U32 MEM_readLE24(const void *memPtr)
+{
+ return MEM_readLE16(memPtr) + (((const BYTE *)memPtr)[2] << 16);
+}
+
+MEM_STATIC void MEM_writeLE24(void *memPtr, U32 val)
+{
+ MEM_writeLE16(memPtr, (U16)val);
+ ((BYTE *)memPtr)[2] = (BYTE)(val >> 16);
+}
+
+MEM_STATIC U32 MEM_readLE32(const void *memPtr)
+{
+ return get_unaligned_le32(memPtr);
+}
+
+MEM_STATIC void MEM_writeLE32(void *memPtr, U32 val32)
+{
+ put_unaligned_le32(val32, memPtr);
+}
+
+MEM_STATIC U64 MEM_readLE64(const void *memPtr)
+{
+ return get_unaligned_le64(memPtr);
+}
+
+MEM_STATIC void MEM_writeLE64(void *memPtr, U64 val64)
+{
+ put_unaligned_le64(val64, memPtr);
+}
+
+MEM_STATIC size_t MEM_readLEST(const void *memPtr)
+{
+ if (MEM_32bits())
+ return (size_t)MEM_readLE32(memPtr);
+ else
+ return (size_t)MEM_readLE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeLEST(void *memPtr, size_t val)
+{
+ if (MEM_32bits())
+ MEM_writeLE32(memPtr, (U32)val);
+ else
+ MEM_writeLE64(memPtr, (U64)val);
+}
+
+/*=== Big endian r/w ===*/
+
+MEM_STATIC U32 MEM_readBE32(const void *memPtr)
+{
+ return get_unaligned_be32(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE32(void *memPtr, U32 val32)
+{
+ put_unaligned_be32(val32, memPtr);
+}
+
+MEM_STATIC U64 MEM_readBE64(const void *memPtr)
+{
+ return get_unaligned_be64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE64(void *memPtr, U64 val64)
+{
+ put_unaligned_be64(val64, memPtr);
+}
+
+MEM_STATIC size_t MEM_readBEST(const void *memPtr)
+{
+ if (MEM_32bits())
+ return (size_t)MEM_readBE32(memPtr);
+ else
+ return (size_t)MEM_readBE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBEST(void *memPtr, size_t val)
+{
+ if (MEM_32bits())
+ MEM_writeBE32(memPtr, (U32)val);
+ else
+ MEM_writeBE64(memPtr, (U64)val);
+}
+
+MEM_STATIC U32 MEM_swap32(U32 in)
+{
+ return swab32(in);
+}
+
+MEM_STATIC U64 MEM_swap64(U64 in)
+{
+ return swab64(in);
+}
+
+MEM_STATIC size_t MEM_swapST(size_t in)
+{
+ if (MEM_32bits())
+ return (size_t)MEM_swap32((U32)in);
+ else
+ return (size_t)MEM_swap64((U64)in);
+}
+
+#endif /* MEM_H_MODULE */
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/squashfs-benchmark.sh b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/squashfs-benchmark.sh
new file mode 100755
index 0000000..02dfd73
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/squashfs-benchmark.sh
@@ -0,0 +1,39 @@
+# !/bin/sh
+set -e
+
+# Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
+# The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and
+# 16 GB of RAM and an SSD.
+
+# $BENCHMARK_DIR is generated with the following commands, from the Ubuntu image
+# ubuntu-16.10-desktop-amd64.iso.
+# > mkdir mnt
+# > sudo mount -o loop ubuntu-16.10-desktop-amd64.iso mnt
+# > cp mnt/casper/filesystem.squashfs .
+# > sudo unsquashfs filesystem.squashfs
+
+# $HOME is on a ext4 filesystem
+BENCHMARK_DIR="$HOME/squashfs-root/"
+BENCHMARK_FS="$HOME/filesystem.squashfs"
+
+# Normalize the environment
+sudo rm -f $BENCHMARK_FS 2> /dev/null > /dev/null || true
+sudo umount /mnt/squashfs 2> /dev/null > /dev/null || true
+
+# Run the benchmark
+echo "Compression"
+echo "sudo mksquashfs $BENCHMARK_DIR $BENCHMARK_FS $@"
+time sudo mksquashfs $BENCHMARK_DIR $BENCHMARK_FS $@ 2> /dev/null > /dev/null
+
+echo "Approximate compression ratio"
+printf "%d / %d\n" \
+ $(sudo du -sx --block-size=1 $BENCHMARK_DIR | cut -f1) \
+ $(sudo du -sx --block-size=1 $BENCHMARK_FS | cut -f1);
+
+# Mount the filesystem
+sudo mount -t squashfs $BENCHMARK_FS /mnt/squashfs
+
+echo "Decompression"
+time sudo tar -c /mnt/squashfs 2> /dev/null | wc -c > /dev/null
+
+sudo umount /mnt/squashfs
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/Makefile b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/Makefile
new file mode 100644
index 0000000..2908839
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/Makefile
@@ -0,0 +1,44 @@
+# ################################################################
+# Copyright (c) Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+LINUX := ../linux
+LINUX_ZSTDLIB := $(LINUX)/lib/zstd
+
+CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG -Wno-deprecated-declarations
+# Don't poison the workspace, it currently doesn't work with static allocation and workspace reuse
+CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE
+
+LINUX_ZSTD_MODULE := $(wildcard $(LINUX_ZSTDLIB)/*.c)
+LINUX_ZSTD_COMMON := $(wildcard $(LINUX_ZSTDLIB)/common/*.c)
+LINUX_ZSTD_COMPRESS := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c)
+LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c)
+LINUX_ZSTD_FILES := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_COMPRESS) $(LINUX_ZSTD_DECOMPRESS)
+LINUX_ZSTD_OBJECTS := $(LINUX_ZSTD_FILES:.c=.o)
+
+liblinuxzstd.a: $(LINUX_ZSTD_OBJECTS)
+ $(AR) $(ARFLAGS) $@ $^
+
+test: test.c liblinuxzstd.a
+ $(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@
+
+static_test: static_test.c
+ $(CC) $(LDFLAGS) $(CPPFLAGS) $(CFLAGS) $^ -o $@
+
+run-test: test static_test
+ ./macro-test.sh
+ ./test
+ ./static_test
+
+.PHONY:
+clean:
+ $(RM) -f $(LINUX_ZSTDLIB)/*.o
+ $(RM) -f $(LINUX_ZSTDLIB)/**/*.o
+ $(RM) -f *.o *.a
+ $(RM) -f test
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/asm/unaligned.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/asm/unaligned.h
new file mode 100644
index 0000000..02c2d74
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/asm/unaligned.h
@@ -0,0 +1,186 @@
+#ifndef ASM_UNALIGNED_H
+#define ASM_UNALIGNED_H
+
+#include
+#include
+
+#ifndef __LITTLE_ENDIAN
+# if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN__)
+# define __LITTLE_ENDIAN 1
+# endif
+#endif
+
+#ifdef __LITTLE_ENDIAN
+# define _IS_LITTLE_ENDIAN 1
+#else
+# define _IS_LITTLE_ENDIAN 0
+#endif
+
+static unsigned _isLittleEndian(void)
+{
+ const union { uint32_t u; uint8_t c[4]; } one = { 1 };
+ assert(_IS_LITTLE_ENDIAN == one.c[0]);
+ return _IS_LITTLE_ENDIAN;
+}
+
+static uint16_t _swap16(uint16_t in)
+{
+ return ((in & 0xF) << 8) + ((in & 0xF0) >> 8);
+}
+
+static uint32_t _swap32(uint32_t in)
+{
+ return __builtin_bswap32(in);
+}
+
+static uint64_t _swap64(uint64_t in)
+{
+ return __builtin_bswap64(in);
+}
+
+/* Little endian */
+static uint16_t get_unaligned_le16(const void* memPtr)
+{
+ uint16_t val;
+ __builtin_memcpy(&val, memPtr, sizeof(val));
+ if (!_isLittleEndian()) _swap16(val);
+ return val;
+}
+
+static uint32_t get_unaligned_le32(const void* memPtr)
+{
+ uint32_t val;
+ __builtin_memcpy(&val, memPtr, sizeof(val));
+ if (!_isLittleEndian()) _swap32(val);
+ return val;
+}
+
+static uint64_t get_unaligned_le64(const void* memPtr)
+{
+ uint64_t val;
+ __builtin_memcpy(&val, memPtr, sizeof(val));
+ if (!_isLittleEndian()) _swap64(val);
+ return val;
+}
+
+static void put_unaligned_le16(uint16_t value, void* memPtr)
+{
+ if (!_isLittleEndian()) value = _swap16(value);
+ __builtin_memcpy(memPtr, &value, sizeof(value));
+}
+
+static void put_unaligned_le32(uint32_t value, void* memPtr)
+{
+ if (!_isLittleEndian()) value = _swap32(value);
+ __builtin_memcpy(memPtr, &value, sizeof(value));
+}
+
+static void put_unaligned_le64(uint64_t value, void* memPtr)
+{
+ if (!_isLittleEndian()) value = _swap64(value);
+ __builtin_memcpy(memPtr, &value, sizeof(value));
+}
+
+/* big endian */
+static uint32_t get_unaligned_be32(const void* memPtr)
+{
+ uint32_t val;
+ __builtin_memcpy(&val, memPtr, sizeof(val));
+ if (_isLittleEndian()) _swap32(val);
+ return val;
+}
+
+static uint64_t get_unaligned_be64(const void* memPtr)
+{
+ uint64_t val;
+ __builtin_memcpy(&val, memPtr, sizeof(val));
+ if (_isLittleEndian()) _swap64(val);
+ return val;
+}
+
+static void put_unaligned_be32(uint32_t value, void* memPtr)
+{
+ if (_isLittleEndian()) value = _swap32(value);
+ __builtin_memcpy(memPtr, &value, sizeof(value));
+}
+
+static void put_unaligned_be64(uint64_t value, void* memPtr)
+{
+ if (_isLittleEndian()) value = _swap64(value);
+ __builtin_memcpy(memPtr, &value, sizeof(value));
+}
+
+/* generic */
+extern void __bad_unaligned_access_size(void);
+
+#define __get_unaligned_le(ptr) ((typeof(*(ptr)))({ \
+ __builtin_choose_expr(sizeof(*(ptr)) == 1, *(ptr), \
+ __builtin_choose_expr(sizeof(*(ptr)) == 2, get_unaligned_le16((ptr)), \
+ __builtin_choose_expr(sizeof(*(ptr)) == 4, get_unaligned_le32((ptr)), \
+ __builtin_choose_expr(sizeof(*(ptr)) == 8, get_unaligned_le64((ptr)), \
+ __bad_unaligned_access_size())))); \
+ }))
+
+#define __get_unaligned_be(ptr) ((typeof(*(ptr)))({ \
+ __builtin_choose_expr(sizeof(*(ptr)) == 1, *(ptr), \
+ __builtin_choose_expr(sizeof(*(ptr)) == 2, get_unaligned_be16((ptr)), \
+ __builtin_choose_expr(sizeof(*(ptr)) == 4, get_unaligned_be32((ptr)), \
+ __builtin_choose_expr(sizeof(*(ptr)) == 8, get_unaligned_be64((ptr)), \
+ __bad_unaligned_access_size())))); \
+ }))
+
+#define __put_unaligned_le(val, ptr) \
+ ({ \
+ void *__gu_p = (ptr); \
+ switch (sizeof(*(ptr))) { \
+ case 1: \
+ *(uint8_t *)__gu_p = (uint8_t)(val); \
+ break; \
+ case 2: \
+ put_unaligned_le16((uint16_t)(val), __gu_p); \
+ break; \
+ case 4: \
+ put_unaligned_le32((uint32_t)(val), __gu_p); \
+ break; \
+ case 8: \
+ put_unaligned_le64((uint64_t)(val), __gu_p); \
+ break; \
+ default: \
+ __bad_unaligned_access_size(); \
+ break; \
+ } \
+ (void)0; \
+ })
+
+#define __put_unaligned_be(val, ptr) \
+ ({ \
+ void *__gu_p = (ptr); \
+ switch (sizeof(*(ptr))) { \
+ case 1: \
+ *(uint8_t *)__gu_p = (uint8_t)(val); \
+ break; \
+ case 2: \
+ put_unaligned_be16((uint16_t)(val), __gu_p); \
+ break; \
+ case 4: \
+ put_unaligned_be32((uint32_t)(val), __gu_p); \
+ break; \
+ case 8: \
+ put_unaligned_be64((uint64_t)(val), __gu_p); \
+ break; \
+ default: \
+ __bad_unaligned_access_size(); \
+ break; \
+ } \
+ (void)0; \
+ })
+
+#if _IS_LITTLE_ENDIAN
+# define get_unaligned __get_unaligned_le
+# define put_unaligned __put_unaligned_le
+#else
+# define get_unaligned __get_unaligned_be
+# define put_unaligned __put_unaligned_be
+#endif
+
+#endif // ASM_UNALIGNED_H
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/compiler.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/compiler.h
new file mode 100644
index 0000000..ea3422e
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/compiler.h
@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_COMPILER_H
+#define LINUX_COMPILER_H
+
+#ifndef inline
+#define inline __inline __attribute__((unused))
+#endif
+
+#ifndef noinline
+#define noinline __attribute__((noinline))
+#endif
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/errno.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/errno.h
new file mode 100644
index 0000000..b247522
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/errno.h
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_ERRNO_H
+#define LINUX_ERRNO_H
+
+#define EINVAL 22
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/kernel.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/kernel.h
new file mode 100644
index 0000000..1f702ab
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/kernel.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_KERNEL_H
+#define LINUX_KERNEL_H
+
+#define WARN_ON(x)
+
+#define PTR_ALIGN(p, a) (typeof(p))ALIGN((unsigned long long)(p), (a))
+#define ALIGN(x, a) ALIGN_MASK((x), (a) - 1)
+#define ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/limits.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/limits.h
new file mode 100644
index 0000000..db9c099
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/limits.h
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_LIMITS_H
+#define LINUX_LIMITS_H
+
+#include
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/math64.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/math64.h
new file mode 100644
index 0000000..8eefa2d
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/math64.h
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_MATH64_H
+#define LINUX_MATH64_H
+
+#define div_u64(dividend, divisor) ((dividend) / (divisor))
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/module.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/module.h
new file mode 100644
index 0000000..be6d20d
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/module.h
@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_MODULE_H
+#define LINUX_MODULE_H
+
+#define EXPORT_SYMBOL(symbol) \
+ void* __##symbol = symbol
+#define MODULE_LICENSE(license)
+#define MODULE_DESCRIPTION(description)
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/printk.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/printk.h
new file mode 100644
index 0000000..eab08e0
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/printk.h
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_PRINTK_H
+#define LINUX_PRINTK_H
+
+#define pr_debug(...)
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/stddef.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/stddef.h
new file mode 100644
index 0000000..8538eb3
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/stddef.h
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_STDDEF_H
+#define LINUX_STDDEF_H
+
+#include
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/swab.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/swab.h
new file mode 100644
index 0000000..783046b
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/swab.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_SWAB_H
+#define LINUX_SWAB_H
+
+#define swab32(x) __builtin_bswap32((x))
+#define swab64(x) __builtin_bswap64((x))
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/types.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/types.h
new file mode 100644
index 0000000..459a457
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/types.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2016-2021, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef LINUX_TYPES_H
+#define LINUX_TYPES_H
+
+#include
+#include
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/xxhash.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/xxhash.h
new file mode 100644
index 0000000..0a43bb2
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/include/linux/xxhash.h
@@ -0,0 +1,746 @@
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://cyan4973.github.io/xxHash/
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+/*
+ * Notice extracted from xxHash homepage:
+ *
+ * xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+ * It also successfully passes all tests from the SMHasher suite.
+ *
+ * Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2
+ * Duo @3GHz)
+ *
+ * Name Speed Q.Score Author
+ * xxHash 5.4 GB/s 10
+ * CrapWow 3.2 GB/s 2 Andrew
+ * MumurHash 3a 2.7 GB/s 10 Austin Appleby
+ * SpookyHash 2.0 GB/s 10 Bob Jenkins
+ * SBox 1.4 GB/s 9 Bret Mulvey
+ * Lookup3 1.2 GB/s 9 Bob Jenkins
+ * SuperFastHash 1.2 GB/s 1 Paul Hsieh
+ * CityHash64 1.05 GB/s 10 Pike & Alakuijala
+ * FNV 0.55 GB/s 5 Fowler, Noll, Vo
+ * CRC32 0.43 GB/s 9
+ * MD5-32 0.33 GB/s 10 Ronald L. Rivest
+ * SHA1-32 0.28 GB/s 10
+ *
+ * Q.Score is a measure of quality of the hash function.
+ * It depends on successfully passing SMHasher test set.
+ * 10 is a perfect score.
+ *
+ * A 64-bits version, named xxh64 offers much better speed,
+ * but for 64-bits applications only.
+ * Name Speed on 64 bits Speed on 32 bits
+ * xxh64 13.8 GB/s 1.9 GB/s
+ * xxh32 6.8 GB/s 6.0 GB/s
+ */
+
+#ifndef XXHASH_H
+#define XXHASH_H
+
+#include
+
+#define XXH_API static inline __attribute__((unused))
+/*-****************************
+ * Simple Hash Functions
+ *****************************/
+
+/**
+ * xxh32() - calculate the 32-bit hash of the input with a given seed.
+ *
+ * @input: The data to hash.
+ * @length: The length of the data to hash.
+ * @seed: The seed can be used to alter the result predictably.
+ *
+ * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
+ *
+ * Return: The 32-bit hash of the data.
+ */
+XXH_API uint32_t xxh32(const void *input, size_t length, uint32_t seed);
+
+/**
+ * xxh64() - calculate the 64-bit hash of the input with a given seed.
+ *
+ * @input: The data to hash.
+ * @length: The length of the data to hash.
+ * @seed: The seed can be used to alter the result predictably.
+ *
+ * This function runs 2x faster on 64-bit systems, but slower on 32-bit systems.
+ *
+ * Return: The 64-bit hash of the data.
+ */
+XXH_API uint64_t xxh64(const void *input, size_t length, uint64_t seed);
+
+/**
+ * xxhash() - calculate wordsize hash of the input with a given seed
+ * @input: The data to hash.
+ * @length: The length of the data to hash.
+ * @seed: The seed can be used to alter the result predictably.
+ *
+ * If the hash does not need to be comparable between machines with
+ * different word sizes, this function will call whichever of xxh32()
+ * or xxh64() is faster.
+ *
+ * Return: wordsize hash of the data.
+ */
+
+static inline unsigned long xxhash(const void *input, size_t length,
+ uint64_t seed)
+{
+#if BITS_PER_LONG == 64
+ return xxh64(input, length, seed);
+#else
+ return xxh32(input, length, seed);
+#endif
+}
+
+/*-****************************
+ * Streaming Hash Functions
+ *****************************/
+
+/*
+ * These definitions are only meant to allow allocation of XXH state
+ * statically, on stack, or in a struct for example.
+ * Do not use members directly.
+ */
+
+/**
+ * struct xxh32_state - private xxh32 state, do not use members directly
+ */
+struct xxh32_state {
+ uint32_t total_len_32;
+ uint32_t large_len;
+ uint32_t v1;
+ uint32_t v2;
+ uint32_t v3;
+ uint32_t v4;
+ uint32_t mem32[4];
+ uint32_t memsize;
+};
+
+/**
+ * struct xxh32_state - private xxh64 state, do not use members directly
+ */
+struct xxh64_state {
+ uint64_t total_len;
+ uint64_t v1;
+ uint64_t v2;
+ uint64_t v3;
+ uint64_t v4;
+ uint64_t mem64[4];
+ uint32_t memsize;
+};
+
+/**
+ * xxh32_reset() - reset the xxh32 state to start a new hashing operation
+ *
+ * @state: The xxh32 state to reset.
+ * @seed: Initialize the hash state with this seed.
+ *
+ * Call this function on any xxh32_state to prepare for a new hashing operation.
+ */
+XXH_API void xxh32_reset(struct xxh32_state *state, uint32_t seed);
+
+/**
+ * xxh32_update() - hash the data given and update the xxh32 state
+ *
+ * @state: The xxh32 state to update.
+ * @input: The data to hash.
+ * @length: The length of the data to hash.
+ *
+ * After calling xxh32_reset() call xxh32_update() as many times as necessary.
+ *
+ * Return: Zero on success, otherwise an error code.
+ */
+XXH_API int xxh32_update(struct xxh32_state *state, const void *input, size_t length);
+
+/**
+ * xxh32_digest() - produce the current xxh32 hash
+ *
+ * @state: Produce the current xxh32 hash of this state.
+ *
+ * A hash value can be produced at any time. It is still possible to continue
+ * inserting input into the hash state after a call to xxh32_digest(), and
+ * generate new hashes later on, by calling xxh32_digest() again.
+ *
+ * Return: The xxh32 hash stored in the state.
+ */
+XXH_API uint32_t xxh32_digest(const struct xxh32_state *state);
+
+/**
+ * xxh64_reset() - reset the xxh64 state to start a new hashing operation
+ *
+ * @state: The xxh64 state to reset.
+ * @seed: Initialize the hash state with this seed.
+ */
+XXH_API void xxh64_reset(struct xxh64_state *state, uint64_t seed);
+
+/**
+ * xxh64_update() - hash the data given and update the xxh64 state
+ * @state: The xxh64 state to update.
+ * @input: The data to hash.
+ * @length: The length of the data to hash.
+ *
+ * After calling xxh64_reset() call xxh64_update() as many times as necessary.
+ *
+ * Return: Zero on success, otherwise an error code.
+ */
+XXH_API int xxh64_update(struct xxh64_state *state, const void *input, size_t length);
+
+/**
+ * xxh64_digest() - produce the current xxh64 hash
+ *
+ * @state: Produce the current xxh64 hash of this state.
+ *
+ * A hash value can be produced at any time. It is still possible to continue
+ * inserting input into the hash state after a call to xxh64_digest(), and
+ * generate new hashes later on, by calling xxh64_digest() again.
+ *
+ * Return: The xxh64 hash stored in the state.
+ */
+XXH_API uint64_t xxh64_digest(const struct xxh64_state *state);
+
+/*-**************************
+ * Utils
+ ***************************/
+
+/**
+ * xxh32_copy_state() - copy the source state into the destination state
+ *
+ * @src: The source xxh32 state.
+ * @dst: The destination xxh32 state.
+ */
+XXH_API void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src);
+
+/**
+ * xxh64_copy_state() - copy the source state into the destination state
+ *
+ * @src: The source xxh64 state.
+ * @dst: The destination xxh64 state.
+ */
+XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src);
+
+/*
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://cyan4973.github.io/xxHash/
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+#include
+#include
+#include
+#include
+#include
+
+/*-*************************************
+ * Macros
+ **************************************/
+#define xxh_rotl32(x, r) ((x << r) | (x >> (32 - r)))
+#define xxh_rotl64(x, r) ((x << r) | (x >> (64 - r)))
+
+#ifdef __LITTLE_ENDIAN
+# define XXH_CPU_LITTLE_ENDIAN 1
+#else
+# define XXH_CPU_LITTLE_ENDIAN 0
+#endif
+
+/*-*************************************
+ * Constants
+ **************************************/
+static const uint32_t PRIME32_1 = 2654435761U;
+static const uint32_t PRIME32_2 = 2246822519U;
+static const uint32_t PRIME32_3 = 3266489917U;
+static const uint32_t PRIME32_4 = 668265263U;
+static const uint32_t PRIME32_5 = 374761393U;
+
+static const uint64_t PRIME64_1 = 11400714785074694791ULL;
+static const uint64_t PRIME64_2 = 14029467366897019727ULL;
+static const uint64_t PRIME64_3 = 1609587929392839161ULL;
+static const uint64_t PRIME64_4 = 9650029242287828579ULL;
+static const uint64_t PRIME64_5 = 2870177450012600261ULL;
+
+/*-**************************
+ * Utils
+ ***************************/
+XXH_API void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src)
+{
+ __builtin_memcpy(dst, src, sizeof(*dst));
+}
+
+XXH_API void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src)
+{
+ __builtin_memcpy(dst, src, sizeof(*dst));
+}
+
+/*-***************************
+ * Simple Hash Functions
+ ****************************/
+static uint32_t xxh32_round(uint32_t seed, const uint32_t input)
+{
+ seed += input * PRIME32_2;
+ seed = xxh_rotl32(seed, 13);
+ seed *= PRIME32_1;
+ return seed;
+}
+
+XXH_API uint32_t xxh32(const void *input, const size_t len, const uint32_t seed)
+{
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t *b_end = p + len;
+ uint32_t h32;
+
+ if (len >= 16) {
+ const uint8_t *const limit = b_end - 16;
+ uint32_t v1 = seed + PRIME32_1 + PRIME32_2;
+ uint32_t v2 = seed + PRIME32_2;
+ uint32_t v3 = seed + 0;
+ uint32_t v4 = seed - PRIME32_1;
+
+ do {
+ v1 = xxh32_round(v1, get_unaligned_le32(p));
+ p += 4;
+ v2 = xxh32_round(v2, get_unaligned_le32(p));
+ p += 4;
+ v3 = xxh32_round(v3, get_unaligned_le32(p));
+ p += 4;
+ v4 = xxh32_round(v4, get_unaligned_le32(p));
+ p += 4;
+ } while (p <= limit);
+
+ h32 = xxh_rotl32(v1, 1) + xxh_rotl32(v2, 7) +
+ xxh_rotl32(v3, 12) + xxh_rotl32(v4, 18);
+ } else {
+ h32 = seed + PRIME32_5;
+ }
+
+ h32 += (uint32_t)len;
+
+ while (p + 4 <= b_end) {
+ h32 += get_unaligned_le32(p) * PRIME32_3;
+ h32 = xxh_rotl32(h32, 17) * PRIME32_4;
+ p += 4;
+ }
+
+ while (p < b_end) {
+ h32 += (*p) * PRIME32_5;
+ h32 = xxh_rotl32(h32, 11) * PRIME32_1;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+
+static uint64_t xxh64_round(uint64_t acc, const uint64_t input)
+{
+ acc += input * PRIME64_2;
+ acc = xxh_rotl64(acc, 31);
+ acc *= PRIME64_1;
+ return acc;
+}
+
+static uint64_t xxh64_merge_round(uint64_t acc, uint64_t val)
+{
+ val = xxh64_round(0, val);
+ acc ^= val;
+ acc = acc * PRIME64_1 + PRIME64_4;
+ return acc;
+}
+
+XXH_API uint64_t xxh64(const void *input, const size_t len, const uint64_t seed)
+{
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t *const b_end = p + len;
+ uint64_t h64;
+
+ if (len >= 32) {
+ const uint8_t *const limit = b_end - 32;
+ uint64_t v1 = seed + PRIME64_1 + PRIME64_2;
+ uint64_t v2 = seed + PRIME64_2;
+ uint64_t v3 = seed + 0;
+ uint64_t v4 = seed - PRIME64_1;
+
+ do {
+ v1 = xxh64_round(v1, get_unaligned_le64(p));
+ p += 8;
+ v2 = xxh64_round(v2, get_unaligned_le64(p));
+ p += 8;
+ v3 = xxh64_round(v3, get_unaligned_le64(p));
+ p += 8;
+ v4 = xxh64_round(v4, get_unaligned_le64(p));
+ p += 8;
+ } while (p <= limit);
+
+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
+ h64 = xxh64_merge_round(h64, v1);
+ h64 = xxh64_merge_round(h64, v2);
+ h64 = xxh64_merge_round(h64, v3);
+ h64 = xxh64_merge_round(h64, v4);
+
+ } else {
+ h64 = seed + PRIME64_5;
+ }
+
+ h64 += (uint64_t)len;
+
+ while (p + 8 <= b_end) {
+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
+
+ h64 ^= k1;
+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
+ p += 8;
+ }
+
+ if (p + 4 <= b_end) {
+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ p += 4;
+ }
+
+ while (p < b_end) {
+ h64 ^= (*p) * PRIME64_5;
+ h64 = xxh_rotl64(h64, 11) * PRIME64_1;
+ p++;
+ }
+
+ h64 ^= h64 >> 33;
+ h64 *= PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= PRIME64_3;
+ h64 ^= h64 >> 32;
+
+ return h64;
+}
+
+/*-**************************************************
+ * Advanced Hash Functions
+ ***************************************************/
+XXH_API void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed)
+{
+ /* use a local state for memcpy() to avoid strict-aliasing warnings */
+ struct xxh32_state state;
+
+ __builtin_memset(&state, 0, sizeof(state));
+ state.v1 = seed + PRIME32_1 + PRIME32_2;
+ state.v2 = seed + PRIME32_2;
+ state.v3 = seed + 0;
+ state.v4 = seed - PRIME32_1;
+ __builtin_memcpy(statePtr, &state, sizeof(state));
+}
+
+XXH_API void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed)
+{
+ /* use a local state for memcpy() to avoid strict-aliasing warnings */
+ struct xxh64_state state;
+
+ __builtin_memset(&state, 0, sizeof(state));
+ state.v1 = seed + PRIME64_1 + PRIME64_2;
+ state.v2 = seed + PRIME64_2;
+ state.v3 = seed + 0;
+ state.v4 = seed - PRIME64_1;
+ __builtin_memcpy(statePtr, &state, sizeof(state));
+}
+
+XXH_API int xxh32_update(struct xxh32_state *state, const void *input, const size_t len)
+{
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t *const b_end = p + len;
+
+ if (input == NULL)
+ return -EINVAL;
+
+ state->total_len_32 += (uint32_t)len;
+ state->large_len |= (len >= 16) | (state->total_len_32 >= 16);
+
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
+ __builtin_memcpy((uint8_t *)(state->mem32) + state->memsize, input, len);
+ state->memsize += (uint32_t)len;
+ return 0;
+ }
+
+ if (state->memsize) { /* some data left from previous update */
+ const uint32_t *p32 = state->mem32;
+
+ __builtin_memcpy((uint8_t *)(state->mem32) + state->memsize, input,
+ 16 - state->memsize);
+
+ state->v1 = xxh32_round(state->v1, get_unaligned_le32(p32));
+ p32++;
+ state->v2 = xxh32_round(state->v2, get_unaligned_le32(p32));
+ p32++;
+ state->v3 = xxh32_round(state->v3, get_unaligned_le32(p32));
+ p32++;
+ state->v4 = xxh32_round(state->v4, get_unaligned_le32(p32));
+ p32++;
+
+ p += 16-state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p <= b_end - 16) {
+ const uint8_t *const limit = b_end - 16;
+ uint32_t v1 = state->v1;
+ uint32_t v2 = state->v2;
+ uint32_t v3 = state->v3;
+ uint32_t v4 = state->v4;
+
+ do {
+ v1 = xxh32_round(v1, get_unaligned_le32(p));
+ p += 4;
+ v2 = xxh32_round(v2, get_unaligned_le32(p));
+ p += 4;
+ v3 = xxh32_round(v3, get_unaligned_le32(p));
+ p += 4;
+ v4 = xxh32_round(v4, get_unaligned_le32(p));
+ p += 4;
+ } while (p <= limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
+
+ if (p < b_end) {
+ __builtin_memcpy(state->mem32, p, (size_t)(b_end-p));
+ state->memsize = (uint32_t)(b_end-p);
+ }
+
+ return 0;
+}
+
+XXH_API uint32_t xxh32_digest(const struct xxh32_state *state)
+{
+ const uint8_t *p = (const uint8_t *)state->mem32;
+ const uint8_t *const b_end = (const uint8_t *)(state->mem32) +
+ state->memsize;
+ uint32_t h32;
+
+ if (state->large_len) {
+ h32 = xxh_rotl32(state->v1, 1) + xxh_rotl32(state->v2, 7) +
+ xxh_rotl32(state->v3, 12) + xxh_rotl32(state->v4, 18);
+ } else {
+ h32 = state->v3 /* == seed */ + PRIME32_5;
+ }
+
+ h32 += state->total_len_32;
+
+ while (p + 4 <= b_end) {
+ h32 += get_unaligned_le32(p) * PRIME32_3;
+ h32 = xxh_rotl32(h32, 17) * PRIME32_4;
+ p += 4;
+ }
+
+ while (p < b_end) {
+ h32 += (*p) * PRIME32_5;
+ h32 = xxh_rotl32(h32, 11) * PRIME32_1;
+ p++;
+ }
+
+ h32 ^= h32 >> 15;
+ h32 *= PRIME32_2;
+ h32 ^= h32 >> 13;
+ h32 *= PRIME32_3;
+ h32 ^= h32 >> 16;
+
+ return h32;
+}
+
+XXH_API int xxh64_update(struct xxh64_state *state, const void *input, const size_t len)
+{
+ const uint8_t *p = (const uint8_t *)input;
+ const uint8_t *const b_end = p + len;
+
+ if (input == NULL)
+ return -EINVAL;
+
+ state->total_len += len;
+
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
+ __builtin_memcpy(((uint8_t *)state->mem64) + state->memsize, input, len);
+ state->memsize += (uint32_t)len;
+ return 0;
+ }
+
+ if (state->memsize) { /* tmp buffer is full */
+ uint64_t *p64 = state->mem64;
+
+ __builtin_memcpy(((uint8_t *)p64) + state->memsize, input,
+ 32 - state->memsize);
+
+ state->v1 = xxh64_round(state->v1, get_unaligned_le64(p64));
+ p64++;
+ state->v2 = xxh64_round(state->v2, get_unaligned_le64(p64));
+ p64++;
+ state->v3 = xxh64_round(state->v3, get_unaligned_le64(p64));
+ p64++;
+ state->v4 = xxh64_round(state->v4, get_unaligned_le64(p64));
+
+ p += 32 - state->memsize;
+ state->memsize = 0;
+ }
+
+ if (p + 32 <= b_end) {
+ const uint8_t *const limit = b_end - 32;
+ uint64_t v1 = state->v1;
+ uint64_t v2 = state->v2;
+ uint64_t v3 = state->v3;
+ uint64_t v4 = state->v4;
+
+ do {
+ v1 = xxh64_round(v1, get_unaligned_le64(p));
+ p += 8;
+ v2 = xxh64_round(v2, get_unaligned_le64(p));
+ p += 8;
+ v3 = xxh64_round(v3, get_unaligned_le64(p));
+ p += 8;
+ v4 = xxh64_round(v4, get_unaligned_le64(p));
+ p += 8;
+ } while (p <= limit);
+
+ state->v1 = v1;
+ state->v2 = v2;
+ state->v3 = v3;
+ state->v4 = v4;
+ }
+
+ if (p < b_end) {
+ __builtin_memcpy(state->mem64, p, (size_t)(b_end-p));
+ state->memsize = (uint32_t)(b_end - p);
+ }
+
+ return 0;
+}
+
+XXH_API uint64_t xxh64_digest(const struct xxh64_state *state)
+{
+ const uint8_t *p = (const uint8_t *)state->mem64;
+ const uint8_t *const b_end = (const uint8_t *)state->mem64 +
+ state->memsize;
+ uint64_t h64;
+
+ if (state->total_len >= 32) {
+ const uint64_t v1 = state->v1;
+ const uint64_t v2 = state->v2;
+ const uint64_t v3 = state->v3;
+ const uint64_t v4 = state->v4;
+
+ h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) +
+ xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18);
+ h64 = xxh64_merge_round(h64, v1);
+ h64 = xxh64_merge_round(h64, v2);
+ h64 = xxh64_merge_round(h64, v3);
+ h64 = xxh64_merge_round(h64, v4);
+ } else {
+ h64 = state->v3 + PRIME64_5;
+ }
+
+ h64 += (uint64_t)state->total_len;
+
+ while (p + 8 <= b_end) {
+ const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p));
+
+ h64 ^= k1;
+ h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4;
+ p += 8;
+ }
+
+ if (p + 4 <= b_end) {
+ h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1;
+ h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ p += 4;
+ }
+
+ while (p < b_end) {
+ h64 ^= (*p) * PRIME64_5;
+ h64 = xxh_rotl64(h64, 11) * PRIME64_1;
+ p++;
+ }
+
+ h64 ^= h64 >> 33;
+ h64 *= PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= PRIME64_3;
+ h64 ^= h64 >> 32;
+
+ return h64;
+}
+
+#endif /* XXHASH_H */
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/macro-test.sh b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/macro-test.sh
new file mode 100755
index 0000000..c688ac0
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/macro-test.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env sh
+
+set -e
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+INCLUDE_DIR="$SCRIPT_DIR/../linux/include"
+LIB_DIR="$SCRIPT_DIR/../linux/lib"
+
+
+print() {
+ printf '%b' "${*}"
+}
+
+println() {
+ printf '%b\n' "${*}"
+}
+
+die() {
+ println "$@" 1>&2
+ exit 1
+}
+
+test_not_present() {
+ print "Testing that '$1' is not present... "
+ grep -r $1 "$INCLUDE_DIR" "$LIB_DIR" && die "Fail!"
+ println "Okay"
+}
+
+println "This test checks that the macro removal process worked as expected"
+println "If this test fails, then freestanding.py wasn't able to remove one of these"
+println "macros from the source code completely. You'll either need to rewrite the check"
+println "or improve freestanding.py."
+println ""
+
+test_not_present "ZSTD_NO_INTRINSICS"
+test_not_present "ZSTD_NO_UNUSED_FUNCTIONS"
+test_not_present "ZSTD_LEGACY_SUPPORT"
+test_not_present "STATIC_BMI2"
+test_not_present "ZSTD_NO_INLINE"
+test_not_present "ZSTD_DLL_EXPORT"
+test_not_present "ZSTD_DLL_IMPORT"
+test_not_present "__ICCARM__"
+test_not_present "_MSC_VER"
+test_not_present "_WIN32"
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/static_test.c b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/static_test.c
new file mode 100644
index 0000000..50c594c
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/static_test.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#include
+#include
+#include
+#include
+
+#include "decompress_sources.h"
+#include
+
+#define CONTROL(x) \
+ do { \
+ if (!(x)) { \
+ fprintf(stderr, "%s:%u: %s failed!\n", __FUNCTION__, __LINE__, #x); \
+ abort(); \
+ } \
+ } while (0)
+
+
+static const char kEmptyZstdFrame[] = {
+ 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x00, 0x01, 0x00, 0x00, 0x99, 0xe9, 0xd8, 0x51
+};
+
+static void test_decompress_unzstd() {
+ fprintf(stderr, "Testing decompress unzstd... ");
+ {
+ size_t const wkspSize = zstd_dctx_workspace_bound();
+ void* wksp = malloc(wkspSize);
+ CONTROL(wksp != NULL);
+ ZSTD_DCtx* dctx = zstd_init_dctx(wksp, wkspSize);
+ CONTROL(dctx != NULL);
+ size_t const dSize = zstd_decompress_dctx(dctx, NULL, 0, kEmptyZstdFrame, sizeof(kEmptyZstdFrame));
+ CONTROL(!zstd_is_error(dSize));
+ CONTROL(dSize == 0);
+ free(wksp);
+ }
+ fprintf(stderr, "Ok\n");
+}
+
+int main(void) {
+ test_decompress_unzstd();
+ return 0;
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/test.c b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/test.c
new file mode 100644
index 0000000..9064be7
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/test/test.c
@@ -0,0 +1,219 @@
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#include
+#include
+#include
+#include
+
+#include
+
+#define CONTROL(x) \
+ do { \
+ if (!(x)) { \
+ fprintf(stderr, "%s:%u: %s failed!\n", __FUNCTION__, __LINE__, #x); \
+ abort(); \
+ } \
+ } while (0)
+
+typedef struct {
+ char *data;
+ char *data2;
+ size_t dataSize;
+ char *comp;
+ size_t compSize;
+} test_data_t;
+
+test_data_t create_test_data(void) {
+ test_data_t data;
+ data.dataSize = 128 * 1024;
+ data.data = malloc(data.dataSize);
+ CONTROL(data.data != NULL);
+ data.data2 = malloc(data.dataSize);
+ CONTROL(data.data2 != NULL);
+ data.compSize = zstd_compress_bound(data.dataSize);
+ data.comp = malloc(data.compSize);
+ CONTROL(data.comp != NULL);
+ memset(data.data, 0, data.dataSize);
+ return data;
+}
+
+static void free_test_data(test_data_t const *data) {
+ free(data->data);
+ free(data->data2);
+ free(data->comp);
+}
+
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+
+static void test_btrfs(test_data_t const *data) {
+ fprintf(stderr, "testing btrfs use cases... ");
+ size_t const size = MIN(data->dataSize, 128 * 1024);
+ for (int level = -1; level < 16; ++level) {
+ zstd_parameters params = zstd_get_params(level, size);
+ CONTROL(params.cParams.windowLog <= 17);
+ size_t const workspaceSize =
+ MAX(zstd_cstream_workspace_bound(¶ms.cParams),
+ zstd_dstream_workspace_bound(size));
+ void *workspace = malloc(workspaceSize);
+ CONTROL(workspace != NULL);
+
+ char const *ip = data->data;
+ char const *iend = ip + size;
+ char *op = data->comp;
+ char *oend = op + data->compSize;
+ {
+ zstd_cstream *cctx = zstd_init_cstream(¶ms, size, workspace, workspaceSize);
+ CONTROL(cctx != NULL);
+ zstd_out_buffer out = {NULL, 0, 0};
+ zstd_in_buffer in = {NULL, 0, 0};
+ for (;;) {
+ if (in.pos == in.size) {
+ in.src = ip;
+ in.size = MIN(4096, iend - ip);
+ in.pos = 0;
+ ip += in.size;
+ }
+
+ if (out.pos == out.size) {
+ out.dst = op;
+ out.size = MIN(4096, oend - op);
+ out.pos = 0;
+ op += out.size;
+ }
+
+ if (ip != iend || in.pos < in.size) {
+ CONTROL(!zstd_is_error(zstd_compress_stream(cctx, &out, &in)));
+ } else {
+ size_t const ret = zstd_end_stream(cctx, &out);
+ CONTROL(!zstd_is_error(ret));
+ if (ret == 0) {
+ break;
+ }
+ }
+ }
+ op += out.pos;
+ }
+
+ ip = data->comp;
+ iend = op;
+ op = data->data2;
+ oend = op + size;
+ {
+ zstd_dstream *dctx = zstd_init_dstream(1ULL << params.cParams.windowLog, workspace, workspaceSize);
+ CONTROL(dctx != NULL);
+ zstd_out_buffer out = {NULL, 0, 0};
+ zstd_in_buffer in = {NULL, 0, 0};
+ for (;;) {
+ if (in.pos == in.size) {
+ in.src = ip;
+ in.size = MIN(4096, iend - ip);
+ in.pos = 0;
+ ip += in.size;
+ }
+
+ if (out.pos == out.size) {
+ out.dst = op;
+ out.size = MIN(4096, oend - op);
+ out.pos = 0;
+ op += out.size;
+ }
+
+ size_t const ret = zstd_decompress_stream(dctx, &out, &in);
+ CONTROL(!zstd_is_error(ret));
+ if (ret == 0) {
+ break;
+ }
+ }
+ }
+ CONTROL(op - data->data2 == data->dataSize);
+ CONTROL(!memcmp(data->data, data->data2, data->dataSize));
+ free(workspace);
+ }
+ fprintf(stderr, "Ok\n");
+}
+
+static void test_decompress_unzstd(test_data_t const *data) {
+ fprintf(stderr, "Testing decompress unzstd... ");
+ size_t cSize;
+ {
+ zstd_parameters params = zstd_get_params(19, 0);
+ size_t const wkspSize = zstd_cctx_workspace_bound(¶ms.cParams);
+ void* wksp = malloc(wkspSize);
+ CONTROL(wksp != NULL);
+ zstd_cctx* cctx = zstd_init_cctx(wksp, wkspSize);
+ CONTROL(cctx != NULL);
+ cSize = zstd_compress_cctx(cctx, data->comp, data->compSize, data->data, data->dataSize, ¶ms);
+ CONTROL(!zstd_is_error(cSize));
+ free(wksp);
+ }
+ {
+ size_t const wkspSize = zstd_dctx_workspace_bound();
+ void* wksp = malloc(wkspSize);
+ CONTROL(wksp != NULL);
+ zstd_dctx* dctx = zstd_init_dctx(wksp, wkspSize);
+ CONTROL(dctx != NULL);
+ size_t const dSize = zstd_decompress_dctx(dctx, data->data2, data->dataSize, data->comp, cSize);
+ CONTROL(!zstd_is_error(dSize));
+ CONTROL(dSize == data->dataSize);
+ CONTROL(!memcmp(data->data, data->data2, data->dataSize));
+ free(wksp);
+ }
+ fprintf(stderr, "Ok\n");
+}
+
+static void test_f2fs() {
+ fprintf(stderr, "testing f2fs uses... ");
+ CONTROL(zstd_min_clevel() < 0);
+ CONTROL(zstd_max_clevel() == 22);
+ fprintf(stderr, "Ok\n");
+}
+
+static char *g_stack = NULL;
+
+static void __attribute__((noinline)) use(void *x) {
+ asm volatile("" : "+r"(x));
+}
+
+static void __attribute__((noinline)) set_stack() {
+
+ char stack[8192];
+ g_stack = stack;
+ memset(g_stack, 0x33, 8192);
+ use(g_stack);
+}
+
+static void __attribute__((noinline)) check_stack() {
+ size_t cleanStack = 0;
+ while (cleanStack < 8192 && g_stack[cleanStack] == 0x33) {
+ ++cleanStack;
+ }
+ size_t const stackSize = 8192 - cleanStack;
+ fprintf(stderr, "Maximum stack size: %zu\n", stackSize);
+ CONTROL(stackSize <= 2048 + 512);
+}
+
+static void test_stack_usage(test_data_t const *data) {
+ set_stack();
+ test_f2fs();
+ test_btrfs(data);
+ test_decompress_unzstd(data);
+ check_stack();
+}
+
+int main(void) {
+ test_data_t data = create_test_data();
+ test_f2fs();
+ test_btrfs(&data);
+ test_decompress_unzstd(&data);
+ test_stack_usage(&data);
+ free_test_data(&data);
+ return 0;
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/zstd_compress_module.c b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/zstd_compress_module.c
new file mode 100644
index 0000000..37d08ff
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/zstd_compress_module.c
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include
+#include
+#include
+#include
+
+#include "common/zstd_deps.h"
+#include "common/zstd_internal.h"
+
+int zstd_min_clevel(void)
+{
+ return ZSTD_minCLevel();
+}
+EXPORT_SYMBOL(zstd_min_clevel);
+
+int zstd_max_clevel(void)
+{
+ return ZSTD_maxCLevel();
+}
+EXPORT_SYMBOL(zstd_max_clevel);
+
+size_t zstd_compress_bound(size_t src_size)
+{
+ return ZSTD_compressBound(src_size);
+}
+EXPORT_SYMBOL(zstd_compress_bound);
+
+zstd_parameters zstd_get_params(int level,
+ unsigned long long estimated_src_size)
+{
+ return ZSTD_getParams(level, estimated_src_size, 0);
+}
+EXPORT_SYMBOL(zstd_get_params);
+
+size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *cparams)
+{
+ return ZSTD_estimateCCtxSize_usingCParams(*cparams);
+}
+EXPORT_SYMBOL(zstd_cctx_workspace_bound);
+
+zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
+{
+ if (workspace == NULL)
+ return NULL;
+ return ZSTD_initStaticCCtx(workspace, workspace_size);
+}
+EXPORT_SYMBOL(zstd_init_cctx);
+
+size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
+ const void *src, size_t src_size, const zstd_parameters *parameters)
+{
+ return ZSTD_compress_advanced(cctx, dst, dst_capacity, src, src_size, NULL, 0, *parameters);
+}
+EXPORT_SYMBOL(zstd_compress_cctx);
+
+size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams)
+{
+ return ZSTD_estimateCStreamSize_usingCParams(*cparams);
+}
+EXPORT_SYMBOL(zstd_cstream_workspace_bound);
+
+zstd_cstream *zstd_init_cstream(const zstd_parameters *parameters,
+ unsigned long long pledged_src_size, void *workspace, size_t workspace_size)
+{
+ zstd_cstream *cstream;
+ size_t ret;
+
+ if (workspace == NULL)
+ return NULL;
+
+ cstream = ZSTD_initStaticCStream(workspace, workspace_size);
+ if (cstream == NULL)
+ return NULL;
+
+ /* 0 means unknown in linux zstd API but means 0 in new zstd API */
+ if (pledged_src_size == 0)
+ pledged_src_size = ZSTD_CONTENTSIZE_UNKNOWN;
+
+ ret = ZSTD_initCStream_advanced(cstream, NULL, 0, *parameters, pledged_src_size);
+ if (ZSTD_isError(ret))
+ return NULL;
+
+ return cstream;
+}
+EXPORT_SYMBOL(zstd_init_cstream);
+
+size_t zstd_reset_cstream(zstd_cstream *cstream,
+ unsigned long long pledged_src_size)
+{
+ return ZSTD_resetCStream(cstream, pledged_src_size);
+}
+EXPORT_SYMBOL(zstd_reset_cstream);
+
+size_t zstd_compress_stream(zstd_cstream *cstream, zstd_out_buffer *output,
+ zstd_in_buffer *input)
+{
+ return ZSTD_compressStream(cstream, output, input);
+}
+EXPORT_SYMBOL(zstd_compress_stream);
+
+size_t zstd_flush_stream(zstd_cstream *cstream, zstd_out_buffer *output)
+{
+ return ZSTD_flushStream(cstream, output);
+}
+EXPORT_SYMBOL(zstd_flush_stream);
+
+size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output)
+{
+ return ZSTD_endStream(cstream, output);
+}
+EXPORT_SYMBOL(zstd_end_stream);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Zstd Compressor");
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/zstd_decompress_module.c b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/zstd_decompress_module.c
new file mode 100644
index 0000000..15005cd
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/zstd_decompress_module.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include
+#include
+#include
+#include
+
+#include "common/zstd_deps.h"
+
+/* Common symbols. zstd_compress must depend on zstd_decompress. */
+
+unsigned int zstd_is_error(size_t code)
+{
+ return ZSTD_isError(code);
+}
+EXPORT_SYMBOL(zstd_is_error);
+
+zstd_error_code zstd_get_error_code(size_t code)
+{
+ return ZSTD_getErrorCode(code);
+}
+EXPORT_SYMBOL(zstd_get_error_code);
+
+const char *zstd_get_error_name(size_t code)
+{
+ return ZSTD_getErrorName(code);
+}
+EXPORT_SYMBOL(zstd_get_error_name);
+
+/* Decompression symbols. */
+
+size_t zstd_dctx_workspace_bound(void)
+{
+ return ZSTD_estimateDCtxSize();
+}
+EXPORT_SYMBOL(zstd_dctx_workspace_bound);
+
+zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size)
+{
+ if (workspace == NULL)
+ return NULL;
+ return ZSTD_initStaticDCtx(workspace, workspace_size);
+}
+EXPORT_SYMBOL(zstd_init_dctx);
+
+size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
+ const void *src, size_t src_size)
+{
+ return ZSTD_decompressDCtx(dctx, dst, dst_capacity, src, src_size);
+}
+EXPORT_SYMBOL(zstd_decompress_dctx);
+
+size_t zstd_dstream_workspace_bound(size_t max_window_size)
+{
+ return ZSTD_estimateDStreamSize(max_window_size);
+}
+EXPORT_SYMBOL(zstd_dstream_workspace_bound);
+
+zstd_dstream *zstd_init_dstream(size_t max_window_size, void *workspace,
+ size_t workspace_size)
+{
+ if (workspace == NULL)
+ return NULL;
+ (void)max_window_size;
+ return ZSTD_initStaticDStream(workspace, workspace_size);
+}
+EXPORT_SYMBOL(zstd_init_dstream);
+
+size_t zstd_reset_dstream(zstd_dstream *dstream)
+{
+ return ZSTD_resetDStream(dstream);
+}
+EXPORT_SYMBOL(zstd_reset_dstream);
+
+size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
+ zstd_in_buffer *input)
+{
+ return ZSTD_decompressStream(dstream, output, input);
+}
+EXPORT_SYMBOL(zstd_decompress_stream);
+
+size_t zstd_find_frame_compressed_size(const void *src, size_t src_size)
+{
+ return ZSTD_findFrameCompressedSize(src, src_size);
+}
+EXPORT_SYMBOL(zstd_find_frame_compressed_size);
+
+size_t zstd_get_frame_header(zstd_frame_header *header, const void *src,
+ size_t src_size)
+{
+ return ZSTD_getFrameHeader(header, src, src_size);
+}
+EXPORT_SYMBOL(zstd_get_frame_header);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Zstd Decompressor");
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/zstd_deps.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/zstd_deps.h
new file mode 100644
index 0000000..853b724
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/linux-kernel/zstd_deps.h
@@ -0,0 +1,125 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*
+ * This file provides common libc dependencies that zstd requires.
+ * The purpose is to allow replacing this file with a custom implementation
+ * to compile zstd without libc support.
+ */
+
+/* Need:
+ * NULL
+ * INT_MAX
+ * UINT_MAX
+ * ZSTD_memcpy()
+ * ZSTD_memset()
+ * ZSTD_memmove()
+ */
+#ifndef ZSTD_DEPS_COMMON
+#define ZSTD_DEPS_COMMON
+
+#include
+#include
+
+#define ZSTD_memcpy(d,s,n) __builtin_memcpy((d),(s),(n))
+#define ZSTD_memmove(d,s,n) __builtin_memmove((d),(s),(n))
+#define ZSTD_memset(d,s,n) __builtin_memset((d),(s),(n))
+
+#endif /* ZSTD_DEPS_COMMON */
+
+/*
+ * Define malloc as always failing. That means the user must
+ * either use ZSTD_customMem or statically allocate memory.
+ * Need:
+ * ZSTD_malloc()
+ * ZSTD_free()
+ * ZSTD_calloc()
+ */
+#ifdef ZSTD_DEPS_NEED_MALLOC
+#ifndef ZSTD_DEPS_MALLOC
+#define ZSTD_DEPS_MALLOC
+
+#define ZSTD_malloc(s) ({ (void)(s); NULL; })
+#define ZSTD_free(p) ((void)(p))
+#define ZSTD_calloc(n,s) ({ (void)(n); (void)(s); NULL; })
+
+#endif /* ZSTD_DEPS_MALLOC */
+#endif /* ZSTD_DEPS_NEED_MALLOC */
+
+/*
+ * Provides 64-bit math support.
+ * Need:
+ * U64 ZSTD_div64(U64 dividend, U32 divisor)
+ */
+#ifdef ZSTD_DEPS_NEED_MATH64
+#ifndef ZSTD_DEPS_MATH64
+#define ZSTD_DEPS_MATH64
+
+#include
+
+static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
+ return div_u64(dividend, divisor);
+}
+
+#endif /* ZSTD_DEPS_MATH64 */
+#endif /* ZSTD_DEPS_NEED_MATH64 */
+
+/*
+ * This is only requested when DEBUGLEVEL >= 1, meaning
+ * it is disabled in production.
+ * Need:
+ * assert()
+ */
+#ifdef ZSTD_DEPS_NEED_ASSERT
+#ifndef ZSTD_DEPS_ASSERT
+#define ZSTD_DEPS_ASSERT
+
+#include
+
+#define assert(x) WARN_ON((x))
+
+#endif /* ZSTD_DEPS_ASSERT */
+#endif /* ZSTD_DEPS_NEED_ASSERT */
+
+/*
+ * This is only requested when DEBUGLEVEL >= 2, meaning
+ * it is disabled in production.
+ * Need:
+ * ZSTD_DEBUG_PRINT()
+ */
+#ifdef ZSTD_DEPS_NEED_IO
+#ifndef ZSTD_DEPS_IO
+#define ZSTD_DEPS_IO
+
+#include
+
+#define ZSTD_DEBUG_PRINT(...) pr_debug(__VA_ARGS__)
+
+#endif /* ZSTD_DEPS_IO */
+#endif /* ZSTD_DEPS_NEED_IO */
+
+/*
+ * Only requested when MSAN is enabled.
+ * Need:
+ * intptr_t
+ */
+#ifdef ZSTD_DEPS_NEED_STDINT
+#ifndef ZSTD_DEPS_STDINT
+#define ZSTD_DEPS_STDINT
+
+/*
+ * The Linux Kernel doesn't provide intptr_t, only uintptr_t, which
+ * is an unsigned long.
+ */
+typedef long intptr_t;
+
+#endif /* ZSTD_DEPS_STDINT */
+#endif /* ZSTD_DEPS_NEED_STDINT */
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/match_finders/README.md b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/match_finders/README.md
new file mode 100644
index 0000000..0f4a3b1
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/match_finders/README.md
@@ -0,0 +1,42 @@
+## Edit Distance Match Finder
+
+```
+/* This match finder leverages techniques used in file comparison algorithms
+ * to find matches between a dictionary and a source file.
+ *
+ * The original motivation for studying this approach was to try and optimize
+ * Zstandard for the use case of patching: the most common scenario being
+ * updating an existing software package with the next version. When patching,
+ * the difference between the old version of the package and the new version
+ * is generally tiny (most of the new file will be identical to
+ * the old one). In more technical terms, the edit distance (the minimal number
+ * of changes required to take one sequence of bytes to another) between the
+ * files would be small relative to the size of the file.
+ *
+ * Various 'diffing' algorithms utilize this notion of edit distance and
+ * the corrensponding concept of a minimal edit script between two
+ * sequences to identify the regions within two files where they differ.
+ * The core algorithm used in this match finder is described in:
+ *
+ * "An O(ND) Difference Algorithm and its Variations", Eugene W. Myers,
+ * Algorithmica Vol. 1, 1986, pp. 251-266,
+ * .
+ *
+ * Additional algorithmic heuristics for speed improvement have also been included.
+ * These we inspired from implementations of various regular and binary diffing
+ * algorithms such as GNU diff, bsdiff, and Xdelta.
+ *
+ * Note: after some experimentation, this approach proved to not provide enough
+ * utility to justify the additional CPU used in finding matches. The one area
+ * where this approach consistenly outperforms Zstandard even on level 19 is
+ * when compressing small files (<10 KB) using a equally small dictionary that
+ * is very similar to the source file. For the use case that this was intended,
+ * (large similar files) this approach by itself took 5-10X longer than zstd-19 and
+ * generally resulted in 2-3X larger files. The core advantage that zstd-19 has
+ * over this appraoch for match finding is the overlapping matches. This approach
+ * cannot find any.
+ *
+ * I'm leaving this in the contrib section in case this ever becomes interesting
+ * to explore again.
+ * */
+```
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/match_finders/zstd_edist.c b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/match_finders/zstd_edist.c
new file mode 100644
index 0000000..aab545f
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/match_finders/zstd_edist.c
@@ -0,0 +1,558 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/*-*************************************
+* Dependencies
+***************************************/
+
+/* Currently relies on qsort when combining contiguous matches. This can probabily
+ * be avoided but would require changes to the algorithm. The qsort is far from
+ * the bottleneck in this algorithm even for medium sized files so it's probably
+ * not worth trying to address */
+#include
+#include
+
+#include "zstd_edist.h"
+#include "mem.h"
+
+/*-*************************************
+* Constants
+***************************************/
+
+/* Just a sential for the entires of the diagnomal matrix */
+#define ZSTD_EDIST_DIAG_MAX (S32)(1 << 30)
+
+/* How large should a snake be to be considered a 'big' snake.
+ * For an explanation of what a 'snake' is with respect to the
+ * edit distance matrix, see the linked paper in zstd_edist.h */
+#define ZSTD_EDIST_SNAKE_THRESH 20
+
+/* After how many iterations should we start to use the heuristic
+ * based on 'big' snakes */
+#define ZSTD_EDIST_SNAKE_ITER_THRESH 200
+
+/* After how many iterations should be just give up and take
+ * the best availabe edit script for this round */
+#define ZSTD_EDIST_EXPENSIVE_THRESH 1024
+
+/*-*************************************
+* Structures
+***************************************/
+
+typedef struct {
+ U32 dictIdx;
+ U32 srcIdx;
+ U32 matchLength;
+} ZSTD_eDist_match;
+
+typedef struct {
+ const BYTE* dict;
+ const BYTE* src;
+ size_t dictSize;
+ size_t srcSize;
+ S32* forwardDiag; /* Entires of the forward diagonal stored here */
+ S32* backwardDiag; /* Entires of the backward diagonal stored here.
+ * Note: this buffer and the 'forwardDiag' buffer
+ * are contiguous. See the ZSTD_eDist_genSequences */
+ ZSTD_eDist_match* matches; /* Accumulate matches of length 1 in this buffer.
+ * In a subsequence post-processing step, we combine
+ * contiguous matches. */
+ U32 nbMatches;
+} ZSTD_eDist_state;
+
+typedef struct {
+ S32 dictMid; /* The mid diagonal for the dictionary */
+ S32 srcMid; /* The mid diagonal for the source */
+ int lowUseHeuristics; /* Should we use heuristics for the low part */
+ int highUseHeuristics; /* Should we use heuristics for the high part */
+} ZSTD_eDist_partition;
+
+/*-*************************************
+* Internal
+***************************************/
+
+static void ZSTD_eDist_diag(ZSTD_eDist_state* state,
+ ZSTD_eDist_partition* partition,
+ S32 dictLow, S32 dictHigh, S32 srcLow,
+ S32 srcHigh, int useHeuristics)
+{
+ S32* const forwardDiag = state->forwardDiag;
+ S32* const backwardDiag = state->backwardDiag;
+ const BYTE* const dict = state->dict;
+ const BYTE* const src = state->src;
+
+ S32 const diagMin = dictLow - srcHigh;
+ S32 const diagMax = dictHigh - srcLow;
+ S32 const forwardMid = dictLow - srcLow;
+ S32 const backwardMid = dictHigh - srcHigh;
+
+ S32 forwardMin = forwardMid;
+ S32 forwardMax = forwardMid;
+ S32 backwardMin = backwardMid;
+ S32 backwardMax = backwardMid;
+ int odd = (forwardMid - backwardMid) & 1;
+ U32 iterations;
+
+ forwardDiag[forwardMid] = dictLow;
+ backwardDiag[backwardMid] = dictHigh;
+
+ /* Main loop for updating diag entries. Unless useHeuristics is
+ * set to false, this loop will run until it finds the minimal
+ * edit script */
+ for (iterations = 1;;iterations++) {
+ S32 diag;
+ int bigSnake = 0;
+
+ if (forwardMin > diagMin) {
+ forwardMin--;
+ forwardDiag[forwardMin - 1] = -1;
+ } else {
+ forwardMin++;
+ }
+
+ if (forwardMax < diagMax) {
+ forwardMax++;
+ forwardDiag[forwardMax + 1] = -1;
+ } else {
+ forwardMax--;
+ }
+
+ for (diag = forwardMax; diag >= forwardMin; diag -= 2) {
+ S32 dictIdx;
+ S32 srcIdx;
+ S32 low = forwardDiag[diag - 1];
+ S32 high = forwardDiag[diag + 1];
+ S32 dictIdx0 = low < high ? high : low + 1;
+
+ for (dictIdx = dictIdx0, srcIdx = dictIdx0 - diag;
+ dictIdx < dictHigh && srcIdx < srcHigh && dict[dictIdx] == src[srcIdx];
+ dictIdx++, srcIdx++) continue;
+
+ if (dictIdx - dictIdx0 > ZSTD_EDIST_SNAKE_THRESH)
+ bigSnake = 1;
+
+ forwardDiag[diag] = dictIdx;
+
+ if (odd && backwardMin <= diag && diag <= backwardMax && backwardDiag[diag] <= dictIdx) {
+ partition->dictMid = dictIdx;
+ partition->srcMid = srcIdx;
+ partition->lowUseHeuristics = 0;
+ partition->highUseHeuristics = 0;
+ return;
+ }
+ }
+
+ if (backwardMin > diagMin) {
+ backwardMin--;
+ backwardDiag[backwardMin - 1] = ZSTD_EDIST_DIAG_MAX;
+ } else {
+ backwardMin++;
+ }
+
+ if (backwardMax < diagMax) {
+ backwardMax++;
+ backwardDiag[backwardMax + 1] = ZSTD_EDIST_DIAG_MAX;
+ } else {
+ backwardMax--;
+ }
+
+
+ for (diag = backwardMax; diag >= backwardMin; diag -= 2) {
+ S32 dictIdx;
+ S32 srcIdx;
+ S32 low = backwardDiag[diag - 1];
+ S32 high = backwardDiag[diag + 1];
+ S32 dictIdx0 = low < high ? low : high - 1;
+
+ for (dictIdx = dictIdx0, srcIdx = dictIdx0 - diag;
+ dictLow < dictIdx && srcLow < srcIdx && dict[dictIdx - 1] == src[srcIdx - 1];
+ dictIdx--, srcIdx--) continue;
+
+ if (dictIdx0 - dictIdx > ZSTD_EDIST_SNAKE_THRESH)
+ bigSnake = 1;
+
+ backwardDiag[diag] = dictIdx;
+
+ if (!odd && forwardMin <= diag && diag <= forwardMax && dictIdx <= forwardDiag[diag]) {
+ partition->dictMid = dictIdx;
+ partition->srcMid = srcIdx;
+ partition->lowUseHeuristics = 0;
+ partition->highUseHeuristics = 0;
+ return;
+ }
+ }
+
+ if (!useHeuristics)
+ continue;
+
+ /* Everything under this point is a heuritic. Using these will
+ * substantially speed up the match finding. In some cases, taking
+ * the total match finding time from several minutes to seconds.
+ * Of course, the caveat is that the edit script found may no longer
+ * be optimal */
+
+ /* Big snake heuristic */
+ if (iterations > ZSTD_EDIST_SNAKE_ITER_THRESH && bigSnake) {
+ {
+ S32 best = 0;
+
+ for (diag = forwardMax; diag >= forwardMin; diag -= 2) {
+ S32 diagDiag = diag - forwardMid;
+ S32 dictIdx = forwardDiag[diag];
+ S32 srcIdx = dictIdx - diag;
+ S32 v = (dictIdx - dictLow) * 2 - diagDiag;
+
+ if (v > 12 * (iterations + (diagDiag < 0 ? -diagDiag : diagDiag))) {
+ if (v > best
+ && dictLow + ZSTD_EDIST_SNAKE_THRESH <= dictIdx && dictIdx <= dictHigh
+ && srcLow + ZSTD_EDIST_SNAKE_THRESH <= srcIdx && srcIdx <= srcHigh) {
+ S32 k;
+ for (k = 1; dict[dictIdx - k] == src[srcIdx - k]; k++) {
+ if (k == ZSTD_EDIST_SNAKE_THRESH) {
+ best = v;
+ partition->dictMid = dictIdx;
+ partition->srcMid = srcIdx;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (best > 0) {
+ partition->lowUseHeuristics = 0;
+ partition->highUseHeuristics = 1;
+ return;
+ }
+ }
+
+ {
+ S32 best = 0;
+
+ for (diag = backwardMax; diag >= backwardMin; diag -= 2) {
+ S32 diagDiag = diag - backwardMid;
+ S32 dictIdx = backwardDiag[diag];
+ S32 srcIdx = dictIdx - diag;
+ S32 v = (dictHigh - dictIdx) * 2 + diagDiag;
+
+ if (v > 12 * (iterations + (diagDiag < 0 ? -diagDiag : diagDiag))) {
+ if (v > best
+ && dictLow < dictIdx && dictIdx <= dictHigh - ZSTD_EDIST_SNAKE_THRESH
+ && srcLow < srcIdx && srcIdx <= srcHigh - ZSTD_EDIST_SNAKE_THRESH) {
+ int k;
+ for (k = 0; dict[dictIdx + k] == src[srcIdx + k]; k++) {
+ if (k == ZSTD_EDIST_SNAKE_THRESH - 1) {
+ best = v;
+ partition->dictMid = dictIdx;
+ partition->srcMid = srcIdx;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (best > 0) {
+ partition->lowUseHeuristics = 1;
+ partition->highUseHeuristics = 0;
+ return;
+ }
+ }
+ }
+
+ /* More general 'too expensive' heuristic */
+ if (iterations >= ZSTD_EDIST_EXPENSIVE_THRESH) {
+ S32 forwardDictSrcBest;
+ S32 forwardDictBest = 0;
+ S32 backwardDictSrcBest;
+ S32 backwardDictBest = 0;
+
+ forwardDictSrcBest = -1;
+ for (diag = forwardMax; diag >= forwardMin; diag -= 2) {
+ S32 dictIdx = MIN(forwardDiag[diag], dictHigh);
+ S32 srcIdx = dictIdx - diag;
+
+ if (srcHigh < srcIdx) {
+ dictIdx = srcHigh + diag;
+ srcIdx = srcHigh;
+ }
+
+ if (forwardDictSrcBest < dictIdx + srcIdx) {
+ forwardDictSrcBest = dictIdx + srcIdx;
+ forwardDictBest = dictIdx;
+ }
+ }
+
+ backwardDictSrcBest = ZSTD_EDIST_DIAG_MAX;
+ for (diag = backwardMax; diag >= backwardMin; diag -= 2) {
+ S32 dictIdx = MAX(dictLow, backwardDiag[diag]);
+ S32 srcIdx = dictIdx - diag;
+
+ if (srcIdx < srcLow) {
+ dictIdx = srcLow + diag;
+ srcIdx = srcLow;
+ }
+
+ if (dictIdx + srcIdx < backwardDictSrcBest) {
+ backwardDictSrcBest = dictIdx + srcIdx;
+ backwardDictBest = dictIdx;
+ }
+ }
+
+ if ((dictHigh + srcHigh) - backwardDictSrcBest < forwardDictSrcBest - (dictLow + srcLow)) {
+ partition->dictMid = forwardDictBest;
+ partition->srcMid = forwardDictSrcBest - forwardDictBest;
+ partition->lowUseHeuristics = 0;
+ partition->highUseHeuristics = 1;
+ } else {
+ partition->dictMid = backwardDictBest;
+ partition->srcMid = backwardDictSrcBest - backwardDictBest;
+ partition->lowUseHeuristics = 1;
+ partition->highUseHeuristics = 0;
+ }
+ return;
+ }
+ }
+}
+
+static void ZSTD_eDist_insertMatch(ZSTD_eDist_state* state,
+ S32 const dictIdx, S32 const srcIdx)
+{
+ state->matches[state->nbMatches].dictIdx = dictIdx;
+ state->matches[state->nbMatches].srcIdx = srcIdx;
+ state->matches[state->nbMatches].matchLength = 1;
+ state->nbMatches++;
+}
+
+static int ZSTD_eDist_compare(ZSTD_eDist_state* state,
+ S32 dictLow, S32 dictHigh, S32 srcLow,
+ S32 srcHigh, int useHeuristics)
+{
+ const BYTE* const dict = state->dict;
+ const BYTE* const src = state->src;
+
+ /* Found matches while traversing from the low end */
+ while (dictLow < dictHigh && srcLow < srcHigh && dict[dictLow] == src[srcLow]) {
+ ZSTD_eDist_insertMatch(state, dictLow, srcLow);
+ dictLow++;
+ srcLow++;
+ }
+
+ /* Found matches while traversing from the high end */
+ while (dictLow < dictHigh && srcLow < srcHigh && dict[dictHigh - 1] == src[srcHigh - 1]) {
+ ZSTD_eDist_insertMatch(state, dictHigh - 1, srcHigh - 1);
+ dictHigh--;
+ srcHigh--;
+ }
+
+ /* If the low and high end end up touching. If we wanted to make
+ * note of the differences like most diffing algorithms do, we would
+ * do so here. In our case, we're only concerned with matches
+ * Note: if you wanted to find the edit distance of the algorithm,
+ * you could just accumulate the cost for an insertion/deletion
+ * below. */
+ if (dictLow == dictHigh) {
+ while (srcLow < srcHigh) {
+ /* Reaching this point means inserting src[srcLow] into
+ * the current position of dict */
+ srcLow++;
+ }
+ } else if (srcLow == srcHigh) {
+ while (dictLow < dictHigh) {
+ /* Reaching this point means deleteing dict[dictLow] from
+ * the current positino of dict */
+ dictLow++;
+ }
+ } else {
+ ZSTD_eDist_partition partition;
+ partition.dictMid = 0;
+ partition.srcMid = 0;
+ ZSTD_eDist_diag(state, &partition, dictLow, dictHigh,
+ srcLow, srcHigh, useHeuristics);
+ if (ZSTD_eDist_compare(state, dictLow, partition.dictMid,
+ srcLow, partition.srcMid, partition.lowUseHeuristics))
+ return 1;
+ if (ZSTD_eDist_compare(state, partition.dictMid, dictHigh,
+ partition.srcMid, srcHigh, partition.highUseHeuristics))
+ return 1;
+ }
+
+ return 0;
+}
+
+static int ZSTD_eDist_matchComp(const void* p, const void* q)
+{
+ S32 const l = ((ZSTD_eDist_match*)p)->srcIdx;
+ S32 const r = ((ZSTD_eDist_match*)q)->srcIdx;
+ return (l - r);
+}
+
+/* The matches from the approach above will all be of the form
+ * (dictIdx, srcIdx, 1). this method combines contiguous matches
+ * of length MINMATCH or greater. Matches less than MINMATCH
+ * are discarded */
+static void ZSTD_eDist_combineMatches(ZSTD_eDist_state* state)
+{
+ /* Create a new buffer to put the combined matches into
+ * and memcpy to state->matches after */
+ ZSTD_eDist_match* combinedMatches =
+ ZSTD_malloc(state->nbMatches * sizeof(ZSTD_eDist_match),
+ ZSTD_defaultCMem);
+
+ U32 nbCombinedMatches = 1;
+ size_t i;
+
+ /* Make sure that the srcIdx and dictIdx are in sorted order.
+ * The combination step won't work otherwise */
+ qsort(state->matches, state->nbMatches, sizeof(ZSTD_eDist_match), ZSTD_eDist_matchComp);
+
+ memcpy(combinedMatches, state->matches, sizeof(ZSTD_eDist_match));
+ for (i = 1; i < state->nbMatches; i++) {
+ ZSTD_eDist_match const match = state->matches[i];
+ ZSTD_eDist_match const combinedMatch =
+ combinedMatches[nbCombinedMatches - 1];
+ if (combinedMatch.srcIdx + combinedMatch.matchLength == match.srcIdx &&
+ combinedMatch.dictIdx + combinedMatch.matchLength == match.dictIdx) {
+ combinedMatches[nbCombinedMatches - 1].matchLength++;
+ } else {
+ /* Discard matches that are less than MINMATCH */
+ if (combinedMatches[nbCombinedMatches - 1].matchLength < MINMATCH) {
+ nbCombinedMatches--;
+ }
+
+ memcpy(combinedMatches + nbCombinedMatches,
+ state->matches + i, sizeof(ZSTD_eDist_match));
+ nbCombinedMatches++;
+ }
+ }
+ memcpy(state->matches, combinedMatches, nbCombinedMatches * sizeof(ZSTD_eDist_match));
+ state->nbMatches = nbCombinedMatches;
+ ZSTD_free(combinedMatches, ZSTD_defaultCMem);
+}
+
+static size_t ZSTD_eDist_convertMatchesToSequences(ZSTD_Sequence* sequences,
+ ZSTD_eDist_state* state)
+{
+ const ZSTD_eDist_match* matches = state->matches;
+ size_t const nbMatches = state->nbMatches;
+ size_t const dictSize = state->dictSize;
+ size_t nbSequences = 0;
+ size_t i;
+ for (i = 0; i < nbMatches; i++) {
+ ZSTD_eDist_match const match = matches[i];
+ U32 const litLength = !i ? match.srcIdx :
+ match.srcIdx - (matches[i - 1].srcIdx + matches[i - 1].matchLength);
+ U32 const offset = (match.srcIdx + dictSize) - match.dictIdx;
+ U32 const matchLength = match.matchLength;
+ sequences[nbSequences].offset = offset;
+ sequences[nbSequences].litLength = litLength;
+ sequences[nbSequences].matchLength = matchLength;
+ nbSequences++;
+ }
+ return nbSequences;
+}
+
+/*-*************************************
+* Interal utils
+***************************************/
+
+static size_t ZSTD_eDist_hamingDist(const BYTE* const a,
+ const BYTE* const b, size_t n)
+{
+ size_t i;
+ size_t dist = 0;
+ for (i = 0; i < n; i++)
+ dist += a[i] != b[i];
+ return dist;
+}
+
+/* This is a pretty naive recursive implementation that should only
+ * be used for quick tests obviously. Don't try and run this on a
+ * GB file or something. There are faster implementations. Use those
+ * if you need to run it for large files. */
+static size_t ZSTD_eDist_levenshteinDist(const BYTE* const s,
+ size_t const sn, const BYTE* const t,
+ size_t const tn)
+{
+ size_t a, b, c;
+
+ if (!sn)
+ return tn;
+ if (!tn)
+ return sn;
+
+ if (s[sn - 1] == t[tn - 1])
+ return ZSTD_eDist_levenshteinDist(
+ s, sn - 1, t, tn - 1);
+
+ a = ZSTD_eDist_levenshteinDist(s, sn - 1, t, tn - 1);
+ b = ZSTD_eDist_levenshteinDist(s, sn, t, tn - 1);
+ c = ZSTD_eDist_levenshteinDist(s, sn - 1, t, tn);
+
+ if (a > b)
+ a = b;
+ if (a > c)
+ a = c;
+
+ return a + 1;
+}
+
+static void ZSTD_eDist_validateMatches(ZSTD_eDist_match* matches,
+ size_t const nbMatches, const BYTE* const dict,
+ size_t const dictSize, const BYTE* const src,
+ size_t const srcSize)
+{
+ size_t i;
+ for (i = 0; i < nbMatches; i++) {
+ ZSTD_eDist_match match = matches[i];
+ U32 const dictIdx = match.dictIdx;
+ U32 const srcIdx = match.srcIdx;
+ U32 const matchLength = match.matchLength;
+
+ assert(dictIdx + matchLength < dictSize);
+ assert(srcIdx + matchLength < srcSize);
+ assert(!memcmp(dict + dictIdx, src + srcIdx, matchLength));
+ }
+}
+
+/*-*************************************
+* API
+***************************************/
+
+size_t ZSTD_eDist_genSequences(ZSTD_Sequence* sequences,
+ const void* dict, size_t dictSize,
+ const void* src, size_t srcSize,
+ int useHeuristics)
+{
+ size_t const nbDiags = dictSize + srcSize + 3;
+ S32* buffer = ZSTD_malloc(nbDiags * 2 * sizeof(S32), ZSTD_defaultCMem);
+ ZSTD_eDist_state state;
+ size_t nbSequences = 0;
+
+ state.dict = (const BYTE*)dict;
+ state.src = (const BYTE*)src;
+ state.dictSize = dictSize;
+ state.srcSize = srcSize;
+ state.forwardDiag = buffer;
+ state.backwardDiag = buffer + nbDiags;
+ state.forwardDiag += srcSize + 1;
+ state.backwardDiag += srcSize + 1;
+ state.matches = ZSTD_malloc(srcSize * sizeof(ZSTD_eDist_match), ZSTD_defaultCMem);
+ state.nbMatches = 0;
+
+ ZSTD_eDist_compare(&state, 0, dictSize, 0, srcSize, 1);
+ ZSTD_eDist_combineMatches(&state);
+ nbSequences = ZSTD_eDist_convertMatchesToSequences(sequences, &state);
+
+ ZSTD_free(buffer, ZSTD_defaultCMem);
+ ZSTD_free(state.matches, ZSTD_defaultCMem);
+
+ return nbSequences;
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/match_finders/zstd_edist.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/match_finders/zstd_edist.h
new file mode 100644
index 0000000..c775a49
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/match_finders/zstd_edist.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This match finder leverages techniques used in file comparison algorithms
+ * to find matches between a dictionary and a source file.
+ *
+ * The original motivation for studying this approach was to try and optimize
+ * Zstandard for the use case of patching: the most common scenario being
+ * updating an existing software package with the next version. When patching,
+ * the difference between the old version of the package and the new version
+ * is generally tiny (most of the new file will be identical to
+ * the old one). In more technical terms, the edit distance (the minimal number
+ * of changes required to take one sequence of bytes to another) between the
+ * files would be small relative to the size of the file.
+ *
+ * Various 'diffing' algorithms utilize this notion of edit distance and
+ * the corrensponding concept of a minimal edit script between two
+ * sequences to identify the regions within two files where they differ.
+ * The core algorithm used in this match finder is described in:
+ *
+ * "An O(ND) Difference Algorithm and its Variations", Eugene W. Myers,
+ * Algorithmica Vol. 1, 1986, pp. 251-266,
+ * .
+ *
+ * Additional algorithmic heuristics for speed improvement have also been included.
+ * These we inspired from implementations of various regular and binary diffing
+ * algorithms such as GNU diff, bsdiff, and Xdelta.
+ *
+ * Note: after some experimentation, this approach proved to not provide enough
+ * utility to justify the additional CPU used in finding matches. The one area
+ * where this approach consistenly outperforms Zstandard even on level 19 is
+ * when compressing small files (<10 KB) using a equally small dictionary that
+ * is very similar to the source file. For the use case that this was intended,
+ * (large similar files) this approach by itself took 5-10X longer than zstd-19 and
+ * generally resulted in 2-3X larger files. The core advantage that zstd-19 has
+ * over this appraoch for match finding is the overlapping matches. This approach
+ * cannot find any.
+ *
+ * I'm leaving this in the contrib section in case this ever becomes interesting
+ * to explore again.
+ * */
+
+#ifndef ZSTD_EDIST_H
+#define ZSTD_EDIST_H
+
+/*-*************************************
+* Dependencies
+***************************************/
+
+#include
+#include "zstd_internal.h" /* ZSTD_Sequence */
+
+/*! ZSTD_eDist_genSequences() :
+ * Will populate the provided ZSTD_Sequence buffer with sequences
+ * based on the optimal or near-optimal (depending on 'useHeuristics')
+ * edit script between 'dict' and 'src.'
+ * @return : the number of sequences found */
+size_t ZSTD_eDist_genSequences(ZSTD_Sequence* sequences,
+ const void* dict, size_t dictSize,
+ const void* src, size_t srcSize,
+ int useHeuristics);
+
+#endif
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/premake/premake4.lua b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/premake/premake4.lua
new file mode 100644
index 0000000..6675e2e
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/premake/premake4.lua
@@ -0,0 +1,6 @@
+-- Include zstd.lua in your GENie or premake4 file, which exposes a project_zstd function
+dofile('zstd.lua')
+
+solution 'example'
+ configurations { 'Debug', 'Release' }
+ project_zstd('../../lib/')
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/premake/zstd.lua b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/premake/zstd.lua
new file mode 100644
index 0000000..df1ace3
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/premake/zstd.lua
@@ -0,0 +1,80 @@
+-- This GENie/premake file copies the behavior of the Makefile in the lib folder.
+-- Basic usage: project_zstd(ZSTD_DIR)
+
+function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy)
+ if compression == nil then compression = true end
+ if decompression == nil then decompression = true end
+ if deprecated == nil then deprecated = false end
+ if dictbuilder == nil then dictbuilder = false end
+
+ if legacy == nil then legacy = 0 end
+
+ if not compression then
+ dictbuilder = false
+ deprecated = false
+ end
+
+ if not decompression then
+ legacy = 0
+ deprecated = false
+ end
+
+ project 'zstd'
+ kind 'StaticLib'
+ language 'C'
+
+ files {
+ dir .. 'zstd.h',
+ dir .. 'common/**.c',
+ dir .. 'common/**.h'
+ }
+
+ if compression then
+ files {
+ dir .. 'compress/**.c',
+ dir .. 'compress/**.h'
+ }
+ end
+
+ if decompression then
+ files {
+ dir .. 'decompress/**.c',
+ dir .. 'decompress/**.h'
+ }
+ end
+
+ if dictbuilder then
+ files {
+ dir .. 'dictBuilder/**.c',
+ dir .. 'dictBuilder/**.h'
+ }
+ end
+
+ if deprecated then
+ files {
+ dir .. 'deprecated/**.c',
+ dir .. 'deprecated/**.h'
+ }
+ end
+
+ if legacy ~= 0 then
+ if legacy >= 8 then
+ files {
+ dir .. 'legacy/zstd_v0' .. (legacy - 7) .. '.*'
+ }
+ end
+ includedirs {
+ dir .. 'legacy'
+ }
+ end
+
+ includedirs {
+ dir,
+ dir .. 'common'
+ }
+
+ defines {
+ 'XXH_NAMESPACE=ZSTD_',
+ 'ZSTD_LEGACY_SUPPORT=' .. legacy
+ }
+end
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/.gitignore b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/.gitignore
new file mode 100644
index 0000000..84e68fb
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/.gitignore
@@ -0,0 +1,2 @@
+# compilation result
+pzstd
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/BUCK b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/BUCK
new file mode 100644
index 0000000..d04eeed
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/BUCK
@@ -0,0 +1,72 @@
+cxx_library(
+ name='libpzstd',
+ visibility=['PUBLIC'],
+ header_namespace='',
+ exported_headers=[
+ 'ErrorHolder.h',
+ 'Logging.h',
+ 'Pzstd.h',
+ ],
+ headers=[
+ 'SkippableFrame.h',
+ ],
+ srcs=[
+ 'Pzstd.cpp',
+ 'SkippableFrame.cpp',
+ ],
+ deps=[
+ ':options',
+ '//contrib/pzstd/utils:utils',
+ '//lib:mem',
+ '//lib:zstd',
+ ],
+)
+
+cxx_library(
+ name='options',
+ visibility=['PUBLIC'],
+ header_namespace='',
+ exported_headers=['Options.h'],
+ srcs=['Options.cpp'],
+ deps=[
+ '//contrib/pzstd/utils:scope_guard',
+ '//lib:zstd',
+ '//programs:util',
+ ],
+)
+
+cxx_binary(
+ name='pzstd',
+ visibility=['PUBLIC'],
+ srcs=['main.cpp'],
+ deps=[
+ ':libpzstd',
+ ':options',
+ ],
+)
+
+# Must run "make googletest" first
+cxx_library(
+ name='gtest',
+ srcs=glob([
+ 'googletest/googletest/src/gtest-all.cc',
+ 'googletest/googlemock/src/gmock-all.cc',
+ 'googletest/googlemock/src/gmock_main.cc',
+ ]),
+ header_namespace='',
+ exported_headers=subdir_glob([
+ ('googletest/googletest/include', '**/*.h'),
+ ('googletest/googlemock/include', '**/*.h'),
+ ]),
+ headers=subdir_glob([
+ ('googletest/googletest', 'src/*.cc'),
+ ('googletest/googletest', 'src/*.h'),
+ ('googletest/googlemock', 'src/*.cc'),
+ ('googletest/googlemock', 'src/*.h'),
+ ]),
+ platform_linker_flags=[
+ ('android', []),
+ ('', ['-lpthread']),
+ ],
+ visibility=['PUBLIC'],
+)
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/ErrorHolder.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/ErrorHolder.h
new file mode 100644
index 0000000..829651c
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/ErrorHolder.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#include
+#include
+#include
+#include
+
+namespace pzstd {
+
+// Coordinates graceful shutdown of the pzstd pipeline
+class ErrorHolder {
+ std::atomic error_;
+ std::string message_;
+
+ public:
+ ErrorHolder() : error_(false) {}
+
+ bool hasError() noexcept {
+ return error_.load();
+ }
+
+ void setError(std::string message) noexcept {
+ // Given multiple possibly concurrent calls, exactly one will ever succeed.
+ bool expected = false;
+ if (error_.compare_exchange_strong(expected, true)) {
+ message_ = std::move(message);
+ }
+ }
+
+ bool check(bool predicate, std::string message) noexcept {
+ if (!predicate) {
+ setError(std::move(message));
+ }
+ return !hasError();
+ }
+
+ std::string getError() noexcept {
+ error_.store(false);
+ return std::move(message_);
+ }
+
+ ~ErrorHolder() {
+ assert(!hasError());
+ }
+};
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Logging.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Logging.h
new file mode 100644
index 0000000..beb160b
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Logging.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#include
+#include
+
+namespace pzstd {
+
+constexpr int kLogError = 1;
+constexpr int kLogInfo = 2;
+constexpr int kLogDebug = 3;
+constexpr int kLogVerbose = 4;
+
+class Logger {
+ std::mutex mutex_;
+ FILE* out_;
+ const int level_;
+
+ using Clock = std::chrono::system_clock;
+ Clock::time_point lastUpdate_;
+ std::chrono::milliseconds refreshRate_;
+
+ public:
+ explicit Logger(int level, FILE* out = stderr)
+ : out_(out), level_(level), lastUpdate_(Clock::now()),
+ refreshRate_(150) {}
+
+
+ bool logsAt(int level) {
+ return level <= level_;
+ }
+
+ template
+ void operator()(int level, const char *fmt, Args... args) {
+ if (level > level_) {
+ return;
+ }
+ std::lock_guard lock(mutex_);
+ std::fprintf(out_, fmt, args...);
+ }
+
+ template
+ void update(int level, const char *fmt, Args... args) {
+ if (level > level_) {
+ return;
+ }
+ std::lock_guard lock(mutex_);
+ auto now = Clock::now();
+ if (now - lastUpdate_ > refreshRate_) {
+ lastUpdate_ = now;
+ std::fprintf(out_, "\r");
+ std::fprintf(out_, fmt, args...);
+ }
+ }
+
+ void clear(int level) {
+ if (level > level_) {
+ return;
+ }
+ std::lock_guard lock(mutex_);
+ std::fprintf(out_, "\r%79s\r", "");
+ }
+};
+
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Makefile b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Makefile
new file mode 100644
index 0000000..25265e7
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Makefile
@@ -0,0 +1,274 @@
+# ################################################################
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+# Standard variables for installation
+DESTDIR ?=
+PREFIX ?= /usr/local
+BINDIR := $(DESTDIR)$(PREFIX)/bin
+
+ZSTDDIR = ../../lib
+PROGDIR = ../../programs
+
+# External program to use to run tests, e.g. qemu or valgrind
+TESTPROG ?=
+# Flags to pass to the tests
+TESTFLAGS ?=
+
+# We use gcc/clang to generate the header dependencies of files
+DEPFLAGS = -MMD -MP -MF $*.Td
+POSTCOMPILE = mv -f $*.Td $*.d
+
+# CFLAGS, CXXFLAGS, CPPFLAGS, and LDFLAGS are for the users to override
+CFLAGS ?= -O3 -Wall -Wextra
+CXXFLAGS ?= -O3 -Wall -Wextra -pedantic
+CPPFLAGS ?=
+LDFLAGS ?=
+
+# PZstd uses legacy APIs
+CFLAGS += -Wno-deprecated-declarations
+
+# Include flags
+PZSTD_INC = -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
+GTEST_INC = -isystem googletest/googletest/include
+
+PZSTD_CPPFLAGS = $(PZSTD_INC)
+PZSTD_CCXXFLAGS =
+PZSTD_CFLAGS = $(PZSTD_CCXXFLAGS)
+PZSTD_CXXFLAGS = $(PZSTD_CCXXFLAGS) -std=c++11
+PZSTD_LDFLAGS =
+EXTRA_FLAGS =
+ALL_CFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CFLAGS) $(PZSTD_CFLAGS)
+ALL_CXXFLAGS = $(EXTRA_FLAGS) $(CPPFLAGS) $(PZSTD_CPPFLAGS) $(CXXFLAGS) $(PZSTD_CXXFLAGS)
+ALL_LDFLAGS = $(EXTRA_FLAGS) $(CXXFLAGS) $(LDFLAGS) $(PZSTD_LDFLAGS)
+
+
+# gtest libraries need to go before "-lpthread" because they depend on it.
+GTEST_LIB = -L googletest/build/googlemock/gtest
+LIBS =
+
+# Compilation commands
+LD_COMMAND = $(CXX) $^ $(ALL_LDFLAGS) $(LIBS) -pthread -o $@
+CC_COMMAND = $(CC) $(DEPFLAGS) $(ALL_CFLAGS) -c $< -o $@
+CXX_COMMAND = $(CXX) $(DEPFLAGS) $(ALL_CXXFLAGS) -c $< -o $@
+
+# Get a list of all zstd files so we rebuild the static library when we need to
+ZSTDCOMMON_FILES := $(wildcard $(ZSTDDIR)/common/*.c) \
+ $(wildcard $(ZSTDDIR)/common/*.h)
+ZSTDCOMP_FILES := $(wildcard $(ZSTDDIR)/compress/*.c) \
+ $(wildcard $(ZSTDDIR)/compress/*.h)
+ZSTDDECOMP_FILES := $(wildcard $(ZSTDDIR)/decompress/*.c) \
+ $(wildcard $(ZSTDDIR)/decompress/*.h)
+ZSTDPROG_FILES := $(wildcard $(PROGDIR)/*.c) \
+ $(wildcard $(PROGDIR)/*.h)
+ZSTD_FILES := $(wildcard $(ZSTDDIR)/*.h) \
+ $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
+ $(ZSTDPROG_FILES)
+
+# List all the pzstd source files so we can determine their dependencies
+PZSTD_SRCS := $(wildcard *.cpp)
+PZSTD_TESTS := $(wildcard test/*.cpp)
+UTILS_TESTS := $(wildcard utils/test/*.cpp)
+ALL_SRCS := $(PZSTD_SRCS) $(PZSTD_TESTS) $(UTILS_TESTS)
+
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
+endif
+
+# Standard targets
+.PHONY: default
+default: all
+
+.PHONY: test-pzstd
+test-pzstd: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd: clean googletest pzstd tests check
+
+.PHONY: test-pzstd32
+test-pzstd32: clean googletest32 all32 check
+
+.PHONY: test-pzstd-tsan
+test-pzstd-tsan: LDFLAGS=-fuse-ld=gold
+test-pzstd-tsan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd-tsan: clean googletest tsan check
+
+.PHONY: test-pzstd-asan
+test-pzstd-asan: LDFLAGS=-fuse-ld=gold
+test-pzstd-asan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
+test-pzstd-asan: clean asan check
+
+.PHONY: check
+check:
+ $(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/RangeTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/ResourcePoolTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/ScopeGuardTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/ThreadPoolTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./utils/test/WorkQueueTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./test/OptionsTest$(EXT) $(TESTFLAGS)
+ $(TESTPROG) ./test/PzstdTest$(EXT) $(TESTFLAGS)
+
+.PHONY: install
+install: PZSTD_CPPFLAGS += -DNDEBUG
+install: pzstd$(EXT)
+ install -d -m 755 $(BINDIR)/
+ install -m 755 pzstd$(EXT) $(BINDIR)/pzstd$(EXT)
+
+.PHONY: uninstall
+uninstall:
+ $(RM) $(BINDIR)/pzstd$(EXT)
+
+# Targets for many different builds
+.PHONY: all
+all: PZSTD_CPPFLAGS += -DNDEBUG
+all: pzstd$(EXT)
+
+.PHONY: debug
+debug: EXTRA_FLAGS += -g
+debug: pzstd$(EXT) tests roundtrip
+
+.PHONY: tsan
+tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC
+tsan: PZSTD_LDFLAGS += -fsanitize=thread
+tsan: debug
+
+.PHONY: asan
+asan: EXTRA_FLAGS += -fsanitize=address
+asan: debug
+
+.PHONY: ubsan
+ubsan: EXTRA_FLAGS += -fsanitize=undefined
+ubsan: debug
+
+.PHONY: all32
+all32: EXTRA_FLAGS += -m32
+all32: all tests roundtrip
+
+.PHONY: debug32
+debug32: EXTRA_FLAGS += -m32
+debug32: debug
+
+.PHONY: asan32
+asan32: EXTRA_FLAGS += -m32
+asan32: asan
+
+.PHONY: tsan32
+tsan32: EXTRA_FLAGS += -m32
+tsan32: tsan
+
+.PHONY: ubsan32
+ubsan32: EXTRA_FLAGS += -m32
+ubsan32: ubsan
+
+# Run long round trip tests
+.PHONY: roundtripcheck
+roundtripcheck: roundtrip check
+ $(TESTPROG) ./test/RoundTripTest$(EXT) $(TESTFLAGS)
+
+# Build the main binary
+pzstd$(EXT): main.o $(PROGDIR)/util.o Options.o Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
+ $(LD_COMMAND)
+
+# Target that depends on all the tests
+.PHONY: tests
+tests: EXTRA_FLAGS += -Wno-deprecated-declarations
+tests: $(patsubst %,%$(EXT),$(basename $(PZSTD_TESTS) $(UTILS_TESTS)))
+
+# Build the round trip tests
+.PHONY: roundtrip
+roundtrip: EXTRA_FLAGS += -Wno-deprecated-declarations
+roundtrip: test/RoundTripTest$(EXT)
+
+# Use the static library that zstd builds for simplicity and
+# so we get the compiler options correct
+$(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
+ CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a
+
+# Rules to build the tests
+test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o \
+ $(PROGDIR)/util.o Options.o \
+ Pzstd.o SkippableFrame.o $(ZSTDDIR)/libzstd.a
+ $(LD_COMMAND)
+
+test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
+test/%Test$(EXT): LIBS += -lgtest -lgtest_main
+test/%Test$(EXT): test/%Test.o $(PROGDIR)/datagen.o \
+ $(PROGDIR)/util.o Options.o Pzstd.o \
+ SkippableFrame.o $(ZSTDDIR)/libzstd.a
+ $(LD_COMMAND)
+
+utils/test/%Test$(EXT): PZSTD_LDFLAGS += $(GTEST_LIB)
+utils/test/%Test$(EXT): LIBS += -lgtest -lgtest_main
+utils/test/%Test$(EXT): utils/test/%Test.o
+ $(LD_COMMAND)
+
+
+GTEST_CMAKEFLAGS =
+
+# Install googletest
+.PHONY: googletest
+googletest: PZSTD_CCXXFLAGS += -fPIC
+googletest:
+ @$(RM) -rf googletest
+ @git clone https://github.com/google/googletest
+ @mkdir -p googletest/build
+ @cd googletest/build && cmake $(GTEST_CMAKEFLAGS) -DCMAKE_CXX_FLAGS="$(ALL_CXXFLAGS)" .. && $(MAKE)
+
+.PHONY: googletest32
+googletest32: PZSTD_CCXXFLAGS += -m32
+googletest32: googletest
+
+.PHONY: googletest-mingw64
+googletest-mingw64: GTEST_CMAKEFLAGS += -G "MSYS Makefiles"
+googletest-mingw64: googletest
+
+.PHONY: clean
+clean:
+ $(RM) -f *.o pzstd$(EXT) *.Td *.d
+ $(RM) -f test/*.o test/*Test$(EXT) test/*.Td test/*.d
+ $(RM) -f utils/test/*.o utils/test/*Test$(EXT) utils/test/*.Td utils/test/*.d
+ $(RM) -f $(PROGDIR)/*.o $(PROGDIR)/*.Td $(PROGDIR)/*.d
+ $(MAKE) -C $(ZSTDDIR) clean
+ @echo Cleaning completed
+
+
+# Cancel implicit rules
+%.o: %.c
+%.o: %.cpp
+
+# Object file rules
+%.o: %.c
+ $(CC_COMMAND)
+ $(POSTCOMPILE)
+
+$(PROGDIR)/%.o: $(PROGDIR)/%.c
+ $(CC_COMMAND)
+ $(POSTCOMPILE)
+
+%.o: %.cpp
+ $(CXX_COMMAND)
+ $(POSTCOMPILE)
+
+test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC)
+test/%.o: test/%.cpp
+ $(CXX_COMMAND)
+ $(POSTCOMPILE)
+
+utils/test/%.o: PZSTD_CPPFLAGS += $(GTEST_INC)
+utils/test/%.o: utils/test/%.cpp
+ $(CXX_COMMAND)
+ $(POSTCOMPILE)
+
+# Dependency file stuff
+.PRECIOUS: %.d test/%.d utils/test/%.d
+
+# Include rules that specify header file dependencies
+-include $(patsubst %,%.d,$(basename $(ALL_SRCS)))
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Options.cpp b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Options.cpp
new file mode 100644
index 0000000..3729222
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Options.cpp
@@ -0,0 +1,424 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#include "Options.h"
+#include "util.h"
+#include "utils/ScopeGuard.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+
+namespace pzstd {
+
+namespace {
+unsigned defaultNumThreads() {
+#ifdef PZSTD_NUM_THREADS
+ return PZSTD_NUM_THREADS;
+#else
+ return std::thread::hardware_concurrency();
+#endif
+}
+
+unsigned parseUnsigned(const char **arg) {
+ unsigned result = 0;
+ while (**arg >= '0' && **arg <= '9') {
+ result *= 10;
+ result += **arg - '0';
+ ++(*arg);
+ }
+ return result;
+}
+
+const char *getArgument(const char *options, const char **argv, int &i,
+ int argc) {
+ if (options[1] != 0) {
+ return options + 1;
+ }
+ ++i;
+ if (i == argc) {
+ std::fprintf(stderr, "Option -%c requires an argument, but none provided\n",
+ *options);
+ return nullptr;
+ }
+ return argv[i];
+}
+
+const std::string kZstdExtension = ".zst";
+constexpr char kStdIn[] = "-";
+constexpr char kStdOut[] = "-";
+constexpr unsigned kDefaultCompressionLevel = 3;
+constexpr unsigned kMaxNonUltraCompressionLevel = 19;
+
+#ifdef _WIN32
+const char nullOutput[] = "nul";
+#else
+const char nullOutput[] = "/dev/null";
+#endif
+
+void notSupported(const char *option) {
+ std::fprintf(stderr, "Operation not supported: %s\n", option);
+}
+
+void usage() {
+ std::fprintf(stderr, "Usage:\n");
+ std::fprintf(stderr, " pzstd [args] [FILE(s)]\n");
+ std::fprintf(stderr, "Parallel ZSTD options:\n");
+ std::fprintf(stderr, " -p, --processes # : number of threads to use for (de)compression (default:)\n");
+
+ std::fprintf(stderr, "ZSTD options:\n");
+ std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);
+ std::fprintf(stderr, " -d, --decompress : decompression\n");
+ std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n");
+ std::fprintf(stderr, " -f, --force : overwrite output without prompting, (de)compress links\n");
+ std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n");
+ std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n");
+ std::fprintf(stderr, " -h, --help : display help and exit\n");
+ std::fprintf(stderr, " -V, --version : display version number and exit\n");
+ std::fprintf(stderr, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n");
+ std::fprintf(stderr, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n");
+ std::fprintf(stderr, " -c, --stdout : force write to standard output, even if it is the console\n");
+#ifdef UTIL_HAS_CREATEFILELIST
+ std::fprintf(stderr, " -r : operate recursively on directories\n");
+#endif
+ std::fprintf(stderr, " --ultra : enable levels beyond %i, up to %i (requires more memory)\n", kMaxNonUltraCompressionLevel, ZSTD_maxCLevel());
+ std::fprintf(stderr, " -C, --check : integrity check (default)\n");
+ std::fprintf(stderr, " --no-check : no integrity check\n");
+ std::fprintf(stderr, " -t, --test : test compressed file integrity\n");
+ std::fprintf(stderr, " -- : all arguments after \"--\" are treated as files\n");
+}
+} // anonymous namespace
+
+Options::Options()
+ : numThreads(defaultNumThreads()), maxWindowLog(23),
+ compressionLevel(kDefaultCompressionLevel), decompress(false),
+ overwrite(false), keepSource(true), writeMode(WriteMode::Auto),
+ checksum(true), verbosity(2) {}
+
+Options::Status Options::parse(int argc, const char **argv) {
+ bool test = false;
+ bool recursive = false;
+ bool ultra = false;
+ bool forceStdout = false;
+ bool followLinks = false;
+ // Local copy of input files, which are pointers into argv.
+ std::vector localInputFiles;
+ for (int i = 1; i < argc; ++i) {
+ const char *arg = argv[i];
+ // Protect against empty arguments
+ if (arg[0] == 0) {
+ continue;
+ }
+ // Everything after "--" is an input file
+ if (!std::strcmp(arg, "--")) {
+ ++i;
+ std::copy(argv + i, argv + argc, std::back_inserter(localInputFiles));
+ break;
+ }
+ // Long arguments that don't have a short option
+ {
+ bool isLongOption = true;
+ if (!std::strcmp(arg, "--rm")) {
+ keepSource = false;
+ } else if (!std::strcmp(arg, "--ultra")) {
+ ultra = true;
+ maxWindowLog = 0;
+ } else if (!std::strcmp(arg, "--no-check")) {
+ checksum = false;
+ } else if (!std::strcmp(arg, "--sparse")) {
+ writeMode = WriteMode::Sparse;
+ notSupported("Sparse mode");
+ return Status::Failure;
+ } else if (!std::strcmp(arg, "--no-sparse")) {
+ writeMode = WriteMode::Regular;
+ notSupported("Sparse mode");
+ return Status::Failure;
+ } else if (!std::strcmp(arg, "--dictID")) {
+ notSupported(arg);
+ return Status::Failure;
+ } else if (!std::strcmp(arg, "--no-dictID")) {
+ notSupported(arg);
+ return Status::Failure;
+ } else {
+ isLongOption = false;
+ }
+ if (isLongOption) {
+ continue;
+ }
+ }
+ // Arguments with a short option simply set their short option.
+ const char *options = nullptr;
+ if (!std::strcmp(arg, "--processes")) {
+ options = "p";
+ } else if (!std::strcmp(arg, "--version")) {
+ options = "V";
+ } else if (!std::strcmp(arg, "--help")) {
+ options = "h";
+ } else if (!std::strcmp(arg, "--decompress")) {
+ options = "d";
+ } else if (!std::strcmp(arg, "--force")) {
+ options = "f";
+ } else if (!std::strcmp(arg, "--stdout")) {
+ options = "c";
+ } else if (!std::strcmp(arg, "--keep")) {
+ options = "k";
+ } else if (!std::strcmp(arg, "--verbose")) {
+ options = "v";
+ } else if (!std::strcmp(arg, "--quiet")) {
+ options = "q";
+ } else if (!std::strcmp(arg, "--check")) {
+ options = "C";
+ } else if (!std::strcmp(arg, "--test")) {
+ options = "t";
+ } else if (arg[0] == '-' && arg[1] != 0) {
+ options = arg + 1;
+ } else {
+ localInputFiles.emplace_back(arg);
+ continue;
+ }
+ assert(options != nullptr);
+
+ bool finished = false;
+ while (!finished && *options != 0) {
+ // Parse the compression level
+ if (*options >= '0' && *options <= '9') {
+ compressionLevel = parseUnsigned(&options);
+ continue;
+ }
+
+ switch (*options) {
+ case 'h':
+ case 'H':
+ usage();
+ return Status::Message;
+ case 'V':
+ std::fprintf(stderr, "PZSTD version: %s.\n", ZSTD_VERSION_STRING);
+ return Status::Message;
+ case 'p': {
+ finished = true;
+ const char *optionArgument = getArgument(options, argv, i, argc);
+ if (optionArgument == nullptr) {
+ return Status::Failure;
+ }
+ if (*optionArgument < '0' || *optionArgument > '9') {
+ std::fprintf(stderr, "Option -p expects a number, but %s provided\n",
+ optionArgument);
+ return Status::Failure;
+ }
+ numThreads = parseUnsigned(&optionArgument);
+ if (*optionArgument != 0) {
+ std::fprintf(stderr,
+ "Option -p expects a number, but %u%s provided\n",
+ numThreads, optionArgument);
+ return Status::Failure;
+ }
+ break;
+ }
+ case 'o': {
+ finished = true;
+ const char *optionArgument = getArgument(options, argv, i, argc);
+ if (optionArgument == nullptr) {
+ return Status::Failure;
+ }
+ outputFile = optionArgument;
+ break;
+ }
+ case 'C':
+ checksum = true;
+ break;
+ case 'k':
+ keepSource = true;
+ break;
+ case 'd':
+ decompress = true;
+ break;
+ case 'f':
+ overwrite = true;
+ forceStdout = true;
+ followLinks = true;
+ break;
+ case 't':
+ test = true;
+ decompress = true;
+ break;
+#ifdef UTIL_HAS_CREATEFILELIST
+ case 'r':
+ recursive = true;
+ break;
+#endif
+ case 'c':
+ outputFile = kStdOut;
+ forceStdout = true;
+ break;
+ case 'v':
+ ++verbosity;
+ break;
+ case 'q':
+ --verbosity;
+ // Ignore them for now
+ break;
+ // Unsupported options from Zstd
+ case 'D':
+ case 's':
+ notSupported("Zstd dictionaries.");
+ return Status::Failure;
+ case 'b':
+ case 'e':
+ case 'i':
+ case 'B':
+ notSupported("Zstd benchmarking options.");
+ return Status::Failure;
+ default:
+ std::fprintf(stderr, "Invalid argument: %s\n", arg);
+ return Status::Failure;
+ }
+ if (!finished) {
+ ++options;
+ }
+ } // while (*options != 0);
+ } // for (int i = 1; i < argc; ++i);
+
+ // Set options for test mode
+ if (test) {
+ outputFile = nullOutput;
+ keepSource = true;
+ }
+
+ // Input file defaults to standard input if not provided.
+ if (localInputFiles.empty()) {
+ localInputFiles.emplace_back(kStdIn);
+ }
+
+ // Check validity of input files
+ if (localInputFiles.size() > 1) {
+ const auto it = std::find(localInputFiles.begin(), localInputFiles.end(),
+ std::string{kStdIn});
+ if (it != localInputFiles.end()) {
+ std::fprintf(
+ stderr,
+ "Cannot specify standard input when handling multiple files\n");
+ return Status::Failure;
+ }
+ }
+ if (localInputFiles.size() > 1 || recursive) {
+ if (!outputFile.empty() && outputFile != nullOutput) {
+ std::fprintf(
+ stderr,
+ "Cannot specify an output file when handling multiple inputs\n");
+ return Status::Failure;
+ }
+ }
+
+ g_utilDisplayLevel = verbosity;
+ // Remove local input files that are symbolic links
+ if (!followLinks) {
+ std::remove_if(localInputFiles.begin(), localInputFiles.end(),
+ [&](const char *path) {
+ bool isLink = UTIL_isLink(path);
+ if (isLink && verbosity >= 2) {
+ std::fprintf(
+ stderr,
+ "Warning : %s is symbolic link, ignoring\n",
+ path);
+ }
+ return isLink;
+ });
+ }
+
+ // Translate input files/directories into files to (de)compress
+ if (recursive) {
+ FileNamesTable* const files = UTIL_createExpandedFNT(localInputFiles.data(), localInputFiles.size(), followLinks);
+ if (files == nullptr) {
+ std::fprintf(stderr, "Error traversing directories\n");
+ return Status::Failure;
+ }
+ auto guard =
+ makeScopeGuard([&] { UTIL_freeFileNamesTable(files); });
+ if (files->tableSize == 0) {
+ std::fprintf(stderr, "No files found\n");
+ return Status::Failure;
+ }
+ inputFiles.resize(files->tableSize);
+ std::copy(files->fileNames, files->fileNames + files->tableSize, inputFiles.begin());
+ } else {
+ inputFiles.resize(localInputFiles.size());
+ std::copy(localInputFiles.begin(), localInputFiles.end(),
+ inputFiles.begin());
+ }
+ localInputFiles.clear();
+ assert(!inputFiles.empty());
+
+ // If reading from standard input, default to standard output
+ if (inputFiles[0] == kStdIn && outputFile.empty()) {
+ assert(inputFiles.size() == 1);
+ outputFile = "-";
+ }
+
+ if (inputFiles[0] == kStdIn && IS_CONSOLE(stdin)) {
+ assert(inputFiles.size() == 1);
+ std::fprintf(stderr, "Cannot read input from interactive console\n");
+ return Status::Failure;
+ }
+ if (outputFile == "-" && IS_CONSOLE(stdout) && !(forceStdout && decompress)) {
+ std::fprintf(stderr, "Will not write to console stdout unless -c or -f is "
+ "specified and decompressing\n");
+ return Status::Failure;
+ }
+
+ // Check compression level
+ {
+ unsigned maxCLevel =
+ ultra ? ZSTD_maxCLevel() : kMaxNonUltraCompressionLevel;
+ if (compressionLevel > maxCLevel || compressionLevel == 0) {
+ std::fprintf(stderr, "Invalid compression level %u.\n", compressionLevel);
+ return Status::Failure;
+ }
+ }
+
+ // Check that numThreads is set
+ if (numThreads == 0) {
+ std::fprintf(stderr, "Invalid arguments: # of threads not specified "
+ "and unable to determine hardware concurrency.\n");
+ return Status::Failure;
+ }
+
+ // Modify verbosity
+ // If we are piping input and output, turn off interaction
+ if (inputFiles[0] == kStdIn && outputFile == kStdOut && verbosity == 2) {
+ verbosity = 1;
+ }
+ // If we are in multi-file mode, turn off interaction
+ if (inputFiles.size() > 1 && verbosity == 2) {
+ verbosity = 1;
+ }
+
+ return Status::Success;
+}
+
+std::string Options::getOutputFile(const std::string &inputFile) const {
+ if (!outputFile.empty()) {
+ return outputFile;
+ }
+ // Attempt to add/remove zstd extension from the input file
+ if (decompress) {
+ int stemSize = inputFile.size() - kZstdExtension.size();
+ if (stemSize > 0 && inputFile.substr(stemSize) == kZstdExtension) {
+ return inputFile.substr(0, stemSize);
+ } else {
+ return "";
+ }
+ } else {
+ return inputFile + kZstdExtension;
+ }
+}
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Options.h b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Options.h
new file mode 100644
index 0000000..924543a
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Options.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#define ZSTD_STATIC_LINKING_ONLY
+#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, pzstd itself is deprecated
+ * and uses deprecated functions
+ */
+#include "zstd.h"
+#undef ZSTD_STATIC_LINKING_ONLY
+
+#include
+#include
+#include
+
+namespace pzstd {
+
+struct Options {
+ enum class WriteMode { Regular, Auto, Sparse };
+
+ unsigned numThreads;
+ unsigned maxWindowLog;
+ unsigned compressionLevel;
+ bool decompress;
+ std::vector inputFiles;
+ std::string outputFile;
+ bool overwrite;
+ bool keepSource;
+ WriteMode writeMode;
+ bool checksum;
+ int verbosity;
+
+ enum class Status {
+ Success, // Successfully parsed options
+ Failure, // Failure to parse options
+ Message // Options specified to print a message (e.g. "-h")
+ };
+
+ Options();
+ Options(unsigned numThreads, unsigned maxWindowLog, unsigned compressionLevel,
+ bool decompress, std::vector inputFiles,
+ std::string outputFile, bool overwrite, bool keepSource,
+ WriteMode writeMode, bool checksum, int verbosity)
+ : numThreads(numThreads), maxWindowLog(maxWindowLog),
+ compressionLevel(compressionLevel), decompress(decompress),
+ inputFiles(std::move(inputFiles)), outputFile(std::move(outputFile)),
+ overwrite(overwrite), keepSource(keepSource), writeMode(writeMode),
+ checksum(checksum), verbosity(verbosity) {}
+
+ Status parse(int argc, const char **argv);
+
+ ZSTD_parameters determineParameters() const {
+ ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0);
+ params.fParams.contentSizeFlag = 0;
+ params.fParams.checksumFlag = checksum;
+ if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) {
+ params.cParams.windowLog = maxWindowLog;
+ params.cParams = ZSTD_adjustCParams(params.cParams, 0, 0);
+ }
+ return params;
+ }
+
+ std::string getOutputFile(const std::string &inputFile) const;
+};
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Pzstd.cpp b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Pzstd.cpp
new file mode 100644
index 0000000..2c09bda
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/pzstd/Pzstd.cpp
@@ -0,0 +1,611 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#include "platform.h" /* Large Files support, SET_BINARY_MODE */
+#include "Pzstd.h"
+#include "SkippableFrame.h"
+#include "utils/FileSystem.h"
+#include "utils/Range.h"
+#include "utils/ScopeGuard.h"
+#include "utils/ThreadPool.h"
+#include "utils/WorkQueue.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+
+namespace pzstd {
+
+namespace {
+#ifdef _WIN32
+const std::string nullOutput = "nul";
+#else
+const std::string nullOutput = "/dev/null";
+#endif
+}
+
+using std::size_t;
+
+static std::uintmax_t fileSizeOrZero(const std::string &file) {
+ if (file == "-") {
+ return 0;
+ }
+ std::error_code ec;
+ auto size = file_size(file, ec);
+ if (ec) {
+ size = 0;
+ }
+ return size;
+}
+
+static std::uint64_t handleOneInput(const Options &options,
+ const std::string &inputFile,
+ FILE* inputFd,
+ const std::string &outputFile,
+ FILE* outputFd,
+ SharedState& state) {
+ auto inputSize = fileSizeOrZero(inputFile);
+ // WorkQueue outlives ThreadPool so in the case of error we are certain
+ // we don't accidentally try to call push() on it after it is destroyed
+ WorkQueue> outs{options.numThreads + 1};
+ std::uint64_t bytesRead;
+ std::uint64_t bytesWritten;
+ {
+ // Initialize the (de)compression thread pool with numThreads
+ ThreadPool executor(options.numThreads);
+ // Run the reader thread on an extra thread
+ ThreadPool readExecutor(1);
+ if (!options.decompress) {
+ // Add a job that reads the input and starts all the compression jobs
+ readExecutor.add(
+ [&state, &outs, &executor, inputFd, inputSize, &options, &bytesRead] {
+ bytesRead = asyncCompressChunks(
+ state,
+ outs,
+ executor,
+ inputFd,
+ inputSize,
+ options.numThreads,
+ options.determineParameters());
+ });
+ // Start writing
+ bytesWritten = writeFile(state, outs, outputFd, options.decompress);
+ } else {
+ // Add a job that reads the input and starts all the decompression jobs
+ readExecutor.add([&state, &outs, &executor, inputFd, &bytesRead] {
+ bytesRead = asyncDecompressFrames(state, outs, executor, inputFd);
+ });
+ // Start writing
+ bytesWritten = writeFile(state, outs, outputFd, options.decompress);
+ }
+ }
+ if (!state.errorHolder.hasError()) {
+ std::string inputFileName = inputFile == "-" ? "stdin" : inputFile;
+ std::string outputFileName = outputFile == "-" ? "stdout" : outputFile;
+ if (!options.decompress) {
+ double ratio = static_cast(bytesWritten) /
+ static_cast(bytesRead + !bytesRead);
+ state.log(kLogInfo, "%-20s :%6.2f%% (%6" PRIu64 " => %6" PRIu64
+ " bytes, %s)\n",
+ inputFileName.c_str(), ratio * 100, bytesRead, bytesWritten,
+ outputFileName.c_str());
+ } else {
+ state.log(kLogInfo, "%-20s: %" PRIu64 " bytes \n",
+ inputFileName.c_str(),bytesWritten);
+ }
+ }
+ return bytesWritten;
+}
+
+static FILE *openInputFile(const std::string &inputFile,
+ ErrorHolder &errorHolder) {
+ if (inputFile == "-") {
+ SET_BINARY_MODE(stdin);
+ return stdin;
+ }
+ // Check if input file is a directory
+ {
+ std::error_code ec;
+ if (is_directory(inputFile, ec)) {
+ errorHolder.setError("Output file is a directory -- ignored");
+ return nullptr;
+ }
+ }
+ auto inputFd = std::fopen(inputFile.c_str(), "rb");
+ if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) {
+ return nullptr;
+ }
+ return inputFd;
+}
+
+static FILE *openOutputFile(const Options &options,
+ const std::string &outputFile,
+ SharedState& state) {
+ if (outputFile == "-") {
+ SET_BINARY_MODE(stdout);
+ return stdout;
+ }
+ // Check if the output file exists and then open it
+ if (!options.overwrite && outputFile != nullOutput) {
+ auto outputFd = std::fopen(outputFile.c_str(), "rb");
+ if (outputFd != nullptr) {
+ std::fclose(outputFd);
+ if (!state.log.logsAt(kLogInfo)) {
+ state.errorHolder.setError("Output file exists");
+ return nullptr;
+ }
+ state.log(
+ kLogInfo,
+ "pzstd: %s already exists; do you wish to overwrite (y/n) ? ",
+ outputFile.c_str());
+ int c = getchar();
+ if (c != 'y' && c != 'Y') {
+ state.errorHolder.setError("Not overwritten");
+ return nullptr;
+ }
+ }
+ }
+ auto outputFd = std::fopen(outputFile.c_str(), "wb");
+ if (!state.errorHolder.check(
+ outputFd != nullptr, "Failed to open output file")) {
+ return nullptr;
+ }
+ return outputFd;
+}
+
+int pzstdMain(const Options &options) {
+ int returnCode = 0;
+ SharedState state(options);
+ for (const auto& input : options.inputFiles) {
+ // Setup the shared state
+ auto printErrorGuard = makeScopeGuard([&] {
+ if (state.errorHolder.hasError()) {
+ returnCode = 1;
+ state.log(kLogError, "pzstd: %s: %s.\n", input.c_str(),
+ state.errorHolder.getError().c_str());
+ }
+ });
+ // Open the input file
+ auto inputFd = openInputFile(input, state.errorHolder);
+ if (inputFd == nullptr) {
+ continue;
+ }
+ auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); });
+ // Open the output file
+ auto outputFile = options.getOutputFile(input);
+ if (!state.errorHolder.check(outputFile != "",
+ "Input file does not have extension .zst")) {
+ continue;
+ }
+ auto outputFd = openOutputFile(options, outputFile, state);
+ if (outputFd == nullptr) {
+ continue;
+ }
+ auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); });
+ // (de)compress the file
+ handleOneInput(options, input, inputFd, outputFile, outputFd, state);
+ if (state.errorHolder.hasError()) {
+ continue;
+ }
+ // Delete the input file if necessary
+ if (!options.keepSource) {
+ // Be sure that we are done and have written everything before we delete
+ if (!state.errorHolder.check(std::fclose(inputFd) == 0,
+ "Failed to close input file")) {
+ continue;
+ }
+ closeInputGuard.dismiss();
+ if (!state.errorHolder.check(std::fclose(outputFd) == 0,
+ "Failed to close output file")) {
+ continue;
+ }
+ closeOutputGuard.dismiss();
+ if (std::remove(input.c_str()) != 0) {
+ state.errorHolder.setError("Failed to remove input file");
+ continue;
+ }
+ }
+ }
+ // Returns 1 if any of the files failed to (de)compress.
+ return returnCode;
+}
+
+/// Construct a `ZSTD_inBuffer` that points to the data in `buffer`.
+static ZSTD_inBuffer makeZstdInBuffer(const Buffer& buffer) {
+ return ZSTD_inBuffer{buffer.data(), buffer.size(), 0};
+}
+
+/**
+ * Advance `buffer` and `inBuffer` by the amount of data read, as indicated by
+ * `inBuffer.pos`.
+ */
+void advance(Buffer& buffer, ZSTD_inBuffer& inBuffer) {
+ auto pos = inBuffer.pos;
+ inBuffer.src = static_cast(inBuffer.src) + pos;
+ inBuffer.size -= pos;
+ inBuffer.pos = 0;
+ return buffer.advance(pos);
+}
+
+/// Construct a `ZSTD_outBuffer` that points to the data in `buffer`.
+static ZSTD_outBuffer makeZstdOutBuffer(Buffer& buffer) {
+ return ZSTD_outBuffer{buffer.data(), buffer.size(), 0};
+}
+
+/**
+ * Split `buffer` and advance `outBuffer` by the amount of data written, as
+ * indicated by `outBuffer.pos`.
+ */
+Buffer split(Buffer& buffer, ZSTD_outBuffer& outBuffer) {
+ auto pos = outBuffer.pos;
+ outBuffer.dst = static_cast(outBuffer.dst) + pos;
+ outBuffer.size -= pos;
+ outBuffer.pos = 0;
+ return buffer.splitAt(pos);
+}
+
+/**
+ * Stream chunks of input from `in`, compress it, and stream it out to `out`.
+ *
+ * @param state The shared state
+ * @param in Queue that we `pop()` input buffers from
+ * @param out Queue that we `push()` compressed output buffers to
+ * @param maxInputSize An upper bound on the size of the input
+ */
+static void compress(
+ SharedState& state,
+ std::shared_ptr in,
+ std::shared_ptr out,
+ size_t maxInputSize) {
+ auto& errorHolder = state.errorHolder;
+ auto guard = makeScopeGuard([&] { out->finish(); });
+ // Initialize the CCtx
+ auto ctx = state.cStreamPool->get();
+ if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_CStream")) {
+ return;
+ }
+ {
+ auto err = ZSTD_CCtx_reset(ctx.get(), ZSTD_reset_session_only);
+ if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
+ return;
+ }
+ }
+
+ // Allocate space for the result
+ auto outBuffer = Buffer(ZSTD_compressBound(maxInputSize));
+ auto zstdOutBuffer = makeZstdOutBuffer(outBuffer);
+ {
+ Buffer inBuffer;
+ // Read a buffer in from the input queue
+ while (in->pop(inBuffer) && !errorHolder.hasError()) {
+ auto zstdInBuffer = makeZstdInBuffer(inBuffer);
+ // Compress the whole buffer and send it to the output queue
+ while (!inBuffer.empty() && !errorHolder.hasError()) {
+ if (!errorHolder.check(
+ !outBuffer.empty(), "ZSTD_compressBound() was too small")) {
+ return;
+ }
+ // Compress
+ auto err =
+ ZSTD_compressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer);
+ if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
+ return;
+ }
+ // Split the compressed data off outBuffer and pass to the output queue
+ out->push(split(outBuffer, zstdOutBuffer));
+ // Forget about the data we already compressed
+ advance(inBuffer, zstdInBuffer);
+ }
+ }
+ }
+ // Write the epilog
+ size_t bytesLeft;
+ do {
+ if (!errorHolder.check(
+ !outBuffer.empty(), "ZSTD_compressBound() was too small")) {
+ return;
+ }
+ bytesLeft = ZSTD_endStream(ctx.get(), &zstdOutBuffer);
+ if (!errorHolder.check(
+ !ZSTD_isError(bytesLeft), ZSTD_getErrorName(bytesLeft))) {
+ return;
+ }
+ out->push(split(outBuffer, zstdOutBuffer));
+ } while (bytesLeft != 0 && !errorHolder.hasError());
+}
+
+/**
+ * Calculates how large each independently compressed frame should be.
+ *
+ * @param size The size of the source if known, 0 otherwise
+ * @param numThreads The number of threads available to run compression jobs on
+ * @param params The zstd parameters to be used for compression
+ */
+static size_t calculateStep(
+ std::uintmax_t size,
+ size_t numThreads,
+ const ZSTD_parameters ¶ms) {
+ (void)size;
+ (void)numThreads;
+ return size_t{1} << (params.cParams.windowLog + 2);
+}
+
+namespace {
+enum class FileStatus { Continue, Done, Error };
+/// Determines the status of the file descriptor `fd`.
+FileStatus fileStatus(FILE* fd) {
+ if (std::feof(fd)) {
+ return FileStatus::Done;
+ } else if (std::ferror(fd)) {
+ return FileStatus::Error;
+ }
+ return FileStatus::Continue;
+}
+} // anonymous namespace
+
+/**
+ * Reads `size` data in chunks of `chunkSize` and puts it into `queue`.
+ * Will read less if an error or EOF occurs.
+ * Returns the status of the file after all of the reads have occurred.
+ */
+static FileStatus
+readData(BufferWorkQueue& queue, size_t chunkSize, size_t size, FILE* fd,
+ std::uint64_t *totalBytesRead) {
+ Buffer buffer(size);
+ while (!buffer.empty()) {
+ auto bytesRead =
+ std::fread(buffer.data(), 1, std::min(chunkSize, buffer.size()), fd);
+ *totalBytesRead += bytesRead;
+ queue.push(buffer.splitAt(bytesRead));
+ auto status = fileStatus(fd);
+ if (status != FileStatus::Continue) {
+ return status;
+ }
+ }
+ return FileStatus::Continue;
+}
+
+std::uint64_t asyncCompressChunks(
+ SharedState& state,
+ WorkQueue>& chunks,
+ ThreadPool& executor,
+ FILE* fd,
+ std::uintmax_t size,
+ size_t numThreads,
+ ZSTD_parameters params) {
+ auto chunksGuard = makeScopeGuard([&] { chunks.finish(); });
+ std::uint64_t bytesRead = 0;
+
+ // Break the input up into chunks of size `step` and compress each chunk
+ // independently.
+ size_t step = calculateStep(size, numThreads, params);
+ state.log(kLogDebug, "Chosen frame size: %zu\n", step);
+ auto status = FileStatus::Continue;
+ while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
+ // Make a new input queue that we will put the chunk's input data into.
+ auto in = std::make_shared();
+ auto inGuard = makeScopeGuard([&] { in->finish(); });
+ // Make a new output queue that compress will put the compressed data into.
+ auto out = std::make_shared();
+ // Start compression in the thread pool
+ executor.add([&state, in, out, step] {
+ return compress(
+ state, std::move(in), std::move(out), step);
+ });
+ // Pass the output queue to the writer thread.
+ chunks.push(std::move(out));
+ state.log(kLogVerbose, "%s\n", "Starting a new frame");
+ // Fill the input queue for the compression job we just started
+ status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead);
+ }
+ state.errorHolder.check(status != FileStatus::Error, "Error reading input");
+ return bytesRead;
+}
+
+/**
+ * Decompress a frame, whose data is streamed into `in`, and stream the output
+ * to `out`.
+ *
+ * @param state The shared state
+ * @param in Queue that we `pop()` input buffers from. It contains
+ * exactly one compressed frame.
+ * @param out Queue that we `push()` decompressed output buffers to
+ */
+static void decompress(
+ SharedState& state,
+ std::shared_ptr in,
+ std::shared_ptr out) {
+ auto& errorHolder = state.errorHolder;
+ auto guard = makeScopeGuard([&] { out->finish(); });
+ // Initialize the DCtx
+ auto ctx = state.dStreamPool->get();
+ if (!errorHolder.check(ctx != nullptr, "Failed to allocate ZSTD_DStream")) {
+ return;
+ }
+ {
+ auto err = ZSTD_DCtx_reset(ctx.get(), ZSTD_reset_session_only);
+ if (!errorHolder.check(!ZSTD_isError(err), ZSTD_getErrorName(err))) {
+ return;
+ }
+ }
+
+ const size_t outSize = ZSTD_DStreamOutSize();
+ Buffer inBuffer;
+ size_t returnCode = 0;
+ // Read a buffer in from the input queue
+ while (in->pop(inBuffer) && !errorHolder.hasError()) {
+ auto zstdInBuffer = makeZstdInBuffer(inBuffer);
+ // Decompress the whole buffer and send it to the output queue
+ while (!inBuffer.empty() && !errorHolder.hasError()) {
+ // Allocate a buffer with at least outSize bytes.
+ Buffer outBuffer(outSize);
+ auto zstdOutBuffer = makeZstdOutBuffer(outBuffer);
+ // Decompress
+ returnCode =
+ ZSTD_decompressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer);
+ if (!errorHolder.check(
+ !ZSTD_isError(returnCode), ZSTD_getErrorName(returnCode))) {
+ return;
+ }
+ // Pass the buffer with the decompressed data to the output queue
+ out->push(split(outBuffer, zstdOutBuffer));
+ // Advance past the input we already read
+ advance(inBuffer, zstdInBuffer);
+ if (returnCode == 0) {
+ // The frame is over, prepare to (maybe) start a new frame
+ ZSTD_initDStream(ctx.get());
+ }
+ }
+ }
+ if (!errorHolder.check(returnCode <= 1, "Incomplete block")) {
+ return;
+ }
+ // We've given ZSTD_decompressStream all of our data, but there may still
+ // be data to read.
+ while (returnCode == 1) {
+ // Allocate a buffer with at least outSize bytes.
+ Buffer outBuffer(outSize);
+ auto zstdOutBuffer = makeZstdOutBuffer(outBuffer);
+ // Pass in no input.
+ ZSTD_inBuffer zstdInBuffer{nullptr, 0, 0};
+ // Decompress
+ returnCode =
+ ZSTD_decompressStream(ctx.get(), &zstdOutBuffer, &zstdInBuffer);
+ if (!errorHolder.check(
+ !ZSTD_isError(returnCode), ZSTD_getErrorName(returnCode))) {
+ return;
+ }
+ // Pass the buffer with the decompressed data to the output queue
+ out->push(split(outBuffer, zstdOutBuffer));
+ }
+}
+
+std::uint64_t asyncDecompressFrames(
+ SharedState& state,
+ WorkQueue
"; + for (l=0; l
" << endl; + continue; + } + + /* comments of type /**< and /*!< are detected and only function declaration is highlighted (bold) */ + if ((line.find("/**<")!=string::npos || line.find("/*!<")!=string::npos) && line.find("*/")!=string::npos) { + sout << "
"; + print_line(sout, line); + sout << "
" << endl; + continue; + } + + spos = line.find("/**="); + if (spos==string::npos) { + spos = line.find("/*!"); + if (spos==string::npos) + spos = line.find("/**"); + if (spos==string::npos) + spos = line.find("/*-"); + if (spos==string::npos) + spos = line.find("/*="); + if (spos==string::npos) + continue; + exclam = line[spos+2]; + } + else exclam = '='; + + comments = get_lines(input, linenum, "*/"); + if (!comments.empty()) comments[0] = line.substr(spos+3); + if (!comments.empty()) comments[comments.size()-1] = comments[comments.size()-1].substr(0, comments[comments.size()-1].find("*/")); + for (l=0; l
";
+ for (l=0; l
" << endl << endl;
+ } else if (exclam == '=') { /* comments of type /*= and /**= mean: use a header and show also all functions until first empty line */
+ trim(comments[0], " ");
+ sout << "
" << comments[0] << "
";
+ for (l=1; l
" << endl << endl;
+ }
+ }
+
+ ostream << "\n\n\n";
+ lines = get_lines(input, ++linenum, "");
+ for (l=0; l
" << endl;
+ } else { /* comments of type /** and /*- mean: this is a comment; use a header for the first line */
+ if (comments.empty()) continue;
+
+ trim(comments[0], " ");
+ sout << "
" << comments[0] << "
";
+ chapters.push_back(comments[0]);
+ chapter++;
+
+ for (l=1; l
" << endl << endl;
+ else
+ sout << "" << version << "
\n";
+
+ ostream << "
\nContents
\n\n";
+ for (size_t i=0; i
\n
\n";
+
+ ostream << sout.str();
+ ostream << "" << endl << "" << endl;
+
+ return 0;
+}
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/.gitignore b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/.gitignore
new file mode 100644
index 0000000..e77c4e4
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/.gitignore
@@ -0,0 +1,2 @@
+# build artifacts
+largeNbDicts
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/Makefile b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/Makefile
new file mode 100644
index 0000000..4c055b0
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/Makefile
@@ -0,0 +1,58 @@
+# ################################################################
+# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR)
+
+CFLAGS ?= -O3
+CFLAGS += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+ -Wstrict-aliasing=1 -Wswitch-enum \
+ -Wstrict-prototypes -Wundef -Wpointer-arith \
+ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+ -Wredundant-decls
+CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
+
+
+default: largeNbDicts
+
+all : largeNbDicts
+
+largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+ $(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+benchfn.o: $(PROGDIR)/benchfn.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+timefn.o: $(PROGDIR)/timefn.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+datagen.o: $(PROGDIR)/datagen.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+util.o: $(PROGDIR)/util.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+xxhash.o : $(LIBDIR)/common/xxhash.c
+ $(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+clean:
+ $(RM) *.o
+ $(MAKE) -C $(LIBDIR) clean > /dev/null
+ $(RM) largeNbDicts
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/README.md b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/README.md
new file mode 100644
index 0000000..f29bcdf
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/README.md
@@ -0,0 +1,25 @@
+largeNbDicts
+=====================
+
+`largeNbDicts` is a benchmark test tool
+dedicated to the specific scenario of
+dictionary decompression using a very large number of dictionaries.
+When dictionaries are constantly changing, they are always "cold",
+suffering from increased latency due to cache misses.
+
+The tool is created in a bid to investigate performance for this scenario,
+and experiment mitigation techniques.
+
+Command line :
+```
+largeNbDicts [Options] filename(s)
+
+Options :
+-r : recursively load all files in subdirectories (default: off)
+-B# : split input into blocks of size # (default: no split)
+-# : use compression level # (default: 3)
+-D # : use # as a dictionary (default: create one)
+-i# : nb benchmark rounds (default: 6)
+--nbDicts=# : set nb of dictionaries to # (default: one per block)
+-h : help (this text)
+```
diff --git a/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/largeNbDicts.c b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/largeNbDicts.c
new file mode 100644
index 0000000..ddbb3e0
--- /dev/null
+++ b/TurboPFor-Integer-Compression/lib/ext/bitshuffle/zstd/contrib/largeNbDicts/largeNbDicts.c
@@ -0,0 +1,998 @@
+/*
+ * Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* largeNbDicts
+ * This is a benchmark test tool
+ * dedicated to the specific case of dictionary decompression
+ * using a very large nb of dictionaries
+ * thus suffering latency from lots of cache misses.
+ * It's created in a bid to investigate performance and find optimizations. */
+
+
+/*--- Dependencies ---*/
+
+#include