From 83861db6b517ff5702d8ce8582774f46150f2cdc Mon Sep 17 00:00:00 2001 From: x Date: Thu, 19 Dec 2019 18:45:14 +0100 Subject: [PATCH] IcBench App: External libs --- ext/FastPFor | 2 +- ext/JASSv2 | 2 +- ext/SPDP_10.c | 4 +- ext/beplug_.h | 10 +- ext/beplugc_.c | 8 +- ext/beplugcs_.c | 9 +- ext/beplugd_.c | 6 +- ext/beplugds_.c | 6 +- ext/beplugr_.h | 2 + ext/c-blosc | 2 +- ext/libext.mak | 254 ++++++++++++++++++++++++++++++++++++++++++++++++ ext/lz4 | 2 +- 12 files changed, 293 insertions(+), 14 deletions(-) create mode 100644 ext/libext.mak diff --git a/ext/FastPFor b/ext/FastPFor index 14281a0..8030e38 160000 --- a/ext/FastPFor +++ b/ext/FastPFor @@ -1 +1 @@ -Subproject commit 14281a020ace4fd0bc3282cf905c6f0b320b8f2f +Subproject commit 8030e3808f857cc12d1d09e264f1ad2a05b1964a diff --git a/ext/JASSv2 b/ext/JASSv2 index 41bd4f7..fc756e5 160000 --- a/ext/JASSv2 +++ b/ext/JASSv2 @@ -1 +1 @@ -Subproject commit 41bd4f7b09b782f77976cd5730245388ffc40d25 +Subproject commit fc756e553bcf5b41be1cf00ad6decf25da1fd6cd diff --git a/ext/SPDP_10.c b/ext/SPDP_10.c index a0e04b2..e825695 100644 --- a/ext/SPDP_10.c +++ b/ext/SPDP_10.c @@ -46,7 +46,7 @@ typedef unsigned char byte_t; typedef unsigned int word_t; -static size_t compress(const byte_t level, const size_t length, byte_t* const buf1, byte_t* const buf2) +static size_t spdp_compress(const byte_t level, const size_t length, byte_t* const buf1, byte_t* const buf2) { word_t* in = (word_t*)buf1; word_t* out = (word_t*)buf2; @@ -119,7 +119,7 @@ static size_t compress(const byte_t level, const size_t length, byte_t* const bu return wpos; } -static void decompress(const byte_t level, const size_t length, byte_t* const buf2, byte_t* const buf1) +static void spdp_decompress(const byte_t level, const size_t length, byte_t* const buf2, byte_t* const buf1) { unsigned int predtabsize = 1 << (level + 9); if (predtabsize > MAX_TABLE_SIZE) predtabsize = MAX_TABLE_SIZE; diff --git a/ext/beplug_.h b/ext/beplug_.h index fcbfe07..77ce5e5 100644 --- a/ext/beplug_.h +++ b/ext/beplug_.h @@ -56,9 +56,15 @@ unsigned char *for_selectx( unsigned char *__restrict in, unsigned n, unsigned * #if C_QMX #include "JASSv2/source/compress_integer_qmx_improved.h" #endif + + #if C_VTENC +#include "VTEnc/vtenc.h" + #endif - #if C_ZLIB -#include + #if defined(ZLIB) +#include "zlib/zlib.h" //#include + +#define USE_LZ #endif #ifdef __cplusplus diff --git a/ext/beplugc_.c b/ext/beplugc_.c index 8e14c1d..82a7fbb 100644 --- a/ext/beplugc_.c +++ b/ext/beplugc_.c @@ -97,6 +97,10 @@ #if C_VARINTG8IU case P_VARINTG8IU: return vintg8enc(in, n, out); #endif + + #if C_VTENC + case P_VTENC: { size_t l; vtenc_list_encode_u32(in, n, out+4, outsize, &l); ctou32(out) = l; return l+4; } + #endif #if C_BITSHUFFLE case P_BITSHUFFLE: BITSHUFFLE( (unsigned char *)in, _n, out,lev); return out + _n; @@ -139,8 +143,8 @@ #endif #if C_ZLIB - case P_ZLIB1: case P_ZLIB2: case P_ZLIB3: case P_ZLIB4: case P_ZLIB5: case P_ZLIB6: case P_ZLIB7: case P_ZLIB8: case P_ZLIB9: - { n *= 4; tpenc( (unsigned char *)in, n, sbuf, 4); uLongf outlen = n; int rc = compress2(out+4, &outlen, sbuf, n, codec-P_ZLIB1+1); if(rc != Z_OK) die("zlib compress2 rc=%d\n", rc); *(unsigned *)out = outlen; return out + 4 + outlen; } + case P_ZLIB: + { n *= 4; tpenc( (unsigned char *)in, n, sbuf, 4); uLongf outlen = n; int rc = compress2(out+4, &outlen, sbuf, n, lev); if(rc != Z_OK) die("zlib compress2 rc=%d\n", rc); *(unsigned *)out = outlen; return out + 4 + outlen; } #endif diff --git a/ext/beplugcs_.c b/ext/beplugcs_.c index 577101e..01d8aad 100644 --- a/ext/beplugcs_.c +++ b/ext/beplugcs_.c @@ -48,6 +48,11 @@ #if C_VARINTG8IU case P_VARINTG8IU: bitdienc32( in, n, pa, -mdelta, mdelta); return vintg8enc(pa, n, out); #endif + + #if C_VTENC + case P_VTENC: { size_t l; vtenc_list_encode_u32(in, n, out+4, outsize, &l); ctou32(out) = l; return l+4; } + #endif + // --------- delta + transpose + lz77 ---------------------------------------------------------------------------------------- #if C_LZTURBO case P_LZT10:{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf,4); struct lzobj lz; lz.srclen = _n; lz.src = sbuf; lz.dst = out; lz.dstlen = _n; lz.level = 0; lz.hbits = 16; return out + lz8c01(&lz); } @@ -74,8 +79,8 @@ #endif #if C_ZLIB - case P_ZLIB1: case P_ZLIB2: case P_ZLIB3: case P_ZLIB4: case P_ZLIB5: case P_ZLIB6: case P_ZLIB7: case P_ZLIB8: case P_ZLIB9: - { bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf, 4); uLongf outlen = _n; int rc = compress2(out+4, &outlen, sbuf, _n, codec-P_ZLIB1+1); if(rc != Z_OK) die("zlib compress2 rc=%d\n", rc); *(unsigned *)out = outlen; return out + 4 + outlen; } + case P_ZLIB: + { bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf, 4); uLongf outlen = _n; int rc = compress2(out+4, &outlen, sbuf, _n, lev); if(rc != Z_OK) die("zlib compress2 rc=%d\n", rc); *(unsigned *)out = outlen; return out + 4 + outlen; } #endif diff --git a/ext/beplugd_.c b/ext/beplugd_.c index 6d9cbde..7ae8016 100644 --- a/ext/beplugd_.c +++ b/ext/beplugd_.c @@ -102,6 +102,10 @@ case P_VARINTG8IU: return vintg8dec(in, n, out); #endif + #if C_VTENC + case P_VTENC: vtenc_list_decode_u32(in+4, ctou32(in), out,n); return in + outlen; + #endif + #if C_BITSHUFFLE case P_BITSHUFFLE:BITUNSHUFFLE(in, outlen, (unsigned char *)out,lev); return in + outlen; #endif @@ -146,7 +150,7 @@ #endif #if C_ZLIB - case P_ZLIB1: case P_ZLIB2: case P_ZLIB3: case P_ZLIB4: case P_ZLIB5: case P_ZLIB6: case P_ZLIB7: case P_ZLIB8: case P_ZLIB9: + case P_ZLIB: { uLongf outsize = outlen; int l = *(unsigned *)in, rc = uncompress(sbuf, &outsize, in+4, l); in += 4 + l; tpdec(sbuf, outlen, (unsigned char *)out,4); } break; #endif diff --git a/ext/beplugds_.c b/ext/beplugds_.c index 2550b15..45f9c0a 100644 --- a/ext/beplugds_.c +++ b/ext/beplugds_.c @@ -48,8 +48,12 @@ case P_VARINTG8IU: in = vintg8dec( in, n, out); bitdidec32(out, n, -mdelta, mdelta); break; #endif + #if C_VTENC + case P_VTENC: vtenc_list_decode_u32(in+4, ctou32(in), out, n); return in + outlen; + #endif + #if C_ZLIB - case P_ZLIB1: case P_ZLIB2: case P_ZLIB3: case P_ZLIB4: case P_ZLIB5: case P_ZLIB6: case P_ZLIB7: case P_ZLIB8: case P_ZLIB9: + case P_ZLIB: { uLongf outsize = outlen; int l = *(unsigned *)in, rc = uncompress(sbuf, &outsize, in+4, l); in += 4 + l; tpdec(sbuf, outlen, (unsigned char *)out,4); bitdidec32(out, n, -mdelta, mdelta); } break; #endif diff --git a/ext/beplugr_.h b/ext/beplugr_.h index f8178f2..45c27ce 100644 --- a/ext/beplugr_.h +++ b/ext/beplugr_.h @@ -55,6 +55,8 @@ { P_ZLIB, "zlib", C_ZLIB, BLK_SIZE, 0, "4,8","TurboPFor's [delta]+byte transpose + zlib" }, { P_BITSHUFFLE, "BitShuffle", C_TURBOPFOR, BLK_SIZE,0,"2,4,8","Bit shuffle (simd)" }, { P_BS_SHUFFLE, "Blosc_Shuffle", C_C_BLOSC, BLK_SIZE,0,"2,4,8","Blosc's shuffle (simd)" }, + { P_VTENC, "vtenc", C_VTENC, BLK_SIZE,0,"2,4,8","VTEnc" }, + #ifdef LZTURBO // #include "../beplugr.h" #endif diff --git a/ext/c-blosc b/ext/c-blosc index aa8ee96..7d0b206 160000 --- a/ext/c-blosc +++ b/ext/c-blosc @@ -1 +1 @@ -Subproject commit aa8ee96d5b584bc311e70b4bec1ee16c33b05d07 +Subproject commit 7d0b2063f1f0864ba4d932a101ec0db2613786b4 diff --git a/ext/libext.mak b/ext/libext.mak new file mode 100644 index 0000000..42a0fbf --- /dev/null +++ b/ext/libext.mak @@ -0,0 +1,254 @@ +#cpp: $(CPPF) +# $(CC) -msse3 $(MSSE) $(MARCH) -w -E -P $(CPPF) + +CFLAGS+=-Iext +CXXFLAGS+=$(DDEBUG) -w -fpermissive -Wall -fno-rtti +CXXFLAGS+=$(SSE) -Iext/FastPFor/headers -std=gnu99 -DUSE_THREADS + +ifeq ($(LZ),1) +LZ4=1 +#ZLIB=1 +#ZSTD=1 +BITSHUFFLE=1 +#BLOSC=1 +endif + +ifeq ($(CODEC1),1) +FASTPFOR=1 +MASKEDVBYTE=1 +SIMDCOMP=1 +SPDP=1 +STREAMVBYTE=1 +#VTENC=1 +SIMPLE8B=1 +XDEFS+=-DCODEC1 +endif + +ifeq ($(CODEC2),1) +LIBFOR=1 +LITTLEINTPACKER=1 +POLYCOM=1 +QMX=1 +RICE=1 +VARINTG8IU=1 +XDEFS+=-DCODEC2 +endif + +#------------------------------------------------ +# use TurboPFor "p4d1enc" instead of the default bitpack functions in "idxcr.c" and "idxqry.c" (inverted index demo) +ifeq ($(TURBOPFOR),1) +CFLAGS+=-D_TURBOPFOR +endif +#ifeq ($(SIMDH),1) +#LIB+=bitunpack128h.o +#endif +#---------------------------------------- +icbench: $(OB) icbench.o plugins.o + $(CXX) $(XDEFS) $^ $(LDFLAGS) -o icbench + +idx: idxcr idxqry idxseg + +icb: icbench + +idxseg: idxseg.o $(LIB) + $(CC) $^ $(LDFLAGS) -o idxseg + +ictest: ictest.o $(LIB) + $(CC) $^ $(LDFLAGS) -o ictest + +ifeq ($(OS), Linux) +para: CFLAGS += -DTHREADMAX=32 +para: idxqry +endif + +idxcr: idxcr.o $(LIB) + $(CC) $^ $(LDFLAGS) -o idxcr $(LFLAGS) + +idxqry: idxqry.o $(LIB) + $(CC) $^ $(LDFLAGS) $(LIBTHREAD) $(LIBRT) -o idxqry $(LFLAGS) + +ext/polycom/optpfd.o: ext/polycom/optpfd.c + $(CC) -O2 $(MARCH) $(CFLAGS) $< -c -o $@ + +varintg8iu.o: ext/varintg8iu.c ext/varintg8iu.h + $(CC) -O2 $(CFLAGS) $(MARCH) -c -std=c99 ext/varintg8iu.c + +#------------------------------------------------------------------- +ifeq ($(BLOSC),1) +LDFLAGS+=-lpthread +CFLAGS+=-DBLOSC + +ext/c-blosc2/blosc/shuffle-sse2.o: ext/c-blosc2/blosc/shuffle-sse2.c + $(CC) -O3 $(CFLAGS) -msse2 -c ext/c-blosc2/blosc/shuffle-sse2.c -o ext/c-blosc2/blosc/shuffle-sse2.o + +ext/c-blosc2/blosc/shuffle-generic.o: ext/c-blosc2/blosc/shuffle-generic.c + $(CC) -O3 $(CFLAGS) -c ext/c-blosc2/blosc/shuffle-generic.c -o ext/c-blosc2/blosc/shuffle-generic.o + +ext/c-blosc2/blosc/shuffle-avx2.o: ext/c-blosc2/blosc/shuffle-avx2.c + $(CC) -O3 $(CFLAGS) -mavx2 -c ext/c-blosc2/blosc/shuffle-avx2.c -o ext/c-blosc2/blosc/shuffle-avx2.o + +ext/c-blosc2/blosc/shuffle-neon.o: ext/c-blosc2/blosc/shuffle-neon.c + $(CC) -O3 $(CFLAGS) -flax-vector-conversions -c ext/c-blosc2/blosc/shuffle-neon.c -o ext/c-blosc2/blosc/shuffle-neon.o + +ext/c-blosc2/blosc/bitshuffle-neon.o: ext/c-blosc2/blosc/bitshuffle-neon.c + $(CC) -O3 $(CFLAGS) -flax-vector-conversions -c ext/c-blosc2/blosc/bitshuffle-neon.c -o ext/c-blosc2/blosc/bitshuffle-neon.o + +OB+=ext/c-blosc2/blosc/blosc2.o ext/c-blosc2/blosc/blosclz.o ext/c-blosc2/blosc/shuffle.o ext/c-blosc2/blosc/shuffle-generic.o \ +ext/c-blosc2/blosc/bitshuffle-generic.o ext/c-blosc2/blosc/btune.o ext/c-blosc2/blosc/fastcopy.o ext/c-blosc2/blosc/delta.o ext/c-blosc2/blosc/timestamp.o ext/c-blosc2/blosc/trunc-prec.o + +ifeq ($(AVX2),1) +CFLAGS+=-DSHUFFLE_AVX2_ENABLED +OB+=ext/c-blosc2/blosc/shuffle-avx2.o ext/c-blosc2/blosc/bitshuffle-avx2.o +endif +ifeq ($(ARCH),aarch64) +CFLAGS+=-DSHUFFLE_NEON_ENABLED +OB+=ext/c-blosc2/blosc/shuffle-neon.o ext/c-blosc2/blosc/bitshuffle-neon.o +else +CFLAGS+=-DSHUFFLE_SSE2_ENABLED +OB+=ext/c-blosc2/blosc/bitshuffle-sse2.o ext/c-blosc2/blosc/shuffle-sse2.o +endif + +else + +ifeq ($(BITSHUFFLE),1) +CFLAGS+=-DBITSHUFFLE -Iext/bitshuffle/lz4 +#-DLZ4_ON +ifeq ($(ARCH),aarch64) +CFLAGS+=-DUSEARMNEON +else +ifeq ($(AVX2),1) +CFLAGS+=-DUSEAVX2 +endif +endif + +OB+=ext/bitshuffle/src/bitshuffle.o ext/bitshuffle/src/iochain.o ext/bitshuffle/src/bitshuffle_core.o +#OB+=ext/bitshuffle/lz4/lz4.o +endif + +endif + +ifeq ($(FASTPFOR),1) +XDEFS+=-DFASTPFOR +OB+=ext/FastPFor/src/bitpacking.o ext/FastPFor/src/simdbitpacking.o ext/FastPFor/src/simdunalignedbitpacking.o ext/fastpfor.o +endif + +ifeq ($(LIBFOR),1) +XDEFS+=-DLIBFOR +OB+=ext/libfor/for.o +endif + +ifeq ($(LITTLEINTPACKER),1) +XDEFS+=-Iext/LittleIntPacker/include -DLITTLEINTPACKER +OB+=ext/LittleIntPacker/src/bitpacking32.o ext/LittleIntPacker/src/turbobitpacking32.o ext/LittleIntPacker/src/scpacking32.o +OB+=ext/LittleIntPacker/src/horizontalpacking32.o +ifeq ($(AVX2),1) +OB+=ext/LittleIntPacker/src/bmipacking32.o +endif +endif + +ifeq ($(LZ4),1) +XDEFS+=-DLZ4 -Ilz4/lib +OB+=ext/lz4/lib/lz4hc.o ext/lz4/lib/lz4.o ext/lz4/lib/lz4frame.o +endif + +ifeq ($(LZTURBO),1) +CFLAGS+=-DLZTURBO +TURBORC=1 +include lzturbo.mak +endif + +ifeq ($(MASKEDVBYTE),1) +XDEFS+=-DMASKEVBYTE +CFLAGS+=-Iext/MaskedVByte/include +OB+=ext/MaskedVByte/src/varintencode.o ext/MaskedVByte/src/varintdecode.o +endif + +ifeq ($(QMX),1) +XDEFS+=-DQMX +OB+=ext/JASSv2/source/compress_integer_qmx_improved.o ext/JASSv2/source/asserts.o +endif + +ifeq ($(RICE),1) +XDEFS+=-DRICE +OB+=ext/rc.o +endif + +ifeq ($(SIMDCOMP),1) +XDEFS+=-Iext/simdcomp/include -DSIMDCOMP +OB+=ext/simdcomp/src/simdintegratedbitpacking.o ext/simdcomp/src/simdcomputil.o ext/simdcomp/src/simdbitpacking.o ext/simdcomp/src/simdpackedselect.o +OB+=ext/simdcomp_/simdfor.o +ifeq ($(AVX2),1) +OB+=ext/simdcomp/src/avxbitpacking.o +endif +endif + +ifeq ($(SPDP),1) +XDEFS+=-DSPDP +endif + +ifeq ($(STREAMVBYTE),1) +CFLAGS+=-Iext/streamvbyte/include -DSTREAMVBYTE +ifeq ($(ARCH),aarch64) +CFLAGS+=-D__ARM_NEON__ +endif +SV=ext/streamvbyte/src/ +OB+=$(SV)streamvbyte_decode.o $(SV)streamvbyte_encode.o $(SV)streamvbytedelta_encode.o $(SV)streamvbytedelta_decode.o $(SV)streamvbyte_0124_encode.o $(SV)streamvbyte_0124_decode.o $(SV)streamvbyte_zigzag.o +endif + +ifeq ($(TURBORC), 1) +XDEFS+=-DTURBORC +OB+=ext/TurboRC/turborcs.o ext/TurboRC/turborcss.o ext/TurboRC/turborcn.o +endif + +ifeq ($(VARINTG8IU),1) +XDEFS+=-DVARINTG8IU +OB+=ext/varintg8iu.o +endif + +ifeq ($(VBZ),1) +XDEFS+=-DVBZ +CXXFLAGS+=-Iext/vbz_compression/third_party -Iext/vbz_compression/vbz -Iext/vbz_compression/third_party/stream_vbyte/include +OB+=ext/vbz_compression/vbz/vbz.o ext/vbz_compression/vbz/v0/vbz_streamvbyte.o ext/vbz_compression/vbz/v1/vbz_streamvbyte.o +endif + +ifeq ($(VTENC),1) +XDEFS+=-DVTENC +OB+=ext/VTEnc/bits.o ext/VTEnc/bitstream.o ext/VTEnc/decode.o ext/VTEnc/encode.o ext/VTEnc/mem.o +endif + +ifeq ($(ZLIB), 1) +XDEFS+=-DZLIB +ifeq ($(HAVE_ZLIB), 1) +ifeq ($(STATIC),1) +OB+=/usr/lib/x86_64-linux-gnu/libz.a +else +OB+=-lz +endif +else +ZD=ext/zlib/ +OB+=$(ZD)adler32.o $(ZD)crc32.o $(ZD)compress.o $(ZD)deflate.o $(ZD)infback.o $(ZD)inffast.o $(ZD)inflate.o $(ZD)inftrees.o $(ZD)trees.o $(ZD)uncompr.o $(ZD)zutil.o +endif +endif + +ifeq ($(ZSTD), 1) +CFLAGS+=-Iext/zstd/lib -Iext/zstd/lib/common +XDEFS+=-DZSTD +OB+=ext/zstd/lib/common/pool.o ext/zstd/lib/common/xxhash.o ext/zstd/lib/common/error_private.o \ + ext/zstd/lib/compress/hist.o ext/zstd/lib/compress/zstd_compress.o ext/zstd/lib/compress/zstd_compress_literals.o ext/zstd/lib/compress/zstd_compress_sequences.o ext/zstd/lib/compress/zstd_double_fast.o ext/zstd/lib/compress/zstd_fast.o ext/zstd/lib/compress/zstd_lazy.o ext/zstd/lib/compress/zstd_ldm.o ext/zstd/lib/compress/zstdmt_compress.o ext/zstd/lib/compress/zstd_opt.o \ + ext/zstd/lib/decompress/zstd_decompress.o ext/zstd/lib/decompress/zstd_decompress_block.o ext/zstd/lib/decompress/zstd_ddict.o ext/zstd/lib/compress/fse_compress.o ext/zstd/lib/common/fse_decompress.o ext/zstd/lib/compress/huf_compress.o ext/zstd/lib/decompress/huf_decompress.o ext/zstd/lib/common/zstd_common.o ext/zstd/lib/common/entropy_common.o +endif + +ifeq ($(POLYCOM),1) +OB+=ext/polycom/optpfd.o +OB+=ext/polycom/polyvbyte.o +endif + +ifeq ($(SIMPLE8B),1) +OB+=ext/simple8b.o +endif + +ifeq ($(EFANO),1) +OB+=eliasfano.o +endif + +CFLAGS+=$(XDEFS) diff --git a/ext/lz4 b/ext/lz4 index e8baeca..0f74983 160000 --- a/ext/lz4 +++ b/ext/lz4 @@ -1 +1 @@ -Subproject commit e8baeca51ef2003d6c9ec21c32f1563fef1065b9 +Subproject commit 0f749838bf29bc0d1df428e23cf3dbb76ec4e9fc