diff --git a/ext/beplug_.h b/ext/beplug_.h deleted file mode 100644 index 77ce5e5..0000000 --- a/ext/beplug_.h +++ /dev/null @@ -1,154 +0,0 @@ - #ifndef __SSSE3__ -#define C_VINTG8IU 0 -#define C_MASKEDVBYTE 0 - #endif - - #if C_FASTPFOR128 -#include "FastPFor/headers/variablebyte.h" -#include "FastPFor/headers/simple16.h" -#include "FastPFor/headers/simple8b_rle.h" - -#include "FastPFor/headers/fastpfor.h" -#include "FastPFor/headers/optpfor.h" - -#include "FastPFor/headers/simdfastpfor.h" -#include "FastPFor/headers/simdoptpfor.h" -#ifndef min -#define min(x,y) (((x)<(y)) ? (x) : (y)) -#endif -#define _mm_store_si128 _mm_storeu_si128 -#define _mm_load_si128 _mm_loadu_si128 -#include "FastPFor/headers/simdgroupsimple.h" -#undef min - #endif - - #if C_SIMPLE8B -#include "simple8b.h" // optimized simple-8b by powturbo - #endif - - #if C_C_BLOSC -#include "c-blosc/blosc/shuffle.h" -#include "c-blosc/blosc/blosc.h" - #endif - - #if C_LIBFOR -#include "libfor/for.h" -unsigned char *for_selectx( unsigned char *__restrict in, unsigned n, unsigned *__restrict out) { unsigned b = in[4], i; for(i = 0; i < n; i++) out[i] = for_select(in, i); return in + 5 + for_compressed_size_bits(n, b); } - #endif - - #if C_LZ4 -#include "lz4/lib/lz4.h" -#include "lz4/lib/lz4hc.h" - #endif - -// #if C_SIMDCOMPLIB -//#include "ext/SIMDCompressionAndIntersection/include/bitpacking.h" -// #endif - - #if C_POLYCOM -#include "vas16c.h" // Simple 16 -#include "vas16d.h" -#include "rc.h" -#include "polycom/optpfd.h" -#include "polycom/polyvbyte.h" - #endif - - #if C_QMX -#include "JASSv2/source/compress_integer_qmx_improved.h" - #endif - - #if C_VTENC -#include "VTEnc/vtenc.h" - #endif - - #if defined(ZLIB) -#include "zlib/zlib.h" //#include - -#define USE_LZ - #endif - -#ifdef __cplusplus -extern "C" { -#endif - - #if C_STREAMVBYTE -#include "streamvbyte/include/streamvbyte.h" -#include "streamvbyte/include/streamvbytedelta.h" -#undef VARINTDECODE_H_ - - #endif - - #if C_MASKEDVBYTE -#include "MaskedVByte/include/varintencode.h" - #undef VARINTDECODE_H_ -#include "MaskedVByte/include/varintdecode.h" - #endif - - #if C_VARINTG8IU -#include "varintg8iu.h" // SIMD Varint G8IU - #endif - #if C_LITTLEPACK -#include "LittleIntPacker/include/bitpacking.h" -#include "LittleIntPacker/include/util.h" - #endif - -#if C_SIMDCOMP128 -#include "simdcomp/include/simdbitpacking.h" -#endif - -#ifdef __cplusplus -} -#endif - - #if C_SIMDCOMP128 -#undef SIMDBITPACKING_H_ -#include "vabyte.h" // Standard Variable Byte -#include "simdcomp/include/simdcomp.h" -unsigned char *simdpackwithoutmaskd1n(uint32_t *in, uint32_t n, uint32_t *out, uint32_t start, uint32_t b) {//checkifdivisibleby(n, 128); const uint32_t * const initout(out); //while(needPaddingTo128Bits(out)) *out++ = 123456; - uint32_t *ip; - for(ip = in; ip != in+(n&~(128-1)); ip += 128,out += 4 * b) - simdpackwithoutmaskd1(start, ip, (__m128i *)out, b); - return (unsigned char *)out; -} - -unsigned char *simdunpackd1n(uint32_t *in, uint32_t n, uint32_t *out, uint32_t start, uint32_t b) { - uint32_t k, *op; - for(op = out; op != out+(n&~(128-1)); op += 128,in += 4 * b) - simdunpackd1(start, (__m128i *)in, out, b); - return (unsigned char *)in; -} - -unsigned char *simdfor_selectx( unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) { - unsigned i; - for(i=0; i < n;i++) out[i] = simdselectFOR(start, (const __m128i *)in, b, i); return in + simdpackFOR_compressedbytes(n, b); -} - #endif - #if C_SIMDCOMP256 -#undef SIMDBITPACKING_H_ -unsigned char *avxpackwithoutmaskn(uint32_t *in, uint32_t n, uint32_t *out, uint32_t b) { - uint32_t *ip; - for(ip = in; ip != in+(n&~(256-1)); ip += 256,out += 8 * b) - avxpackwithoutmask(ip, (__m256i *)out, b); - return (unsigned char *)simdpack_shortlength(ip, n & (256-1), (__m128i *)out, b); -} -unsigned char *avxunpackn(uint32_t *in, uint32_t n, uint32_t *out, uint32_t b) { - uint32_t k, *op; - for(op = out; op != out+(n&~(256-1)); op += 256,in += 8 * b) - avxunpack((__m256i *)in, op, b); - return (unsigned char *)simdunpack_shortlength((__m128i *)in, n & (256-1), op, b); -} - #endif - - #if C_BITSHUFFLE -#define __STDC_VERSION__ 199901L -#include "bitshuffle/src/bitshuffle.h" - #endif - - #if C_C_BLOSC -#define BITSHUFFLE(in,n,out,esize) bitshuffle( esize, (n)/esize, (unsigned char *)in, out, NULL) //crash -#define BITUNSHUFFLE(in,n,out,esize) bitunshuffle( esize, (n)/esize, (unsigned char *)in, (unsigned char *)out, NULL) - #else -#define BITSHUFFLE(in,n,out,esize) bshuf_bitshuffle( in, out, (n)/esize, esize, 0); memcpy((char *)out+((n)&(~(8*esize-1))),(char *)in+((n)&(~(8*esize-1))),(n)&(8*esize-1)) -#define BITUNSHUFFLE(in,n,out,esize) bshuf_bitunshuffle(in, out, (n)/esize, esize, 0); memcpy((char *)out+((n)&(~(8*esize-1))),(char *)in+((n)&(~(8*esize-1))),(n)&(8*esize-1)) - #endif - diff --git a/ext/beplugc_.c b/ext/beplugc_.c deleted file mode 100644 index 82a7fbb..0000000 --- a/ext/beplugc_.c +++ /dev/null @@ -1,150 +0,0 @@ - //----- External libs ---------- - #if C_SIMPLE8B - case AM_SIMPLE8B:return vs8benc( in, n, out); - #endif - - #if C_FASTPFOR128 - case FP_FASTPFOR: { - size_t nvalue = outsize/4; - FastPForLib::FastPFor<4> ic; ic.encodeArray((const uint32_t *)in, n & (~127), (uint32_t *)(out+4), nvalue); - if(n & 127) { - size_t nvalue2 = outsize/4 - nvalue; - FastPForLib::VariableByte vc; vc.encodeArray((const uint32_t *)(in + (n & (~127))), n & 127, (uint32_t *)(out + 4 + nvalue*4), nvalue2); - nvalue += nvalue2; - } - ctou32(out) = nvalue; - return out+4+nvalue*4; - } - case FP_SIMDFASTPFOR: { - size_t nvalue = outsize/4; - FastPForLib::SIMDFastPFor<4> ic; ic.encodeArray((const uint32_t *)in, n & (~127), (uint32_t *)(out+4), nvalue); - if(n & 127) { - size_t nvalue2 = outsize/4 - nvalue; - FastPForLib::VariableByte vc; vc.encodeArray((const uint32_t *)(in + (n & (~127))), n & 127, (uint32_t *)(out + 4 + nvalue*4), nvalue2); - nvalue += nvalue2; - } - ctou32(out) = nvalue; - return out+4+nvalue*4; - } - case FP_SIMDOPTPFOR: { - size_t nvalue = outsize/4; - FastPForLib::SIMDOPTPFor<4> ic; ic.encodeArray((const uint32_t *)in, n & (~127), (uint32_t *)(out+4), nvalue); - if(n & 127) { - size_t nvalue2 = outsize/4 - nvalue; - FastPForLib::VariableByte vc; vc.encodeArray((const uint32_t *)(in + (n & (~127))), n & 127, (uint32_t *)(out + 4 + nvalue*4), nvalue2); - nvalue += nvalue2; - } - ctou32(out) = nvalue; - return out+4+nvalue*4; - } - case FP_GROUPSIMPLE: { - size_t nvalue = outsize/4; - FastPForLib::SIMDGroupSimple ic; ic.encodeArray((const uint32_t *)in, n & (~127), (uint32_t *)(out+4), nvalue); - if(n & 127) { - size_t nvalue2 = outsize/4 - nvalue; - FastPForLib::VariableByte vc; vc.encodeArray((const uint32_t *)(in + (n & (~127))), n & 127, (uint32_t *)(out + 4 + nvalue*4), nvalue2); - nvalue += nvalue2; - } - ctou32(out) = nvalue; - return out+4+nvalue*4; - } - case FP_VBYTE: { size_t nvalue=outsize/4; FastPForLib::VariableByte ic; ic.encodeArray((const uint32_t *)in, (const size_t)n, (uint32_t *)(out+4), nvalue); ctou32(out)=nvalue; return out+4+nvalue*4; } - case FP_SIMPLE8BRLE:{ size_t nvalue=outsize/4; FastPForLib::Simple8b_RLE ic; ic.encodeArray((const uint32_t *)in, (const size_t)n, (uint32_t *)(out+4), nvalue); ctou32(out)=nvalue; return out+4+nvalue*4; } - #endif - - #if C_LIBFOR - case LF_FORX: //if(b < 0) b = maxbits(in), *out++ = b; return out + for_compress_bits(in, out, n, 0, b); - case LF_FOR: return out + for_compress_unsorted(in, out, n); - #endif - - #if C_LITTLEPACK - case LI_PACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } pack32( in, n, b, out); return out + byte_count(n,b); - case LI_HORPACK: - case LI_TURBOPACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } turbopack32(in, n, b, out); return out + byte_count(n,b); - case LI_SCPACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } scpack32( in, n, b, out); return out + byte_count(n,b); - #ifdef __AVX2__ - case LI_BMIPACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } bmipack32( in, n, b, out); return out + byte_count(n,b); - #endif - #endif - - #if C_MASKEDVBYTE - case P_MASKEDVBYTE: return out + vbyte_encode(in, n, out); - #endif - - #if C_POLYCOM - case PC_VBYTE: return vbpolyenc(in, n, out); - case PC_SIMPLE16: return vs16enc( in, n, (unsigned *)out); - case PC_RICE: return rcenc32( in, n, (unsigned *)out); - case PC_OPTPFD: return optpfdenc32(in, n, out); //if(n < 128) return vbyteenc(in, n, (unsigned *)out); else { unsigned tmp[2048]; for(i = 0; i < n; i++) tmp[i] = in[i]; return out + OPT4(tmp, n, (unsigned *)out); } - #endif - //encode(void *encoded, size_t encoded_buffer_length, const integer *source, size_t source_integers) - #if C_QMX - case P_QMX: { JASS::compress_integer_qmx_improved qmx; unsigned r=qmx.encode(out+4, outsize, (uint32_t *)in, n); ctou32(out)=r; return out+4+r; } - #endif - - #if C_SIMDCOMP128 - //case SC_PACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } return fastpackwithoutmask32(in, n, (uint32_t *)out, b); - case SC_FOR: - case SC_FORDA: - case SC_SIMDPACK128:if(b < 0) b = maxbits(in), *out++ = b; return (unsigned char *)simdpack_length(in, n, (__m128i *)out, b); - #endif - #if C_SIMDCOMP256 - case SC_SIMDPACK256:if(b < 0) b = avxmaxbits(in), *out++ = b; return (unsigned char *)avxpackwithoutmaskn(in, n, (unsigned *)out, b); - #endif - #if C_STREAMVBYTE - case P_STREAMVBYTE: return out + streamvbyte_encode(in, n, out); - #endif - #if C_VARINTG8IU - case P_VARINTG8IU: return vintg8enc(in, n, out); - #endif - - #if C_VTENC - case P_VTENC: { size_t l; vtenc_list_encode_u32(in, n, out+4, outsize, &l); ctou32(out) = l; return l+4; } - #endif - - #if C_BITSHUFFLE - case P_BITSHUFFLE: BITSHUFFLE( (unsigned char *)in, _n, out,lev); return out + _n; - #endif - #if C_C_BLOSC - case P_BS_SHUFFLE: shuffle( lev, _n, (unsigned char *)in, out); return out + _n; - #endif - // --------- transpose + lz77 ---------------------------------------- - #if C_LZTURBO - case P_LZT10: { n *= 4; tpenc( (unsigned char *)in, n, sbuf,4); struct lzobj lz; lz.srclen = n; lz.src = sbuf; lz.dst = out; lz.dstlen = n; lz.level = 0; lz.hbits = 16; return out + lz8c01(&lz); } - case P_LZT20: { n *= 4; tpenc( (unsigned char *)in, n, sbuf,4); struct lzobj lz; lz.srclen = n; lz.src = sbuf; lz.dst = out; lz.dstlen = n; lz.level = 0; lz.hbits = 16; return out + lzbc01(&lz); } - case P_LZT22: { n *= 4; tpenc( (unsigned char *)in, n, sbuf,4); struct lzobj lz; lz.srclen = n; lz.src = sbuf; lz.dst = out; lz.dstlen = n; lz.level = 2; lz.hbits = 26; return out + lzbc2(&lz); } - case P_LZT32: { n *= 4; tpenc( (unsigned char *)in, n, sbuf,4); struct lzobj lz; lz.srclen = n; lz.src = sbuf; lz.dst = out; lz.dstlen = n; lz.level = 2; lz.hbits = 26; return out + lzhc2(&lz); } - //case P_VSHUF: { unsigned char *p = vsenc32(in, n, sbuf); p = mheenc(sbuf, p-sbuf, MH_BLK, out+2, out+_n+256); ctou16(out) = p-(out+2); printf("L=%d ", p-(out+2)); return p; } //out + anscompress( sbuf, xb, out, out+_n); - //case P_VSHUF: { unsigned char *p = mheenc(in, _n, MH_BLK, out+4, out+_n+256); ctou32(out) = p-(out+4); return p; } //out + anscompress( sbuf, xb, out, out+_n); - case P_VSHUF: { unsigned char *p;/* = vsenc32(in, n, sbuf)*/ n *= 4; tpenc( (unsigned char *)in, n, sbuf, 4); p = mheenc(sbuf, n/*p-sbuf*/, MH_BLK, out+4, out+n+256); ctou32(out) = p-(out+4); return p; } //out + anscompress( sbuf, xb, out, out+_n); - #endif - - #if C_BITSHUFFLE - //case TB_VSIMPLE_S1: BITSHUFFLE( (unsigned char *)in, _n, sbuf,4); return vsenc32((unsigned *)sbuf, n, out); - #endif - #if C_LZ4 - #if C_BITSHUFFLE - case LZ4_BIT: BITSHUFFLE( (unsigned char *)in, _n, sbuf,lev);return out + LZ4_compress((char *)sbuf, (char *)out, _n); - case LZ4_BITX: bitxenc32(in, n, (unsigned *)out, 0); BITSHUFFLE((unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); - #endif - case LZ4_ : return out + LZ4_compress((char *)in, (char *)out, _n); - case LZ4_NIBBLE: tp4enc((unsigned char *)in, _n, sbuf, lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); -// case LZ4_NIBBLE: loenc64((uint64_t *)in, (_n+7)/8, 3, (uint64_t *)out); tp4enc((unsigned char *)out, _n, sbuf, lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); - case LZ4_NIBBLEX:{bitxenc32(in, n, (unsigned *)out, 0); tp4enc((unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); } - case LZ4_BYTE: tpenc( (unsigned char *)in, _n, sbuf, lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); - //case LZ4_FP8: fp8zenc( (unsigned char *)in, _n, sbuf, sbuf+_n*2); return out + LZ4_compress((char *)sbuf, (char *)out, _n); - #endif - - #if C_C_BLOSC - case P_BS_LZ: - case P_BS_LZ4: - case P_BS_ZLIB: return out + blosc_compress(lev, BLOSC_SHUFFLE, 4/*typesize*/, _n, in, out, _n+BLOSC_MAX_OVERHEAD); - //return out + blosc_compress(1/*clevel*/, BLOSC_BITSHUFFLE, 4/*typesize*/, _n, in, out, _n+BLOSC_MAX_OVERHEAD); - #endif - - #if C_ZLIB - case P_ZLIB: - { n *= 4; tpenc( (unsigned char *)in, n, sbuf, 4); uLongf outlen = n; int rc = compress2(out+4, &outlen, sbuf, n, lev); if(rc != Z_OK) die("zlib compress2 rc=%d\n", rc); *(unsigned *)out = outlen; return out + 4 + outlen; } - #endif - - diff --git a/ext/beplugcs_.c b/ext/beplugcs_.c deleted file mode 100644 index 01d8aad..0000000 --- a/ext/beplugcs_.c +++ /dev/null @@ -1,86 +0,0 @@ - //-------- External libs ---------- - #if C_SIMPLE8B - case AM_SIMPLE8B: b = bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); if(b>28) die("simple-8b overflow.bits size>28\n"); - return vs8benc( pa, n, out); - #endif - case P_COPY: return u32enc( in, n, (unsigned *)out); - case P_MCPY: memcpy(out, _in, _n); return out+_n; - #if C_FASTPFOR - case FP_VBYTE: bitdienc32( in, n, pa, -mdelta, mdelta); return vbyteenc( pa, n, (unsigned *)out); - #endif - - #if C_MASKEDVBYTE - case P_MASKEDVBYTE: return out+vbyte_encode_delta(in, n, out, 0); - #endif - - #if C_LIBFOR - case LF_FORX: //b = bitf32( in+1, --n, in[0]); *(unsigned *)out = in[0]; out+= 4; *out++=b; return out + for_compress_bits( in+1, out, n, in[0], b); //if(b < 0) b = maxbits(in), *out++ = b; return out + for_compress_bits(in, out, n, 0, b); - case LF_FOR: return out + for_compress_sorted(in, out, n); - #endif - - #if C_POLYCOM - case PC_VBYTE: bitdienc32( in, n, pa, -mdelta, mdelta);return vbpolyenc(pa, n, out); - case PC_SIMPLE16: b = bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); if(b>28) die("simple16 overflow.bits size>28\n"); - return vs16enc( pa, n, (unsigned *)out); - case PC_OPTPFD: b = bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); if(b>28) die("optp4 overflow.bits size>28\n"); - return optpfdenc32(pa, n, out); - /*b = bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); if(b>28) die("optp4 overflow.bits size>28\n"); - if(n < 128) return vbyteenc( pa, n, (unsigned *)out); - else { return out + OPT4(pa, n, (unsigned *)out); }*/ - #endif - - #if C_QMX - case P_QMX: { bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); JASS::compress_integer_qmx_improved qmx; if(n) { unsigned r=qmx.encode(out+4, outsize, (uint32_t *)pa, n); ctou32(out) = r; return out+4+r; } return out; } - #endif - - #if C_SIMDCOMP - case SC_FOR: - case SC_FORDA: x = *in++; b = bitf32( in, --n, x); vbxput32(out, x); *out++=b; return (unsigned char *)simdpackFOR_length(x, in, n, (__m128i *)out, b); - case SC_SIMDPACK128: - if(n < 129) { bitdienc32( in, n, pa, -mdelta, mdelta); return vbyteenc((unsigned *)pa, n, (unsigned *)out); } - else { b = simdmaxbitsd1(in[0], in+1); vbxput32(out, in[0]); *out++=b; return simdpackwithoutmaskd1n(in+1, n-1, (unsigned *)out, in[0], b); } - #endif else { simdpackFOR(in[0], (in+1), (__m128i *)out, b); return out + simdpackFOR_compressedbytes(n, b); } - - #if C_STREAMVBYTE - case P_STREAMVBYTE: return out + streamvbyte_delta_encode(in, n, out, mdelta); - #endif - - #if C_VARINTG8IU - case P_VARINTG8IU: bitdienc32( in, n, pa, -mdelta, mdelta); return vintg8enc(pa, n, out); - #endif - - #if C_VTENC - case P_VTENC: { size_t l; vtenc_list_encode_u32(in, n, out+4, outsize, &l); ctou32(out) = l; return l+4; } - #endif - - // --------- delta + transpose + lz77 ---------------------------------------------------------------------------------------- - #if C_LZTURBO - case P_LZT10:{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf,4); struct lzobj lz; lz.srclen = _n; lz.src = sbuf; lz.dst = out; lz.dstlen = _n; lz.level = 0; lz.hbits = 16; return out + lz8c01(&lz); } - case P_LZT20:{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf,4); struct lzobj lz; lz.srclen = _n; lz.src = sbuf; lz.dst = out; lz.dstlen = _n; lz.level = 0; lz.hbits = 16; return out + lzbc01(&lz); } - case P_LZT22:{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf,4); struct lzobj lz; lz.srclen = _n; lz.src = sbuf; lz.dst = out; lz.dstlen = _n; lz.level = 2; lz.hbits = 26; return out + lzbc2(&lz); } - case P_VSHUF:{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf,4); unsigned char *p = mheenc(sbuf, _n, MH_BLK, out+4, out+_n+MH_BLK); ctou32(out) = p-(out+4); return p; } - //case P_VSHUF:{ bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); unsigned char *p=vsenc32(pa, n, sbuf); p = mheenc(sbuf, p-sbuf, MH_BLK, out+4, out+_n+1024); ctou32(out) = p-(out+4); return p; } - #endif - - #if C_C_BLOSC - case P_BS_LZ: - case P_BS_LZ4: - case P_BS_ZLIB: return out + blosc_compress(lev, BLOSC_SHUFFLE, 4/*typesize*/, _n, in, out, _n+BLOSC_MAX_OVERHEAD); - //case P_BS_ZLIB_1: return out + blosc_compress(lev, BLOSC_SHUFFLE/*BLOSC_DELTA*/, 4/*typesize*/, _n, in, out, _n+BLOSC_MAX_OVERHEAD); - #endif - - #if C_LZ4 - #if C_BITSHUFFLE - case LZ4_BIT: bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); BITSHUFFLE((unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); - #endif - case LZ4_ : return out + LZ4_compress((char *)in, (char *)out, _n); - case LZ4_NIBBLE:{ bitdienc32(in, n, (uint32_t *)out, -mdelta, mdelta); tp4enc((unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); } - case LZ4_BYTE: { bitdienc32(in, n, (uint32_t *)out, -mdelta, mdelta); tpenc( (unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); } - #endif - - #if C_ZLIB - case P_ZLIB: - { bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf, 4); uLongf outlen = _n; int rc = compress2(out+4, &outlen, sbuf, _n, lev); if(rc != Z_OK) die("zlib compress2 rc=%d\n", rc); *(unsigned *)out = outlen; return out + 4 + outlen; } - #endif - - diff --git a/ext/beplugd_.c b/ext/beplugd_.c deleted file mode 100644 index 7ae8016..0000000 --- a/ext/beplugd_.c +++ /dev/null @@ -1,156 +0,0 @@ - //---- external libs ------------- - case P_COPY: return u32dec( (unsigned *)in, n, out); - case P_MCPY: memcpy(_out, in, _n); return in+_n; - - #if C_SIMPLE8B - case AM_SIMPLE8B: return vs8bdec( in, n, out); - #endif - - #if C_FASTPFOR128 - case FP_FASTPFOR: { - size_t nvalue = n; - FastPForLib::FastPFor<4> ic; const uint32_t *ip = ic.decodeArray((const uint32_t *)(in+4), ctou32(in), out, nvalue); - if(n & 127) { - nvalue = n - nvalue; - FastPForLib::VariableByte vc; - return (unsigned char *)vc.decodeArray(ip, (const uint32_t *)in+1+ctou32(in) - ip, out + (n&(~127)), nvalue); //return vbdec32((unsigned char *)ip, n & 127, out + mynvalue1); - } - return (unsigned char *)ip; - } - - case FP_SIMDFASTPFOR: { - size_t nvalue = n; - FastPForLib::SIMDFastPFor<4> ic; const uint32_t *ip = ic.decodeArray((const uint32_t *)(in+4), ctou32(in), out, nvalue); - if(n & 127) { - nvalue = n - nvalue; - FastPForLib::VariableByte vc; - return (unsigned char *)vc.decodeArray(ip, (const uint32_t *)in+1+ctou32(in) - ip, out + (n&(~127)), nvalue); //return vbdec32((unsigned char *)ip, n & 127, out + mynvalue1); - } - return (unsigned char *)ip; - } - case FP_SIMDOPTPFOR: { - size_t nvalue = n; - FastPForLib::SIMDOPTPFor<4> ic; const uint32_t *ip = ic.decodeArray((const uint32_t *)(in+4), ctou32(in), out, nvalue); - if(n & 127) { - nvalue = n - nvalue; - FastPForLib::VariableByte vc; - return (unsigned char *)vc.decodeArray(ip, (const uint32_t *)in+1+ctou32(in) - ip, out + (n&(~127)), nvalue); //return vbdec32((unsigned char *)ip, n & 127, out + mynvalue1); - } - return (unsigned char *)ip; - } - case FP_GROUPSIMPLE: { - size_t nvalue = n; - FastPForLib::SIMDGroupSimple ic; const uint32_t *ip = ic.decodeArray((const uint32_t *)(in+4), ctou32(in), out, nvalue); - if(n & 127) { - nvalue = n - nvalue; - FastPForLib::VariableByte vc; - return (unsigned char *)vc.decodeArray(ip, (const uint32_t *)in+1+ctou32(in) - ip, out + (n&(~127)), nvalue); //return vbdec32((unsigned char *)ip, n & 127, out + mynvalue1); - } - return (unsigned char *)ip; - } - - case FP_VBYTE: //return vbytedec( in, n, out); - { size_t nvalue=n; FastPForLib::VariableByte ic; return (unsigned char *)ic.decodeArray((const uint32_t *)(in+4), ctou32(in), (uint32_t *)out, nvalue); } - case FP_SIMPLE8BRLE: - { size_t nvalue=n; FastPForLib::Simple8b_RLE ic; ic.decodeArray((const uint32_t *)(in+4), ctou32(in), (uint32_t *)out, nvalue); return in+4+ctou32(in)*4; } - #endif - - #if C_LIBFOR - case LF_FORX: return for_selectx(in, n, out); //if(b < 0) b = *in++; return for_selectx(in, n, out, 0, b);// - case LF_FOR: return in + for_uncompress(in, out, n); //if(b < 0) b = *in++; return in + for_uncompress_bits(in, out, n, 0, b); //return in + for_uncompress(in, out, n); - #endif - - #if C_LITTLEPACK - case LI_PACK: if(b < 0) b = *in++; unpack32( in, n, b, out); return in + byte_count(n,b); - case LI_TURBOPACK: if(b < 0) b = *in++; turbounpack32( in, n, b, out); return in + byte_count(n,b); - case LI_SCPACK: if(b < 0) b = *in++; scunpack32( in, n, b, out); return in + byte_count(n,b); - case LI_HORPACK: if(b < 0) b = *in++; horizontalunpack32(in, n, b, out); return in + byte_count(n,b); - #ifdef __AVX2__ - case LI_BMIPACK: if(b < 0) b = *in++; bmiunpack32( in, n, b, out); return in + byte_count(n,b); - #endif - #endif - - #if C_MASKEDVBYTE - case P_MASKEDVBYTE: return in + masked_vbyte_decode(in, out, n); - #endif - - #if C_POLYCOM - case PC_VBYTE: return vbpolydec(in, n, out); - case PC_SIMPLE16: return vs16dec( (unsigned *)in, n, out); - case PC_RICE: return rcdec32( (unsigned *)in, n, out); - case PC_OPTPFD : return optpfddec32( in, n, out); //if(n < 128) return vbytedec(in, n, out); else { unsigned all_array[2048]; return (unsigned char *)detailed_p4_decode(out, (unsigned *)in, all_array); } - #endif - - #if C_SIMDCOMP128 - //case SC_PACK: if(b < 0) b = *in++; return fastunpack32((uint32_t *)in, n, out, b); - case SC_FOR: - case SC_SIMDPACK128: if(b < 0) b = *in++; return (unsigned char *)simdunpack_length( (__m128i *)in, n, out, b); - case SC_FORDA: if(b < 0) b = *in++; return simdfor_selectx(in, n, out, 0, b); - #endif - #if C_SIMDCOMP256 - case SC_SIMDPACK256: if(b < 0) b = *in++; return (unsigned char *)avxunpackn( (unsigned *)in, n, out, b); - #endif - - #if C_STREAMVBYTE - case P_STREAMVBYTE: return in + streamvbyte_decode(in, out, n); - #endif - #if C_QMX - case P_QMX: { JASS::compress_integer_qmx_improved qmx; qmx.decode(out, n, in+4, ctou32(in)); return in+4+ctou32(in); } - #endif - - #if C_VARINTG8IU - case P_VARINTG8IU: return vintg8dec(in, n, out); - #endif - - #if C_VTENC - case P_VTENC: vtenc_list_decode_u32(in+4, ctou32(in), out,n); return in + outlen; - #endif - - #if C_BITSHUFFLE - case P_BITSHUFFLE:BITUNSHUFFLE(in, outlen, (unsigned char *)out,lev); return in + outlen; - #endif - #if C_C_BLOSC - case P_BS_SHUFFLE:unshuffle( lev, outlen, (unsigned char *)in, (unsigned char *)out);return in + outlen; - #endif - - //---------- transpose + lz77 ---------------------- - #if C_LZTURBO - case P_LZT10: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lz8d(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); } break; - case P_LZT20: - case P_LZT22: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lzbd(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); } break; - case P_LZT32: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lzhd(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); } break; - //case P_VSHUF: { unsigned inlen = ctou16(in); printf("l=%d ", inlen); in+=2; in = mhdec(in, inlen, MH_BLK, sbuf, outlen); vsdec32(sbuf, n, out); } break;// in = ansdecompress( in, outlen, sbuf); - //case P_VSHUF: { unsigned inlen = ctou32(in); in+=4; in = mhdec(in, inlen, MH_BLK, out, outlen); } break;// in = ansdecompress( in, outlen, sbuf); - case P_VSHUF: { unsigned inlen = ctou32(in); in+=4; in = mhdec(in, inlen, MH_BLK, sbuf, outlen); tpdec(sbuf, outlen, (unsigned char *)out,4); /*vsdec32(sbuf, n, out);*/ } break;// in = ansdecompress( in, outlen, sbuf); - #endif - - // #if C_BITSHUFFLE - //case TB_VSIMPLE_S1: in = vsdec32(in, n, (unsigned *)sbuf); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,4); break; - // #endif - //case TB_VSIMPLE_S4: in = vsdec32(in, n, (unsigned *)sbuf); tp4dec(sbuf, _n, (unsigned char *)out,4); break; - //case TB_VSIMPLE_S8: in = vsdec32(in, n, (unsigned *)sbuf); tpdec( sbuf, _n, (unsigned char *)out,4); break; - - #if C_LZ4 - #if C_BITSHUFFLE - case LZ4_BIT: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,lev); break; - case LZ4_BITX: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,lev); bitxdec32(out, n, 0); break; - #endif - case LZ4_ : in += LZ4_decompress_fast((char *)in, (char *)out, outlen); break; - //case LZ4_NIBBLE: in += LZ4_decompress_fast((char *)in, (char *)out, outlen); tp4dec((unsigned char *)out, outlen, (unsigned char *)sbuf,lev); lodec64((uint64_t *)sbuf, (outlen+7)/8, 3, (uint64_t *)out); break; - case LZ4_NIBBLE: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tp4dec(sbuf, outlen, (unsigned char *)out,lev); break; - case LZ4_NIBBLEX:in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tp4dec(sbuf, outlen, (unsigned char *)out,lev); bitxdec32(out, n, 0); break; - case LZ4_BYTE: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tpdec( sbuf, outlen, (unsigned char *)out,lev); break; - //case LZ4_FP8: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); fpzdec64(sbuf, sbuf+outlen*2, outlen, (unsigned char *)out); break; - #endif - - #if C_C_BLOSC - case P_BS_LZ: - case P_BS_LZ4: - case P_BS_ZLIB: { blosc_decompress(in, out, outlen); size_t nbytes, cbytes,blocksize; blosc_cbuffer_sizes(in, &nbytes, &cbytes, &blocksize); return in+cbytes; } - #endif - - #if C_ZLIB - case P_ZLIB: - { uLongf outsize = outlen; int l = *(unsigned *)in, rc = uncompress(sbuf, &outsize, in+4, l); in += 4 + l; tpdec(sbuf, outlen, (unsigned char *)out,4); } break; - #endif - diff --git a/ext/beplugds_.c b/ext/beplugds_.c deleted file mode 100644 index 45f9c0a..0000000 --- a/ext/beplugds_.c +++ /dev/null @@ -1,87 +0,0 @@ - //-------- Externa libs - case P_COPY: return u32dec( (unsigned *)in, n, out); - case P_MCPY: memcpy(_out, in, _n); return _out+_n; - - #if C_SIMPLE8B - case AM_SIMPLE8B: vbxget32(in, x);*out++ = x; --n; in = vs8bdec(in, n, out); bitdidec32(out, n, x+mdelta, mdelta); break; - #endif - - #if C_FASTPFOR - case FP_VBYTE: in = vbytedec( in, n, out); bitdidec32(out, n, -mdelta, mdelta); break; - #endif - - #if C_LIBFOR - case LF_FORX: return for_selectx(in, n, out); //{ out[0] = *(unsigned *)in, b = in[4]; return for_selectx(in+5, n-1, out+1, out[0], b); } - case LF_FOR: return in + for_uncompress(in, out, n); - #endif - - #if C_MASKEDVBYTE - case P_MASKEDVBYTE: in += masked_vbyte_decode_delta(in, out, n, 0); break; - #endif - - #if C_POLYCOM - case PC_VBYTE: in = vbpolydec( in, n, out); bitdidec32(out, n, -mdelta, mdelta); break; - case PC_SIMPLE16: vbxget32(in, x); *out = x; in = vs16dec((unsigned *)in, n-1, out+1); bitdidec32(out, n, -mdelta, mdelta); break; - case PC_OPTPFD: vbxget32(in, x); *out = x; in = optpfddec32(in,n-1,out+1); bitdidec32(out+1, n-1, x, mdelta); break; - /*if(n < 129) in = vbytedec(in, n, out); - else { _VBGET32(in, x, *out = x); unsigned all_array[2048]; in = (unsigned char *)detailed_p4_decode(out+1, (unsigned *)in, all_array); } - bitdidec32(out, n, -mdelta, mdelta); break;*/ - #endif - #if C_QMX - case P_QMX: { vbxget32(in, x); *out = x; if(n>1) { unsigned l = *(unsigned *)in; JASS::compress_integer_qmx_improved qmx; qmx.decode(out+1, n-1, in+4, l); bitdidec32(out+1, n-1, x, mdelta); return in+=4+l; } return in; } - #endif - - #if C_SIMDCOMP - case SC_SIMDPACK128: - if(n < 129) { in = vbytedec(in, n, out); bitdidec32(out, n, -mdelta, mdelta); } - else { vbxget32(in, x);*out = x; b = *in++; in = simdunpackd1n((uint32_t *)in, n-1, out+1, out[0], b); } break; - - case SC_FOR: vbxget32(in, x);*out = x; b = *in++; if(n < 129) return (unsigned char *)simdunpackFOR_length(x, (const __m128i *)in, n-1, (unsigned *)(out+1), b); else { simdunpackFOR(x, (const __m128i *)in, (unsigned *)(out+1), b); return in + simdpackFOR_compressedbytes(n-1, b); } - case SC_FORDA: { vbxget32(in, x);*out++ = x; --n; b = *in++; unsigned i; for(i=0; i < n;i++) out[i] = simdselectFOR(x, (const __m128i *)in, b, i); return in + simdpackFOR_compressedbytes(n, b); } - #endif - - #if C_STREAMVBYTE - case P_STREAMVBYTE: return in + streamvbyte_delta_decode(in, out, n, mdelta); - #endif - - #if C_VARINTG8IU - case P_VARINTG8IU: in = vintg8dec( in, n, out); bitdidec32(out, n, -mdelta, mdelta); break; - #endif - - #if C_VTENC - case P_VTENC: vtenc_list_decode_u32(in+4, ctou32(in), out, n); return in + outlen; - #endif - - #if C_ZLIB - case P_ZLIB: - { uLongf outsize = outlen; int l = *(unsigned *)in, rc = uncompress(sbuf, &outsize, in+4, l); in += 4 + l; tpdec(sbuf, outlen, (unsigned char *)out,4); bitdidec32(out, n, -mdelta, mdelta); } break; - #endif - - #if C_C_BLOSC - case P_BS_LZ: - case P_BS_LZ4: - case P_BS_ZLIB: { blosc_decompress(in, out, outlen); size_t nbytes, cbytes,blocksize; blosc_cbuffer_sizes(in, &nbytes, &cbytes, &blocksize); return in+cbytes; } - #endif - - #if C_LZTURBO - case P_LZT10: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lz8d(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); bitdidec32(out, n, -mdelta, mdelta); } break; - case P_LZT20: - case P_LZT22: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lzbd(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); bitdidec32(out, n, -mdelta, mdelta); } break; - //case P_VSHUF: vbxget32(in, x);*out = x; in += ansdecompress( in, (n-1)*4, sbuf); vsdec32( sbuf, n-1, out+1); bitdidec32(out, n, -mdelta, mdelta); break; - - case P_VSHUF: { unsigned inlen = ctou32(in); in+=4; in = mhdec(in, inlen, MH_BLK, sbuf, outlen); tpdec(sbuf, outlen, (unsigned char *)out,4); bitdidec32(out, n, -mdelta, mdelta); } break; - //case P_VSHUF: { vbxget32(in, x);*out = x; unsigned inlen = ctou32(in); in+=4; in = mhdec(in, inlen, MH_BLK, sbuf, outlen); vsdec32(sbuf, n-1, out+1); bitdidec32(out+1, n-1, x, mdelta); } break; - #endif - //---------- delta + transpose + lz77 ---------------------- - #if C_LZ4 - #if C_BITSHUFFLE - case LZ4_BIT: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,lev); bitdidec32(out, n, -mdelta, mdelta); break; - case LZ4_BITX: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,lev); bitxdec32(out, n, mdelta); break; - #endif - case LZ4_ : in += LZ4_decompress_fast((char *)in, (char *)out, outlen); break; - case LZ4_NIBBLE:in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tp4dec(sbuf, outlen, (unsigned char *)out,lev); bitdidec32(out, n, -mdelta, mdelta); break; - case LZ4_NIBBLEX:in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tp4dec(sbuf, outlen, (unsigned char *)out,lev); bitxdec32(out, n, mdelta); break; - case LZ4_BYTE: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tpdec( sbuf, outlen, (unsigned char *)out,lev); bitdidec32(out, n, -mdelta, mdelta); break; - #endif - - diff --git a/ext/beplugi_.c b/ext/beplugi_.c deleted file mode 100644 index b49bb5b..0000000 --- a/ext/beplugi_.c +++ /dev/null @@ -1,12 +0,0 @@ - #if C_C_BLOSC - case P_BS_LZ4: blosc_init(); blosc_set_nthreads(1); blosc_set_compressor(BLOSC_LZ4_COMPNAME);break; - case P_BS_ZLIB: blosc_init(); blosc_set_nthreads(1); blosc_set_compressor(BLOSC_ZLIB_COMPNAME); break; - case P_BS_LZ: blosc_init(); blosc_set_nthreads(1); blosc_set_compressor(BLOSC_BLOSCLZ_COMPNAME); break; - #endif - #if C_VARINTG8IU - case P_VARINTG8IU: VarIntG8IU(); - #endif - #ifdef LZTURBO - // #include "../beplug0.h" - #endif -