daiduo/TurboPFor-Integer-Compression
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
x
2023-03-17 20:10:09 +01:00
parent 6361c773e5
commit 09eb5f2f13
6 changed files with 0 additions and 645 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
ext/beplug_.h
View File

@ -1,154 +0,0 @@
#ifndef __SSSE3__
#define C_VINTG8IU 0
#define C_MASKEDVBYTE 0
#endif
#if C_FASTPFOR128
#include "FastPFor/headers/variablebyte.h"
#include "FastPFor/headers/simple16.h"
#include "FastPFor/headers/simple8b_rle.h"
#include "FastPFor/headers/fastpfor.h"
#include "FastPFor/headers/optpfor.h"
#include "FastPFor/headers/simdfastpfor.h"
#include "FastPFor/headers/simdoptpfor.h"
#ifndef min
#define min(x,y) (((x)<(y)) ? (x) : (y))
#endif
#define _mm_store_si128 _mm_storeu_si128
#define _mm_load_si128 _mm_loadu_si128
#include "FastPFor/headers/simdgroupsimple.h"
#undef min
#endif
#if C_SIMPLE8B
#include "simple8b.h" // optimized simple-8b by powturbo
#endif
#if C_C_BLOSC
#include "c-blosc/blosc/shuffle.h"
#include "c-blosc/blosc/blosc.h"
#endif
#if C_LIBFOR
#include "libfor/for.h"
unsigned char *for_selectx( unsigned char *__restrict in, unsigned n, unsigned *__restrict out) { unsigned b = in[4], i; for(i = 0; i < n; i++) out[i] = for_select(in, i); return in + 5 + for_compressed_size_bits(n, b); }
#endif
#if C_LZ4
#include "lz4/lib/lz4.h"
#include "lz4/lib/lz4hc.h"
#endif
// #if C_SIMDCOMPLIB
//#include "ext/SIMDCompressionAndIntersection/include/bitpacking.h"
// #endif
#if C_POLYCOM
#include "vas16c.h" // Simple 16
#include "vas16d.h"
#include "rc.h"
#include "polycom/optpfd.h"
#include "polycom/polyvbyte.h"
#endif
#if C_QMX
#include "JASSv2/source/compress_integer_qmx_improved.h"
#endif
#if C_VTENC
#include "VTEnc/vtenc.h"
#endif
#if defined(ZLIB)
#include "zlib/zlib.h" //#include <zlib.h>
#define USE_LZ
#endif
#ifdef __cplusplus
extern "C" {
#endif
#if C_STREAMVBYTE
#include "streamvbyte/include/streamvbyte.h"
#include "streamvbyte/include/streamvbytedelta.h"
#undef VARINTDECODE_H_
#endif
#if C_MASKEDVBYTE
#include "MaskedVByte/include/varintencode.h"
#undef VARINTDECODE_H_
#include "MaskedVByte/include/varintdecode.h"
#endif
#if C_VARINTG8IU
#include "varintg8iu.h" // SIMD Varint G8IU
#endif
#if C_LITTLEPACK
#include "LittleIntPacker/include/bitpacking.h"
#include "LittleIntPacker/include/util.h"
#endif
#if C_SIMDCOMP128
#include "simdcomp/include/simdbitpacking.h"
#endif
#ifdef __cplusplus
}
#endif
#if C_SIMDCOMP128
#undef SIMDBITPACKING_H_
#include "vabyte.h" // Standard Variable Byte
#include "simdcomp/include/simdcomp.h"
unsigned char *simdpackwithoutmaskd1n(uint32_t *in, uint32_t n, uint32_t *out, uint32_t start, uint32_t b) {//checkifdivisibleby(n, 128); const uint32_t * const initout(out); //while(needPaddingTo128Bits(out)) *out++ = 123456;
uint32_t *ip;
for(ip = in; ip != in+(n&~(128-1)); ip += 128,out += 4 * b)
simdpackwithoutmaskd1(start, ip, (__m128i *)out, b);
return (unsigned char *)out;
}
unsigned char *simdunpackd1n(uint32_t *in, uint32_t n, uint32_t *out, uint32_t start, uint32_t b) {
uint32_t k, *op;
for(op = out; op != out+(n&~(128-1)); op += 128,in += 4 * b)
simdunpackd1(start, (__m128i *)in, out, b);
return (unsigned char *)in;
}
unsigned char *simdfor_selectx( unsigned char *__restrict in, unsigned n, unsigned *__restrict out, unsigned start, unsigned b) {
unsigned i;
for(i=0; i < n;i++) out[i] = simdselectFOR(start, (const __m128i *)in, b, i); return in + simdpackFOR_compressedbytes(n, b);
}
#endif
#if C_SIMDCOMP256
#undef SIMDBITPACKING_H_
unsigned char *avxpackwithoutmaskn(uint32_t *in, uint32_t n, uint32_t *out, uint32_t b) {
uint32_t *ip;
for(ip = in; ip != in+(n&~(256-1)); ip += 256,out += 8 * b)
avxpackwithoutmask(ip, (__m256i *)out, b);
return (unsigned char *)simdpack_shortlength(ip, n & (256-1), (__m128i *)out, b);
}
unsigned char *avxunpackn(uint32_t *in, uint32_t n, uint32_t *out, uint32_t b) {
uint32_t k, *op;
for(op = out; op != out+(n&~(256-1)); op += 256,in += 8 * b)
avxunpack((__m256i *)in, op, b);
return (unsigned char *)simdunpack_shortlength((__m128i *)in, n & (256-1), op, b);
}
#endif
#if C_BITSHUFFLE
#define __STDC_VERSION__ 199901L
#include "bitshuffle/src/bitshuffle.h"
#endif
#if C_C_BLOSC
#define BITSHUFFLE(in,n,out,esize) bitshuffle( esize, (n)/esize, (unsigned char *)in, out, NULL) //crash
#define BITUNSHUFFLE(in,n,out,esize) bitunshuffle( esize, (n)/esize, (unsigned char *)in, (unsigned char *)out, NULL)
#else
#define BITSHUFFLE(in,n,out,esize) bshuf_bitshuffle( in, out, (n)/esize, esize, 0); memcpy((char *)out+((n)&(~(8*esize-1))),(char *)in+((n)&(~(8*esize-1))),(n)&(8*esize-1))
#define BITUNSHUFFLE(in,n,out,esize) bshuf_bitunshuffle(in, out, (n)/esize, esize, 0); memcpy((char *)out+((n)&(~(8*esize-1))),(char *)in+((n)&(~(8*esize-1))),(n)&(8*esize-1))
#endif

150
ext/beplugc_.c
View File

@ -1,150 +0,0 @@
//----- External libs ----------
#if C_SIMPLE8B
case AM_SIMPLE8B:return vs8benc( in, n, out);
#endif
#if C_FASTPFOR128
case FP_FASTPFOR: {
size_t nvalue = outsize/4;
FastPForLib::FastPFor<4> ic; ic.encodeArray((const uint32_t *)in, n & (~127), (uint32_t *)(out+4), nvalue);
if(n & 127) {
size_t nvalue2 = outsize/4 - nvalue;
FastPForLib::VariableByte vc; vc.encodeArray((const uint32_t *)(in + (n & (~127))), n & 127, (uint32_t *)(out + 4 + nvalue*4), nvalue2);
nvalue += nvalue2;
}
ctou32(out) = nvalue;
return out+4+nvalue*4;
}
case FP_SIMDFASTPFOR: {
size_t nvalue = outsize/4;
FastPForLib::SIMDFastPFor<4> ic; ic.encodeArray((const uint32_t *)in, n & (~127), (uint32_t *)(out+4), nvalue);
if(n & 127) {
size_t nvalue2 = outsize/4 - nvalue;
FastPForLib::VariableByte vc; vc.encodeArray((const uint32_t *)(in + (n & (~127))), n & 127, (uint32_t *)(out + 4 + nvalue*4), nvalue2);
nvalue += nvalue2;
}
ctou32(out) = nvalue;
return out+4+nvalue*4;
}
case FP_SIMDOPTPFOR: {
size_t nvalue = outsize/4;
FastPForLib::SIMDOPTPFor<4> ic; ic.encodeArray((const uint32_t *)in, n & (~127), (uint32_t *)(out+4), nvalue);
if(n & 127) {
size_t nvalue2 = outsize/4 - nvalue;
FastPForLib::VariableByte vc; vc.encodeArray((const uint32_t *)(in + (n & (~127))), n & 127, (uint32_t *)(out + 4 + nvalue*4), nvalue2);
nvalue += nvalue2;
}
ctou32(out) = nvalue;
return out+4+nvalue*4;
}
case FP_GROUPSIMPLE: {
size_t nvalue = outsize/4;
FastPForLib::SIMDGroupSimple<false,false> ic; ic.encodeArray((const uint32_t *)in, n & (~127), (uint32_t *)(out+4), nvalue);
if(n & 127) {
size_t nvalue2 = outsize/4 - nvalue;
FastPForLib::VariableByte vc; vc.encodeArray((const uint32_t *)(in + (n & (~127))), n & 127, (uint32_t *)(out + 4 + nvalue*4), nvalue2);
nvalue += nvalue2;
}
ctou32(out) = nvalue;
return out+4+nvalue*4;
}
case FP_VBYTE: { size_t nvalue=outsize/4; FastPForLib::VariableByte ic; ic.encodeArray((const uint32_t *)in, (const size_t)n, (uint32_t *)(out+4), nvalue); ctou32(out)=nvalue; return out+4+nvalue*4; }
case FP_SIMPLE8BRLE:{ size_t nvalue=outsize/4; FastPForLib::Simple8b_RLE<true> ic; ic.encodeArray((const uint32_t *)in, (const size_t)n, (uint32_t *)(out+4), nvalue); ctou32(out)=nvalue; return out+4+nvalue*4; }
#endif
#if C_LIBFOR
case LF_FORX: //if(b < 0) b = maxbits(in), *out++ = b; return out + for_compress_bits(in, out, n, 0, b);
case LF_FOR: return out + for_compress_unsorted(in, out, n);
#endif
#if C_LITTLEPACK
case LI_PACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } pack32( in, n, b, out); return out + byte_count(n,b);
case LI_HORPACK:
case LI_TURBOPACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } turbopack32(in, n, b, out); return out + byte_count(n,b);
case LI_SCPACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } scpack32( in, n, b, out); return out + byte_count(n,b);
#ifdef __AVX2__
case LI_BMIPACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } bmipack32( in, n, b, out); return out + byte_count(n,b);
#endif
#endif
#if C_MASKEDVBYTE
case P_MASKEDVBYTE: return out + vbyte_encode(in, n, out);
#endif
#if C_POLYCOM
case PC_VBYTE: return vbpolyenc(in, n, out);
case PC_SIMPLE16: return vs16enc( in, n, (unsigned *)out);
case PC_RICE: return rcenc32( in, n, (unsigned *)out);
case PC_OPTPFD: return optpfdenc32(in, n, out); //if(n < 128) return vbyteenc(in, n, (unsigned *)out); else { unsigned tmp[2048]; for(i = 0; i < n; i++) tmp[i] = in[i]; return out + OPT4(tmp, n, (unsigned *)out); }
#endif
//encode(void *encoded, size_t encoded_buffer_length, const integer *source, size_t source_integers)
#if C_QMX
case P_QMX: { JASS::compress_integer_qmx_improved qmx; unsigned r=qmx.encode(out+4, outsize, (uint32_t *)in, n); ctou32(out)=r; return out+4+r; }
#endif
#if C_SIMDCOMP128
//case SC_PACK: if(b < 0) { b = maxbits_length(in,n); *out++ = b; } return fastpackwithoutmask32(in, n, (uint32_t *)out, b);
case SC_FOR:
case SC_FORDA:
case SC_SIMDPACK128:if(b < 0) b = maxbits(in), *out++ = b; return (unsigned char *)simdpack_length(in, n, (__m128i *)out, b);
#endif
#if C_SIMDCOMP256
case SC_SIMDPACK256:if(b < 0) b = avxmaxbits(in), *out++ = b; return (unsigned char *)avxpackwithoutmaskn(in, n, (unsigned *)out, b);
#endif
#if C_STREAMVBYTE
case P_STREAMVBYTE: return out + streamvbyte_encode(in, n, out);
#endif
#if C_VARINTG8IU
case P_VARINTG8IU: return vintg8enc(in, n, out);
#endif
#if C_VTENC
case P_VTENC: { size_t l; vtenc_list_encode_u32(in, n, out+4, outsize, &l); ctou32(out) = l; return l+4; }
#endif
#if C_BITSHUFFLE
case P_BITSHUFFLE: BITSHUFFLE( (unsigned char *)in, _n, out,lev); return out + _n;
#endif
#if C_C_BLOSC
case P_BS_SHUFFLE: shuffle( lev, _n, (unsigned char *)in, out); return out + _n;
#endif
// --------- transpose + lz77 ----------------------------------------
#if C_LZTURBO
case P_LZT10: { n *= 4; tpenc( (unsigned char *)in, n, sbuf,4); struct lzobj lz; lz.srclen = n; lz.src = sbuf; lz.dst = out; lz.dstlen = n; lz.level = 0; lz.hbits = 16; return out + lz8c01(&lz); }
case P_LZT20: { n *= 4; tpenc( (unsigned char *)in, n, sbuf,4); struct lzobj lz; lz.srclen = n; lz.src = sbuf; lz.dst = out; lz.dstlen = n; lz.level = 0; lz.hbits = 16; return out + lzbc01(&lz); }
case P_LZT22: { n *= 4; tpenc( (unsigned char *)in, n, sbuf,4); struct lzobj lz; lz.srclen = n; lz.src = sbuf; lz.dst = out; lz.dstlen = n; lz.level = 2; lz.hbits = 26; return out + lzbc2(&lz); }
case P_LZT32: { n *= 4; tpenc( (unsigned char *)in, n, sbuf,4); struct lzobj lz; lz.srclen = n; lz.src = sbuf; lz.dst = out; lz.dstlen = n; lz.level = 2; lz.hbits = 26; return out + lzhc2(&lz); }
//case P_VSHUF: { unsigned char *p = vsenc32(in, n, sbuf); p = mheenc(sbuf, p-sbuf, MH_BLK, out+2, out+_n+256); ctou16(out) = p-(out+2); printf("L=%d ", p-(out+2)); return p; } //out + anscompress( sbuf, xb, out, out+_n);
//case P_VSHUF: { unsigned char *p = mheenc(in, _n, MH_BLK, out+4, out+_n+256); ctou32(out) = p-(out+4); return p; } //out + anscompress( sbuf, xb, out, out+_n);
case P_VSHUF: { unsigned char *p;/* = vsenc32(in, n, sbuf)*/ n *= 4; tpenc( (unsigned char *)in, n, sbuf, 4); p = mheenc(sbuf, n/*p-sbuf*/, MH_BLK, out+4, out+n+256); ctou32(out) = p-(out+4); return p; } //out + anscompress( sbuf, xb, out, out+_n);
#endif
#if C_BITSHUFFLE
//case TB_VSIMPLE_S1: BITSHUFFLE( (unsigned char *)in, _n, sbuf,4); return vsenc32((unsigned *)sbuf, n, out);
#endif
#if C_LZ4
#if C_BITSHUFFLE
case LZ4_BIT: BITSHUFFLE( (unsigned char *)in, _n, sbuf,lev);return out + LZ4_compress((char *)sbuf, (char *)out, _n);
case LZ4_BITX: bitxenc32(in, n, (unsigned *)out, 0); BITSHUFFLE((unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n);
#endif
case LZ4_ : return out + LZ4_compress((char *)in, (char *)out, _n);
case LZ4_NIBBLE: tp4enc((unsigned char *)in, _n, sbuf, lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n);
// case LZ4_NIBBLE: loenc64((uint64_t *)in, (_n+7)/8, 3, (uint64_t *)out); tp4enc((unsigned char *)out, _n, sbuf, lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n);
case LZ4_NIBBLEX:{bitxenc32(in, n, (unsigned *)out, 0); tp4enc((unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); }
case LZ4_BYTE: tpenc( (unsigned char *)in, _n, sbuf, lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n);
//case LZ4_FP8: fp8zenc( (unsigned char *)in, _n, sbuf, sbuf+_n*2); return out + LZ4_compress((char *)sbuf, (char *)out, _n);
#endif
#if C_C_BLOSC
case P_BS_LZ:
case P_BS_LZ4:
case P_BS_ZLIB: return out + blosc_compress(lev, BLOSC_SHUFFLE, 4/*typesize*/, _n, in, out, _n+BLOSC_MAX_OVERHEAD);
//return out + blosc_compress(1/*clevel*/, BLOSC_BITSHUFFLE, 4/*typesize*/, _n, in, out, _n+BLOSC_MAX_OVERHEAD);
#endif
#if C_ZLIB
case P_ZLIB:
{ n *= 4; tpenc( (unsigned char *)in, n, sbuf, 4); uLongf outlen = n; int rc = compress2(out+4, &outlen, sbuf, n, lev); if(rc != Z_OK) die("zlib compress2 rc=%d\n", rc); *(unsigned *)out = outlen; return out + 4 + outlen; }
#endif

86
ext/beplugcs_.c
View File

@ -1,86 +0,0 @@
//-------- External libs ----------
#if C_SIMPLE8B
case AM_SIMPLE8B: b = bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); if(b>28) die("simple-8b overflow.bits size>28\n");
return vs8benc( pa, n, out);
#endif
case P_COPY: return u32enc( in, n, (unsigned *)out);
case P_MCPY: memcpy(out, _in, _n); return out+_n;
#if C_FASTPFOR
case FP_VBYTE: bitdienc32( in, n, pa, -mdelta, mdelta); return vbyteenc( pa, n, (unsigned *)out);
#endif
#if C_MASKEDVBYTE
case P_MASKEDVBYTE: return out+vbyte_encode_delta(in, n, out, 0);
#endif
#if C_LIBFOR
case LF_FORX: //b = bitf32( in+1, --n, in[0]); *(unsigned *)out = in[0]; out+= 4; *out++=b; return out + for_compress_bits( in+1, out, n, in[0], b); //if(b < 0) b = maxbits(in), *out++ = b; return out + for_compress_bits(in, out, n, 0, b);
case LF_FOR: return out + for_compress_sorted(in, out, n);
#endif
#if C_POLYCOM
case PC_VBYTE: bitdienc32( in, n, pa, -mdelta, mdelta);return vbpolyenc(pa, n, out);
case PC_SIMPLE16: b = bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); if(b>28) die("simple16 overflow.bits size>28\n");
return vs16enc( pa, n, (unsigned *)out);
case PC_OPTPFD: b = bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); if(b>28) die("optp4 overflow.bits size>28\n");
return optpfdenc32(pa, n, out);
/*b = bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); if(b>28) die("optp4 overflow.bits size>28\n");
if(n < 128) return vbyteenc( pa, n, (unsigned *)out);
else { return out + OPT4(pa, n, (unsigned *)out); }*/
#endif
#if C_QMX
case P_QMX: { bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); JASS::compress_integer_qmx_improved qmx; if(n) { unsigned r=qmx.encode(out+4, outsize, (uint32_t *)pa, n); ctou32(out) = r; return out+4+r; } return out; }
#endif
#if C_SIMDCOMP
case SC_FOR:
case SC_FORDA: x = *in++; b = bitf32( in, --n, x); vbxput32(out, x); *out++=b; return (unsigned char *)simdpackFOR_length(x, in, n, (__m128i *)out, b);
case SC_SIMDPACK128:
if(n < 129) { bitdienc32( in, n, pa, -mdelta, mdelta); return vbyteenc((unsigned *)pa, n, (unsigned *)out); }
else { b = simdmaxbitsd1(in[0], in+1); vbxput32(out, in[0]); *out++=b; return simdpackwithoutmaskd1n(in+1, n-1, (unsigned *)out, in[0], b); }
#endif else { simdpackFOR(in[0], (in+1), (__m128i *)out, b); return out + simdpackFOR_compressedbytes(n, b); }
#if C_STREAMVBYTE
case P_STREAMVBYTE: return out + streamvbyte_delta_encode(in, n, out, mdelta);
#endif
#if C_VARINTG8IU
case P_VARINTG8IU: bitdienc32( in, n, pa, -mdelta, mdelta); return vintg8enc(pa, n, out);
#endif
#if C_VTENC
case P_VTENC: { size_t l; vtenc_list_encode_u32(in, n, out+4, outsize, &l); ctou32(out) = l; return l+4; }
#endif
// --------- delta + transpose + lz77 ----------------------------------------------------------------------------------------
#if C_LZTURBO
case P_LZT10:{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf,4); struct lzobj lz; lz.srclen = _n; lz.src = sbuf; lz.dst = out; lz.dstlen = _n; lz.level = 0; lz.hbits = 16; return out + lz8c01(&lz); }
case P_LZT20:{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf,4); struct lzobj lz; lz.srclen = _n; lz.src = sbuf; lz.dst = out; lz.dstlen = _n; lz.level = 0; lz.hbits = 16; return out + lzbc01(&lz); }
case P_LZT22:{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf,4); struct lzobj lz; lz.srclen = _n; lz.src = sbuf; lz.dst = out; lz.dstlen = _n; lz.level = 2; lz.hbits = 26; return out + lzbc2(&lz); }
case P_VSHUF:{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf,4); unsigned char *p = mheenc(sbuf, _n, MH_BLK, out+4, out+_n+MH_BLK); ctou32(out) = p-(out+4); return p; }
//case P_VSHUF:{ bitdienc32( in+1, --n, pa, in[0], mdelta); vbxput32(out, in[0]); unsigned char *p=vsenc32(pa, n, sbuf); p = mheenc(sbuf, p-sbuf, MH_BLK, out+4, out+_n+1024); ctou32(out) = p-(out+4); return p; }
#endif
#if C_C_BLOSC
case P_BS_LZ:
case P_BS_LZ4:
case P_BS_ZLIB: return out + blosc_compress(lev, BLOSC_SHUFFLE, 4/*typesize*/, _n, in, out, _n+BLOSC_MAX_OVERHEAD);
//case P_BS_ZLIB_1: return out + blosc_compress(lev, BLOSC_SHUFFLE/*BLOSC_DELTA*/, 4/*typesize*/, _n, in, out, _n+BLOSC_MAX_OVERHEAD);
#endif
#if C_LZ4
#if C_BITSHUFFLE
case LZ4_BIT: bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); BITSHUFFLE((unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n);
#endif
case LZ4_ : return out + LZ4_compress((char *)in, (char *)out, _n);
case LZ4_NIBBLE:{ bitdienc32(in, n, (uint32_t *)out, -mdelta, mdelta); tp4enc((unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); }
case LZ4_BYTE: { bitdienc32(in, n, (uint32_t *)out, -mdelta, mdelta); tpenc( (unsigned char *)out, _n, sbuf,lev); return out + LZ4_compress((char *)sbuf, (char *)out, _n); }
#endif
#if C_ZLIB
case P_ZLIB:
{ bitdienc32(in, n, (unsigned *)out, -mdelta, mdelta); tpenc((unsigned char *)out, _n, sbuf, 4); uLongf outlen = _n; int rc = compress2(out+4, &outlen, sbuf, _n, lev); if(rc != Z_OK) die("zlib compress2 rc=%d\n", rc); *(unsigned *)out = outlen; return out + 4 + outlen; }
#endif

156
ext/beplugd_.c
View File

@ -1,156 +0,0 @@
//---- external libs -------------
case P_COPY: return u32dec( (unsigned *)in, n, out);
case P_MCPY: memcpy(_out, in, _n); return in+_n;
#if C_SIMPLE8B
case AM_SIMPLE8B: return vs8bdec( in, n, out);
#endif
#if C_FASTPFOR128
case FP_FASTPFOR: {
size_t nvalue = n;
FastPForLib::FastPFor<4> ic; const uint32_t *ip = ic.decodeArray((const uint32_t *)(in+4), ctou32(in), out, nvalue);
if(n & 127) {
nvalue = n - nvalue;
FastPForLib::VariableByte vc;
return (unsigned char *)vc.decodeArray(ip, (const uint32_t *)in+1+ctou32(in) - ip, out + (n&(~127)), nvalue); //return vbdec32((unsigned char *)ip, n & 127, out + mynvalue1);
}
return (unsigned char *)ip;
}
case FP_SIMDFASTPFOR: {
size_t nvalue = n;
FastPForLib::SIMDFastPFor<4> ic; const uint32_t *ip = ic.decodeArray((const uint32_t *)(in+4), ctou32(in), out, nvalue);
if(n & 127) {
nvalue = n - nvalue;
FastPForLib::VariableByte vc;
return (unsigned char *)vc.decodeArray(ip, (const uint32_t *)in+1+ctou32(in) - ip, out + (n&(~127)), nvalue); //return vbdec32((unsigned char *)ip, n & 127, out + mynvalue1);
}
return (unsigned char *)ip;
}
case FP_SIMDOPTPFOR: {
size_t nvalue = n;
FastPForLib::SIMDOPTPFor<4> ic; const uint32_t *ip = ic.decodeArray((const uint32_t *)(in+4), ctou32(in), out, nvalue);
if(n & 127) {
nvalue = n - nvalue;
FastPForLib::VariableByte vc;
return (unsigned char *)vc.decodeArray(ip, (const uint32_t *)in+1+ctou32(in) - ip, out + (n&(~127)), nvalue); //return vbdec32((unsigned char *)ip, n & 127, out + mynvalue1);
}
return (unsigned char *)ip;
}
case FP_GROUPSIMPLE: {
size_t nvalue = n;
FastPForLib::SIMDGroupSimple<false,false> ic; const uint32_t *ip = ic.decodeArray((const uint32_t *)(in+4), ctou32(in), out, nvalue);
if(n & 127) {
nvalue = n - nvalue;
FastPForLib::VariableByte vc;
return (unsigned char *)vc.decodeArray(ip, (const uint32_t *)in+1+ctou32(in) - ip, out + (n&(~127)), nvalue); //return vbdec32((unsigned char *)ip, n & 127, out + mynvalue1);
}
return (unsigned char *)ip;
}
case FP_VBYTE: //return vbytedec( in, n, out);
{ size_t nvalue=n; FastPForLib::VariableByte ic; return (unsigned char *)ic.decodeArray((const uint32_t *)(in+4), ctou32(in), (uint32_t *)out, nvalue); }
case FP_SIMPLE8BRLE:
{ size_t nvalue=n; FastPForLib::Simple8b_RLE<true> ic; ic.decodeArray((const uint32_t *)(in+4), ctou32(in), (uint32_t *)out, nvalue); return in+4+ctou32(in)*4; }
#endif
#if C_LIBFOR
case LF_FORX: return for_selectx(in, n, out); //if(b < 0) b = *in++; return for_selectx(in, n, out, 0, b);//
case LF_FOR: return in + for_uncompress(in, out, n); //if(b < 0) b = *in++; return in + for_uncompress_bits(in, out, n, 0, b); //return in + for_uncompress(in, out, n);
#endif
#if C_LITTLEPACK
case LI_PACK: if(b < 0) b = *in++; unpack32( in, n, b, out); return in + byte_count(n,b);
case LI_TURBOPACK: if(b < 0) b = *in++; turbounpack32( in, n, b, out); return in + byte_count(n,b);
case LI_SCPACK: if(b < 0) b = *in++; scunpack32( in, n, b, out); return in + byte_count(n,b);
case LI_HORPACK: if(b < 0) b = *in++; horizontalunpack32(in, n, b, out); return in + byte_count(n,b);
#ifdef __AVX2__
case LI_BMIPACK: if(b < 0) b = *in++; bmiunpack32( in, n, b, out); return in + byte_count(n,b);
#endif
#endif
#if C_MASKEDVBYTE
case P_MASKEDVBYTE: return in + masked_vbyte_decode(in, out, n);
#endif
#if C_POLYCOM
case PC_VBYTE: return vbpolydec(in, n, out);
case PC_SIMPLE16: return vs16dec( (unsigned *)in, n, out);
case PC_RICE: return rcdec32( (unsigned *)in, n, out);
case PC_OPTPFD : return optpfddec32( in, n, out); //if(n < 128) return vbytedec(in, n, out); else { unsigned all_array[2048]; return (unsigned char *)detailed_p4_decode(out, (unsigned *)in, all_array); }
#endif
#if C_SIMDCOMP128
//case SC_PACK: if(b < 0) b = *in++; return fastunpack32((uint32_t *)in, n, out, b);
case SC_FOR:
case SC_SIMDPACK128: if(b < 0) b = *in++; return (unsigned char *)simdunpack_length( (__m128i *)in, n, out, b);
case SC_FORDA: if(b < 0) b = *in++; return simdfor_selectx(in, n, out, 0, b);
#endif
#if C_SIMDCOMP256
case SC_SIMDPACK256: if(b < 0) b = *in++; return (unsigned char *)avxunpackn( (unsigned *)in, n, out, b);
#endif
#if C_STREAMVBYTE
case P_STREAMVBYTE: return in + streamvbyte_decode(in, out, n);
#endif
#if C_QMX
case P_QMX: { JASS::compress_integer_qmx_improved qmx; qmx.decode(out, n, in+4, ctou32(in)); return in+4+ctou32(in); }
#endif
#if C_VARINTG8IU
case P_VARINTG8IU: return vintg8dec(in, n, out);
#endif
#if C_VTENC
case P_VTENC: vtenc_list_decode_u32(in+4, ctou32(in), out,n); return in + outlen;
#endif
#if C_BITSHUFFLE
case P_BITSHUFFLE:BITUNSHUFFLE(in, outlen, (unsigned char *)out,lev); return in + outlen;
#endif
#if C_C_BLOSC
case P_BS_SHUFFLE:unshuffle( lev, outlen, (unsigned char *)in, (unsigned char *)out);return in + outlen;
#endif
//---------- transpose + lz77 ----------------------
#if C_LZTURBO
case P_LZT10: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lz8d(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); } break;
case P_LZT20:
case P_LZT22: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lzbd(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); } break;
case P_LZT32: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lzhd(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); } break;
//case P_VSHUF: { unsigned inlen = ctou16(in); printf("l=%d ", inlen); in+=2; in = mhdec(in, inlen, MH_BLK, sbuf, outlen); vsdec32(sbuf, n, out); } break;// in = ansdecompress( in, outlen, sbuf);
//case P_VSHUF: { unsigned inlen = ctou32(in); in+=4; in = mhdec(in, inlen, MH_BLK, out, outlen); } break;// in = ansdecompress( in, outlen, sbuf);
case P_VSHUF: { unsigned inlen = ctou32(in); in+=4; in = mhdec(in, inlen, MH_BLK, sbuf, outlen); tpdec(sbuf, outlen, (unsigned char *)out,4); /*vsdec32(sbuf, n, out);*/ } break;// in = ansdecompress( in, outlen, sbuf);
#endif
// #if C_BITSHUFFLE
//case TB_VSIMPLE_S1: in = vsdec32(in, n, (unsigned *)sbuf); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,4); break;
// #endif
//case TB_VSIMPLE_S4: in = vsdec32(in, n, (unsigned *)sbuf); tp4dec(sbuf, _n, (unsigned char *)out,4); break;
//case TB_VSIMPLE_S8: in = vsdec32(in, n, (unsigned *)sbuf); tpdec( sbuf, _n, (unsigned char *)out,4); break;
#if C_LZ4
#if C_BITSHUFFLE
case LZ4_BIT: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,lev); break;
case LZ4_BITX: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,lev); bitxdec32(out, n, 0); break;
#endif
case LZ4_ : in += LZ4_decompress_fast((char *)in, (char *)out, outlen); break;
//case LZ4_NIBBLE: in += LZ4_decompress_fast((char *)in, (char *)out, outlen); tp4dec((unsigned char *)out, outlen, (unsigned char *)sbuf,lev); lodec64((uint64_t *)sbuf, (outlen+7)/8, 3, (uint64_t *)out); break;
case LZ4_NIBBLE: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tp4dec(sbuf, outlen, (unsigned char *)out,lev); break;
case LZ4_NIBBLEX:in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tp4dec(sbuf, outlen, (unsigned char *)out,lev); bitxdec32(out, n, 0); break;
case LZ4_BYTE: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tpdec( sbuf, outlen, (unsigned char *)out,lev); break;
//case LZ4_FP8: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); fpzdec64(sbuf, sbuf+outlen*2, outlen, (unsigned char *)out); break;
#endif
#if C_C_BLOSC
case P_BS_LZ:
case P_BS_LZ4:
case P_BS_ZLIB: { blosc_decompress(in, out, outlen); size_t nbytes, cbytes,blocksize; blosc_cbuffer_sizes(in, &nbytes, &cbytes, &blocksize); return in+cbytes; }
#endif
#if C_ZLIB
case P_ZLIB:
{ uLongf outsize = outlen; int l = *(unsigned *)in, rc = uncompress(sbuf, &outsize, in+4, l); in += 4 + l; tpdec(sbuf, outlen, (unsigned char *)out,4); } break;
#endif

87
ext/beplugds_.c
View File

@ -1,87 +0,0 @@
//-------- Externa libs
case P_COPY: return u32dec( (unsigned *)in, n, out);
case P_MCPY: memcpy(_out, in, _n); return _out+_n;
#if C_SIMPLE8B
case AM_SIMPLE8B: vbxget32(in, x);*out++ = x; --n; in = vs8bdec(in, n, out); bitdidec32(out, n, x+mdelta, mdelta); break;
#endif
#if C_FASTPFOR
case FP_VBYTE: in = vbytedec( in, n, out); bitdidec32(out, n, -mdelta, mdelta); break;
#endif
#if C_LIBFOR
case LF_FORX: return for_selectx(in, n, out); //{ out[0] = *(unsigned *)in, b = in[4]; return for_selectx(in+5, n-1, out+1, out[0], b); }
case LF_FOR: return in + for_uncompress(in, out, n);
#endif
#if C_MASKEDVBYTE
case P_MASKEDVBYTE: in += masked_vbyte_decode_delta(in, out, n, 0); break;
#endif
#if C_POLYCOM
case PC_VBYTE: in = vbpolydec( in, n, out); bitdidec32(out, n, -mdelta, mdelta); break;
case PC_SIMPLE16: vbxget32(in, x); *out = x; in = vs16dec((unsigned *)in, n-1, out+1); bitdidec32(out, n, -mdelta, mdelta); break;
case PC_OPTPFD: vbxget32(in, x); *out = x; in = optpfddec32(in,n-1,out+1); bitdidec32(out+1, n-1, x, mdelta); break;
/*if(n < 129) in = vbytedec(in, n, out);
else { _VBGET32(in, x, *out = x); unsigned all_array[2048]; in = (unsigned char *)detailed_p4_decode(out+1, (unsigned *)in, all_array); }
bitdidec32(out, n, -mdelta, mdelta); break;*/
#endif
#if C_QMX
case P_QMX: { vbxget32(in, x); *out = x; if(n>1) { unsigned l = *(unsigned *)in; JASS::compress_integer_qmx_improved qmx; qmx.decode(out+1, n-1, in+4, l); bitdidec32(out+1, n-1, x, mdelta); return in+=4+l; } return in; }
#endif
#if C_SIMDCOMP
case SC_SIMDPACK128:
if(n < 129) { in = vbytedec(in, n, out); bitdidec32(out, n, -mdelta, mdelta); }
else { vbxget32(in, x);*out = x; b = *in++; in = simdunpackd1n((uint32_t *)in, n-1, out+1, out[0], b); } break;
case SC_FOR: vbxget32(in, x);*out = x; b = *in++; if(n < 129) return (unsigned char *)simdunpackFOR_length(x, (const __m128i *)in, n-1, (unsigned *)(out+1), b); else { simdunpackFOR(x, (const __m128i *)in, (unsigned *)(out+1), b); return in + simdpackFOR_compressedbytes(n-1, b); }
case SC_FORDA: { vbxget32(in, x);*out++ = x; --n; b = *in++; unsigned i; for(i=0; i < n;i++) out[i] = simdselectFOR(x, (const __m128i *)in, b, i); return in + simdpackFOR_compressedbytes(n, b); }
#endif
#if C_STREAMVBYTE
case P_STREAMVBYTE: return in + streamvbyte_delta_decode(in, out, n, mdelta);
#endif
#if C_VARINTG8IU
case P_VARINTG8IU: in = vintg8dec( in, n, out); bitdidec32(out, n, -mdelta, mdelta); break;
#endif
#if C_VTENC
case P_VTENC: vtenc_list_decode_u32(in+4, ctou32(in), out, n); return in + outlen;
#endif
#if C_ZLIB
case P_ZLIB:
{ uLongf outsize = outlen; int l = *(unsigned *)in, rc = uncompress(sbuf, &outsize, in+4, l); in += 4 + l; tpdec(sbuf, outlen, (unsigned char *)out,4); bitdidec32(out, n, -mdelta, mdelta); } break;
#endif
#if C_C_BLOSC
case P_BS_LZ:
case P_BS_LZ4:
case P_BS_ZLIB: { blosc_decompress(in, out, outlen); size_t nbytes, cbytes,blocksize; blosc_cbuffer_sizes(in, &nbytes, &cbytes, &blocksize); return in+cbytes; }
#endif
#if C_LZTURBO
case P_LZT10: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lz8d(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); bitdidec32(out, n, -mdelta, mdelta); } break;
case P_LZT20:
case P_LZT22: { struct lzobj lz; lz.dstlen = outlen; lz.src = in; lz.dst = sbuf; lz.level = 0; in += lzbd(&lz); tpdec(sbuf, outlen, (unsigned char *)out,4); bitdidec32(out, n, -mdelta, mdelta); } break;
//case P_VSHUF: vbxget32(in, x);*out = x; in += ansdecompress( in, (n-1)*4, sbuf); vsdec32( sbuf, n-1, out+1); bitdidec32(out, n, -mdelta, mdelta); break;
case P_VSHUF: { unsigned inlen = ctou32(in); in+=4; in = mhdec(in, inlen, MH_BLK, sbuf, outlen); tpdec(sbuf, outlen, (unsigned char *)out,4); bitdidec32(out, n, -mdelta, mdelta); } break;
//case P_VSHUF: { vbxget32(in, x);*out = x; unsigned inlen = ctou32(in); in+=4; in = mhdec(in, inlen, MH_BLK, sbuf, outlen); vsdec32(sbuf, n-1, out+1); bitdidec32(out+1, n-1, x, mdelta); } break;
#endif
//---------- delta + transpose + lz77 ----------------------
#if C_LZ4
#if C_BITSHUFFLE
case LZ4_BIT: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,lev); bitdidec32(out, n, -mdelta, mdelta); break;
case LZ4_BITX: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); BITUNSHUFFLE(sbuf, outlen, (unsigned char *)out,lev); bitxdec32(out, n, mdelta); break;
#endif
case LZ4_ : in += LZ4_decompress_fast((char *)in, (char *)out, outlen); break;
case LZ4_NIBBLE:in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tp4dec(sbuf, outlen, (unsigned char *)out,lev); bitdidec32(out, n, -mdelta, mdelta); break;
case LZ4_NIBBLEX:in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tp4dec(sbuf, outlen, (unsigned char *)out,lev); bitxdec32(out, n, mdelta); break;
case LZ4_BYTE: in += LZ4_decompress_fast((char *)in, (char *)sbuf, outlen); tpdec( sbuf, outlen, (unsigned char *)out,lev); bitdidec32(out, n, -mdelta, mdelta); break;
#endif

12
ext/beplugi_.c
View File

@ -1,12 +0,0 @@
#if C_C_BLOSC
case P_BS_LZ4: blosc_init(); blosc_set_nthreads(1); blosc_set_compressor(BLOSC_LZ4_COMPNAME);break;
case P_BS_ZLIB: blosc_init(); blosc_set_nthreads(1); blosc_set_compressor(BLOSC_ZLIB_COMPNAME); break;
case P_BS_LZ: blosc_init(); blosc_set_nthreads(1); blosc_set_compressor(BLOSC_BLOSCLZ_COMPNAME); break;
#endif
#if C_VARINTG8IU
case P_VARINTG8IU: VarIntG8IU();
#endif
#ifdef LZTURBO
// #include "../beplug0.h"
#endif