hcq/TurboPFor-Integer-Compression
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IcBench: Benchmark App

Browse Source
This commit is contained in:
x
2017-01-29 14:09:46 +01:00
parent 4f6e37bcfd
commit 73e22cd8f7
1 changed files with 151 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

234
icbench.c
View File

@ -169,14 +169,22 @@ int strpref(char **str, int n, char sep1, char sep2) {
void memrcpy(unsigned char *out, unsigned char *in, unsigned n) { int i; for(i = 0; i < n; i++) out[i] = ~in[i]; } void memrcpy(unsigned char *out, unsigned char *in, unsigned n) { int i; for(i = 0; i < n; i++) out[i] = ~in[i]; }
int memcheck(unsigned char *in, unsigned n, unsigned char *cpy, int cmp) { int memcheck(unsigned char *_in, unsigned _n, unsigned char *_cpy, int cmp) {
unsigned *in = _in,*cpy=_cpy,n = (_n+3)/4;
int i; int i;
if(cmp <= 1) if(cmp <= 1)
return 0; return 0;
for(i = 0; i < n; i++) for(i = 0; i < n; i++)
if(in[i] != cpy[i]) { if(in[i] != cpy[i]) {
if(cmp > 3) abort(); // crash (AFL) fuzzing if(cmp > 4) abort(); // crash (AFL) fuzzing
printf("ERROR at %d:%x, %x\n", i, in[i], cpy[i]); printf("ERROR in[%d]=%x, dec[%d]=%x\n", i, in[i], i, cpy[i]);
if(cmp>3) {
int j;
for(j=i & 0xffffff80u; j < i+128;j++) { unsigned e = in[j] != cpy[j];
if(e) printf("#%d:%x,%x ", j, in[j], cpy[j]);else printf("%d:%x ", j, in[j]);
}
printf("\n");
}
if(cmp > 2) exit(EXIT_FAILURE); if(cmp > 2) exit(EXIT_FAILURE);
return i+1; return i+1;
} }
@ -237,16 +245,17 @@ void plugsprt(void) {
struct plugg *pg; struct plugg *pg;
printf("Codec group:\n"); printf("Codec group:\n");
for(pg = plugg; pg->id[0]; pg++) for(pg = plugg; pg->id[0]; pg++)
printf("%-16s %s %s\n", pg->id, pg->desc, pg->s); printf("%-16s %s %s\n", pg->id, pg->desc, pg->name);
printf("\nPlugins:\n"); printf("\n\n%-16s %-48s %s\n", "Function","Description", "level");
printf("\n%-16s %-48s %s\n", "--------","-----------", "-----");
for(gs = plugs; gs->id >= 0; gs++) for(gs = plugs; gs->id >= 0; gs++)
if(gs->codec) if(gs->codec)
{ printf("%s %s\n", gs->s, gs->lev?gs->lev:""); fflush(stdout);} printf("%-16s %-48s %s\n", gs->name, gs->desc?gs->desc:"", gs->lev?gs->lev:"");
} }
void plugsprtv(FILE *f, int fmt) { void plugsprtv(FILE *f, int fmt) {
struct plugs *gs; struct codecs *co;
char *pv = ""; char *pv = "";
switch(fmt) { switch(fmt) {
@ -259,25 +268,25 @@ void plugsprtv(FILE *f, int fmt) {
break; break;
} }
for(gs = plugs; gs->id >= 0; gs++) for(co = codecs; co->coid >= 0; co++)
if(gs->codec && strcmp(gs->name,pv)) { if(co->coid) {
pv = gs->name; pv = co->name;
char name[65],ver[33]; char name[65],ver[33];
ver[0] = 0; ver[0] = 0;
codver(gs->id, gs->ver, ver); codver(co->coid, co->ver, ver);
sprintf(name, "%s v%s", gs->name, ver); sprintf(name, "%s %c%s", co->name, co->ver?'v':' ', ver);
switch(fmt) { switch(fmt) {
case FMT_VBULLETIN: case FMT_VBULLETIN:
fprintf(f, "[*][URL=\"%s\"]%s[/URL] %s\n", gs->url, name, gs->lic ); fprintf(f, "[*][URL=\"%s\"]%s[/URL]\n", co->url, name );
break; break;
case FMT_HTML : case FMT_HTML :
fprintf(f, "<li><a href=\"%s\">%s</a> %s\n", gs->url, name, gs->lic ); fprintf(f, "<li><a href=\"%s\">%s</a>\n", co->url, name );
break; break;
case FMT_MARKDOWN : case FMT_MARKDOWN :
fprintf(f, " - [%s](%s) %s\n", name, gs->url, gs->lic ); fprintf(f, " - [%s](%s)\n", name, co->url);
break; break;
default: default:
fprintf(f, "%-24s\t%s\t%s\n", name, gs->lic, gs->url ); fprintf(f, "%-24s\t%s\n", name, co->url );
} }
} }
@ -294,7 +303,7 @@ void plugsprtv(FILE *f, int fmt) {
//------------------ plugin: process ---------------------------------- //------------------ plugin: process ----------------------------------
struct plug { struct plug {
int id,err,blksize,lev; int id,err,blksize,lev;
char *s,prm[17],tms[20]; char *name,prm[17],tms[20];
long long len,memc,memd; long long len,memc,memd;
double tc,td; double tc,td;
}; };
@ -313,7 +322,7 @@ int plugins(struct plug *plug, struct plugs *gs, int *pk, unsigned bsize, int bs
memset(p, 0, sizeof(struct plug)); memset(p, 0, sizeof(struct plug));
p->id = gs->id; p->id = gs->id;
p->err = 0; p->err = 0;
p->s = gs->s; p->name = gs->name;
p->lev = lev; p->lev = lev;
strncpy(p->prm, prm?prm:(char *)"", 16); strncpy(p->prm, prm?prm:(char *)"", 16);
p->prm[16] = 0; p->prm[16] = 0;
@ -368,7 +377,7 @@ int plugreg(struct plug *plug, char *cmd, int k, int bsize, int bsizex) {
if(!*name) if(!*name)
break; break;
for(gs = plugs; gs->id >= 0; gs++) for(gs = plugs; gs->id >= 0; gs++)
if(gs->codec && !strcasecmp(gs->s, name) ) { if(!strcasecmp(gs->name, name) ) {
char s[33],*q; char s[33],*q;
sprintf(s,"%d", lev); sprintf(s,"%d", lev);
found++; found++;
@ -378,9 +387,11 @@ int plugreg(struct plug *plug, char *cmd, int k, int bsize, int bsizex) {
} }
break; break;
} }
if(found<2 && !ignore) { if((found<2 || !gs->codec) && !ignore) {
if(!found) if(!found)
fprintf(stderr, "codec '%s' not found\n", name); fprintf(stderr, "codec '%s' not found\n", name);
else if(!gs->codec)
fprintf(stderr, "codec '%s' not compiled in icbench\n", name);
else if(lev<0) else if(lev<0)
fprintf(stderr, "level [%s] not specified for codec '%s'\n", gs->lev, name ); fprintf(stderr, "level [%s] not specified for codec '%s'\n", gs->lev, name );
else if(gs->lev && gs->lev[0]) else if(gs->lev && gs->lev[0])
@ -389,6 +400,8 @@ int plugreg(struct plug *plug, char *cmd, int k, int bsize, int bsizex) {
fprintf(stderr, "codec '%s' has no levels\n", name); fprintf(stderr, "codec '%s' has no levels\n", name);
exit(0); exit(0);
} }
while(isspace(*cmd)) while(isspace(*cmd))
cmd++; cmd++;
if(*cmd != ',' && (*cmd < '0' || *cmd > '9')) if(*cmd != ',' && (*cmd < '0' || *cmd > '9'))
@ -521,9 +534,9 @@ void plugprt(struct plug *plug, long long totinlen, char *finame, int fmt, FILE
tc = TMIS(totinlen,plug->tc), td = TMIS(totinlen,plug->td); tc = TMIS(totinlen,plug->tc), td = TMIS(totinlen,plug->td);
char name[65]; char name[65];
if(plug->lev >= 0) if(plug->lev >= 0)
sprintf(name, "%s%s %d%s", plug->err?"?":"", plug->s, plug->lev, plug->prm); sprintf(name, "%s%s %d%s", plug->err?"?":"", plug->name, plug->lev, plug->prm);
else else
sprintf(name, "%s%s%s", plug->err?"?":"", plug->s, plug->prm); sprintf(name, "%s%s%s", plug->err?"?":"", plug->name, plug->prm);
int c = 0, d = 0, n = 0; int c = 0, d = 0, n = 0;
switch(fmt) { switch(fmt) {
@ -624,9 +637,9 @@ void plugprtp(struct plug *plug, long long totinlen, char *finame, int fmt, int
int i; int i;
char name[65]; char name[65];
if(plug->lev>=0) if(plug->lev>=0)
sprintf(name, "%s%s%s%d%s", plug->err?"?":"", plug->s, fmt==FMT_MARKDOWN?"_":" ", plug->lev, plug->prm); sprintf(name, "%s%s%s%d%s", plug->err?"?":"", plug->name, fmt==FMT_MARKDOWN?"_":" ", plug->lev, plug->prm);
else else
sprintf(name, "%s%s%s", plug->err?"?":"", plug->s, plug->prm); sprintf(name, "%s%s%s", plug->err?"?":"", plug->name, plug->prm);
if(fmt == FMT_HTML) if(fmt == FMT_HTML)
fprintf(f, "<tr><td>%s</td>", name); fprintf(f, "<tr><td>%s</td>", name);
else else
@ -696,9 +709,9 @@ void plugplot(struct plug *plug, long long totinlen, int fmt, int speedup, char
int i; int i;
char name[65]; char name[65];
if(plug->lev>=0) if(plug->lev>=0)
sprintf(name, "%s%s_%d%s", plug->err?"?":"", plug->s, plug->lev, plug->prm); sprintf(name, "%s%s_%d%s", plug->err?"?":"", plug->name, plug->lev, plug->prm);
else else
sprintf(name, "%s%s%s", plug->err?"?":"", plug->s, plug->prm); sprintf(name, "%s%s%s", plug->err?"?":"", plug->name, plug->prm);
strcat(s,name); strcat(s,","); strcat(s,name); strcat(s,",");
fprintf(f, "var %s = { x: [", name); fprintf(f, "var %s = { x: [", name);
@ -742,7 +755,7 @@ int libcmp(const struct plug *e1, const struct plug *e2) {
} }
int libcmpn(const struct plug *e1, const struct plug *e2) { int libcmpn(const struct plug *e1, const struct plug *e2) {
int c = strcmp(e1->s, e2->s); int c = strcmp(e1->name, e2->name);
if(c < 0) if(c < 0)
return -1; return -1;
else if(c > 0) else if(c > 0)
@ -765,7 +778,7 @@ void plugplotc(struct plug *plug, int k, long long totinlen, int fmt, int speedu
if(g->id <= P_MCPY && !plotmcpy) if(g->id <= P_MCPY && !plotmcpy)
continue; continue;
else { else {
if(strcmp(g->s, name)) { if(strcmp(g->name, name)) {
if(name[0]) { if(name[0]) {
fprintf(f, "],\ny: ["); fprintf(f, "],\ny: [");
for(p = gs; p < g; p++) for(p = gs; p < g; p++)
@ -778,16 +791,16 @@ void plugplotc(struct plug *plug, int k, long long totinlen, int fmt, int speedu
txt[0] = 0; txt[0] = 0;
} }
gs = g; gs = g;
strcpy(name, g->s); strcpy(name, g->name);
fprintf(f, "var %s = {\n x: [", g->s); fprintf(f, "var %s = {\n x: [", g->name);
strcat(s, g->s); strcat(s, g->name);
} else { } else {
fprintf(f, ","); fprintf(f, ",");
strcat(txt, ","); strcat(txt, ",");
} }
if(g->lev >= 0) { if(g->lev >= 0) {
char ts[33]; char ts[33];
sprintf(ts, "'%s%s%d%s'", divxy>=2?"":g->s, divxy>=2?"":",", g->lev, g->prm); sprintf(ts, "'%s%s%d%s'", divxy>=2?"":g->name, divxy>=2?"":",", g->lev, g->prm);
strcat(txt, ts); strcat(txt, ts);
} }
double t = (speedup&1)?g->tc:g->td; double t = (speedup&1)?g->tc:g->td;
@ -903,9 +916,9 @@ int plugread(struct plug *plug, char *finame, long long *totinlen) {
p->prm[0]=0; p->prm[0]=0;
int i; int i;
for(i = 0; plugs[i].id >=0; i++) for(i = 0; plugs[i].id >=0; i++)
if(!strcmp(name, plugs[i].s)) { if(!strcmp(name, plugs[i].name)) {
p->s = plugs[i].s; p->name = plugs[i].name;
p->id = plugs[i].id; if(verbose>1) { fprintf(stdout, "%s\t%"PRId64"\t%"PRId64"\t%.6f\t%.6f\t%s\t%d%s\t%s\t%"PRId64"\t%"PRId64"\n", s, *totinlen, p->len, p->td, p->tc, p->s, p->lev, p->prm, p->tms, p->memc, p->memd); fflush(stdout); } p->id = plugs[i].id; if(verbose>1) { fprintf(stdout, "%s\t%"PRId64"\t%"PRId64"\t%.6f\t%.6f\t%s\t%d%s\t%s\t%"PRId64"\t%"PRId64"\n", s, *totinlen, p->len, p->td, p->tc, p->name, p->lev, p->prm, p->tms, p->memc, p->memd); fflush(stdout); }
p++; p++;
break; break;
} }
@ -915,15 +928,15 @@ int plugread(struct plug *plug, char *finame, long long *totinlen) {
} }
//----------------------------------- Benchmark ----------------------------------------------------------------------------- //----------------------------------- Benchmark -----------------------------------------------------------------------------
static int mcpy, mode, tincx, fuzz; static int be_mcpy, be_nblocks, fuzz;
int becomp(unsigned char *_in, unsigned _inlen, unsigned char *_out, unsigned outsize, unsigned bsize, int id, int lev, char *prm, int ifmt, CODCOMP codcomp) { int becomp(unsigned char *_in, unsigned _inlen, unsigned char *_out, unsigned outsize, unsigned bsize, int id, int lev, char *prm, int be_mindelta, CODCOMP codcomp) {
unsigned char *op,*oe = _out + outsize; if(!_inlen) return 0; unsigned char *op,*oe = _out + outsize; if(!_inlen) return 0;
if(ifmt >=0 && !mode) { if((be_mindelta == 0 || be_mindelta == 1) && !be_nblocks) {
unsigned *in = (unsigned *)_in,i; unsigned *in = (unsigned *)_in,i;
for(i = 1; i < _inlen/4; i++) for(i = 1; i < _inlen/4; i++)
if(in[i] < in[i-1]+ifmt) { if(in[i] < in[i-1]+be_mindelta) {
fprintf(stderr, "WARNING: IDs not sorted %d:%u,%u\n", i, in[i-1], in[i]);fflush(stderr); fprintf(stderr, "WARNING: IDs not sorted %d:%u,%u\n", i, in[i-1], in[i]);fflush(stderr);
break; break;
} }
@ -935,7 +948,7 @@ int becomp(unsigned char *_in, unsigned _inlen, unsigned char *_out, unsigned ou
unsigned char *in,*ip; unsigned char *in,*ip;
for(op = _out, in = _in; in < _in+_inlen; ) { for(op = _out, in = _in; in < _in+_inlen; ) {
unsigned inlen = _inlen,bs; unsigned inlen = _inlen,bs;
if(mode) { blknum++; if(be_nblocks) { blknum++;
inlen = ctou32(in); in+=4; inlen = ctou32(in); in+=4;
vbput32(op, inlen); // ctou32(op) = inlen; op+=4; vbput32(op, inlen); // ctou32(op) = inlen; op+=4;
inlen *= 4; if(in+inlen>_in+_inlen) die("FATAL buffer overflow error"); inlen *= 4; if(in+inlen>_in+_inlen) die("FATAL buffer overflow error");
@ -943,7 +956,7 @@ int becomp(unsigned char *_in, unsigned _inlen, unsigned char *_out, unsigned ou
for(ip = in, in += inlen; ip < in; ) { for(ip = in, in += inlen; ip < in; ) {
unsigned iplen = in - ip; iplen = min(iplen, bsize); // cnt++; csize += iplen; unsigned iplen = in - ip; iplen = min(iplen, bsize); // cnt++; csize += iplen;
op = codcomp(ip, iplen, op, oe-op, id, lev, prm, ifmt); op = codcomp(ip, iplen, op, oe-op, id, lev, prm, be_mindelta);
ip += iplen; ip += iplen;
if(op > _out+outsize) if(op > _out+outsize)
die("Compress overflow error %llu, %u in lib=%d\n", outsize, (int)(ptrdiff_t)(op - _out), id); die("Compress overflow error %llu, %u in lib=%d\n", outsize, (int)(ptrdiff_t)(op - _out), id);
@ -953,7 +966,7 @@ int becomp(unsigned char *_in, unsigned _inlen, unsigned char *_out, unsigned ou
return op - _out; return op - _out;
} }
int bedecomp(unsigned char *_in, int _inlen, unsigned char *_out, unsigned _outlen, unsigned bsize, int id, int lev, char *prm, int ifmt, CODDECOMP coddecomp) { int bedecomp(unsigned char *_in, int _inlen, unsigned char *_out, unsigned _outlen, unsigned bsize, int id, int lev, char *prm, int be_mindelta, CODDECOMP coddecomp) {
unsigned char *ip; unsigned char *ip;
TMDEF; TMDEF;
@ -961,7 +974,7 @@ int bedecomp(unsigned char *_in, int _inlen, unsigned char *_out, unsigned _outl
unsigned char *out,*op; unsigned char *out,*op;
for(ip = _in, out = _out; out < _out+_outlen;) { for(ip = _in, out = _out; out < _out+_outlen;) {
unsigned outlen=_outlen,bs; unsigned outlen=_outlen,bs;
if(mode) { if(be_nblocks) {
vbget32(ip, outlen); //outlen = ctou32(ip); ip += 4; vbget32(ip, outlen); //outlen = ctou32(ip); ip += 4;
ctou32(out) = outlen; out += 4; ctou32(out) = outlen; out += 4;
outlen *= 4; if(out+outlen >_out+_outlen) die("FATAL: decompress overflow output error %d ", outlen); outlen *= 4; if(out+outlen >_out+_outlen) die("FATAL: decompress overflow output error %d ", outlen);
@ -969,7 +982,7 @@ int bedecomp(unsigned char *_in, int _inlen, unsigned char *_out, unsigned _outl
for(op = out, out += outlen; op < out; ) { for(op = out, out += outlen; op < out; ) {
unsigned oplen = out - op; unsigned oplen = out - op;
oplen = min(oplen, bsize); oplen = min(oplen, bsize);
ip = coddecomp(ip, 0, op, oplen, id, lev, prm, ifmt); if(ip >_in+_inlen) die("FATAL inlen %d,%d ", _inlen, ip-(_in+_inlen)); ip = coddecomp(ip, 0, op, oplen, id, lev, prm, be_mindelta); if(ip-_in>_inlen) die("FATAL inlen %d,%d ", _inlen, ip-_in);
op += oplen; op += oplen;
} }
} }
@ -1008,17 +1021,18 @@ int getpagesize() {
unsigned mininlen; unsigned mininlen;
struct cod { CODCOMP comp; CODDECOMP decomp; }; struct cod { CODCOMP comp; CODDECOMP decomp; };
struct cod cods[] = { { codcomp, coddecomp }, {codcomps, coddecomps }}; struct cod cods[] = { { codcomp, coddecomp }, {codcomps, coddecomps }, {codcompz, coddecompz } };
int be_mindelta=-1,mdelta=0;
unsigned long long filen; unsigned long long filen;
size_t insizem; size_t insizem;
char name[65]; char name[65];
int ifmt=-1,mdelta=0;
unsigned long long plugbench(struct plug *plug, unsigned char *in, unsigned inlen, unsigned char *out, unsigned outsize, unsigned long long plugbench(struct plug *plug, unsigned char *in, unsigned inlen, unsigned char *out, unsigned outsize,
unsigned char *_cpy, unsigned bsize, struct plug *plugr, int tid, char *finame) { unsigned char *_cpy, unsigned bsize, struct plug *plugr, int tid, char *finame) {
double tc = 0.0, td = 0.0; double tc = 0.0, td = 0.0;
unsigned outlen = becomp(in, inlen, out, outsize, bsize, plug->id, plug->lev, plug->prm, ifmt, cods[ifmt<0?0:1].comp); int be_mode = be_mindelta<0?0:(be_mindelta>1?2:1);
unsigned outlen = becomp(in, inlen, out, outsize, bsize, plug->id, plug->lev, plug->prm, be_mindelta, cods[be_mode].comp);
plug->len += outlen; plug->len += outlen;
plug->tc += (tc += (double)tm_tm/(double)tm_rm); plug->tc += (tc += (double)tm_tm/(double)tm_rm);
if(!outlen) plug->tc = 0; if(!outlen) plug->tc = 0;
@ -1026,12 +1040,12 @@ unsigned long long plugbench(struct plug *plug, unsigned char *in, unsigned inle
if(cmp && outlen) { if(cmp && outlen) {
unsigned char *cpy = _cpy; unsigned char *cpy = _cpy;
if(_cpy != in) memrcpy(cpy, in, inlen); if(_cpy != in) memrcpy(cpy, in, inlen);
unsigned cpylen = bedecomp(out, outlen, cpy, inlen, bsize, plug->id, plug->lev, plug->prm, ifmt, cods[ifmt<0?0:1].decomp); unsigned cpylen = bedecomp(out, outlen, cpy, inlen, bsize, plug->id, plug->lev, plug->prm, be_mindelta, cods[be_mode].decomp);
td = (double)tm_tm/(double)tm_rm; if(verbose /*&& inlen == filen*/) { printf("%8.2f %-16s%s\n", TMIS(inlen,td), name, finame); } td = (double)tm_tm/(double)tm_rm; if(verbose /*&& inlen == filen*/) { printf("%8.2f %c%-16s%s\n", TMIS(inlen,td), outlen?' ':'?',name, finame); }
int e = memcheck(in, inlen, cpy, cmp); int e = memcheck(in, inlen, cpy, cmp);
plug->err = plug->err?plug->err:e; plug->err = plug->err?plug->err:e;
plug->td += td; plug->td += td;
} else if(verbose /*&& inlen == filen*/) { printf("%8.2f %-16s%s\n", 0.0, name, finame); } } else if(verbose /*&& inlen == filen*/) { printf("%8.2f %c%-16s%s\n", 0.0, outlen?' ':'?', name, finame); }
if(!outlen) plug->err++; if(!outlen) plug->err++;
return outlen; return outlen;
} }
@ -1088,21 +1102,21 @@ void zipfgen(unsigned *a, unsigned n, double alpha, unsigned x1, unsigned x2) {
// 0 1 2 3 4 5 6 7, 8 // 0 1 2 3 4 5 6 7, 8
enum { T_0, T_UINT8, T_UINT16, T_UINT24, T_UINT32, T_UINT40, T_UINT48, T_UINT56, T_UINT64, T_CHAR, T_TXT, T_TST }; enum { T_0, T_UINT8, T_UINT16, T_UINT24, T_UINT32, T_UINT40, T_UINT48, T_UINT56, T_UINT64, T_CHAR, T_TXT, T_TST };
int mdelta; int mdelta;
unsigned rm=0,rx=1<<20; unsigned rm=0,rx=255;
#define OVD (10*MB) #define OVD (10*MB)
#define IPUSH(in,n,isize, nmax,u) { if(n >= nmax) { nmax = nmax?(nmax << 1):(1<<20); in = realloc(in, nmax*isize+OVD); if(!in) die("malloc err=%u", nmax); }\ #define IPUSH(in,n,isize, nmax,u) { if(n >= nmax) { nmax = nmax?(nmax << 1):(1<<20); in = realloc(in, nmax*isize+OVD); if(!in) die("malloc err=%u", nmax); }\
ctou32(in+n*isize) = u; n++; \ ctou32(in+n*isize) = u; n++; if(verbose>5) printf("%u,",(unsigned)u );\
} }
unsigned befgen(unsigned char **_in, unsigned n, int fmt, unsigned isize, FILE *fi) { unsigned befgen(unsigned char **_in, unsigned n, int fmt, unsigned isize, FILE *fi, int kid, int skiph, int decs, int divs) {
unsigned char *in = *_in,*ip; unsigned nmax = 0; unsigned char *in = *_in,*ip; unsigned nmax = 0;
if(!fi) { printf("zipf alpha=%.2f range[%u..%u].n=%u\n ", a, rm, rx, n); if(!fi) { printf("zipf alpha=%.2f range[%u..%u].n=%u\n ", a, rm, rx, n);
in = malloc(n*isize+OVD); if(!in) die("malloc err=%u", nmax); in = malloc(n*isize+OVD); if(!in) die("malloc err=%u", nmax);
zipfgen((unsigned *)in, n, a, rm, rx); zipfgen((unsigned *)in, n, a, rm, rx);
int i;for(i = 1; i <= n; i++) xbits[bsr32(ctou32(in+i*4))]++; int i;for(i = 1; i <= n; i++) xbits[bsr32(ctou32(in+i*4))]++;
if(ifmt >= 0) { stprint("delta"); if(be_mindelta == 0 || be_mindelta == 1) { stprint("delta");
unsigned *ip = (unsigned *)in; int v; unsigned *ip = (unsigned *)in, v;
for(ip[0]=0,v = 1; v < n; v++) { for(ip[0]=0,v = 1; v < n; v++) {
ip[v] += ip[v-1] + ifmt; if(ip[v]>=(1u<<31)) die("overflow generating sorted array\n" ); ip[v] += ip[v-1] + be_mindelta; if(ip[v]>=(1u<<31)) die("overflow generating sorted array\n" );
} }
} else stprint(""); } else stprint("");
*_in = in; *_in = in;
@ -1112,22 +1126,45 @@ unsigned befgen(unsigned char **_in, unsigned n, int fmt, unsigned isize, FILE *
n = 0; //printf("fmt=%d,", fmt);fflush(stdout); n = 0; //printf("fmt=%d,", fmt);fflush(stdout);
#define LSIZE 1024 #define LSIZE 1024
char s[LSIZE+1]; char s[LSIZE+1];
if(skiph) {
fgets(s, LSIZE, fi); if(verbose>5) printf("skip first line\n");
}
double pre = decs?pow(10.0f,(float)decs):1;
pre /= divs;
switch(fmt) { switch(fmt) {
case T_TXT: if(verbose) printf("reading text file\n"); case T_TXT: if(verbose) printf("reading text file\n");
while(fgets(s, LSIZE, fi)) { while(fgets(s, LSIZE, fi)) {
s[strlen(s) - 1] = 0; //printf("%5s,", s); s[strlen(s) - 1] = 0;
unsigned long long u = strtoull(s, NULL, 10) - mdelta; unsigned char *p = s,*q;
int k;
for(k = 0; *p; ) {
while(*p && (*p < '0' || *p > '9') ) *p++;
unsigned long long u = strtoull(p, &q, 10) - mdelta;
if(*q == '.') {
u = round(strtod(p, &q)*pre) - mdelta;
}
p = q;
if(++k == kid) {
IPUSH(in,n,isize,nmax,u); IPUSH(in,n,isize,nmax,u);
break;
}
while(*p && (*p >= '0' && *p <= '9' || *p == '.') ) *p++;
}
} }
break; break;
case T_CHAR: case T_CHAR:
for(;;) { for(;;) {
char *p = s; char *p = s,*q;
int c; int c;
while((c = getc(fi)) >= '0' && c <= '9') while((c = getc(fi)) >= '0' && c <= '9')
if(p - s < LSIZE) *p++ = c; if(p - s < LSIZE) *p++ = c;
*p = 0; *p = 0;
unsigned long long u = strtoull(s, NULL, 10) - mdelta; unsigned long long u = strtoull(s, &p, 10) - mdelta;
if(*p == '.')
u = round(strtod(s, &q)*pre) - mdelta;
IPUSH(in,n,isize,nmax,u); IPUSH(in,n,isize,nmax,u);
if(c == EOF) break; if(c == EOF) break;
} }
@ -1170,8 +1207,13 @@ void usage(char *pgm) {
fprintf(stderr, " -D No process real-time priority setting\n"); fprintf(stderr, " -D No process real-time priority setting\n");
fprintf(stderr, "Check:\n"); fprintf(stderr, "Check:\n");
fprintf(stderr, " -C# #=0 compress only, #=1 ignore errors, #=2 exit on error, #=3 crash on error\n"); fprintf(stderr, " -C# #=0 compress only, #=1 ignore errors, #=2 exit on error, #=3 crash on error\n");
fprintf(stderr, " -fXs X = file format: 1=int8,2=int16,4=int32(4=default), t=text:one integer per line c=text:integers separated by non digit char\n"); fprintf(stderr, " -f[X[c][H]][.d][s] X = file format:\n");
fprintf(stderr, " s = sorted,S=sorted unique\n"); fprintf(stderr, " [1=int8], [2=int16], [4=int32(default)], [f:float] [d:double]\n");
fprintf(stderr, " t = text:one integer per line, c=column number in multiple columns line\n");
fprintf(stderr, " c = text:integers separated by non digit char\n");
fprintf(stderr, " . = decimal digits (default 2)\n");
fprintf(stderr, " H = skip first line\n");
fprintf(stderr, " s = sorted (increasing), S=sorted (strictly increasing, no duplicates), z=ZigZag\n");
// fprintf(stderr, " -z# check reading/writing outside bounds: #=1 compress, #=2 decompress, #3:both\n"); // fprintf(stderr, " -z# check reading/writing outside bounds: #=1 compress, #=2 decompress, #3:both\n");
fprintf(stderr, "Output:\n"); fprintf(stderr, "Output:\n");
fprintf(stderr, " -v# # = verbosity 0..3 (default 1)\n"); fprintf(stderr, " -v# # = verbosity 0..3 (default 1)\n");
@ -1196,6 +1238,7 @@ void usage(char *pgm) {
BEUSAGE; BEUSAGE;
fprintf(stderr, "ex. ./icbench file -eVBYTE/turbopfor ZIPF\n"); fprintf(stderr, "ex. ./icbench file -eVBYTE/turbopfor ZIPF\n");
fprintf(stderr, "ex. ./icbench -eTurboPFor -fS -r gov2.sorted\n"); fprintf(stderr, "ex. ./icbench -eTurboPFor -fS -r gov2.sorted\n");
fprintf(stderr, "ex. ./icbench -eTurboPFor -fS -ft3.1H ratings.csv \n");
exit(0); exit(0);
} }
@ -1215,6 +1258,7 @@ void printfile(char *finame, int xstdout, int fmt, char *rem) {
plugprts(plug, k, s, xstdout, totinlen, fmt, rem); plugprts(plug, k, s, xstdout, totinlen, fmt, rem);
} }
int be_rand=1;
void ftest(struct plug *plug, unsigned k,unsigned n, unsigned bsize) { void ftest(struct plug *plug, unsigned k,unsigned n, unsigned bsize) {
if(!n) if(!n)
n = 25000000; n = 25000000;
@ -1222,28 +1266,30 @@ void ftest(struct plug *plug, unsigned k,unsigned n, unsigned bsize) {
unsigned char *out = malloc(n*5+OVD); if(!out) die("malloc err=%u", n*5); unsigned char *out = malloc(n*5+OVD); if(!out) die("malloc err=%u", n*5);
unsigned *cpy = malloc(n*4+OVD),b,i; if(!cpy) die("malloc err=%u", n*4); unsigned *cpy = malloc(n*4+OVD),b,i; if(!cpy) die("malloc err=%u", n*4);
char s[33]; char s[33];
s[0] = 0; printf("bittest: %u-%u, n=%d\n", rm, rx, n); fflush(stdout); s[0] = 0; printf("bittest: %u-%u, n=%u\n", rm, rx, n); fflush(stdout);
for(b = rm; b <= min(rx,32); b++) { /*srand(time(NULL));*/ sprintf(s,"b=%d", b); for(b = rm; b <= min(rx,32); b++) {
for(i = 0; i < n; i++) srand(time(NULL)); sprintf(s,"b=%d", b);
in[i] = /*(unsigned)rand() &*/ ((1ull << b)-1); for(i = 0; i < n; i++) in[i] = (be_rand?rand():(uint32_t)(-1)) & ((1ull << b)-1);
if(ifmt >= 0) in[n-1] = ((1ull << b)-1);
if(be_mindelta == 0 || be_mindelta == 1)
for(in[0]=0,i = 1; i < n; i++) { for(in[0]=0,i = 1; i < n; i++) {
unsigned long long v = (unsigned long long)in[i-1] + in[i] + ifmt; if(v >= (1ull<<32)) die("overflow generating sorted array at %d,%x+%x->v=%llx\n", i, in[i], in[i-1], v ); unsigned long long v = (unsigned long long)in[i-1] + in[i] + be_mindelta; if(v >= (1ull<<32)) die("overflow generating sorted array at %d,%x+%x->v=%llx\n", i, in[i], in[i-1], v );
in[i] = v; in[i] = v;
} }
struct plug *p; struct plug *p;
for(p = plug; p < plug+k; p++) { for(p = plug; p < plug+k; p++) {
if(p->lev >= 0) if(p->lev >= 0)
sprintf(name, "%s %d%s", p->s, p->lev, p->prm); sprintf(name, "%s %d%s", p->name, p->lev, p->prm);
else else
sprintf(name, "%s%s", p->s, p->prm); sprintf(name, "%s%s", p->name, p->prm);
codini(n, p->id); codini(n, p->id);
unsigned pbsize = p->blksize?p->blksize:bsize; unsigned pbsize = p->blksize?p->blksize:bsize;
if(ifmt >= 0) if(be_mindelta >= 0)
pbsize += 4; // start stored w. variable byte for delta coding pbsize += 4; // start stored w. variable byte for delta coding
p->len = p->tc = p->td = 0; sprintf(s, "%d, %d", b, pbsize/4); p->len = p->tc = p->td = 0; sprintf(s, "%d, %d", b, pbsize/4);
unsigned outlen = plugbench(p, in, n*4, out, n*5, cpy, pbsize, plugr,tid, s); unsigned outlen = plugbench(p, in, n*4, out, n*5, cpy, pbsize, plugr,tid, s);
if(!outlen) die("Codec error or codec not available\n");
} }
} }
} }
@ -1255,7 +1301,7 @@ extern int _CRT_glob = 1;
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
int xstdout=-1,xstdin=-1; int xstdout=-1,xstdin=-1;
int recurse = 0, xplug = 0,tm_Repk=1,plot=-1,fmt=0,fno,merge=0,rprio=1,dfmt = 0; int recurse = 0, xplug = 0,tm_Repk=1,plot=-1,fmt=0,fno,merge=0,rprio=1,dfmt = 0,kid=1,skiph=0,decs=2,divs=1;
unsigned bsize = 128*4, bsizex=0, n=25000000; unsigned bsize = 128*4, bsizex=0, n=25000000;
unsigned long long filenmax = 0; unsigned long long filenmax = 0;
char *scmd = NULL,*trans=NULL,*beb=NULL,*rem="",s[2049]; char *scmd = NULL,*trans=NULL,*beb=NULL,*rem="",s[2049];
@ -1279,15 +1325,19 @@ int main(int argc, char* argv[]) {
case 'b': bsize = argtoi(optarg,1)*4; bsizex++; break; case 'b': bsize = argtoi(optarg,1)*4; bsizex++; break;
case 'B': filenmax = argtol(optarg); break; case 'B': filenmax = argtol(optarg); break;
case 'C': cmp = atoi(optarg); break; case 'C': cmp = atoi(optarg); break;
// case 'c': ifmt = atoi(optarg); break;
case 'd': mdelta = atoi(optarg); break; case 'd': mdelta = atoi(optarg); break;
case 'e': scmd = optarg; break; case 'e': scmd = optarg; break;
case 'f': { char *s = optarg; case 'f': { char *s = optarg;
if(*s=='c') { dfmt = T_CHAR; s++;} if(*s=='c') { dfmt = T_CHAR; s++;}
else if(*s=='t') { dfmt = T_TXT; s++;} else if(*s=='t') { dfmt = T_TXT; if(*++s > '0' && *s <= '9') { kid = s[0] - '0'; s++; } }
else if(*s=='u') { dfmt = T_TST; s++;} else if(*s=='u') { dfmt = T_TST; s++;}
else if(*s=='1' || *s=='2') { dfmt = s[0]-'0';s++;} //else if(*s=='f') { dfmt = T_FLT; s++;}
switch(*s) { case 's': ifmt = 0; break; case 'S': ifmt = 1;break; case 'z': ifmt = 2;break; } //else if(*s=='d') { dfmt = T_DBL; s++;}
else if(*s=='1' || *s=='2') { dfmt = s[0]-'0'; s++; }
if(*s == '.') { if(*++s >= '0' && *s <= '9') { decs = s[0] - '0'; s++; } }
if(*s == 'v') { divs = strtod(++s, &s); }
if(*s == 'H') { skiph++; s++; }
switch(*s) { case 's': be_mindelta = 0; break; case 'S': be_mindelta = 1;break; case 'z': be_mindelta = 2;break; }
} break; } break;
case 'F': fac = strtod(optarg, NULL); break; case 'F': fac = strtod(optarg, NULL); break;
// case 'f': fuzz = atoi(optarg); break; // case 'f': fuzz = atoi(optarg); break;
@ -1308,10 +1358,10 @@ int main(int argc, char* argv[]) {
case 'S': speedup = atoi(optarg); break; case 'S': speedup = atoi(optarg); break;
case 'l': xplug = atoi(optarg); break; case 'l': xplug = atoi(optarg); break;
case 'r': mode++; break; case 'r': be_nblocks++; break;
case 'o': xstdout++; break; case 'o': xstdout++; break;
case 'p': fmt = atoi(optarg); break; case 'p': fmt = atoi(optarg); break;
case 'P': mcpy++; break; case 'P': be_mcpy++; break;
case 'Q': divxy = atoi(optarg); case 'Q': divxy = atoi(optarg);
if(divxy>3) divxy=3; break; if(divxy>3) divxy=3; break;
case 'D': rprio=0; break; case 'D': rprio=0; break;
@ -1323,6 +1373,7 @@ int main(int argc, char* argv[]) {
case '2': ylog = ylog?0:1; break; case '2': ylog = ylog?0:1; break;
case '3': xlog2 = xlog2?0:1; break; case '3': xlog2 = xlog2?0:1; break;
case '4': ylog2 = ylog2?0:1; break; case '4': ylog2 = ylog2?0:1; break;
case 'R': be_rand=0; break;
#ifdef LZTURBO #ifdef LZTURBO
case 'c': beb = optarg; break; case 'c': beb = optarg; break;
#else #else
@ -1331,7 +1382,7 @@ int main(int argc, char* argv[]) {
case 'n': n = argtoi(optarg,1); break; case 'n': n = argtoi(optarg,1); break;
case 'm': rm = argtoi(optarg,1); break; case 'm': rm = argtoi(optarg,1); break;
case 'M': rx = argtoi(optarg,1); break; case 'M': rx = argtoi(optarg,1); break;
// case 'z': vstest64(atoi(optarg),rm,rx,n); break; //case 'z': vstest64(atoi(optarg),rm,rx,n); break;
BEOPT; BEOPT;
case 'h': case 'h':
default: default:
@ -1378,7 +1429,7 @@ int main(int argc, char* argv[]) {
for(i = 0; plugg[i].id[0]; i++) for(i = 0; plugg[i].id[0]; i++)
if(!strcmp(scmd, plugg[i].id)) { if(!strcmp(scmd, plugg[i].id)) {
strcat(s, "ON/"); strcat(s, "ON/");
strcat(s, plugg[i].s); strcat(s, plugg[i].name);
strcat(s, "OFF/"); strcat(s, "OFF/");
break; break;
} }
@ -1410,7 +1461,7 @@ int main(int argc, char* argv[]) {
finame = argvx[fno]; if(verbose > 1) printf("%s\n", finame); finame = argvx[fno]; if(verbose > 1) printf("%s\n", finame);
FILE *fi = NULL; FILE *fi = NULL;
if(!strcmp(finame,"ZIPF")) { sprintf(sfiname, "ZIPF%.2f_%u-%u", a,rm,rx); if(!strcmp(finame,"ZIPF")) { sprintf(sfiname, "ZIPF%.2f_%u-%u", a,rm,rx);
strcat(sfiname,sifmt[ifmt+1]); strcat(sfiname,sifmt[be_mindelta+1]);
finame = sfiname; finame = sfiname;
if(!dfmt) dfmt = T_UINT32; if(!dfmt) dfmt = T_UINT32;
} else { } else {
@ -1425,7 +1476,7 @@ int main(int argc, char* argv[]) {
unsigned char *_in = NULL; unsigned char *_in = NULL;
int inlen; int inlen;
if(dfmt) { if(dfmt) {
if(!_in) { filen = inlen = befgen(&_in, n, dfmt, 4, fi); if(!_in) { filen = inlen = befgen(&_in, n, dfmt, 4, fi, kid, skiph, decs, divs);
insize = min(filen,(1u<<MAP_BITS)); if(filen < mininlen) insize = mininlen; insize = min(filen,(1u<<MAP_BITS)); if(filen < mininlen) insize = mininlen;
insizem = (fuzz&3)?SIZE_ROUNDUP(insize, pagesize):(insize+INOVD); insizem = (fuzz&3)?SIZE_ROUNDUP(insize, pagesize):(insize+INOVD);
} }
@ -1451,13 +1502,13 @@ int main(int argc, char* argv[]) {
struct plug *p; struct plug *p;
for(p = plug; p < plug+k; p++) { for(p = plug; p < plug+k; p++) {
if(p->lev >= 0) if(p->lev >= 0)
sprintf(name, "%s %d%s", p->s, p->lev, p->prm); sprintf(name, "%s %d%s", p->name, p->lev, p->prm);
else else
sprintf(name, "%s%s", p->s, p->prm); sprintf(name, "%s%s", p->name, p->prm);
codini(insize, p->id); codini(insize, p->id);
unsigned pbsize = p->blksize?p->blksize:bsize; unsigned pbsize = p->blksize?p->blksize:bsize;
if(ifmt >= 0) if(be_mindelta >= 0)
pbsize += 4; // start stored w. variable byte for delta coding pbsize += 4; // start stored w. variable byte for delta coding
//printf("bsize=%d, ", pbsize); //printf("bsize=%d, ", pbsize);
p->len = p->tc = p->td = 0; blknum = 0; p->len = p->tc = p->td = 0; blknum = 0;
@ -1468,7 +1519,7 @@ int main(int argc, char* argv[]) {
} else { } else {
ftotinlen = 0; ftotinlen = 0;
fseek(fi, 0, SEEK_SET); fseek(fi, 0, SEEK_SET);
if(mode) { if(be_nblocks) {
unsigned char *ip = _in; unsigned num; unsigned char *ip = _in; unsigned num;
while(fread(&num, 1, 4, fi) == 4 && num) { //if(num < rm || num > rx) { fseeko(fi, num*4, SEEK_CUR); continue; } while(fread(&num, 1, 4, fi) == 4 && num) { //if(num < rm || num > rx) { fseeko(fi, num*4, SEEK_CUR); continue; }
num *= 4; num *= 4;
@ -1481,9 +1532,9 @@ int main(int argc, char* argv[]) {
ctou32(ip) = num/4; ip += 4; ctou32(ip) = num/4; ip += 4;
if(fread(ip, 1, num, fi) != num) if(fread(ip, 1, num, fi) != num)
break; break;
if(ifmt >=0) { if(be_mindelta >=0) {
unsigned *p = (unsigned *)ip,i; unsigned *p = (unsigned *)ip,i;
for(i = 1; i < num/4; i++) if(p[i] < p[i-1]+ifmt) { printf("Warning: IDs not sorted :%d:%d,%d\n", i, p[i-1], p[i] );break; } for(i = 1; i < num/4; i++) if(p[i] < p[i-1]+be_mindelta) { printf("Warning: IDs not sorted :%d:%d,%d\n", i, p[i-1], p[i] );break; }
} }
ip += num; ip += num;
} }
@ -1550,7 +1601,7 @@ int main(int argc, char* argv[]) {
fprintf(fo, "dataset\tsize\tcsize\tdtime\tctime\tcodec\tlevel\tparam\tcmem\tdmem\ttime\n"); fprintf(fo, "dataset\tsize\tcsize\tdtime\tctime\tcodec\tlevel\tparam\tcmem\tdmem\ttime\n");
for(p = plug; p < plug+k; p++) { for(p = plug; p < plug+k; p++) {
for(g = pluga; g < pluga+gk; g++) for(g = pluga; g < pluga+gk; g++)
if(g->id >= 0 && !strcmp(g->s, p->s) && g->lev == p->lev && !strcmp(g->prm, p->prm)) { if(g->id >= 0 && !strcmp(g->name, p->name) && g->lev == p->lev && !strcmp(g->prm, p->prm)) {
if(g->len == p->len) { if(g->len == p->len) {
int u=0; int u=0;
if(g->td < p->td || p->td < 0.01) if(g->td < p->td || p->td < 0.01)
@ -1564,12 +1615,13 @@ int main(int argc, char* argv[]) {
g->id = -1; g->id = -1;
break; break;
} }
fprintf(fo, "%s\t%"PRId64"\t%"PRId64"\t%.6f\t%.6f\t%s\t%d\t%s\t%"PRId64"\t%"PRId64"\t%s\n", finame, totinlen, p->len, p->td, p->tc, p->s, p->lev, p->prm[0]?p->prm:"?", p->memc, p->memd, p->tms[0]?p->tms:tms); fprintf(fo, "%s\t%"PRId64"\t%"PRId64"\t%.6f\t%.6f\t%s\t%d\t%s\t%"PRId64"\t%"PRId64"\t%s\n", finame, totinlen, p->len, p->td, p->tc, p->name, p->lev, p->prm[0]?p->prm:"?", p->memc, p->memd, p->tms[0]?p->tms:tms);
} }
for(g = pluga; g < pluga+gk; g++) for(g = pluga; g < pluga+gk; g++)
if(g->id >= 0) if(g->id >= 0)
fprintf(fo, "%s\t%"PRId64"\t%"PRId64"\t%.6f\t%.6f\t%s\t%d\t%s\t%"PRId64"\t%"PRId64"\t%s\n", finame, totinlen, g->len, g->td, g->tc, g->s, g->lev, g->prm[0]?g->prm:"?", g->memc, g->memd, g->tms[0]?g->tms:tms); fprintf(fo, "%s\t%"PRId64"\t%"PRId64"\t%.6f\t%.6f\t%s\t%d\t%s\t%"PRId64"\t%"PRId64"\t%s\n", finame, totinlen, g->len, g->td, g->tc, g->name, g->lev, g->prm[0]?g->prm:"?", g->memc, g->memd, g->tms[0]?g->tms:tms);
fclose(fo); fclose(fo);
printfile(s, 0, FMT_TEXT, rem); printfile(s, 0, FMT_TEXT, rem);
} }
} }