180 lines
7.6 KiB
C
180 lines
7.6 KiB
C
/**
|
|
Copyright (C) powturbo 2013-2023
|
|
SPDX-License-Identifier: GPL v2 License
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 2 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License along
|
|
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
- homepage : https://sites.google.com/site/powturbo/
|
|
- github : https://github.com/powturbo
|
|
- twitter : https://twitter.com/powturbo
|
|
- email : powturbo [_AT_] gmail [_DOT_] com
|
|
**/
|
|
// idxcr.c - "Integer Compression" Create inverted index for using by idxqry for benchmarking
|
|
#define _LARGEFILE64_SOURCE 1
|
|
#define _FILE_OFFSET_BITS 64
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <stdio.h>
|
|
#include <math.h>
|
|
#include <limits.h>
|
|
#include <getopt.h>
|
|
#include <sys/stat.h>
|
|
#ifdef __APPLE__
|
|
#ifdef HAVE_MALLOC_MALLOC
|
|
#include <sys/malloc.h>
|
|
#else
|
|
#include <malloc/malloc.h>
|
|
#endif
|
|
#else
|
|
#include <malloc.h>
|
|
#endif
|
|
|
|
#include "include_/conf.h"
|
|
#include "include_/vint.h"
|
|
#include "include_/bitpack.h"
|
|
#include "include_/vp4.h"
|
|
#include "include_/vlcbyte.h"
|
|
#include "idx.h"
|
|
#ifndef min
|
|
#define min(x,y) (((x)<(y)) ? (x) : (y))
|
|
#endif
|
|
|
|
#ifdef NSIMD
|
|
#define bitpack128v32 bitpack32
|
|
#endif
|
|
|
|
//---------------------------------------------------------------------------------------------------------------
|
|
#define DELTA( __in, __n, __b) do { unsigned _v; for(__b=0,_v = __n-1; _v > 0; --_v) __in[_v] = (__in[_v] - __in[_v-1]) - 1, __b |= __in[_v]; __b = bsr32(__b); } while(0)
|
|
|
|
#define TERMNUM 2000000
|
|
int verb;
|
|
|
|
void usage() {
|
|
fprintf(stderr, "\nTurboPFor Copyright (c) 2013-2023 Powturbo %s\n", __DATE__);
|
|
fprintf(stderr, "https://github.com/powturbo/TurboPFor\n\n");
|
|
fprintf(stderr, "Create inverted index from 'Document identifier data set' format\n");
|
|
fprintf(stderr, "See http://lemire.me/data/integercompression2014.html'\n");
|
|
fprintf(stderr, "Usage: idxcr <docid file> <destination dir>\n");
|
|
fprintf(stderr, "ex. idxcr clueweb09.sorted idxdir\n\n");
|
|
fprintf(stderr, "ex. index partitions generated from idxseg\n\n");
|
|
fprintf(stderr, "ex. idxcr ./idxcr gov2.sorted.s* .\n\n");
|
|
exit(-1);
|
|
}
|
|
|
|
int main(int argc, char *argv[]) {
|
|
int fno,c, digit_optind = 0, this_option_optind = optind ? optind : 1, option_index = 0; char *path="";
|
|
static struct option long_options[] = { {"r", 0, 0, 'r'}, {0,0, 0, 0} };
|
|
for(;;) {
|
|
if((c = getopt_long(argc, argv, "xv:", long_options, &option_index)) == -1) break;
|
|
switch(c) {
|
|
case 0 : printf("Option %s", long_options[option_index].name);
|
|
if(optarg) printf (" with arg %s", optarg); printf ("\n"); break;
|
|
case 'v': verb = atoi(optarg); break;
|
|
default: die("unknown option: %c \n", optopt);
|
|
}
|
|
}
|
|
if(argc - optind < 2) usage();
|
|
path = argv[--argc];
|
|
|
|
for(fno = optind; fno < argc; fno++) {
|
|
char outname[257], *inname = argv[fno];
|
|
strcpy(outname, path);
|
|
char *p = strrchr(inname,'/');
|
|
if(!p) p = strrchr(inname,'\\'); if(!p) p=inname;
|
|
strcat(outname, p); strcat(outname,".i");
|
|
|
|
FILE *fi = fopen(inname, "rb"); if(!fi) { fprintf(stderr, "open error '%s'", inname); perror(inname); exit(-1); } int fdi = fileno(fi);
|
|
FILE *fo = fopen(outname,"wb"),*fm; if(!fo) { fprintf(stderr, "creat error '%s'", outname); perror(outname); exit(-1); } fprintf(stderr, "file='%s'", outname);
|
|
fseeko(fo, sizeof(unsigned)+sizeof(unsigned long long), SEEK_SET);
|
|
|
|
tmap_t *tmap = calloc(1, TERMNUM*sizeof(tmap_t));
|
|
if(!tmap) die("malloc error\n");
|
|
unsigned *in = NULL,*ip,*ep,num,tid=0,numx=0,outsize;
|
|
unsigned char *out = NULL;
|
|
unsigned long long fofs; unsigned long long postsize=0,skipsize = 0;
|
|
|
|
while(fread(&num, 1, 4, fi) == 4) { // read number of docid in term
|
|
if(!num) { ++tid; continue; }
|
|
unsigned bnum = (num+BLK_DIDNUM-1)/BLK_DIDNUM;
|
|
if(num > numx) {
|
|
numx = num;
|
|
in = realloc(in, num*4+64);
|
|
outsize = num*4+bnum*sizeof(unsigned)*SKIP_SIZE+1024;
|
|
out = realloc(out, outsize);
|
|
if(!in || !out) die("malloc err=%u", num);
|
|
}
|
|
if(fread(in, 4, num, fi) != num) break; // read docid list
|
|
|
|
unsigned char *op = out,*_op;
|
|
vbput32(op, num); // store f_t
|
|
|
|
unsigned *pix = (unsigned *)op;
|
|
if(num > BLK_DIDNUM) {
|
|
op += bnum*sizeof(unsigned)*SKIP_SIZE; skipsize += op-out;
|
|
}
|
|
for(_op = op, ip = in, ep = ip+num; ip < ep; ) {
|
|
unsigned n = min(ep-ip, BLK_DIDNUM), b = 0,bx; if(op+5*n > out+outsize) die("output buffer too small\n");
|
|
if(n > 1) {
|
|
#ifndef _TURBOPFOR
|
|
DELTA(ip, n, b); //bitdenc32( in+1, --n, pa, in[0], mode);
|
|
// #ifdef _TURBOPFOR
|
|
//b = _p4bits32(ip+1, n-1, &bx);
|
|
#endif
|
|
}
|
|
#ifdef SKIP_S
|
|
unsigned u = ip[0]<<SKIP_S|b; // save width
|
|
#else
|
|
unsigned u = ip[0]; // First docid
|
|
#endif
|
|
|
|
if(num > BLK_DIDNUM) { // skip/index. docid[0] and offset to compressed block
|
|
*pix = u;
|
|
#if SKIP_SIZE == 2
|
|
pix[bnum] = op - _op; // save posting offset
|
|
#endif
|
|
pix++;
|
|
} else vbput32(op, u); // skip not needed
|
|
|
|
if(n > 1) {
|
|
#ifdef _TURBOPFOR
|
|
op = n==129?p4d1enc128v32( ip+1, n-1, op, u):p4d1enc32( ip+1, n-1, op, u);//*op++ = bx; op = n==129?_p4enc128v32( ip+1, n-1, op, b, bx):_p4enc32( ip+1, n-1, op, b, bx);
|
|
#else
|
|
#ifndef SKIP_S
|
|
*op++ = b;
|
|
#endif
|
|
op = n==129?bitpack128v32( ip+1, n-1, op, b) :bitpack32(ip+1, n-1, op, b);
|
|
#endif
|
|
}
|
|
ip += n;
|
|
}
|
|
fofs = ftello(fo);
|
|
tmap_t *t = &tmap[tid++];
|
|
TIDMAPSET(t, fofs);
|
|
if(fwrite(out, 1, op-out, fo) != op-out) die("fwrite error\n"); postsize += op-out;
|
|
}
|
|
fofs = ftello(fo); // write termmap
|
|
if(fwrite(tmap, sizeof(tmap_t), tid, fo) != tid) die("fwrite error\n");
|
|
|
|
fseeko(fo, 0, SEEK_SET);
|
|
if(fwrite(&fofs, sizeof(unsigned long long), 1, fo) != 1) die("fwrite error\n");
|
|
if(fwrite(&tid, sizeof(unsigned), 1, fo) != 1) die("fwrite error\n"); fofs = ftello(fi);
|
|
fclose(fi); fclose(fo);
|
|
if(in) { free(in); free(out); }
|
|
free(tmap);
|
|
printf("\nterms=%u size=[tmap=%u skip=%llu post=%llu total=%llu, ratio=%.2f %%\n", tid, (unsigned)(tid*sizeof(tmap_t)),
|
|
skipsize, postsize-skipsize, postsize+tid*sizeof(tmap_t)+12, (double)postsize*100/(double)fofs );
|
|
}
|
|
}
|