daiduo/TurboPFor-Integer-Compression
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Inverted index: header for idxcr/idxqry

Browse Source
This commit is contained in:
x
2019-12-21 15:19:29 +01:00
parent 9d35489b29
commit 590da7331d
1 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
idx.h
View File

@ -1,7 +1,7 @@
/** /**
Copyright (C) powturbo 2013-2019 Copyright (C) powturbo 2013-2019
GPL v2 License GPL v2 License
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or the Free Software Foundation; either version 2 of the License, or
@ -21,28 +21,28 @@
- twitter : https://twitter.com/powturbo - twitter : https://twitter.com/powturbo
- email : powturbo [_AT_] gmail [_DOT_] com - email : powturbo [_AT_] gmail [_DOT_] com
**/ **/
// "Integer Compression" header for idxcr/idxqry // "Integer Compression" header for idxcr/idxqry
#include <stdint.h> #include <stdint.h>
#define BLK_DIDNUM (128+1) // Block size 128 + 1 (1 stored in skips) #define BLK_DIDNUM (128+1) // Block size 128 + 1 (1 stored in skips)
// compressed size for 62 GB clueweb09.sorted // compressed size for 62 GB clueweb09.sorted
// Defaut is bitpackv/bitunpackv 18 GB // Defaut is bitpackv/bitunpackv 18 GB
//#define _TURBOPFOR // for compact version 12 GB //#define _TURBOPFOR // for compact version 12 GB
#ifdef _TURBOPFOR #ifdef _TURBOPFOR
//#define SKIP_S 6 //#define SKIP_S 6
#define SKIP_SIZE 2 // always no implicit skip. #define SKIP_SIZE 2 // always no implicit skip.
#else #else
//#define SKIP_S 5 //#define SKIP_S 5
#define SKIP_SIZE 2 // no implicit skips #define SKIP_SIZE 2 // no implicit skips
//#define SKIP_SIZE 1 // implicit skips //#define SKIP_SIZE 1 // implicit skips
#endif #endif
#define SKIP_M ((1<<SKIP_S)-1) #define SKIP_M ((1<<SKIP_S)-1)
#ifdef _WIN32 #ifdef _WIN32
#define __off64_t _off64_t #define __off64_t _off64_t
#endif #endif
//-------------------------- Mapping term id <-> posting offset in file ---------------------------------- //-------------------------- Mapping term id <-> posting offset in file ----------------------------------
typedef struct { uint8_t offseth; uint32_t offsetl; } __attribute__ ((packed)) tmap_t; // 40 bits offsets -> 1 Terabyte typedef struct { uint8_t offseth; uint32_t offsetl; } __attribute__ ((packed)) tmap_t; // 40 bits offsets -> 1 Terabyte