diff --git a/idx.h b/idx.h index 18282f3..a70fd70 100644 --- a/idx.h +++ b/idx.h @@ -1,20 +1,52 @@ +/** + Copyright (C) powturbo 2013-2015 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + - homepage : https://sites.google.com/site/powturbo/ + - github : https://github.com/powturbo + - twitter : https://twitter.com/powturbo + - email : powturbo [_AT_] gmail [_DOT_] com +**/ + +// idx.h - "Integer Compression" header for idxcr/idxqry #include -#define BLK_DIDNUM (128+1) // // Block size 128 + 1 (1 stored in skips) +#define BLK_DIDNUM (128+1) // Block size 128 + 1 (1 stored in skips) + // compressed size for 62 GB clueweb09.sorted + // Defaut is bitpackv/bitunpackv 18 GB +//#define _TURBOPFOR // for compact version 12 GB -// Compression method. Set only one METHOD! - // compressed size for 62 GB clueweb09.sorted - // Defaut is bitpack/bitunpack 18 GB -#define USE_SIMDPACK // SIMD Bitpacking 18 GB -//#define USE_TURBOPFOR // for compact version 12 GB -//#define USE_TURBOPACKD + #ifdef _TURBOPFOR +#define SKIP_S 6 +#define SKIP_SIZE 2 // no implicit skip + #else +//#define SKIP_S 5 +#define SKIP_SIZE 2 // no implicit skips +//#define SKIP_SIZE 1 // implicit skips + #endif +#define SKIP_M ((1< posting offset in file ---------------------------------- typedef struct { uint8_t offseth; uint32_t offsetl; } __attribute__ ((packed)) tmap_t; // 40 bits offsets -> 1 Terabyte #define TIDMAPSET(__t, __ofs) { (__t)->offseth = (__ofs)>>32; (__t)->offsetl = (__ofs) & 0xffffffff; } #define TIDMAPGET(__t) ((__off64_t)(__t)->offseth << 32 | (__t)->offsetl) #define TIDMAP(__fdm, __tid) ({ char *_bp = __fdm; tmap_t *_t = (tmap_t *)&_bp[(__tid)*sizeof(tmap_t)]; TIDMAPGET(_t); }) -//-------------------------------------------------------------------------------------------------------- -