daiduo/TurboPFor-Integer-Compression
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix some bug to support java jni

Browse Source
This commit is contained in:
spcxtelsa
2020-09-25 17:59:14 +08:00
committed by spcxtesla
parent 4ab9f5b0e0
commit 302e5f1615
4 changed files with 60 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
java/jic.java
View File

@ -140,8 +140,8 @@ public class jic {
static {
try {
//System.loadLibrary("libic.so");
System.load("/home/x/TurboPFor/libic.so");
System.loadLibrary("ic");
// System.load("/home/x/TurboPFor/libic.so");
} catch (UnsatisfiedLinkError e) {
System.err.println("can't load Native TurboPFor from '" + System.getProperty("java.library.path") + "'" + e);
System.exit(1);

53
java/jicbench.java
View File

@ -23,9 +23,9 @@
**/
// icbench - "Integer Compression" Java Critical Native Interface
/* Usage: (actually no makefile available)
/* Usage: (make java/jicbench)
1 - generate header jic.h
$ cd ~/TurboPFor/java
$ cd TurboPFor/java
$ javah -jni jic
$ cp jic.h ..
@ -34,58 +34,51 @@ $ javac jic.java
$ javac jicbench.java
3 - compile & link a shared library
$ cd ~/TurboPFor
$ cd ..
$ gcc -O3 -w -march=native -fstrict-aliasing -m64 -shared -fPIC -I/usr/lib/jvm/default-java/include -I/usr/lib/jvm/default-java/include/linux bitpack.c bitunpack.c vp4c.c vp4d.c vsimple.c vint.c bitutil.c jic.c -o libic.so
$ Search "/usr/lib/" for the file "jni.h" and replace the JDK name "default-java" if necessary (example by "java-8-openjdk-amd64").
4 - copy "libic.so" to java library directory
5 - start icbench
$java icbench
5 - start jicbench
$java jicbench
*/
class jicbench {
// Note: this is a simple interface test not a real benchmark
public static void main(String args[]) {
jic ic = new jic();
final int[] in = new int[256];
final byte[] out = new byte[256*5];
final int[] cpy = new int[256];
int bnum = 12500;
for (int i = 0; i < 128; ++i) {
jic ic = new jic();
final int[] in = new int[bnum];
final byte[] out = new byte[bnum*4];
// final byte[] out = new byte[bnum*4*5/3+1024];
final int[] cpy = new int[bnum];
for (int i = 0; i < bnum; ++i) {
in[i] = i;
cpy[i] = 0;
}
long t0 = System.currentTimeMillis();
int b=0, bnum = 125000000; // 16 billions integers. 64 GB
for (int i = 0; i < bnum; ++i) {
//ic.vbenc32(in, 128, out);
//ic.vsenc32(in, 128, out);
//ic.p4nenc32(in, 128, out);
b = ic.bit32(in, 128);
ic.bitpack128v32( in, 128, out, b);
}
ic.p4nenc32(in, bnum, out);
long t = System.currentTimeMillis() - t0;
System.out.println("encode time'" + t + "'");
t0 = System.currentTimeMillis();
ic.p4ndec32(out, bnum, cpy);
t = System.currentTimeMillis() - t0;
System.out.println("decode time'" + t + "'");
for (int i = 0; i < bnum; ++i) {
//ic.vbenc32(out, 128, cpy);
//ic.vsenc32(out, 128, cpy);
//ic.p4nenc32(in, 128, out);
ic.bitunpack128v32(out, 128, cpy, b);
}
for (int i = 0; i < 128; ++i) {
if(in[i] != cpy[i]) {
System.err.println("Error at'" + i + "'");
System.exit(1);
}
}
t = System.currentTimeMillis() - t0;
System.out.println("decode time'" + t + "'");
System.out.println("check pass");
}
}

31
jic.c
View File

@ -24,6 +24,7 @@
// ic - "Integer Compression" Java Critical Native Interface / JNI
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include "bitutil.h"
#include "vint.h"
#include "vsimple.h"
@ -56,8 +57,8 @@
#define JNIEI (*env)->ReleasePrimitiveArrayCritical(env, _in, in, JNI_ABORT);
#define JNIBIT( __func, _in, n) JNIBI; jint l = __func ((unsigned *)in, n ); JNIEI; return l
#define JNIBITS(__func, _in, n, x) JNIBI; jint l = __func ((unsigned *)in, n, x); JNIEI; return l
#define JNIBIT( __func, _in, n) JNIBI; jint l = __func ((unsigned *)in, n, 0 ); JNIEI; return l
#define JNIBITS(__func, _in, n, x) JNIBI; jint l = __func ((unsigned *)in, n, 0, x); JNIEI; return l
//---------------------------------- variable byte ------------------------------------------------------------------------------------------
JNIEXPORT jint JNICALL Java_jic_vbenc32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNIENC(vbenc32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_vbenc32( jint i, jint *in, jint n, jint o, jbyte *out ) { return vbenc32( (unsigned *)in, n, (unsigned char *)out ) - (unsigned char *)out; }
@ -73,20 +74,20 @@ JNIEXPORT jint JNICALL Java_jic_vsenc32( JNIEnv *env, jclass cls, jin
JNIEXPORT jint JNICALL Java_jic_vsdec32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNIDEC(vsdec32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_vsdec32( jint i, jbyte *in, jint n, jint o, jint *out ) { return vsdec32( (unsigned char *)in, n, (unsigned *)out ) - (unsigned char *)in; }
//--------------------------------- TurboPFor: PFor/PForDelta -----------------------------------------------------------------
// High level API - n = unlimited
JNIEXPORT jint JNICALL Java_jic_p4nenc32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNNIENC(p4nenc32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nenc32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4nenc32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4ndec32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNNIDEC(p4ndec32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4ndec32( jint i, jbyte *in, jint n, jint o, jint *out ) { return p4ndec32( (unsigned char *)in, n, (unsigned *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nenc32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNINENC(p4nenc32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nenc32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4nenc32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4ndec32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNINDEC(p4ndec32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4ndec32( jint i, jbyte *in, jint n, jint o, jint *out ) { return p4ndec32( (unsigned char *)in, n, (unsigned *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nenc128v32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNNIENC(p4nenc128v32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nenc128v32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4nenc128v32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4ndec128v32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNNIDEC(p4ndec128v32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4ndec128v32( jint i, jbyte *in, jint n, jint o, jint *out ) { return p4ndec128v32( (unsigned char *)in, n, (unsigned *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nenc128v32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNINENC(p4nenc128v32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nenc128v32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4nenc128v32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4ndec128v32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNINDEC(p4ndec128v32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4ndec128v32( jint i, jbyte *in, jint n, jint o, jint *out ) { return p4ndec128v32( (unsigned char *)in, n, (unsigned *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nenc256v32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNNIENC(p4nenc256v32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nenc256v32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4nenc256v32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4ndec256v32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNNIDEC(p4ndec256v32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4ndec256v32( jint i, jbyte *in, jint n, jint o, jint *out ) { return p4ndec256v32( (unsigned char *)in, n, (unsigned *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nenc256v32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNINENC(p4nenc256v32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nenc256v32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4nenc256v32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4ndec256v32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNINDEC(p4ndec256v32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4ndec256v32( jint i, jbyte *in, jint n, jint o, jint *out ) { return p4ndec256v32( (unsigned char *)in, n, (unsigned *)out ); }
//--------
JNIEXPORT jint JNICALL Java_jic_p4ndenc32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNNIENC(p4ndenc32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4ndenc32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4ndenc32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nddec32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNNIDEC(p4nddec32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nddec32( jint i, jbyte *in, jint n, jint o, jint *out, jint start ) { return p4nddec32( (unsigned char *)in, n, (unsigned *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4ndenc32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNINENC(p4ndenc32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4ndenc32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4ndenc32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nddec32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNINDEC(p4nddec32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nddec32( jint i, jbyte *in, jint n, jint o, jint *out, jint start ) { return p4nddec32( (unsigned char *)in, n, (unsigned *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nd1enc32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNNIENC(p4nd1enc32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nd1enc32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4nd1enc32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nd1dec32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNNIDEC(p4nd1dec32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nd1dec32( jint i, jbyte *in, jint n, jint o, jint *out, jint start ) { return p4nd1dec32( (unsigned char *)in, n, (unsigned *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nd1enc32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNINENC(p4nd1enc32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nd1enc32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4nd1enc32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nd1dec32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNINDEC(p4nd1dec32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nd1dec32( jint i, jbyte *in, jint n, jint o, jint *out, jint start ) { return p4nd1dec32( (unsigned char *)in, n, (unsigned *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nzenc32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out ) { JNINENC(p4nzenc32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nzenc32( jint i, jint *in, jint n, jint o, jbyte *out ) { return p4nzenc32( (unsigned *)in, n, (unsigned char *)out ); }
JNIEXPORT jint JNICALL Java_jic_p4nzdec32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out ) { JNINDEC(p4nzdec32, _in, n, _out ); } JNIEXPORT jint JNICALL JavaCritical_jic_p4nzdec32( jint i, jbyte *in, jint n, jint o, jint *out, jint start ) { return p4nzdec32( (unsigned char *)in, n, (unsigned *)out ); }
@ -166,7 +167,7 @@ JNIEXPORT jint JNICALL Java_jic_bitdunpack256v32( JNIEnv *env, jclass cls, jbyt
JNIEXPORT jint JNICALL Java_jic_bitd1pack256v32( JNIEnv *env, jclass cls, jintArray _in, jint n, jbyteArray _out, jint start, jint b) { JNIENCSB(bitd1pack256v32, _in, n, _out, start, b); } JNIEXPORT jint JNICALL JavaCritical_jic_bitd1pack256v32( jint i, jint *in, jint n, jint o, jbyte *out, jint start, jint b) { return bitd1pack256v32( (unsigned *)in, n, (unsigned char *)out, start, b) - (unsigned char *)out; }
JNIEXPORT jint JNICALL Java_jic_bitd1unpack256v32( JNIEnv *env, jclass cls, jbyteArray _in, jint n, jintArray _out, jint start, jint b) { JNIDECSB(bitd1unpack256v32, _in, n, _out, start, b); } JNIEXPORT jint JNICALL JavaCritical_jic_bitd1unpack256v32( jint i, jbyte *in, jint n, jint o, jint *out, jint start, jint b) { return bitd1unpack256v32( (unsigned char *)in, n, (unsigned *)out, start, b) - (unsigned char *)in; }
//--------------------------------- bitutil --------------------------------------------------------------------------------------------------
JNIEXPORT jint JNICALL Java_jic_bit32( JNIEnv *env, jclass cls, jintArray _in, jint n ) { JNIBIT( bit32, _in, n ); } JNIEXPORT jint JNICALL JavaCritical_jic_bit32( jint i, jint *in, jint n ) { return bit32( (unsigned *)in, n ); }
JNIEXPORT jint JNICALL Java_jic_bitd32( JNIEnv *env, jclass cls, jintArray _in, jint n, jint start ) { JNIBITS(bitd32, _in, n, start ); } JNIEXPORT jint JNICALL JavaCritical_jic_bitd32( jint i, jint *in, jint n, jint start ) { return bitd32( (unsigned *)in, n, start ); }
JNIEXPORT jint JNICALL Java_jic_bitd132( JNIEnv *env, jclass cls, jintArray _in, jint n, jint start ) { JNIBITS(bitd132, _in, n, start ); } JNIEXPORT jint JNICALL JavaCritical_jic_bitd132( jint i, jint *in, jint n, jint start ) { return bitd132( (unsigned *)in, n, start ); }
JNIEXPORT jint JNICALL Java_jic_bit32( JNIEnv *env, jclass cls, jintArray _in, jint n ) { JNIBIT( bit32, _in, n ); } JNIEXPORT jint JNICALL JavaCritical_jic_bit32( jint i, jint *in, jint n ) { return bit32( (unsigned *)in, n, 0 ); }
JNIEXPORT jint JNICALL Java_jic_bitd32( JNIEnv *env, jclass cls, jintArray _in, jint n, jint start ) { JNIBITS(bitd32, _in, n, start ); } JNIEXPORT jint JNICALL JavaCritical_jic_bitd32( jint i, jint *in, jint n, jint start ) { return bitd32( (unsigned *)in, n, 0, start ); }
JNIEXPORT jint JNICALL Java_jic_bitd132( JNIEnv *env, jclass cls, jintArray _in, jint n, jint start ) { JNIBITS(bitd132, _in, n, start ); } JNIEXPORT jint JNICALL JavaCritical_jic_bitd132( jint i, jint *in, jint n, jint start ) { return bitd132( (unsigned *)in, n, 0, start ); }

21
makefile
View File

@ -15,12 +15,13 @@ CXX ?= g++
CL = $(CC)
#DEBUG=-DDEBUG -g
JAVA_HOME ?= /usr/lib/jvm/java-8-openjdk-amd64
PREFIX ?= /usr/local
DIRBIN ?= $(PREFIX)/bin
DIRINC ?= $(PREFIX)/include
DIRLIB ?= $(PREFIX)/lib
OPT=-fstrict-aliasing
OPT=-fstrict-aliasing -fPIC
ifeq (,$(findstring clang, $(CC)))
OPT+=-falign-loops
endif
@ -120,6 +121,24 @@ endif
libic.a: $(LIB)
ar cr $@ $+
libic.so : $(LIB)
$(CC) -shared $+ -o $@
JAVA_SUBDIR = java
jic.h: $(JAVA_SUBDIR)/jic.java
# cp jic.h ..
cd $(JAVA_SUBDIR) && javah -jni jic && cp jic.h ..
# $(JAVA_SUBDIR)/jic.class : $(JAVA_SUBDIR)/jic.java
# cd $(JAVA_SUBDIR) && javac jic.java
$(JAVA_SUBDIR)/libic.so : libic.a jic.h jic.c
$(CC) -shared $(CFLAGS) -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux jic.c libic.a -o $(JAVA_SUBDIR)/libic.so
$(JAVA_SUBDIR)/jicbench : $(JAVA_SUBDIR)/jicbench.java $(JAVA_SUBDIR)/libic.so
cd $(JAVA_SUBDIR) && javac jicbench.java && java -Djava.library.path=. jicbench
icapp: icapp.o libic.a $(OB)
$(CL) $^ $(LDFLAGS) -o icapp