database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug] Java Version BitmapValue deserialized failed when only has 32-bit bitmap (#4884)

Browse Source
This commit is contained in:
wangbo
2020-11-16 21:54:07 +08:00
committed by GitHub
parent e706a6bca4
commit 2af4bc294f
2 changed files with 29 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java
View File

@ -24,6 +24,7 @@ import org.roaringbitmap.BitmapDataProvider;
import org.roaringbitmap.BitmapDataProviderSupplier;
import org.roaringbitmap.IntConsumer;
import org.roaringbitmap.IntIterator;
import org.roaringbitmap.InvalidRoaringFormat;
import org.roaringbitmap.RoaringBitmap;
import org.roaringbitmap.RoaringBitmapSupplier;
import org.roaringbitmap.Util;
@ -1345,16 +1346,22 @@ public class Roaring64Map {
this.clear();
highToBitmap = new TreeMap<>();
long nbHighs = 1;
if (bitmapType == BitmapValue.BITMAP64) {
nbHighs = Codec.decodeVarint64(in);
if (bitmapType == BitmapValue.BITMAP32) {
RoaringBitmap provider = new RoaringBitmap();
provider.deserialize(in);
highToBitmap.put(0, provider);
return;
}
if (bitmapType != BitmapValue.BITMAP64) {
throw new InvalidRoaringFormat("invalid bitmap type");
}
long nbHighs = Codec.decodeVarint64(in);
for (int i = 0; i < nbHighs; i++) {
int high = in.readInt();
RoaringBitmap provider = new RoaringBitmap();
provider.deserialize(in);
highToBitmap.put(high, provider);
}

21
fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java
View File

@ -324,13 +324,11 @@ public class BitmapValueTest {
Assert.assertTrue(serializeSingleValueBitmapValue.equals(deserializeSingleValueBitmapValue));
// bitmap
// case 1 : 32-bit bitmap
BitmapValue serializeBitmapBitmapValue = new BitmapValue();
for (int i = 0; i < 10; i++) {
serializeBitmapBitmapValue.add(i);
}
for (long i = Long.MAX_VALUE; i > Long.MAX_VALUE - 10; i--) {
serializeBitmapBitmapValue.add(i);
}
ByteArrayOutputStream bitmapOutputStream = new ByteArrayOutputStream();
DataOutput bitmapOutput = new DataOutputStream(bitmapOutputStream);
serializeBitmapBitmapValue.serialize(bitmapOutput);
@ -340,6 +338,23 @@ public class BitmapValueTest {
deserializeBitmapBitmapValue.deserialize(bitmapInputStream);
Assert.assertTrue(serializeBitmapBitmapValue.equals(deserializeBitmapBitmapValue));
// bitmap
// case 2 : 64-bit bitmap
BitmapValue serializeBitmapBitmapValue64 = new BitmapValue();
for (long i = Long.MAX_VALUE; i > Long.MAX_VALUE - 10; i--) {
serializeBitmapBitmapValue64.add(i);
}
ByteArrayOutputStream bitmapOutputStream64 = new ByteArrayOutputStream();
DataOutput bitmapOutput64 = new DataOutputStream(bitmapOutputStream64);
serializeBitmapBitmapValue64.serialize(bitmapOutput64);
DataInputStream bitmapInputStream64 = new DataInputStream(new ByteArrayInputStream(bitmapOutputStream64.toByteArray()));
BitmapValue deserializeBitmapBitmapValue64 = new BitmapValue();
deserializeBitmapBitmapValue64.deserialize(bitmapInputStream64);
Assert.assertTrue(serializeBitmapBitmapValue64.equals(deserializeBitmapBitmapValue64));
}
@Test