diff --git a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java index 56ad1a7a9c..a606766d1a 100644 --- a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java +++ b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/Roaring64Map.java @@ -24,6 +24,7 @@ import org.roaringbitmap.BitmapDataProvider; import org.roaringbitmap.BitmapDataProviderSupplier; import org.roaringbitmap.IntConsumer; import org.roaringbitmap.IntIterator; +import org.roaringbitmap.InvalidRoaringFormat; import org.roaringbitmap.RoaringBitmap; import org.roaringbitmap.RoaringBitmapSupplier; import org.roaringbitmap.Util; @@ -1345,16 +1346,22 @@ public class Roaring64Map { this.clear(); highToBitmap = new TreeMap<>(); - long nbHighs = 1; - if (bitmapType == BitmapValue.BITMAP64) { - nbHighs = Codec.decodeVarint64(in); + if (bitmapType == BitmapValue.BITMAP32) { + RoaringBitmap provider = new RoaringBitmap(); + provider.deserialize(in); + highToBitmap.put(0, provider); + return; } + if (bitmapType != BitmapValue.BITMAP64) { + throw new InvalidRoaringFormat("invalid bitmap type"); + } + + long nbHighs = Codec.decodeVarint64(in); for (int i = 0; i < nbHighs; i++) { int high = in.readInt(); RoaringBitmap provider = new RoaringBitmap(); provider.deserialize(in); - highToBitmap.put(high, provider); } diff --git a/fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java b/fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java index d9c71e818c..e90e8851d3 100644 --- a/fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java +++ b/fe/spark-dpp/src/test/java/org/apache/doris/load/loadv2/dpp/BitmapValueTest.java @@ -324,13 +324,11 @@ public class BitmapValueTest { Assert.assertTrue(serializeSingleValueBitmapValue.equals(deserializeSingleValueBitmapValue)); // bitmap + // case 1 : 32-bit bitmap BitmapValue serializeBitmapBitmapValue = new BitmapValue(); for (int i = 0; i < 10; i++) { serializeBitmapBitmapValue.add(i); } - for (long i = Long.MAX_VALUE; i > Long.MAX_VALUE - 10; i--) { - serializeBitmapBitmapValue.add(i); - } ByteArrayOutputStream bitmapOutputStream = new ByteArrayOutputStream(); DataOutput bitmapOutput = new DataOutputStream(bitmapOutputStream); serializeBitmapBitmapValue.serialize(bitmapOutput); @@ -340,6 +338,23 @@ public class BitmapValueTest { deserializeBitmapBitmapValue.deserialize(bitmapInputStream); Assert.assertTrue(serializeBitmapBitmapValue.equals(deserializeBitmapBitmapValue)); + + + // bitmap + // case 2 : 64-bit bitmap + BitmapValue serializeBitmapBitmapValue64 = new BitmapValue(); + for (long i = Long.MAX_VALUE; i > Long.MAX_VALUE - 10; i--) { + serializeBitmapBitmapValue64.add(i); + } + ByteArrayOutputStream bitmapOutputStream64 = new ByteArrayOutputStream(); + DataOutput bitmapOutput64 = new DataOutputStream(bitmapOutputStream64); + serializeBitmapBitmapValue64.serialize(bitmapOutput64); + + DataInputStream bitmapInputStream64 = new DataInputStream(new ByteArrayInputStream(bitmapOutputStream64.toByteArray())); + BitmapValue deserializeBitmapBitmapValue64 = new BitmapValue(); + deserializeBitmapBitmapValue64.deserialize(bitmapInputStream64); + + Assert.assertTrue(serializeBitmapBitmapValue64.equals(deserializeBitmapBitmapValue64)); } @Test