diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java index 76d21072c8..e981c4d041 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java @@ -358,8 +358,9 @@ public class ColumnType { String keyValue = lowerCaseType.substring(4, lowerCaseType.length() - 1); int index = findNextNestedField(keyValue); if (index != keyValue.length() && index != 0) { - ColumnType keyType = parseType("key", keyValue.substring(0, index)); - ColumnType valueType = parseType("value", keyValue.substring(index + 1)); + ColumnType keyType = parseType("key", keyValue.substring(0, index).trim()); + ColumnType valueType = + parseType("value", keyValue.substring(index + 1).trim()); ColumnType mapType = new ColumnType(columnName, Type.MAP); mapType.setChildTypes(Arrays.asList(keyType, valueType)); return mapType; diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index a8783fe529..c55b80b6e8 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -20,6 +20,9 @@ package org.apache.doris.paimon; import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.ColumnValue; +import org.apache.paimon.data.DataGetters; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; import java.math.BigDecimal; @@ -32,12 +35,18 @@ import java.util.List; public class PaimonColumnValue implements ColumnValue { private int idx; - private InternalRow record; - ColumnType dorisType; + private DataGetters record; + private ColumnType dorisType; public PaimonColumnValue() { } + public PaimonColumnValue(DataGetters record, int idx, ColumnType columnType) { + this.idx = idx; + this.record = record; + this.dorisType = columnType; + } + public void setIdx(int idx, ColumnType dorisType) { this.idx = idx; this.dorisType = dorisType; @@ -130,12 +139,29 @@ public class PaimonColumnValue implements ColumnValue { @Override public void unpackArray(List values) { - + InternalArray recordArray = record.getArray(idx); + for (int i = 0; i < recordArray.size(); i++) { + PaimonColumnValue arrayColumnValue = new PaimonColumnValue((DataGetters) recordArray, i, + dorisType.getChildTypes().get(0)); + values.add(arrayColumnValue); + } } @Override public void unpackMap(List keys, List values) { - + InternalMap map = record.getMap(idx); + InternalArray key = map.keyArray(); + for (int i = 0; i < key.size(); i++) { + PaimonColumnValue keyColumnValue = new PaimonColumnValue((DataGetters) key, i, + dorisType.getChildTypes().get(0)); + keys.add(keyColumnValue); + } + InternalArray value = map.valueArray(); + for (int i = 0; i < value.size(); i++) { + PaimonColumnValue valueColumnValue = new PaimonColumnValue((DataGetters) value, i, + dorisType.getChildTypes().get(1)); + values.add(valueColumnValue); + } } @Override diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 4e3cda8222..524f05edd2 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -40,7 +40,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - public class PaimonJniScanner extends JniScanner { private static final Logger LOG = LoggerFactory.getLogger(PaimonJniScanner.class); private static final String PAIMON_OPTION_PREFIX = "paimon_option_prefix."; diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTableCache.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTableCache.java index caf1f156de..f57ffeb559 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTableCache.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTableCache.java @@ -34,7 +34,6 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; - public class PaimonTableCache { private static final Logger LOG = LoggerFactory.getLogger(PaimonTableCache.class); // Max cache num of paimon table @@ -83,7 +82,6 @@ public class PaimonTableCache { } } - public static class TableExt { private Table table; private long createTime; diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java index b2a9450a6d..575809c5df 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java @@ -45,6 +45,10 @@ import org.apache.paimon.types.VarCharType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /** * Convert paimon type to doris type. */ @@ -56,9 +60,11 @@ public class PaimonTypeUtils { public static ColumnType fromPaimonType(String columnName, DataType type) { PaimonColumnType paimonColumnType = type.accept(PaimonToDorisTypeVisitor.INSTANCE); - return new ColumnType(columnName, paimonColumnType.getType(), paimonColumnType.getLength(), + ColumnType columnType = new ColumnType(columnName, paimonColumnType.getType(), paimonColumnType.getLength(), paimonColumnType.getPrecision(), paimonColumnType.getScale()); + columnType.setChildTypes(paimonColumnType.getChildTypes()); + return columnType; } private static class PaimonToDorisTypeVisitor extends DataTypeDefaultVisitor { @@ -153,7 +159,10 @@ public class PaimonTypeUtils { @Override public PaimonColumnType visit(ArrayType arrayType) { - return this.defaultMethod(arrayType); + PaimonColumnType paimonColumnType = new PaimonColumnType(Type.ARRAY); + ColumnType elementColumnType = fromPaimonType("dummy-element", arrayType.getElementType()); + paimonColumnType.setChildTypes(Collections.singletonList(elementColumnType)); + return paimonColumnType; } @Override @@ -163,7 +172,11 @@ public class PaimonTypeUtils { @Override public PaimonColumnType visit(MapType mapType) { - return this.defaultMethod(mapType); + PaimonColumnType paimonColumnType = new PaimonColumnType(Type.MAP); + ColumnType key = fromPaimonType("dummy-key", mapType.getKeyType()); + ColumnType value = fromPaimonType("dummy-value", mapType.getValueType()); + paimonColumnType.setChildTypes(Arrays.asList(key, value)); + return paimonColumnType; } @Override @@ -184,6 +197,7 @@ public class PaimonTypeUtils { private int length; private int precision; private int scale; + private List childTypes; public PaimonColumnType(Type type) { this.type = type; @@ -225,5 +239,13 @@ public class PaimonTypeUtils { public void setPrecision(int precision) { this.precision = precision; } + + public void setChildTypes(List childTypes) { + this.childTypes = childTypes; + } + + public List getChildTypes() { + return childTypes; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java index 2ad593b1b3..679a784297 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java @@ -31,8 +31,10 @@ import org.apache.logging.log4j.Logger; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.AbstractFileStoreTable; import org.apache.paimon.table.Table; +import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.MapType; import java.util.HashMap; import java.util.List; @@ -111,6 +113,14 @@ public class PaimonExternalTable extends ExternalTable { case TIMESTAMP_WITHOUT_TIME_ZONE: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: return ScalarType.createDatetimeV2Type(PAIMON_DATETIME_SCALE_MS); + case ARRAY: + ArrayType arrayType = (ArrayType) dataType; + Type innerType = paimonPrimitiveTypeToDorisType(arrayType.getElementType()); + return org.apache.doris.catalog.ArrayType.create(innerType, true); + case MAP: + MapType mapType = (MapType) dataType; + return new org.apache.doris.catalog.MapType( + paimonTypeToDorisType(mapType.getKeyType()), paimonTypeToDorisType(mapType.getValueType())); case TIME_WITHOUT_TIME_ZONE: return Type.UNSUPPORTED; default: diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out index 4918db4555..bc1a15cb36 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out @@ -55,8 +55,8 @@ 1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 -- !c19 -- -11 22 aa bb cc 1 2 a b c +11 22 aa bb cc -- !c20 -- 1 2 a b c @@ -66,3 +66,21 @@ -- !c22 -- +-- !c23 -- +1 [1111, 2222, 3333] {"a_test":1} +2 [4444, 5555, 6666] {"b_test":0, "bbb":1} +3 [7777, 8888, 9999] {"c_test":1, "ccc":0} + +-- !c24 -- +1 [1111, 2222, 3333] {"a_test":1} + +-- !c25 -- +true \N \N \N +\N false true \N +\N \N \N false + +-- !c26 -- +3333 +6666 +9999 + diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy index ad72a47e64..0b9b35d7aa 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy @@ -72,10 +72,14 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_ def c15 = """select * from all_table where c15='a';""" def c16 = """select * from all_table where c16=true;""" def c18 = """select * from all_table where c18='2023-08-13 09:32:38.53';""" - def c19 = """select * from auto_bucket;""" + def c19 = """select * from auto_bucket order by user_id;""" def c20 = """select * from auto_bucket where dt="b";""" def c21 = """select * from auto_bucket where dt="b" and hh="c";""" def c22 = """select * from auto_bucket where dt="d";""" + def c23 = """select * from complex_tab order by c1;""" + def c24 = """select * from complex_tab where c1=1;""" + def c26 = """select array_max(c2) from complex_tab""" + def c25 = """select c3['a_test'], c3['b_test'], c3['bbb'], c3['ccc'] from complex_tab""" String hdfs_port = context.config.otherConfigs.get("hdfs_port") String catalog_name = "paimon1" @@ -111,5 +115,9 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_ qt_c20 c20 qt_c21 c21 qt_c22 c22 + qt_c23 c23 + qt_c24 c24 + qt_c25 c25 + qt_c26 c26 } }