From 67dc68630ba08a7c4ce66ba7e68f84f5c4d16ff5 Mon Sep 17 00:00:00 2001 From: amory Date: Fri, 19 May 2023 17:43:36 +0800 Subject: [PATCH] [Improve](complex-type)improve array/map/struct creating and function with decimalv3 (#19830) --- .../java/org/apache/doris/catalog/Type.java | 41 +++++++++++++++++++ .../doris/analysis/FunctionCallExpr.java | 31 ++++++++++---- .../org/apache/doris/catalog/Function.java | 2 +- .../test_array_with_scale_type.out | 4 +- 4 files changed, 67 insertions(+), 11 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index 3de0eadea6..48228a6c06 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -199,6 +199,9 @@ public abstract class Type { mapSubTypes.add(FLOAT); mapSubTypes.add(DOUBLE); mapSubTypes.add(DECIMALV2); + mapSubTypes.add(DECIMAL32); // same DEFAULT_DECIMALV3 + mapSubTypes.add(DECIMAL64); + mapSubTypes.add(DECIMAL128); mapSubTypes.add(DATE); mapSubTypes.add(DATETIME); mapSubTypes.add(DATEV2); @@ -214,6 +217,9 @@ public abstract class Type { structSubTypes.add(FLOAT); structSubTypes.add(DOUBLE); structSubTypes.add(DECIMALV2); + structSubTypes.add(DECIMAL32); // same DEFAULT_DECIMALV3 + structSubTypes.add(DECIMAL64); + structSubTypes.add(DECIMAL128); structSubTypes.add(DATE); structSubTypes.add(DATETIME); structSubTypes.add(DATEV2); @@ -341,6 +347,41 @@ public abstract class Type { return isScalarType(PrimitiveType.DECIMALV2); } + public boolean typeContainsPrecision() { + if (PrimitiveType.typeWithPrecision.contains(this.getPrimitiveType())) { + return true; + } else if (isStructType()) { + for (StructField field : ((StructType) this).getFields()) { + if (PrimitiveType.typeWithPrecision.contains(field.getType().getPrimitiveType())) { + return true; + } + } + } else if (isMapType()) { + return PrimitiveType.typeWithPrecision.contains(((MapType) this).getKeyType().getPrimitiveType()) + || PrimitiveType.typeWithPrecision.contains(((MapType) this).getValueType().getPrimitiveType()); + } else if (isArrayType()) { + return PrimitiveType.typeWithPrecision.contains(((ArrayType) this).getItemType().getPrimitiveType()); + } + return false; + } + + public boolean isDecimalV3OrContainsDecimalV3() { + if (isDecimalV3()) { + return true; + } else if (isStructType()) { + for (StructField field : ((StructType) this).getFields()) { + if (field.getType().isDecimalV3()) { + return true; + } + } + } else if (isMapType()) { + return ((MapType) this).getKeyType().isDecimalV3() || ((MapType) this).getValueType().isDecimalV3(); + } else if (isArrayType()) { + return ((ArrayType) this).getItemType().isDecimalV3(); + } + return false; + } + public boolean isDecimalV3() { return isScalarType(PrimitiveType.DECIMAL32) || isScalarType(PrimitiveType.DECIMAL64) || isScalarType(PrimitiveType.DECIMAL128); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index df18e17880..f138426a1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1654,7 +1654,7 @@ public class FunctionCallExpr extends Expr { && ((ArrayType) args[ix]).getItemType().isDecimalV3()))) { continue; } else if (!argTypes[i].matchesType(args[ix]) - && (!fn.getReturnType().isDecimalV3() + && (!fn.getReturnType().isDecimalV3OrContainsDecimalV3() || (argTypes[i].isValid() && !argTypes[i].isDecimalV3() && args[ix].isDecimalV3()))) { uncheckedCastChild(args[ix], i); } @@ -1748,14 +1748,28 @@ public class FunctionCallExpr extends Expr { this.type = children.get(1).getType(); } } else if (fnName.getFunction().equalsIgnoreCase("named_struct")) { - List fieldNames = Lists.newArrayList(); - for (int i = 0; i < children.size(); i++) { - if ((i & 1) == 0) { - StringLiteral nameLiteral = (StringLiteral) children.get(i); - fieldNames.add(nameLiteral.getStringValue()); + ArrayList newFields = Lists.newArrayList(); + ArrayList originFields = ((StructType) type).getFields(); + for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) { + Type fieldType = originFields.get(i + i >> 2).getType(); + if (fieldType.isDecimalV3() || fieldType.isDatetimeV2()) { + fieldType = children.get(i + 1).type; } + StringLiteral nameLiteral = (StringLiteral) children.get(i); + newFields.add(new StructField(nameLiteral.getStringValue(), fieldType)); } - this.type = ((StructType) type).replaceFieldsWithNames(fieldNames); + this.type = new StructType(newFields); + } else if (fnName.getFunction().equalsIgnoreCase("struct")) { + ArrayList newFields = Lists.newArrayList(); + ArrayList originFields = ((StructType) type).getFields(); + for (int i = 0; i < children.size(); i++) { + Type fieldType = originFields.get(i).getType(); + if (originFields.get(i).getType().isDecimalV3() || originFields.get(i).getType().isDatetimeV2()) { + fieldType = children.get(i).type; + } + newFields.add(new StructField(fieldType)); + } + this.type = new StructType(newFields); } else if (fnName.getFunction().equalsIgnoreCase("array_distinct") || fnName.getFunction() .equalsIgnoreCase("array_remove") || fnName.getFunction().equalsIgnoreCase("array_sort") || fnName.getFunction().equalsIgnoreCase("array_reverse_sort") @@ -1772,7 +1786,8 @@ public class FunctionCallExpr extends Expr { || fnName.getFunction().equalsIgnoreCase("array_shuffle") || fnName.getFunction().equalsIgnoreCase("shuffle") || fnName.getFunction().equalsIgnoreCase("array_except") - || fnName.getFunction().equalsIgnoreCase("array_concat")) { + || fnName.getFunction().equalsIgnoreCase("array_concat") + || fnName.getFunction().equalsIgnoreCase("array_apply")) { if (children.size() > 0) { this.type = children.get(0).getType(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 0ae8247579..f4d65bc593 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -523,7 +523,7 @@ public class Function implements Writable { } // For types with different precisions and scales, return type only indicates a type with default // precision and scale so we need to transform it to the correct type. - if (PrimitiveType.typeWithPrecision.contains(realReturnType.getPrimitiveType())) { + if (realReturnType.typeContainsPrecision()) { fn.setRetType(realReturnType.toThrift()); } else { fn.setRetType(getReturnType().toThrift()); diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out index 56a04d45bb..294fe7f226 100644 --- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out @@ -91,8 +91,8 @@ [] -- !select -- -[2022-12-01 22:23:24, 2022-12-01 23:23:24] -[2022-12-02 22:23:24, 2022-12-02 23:23:24] +[2022-12-01 22:23:24.999, 2022-12-01 23:23:24.999] +[2022-12-02 22:23:24.999, 2022-12-02 23:23:24.999] -- !select -- \N