From e02747e9761aec784e1a0aed8a60e6d489b3ba8c Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Tue, 29 Aug 2023 20:41:24 +0800 Subject: [PATCH] [feature](Nereids) support struct type (#23597) 1. support struct data type 2. add array / map / struct literal syntax 3. fix array union / intersect / except type coercion 4. fix explict cast data type check for array 5. fix bound function type coercion --- .../apache/doris/catalog/PrimitiveType.java | 22 +++- .../org/apache/doris/nereids/DorisLexer.g4 | 4 +- .../org/apache/doris/nereids/DorisParser.g4 | 12 +- .../doris/catalog/BuiltinScalarFunctions.java | 6 + .../nereids/parser/LogicalPlanBuilder.java | 46 +++++++- .../rules/expression/check/CheckCast.java | 22 ++-- .../expression/rules/FunctionBinder.java | 18 +++ .../nereids/rules/rewrite/CheckDataTypes.java | 8 +- .../nereids/trees/expressions/Expression.java | 15 ++- .../functions/ComputeSignature.java | 7 +- .../functions/ComputeSignatureHelper.java | 18 ++- .../functions/scalar/ArrayExcept.java | 3 +- .../functions/scalar/ArrayIntersect.java | 3 +- .../functions/scalar/ArrayUnion.java | 3 +- .../functions/scalar/CreateMap.java | 4 +- .../functions/scalar/CreateNamedStruct.java | 105 ++++++++++++++++++ .../functions/scalar/CreateStruct.java | 77 +++++++++++++ .../functions/scalar/StructElement.java | 99 +++++++++++++++++ .../literal/StringLikeLiteral.java | 5 + .../expressions/literal/StringLiteral.java | 5 - .../expressions/literal/VarcharLiteral.java | 5 - .../visitor/ScalarFunctionVisitor.java | 17 +++ .../apache/doris/nereids/types/DataType.java | 7 +- .../doris/nereids/types/StructField.java | 104 +++++++++++++++++ .../doris/nereids/types/StructType.java | 65 +++++++++-- .../doris/nereids/util/TypeCoercionUtils.java | 66 ++++++++++- .../doris/nereids/UnsupportedTypeTest.java | 8 +- .../test_complex_type.groovy | 63 +++++++++++ 28 files changed, 743 insertions(+), 74 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateStruct.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java create mode 100644 regression-test/suites/nereids_syntax_p0/test_complex_type.groovy diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index d35750ee69..50fddda0f3 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -169,6 +169,8 @@ public enum PrimitiveType { builder.put(TINYINT, DECIMAL128); builder.put(TINYINT, VARCHAR); builder.put(TINYINT, STRING); + builder.put(TINYINT, TIME); + builder.put(TINYINT, TIMEV2); // Smallint builder.put(SMALLINT, BOOLEAN); builder.put(SMALLINT, TINYINT); @@ -188,6 +190,8 @@ public enum PrimitiveType { builder.put(SMALLINT, DECIMAL128); builder.put(SMALLINT, VARCHAR); builder.put(SMALLINT, STRING); + builder.put(SMALLINT, TIME); + builder.put(SMALLINT, TIMEV2); // Int builder.put(INT, BOOLEAN); builder.put(INT, TINYINT); @@ -207,6 +211,8 @@ public enum PrimitiveType { builder.put(INT, DECIMAL128); builder.put(INT, VARCHAR); builder.put(INT, STRING); + builder.put(INT, TIME); + builder.put(INT, TIMEV2); // Bigint builder.put(BIGINT, BOOLEAN); builder.put(BIGINT, TINYINT); @@ -226,6 +232,8 @@ public enum PrimitiveType { builder.put(BIGINT, DECIMAL128); builder.put(BIGINT, VARCHAR); builder.put(BIGINT, STRING); + builder.put(BIGINT, TIME); + builder.put(BIGINT, TIMEV2); // Largeint builder.put(LARGEINT, BOOLEAN); builder.put(LARGEINT, TINYINT); @@ -245,6 +253,8 @@ public enum PrimitiveType { builder.put(LARGEINT, DECIMAL128); builder.put(LARGEINT, VARCHAR); builder.put(LARGEINT, STRING); + builder.put(LARGEINT, TIME); + builder.put(LARGEINT, TIMEV2); // Float builder.put(FLOAT, BOOLEAN); builder.put(FLOAT, TINYINT); @@ -264,6 +274,8 @@ public enum PrimitiveType { builder.put(FLOAT, DECIMAL128); builder.put(FLOAT, VARCHAR); builder.put(FLOAT, STRING); + builder.put(FLOAT, TIME); + builder.put(FLOAT, TIMEV2); // Double builder.put(DOUBLE, BOOLEAN); builder.put(DOUBLE, TINYINT); @@ -283,6 +295,8 @@ public enum PrimitiveType { builder.put(DOUBLE, DECIMAL128); builder.put(DOUBLE, VARCHAR); builder.put(DOUBLE, STRING); + builder.put(DOUBLE, TIME); + builder.put(DOUBLE, TIMEV2); // Date builder.put(DATE, BOOLEAN); builder.put(DATE, TINYINT); @@ -379,6 +393,8 @@ public enum PrimitiveType { builder.put(CHAR, DECIMAL128); builder.put(CHAR, VARCHAR); builder.put(CHAR, STRING); + builder.put(CHAR, TIME); + builder.put(CHAR, TIMEV2); // Varchar builder.put(VARCHAR, BOOLEAN); builder.put(VARCHAR, TINYINT); @@ -399,8 +415,10 @@ public enum PrimitiveType { builder.put(VARCHAR, VARCHAR); builder.put(VARCHAR, JSONB); builder.put(VARCHAR, STRING); + builder.put(VARCHAR, TIME); + builder.put(VARCHAR, TIMEV2); - // Varchar + // String builder.put(STRING, BOOLEAN); builder.put(STRING, TINYINT); builder.put(STRING, SMALLINT); @@ -420,6 +438,8 @@ public enum PrimitiveType { builder.put(STRING, VARCHAR); builder.put(STRING, JSONB); builder.put(STRING, STRING); + builder.put(STRING, TIME); + builder.put(STRING, TIMEV2); // DecimalV2 builder.put(DECIMALV2, BOOLEAN); diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index fa0e696ff3..35fe7f26a2 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -82,6 +82,8 @@ COMMA: ','; DOT: '.'; LEFT_BRACKET: '['; RIGHT_BRACKET: ']'; +LEFT_BRACE: '{'; +RIGHT_BRACE: '}'; // TODO: add a doc to list reserved words @@ -232,7 +234,6 @@ LAST: 'LAST'; LATERAL: 'LATERAL'; LAZY: 'LAZY'; LEADING: 'LEADING'; -LEFT_BRACE: '{'; LEFT: 'LEFT'; LIKE: 'LIKE'; ILIKE: 'ILIKE'; @@ -319,7 +320,6 @@ RESTRICT: 'RESTRICT'; RESTRICTIVE: 'RESTRICTIVE'; REVOKE: 'REVOKE'; REWRITTEN: 'REWRITTEN'; -RIGHT_BRACE: '}'; RIGHT: 'RIGHT'; // original optimizer only support REGEXP, the new optimizer should be consistent with it RLIKE: 'RLIKE'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index dd9849376a..891c3a01b8 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -401,6 +401,7 @@ primaryExpression | CASE value=expression whenClause+ (ELSE elseExpression=expression)? END #simpleCase | name=CAST LEFT_PAREN expression AS dataType RIGHT_PAREN #cast | constant #constantDefault + | interval #intervalLiteral | ASTERISK #star | qualifiedName DOT ASTERISK #star | functionIdentifier LEFT_PAREN ((DISTINCT|ALL)? arguments+=expression @@ -468,11 +469,14 @@ specifiedPartition constant : NULL #nullLiteral - | interval #intervalLiteral - | type=(DATE | DATEV2 | TIMESTAMP) STRING_LITERAL #typeConstructor + | type=(DATE | DATEV2 | TIMESTAMP) STRING_LITERAL #typeConstructor | number #numericLiteral | booleanValue #booleanLiteral - | STRING_LITERAL #stringLiteral + | STRING_LITERAL #stringLiteral + | LEFT_BRACKET items+=constant (COMMA items+=constant)* RIGHT_BRACKET #arrayLiteral + | LEFT_BRACE items+=constant COLON items+=constant + (COMMA items+=constant COLON items+=constant)* RIGHT_BRACE #mapLiteral + | LEFT_BRACE items+=constant (COMMA items+=constant)* RIGHT_BRACE #structLiteral ; comparisonOperator @@ -543,7 +547,7 @@ quotedIdentifier ; number - : MINUS? INTEGER_VALUE #integerLiteral + : MINUS? INTEGER_VALUE #integerLiteral | MINUS? (EXPONENT_VALUE | DECIMAL_VALUE) #decimalLiteral ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 7102526efd..8267b4f952 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -93,6 +93,8 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz; import org.apache.doris.nereids.trees.expressions.functions.scalar.Cos; import org.apache.doris.nereids.trees.expressions.functions.scalar.CountEqual; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateNamedStruct; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct; import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentCatalog; import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentDate; import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentTime; @@ -305,6 +307,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.StartsWith; import org.apache.doris.nereids.trees.expressions.functions.scalar.StrLeft; import org.apache.doris.nereids.trees.expressions.functions.scalar.StrRight; import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement; import org.apache.doris.nereids.trees.expressions.functions.scalar.SubBitmap; import org.apache.doris.nereids.trees.expressions.functions.scalar.SubReplace; import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring; @@ -436,6 +439,8 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Cos.class, "cos"), scalar(CountEqual.class, "countequal"), scalar(CreateMap.class, "map"), + scalar(CreateStruct.class, "struct"), + scalar(CreateNamedStruct.class, "named_struct"), scalar(CurrentCatalog.class, "current_catalog"), scalar(CurrentDate.class, "curdate", "current_date"), scalar(CurrentTime.class, "curtime", "current_time"), @@ -610,6 +615,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Sign.class, "sign"), scalar(Sin.class, "sin"), scalar(Sleep.class, "sleep"), + scalar(StructElement.class, "struct_element"), scalar(Sm3.class, "sm3"), scalar(Sm3sum.class, "sm3sum"), scalar(Sm4Decrypt.class, "sm4_decrypt"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 708b2ef86d..c7e7a19361 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.DorisParser.AliasQueryContext; import org.apache.doris.nereids.DorisParser.AliasedQueryContext; import org.apache.doris.nereids.DorisParser.ArithmeticBinaryContext; import org.apache.doris.nereids.DorisParser.ArithmeticUnaryContext; +import org.apache.doris.nereids.DorisParser.ArrayLiteralContext; import org.apache.doris.nereids.DorisParser.ArraySliceContext; import org.apache.doris.nereids.DorisParser.BitOperationContext; import org.apache.doris.nereids.DorisParser.BooleanExpressionContext; @@ -41,6 +42,8 @@ import org.apache.doris.nereids.DorisParser.ColumnReferenceContext; import org.apache.doris.nereids.DorisParser.CommentJoinHintContext; import org.apache.doris.nereids.DorisParser.CommentRelationHintContext; import org.apache.doris.nereids.DorisParser.ComparisonContext; +import org.apache.doris.nereids.DorisParser.ComplexColTypeContext; +import org.apache.doris.nereids.DorisParser.ComplexColTypeListContext; import org.apache.doris.nereids.DorisParser.ComplexDataTypeContext; import org.apache.doris.nereids.DorisParser.ConstantContext; import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext; @@ -74,6 +77,7 @@ import org.apache.doris.nereids.DorisParser.LateralViewContext; import org.apache.doris.nereids.DorisParser.LimitClauseContext; import org.apache.doris.nereids.DorisParser.LogicalBinaryContext; import org.apache.doris.nereids.DorisParser.LogicalNotContext; +import org.apache.doris.nereids.DorisParser.MapLiteralContext; import org.apache.doris.nereids.DorisParser.MultiStatementsContext; import org.apache.doris.nereids.DorisParser.MultipartIdentifierContext; import org.apache.doris.nereids.DorisParser.NamedExpressionContext; @@ -105,6 +109,7 @@ import org.apache.doris.nereids.DorisParser.SortItemContext; import org.apache.doris.nereids.DorisParser.StarContext; import org.apache.doris.nereids.DorisParser.StatementDefaultContext; import org.apache.doris.nereids.DorisParser.StringLiteralContext; +import org.apache.doris.nereids.DorisParser.StructLiteralContext; import org.apache.doris.nereids.DorisParser.SubqueryContext; import org.apache.doris.nereids.DorisParser.SubqueryExpressionContext; import org.apache.doris.nereids.DorisParser.SystemVariableContext; @@ -187,7 +192,10 @@ import org.apache.doris.nereids.trees.expressions.WindowFrame; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Array; import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySlice; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.DaysSub; @@ -264,6 +272,8 @@ import org.apache.doris.nereids.trees.plans.logical.UsingJoin; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.coercion.CharacterType; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.policy.FilterType; @@ -1311,7 +1321,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { } @Override - public Expression visitTypeConstructor(TypeConstructorContext ctx) { + public Literal visitTypeConstructor(TypeConstructorContext ctx) { String value = ctx.STRING_LITERAL().getText(); value = value.substring(1, value.length() - 1); String type = ctx.type.getText().toUpperCase(); @@ -1367,7 +1377,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { * Create a NULL literal expression. */ @Override - public Expression visitNullLiteral(NullLiteralContext ctx) { + public Literal visitNullLiteral(NullLiteralContext ctx) { return new NullLiteral(); } @@ -1444,6 +1454,24 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { return sb.toString(); } + @Override + public Object visitArrayLiteral(ArrayLiteralContext ctx) { + Literal[] items = ctx.items.stream().map(this::typedVisit).toArray(Literal[]::new); + return new Array(items); + } + + @Override + public Object visitMapLiteral(MapLiteralContext ctx) { + Literal[] items = ctx.items.stream().map(this::typedVisit).toArray(Literal[]::new); + return new CreateMap(items); + } + + @Override + public Object visitStructLiteral(StructLiteralContext ctx) { + Literal[] items = ctx.items.stream().map(this::typedVisit).toArray(Literal[]::new); + return new CreateStruct(items); + } + @Override public Expression visitParenthesizedExpression(ParenthesizedExpressionContext ctx) { return getExpression(ctx.expression()); @@ -2084,13 +2112,25 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { case DorisParser.MAP: return MapType.of(typedVisit(ctx.dataType(0)), typedVisit(ctx.dataType(1))); case DorisParser.STRUCT: - throw new AnalysisException("do not support STRUCT type for Nereids"); + return new StructType(visitComplexColTypeList(ctx.complexColTypeList())); default: throw new AnalysisException("do not support " + ctx.complex.getText() + " type for Nereids"); } }); } + @Override + public List visitComplexColTypeList(ComplexColTypeListContext ctx) { + return ctx.complexColType().stream().map(this::visitComplexColType).collect(ImmutableList.toImmutableList()); + } + + @Override + public StructField visitComplexColType(ComplexColTypeContext ctx) { + String comment = ctx.commentSpec().STRING_LITERAL().getText(); + comment = escapeBackSlash(comment.substring(1, comment.length() - 1)); + return new StructField(ctx.identifier().getText(), typedVisit(ctx.dataType()), true, comment); + } + private Expression parseFunctionWithOrderKeys(String functionName, boolean isDistinct, List params, List orderKeys, ParserRuleContext ctx) { if (functionName.equalsIgnoreCase("group_concat")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java index 2a5b83a06f..84abcb9d43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java @@ -25,9 +25,11 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; +import org.apache.doris.nereids.types.coercion.CharacterType; -import java.util.Map; +import java.util.List; /** * check cast valid @@ -42,7 +44,7 @@ public class CheckCast extends AbstractExpressionRewriteRule { DataType originalType = cast.child().getDataType(); DataType targetType = cast.getDataType(); if (!check(originalType, targetType)) { - throw new AnalysisException("cannot cast " + originalType + " to " + targetType); + throw new AnalysisException("cannot cast " + originalType.toSql() + " to " + targetType.toSql()); } return cast; } @@ -54,17 +56,19 @@ public class CheckCast extends AbstractExpressionRewriteRule { return check(((MapType) originalType).getKeyType(), ((MapType) targetType).getKeyType()) && check(((MapType) originalType).getValueType(), ((MapType) targetType).getValueType()); } else if (originalType instanceof StructType && targetType instanceof StructType) { - Map targetItems = ((StructType) targetType).getItems(); - for (Map.Entry entry : ((StructType) originalType).getItems().entrySet()) { - if (targetItems.containsKey(entry.getKey())) { - if (!check(entry.getValue(), targetItems.get(entry.getKey()))) { - return false; - } - } else { + List targetFields = ((StructType) targetType).getFields(); + List originalFields = ((StructType) originalType).getFields(); + if (targetFields.size() != originalFields.size()) { + return false; + } + for (int i = 0; i < targetFields.size(); i++) { + if (!targetFields.get(i).equals(originalFields.get(i))) { return false; } } return true; + } else if (originalType instanceof CharacterType && targetType instanceof StructType) { + return true; } else { return checkPrimitiveType(originalType, targetType); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java index 69a3499630..e049254368 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java @@ -46,6 +46,7 @@ import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdfBuilder; import org.apache.doris.nereids.trees.expressions.typecoercion.ImplicitCastInputTypes; +import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -116,6 +117,13 @@ public class FunctionBinder extends AbstractExpressionRewriteRule { } } + @Override + public Expression visitBoundFunction(BoundFunction boundFunction, ExpressionRewriteContext context) { + boundFunction = (BoundFunction) boundFunction.withChildren(boundFunction.children().stream() + .map(e -> e.accept(this, context)).collect(Collectors.toList())); + return TypeCoercionUtils.processBoundFunction(boundFunction); + } + /** * gets the method for calculating the time. * e.g. YEARS_ADD、YEARS_SUB、DAYS_ADD 、DAYS_SUB @@ -271,4 +279,14 @@ public class FunctionBinder extends AbstractExpressionRewriteRule { } return match.withChildren(left, right); } + + @Override + public Expression visitCast(Cast cast, ExpressionRewriteContext context) { + cast = (Cast) super.visitCast(cast, context); + // NOTICE: just for compatibility with legacy planner. + if (cast.child().getDataType() instanceof ArrayType || cast.getDataType() instanceof ArrayType) { + TypeCoercionUtils.checkCanCastTo(cast.child().getDataType(), cast.getDataType()); + } + return cast; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckDataTypes.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckDataTypes.java index d42fc51284..e3ccfa689f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckDataTypes.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckDataTypes.java @@ -40,8 +40,7 @@ import java.util.Set; */ public class CheckDataTypes implements CustomRewriter { - private static final Set> UNSUPPORTED_TYPE = ImmutableSet.of( - StructType.class, JsonType.class); + private static final Set> UNSUPPORTED_TYPE = ImmutableSet.of(JsonType.class); @Override public Plan rewriteRoot(Plan rootPlan, JobContext jobContext) { @@ -92,8 +91,9 @@ public class CheckDataTypes implements CustomRewriter { } else if (dataType instanceof MapType) { checkTypes(((MapType) dataType).getKeyType()); checkTypes(((MapType) dataType).getValueType()); - } - if (UNSUPPORTED_TYPE.contains(dataType.getClass())) { + } else if (dataType instanceof StructType) { + ((StructType) dataType).getFields().forEach(f -> this.checkTypes(f.getDataType())); + } else if (UNSUPPORTED_TYPE.contains(dataType.getClass())) { throw new AnalysisException(String.format("type %s is unsupported for Nereids", dataType)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index 86122af5ad..fd4ad63f6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.coercion.AnyDataType; @@ -129,7 +130,19 @@ public abstract class Expression extends AbstractTreeNode implements && checkInputDataTypesWithExpectType( ((MapType) input).getValueType(), ((MapType) expected).getValueType()); } else if (input instanceof StructType && expected instanceof StructType) { - throw new AnalysisException("not support struct type now."); + List inputFields = ((StructType) input).getFields(); + List expectedFields = ((StructType) expected).getFields(); + if (inputFields.size() != expectedFields.size()) { + return false; + } + for (int i = 0; i < inputFields.size(); i++) { + if (!checkInputDataTypesWithExpectType( + inputFields.get(i).getDataType(), + expectedFields.get(i).getDataType())) { + return false; + } + } + return true; } else { return checkPrimitiveInputDataTypesWithExpectType(input, expected); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignature.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignature.java index 549053f89b..a755e3c517 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignature.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignature.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.trees.expressions.functions; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.annotation.Developing; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.functions.ComputeSignatureHelper.ComputeSignatureChain; import org.apache.doris.nereids.trees.expressions.typecoercion.ImplicitCastInputTypes; import org.apache.doris.nereids.types.ArrayType; @@ -117,7 +116,7 @@ public interface ComputeSignature extends FunctionTrait, ImplicitCastInputTypes .get(); } - /** default computeSignature */ + /** use processor to process computeSignature */ static boolean processComplexType(DataType signatureType, DataType realType, BiFunction processor) { if (signatureType instanceof ArrayType && realType instanceof ArrayType) { @@ -127,7 +126,9 @@ public interface ComputeSignature extends FunctionTrait, ImplicitCastInputTypes return processor.apply(((MapType) signatureType).getKeyType(), ((MapType) realType).getKeyType()) && processor.apply(((MapType) signatureType).getValueType(), ((MapType) realType).getValueType()); } else if (signatureType instanceof StructType && realType instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + // TODO: do not support struct type now + // throw new AnalysisException("do not support struct type now"); + return true; } else { return processor.apply(signatureType, realType); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java index a8e03153ae..223b793a9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java @@ -73,7 +73,8 @@ public class ComputeSignatureHelper { collectAnyDataType(((MapType) sigType).getKeyType(), NullType.INSTANCE, indexToArgumentTypes); collectAnyDataType(((MapType) sigType).getValueType(), NullType.INSTANCE, indexToArgumentTypes); } else if (sigType instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + // TODO: do not support struct type now + // throw new AnalysisException("do not support struct type now"); } else { if (sigType instanceof AnyDataType && ((AnyDataType) sigType).getIndex() >= 0) { List dataTypes = indexToArgumentTypes.computeIfAbsent( @@ -90,7 +91,8 @@ public class ComputeSignatureHelper { collectAnyDataType(((MapType) sigType).getValueType(), ((MapType) expressionType).getValueType(), indexToArgumentTypes); } else if (sigType instanceof StructType && expressionType instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + // TODO: do not support struct type now + // throw new AnalysisException("do not support struct type now"); } else { if (sigType instanceof AnyDataType && ((AnyDataType) sigType).getIndex() >= 0) { List dataTypes = indexToArgumentTypes.computeIfAbsent( @@ -112,7 +114,8 @@ public class ComputeSignatureHelper { collectFollowToAnyDataType(((MapType) sigType).getValueType(), NullType.INSTANCE, indexToArgumentTypes, allNullTypeIndex); } else if (sigType instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + // TODO: do not support struct type now + // throw new AnalysisException("do not support struct type now"); } else { if (sigType instanceof FollowToAnyDataType && allNullTypeIndex.contains(((FollowToAnyDataType) sigType).getIndex())) { @@ -130,7 +133,8 @@ public class ComputeSignatureHelper { collectFollowToAnyDataType(((MapType) sigType).getValueType(), ((MapType) expressionType).getValueType(), indexToArgumentTypes, allNullTypeIndex); } else if (sigType instanceof StructType && expressionType instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + // TODO: do not support struct type now + // throw new AnalysisException("do not support struct type now"); } else { if (sigType instanceof FollowToAnyDataType && allNullTypeIndex.contains(((FollowToAnyDataType) sigType).getIndex())) { @@ -149,7 +153,9 @@ public class ComputeSignatureHelper { return MapType.of(replaceAnyDataType(((MapType) dataType).getKeyType(), indexToCommonTypes), replaceAnyDataType(((MapType) dataType).getValueType(), indexToCommonTypes)); } else if (dataType instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + // TODO: do not support struct type now + // throw new AnalysisException("do not support struct type now"); + return dataType; } else { if (dataType instanceof AnyDataType && ((AnyDataType) dataType).getIndex() >= 0) { Optional optionalDataType = indexToCommonTypes.get(((AnyDataType) dataType).getIndex()); @@ -177,7 +183,7 @@ public class ComputeSignatureHelper { DataType sigType; if (i >= signature.argumentsTypes.size()) { sigType = signature.getVarArgType().orElseThrow( - () -> new IllegalStateException("function arity not match with signature")); + () -> new AnalysisException("function arity not match with signature")); } else { sigType = signature.argumentsTypes.get(i); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayExcept.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayExcept.java index b87460b766..c1edd9f2e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayExcept.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayExcept.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.coercion.AnyDataType; -import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -40,7 +39,7 @@ public class ArrayExcept extends ScalarFunction implements ExplicitlyCastableSig public static final List SIGNATURES = ImmutableList.of( FunctionSignature.retArgType(0) - .args(ArrayType.of(new AnyDataType(0)), ArrayType.of(new FollowToAnyDataType(0))) + .args(ArrayType.of(new AnyDataType(0)), ArrayType.of(new AnyDataType(0))) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayIntersect.java index 0dc4f38151..0811304e51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayIntersect.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.coercion.AnyDataType; -import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -40,7 +39,7 @@ public class ArrayIntersect extends ScalarFunction implements ExplicitlyCastable public static final List SIGNATURES = ImmutableList.of( FunctionSignature.retArgType(0) - .args(ArrayType.of(new AnyDataType(0)), ArrayType.of(new FollowToAnyDataType(0))) + .args(ArrayType.of(new AnyDataType(0)), ArrayType.of(new AnyDataType(0))) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayUnion.java index 93e1ca862f..a1152b2c6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayUnion.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.coercion.AnyDataType; -import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -40,7 +39,7 @@ public class ArrayUnion extends ScalarFunction implements ExplicitlyCastableSign public static final List SIGNATURES = ImmutableList.of( FunctionSignature.retArgType(0) - .args(ArrayType.of(new AnyDataType(0)), ArrayType.of(new FollowToAnyDataType(0))) + .args(ArrayType.of(new AnyDataType(0)), ArrayType.of(new AnyDataType(0))) ); /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java index 2ffe1b4ea9..fe966f4e38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; /** - * ScalarFunction 'array'. This class is generated by GenerateFunction. + * ScalarFunction 'map'. */ public class CreateMap extends ScalarFunction implements ExplicitlyCastableSignature, AlwaysNotNullable { @@ -59,7 +59,7 @@ public class CreateMap extends ScalarFunction @Override public void checkLegalityBeforeTypeCoercion() { if (arity() % 2 != 0) { - throw new AnalysisException("map can't be odd parameters, need even parameters " + this); + throw new AnalysisException("map can't be odd parameters, need even parameters " + this.toSql()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java new file mode 100644 index 0000000000..0184fed65a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateNamedStruct.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; + +import java.util.List; +import java.util.Set; + +/** + * ScalarFunction 'named_struct'. + */ +public class CreateNamedStruct extends ScalarFunction + implements ExplicitlyCastableSignature, AlwaysNotNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(StructType.SYSTEM_DEFAULT).args() + ); + + /** + * constructor with 0 or more arguments. + */ + public CreateNamedStruct(Expression... varArgs) { + super("named_struct", varArgs); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (arity() % 2 != 0) { + throw new AnalysisException("named_struct can't be odd parameters, need even parameters " + this.toSql()); + } + Set names = Sets.newHashSet(); + for (int i = 0; i < arity(); i = i + 2) { + if (!(child(i) instanceof StringLikeLiteral)) { + throw new AnalysisException("named_struct only allows" + + " constant string parameter in odd position: " + this); + } else { + String name = ((StringLikeLiteral) child(i)).getStringValue(); + if (names.contains(name)) { + throw new AnalysisException("The name of the struct field cannot be repeated." + + " same name fields are " + name); + } else { + names.add(name); + } + } + } + } + + /** + * withChildren. + */ + @Override + public CreateNamedStruct withChildren(List children) { + return new CreateNamedStruct(children.toArray(new Expression[0])); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitCreateNamedStruct(this, context); + } + + @Override + public List getSignatures() { + if (arity() == 0) { + return SIGNATURES; + } else { + ImmutableList.Builder structFields = ImmutableList.builder(); + for (int i = 0; i < arity(); i = i + 2) { + StringLikeLiteral nameLiteral = (StringLikeLiteral) child(i); + structFields.add(new StructField(nameLiteral.getStringValue(), + children.get(i + 1).getDataType(), true, "")); + } + return ImmutableList.of(FunctionSignature.ret(new StructType(structFields.build())) + .args(children.stream().map(ExpressionTrait::getDataType).toArray(DataType[]::new))); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateStruct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateStruct.java new file mode 100644 index 0000000000..cba64fdcce --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateStruct.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'struct'. + */ +public class CreateStruct extends ScalarFunction + implements ExplicitlyCastableSignature, AlwaysNotNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(StructType.SYSTEM_DEFAULT).args() + ); + + /** + * constructor with 0 or more arguments. + */ + public CreateStruct(Expression... varArgs) { + super("struct", varArgs); + } + + /** + * withChildren. + */ + @Override + public CreateStruct withChildren(List children) { + return new CreateStruct(children.toArray(new Expression[0])); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitCreateStruct(this, context); + } + + @Override + public List getSignatures() { + if (arity() == 0) { + return SIGNATURES; + } else { + ImmutableList.Builder structFields = ImmutableList.builder(); + for (int i = 0; i < arity(); i++) { + structFields.add(new StructField(String.valueOf(i + 1), children.get(i).getDataType(), true, "")); + } + return ImmutableList.of(FunctionSignature.ret(new StructType(structFields.build())) + .args(children.stream().map(ExpressionTrait::getDataType).toArray(DataType[]::new))); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java new file mode 100644 index 0000000000..d21b365947 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.SearchSignature; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'struct_element'. + */ +public class StructElement extends ScalarFunction + implements ExplicitlyCastableSignature, AlwaysNullable { + + /** + * constructor with 0 or more arguments. + */ + public StructElement(Expression arg0, Expression arg1) { + super("struct_element", arg0, arg1); + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (!(child(0).getDataType() instanceof StructType)) { + SearchSignature.throwCanNotFoundFunctionException(this.getName(), this.getArguments()); + } + if (!(child(1) instanceof StringLikeLiteral || child(1) instanceof IntegerLikeLiteral)) { + throw new AnalysisException("struct_element only allows" + + " constant int or string second parameter: " + this.toSql()); + } + } + + /** + * withChildren. + */ + @Override + public StructElement withChildren(List children) { + Preconditions.checkArgument(children.size() == 2, "children size should be 2"); + return new StructElement(children.get(0), children.get(1)); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitStructElement(this, context); + } + + @Override + public List getSignatures() { + StructType structArgType = (StructType) child(0).getDataType(); + DataType retType; + if (child(1) instanceof IntegerLikeLiteral) { + int offset = ((IntegerLikeLiteral) child(1)).getIntValue(); + if (offset <= 0 || offset > structArgType.getFields().size()) { + throw new AnalysisException("the specified field index out of bound: " + this.toSql()); + } else { + retType = structArgType.getFields().get(offset - 1).getDataType(); + } + } else if (child(1) instanceof StringLikeLiteral) { + String name = ((StringLikeLiteral) child(1)).getStringValue(); + if (!structArgType.getNameToFields().containsKey(name)) { + throw new AnalysisException("the specified field name " + name + " was not found: " + this.toSql()); + } else { + retType = structArgType.getNameToFields().get(name).getDataType(); + } + } else { + throw new AnalysisException("struct_element only allows" + + " constant int or string second parameter: " + this.toSql()); + } + return ImmutableList.of(FunctionSignature.ret(retType).args(structArgType, child(1).getDataType())); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java index 480ac996a1..b9cdd7eced 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java @@ -43,4 +43,9 @@ public abstract class StringLikeLiteral extends Literal { } return (double) v; } + + @Override + public String toString() { + return "'" + value + "'"; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLiteral.java index 588adbc0aa..603749c60b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLiteral.java @@ -52,9 +52,4 @@ public class StringLiteral extends StringLikeLiteral { public LiteralExpr toLegacyLiteral() { return new org.apache.doris.analysis.StringLiteral(value); } - - @Override - public String toString() { - return "'" + value + "'"; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarcharLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarcharLiteral.java index 40f3f8f8ad..75161dd000 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarcharLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarcharLiteral.java @@ -50,9 +50,4 @@ public class VarcharLiteral extends StringLikeLiteral { public LiteralExpr toLegacyLiteral() { return new StringLiteral(value); } - - @Override - public String toString() { - return "'" + value + "'"; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index f77d241898..0f37d870b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -97,6 +97,8 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz; import org.apache.doris.nereids.trees.expressions.functions.scalar.Cos; import org.apache.doris.nereids.trees.expressions.functions.scalar.CountEqual; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateNamedStruct; +import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct; import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentCatalog; import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentDate; import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentTime; @@ -309,6 +311,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.StartsWith; import org.apache.doris.nereids.trees.expressions.functions.scalar.StrLeft; import org.apache.doris.nereids.trees.expressions.functions.scalar.StrRight; import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement; import org.apache.doris.nereids.trees.expressions.functions.scalar.SubBitmap; import org.apache.doris.nereids.trees.expressions.functions.scalar.SubReplace; import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring; @@ -1695,4 +1698,18 @@ public interface ScalarFunctionVisitor { default R visitMapValues(MapValues mapValues, C context) { return visitScalarFunction(mapValues, context); } + + // struct function + + default R visitCreateStruct(CreateStruct createStruct, C context) { + return visitScalarFunction(createStruct, context); + } + + default R visitCreateNamedStruct(CreateNamedStruct createNamedStruct, C context) { + return visitScalarFunction(createNamedStruct, context); + } + + default R visitStructElement(StructElement structElement, C context) { + return visitScalarFunction(structElement, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java index b45b70b9f4..5f233c959b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java @@ -290,8 +290,11 @@ public abstract class DataType { } else if (type.isJsonbType()) { return JsonType.INSTANCE; } else if (type.isStructType()) { - // TODO: support struct type really - return StructType.INSTANCE; + List structFields = ((org.apache.doris.catalog.StructType) (type)).getFields().stream() + .map(cf -> new StructField(cf.getName(), fromCatalogType(cf.getType()), + cf.getContainsNull(), cf.getComment())) + .collect(ImmutableList.toImmutableList()); + return new StructType(structFields); } else if (type.isMapType()) { org.apache.doris.catalog.MapType mapType = (org.apache.doris.catalog.MapType) type; return MapType.of(fromCatalogType(mapType.getKeyType()), fromCatalogType(mapType.getValueType())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java new file mode 100644 index 0000000000..d42d7f960a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.types; + +import org.apache.doris.nereids.util.Utils; + +import java.util.Objects; + +/** + * A field inside a StructType. + */ +public class StructField { + + private final String name; + private final DataType dataType; + private final boolean nullable; + private final String comment; + + /** + * StructField Constructor + * @param name The name of this field + * @param dataType The data type of this field + * @param nullable Indicates if values of this field can be `null` values + */ + public StructField(String name, DataType dataType, boolean nullable, String comment) { + this.name = Objects.requireNonNull(name, "name should not be null"); + this.dataType = Objects.requireNonNull(dataType, "dataType should not be null"); + this.nullable = nullable; + this.comment = Objects.requireNonNull(comment, "comment should not be null"); + } + + public String getName() { + return name; + } + + public DataType getDataType() { + return dataType; + } + + public boolean isNullable() { + return nullable; + } + + public String getComment() { + return comment; + } + + public StructField withDataType(DataType dataType) { + return new StructField(name, dataType, nullable, comment); + } + + public org.apache.doris.catalog.StructField toCatalogDataType() { + return new org.apache.doris.catalog.StructField( + name, dataType.toCatalogDataType(), comment, nullable); + } + + public String toSql() { + return name + ":" + dataType.toSql() + + (nullable ? " " : " NOT NULL") + + (comment.isEmpty() ? "" : " COMMENT " + comment); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StructField that = (StructField) o; + return nullable == that.nullable && Objects.equals(name, that.name) && Objects.equals(dataType, + that.dataType); + } + + @Override + public int hashCode() { + return Objects.hash(name, dataType, nullable); + } + + @Override + public String toString() { + return Utils.toSqlString("StructField", + "name", name, + "dataType", dataType, + "nullable", nullable, + "comment", comment); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java index 9359df15f7..e4ca5e9db7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java @@ -19,12 +19,18 @@ package org.apache.doris.nereids.types; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.annotation.Developing; +import org.apache.doris.nereids.exceptions.AnalysisException; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSortedMap; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Struct type in Nereids. @@ -32,28 +38,43 @@ import java.util.Objects; @Developing public class StructType extends DataType { - public static final StructType INSTANCE = new StructType(); + public static final StructType SYSTEM_DEFAULT = new StructType(); public static final int WIDTH = 24; - private final Map items; + private final List fields; + private final Supplier> nameToFields; private StructType() { - items = ImmutableMap.of(); + nameToFields = Suppliers.memoize(ImmutableMap::of); + fields = ImmutableList.of(); } - public StructType(Map items) { - this.items = ImmutableSortedMap.copyOf(Objects.requireNonNull(items, "items should not be null"), - String.CASE_INSENSITIVE_ORDER); + /** + * construct struct type. + */ + public StructType(List fields) { + this.fields = ImmutableList.copyOf(Objects.requireNonNull(fields, "fields should not be null")); + this.nameToFields = Suppliers.memoize(() -> this.fields.stream().collect(ImmutableMap.toImmutableMap( + StructField::getName, f -> f, (f1, f2) -> { + throw new AnalysisException("The name of the struct field cannot be repeated." + + " same name fields are " + f1 + " and " + f2); + }))); } - public Map getItems() { - return items; + public List getFields() { + return fields; + } + + public Map getNameToFields() { + return nameToFields.get(); } @Override public Type toCatalogDataType() { - return Type.STRUCT; + return new org.apache.doris.catalog.StructType(fields.stream() + .map(StructField::toCatalogDataType) + .collect(Collectors.toCollection(ArrayList::new))); } @Override @@ -68,7 +89,22 @@ public class StructType extends DataType { @Override public boolean equals(Object o) { - return o instanceof StructType; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + StructType that = (StructType) o; + return Objects.equals(fields, that.fields); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), fields); } @Override @@ -78,6 +114,11 @@ public class StructType extends DataType { @Override public String toSql() { - return "STRUCT"; + return "STRUCT<" + fields.stream().map(StructField::toSql).collect(Collectors.joining(", ")) + ">"; + } + + @Override + public String toString() { + return "STRUCT<" + fields.stream().map(StructField::toString).collect(Collectors.joining(", ")) + ">"; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index badde85c2e..185c445647 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -85,6 +85,7 @@ import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.TimeType; import org.apache.doris.nereids.types.TimeV2Type; @@ -153,7 +154,22 @@ public class TypeCoercionUtils { } return Optional.empty(); } else if (input instanceof StructType && expected instanceof StructType) { - throw new AnalysisException("not support struct type now."); + List inputFields = ((StructType) input).getFields(); + List expectedFields = ((StructType) expected).getFields(); + if (inputFields.size() != expectedFields.size()) { + return Optional.empty(); + } + List newFields = Lists.newArrayList(); + for (int i = 0; i < inputFields.size(); i++) { + Optional newDataType = implicitCast(inputFields.get(i).getDataType(), + expectedFields.get(i).getDataType()); + if (newDataType.isPresent()) { + newFields.add(inputFields.get(i).withDataType(newDataType.get())); + } else { + return Optional.empty(); + } + } + return Optional.of(new StructType(newFields)); } else { return implicitCastPrimitive(input, expected); } @@ -247,7 +263,7 @@ public class TypeCoercionUtils { return hasCharacterType(((MapType) dataType).getKeyType()) || hasCharacterType(((MapType) dataType).getValueType()); } else if (dataType instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + return ((StructType) dataType).getFields().stream().anyMatch(f -> hasCharacterType(f.getDataType())); } return dataType instanceof CharacterType; } @@ -293,7 +309,17 @@ public class TypeCoercionUtils { return matchesType(((MapType) input).getKeyType(), ((MapType) target).getKeyType()) && matchesType(((MapType) input).getValueType(), ((MapType) target).getValueType()); } else if (input instanceof StructType && target instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + List inputFields = ((StructType) input).getFields(); + List targetFields = ((StructType) target).getFields(); + if (inputFields.size() != targetFields.size()) { + return false; + } + for (int i = 0; i < inputFields.size(); i++) { + if (!matchesType(inputFields.get(i).getDataType(), targetFields.get(i).getDataType())) { + return false; + } + } + return true; } else { if (input instanceof NullType) { return false; @@ -994,7 +1020,22 @@ public class TypeCoercionUtils { return Optional.of(MapType.of(keyType.get(), valueType.get())); } } else if (left instanceof StructType && right instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + List leftFields = ((StructType) left).getFields(); + List rightFields = ((StructType) right).getFields(); + if (leftFields.size() != rightFields.size()) { + return Optional.empty(); + } + List newFields = Lists.newArrayList(); + for (int i = 0; i < leftFields.size(); i++) { + Optional newDataType = findCommonComplexTypeForComparison(leftFields.get(i).getDataType(), + rightFields.get(i).getDataType(), intStringToString); + if (newDataType.isPresent()) { + newFields.add(leftFields.get(i).withDataType(newDataType.get())); + } else { + return Optional.empty(); + } + } + return Optional.of(new StructType(newFields)); } return Optional.empty(); } @@ -1192,7 +1233,22 @@ public class TypeCoercionUtils { return Optional.of(MapType.of(keyType.get(), valueType.get())); } } else if (left instanceof StructType && right instanceof StructType) { - throw new AnalysisException("do not support struct type now"); + List leftFields = ((StructType) left).getFields(); + List rightFields = ((StructType) right).getFields(); + if (leftFields.size() != rightFields.size()) { + return Optional.empty(); + } + List newFields = Lists.newArrayList(); + for (int i = 0; i < leftFields.size(); i++) { + Optional newDataType = findCommonComplexTypeForCaseWhen(leftFields.get(i).getDataType(), + rightFields.get(i).getDataType()); + if (newDataType.isPresent()) { + newFields.add(leftFields.get(i).withDataType(newDataType.get())); + } else { + return Optional.empty(); + } + } + return Optional.of(new StructType(newFields)); } return Optional.empty(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/UnsupportedTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/UnsupportedTypeTest.java index 0f379ef30e..20c26286c2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/UnsupportedTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/UnsupportedTypeTest.java @@ -63,23 +63,23 @@ public class UnsupportedTypeTest extends TestWithFeService { "select karr from type_tb", "select array_range(10)", "select kmap from type_tb1", + "select * from type_tb1", "select jsonb_parse('{\"k1\":\"v31\",\"k2\":300}')", "select * from type_tb", - "select * from type_tb1", }; Class[] exceptions = { null, null, null, null, - AnalysisException.class, + null, AnalysisException.class, AnalysisException.class }; - for (int i = 0; i < 3; ++i) { + for (int i = 0; i < 5; ++i) { runPlanner(sqls[i]); } - for (int i = 4; i < sqls.length; ++i) { + for (int i = 5; i < sqls.length; ++i) { int iCopy = i; Assertions.assertThrows(exceptions[i], () -> runPlanner(sqls[iCopy])); } diff --git a/regression-test/suites/nereids_syntax_p0/test_complex_type.groovy b/regression-test/suites/nereids_syntax_p0/test_complex_type.groovy new file mode 100644 index 0000000000..145e3d2eb3 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/test_complex_type.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_complex_type") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + // array literal + sql "select [1, 2, 3, 4]" + + // map literal + sql "select {1:2, 3:4}" + + // struct literal + sql "select {1, 2, 3, 4}" + + // struct constructor + sql "select struct(1, 2, 3, 4)" + + // named struct constructor + sql "select named_struct('x', 1, 'y', 2)" + + test { + sql "select named_struct('x', 1, 'x', 2)" + exception "The name of the struct field cannot be repeated" + } + + // struct element with int + sql "select struct_element(struct('1', '2', '3', '4'), 1);" + + test { + sql "select struct_element(struct('1', '2', '3', '4'), 5);" + exception "the specified field index out of bound" + } + + test { + sql "select struct_element(struct('1', '2', '3', '4'), -1);" + exception "the specified field index out of bound" + } + + // struct element with string + sql "select struct_element(named_struct('1', '2', '3', '4'), '1');" + + test { + sql "select struct_element(named_struct('1', '2', '3', '4'), '5')" + exception "the specified field name 5 was not found" + } +} +