From 3786ffec51c1ae0226c0c981e2ea48ce9274da0e Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Fri, 25 Aug 2023 11:01:50 +0800 Subject: [PATCH] [opt](Nereids) add some array functions (#23324) 1. rename TVFProperties to Properties 2. add generating function explode and explode_outer 3. fix concat_ws could not apply on array 4. check tokenize second argument format on FE 5. add test case for concat_ws, tokenize, explode, explode_outer and split_by_string --- .../org/apache/doris/nereids/DorisParser.g4 | 15 +- .../BuiltinTableGeneratingFunctions.java | 4 + .../nereids/analyzer/UnboundTVFRelation.java | 10 +- .../nereids/parser/LogicalPlanBuilder.java | 30 +- .../doris/nereids/parser/NereidsParser.java | 5 + .../rules/analysis/BindExpression.java | 4 +- .../{TVFProperties.java => Properties.java} | 13 +- .../functions/generator/Explode.java | 68 ++ .../functions/generator/ExplodeOuter.java | 68 ++ .../functions/scalar/ConcatWs.java | 4 +- .../functions/scalar/Tokenize.java | 19 + .../expressions/functions/table/Hdfs.java | 4 +- .../expressions/functions/table/Numbers.java | 8 +- .../trees/expressions/functions/table/S3.java | 4 +- .../functions/table/TableValuedFunction.java | 8 +- .../visitor/ExpressionVisitor.java | 6 +- .../TableGeneratingFunctionVisitor.java | 10 + .../nereids_function_p0/gen_function/gen.out | 964 ++++++++++++++++++ .../scalar_function/Array.out | 174 ++++ .../nereids_function_p0/scalar_function/C.out | 111 +- .../gen_function/gen.groovy | 45 + .../scalar_function/Array.groovy | 41 + .../scalar_function/C.groovy | 6 + 23 files changed, 1565 insertions(+), 56 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/{TVFProperties.java => Properties.java} (89%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuter.java diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 111d30f394..2b014d3610 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -55,6 +55,11 @@ statement whereClause #delete ; +propertiesStatment + : properties+=property (COMMA properties+=property)* + ; + + // -----------------Command accessories----------------- identifierOrText @@ -89,7 +94,7 @@ planType outFileClause : INTO OUTFILE filePath=constant (FORMAT AS format=identifier)? - (PROPERTIES LEFT_PAREN properties+=tvfProperty (COMMA properties+=tvfProperty)* RIGHT_PAREN)? + (PROPERTIES LEFT_PAREN properties+=property (COMMA properties+=property)* RIGHT_PAREN)? ; query @@ -265,15 +270,15 @@ relationPrimary : multipartIdentifier specifiedPartition? tabletList? tableAlias relationHint? lateralView* #tableName | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN - (properties+=tvfProperty (COMMA properties+=tvfProperty)*)? + (properties+=property (COMMA properties+=property)*)? RIGHT_PAREN tableAlias #tableValuedFunction ; -tvfProperty - : key=tvfPropertyItem EQ value=tvfPropertyItem +property + : key=propertyItem EQ value=propertyItem ; -tvfPropertyItem : identifier | constant ; +propertyItem : identifier | constant ; tableAlias : (AS? strictIdentifier identifierList?)? diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java index b1dd9c8306..cdcebc4be9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableGeneratingFunctions.java @@ -17,6 +17,7 @@ package org.apache.doris.catalog; +import org.apache.doris.nereids.trees.expressions.functions.generator.Explode; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeBitmap; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeBitmapOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayDouble; @@ -29,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJso import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayStringOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbers; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbersOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplitOuter; @@ -44,6 +46,8 @@ import java.util.List; */ public class BuiltinTableGeneratingFunctions implements FunctionHelper { public final List tableGeneratingFunctions = ImmutableList.of( + tableGenerating(Explode.class, "explode"), + tableGenerating(ExplodeOuter.class, "explode_outer"), tableGenerating(ExplodeNumbers.class, "explode_numbers"), tableGenerating(ExplodeNumbersOuter.class, "explode_numbers_outer"), tableGenerating(ExplodeBitmap.class, "explode_bitmap"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java index 83c9d4fa28..51ffb722b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java @@ -22,8 +22,8 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -41,13 +41,13 @@ import java.util.Optional; public class UnboundTVFRelation extends LogicalRelation implements TVFRelation, Unbound { private final String functionName; - private final TVFProperties properties; + private final Properties properties; - public UnboundTVFRelation(RelationId id, String functionName, TVFProperties properties) { + public UnboundTVFRelation(RelationId id, String functionName, Properties properties) { this(id, functionName, properties, Optional.empty(), Optional.empty()); } - public UnboundTVFRelation(RelationId id, String functionName, TVFProperties properties, + public UnboundTVFRelation(RelationId id, String functionName, Properties properties, Optional groupExpression, Optional logicalProperties) { super(id, PlanType.LOGICAL_UNBOUND_TVF_RELATION, groupExpression, logicalProperties); this.functionName = Objects.requireNonNull(functionName, "functionName can not be null"); @@ -58,7 +58,7 @@ public class UnboundTVFRelation extends LogicalRelation implements TVFRelation, return functionName; } - public TVFProperties getProperties() { + public Properties getProperties() { return properties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 47c5796e3b..ac5d60af46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -80,6 +80,9 @@ import org.apache.doris.nereids.DorisParser.PlanTypeContext; import org.apache.doris.nereids.DorisParser.PredicateContext; import org.apache.doris.nereids.DorisParser.PredicatedContext; import org.apache.doris.nereids.DorisParser.PrimitiveDataTypeContext; +import org.apache.doris.nereids.DorisParser.PropertiesStatmentContext; +import org.apache.doris.nereids.DorisParser.PropertyContext; +import org.apache.doris.nereids.DorisParser.PropertyItemContext; import org.apache.doris.nereids.DorisParser.QualifiedNameContext; import org.apache.doris.nereids.DorisParser.QueryContext; import org.apache.doris.nereids.DorisParser.QueryOrganizationContext; @@ -103,8 +106,6 @@ import org.apache.doris.nereids.DorisParser.TableNameContext; import org.apache.doris.nereids.DorisParser.TableValuedFunctionContext; import org.apache.doris.nereids.DorisParser.TimestampaddContext; import org.apache.doris.nereids.DorisParser.TimestampdiffContext; -import org.apache.doris.nereids.DorisParser.TvfPropertyContext; -import org.apache.doris.nereids.DorisParser.TvfPropertyItemContext; import org.apache.doris.nereids.DorisParser.TypeConstructorContext; import org.apache.doris.nereids.DorisParser.UnitIdentifierContext; import org.apache.doris.nereids.DorisParser.UpdateAssignmentContext; @@ -165,11 +166,11 @@ import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Regexp; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.Subtract; -import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.WindowExpression; @@ -394,6 +395,17 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { return logicalPlans; } + @Override + public Properties visitPropertiesStatment(PropertiesStatmentContext ctx) { + Builder map = ImmutableMap.builder(); + for (PropertyContext argument : ctx.properties) { + String key = parsePropertyItem(argument.key); + String value = parsePropertyItem(argument.value); + map.put(key, value); + } + return new Properties(map.build()); + } + /* ******************************************************************************************** * Plan parsing * ******************************************************************************************** */ @@ -624,13 +636,13 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { String functionName = ctx.tvfName.getText(); Builder map = ImmutableMap.builder(); - for (TvfPropertyContext argument : ctx.properties) { - String key = parseTVFPropertyItem(argument.key); - String value = parseTVFPropertyItem(argument.value); + for (PropertyContext argument : ctx.properties) { + String key = parsePropertyItem(argument.key); + String value = parsePropertyItem(argument.value); map.put(key, value); } LogicalPlan relation = new UnboundTVFRelation(StatementScopeIdGenerator.newRelationId(), - functionName, new TVFProperties(map.build())); + functionName, new Properties(map.build())); return withTableAlias(relation, ctx.tableAlias()); }); } @@ -1499,7 +1511,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { format = ctx.format.getText(); } Map properties = Maps.newHashMap(); - for (TvfPropertyContext argument : ctx.properties) { + for (PropertyContext argument : ctx.properties) { String key = parseConstant(argument.key.constant()); String value = parseConstant(argument.value.constant()); properties.put(key, value); @@ -1935,7 +1947,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { } } - private String parseTVFPropertyItem(TvfPropertyItemContext item) { + private String parsePropertyItem(PropertyItemContext item) { if (item.constant() != null) { return parseConstant(item.constant()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 895f057a69..4ce8098a53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.DorisParser; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.DataType; @@ -79,6 +80,10 @@ public class NereidsParser { return parse(dataType, DorisParser::dataType); } + public Properties parseProperties(String properties) { + return parse(properties, DorisParser::propertiesStatment); + } + private T parse(String sql, Function parseFunction) { ParserRuleContext tree = toAst(sql, parseFunction); LogicalPlanBuilder logicalPlanBuilder = new LogicalPlanBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 331013549f..4981d84f4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -39,9 +39,9 @@ import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; @@ -697,7 +697,7 @@ public class BindExpression implements AnalysisRuleFactory { FunctionRegistry functionRegistry = env.getFunctionRegistry(); String functionName = unboundTVFRelation.getFunctionName(); - TVFProperties arguments = unboundTVFRelation.getProperties(); + Properties arguments = unboundTVFRelation.getProperties(); FunctionBuilder functionBuilder = functionRegistry.findFunctionBuilder(functionName, arguments); BoundFunction function = functionBuilder.build(functionName, arguments); if (!(function instanceof TableValuedFunction)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TVFProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java similarity index 89% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TVFProperties.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java index 6a8cfc585d..9c8e22e65b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TVFProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Properties.java @@ -30,12 +30,13 @@ import java.util.Objects; import java.util.stream.Collectors; /** - * TVFProperties + * Properties */ -public class TVFProperties extends Expression implements LeafExpression { +public class Properties extends Expression implements LeafExpression { + private final Map keyValues; - public TVFProperties(Map properties) { + public Properties(Map properties) { super(ImmutableList.of()); this.keyValues = Objects.requireNonNull(properties, "properties can not be null"); } @@ -65,7 +66,7 @@ public class TVFProperties extends Expression implements LeafExpression { @Override public String toString() { - return "TVFProperties(" + toSql() + ")"; + return "Properties(" + toSql() + ")"; } @Override @@ -79,7 +80,7 @@ public class TVFProperties extends Expression implements LeafExpression { if (!super.equals(o)) { return false; } - TVFProperties that = (TVFProperties) o; + Properties that = (Properties) o; return Objects.equals(keyValues, that.keyValues); } @@ -90,6 +91,6 @@ public class TVFProperties extends Expression implements LeafExpression { @Override public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitTVFProperties(this, context); + return visitor.visitProperties(this, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java new file mode 100644 index 0000000000..7fb87b055f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * explode([1, 2, 3]), generate 3 lines include 1, 2 and 3. + */ +public class Explode extends TableGeneratingFunction implements BinaryExpression, AlwaysNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(new FollowToAnyDataType(0)).args(ArrayType.of(new AnyDataType(0))) + ); + + /** + * constructor with 1 argument. + */ + public Explode(Expression arg) { + super("explode", arg); + } + + /** + * withChildren. + */ + @Override + public Explode withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new Explode(children.get(0)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitExplode(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuter.java new file mode 100644 index 0000000000..5d90546ebe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeOuter.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.generator; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * explode([1, 2, 3]), generate 3 lines include 1, 2 and 3. + */ +public class ExplodeOuter extends TableGeneratingFunction implements BinaryExpression, AlwaysNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(new FollowToAnyDataType(0)).args(ArrayType.of(new AnyDataType(0))) + ); + + /** + * constructor with 1 argument. + */ + public ExplodeOuter(Expression arg) { + super("explode_outer", arg); + } + + /** + * withChildren. + */ + @Override + public ExplodeOuter withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new ExplodeOuter(children.get(0)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitExplodeOuter(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConcatWs.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConcatWs.java index 107c4d2cd2..85be060a96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConcatWs.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConcatWs.java @@ -38,10 +38,10 @@ public class ConcatWs extends ScalarFunction implements ExplicitlyCastableSignature { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) - .varArgs(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) .args(VarcharType.SYSTEM_DEFAULT, ArrayType.of(VarcharType.SYSTEM_DEFAULT)), + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) + .varArgs(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(StringType.INSTANCE).varArgs(StringType.INSTANCE, StringType.INSTANCE) ); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Tokenize.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Tokenize.java index 7380a6d12a..8186f87864 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Tokenize.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Tokenize.java @@ -18,9 +18,12 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; @@ -50,6 +53,22 @@ public class Tokenize extends ScalarFunction super("tokenize", arg0, arg1); } + @Override + public void checkLegalityAfterRewrite() { + if (!(child(1) instanceof StringLikeLiteral)) { + throw new AnalysisException("tokenize second argument must be string literal"); + } + String properties = ((StringLikeLiteral) child(1)).value; + if (properties == null || properties.isEmpty()) { + return; + } + try { + new NereidsParser().parseProperties(((StringLikeLiteral) child(1)).value); + } catch (Throwable e) { + throw new AnalysisException("tokenize second argument must be properties format"); + } + } + /** * withChildren. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java index 29abbcee22..167c9b3e74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java @@ -19,8 +19,8 @@ package org.apache.doris.nereids.trees.expressions.functions.table; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.statistics.Statistics; @@ -33,7 +33,7 @@ import java.util.Map; /** hdfs */ public class Hdfs extends TableValuedFunction { - public Hdfs(TVFProperties properties) { + public Hdfs(Properties properties) { super("hdfs", properties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Numbers.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Numbers.java index e5af7477a2..303f862f69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Numbers.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Numbers.java @@ -24,8 +24,8 @@ import org.apache.doris.common.NereidsException; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.statistics.ColumnStatistic; @@ -42,7 +42,7 @@ import java.util.Map; /** Numbers */ public class Numbers extends TableValuedFunction { - public Numbers(TVFProperties properties) { + public Numbers(Properties properties) { super("numbers", properties); } @@ -100,7 +100,7 @@ public class Numbers extends TableValuedFunction { @Override public Numbers withChildren(List children) { Preconditions.checkArgument(children().size() == 1 - && children().get(0) instanceof TVFProperties); - return new Numbers((TVFProperties) children.get(0)); + && children().get(0) instanceof Properties); + return new Numbers((Properties) children.get(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java index 90e72f89f7..b74fc61a4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java @@ -19,8 +19,8 @@ package org.apache.doris.nereids.trees.expressions.functions.table; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.statistics.Statistics; @@ -33,7 +33,7 @@ import java.util.Map; /** s3 */ public class S3 extends TableValuedFunction { - public S3(TVFProperties properties) { + public S3(Properties properties) { super("s3", properties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java index eae1e25201..29019f6139 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java @@ -22,8 +22,8 @@ import org.apache.doris.catalog.FunctionGenTable; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; @@ -53,7 +53,7 @@ public abstract class TableValuedFunction extends BoundFunction implements Unary } }); - public TableValuedFunction(String functionName, TVFProperties tvfProperties) { + public TableValuedFunction(String functionName, Properties tvfProperties) { super(functionName, tvfProperties); } @@ -61,8 +61,8 @@ public abstract class TableValuedFunction extends BoundFunction implements Unary public abstract Statistics computeStats(List slots); - public TVFProperties getTVFProperties() { - return (TVFProperties) child(0); + public Properties getTVFProperties() { + return (Properties) child(0); } public final String getTableName() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index f056939ce5..73c1b608cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -64,12 +64,12 @@ import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.expressions.Subtract; -import org.apache.doris.nereids.trees.expressions.TVFProperties; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; import org.apache.doris.nereids.trees.expressions.UnaryArithmetic; import org.apache.doris.nereids.trees.expressions.UnaryOperator; @@ -415,8 +415,8 @@ public abstract class ExpressionVisitor return visit(variableDesc, context); } - public R visitTVFProperties(TVFProperties tvfProperties, C context) { - return visit(tvfProperties, context); + public R visitProperties(Properties properties, C context) { + return visit(properties, context); } public R visitInterval(Interval interval, C context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java index 040b279df8..5046e3c205 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableGeneratingFunctionVisitor.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.expressions.visitor; +import org.apache.doris.nereids.trees.expressions.functions.generator.Explode; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeBitmap; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeBitmapOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayDouble; @@ -29,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJso import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeJsonArrayStringOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbers; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeNumbersOuter; +import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplit; import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeSplitOuter; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; @@ -39,6 +41,14 @@ import org.apache.doris.nereids.trees.expressions.functions.generator.TableGener public interface TableGeneratingFunctionVisitor { R visitTableGeneratingFunction(TableGeneratingFunction tableGeneratingFunction, C context); + default R visitExplode(Explode explode, C context) { + return visitTableGeneratingFunction(explode, context); + } + + default R visitExplodeOuter(ExplodeOuter explodeOuter, C context) { + return visitTableGeneratingFunction(explodeOuter, context); + } + default R visitExplodeNumbers(ExplodeNumbers explodeNumbers, C context) { return visitTableGeneratingFunction(explodeNumbers, context); } diff --git a/regression-test/data/nereids_function_p0/gen_function/gen.out b/regression-test/data/nereids_function_p0/gen_function/gen.out index ce8d8f8de1..17f86d875e 100644 --- a/regression-test/data/nereids_function_p0/gen_function/gen.out +++ b/regression-test/data/nereids_function_p0/gen_function/gen.out @@ -593,3 +593,967 @@ 11 d 11 a +-- !sql_explode_json_array_int_Varchar -- +\N 1 +\N 2 +\N 3 +0 1 +0 2 +0 3 +1 1 +1 2 +1 3 +2 1 +2 2 +2 3 +3 1 +3 2 +3 3 +4 1 +4 2 +4 3 +5 1 +5 2 +5 3 +6 1 +6 2 +6 3 +7 1 +7 2 +7 3 +8 1 +8 2 +8 3 +9 1 +9 2 +9 3 +10 1 +10 2 +10 3 +11 1 +11 2 +11 3 + +-- !sql_explode_json_array_double_Varchar -- +\N 1.1 +\N 2.2 +\N 3.3 +0 1.1 +0 2.2 +0 3.3 +1 1.1 +1 2.2 +1 3.3 +2 1.1 +2 2.2 +2 3.3 +3 1.1 +3 2.2 +3 3.3 +4 1.1 +4 2.2 +4 3.3 +5 1.1 +5 2.2 +5 3.3 +6 1.1 +6 2.2 +6 3.3 +7 1.1 +7 2.2 +7 3.3 +8 1.1 +8 2.2 +8 3.3 +9 1.1 +9 2.2 +9 3.3 +10 1.1 +10 2.2 +10 3.3 +11 1.1 +11 2.2 +11 3.3 + +-- !sql_explode_json_array_string_Varchar -- +\N 1 +\N 2 +\N 3 +0 1 +0 2 +0 3 +1 1 +1 2 +1 3 +2 1 +2 2 +2 3 +3 1 +3 2 +3 3 +4 1 +4 2 +4 3 +5 1 +5 2 +5 3 +6 1 +6 2 +6 3 +7 1 +7 2 +7 3 +8 1 +8 2 +8 3 +9 1 +9 2 +9 3 +10 1 +10 2 +10 3 +11 1 +11 2 +11 3 + +-- !sql_explode_json_array_json_Varchar -- +\N {"id":1,"name":"John"} +\N {"id":2,"name":"Mary"} +\N {"id":3,"name":"Bob"} +0 {"id":1,"name":"John"} +0 {"id":2,"name":"Mary"} +0 {"id":3,"name":"Bob"} +1 {"id":1,"name":"John"} +1 {"id":2,"name":"Mary"} +1 {"id":3,"name":"Bob"} +2 {"id":1,"name":"John"} +2 {"id":2,"name":"Mary"} +2 {"id":3,"name":"Bob"} +3 {"id":1,"name":"John"} +3 {"id":2,"name":"Mary"} +3 {"id":3,"name":"Bob"} +4 {"id":1,"name":"John"} +4 {"id":2,"name":"Mary"} +4 {"id":3,"name":"Bob"} +5 {"id":1,"name":"John"} +5 {"id":2,"name":"Mary"} +5 {"id":3,"name":"Bob"} +6 {"id":1,"name":"John"} +6 {"id":2,"name":"Mary"} +6 {"id":3,"name":"Bob"} +7 {"id":1,"name":"John"} +7 {"id":2,"name":"Mary"} +7 {"id":3,"name":"Bob"} +8 {"id":1,"name":"John"} +8 {"id":2,"name":"Mary"} +8 {"id":3,"name":"Bob"} +9 {"id":1,"name":"John"} +9 {"id":2,"name":"Mary"} +9 {"id":3,"name":"Bob"} +10 {"id":1,"name":"John"} +10 {"id":2,"name":"Mary"} +10 {"id":3,"name":"Bob"} +11 {"id":1,"name":"John"} +11 {"id":2,"name":"Mary"} +11 {"id":3,"name":"Bob"} + +-- !sql_explode_Double -- +0 0.1 +1 0.2 +10 1.1 +11 1.2 +2 0.3 +3 0.4 +4 0.5 +5 0.6 +6 0.7 +7 0.8 +8 0.9 +9 1.0 + +-- !sql_explode_Double_notnull -- +0 0.1 +1 0.2 +10 1.1 +11 1.2 +2 0.3 +3 0.4 +4 0.5 +5 0.6 +6 0.7 +7 0.8 +8 0.9 +9 1.0 + +-- !sql_explode_Float -- +0 1.0 +1 2.0 +10 11.0 +11 12.0 +2 3.0 +3 4.0 +4 5.0 +5 6.0 +6 7.0 +7 8.0 +8 9.0 +9 10.0 + +-- !sql_explode_Float_notnull -- +0 1.0 +1 2.0 +10 11.0 +11 12.0 +2 3.0 +3 4.0 +4 5.0 +5 6.0 +6 7.0 +7 8.0 +8 9.0 +9 10.0 + +-- !sql_explode_LargeInt -- +0 1 +1 2 +10 11 +11 12 +2 3 +3 4 +4 5 +5 6 +6 7 +7 8 +8 9 +9 10 + +-- !sql_explode_LargeInt_notnull -- +0 1 +1 2 +10 11 +11 12 +2 3 +3 4 +4 5 +5 6 +6 7 +7 8 +8 9 +9 10 + +-- !sql_explode_BigInt -- +0 1 +1 2 +10 11 +11 12 +2 3 +3 4 +4 5 +5 6 +6 7 +7 8 +8 9 +9 10 + +-- !sql_explode_BigInt_notnull -- +0 1 +1 2 +10 11 +11 12 +2 3 +3 4 +4 5 +5 6 +6 7 +7 8 +8 9 +9 10 + +-- !sql_explode_SmallInt -- +0 1 +1 2 +10 11 +11 12 +2 3 +3 4 +4 5 +5 6 +6 7 +7 8 +8 9 +9 10 + +-- !sql_explode_SmallInt_notnull -- +0 1 +1 2 +10 11 +11 12 +2 3 +3 4 +4 5 +5 6 +6 7 +7 8 +8 9 +9 10 + +-- !sql_explode_Integer -- +0 1 +1 2 +10 11 +11 12 +2 3 +3 4 +4 5 +5 6 +6 7 +7 8 +8 9 +9 10 + +-- !sql_explode_Integer_notnull -- +0 1 +1 2 +10 11 +11 12 +2 3 +3 4 +4 5 +5 6 +6 7 +7 8 +8 9 +9 10 + +-- !sql_explode_TinyInt -- +0 0 +1 0 +10 1 +11 1 +2 0 +3 0 +4 0 +5 0 +6 0 +7 1 +8 1 +9 1 + +-- !sql_explode_TinyInt_notnull -- +0 0 +1 0 +10 1 +11 1 +2 0 +3 0 +4 0 +5 0 +6 0 +7 1 +8 1 +9 1 + +-- !sql_explode_DecimalV3 -- +0 0.100000000 +0 0.100000000 +1 0.200000000 +1 0.200000000 +10 1.100000000 +10 1.100000000 +11 1.200000000 +11 1.200000000 +2 0.300000000 +2 0.300000000 +3 0.400000000 +3 0.400000000 +4 0.500000000 +4 0.500000000 +5 0.600000000 +5 0.600000000 +6 0.700000000 +6 0.700000000 +7 0.800000000 +7 0.800000000 +8 0.900000000 +8 0.900000000 +9 1.000000000 +9 1.000000000 + +-- !sql_explode_DecimalV3_notnull -- +0 0.100000000 +0 0.100000000 +1 0.200000000 +1 0.200000000 +10 1.100000000 +10 1.100000000 +11 1.200000000 +11 1.200000000 +2 0.300000000 +2 0.300000000 +3 0.400000000 +3 0.400000000 +4 0.500000000 +4 0.500000000 +5 0.600000000 +5 0.600000000 +6 0.700000000 +6 0.700000000 +7 0.800000000 +7 0.800000000 +8 0.900000000 +8 0.900000000 +9 1.000000000 +9 1.000000000 + +-- !sql_explode_Boolean -- +0 false +1 false +10 true +11 true +2 false +3 false +4 false +5 false +6 false +7 true +8 true +9 true + +-- !sql_explode_Boolean_notnull -- +0 false +1 false +10 true +11 true +2 false +3 false +4 false +5 false +6 false +7 true +8 true +9 true + +-- !sql_explode_Char -- +0 char11 +0 char21 +0 char31 +1 char12 +1 char22 +1 char32 +10 char12 +10 char22 +10 char32 +11 char13 +11 char23 +11 char33 +2 char13 +2 char23 +2 char33 +3 char11 +3 char21 +3 char31 +4 char12 +4 char22 +4 char32 +5 char13 +5 char23 +5 char33 +6 char11 +6 char21 +6 char31 +7 char12 +7 char22 +7 char32 +8 char13 +8 char23 +8 char33 +9 char11 +9 char21 +9 char31 + +-- !sql_explode_Char_notnull -- +0 char11 +0 char21 +0 char31 +1 char12 +1 char22 +1 char32 +10 char12 +10 char22 +10 char32 +11 char13 +11 char23 +11 char33 +2 char13 +2 char23 +2 char33 +3 char11 +3 char21 +3 char31 +4 char12 +4 char22 +4 char32 +5 char13 +5 char23 +5 char33 +6 char11 +6 char21 +6 char31 +7 char12 +7 char22 +7 char32 +8 char13 +8 char23 +8 char33 +9 char11 +9 char21 +9 char31 + +-- !sql_explode_Varchar -- +0 char11 +0 char21 +0 char31 +0 varchar11 +0 varchar21 +0 varchar31 +1 char12 +1 char22 +1 char32 +1 varchar12 +1 varchar22 +1 varchar32 +10 char12 +10 char22 +10 char32 +10 varchar12 +10 varchar22 +10 varchar32 +11 char13 +11 char23 +11 char33 +11 varchar13 +11 varchar23 +11 varchar33 +2 char13 +2 char23 +2 char33 +2 varchar13 +2 varchar23 +2 varchar33 +3 char11 +3 char21 +3 char31 +3 varchar11 +3 varchar21 +3 varchar31 +4 char12 +4 char22 +4 char32 +4 varchar12 +4 varchar22 +4 varchar32 +5 char13 +5 char23 +5 char33 +5 varchar13 +5 varchar23 +5 varchar33 +6 char11 +6 char21 +6 char31 +6 varchar11 +6 varchar21 +6 varchar31 +7 char12 +7 char22 +7 char32 +7 varchar12 +7 varchar22 +7 varchar32 +8 char13 +8 char23 +8 char33 +8 varchar13 +8 varchar23 +8 varchar33 +9 char11 +9 char21 +9 char31 +9 varchar11 +9 varchar21 +9 varchar31 + +-- !sql_explode_Varchar_notnull -- +0 char11 +0 char21 +0 char31 +0 varchar11 +0 varchar21 +0 varchar31 +1 char12 +1 char22 +1 char32 +1 varchar12 +1 varchar22 +1 varchar32 +10 char12 +10 char22 +10 char32 +10 varchar12 +10 varchar22 +10 varchar32 +11 char13 +11 char23 +11 char33 +11 varchar13 +11 varchar23 +11 varchar33 +2 char13 +2 char23 +2 char33 +2 varchar13 +2 varchar23 +2 varchar33 +3 char11 +3 char21 +3 char31 +3 varchar11 +3 varchar21 +3 varchar31 +4 char12 +4 char22 +4 char32 +4 varchar12 +4 varchar22 +4 varchar32 +5 char13 +5 char23 +5 char33 +5 varchar13 +5 varchar23 +5 varchar33 +6 char11 +6 char21 +6 char31 +6 varchar11 +6 varchar21 +6 varchar31 +7 char12 +7 char22 +7 char32 +7 varchar12 +7 varchar22 +7 varchar32 +8 char13 +8 char23 +8 char33 +8 varchar13 +8 varchar23 +8 varchar33 +9 char11 +9 char21 +9 char31 +9 varchar11 +9 varchar21 +9 varchar31 + +-- !sql_explode_String -- +0 char21 +0 char31 +0 string1 +0 varchar11 +0 varchar21 +0 varchar31 +1 char22 +1 char32 +1 string2 +1 varchar12 +1 varchar22 +1 varchar32 +10 char22 +10 char32 +10 string2 +10 varchar12 +10 varchar22 +10 varchar32 +11 char23 +11 char33 +11 string3 +11 varchar13 +11 varchar23 +11 varchar33 +2 char23 +2 char33 +2 string3 +2 varchar13 +2 varchar23 +2 varchar33 +3 char21 +3 char31 +3 string1 +3 varchar11 +3 varchar21 +3 varchar31 +4 char22 +4 char32 +4 string2 +4 varchar12 +4 varchar22 +4 varchar32 +5 char23 +5 char33 +5 string3 +5 varchar13 +5 varchar23 +5 varchar33 +6 char21 +6 char31 +6 string1 +6 varchar11 +6 varchar21 +6 varchar31 +7 char22 +7 char32 +7 string2 +7 varchar12 +7 varchar22 +7 varchar32 +8 char23 +8 char33 +8 string3 +8 varchar13 +8 varchar23 +8 varchar33 +9 char21 +9 char31 +9 string1 +9 varchar11 +9 varchar21 +9 varchar31 + +-- !sql_explode_String_notnull -- +0 char21 +0 char31 +0 string1 +0 varchar11 +0 varchar21 +0 varchar31 +1 char22 +1 char32 +1 string2 +1 varchar12 +1 varchar22 +1 varchar32 +10 char22 +10 char32 +10 string2 +10 varchar12 +10 varchar22 +10 varchar32 +11 char23 +11 char33 +11 string3 +11 varchar13 +11 varchar23 +11 varchar33 +2 char23 +2 char33 +2 string3 +2 varchar13 +2 varchar23 +2 varchar33 +3 char21 +3 char31 +3 string1 +3 varchar11 +3 varchar21 +3 varchar31 +4 char22 +4 char32 +4 string2 +4 varchar12 +4 varchar22 +4 varchar32 +5 char23 +5 char33 +5 string3 +5 varchar13 +5 varchar23 +5 varchar33 +6 char21 +6 char31 +6 string1 +6 varchar11 +6 varchar21 +6 varchar31 +7 char22 +7 char32 +7 string2 +7 varchar12 +7 varchar22 +7 varchar32 +8 char23 +8 char33 +8 string3 +8 varchar13 +8 varchar23 +8 varchar33 +9 char21 +9 char31 +9 string1 +9 varchar11 +9 varchar21 +9 varchar31 + +-- !sql_explode_DatetimeV2 -- +0 2012-03-01T01:00:01 +0 2012-03-01T01:00:01 +0 2012-03-01T01:00:01 +0 2012-03-01T01:00:01 +1 2012-03-02T02:01:02 +1 2012-03-02T02:01:02 +1 2012-03-02T02:01:02 +1 2012-03-02T02:01:02 +10 2012-03-11T11:10:11 +10 2012-03-11T11:10:11 +10 2012-03-11T11:10:11 +10 2012-03-11T11:10:11 +11 2012-03-12T12:11:12 +11 2012-03-12T12:11:12 +11 2012-03-12T12:11:12 +11 2012-03-12T12:11:12 +2 2012-03-03T03:02:03 +2 2012-03-03T03:02:03 +2 2012-03-03T03:02:03 +2 2012-03-03T03:02:03 +3 2012-03-04T04:03:04 +3 2012-03-04T04:03:04 +3 2012-03-04T04:03:04 +3 2012-03-04T04:03:04 +4 2012-03-05T05:04:05 +4 2012-03-05T05:04:05 +4 2012-03-05T05:04:05 +4 2012-03-05T05:04:05 +5 2012-03-06T06:05:06 +5 2012-03-06T06:05:06 +5 2012-03-06T06:05:06 +5 2012-03-06T06:05:06 +6 2012-03-07T07:06:07 +6 2012-03-07T07:06:07 +6 2012-03-07T07:06:07 +6 2012-03-07T07:06:07 +7 2012-03-08T08:07:08 +7 2012-03-08T08:07:08 +7 2012-03-08T08:07:08 +7 2012-03-08T08:07:08 +8 2012-03-09T09:08:09 +8 2012-03-09T09:08:09 +8 2012-03-09T09:08:09 +8 2012-03-09T09:08:09 +9 2012-03-10T10:09:10 +9 2012-03-10T10:09:10 +9 2012-03-10T10:09:10 +9 2012-03-10T10:09:10 + +-- !sql_explode_DatetimeV2_notnull -- +0 2012-03-01T01:00:01 +0 2012-03-01T01:00:01 +0 2012-03-01T01:00:01 +0 2012-03-01T01:00:01 +1 2012-03-02T02:01:02 +1 2012-03-02T02:01:02 +1 2012-03-02T02:01:02 +1 2012-03-02T02:01:02 +10 2012-03-11T11:10:11 +10 2012-03-11T11:10:11 +10 2012-03-11T11:10:11 +10 2012-03-11T11:10:11 +11 2012-03-12T12:11:12 +11 2012-03-12T12:11:12 +11 2012-03-12T12:11:12 +11 2012-03-12T12:11:12 +2 2012-03-03T03:02:03 +2 2012-03-03T03:02:03 +2 2012-03-03T03:02:03 +2 2012-03-03T03:02:03 +3 2012-03-04T04:03:04 +3 2012-03-04T04:03:04 +3 2012-03-04T04:03:04 +3 2012-03-04T04:03:04 +4 2012-03-05T05:04:05 +4 2012-03-05T05:04:05 +4 2012-03-05T05:04:05 +4 2012-03-05T05:04:05 +5 2012-03-06T06:05:06 +5 2012-03-06T06:05:06 +5 2012-03-06T06:05:06 +5 2012-03-06T06:05:06 +6 2012-03-07T07:06:07 +6 2012-03-07T07:06:07 +6 2012-03-07T07:06:07 +6 2012-03-07T07:06:07 +7 2012-03-08T08:07:08 +7 2012-03-08T08:07:08 +7 2012-03-08T08:07:08 +7 2012-03-08T08:07:08 +8 2012-03-09T09:08:09 +8 2012-03-09T09:08:09 +8 2012-03-09T09:08:09 +8 2012-03-09T09:08:09 +9 2012-03-10T10:09:10 +9 2012-03-10T10:09:10 +9 2012-03-10T10:09:10 +9 2012-03-10T10:09:10 + +-- !sql_explode_DateV2 -- +0 2012-03-01 +0 2012-03-01 +1 2012-03-02 +1 2012-03-02 +10 2012-03-11 +10 2012-03-11 +11 2012-03-12 +11 2012-03-12 +2 2012-03-03 +2 2012-03-03 +3 2012-03-04 +3 2012-03-04 +4 2012-03-05 +4 2012-03-05 +5 2012-03-06 +5 2012-03-06 +6 2012-03-07 +6 2012-03-07 +7 2012-03-08 +7 2012-03-08 +8 2012-03-09 +8 2012-03-09 +9 2012-03-10 +9 2012-03-10 + +-- !sql_explode_DateV2_notnull -- +0 2012-03-01 +0 2012-03-01 +1 2012-03-02 +1 2012-03-02 +10 2012-03-11 +10 2012-03-11 +11 2012-03-12 +11 2012-03-12 +2 2012-03-03 +2 2012-03-03 +3 2012-03-04 +3 2012-03-04 +4 2012-03-05 +4 2012-03-05 +5 2012-03-06 +5 2012-03-06 +6 2012-03-07 +6 2012-03-07 +7 2012-03-08 +7 2012-03-08 +8 2012-03-09 +8 2012-03-09 +9 2012-03-10 +9 2012-03-10 + diff --git a/regression-test/data/nereids_function_p0/scalar_function/Array.out b/regression-test/data/nereids_function_p0/scalar_function/Array.out index dad029a71b..a255a65416 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/Array.out +++ b/regression-test/data/nereids_function_p0/scalar_function/Array.out @@ -9540,3 +9540,177 @@ true 2 2 +-- !sql_split_by_string_Char -- +["char11"] +["char11"] +["char11"] +["char11"] +["char12"] +["char12"] +["char12"] +["char12"] +["char13"] +["char13"] +["char13"] +["char13"] +["null"] + +-- !sql_split_by_string_Char_notnull -- +["char11"] +["char11"] +["char11"] +["char11"] +["char12"] +["char12"] +["char12"] +["char12"] +["char13"] +["char13"] +["char13"] +["char13"] + +-- !sql_split_by_string_VarChar -- +["null"] +["varchar11"] +["varchar11"] +["varchar11"] +["varchar11"] +["varchar12"] +["varchar12"] +["varchar12"] +["varchar12"] +["varchar13"] +["varchar13"] +["varchar13"] +["varchar13"] + +-- !sql_split_by_string_VarChar_notnull -- +["varchar11"] +["varchar11"] +["varchar11"] +["varchar11"] +["varchar12"] +["varchar12"] +["varchar12"] +["varchar12"] +["varchar13"] +["varchar13"] +["varchar13"] +["varchar13"] + +-- !sql_split_by_string_String -- +["null"] +["string1"] +["string1"] +["string1"] +["string1"] +["string2"] +["string2"] +["string2"] +["string2"] +["string3"] +["string3"] +["string3"] +["string3"] + +-- !sql_split_by_string_String_notnull -- +["string1"] +["string1"] +["string1"] +["string1"] +["string2"] +["string2"] +["string2"] +["string2"] +["string3"] +["string3"] +["string3"] +["string3"] + +-- !sql_tokenize_Char -- +["char11"] +["char11"] +["char11"] +["char11"] +["char12"] +["char12"] +["char12"] +["char12"] +["char13"] +["char13"] +["char13"] +["char13"] +["null"] + +-- !sql_tokenize_Char_notnull -- +["char11"] +["char11"] +["char11"] +["char11"] +["char12"] +["char12"] +["char12"] +["char12"] +["char13"] +["char13"] +["char13"] +["char13"] + +-- !sql_tokenize_VarChar -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !sql_tokenize_VarChar_notnull -- +["varchar11"] +["varchar11"] +["varchar11"] +["varchar11"] +["varchar12"] +["varchar12"] +["varchar12"] +["varchar12"] +["varchar13"] +["varchar13"] +["varchar13"] +["varchar13"] + +-- !sql_tokenize_String -- +["null"] +["string1"] +["string1"] +["string1"] +["string1"] +["string2"] +["string2"] +["string2"] +["string2"] +["string3"] +["string3"] +["string3"] +["string3"] + +-- !sql_tokenize_String_notnull -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + diff --git a/regression-test/data/nereids_function_p0/scalar_function/C.out b/regression-test/data/nereids_function_p0/scalar_function/C.out index 607f08be34..7f60e9d43f 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/C.out +++ b/regression-test/data/nereids_function_p0/scalar_function/C.out @@ -1,29 +1,29 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql_cbrt_Double -- \N -0.46415888336127786 -0.5848035476425731 -0.6694329500821694 -0.7368062997280772 +0.4641588833612779 +0.5848035476425733 +0.6694329500821695 +0.7368062997280773 0.7937005259840998 0.8434326653017492 0.8879040017426008 -0.9283177667225557 -0.9654893846056297 +0.9283177667225558 +0.9654893846056298 1.0 1.0322801154563672 1.0626585691826111 -- !sql_cbrt_Double_notnull -- -0.46415888336127786 -0.5848035476425731 -0.6694329500821694 -0.7368062997280772 +0.4641588833612779 +0.5848035476425733 +0.6694329500821695 +0.7368062997280773 0.7937005259840998 0.8434326653017492 0.8879040017426008 -0.9283177667225557 -0.9654893846056297 +0.9283177667225558 +0.9654893846056298 1.0 1.0322801154563672 1.0626585691826111 @@ -811,6 +811,93 @@ string3 string3 string3 +-- !sql_concat_ws_Varchar -- +null +varchar11 +varchar11 +varchar11 +varchar11 +varchar12 +varchar12 +varchar12 +varchar12 +varchar13 +varchar13 +varchar13 +varchar13 + +-- !sql_concat_ws_Varchar_notnull -- +varchar11 +varchar11 +varchar11 +varchar11 +varchar12 +varchar12 +varchar12 +varchar12 +varchar13 +varchar13 +varchar13 +varchar13 + +-- !sql_concat_ws_String -- +null +string1 +string1 +string1 +string1 +string2 +string2 +string2 +string2 +string3 +string3 +string3 +string3 + +-- !sql_concat_ws_String_notnull -- +string1 +string1 +string1 +string1 +string2 +string2 +string2 +string2 +string3 +string3 +string3 +string3 + +-- !sql_concat_ws_ArrayVarchar -- + +char11, char21, char31, varchar11, varchar21, varchar31 +char11, char21, char31, varchar11, varchar21, varchar31 +char11, char21, char31, varchar11, varchar21, varchar31 +char11, char21, char31, varchar11, varchar21, varchar31 +char12, char22, char32, varchar12, varchar22, varchar32 +char12, char22, char32, varchar12, varchar22, varchar32 +char12, char22, char32, varchar12, varchar22, varchar32 +char12, char22, char32, varchar12, varchar22, varchar32 +char13, char23, char33, varchar13, varchar23, varchar33 +char13, char23, char33, varchar13, varchar23, varchar33 +char13, char23, char33, varchar13, varchar23, varchar33 +char13, char23, char33, varchar13, varchar23, varchar33 + +-- !sql_concat_ws_ArrayVarchar_notnull -- +char11, char21, char31, varchar11, varchar21, varchar31 +char11, char21, char31, varchar11, varchar21, varchar31 +char11, char21, char31, varchar11, varchar21, varchar31 +char11, char21, char31, varchar11, varchar21, varchar31 +char12, char22, char32, varchar12, varchar22, varchar32 +char12, char22, char32, varchar12, varchar22, varchar32 +char12, char22, char32, varchar12, varchar22, varchar32 +char12, char22, char32, varchar12, varchar22, varchar32 +char13, char23, char33, varchar13, varchar23, varchar33 +char13, char23, char33, varchar13, varchar23, varchar33 +char13, char23, char33, varchar13, varchar23, varchar33 +char13, char23, char33, varchar13, varchar23, varchar33 + -- !sql_conv_BigInt_TinyInt_TinyInt -- \N \N diff --git a/regression-test/suites/nereids_function_p0/gen_function/gen.groovy b/regression-test/suites/nereids_function_p0/gen_function/gen.groovy index 4ca054ab25..547ee40a22 100644 --- a/regression-test/suites/nereids_function_p0/gen_function/gen.groovy +++ b/regression-test/suites/nereids_function_p0/gen_function/gen.groovy @@ -19,6 +19,7 @@ suite("nereids_gen_fn") { sql 'use regression_test_nereids_function_p0' sql 'set enable_nereids_planner=true' sql 'set enable_fallback_to_original_planner=false' + qt_sql_explode_bitmap_Bitmap ''' select id, e from fn_test lateral view explode_bitmap(to_bitmap(kbint)) lv as e order by id, e''' qt_sql_explode_bitmap_Bitmap_notnull ''' @@ -49,4 +50,48 @@ suite("nereids_gen_fn") { qt_sql_explode_split_outer_Varchar_Varchar_notnull ''' select id, e from fn_test lateral view explode_split_outer('a, b, c, d', ',') lv as e order by id, e''' + qt_sql_explode_json_array_int_Varchar ''' + select id, e from fn_test lateral view explode_json_array_int('[1, 2, 3]') lv as e order by id, e''' + + qt_sql_explode_json_array_double_Varchar ''' + select id, e from fn_test lateral view explode_json_array_double('[1.1, 2.2, 3.3]') lv as e order by id, e''' + + qt_sql_explode_json_array_string_Varchar ''' + select id, e from fn_test lateral view explode_json_array_string('["1", "2", "3"]') lv as e order by id, e''' + + qt_sql_explode_json_array_json_Varchar ''' + select id, e from fn_test lateral view explode_json_array_json('[{"id":1,"name":"John"},{"id":2,"name":"Mary"},{"id":3,"name":"Bob"}]') lv as e order by id, e''' + + // explode + order_qt_sql_explode_Double "select id, e from fn_test lateral view explode(kadbl) lv as e order by id, e" + order_qt_sql_explode_Double_notnull "select id, e from fn_test_not_nullable lateral view explode(kadbl) lv as e order by id, e" + order_qt_sql_explode_Float "select id, e from fn_test lateral view explode(kafloat) lv as e order by id, e" + order_qt_sql_explode_Float_notnull "select id, e from fn_test_not_nullable lateral view explode(kafloat) lv as e order by id, e" + order_qt_sql_explode_LargeInt "select id, e from fn_test lateral view explode(kalint) lv as e order by id, e" + order_qt_sql_explode_LargeInt_notnull "select id, e from fn_test_not_nullable lateral view explode(kalint) lv as e order by id, e" + order_qt_sql_explode_BigInt "select id, e from fn_test lateral view explode(kabint) lv as e order by id, e" + order_qt_sql_explode_BigInt_notnull "select id, e from fn_test_not_nullable lateral view explode(kabint) lv as e order by id, e" + order_qt_sql_explode_SmallInt "select id, e from fn_test lateral view explode(kasint) lv as e order by id, e" + order_qt_sql_explode_SmallInt_notnull "select id, e from fn_test_not_nullable lateral view explode(kasint) lv as e order by id, e" + order_qt_sql_explode_Integer "select id, e from fn_test lateral view explode(kaint) lv as e order by id, e" + order_qt_sql_explode_Integer_notnull "select id, e from fn_test_not_nullable lateral view explode(kaint) lv as e order by id, e" + order_qt_sql_explode_TinyInt "select id, e from fn_test lateral view explode(katint) lv as e order by id, e" + order_qt_sql_explode_TinyInt_notnull "select id, e from fn_test_not_nullable lateral view explode(katint) lv as e order by id, e" + order_qt_sql_explode_DecimalV3 "select id, e from fn_test lateral view explode(kadcml) lv as e order by id, e" + order_qt_sql_explode_DecimalV3_notnull "select id, e from fn_test_not_nullable lateral view explode(kadcml) lv as e order by id, e" + + order_qt_sql_explode_Boolean "select id, e from fn_test lateral view explode(kabool) lv as e order by id, e" + order_qt_sql_explode_Boolean_notnull "select id, e from fn_test_not_nullable lateral view explode(kabool) lv as e order by id, e" + + order_qt_sql_explode_Char "select id, e from fn_test lateral view explode(kachr) lv as e order by id, e" + order_qt_sql_explode_Char_notnull "select id, e from fn_test_not_nullable lateral view explode(kachr) lv as e order by id, e" + order_qt_sql_explode_Varchar "select id, e from fn_test lateral view explode(kavchr) lv as e order by id, e" + order_qt_sql_explode_Varchar_notnull "select id, e from fn_test_not_nullable lateral view explode(kavchr) lv as e order by id, e" + order_qt_sql_explode_String "select id, e from fn_test lateral view explode(kastr) lv as e order by id, e" + order_qt_sql_explode_String_notnull "select id, e from fn_test_not_nullable lateral view explode(kastr) lv as e order by id, e" + + order_qt_sql_explode_DatetimeV2 "select id, e from fn_test lateral view explode(kadtmv2) lv as e order by id, e" + order_qt_sql_explode_DatetimeV2_notnull "select id, e from fn_test_not_nullable lateral view explode(kadtmv2) lv as e order by id, e" + order_qt_sql_explode_DateV2 "select id, e from fn_test lateral view explode(kadtv2) lv as e order by id, e" + order_qt_sql_explode_DateV2_notnull "select id, e from fn_test_not_nullable lateral view explode(kadtv2) lv as e order by id, e" } diff --git a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy index e2466d24b1..507540e4c9 100644 --- a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy +++ b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy @@ -809,4 +809,45 @@ suite("nereids_scalar_fn_Array") { order_qt_sql_size_DateV2 "select size(kadtv2) from fn_test" order_qt_sql_size_DateV2_notnull "select size(kadtv2) from fn_test_not_nullable" + // split_by_string + order_qt_sql_split_by_string_Char "select split_by_string(kchrs1, ',') from fn_test" + order_qt_sql_split_by_string_Char_notnull "select split_by_string(kchrs1, ',') from fn_test_not_nullable" + order_qt_sql_split_by_string_VarChar "select split_by_string(kvchrs1, ',') from fn_test" + order_qt_sql_split_by_string_VarChar_notnull "select split_by_string(kvchrs1, ',') from fn_test_not_nullable" + order_qt_sql_split_by_string_String "select split_by_string(kstr, ',') from fn_test" + order_qt_sql_split_by_string_String_notnull "select split_by_string(kstr, ',') from fn_test_not_nullable" + + // tokenize + order_qt_sql_tokenize_Char "select tokenize(kchrs1, '') from fn_test" + order_qt_sql_tokenize_Char_notnull "select tokenize(kchrs1, '') from fn_test_not_nullable" + order_qt_sql_tokenize_VarChar "select tokenize(kvchrs1, null) from fn_test" + order_qt_sql_tokenize_VarChar_notnull "select tokenize(kvchrs1, '') from fn_test_not_nullable" + order_qt_sql_tokenize_String "select tokenize(kstr, '') from fn_test" + order_qt_sql_tokenize_String_notnull "select tokenize(kstr, null) from fn_test_not_nullable" + + test { + sql "select tokenize('arg1','xxx = yyy,zzz');" + check{result, exception, startTime, endTime -> + assertTrue(exception != null) + logger.info(exception.message) + } + } + + test { + sql "select tokenize('arg1','2');" + check{result, exception, startTime, endTime -> + assertTrue(exception != null) + logger.info(exception.message) + } + } + + test { + sql "select tokenize(kstr, kstr) from fn_test" + check{result, exception, startTime, endTime -> + assertTrue(exception != null) + logger.info(exception.message) + } + } + + } \ No newline at end of file diff --git a/regression-test/suites/nereids_function_p0/scalar_function/C.groovy b/regression-test/suites/nereids_function_p0/scalar_function/C.groovy index e6b124e9a3..47ff0b9b5d 100644 --- a/regression-test/suites/nereids_function_p0/scalar_function/C.groovy +++ b/regression-test/suites/nereids_function_p0/scalar_function/C.groovy @@ -75,6 +75,12 @@ suite("nereids_scalar_fn_C") { qt_sql_concat_Varchar_notnull "select concat(kvchrs1) from fn_test_not_nullable order by kvchrs1" qt_sql_concat_String "select concat(kstr) from fn_test order by kstr" qt_sql_concat_String_notnull "select concat(kstr) from fn_test_not_nullable order by kstr" + qt_sql_concat_ws_Varchar "select concat_ws(', ', kvchrs1) from fn_test order by kvchrs1" + qt_sql_concat_ws_Varchar_notnull "select concat_ws(', ', kvchrs1) from fn_test_not_nullable order by kvchrs1" + qt_sql_concat_ws_String "select concat_ws(', ', kstr) from fn_test order by kstr" + qt_sql_concat_ws_String_notnull "select concat_ws(', ', kstr) from fn_test_not_nullable order by kstr" + order_qt_sql_concat_ws_ArrayVarchar "select concat_ws(', ', kavchr) from fn_test" + order_qt_sql_concat_ws_ArrayVarchar_notnull "select concat_ws(', ', kavchr) from fn_test_not_nullable" sql "select connection_id() from fn_test" sql "select connection_id() from fn_test_not_nullable" qt_sql_conv_BigInt_TinyInt_TinyInt "select conv(kbint, ktint, ktint) from fn_test order by kbint, ktint, ktint"