From cf04c9c300704ce14d5730437459bb657da818bb Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 12 Mar 2024 17:09:38 +0800 Subject: [PATCH] [enhancement](Nereids) refine and speedup analyzer (#31792) (#32111) ## Proposed changes 1. check data type whether can applied should not throw exception when real data type is subclass of signature data type 2. merge `SlotBinder` and `FunctionBinder` to `ExpressionAnalyzer` to skip rewrite the whole expression tree multiple times. 3. `ExpressionAnalyzer.buildCustomSlotBinderAnalyzer()` provide more refined code to bind slot by different parts and different priority 4. the origin slot binder has O(n^2) complexity, this pr use `Scope.nameToSlot` to support O(n) bind 5. modify some `Collection.stream()` to `ImmutableXxx.builder()` to remove some method call which are difficult to inline by jvm in the hot path, e.g. `Expression.` and `AbstractTreeNode.` 6. modify some `ImmutableXxx.copyOf(xxx)` to `Utils.fastToImmutableList(xxx)` to skip addition copy of the array 7. set init size to `Immutable.builder()` to skip some useless resize 8. lazy compute and cache some heavy operations, like `Scope.nameToSlot` and `CaseWhen.computeDataTypesForCoercion()` (cherry picked from commit 83c2f5a95827136aac4f0a78c5e841e9a099858c) --- .../doris/catalog/FunctionRegistry.java | 22 +- .../doris/catalog/FunctionSignature.java | 3 +- .../apache/doris/nereids/CascadesContext.java | 9 + .../nereids/analyzer/ComplexDataType.java | 22 + .../doris/nereids/analyzer/MappingSlot.java | 115 ++ .../apache/doris/nereids/analyzer/Scope.java | 37 +- .../jobs/rewrite/PlanTreeRewriteJob.java | 18 +- .../analysis/AvgDistinctToSumDivCount.java | 2 +- .../rules/analysis/BindExpression.java | 1229 ++++++++--------- .../rules/analysis/BindSlotWithPaths.java | 2 +- .../nereids/rules/analysis/CheckAnalysis.java | 12 +- .../analysis/EliminateLogicalSelectHint.java | 2 +- .../rules/analysis/ExpressionAnalyzer.java | 807 +++++++++++ .../nereids/rules/analysis/SlotBinder.java | 2 +- .../rules/analysis/SubExprAnalyzer.java | 19 +- .../PushDownLimitDistinctThroughUnion.java | 2 +- .../rules/rewrite/PushProjectIntoUnion.java | 2 +- .../rewrite/PushProjectThroughUnion.java | 2 +- .../SelectMaterializedIndexWithAggregate.java | 2 +- .../doris/nereids/trees/AbstractTreeNode.java | 10 +- .../apache/doris/nereids/trees/TreeNode.java | 17 +- .../nereids/trees/expressions/CaseWhen.java | 31 +- .../nereids/trees/expressions/Expression.java | 65 +- .../trees/expressions/SlotReference.java | 4 +- .../functions/ComputeSignatureHelper.java | 35 +- .../ExplicitlyCastableSignature.java | 4 + .../functions/IdenticalSignature.java | 4 + .../ImplicitlyCastableSignature.java | 10 +- .../functions/NullOrIdenticalSignature.java | 4 + .../visitor/DefaultExpressionRewriter.java | 48 +- .../trees/plans/logical/LogicalGenerate.java | 6 +- .../trees/plans/logical/LogicalJoin.java | 6 +- .../trees/plans/logical/LogicalOlapScan.java | 21 +- .../trees/plans/logical/LogicalProject.java | 13 +- .../plans/logical/LogicalSetOperation.java | 30 +- .../trees/plans/logical/LogicalUnion.java | 4 +- .../plans/visitor/InferPlanOutputAlias.java | 72 +- .../apache/doris/nereids/types/ArrayType.java | 3 +- .../apache/doris/nereids/types/MapType.java | 3 +- .../doris/nereids/types/StructType.java | 3 +- .../doris/nereids/util/ExpressionUtils.java | 169 +-- .../apache/doris/nereids/util/JoinUtils.java | 8 +- .../apache/doris/nereids/util/PlanUtils.java | 18 + .../doris/nereids/util/TypeCoercionUtils.java | 82 +- .../org/apache/doris/nereids/util/Utils.java | 69 + .../doris/regression/RegressionTest.groovy | 11 +- .../doris/regression/util/OutputUtils.groovy | 2 +- 47 files changed, 2097 insertions(+), 964 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/ComplexDataType.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/MappingSlot.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java index 82a09d7e04..5552b3d1d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java @@ -98,10 +98,15 @@ public class FunctionRegistry { String combinatorSuffix = AggCombinerFunctionBuilder.getCombinatorSuffix(name); functionBuilders = name2InternalBuiltinBuilders.get(nestedName.toLowerCase()); if (functionBuilders != null) { - functionBuilders = functionBuilders.stream() - .map(builder -> new AggCombinerFunctionBuilder(combinatorSuffix, builder)) - .filter(functionBuilder -> functionBuilder.canApply(arguments)) - .collect(Collectors.toList()); + List candidateBuilders = Lists.newArrayListWithCapacity(functionBuilders.size()); + for (FunctionBuilder functionBuilder : functionBuilders) { + AggCombinerFunctionBuilder combinerBuilder + = new AggCombinerFunctionBuilder(combinatorSuffix, functionBuilder); + if (combinerBuilder.canApply(arguments)) { + candidateBuilders.add(combinerBuilder); + } + } + functionBuilders = candidateBuilders; } } } @@ -115,9 +120,12 @@ public class FunctionRegistry { } // check the arity and type - List candidateBuilders = functionBuilders.stream() - .filter(functionBuilder -> functionBuilder.canApply(arguments)) - .collect(Collectors.toList()); + List candidateBuilders = Lists.newArrayListWithCapacity(arguments.size()); + for (FunctionBuilder functionBuilder : functionBuilders) { + if (functionBuilder.canApply(arguments)) { + candidateBuilders.add(functionBuilder); + } + } if (candidateBuilders.isEmpty()) { String candidateHints = getCandidateHint(name, functionBuilders); throw new AnalysisException("Can not found function '" + qualifiedName diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSignature.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSignature.java index 1cb1573233..4ff9448a33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSignature.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSignature.java @@ -20,6 +20,7 @@ package org.apache.doris.catalog; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.coercion.FollowToArgumentType; +import org.apache.doris.nereids.util.Utils; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; @@ -40,7 +41,7 @@ public class FunctionSignature { private FunctionSignature(DataType returnType, boolean hasVarArgs, List argumentsTypes) { this.returnType = Objects.requireNonNull(returnType, "returnType is not null"); - this.argumentsTypes = ImmutableList.copyOf( + this.argumentsTypes = Utils.fastToImmutableList( Objects.requireNonNull(argumentsTypes, "argumentsTypes is not null")); this.hasVarArgs = hasVarArgs; this.arity = argumentsTypes.size(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 18fad8d294..8e4a47938e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -72,6 +72,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.HashMap; @@ -92,6 +94,7 @@ import javax.annotation.Nullable; * Context used in memo. */ public class CascadesContext implements ScheduleContext { + private static final Logger LOG = LogManager.getLogger(CascadesContext.class); // in analyze/rewrite stage, the plan will storage in this field private Plan plan; @@ -713,4 +716,10 @@ public class CascadesContext implements ScheduleContext { task.run(); } } + + public void printPlanProcess() { + for (PlanProcess row : planProcesses) { + LOG.info("RULE: " + row.ruleName + "\nBEFORE:\n" + row.beforeShape + "\nafter:\n" + row.afterShape); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/ComplexDataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/ComplexDataType.java new file mode 100644 index 0000000000..45389c9589 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/ComplexDataType.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +/** ComplexDataType */ +public interface ComplexDataType { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/MappingSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/MappingSlot.java new file mode 100644 index 0000000000..7e5b92e5a2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/MappingSlot.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; + +import java.util.List; + +/** + * MappingSlot. + * mapping slot to an expression, use to replace slot to this expression. + * this class only use in Scope, and **NEVER** appear in the expression tree + */ +public class MappingSlot extends Slot { + private final Slot slot; + private final Expression mappingExpression; + + public MappingSlot(Slot slot, Expression mappingExpression) { + this.slot = slot; + this.mappingExpression = mappingExpression; + } + + public Slot getRealSlot() { + return slot; + } + + @Override + public List getQualifier() { + return slot.getQualifier(); + } + + public Expression getMappingExpression() { + return mappingExpression; + } + + @Override + public ExprId getExprId() { + return slot.getExprId(); + } + + @Override + public String getName() throws UnboundException { + return slot.getName(); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitSlot(this, context); + } + + @Override + public boolean nullable() { + return slot.nullable(); + } + + @Override + public String toSql() { + return slot.toSql(); + } + + @Override + public String toString() { + return slot.toString(); + } + + @Override + public DataType getDataType() throws UnboundException { + return slot.getDataType(); + } + + @Override + public String getInternalName() { + return slot.getInternalName(); + } + + @Override + public Slot withName(String name) { + return this; + } + + @Override + public Slot withNullable(boolean newNullable) { + return this; + } + + @Override + public Slot withExprId(ExprId exprId) { + return this; + } + + @Override + public Slot withQualifier(List qualifier) { + return this; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java index 7976261c7a..a95e562f7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java @@ -19,14 +19,19 @@ package org.apache.doris.nereids.analyzer; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.util.Utils; -import com.google.common.collect.ImmutableList; +import com.google.common.base.Suppliers; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; import com.google.common.collect.Sets; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; /** * The slot range required for expression analyze. @@ -58,16 +63,19 @@ public class Scope { private final List slots; private final Optional ownerSubquery; private final Set correlatedSlots; + private final Supplier> nameToSlot; - public Scope(Optional outerScope, List slots, Optional subqueryExpr) { - this.outerScope = outerScope; - this.slots = ImmutableList.copyOf(Objects.requireNonNull(slots, "slots can not be null")); - this.ownerSubquery = subqueryExpr; - this.correlatedSlots = Sets.newLinkedHashSet(); + public Scope(List slots) { + this(Optional.empty(), slots, Optional.empty()); } - public Scope(List slots) { - this(Optional.empty(), slots, Optional.empty()); + /** Scope */ + public Scope(Optional outerScope, List slots, Optional subqueryExpr) { + this.outerScope = Objects.requireNonNull(outerScope, "outerScope can not be null"); + this.slots = Utils.fastToImmutableList(Objects.requireNonNull(slots, "slots can not be null")); + this.ownerSubquery = Objects.requireNonNull(subqueryExpr, "subqueryExpr can not be null"); + this.correlatedSlots = Sets.newLinkedHashSet(); + this.nameToSlot = Suppliers.memoize(this::buildNameToSlot); } public List getSlots() { @@ -85,4 +93,17 @@ public class Scope { public Set getCorrelatedSlots() { return correlatedSlots; } + + /** findSlotIgnoreCase */ + public List findSlotIgnoreCase(String slotName) { + return nameToSlot.get().get(slotName.toUpperCase(Locale.ROOT)); + } + + private ListMultimap buildNameToSlot() { + ListMultimap map = LinkedListMultimap.create(slots.size()); + for (Slot slot : slots) { + map.put(slot.getName().toUpperCase(Locale.ROOT), slot); + } + return map; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java index e3c58f36ee..affbb9196c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.jobs.rewrite; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.PlanProcess; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.jobs.Job; import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; @@ -27,8 +28,6 @@ import org.apache.doris.nereids.pattern.Pattern; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.trees.plans.Plan; -import com.google.common.base.Preconditions; - import java.util.List; /** PlanTreeRewriteJob */ @@ -38,11 +37,9 @@ public abstract class PlanTreeRewriteJob extends Job { super(type, context); } - protected RewriteResult rewrite(Plan plan, List rules, RewriteJobContext rewriteJobContext) { - // boolean traceEnable = isTraceEnable(context); - boolean isRewriteRoot = rewriteJobContext.isRewriteRoot(); + protected final RewriteResult rewrite(Plan plan, List rules, RewriteJobContext rewriteJobContext) { CascadesContext cascadesContext = context.getCascadesContext(); - cascadesContext.setIsRewriteRoot(isRewriteRoot); + cascadesContext.setIsRewriteRoot(rewriteJobContext.isRewriteRoot()); boolean showPlanProcess = cascadesContext.showPlanProcess(); for (Rule rule : rules) { @@ -52,8 +49,9 @@ public abstract class PlanTreeRewriteJob extends Job { Pattern pattern = (Pattern) rule.getPattern(); if (pattern.matchPlanTree(plan)) { List newPlans = rule.transform(plan, cascadesContext); - Preconditions.checkState(newPlans.size() == 1, - "Rewrite rule should generate one plan: " + rule.getRuleType()); + if (newPlans.size() != 1) { + throw new AnalysisException("Rewrite rule should generate one plan: " + rule.getRuleType()); + } Plan newPlan = newPlans.get(0); if (!newPlan.deepEquals(plan)) { // don't remove this comment, it can help us to trace some bug when developing. @@ -78,13 +76,13 @@ public abstract class PlanTreeRewriteJob extends Job { return new RewriteResult(false, plan); } - protected Plan linkChildrenAndParent(Plan plan, RewriteJobContext rewriteJobContext) { + protected final Plan linkChildrenAndParent(Plan plan, RewriteJobContext rewriteJobContext) { Plan newPlan = linkChildren(plan, rewriteJobContext.childrenContext); rewriteJobContext.setResult(newPlan); return newPlan; } - protected Plan linkChildren(Plan plan, RewriteJobContext[] childrenContext) { + protected final Plan linkChildren(Plan plan, RewriteJobContext[] childrenContext) { boolean changed = false; Plan[] newChildren = new Plan[childrenContext.length]; for (int i = 0; i < childrenContext.length; ++i) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java index 4ed79e7384..8c27ee8e18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java @@ -59,7 +59,7 @@ public class AvgDistinctToSumDivCount extends OneRewriteRuleFactory { })); if (!avgToSumDivCount.isEmpty()) { List newOutput = agg.getOutputExpressions().stream() - .map(expr -> (NamedExpression) ExpressionUtils.replace(expr, avgToSumDivCount)) + .map(expr -> ExpressionUtils.replaceNameExpression(expr, avgToSumDivCount)) .collect(ImmutableList.toImmutableList()); return new LogicalAggregate<>(agg.getGroupByExpressions(), newOutput, agg.child()); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 9704de7e25..a561507fb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -20,21 +20,22 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FunctionRegistry; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.MappingSlot; import org.apache.doris.nereids.analyzer.Scope; -import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.LogicalPlanBuilder; +import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.AppliedAwareRule.AppliedAwareRuleCondition; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; -import org.apache.doris.nereids.rules.expression.rules.FunctionBinder; -import org.apache.doris.nereids.trees.UnaryNode; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.DefaultValueSlot; @@ -46,7 +47,6 @@ import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; -import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; @@ -56,15 +56,13 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; -import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; @@ -88,44 +86,38 @@ import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; -import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * BindSlotReference. */ @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class BindExpression implements AnalysisRuleFactory { - - private Scope toScope(CascadesContext cascadesContext, List slots) { - Optional outerScope = cascadesContext.getOuterScope(); - if (outerScope.isPresent()) { - return new Scope(outerScope, slots, outerScope.get().getSubquery()); - } else { - return new Scope(slots); - } - } + public static final Logger LOG = LogManager.getLogger(NereidsPlanner.class); @Override public List buildRules() { @@ -150,555 +142,511 @@ public class BindExpression implements AnalysisRuleFactory { return ImmutableList.of( RuleType.BINDING_PROJECT_SLOT.build( - logicalProject().thenApply(ctx -> { - LogicalProject project = ctx.root; - List boundProjections = - bindSlot(project.getProjects(), project.child(), ctx.cascadesContext); - if (boundProjections.stream().anyMatch(BoundStar.class::isInstance)) { - List boundExceptions = project.getExcepts().isEmpty() ? ImmutableList.of() - : bindSlot(project.getExcepts(), project.child(), ctx.cascadesContext); - boundProjections = flatBoundStar(boundProjections, boundExceptions); - } - boundProjections = boundProjections.stream() - .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) - .collect(ImmutableList.toImmutableList()); - return project.withProjects(boundProjections); - }) + logicalProject().thenApply(this::bindProject) ), RuleType.BINDING_FILTER_SLOT.build( - logicalFilter().thenApply(ctx -> { - LogicalFilter filter = ctx.root; - Set boundConjuncts = filter.getConjuncts().stream() - .map(expr -> bindSlot(expr, filter.child(), ctx.cascadesContext)) - .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) - .map(expr -> TypeCoercionUtils.castIfNotSameType(expr, BooleanType.INSTANCE)) - .collect(ImmutableSet.toImmutableSet()); - return new LogicalFilter<>(boundConjuncts, filter.child()); - }) + logicalFilter().thenApply(this::bindFilter) ), - RuleType.BINDING_USING_JOIN_SLOT.build( - usingJoin().thenApply(ctx -> { - UsingJoin using = ctx.root; - LogicalJoin lj = new LogicalJoin<>(using.getJoinType() == JoinType.CROSS_JOIN - ? JoinType.INNER_JOIN : using.getJoinType(), - using.getHashJoinConjuncts(), - using.getOtherJoinConjuncts(), using.getDistributeHint(), using.getMarkJoinSlotReference(), - using.children(), null); - List unboundSlots = lj.getHashJoinConjuncts(); - Set slotNames = new HashSet<>(); - List leftOutput = new ArrayList<>(lj.left().getOutput()); - // Suppose A JOIN B USING(name) JOIN C USING(name), [A JOIN B] is the left node, in this case, - // C should combine with table B on C.name=B.name. so we reverse the output to make sure that - // the most right slot is matched with priority. - Collections.reverse(leftOutput); - List leftSlots = new ArrayList<>(); - Scope scope = toScope(ctx.cascadesContext, leftOutput.stream() - .filter(s -> !slotNames.contains(s.getName())) - .peek(s -> slotNames.add(s.getName())) - .collect(Collectors.toList())); - for (Expression unboundSlot : unboundSlots) { - Expression expression = new SlotBinder(scope, ctx.cascadesContext).bind(unboundSlot); - leftSlots.add(expression); - } - slotNames.clear(); - scope = toScope(ctx.cascadesContext, lj.right().getOutput().stream() - .filter(s -> !slotNames.contains(s.getName())) - .peek(s -> slotNames.add(s.getName())) - .collect(Collectors.toList())); - List rightSlots = new ArrayList<>(); - for (Expression unboundSlot : unboundSlots) { - Expression expression = new SlotBinder(scope, ctx.cascadesContext).bind(unboundSlot); - rightSlots.add(expression); - } - int size = leftSlots.size(); - List hashEqExpr = new ArrayList<>(); - for (int i = 0; i < size; i++) { - hashEqExpr.add(new EqualTo(leftSlots.get(i), rightSlots.get(i))); - } - return lj.withJoinConjuncts(hashEqExpr, lj.getOtherJoinConjuncts(), null); - }) + usingJoin().thenApply(this::bindUsingJoin) ), RuleType.BINDING_JOIN_SLOT.build( - logicalJoin().thenApply(ctx -> { - LogicalJoin join = ctx.root; - List cond = join.getOtherJoinConjuncts().stream() - .map(expr -> bindSlot(expr, join.children(), ctx.cascadesContext)) - .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) - .map(expr -> TypeCoercionUtils.castIfNotSameType(expr, BooleanType.INSTANCE)) - .collect(Collectors.toList()); - List hashJoinConjuncts = join.getHashJoinConjuncts().stream() - .map(expr -> bindSlot(expr, join.children(), ctx.cascadesContext)) - .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) - .map(expr -> TypeCoercionUtils.castIfNotSameType(expr, BooleanType.INSTANCE)) - .collect(Collectors.toList()); - return new LogicalJoin<>(join.getJoinType(), - hashJoinConjuncts, cond, join.getDistributeHint(), join.getMarkJoinSlotReference(), - join.children(), null); - }) + logicalJoin().thenApply(this::bindJoin) ), RuleType.BINDING_AGGREGATE_SLOT.build( - logicalAggregate().thenApply(ctx -> { - LogicalAggregate agg = ctx.root; - List output = agg.getOutputExpressions().stream() - .map(expr -> bindSlot(expr, agg.child(), ctx.cascadesContext)) - .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) - .collect(ImmutableList.toImmutableList()); - - // The columns referenced in group by are first obtained from the child's output, - // and then from the node's output - Set duplicatedSlotNames = new HashSet<>(); - Map childOutputsToExpr = agg.child().getOutput().stream() - .collect(Collectors.toMap(Slot::getName, Slot::toSlot, - (oldExpr, newExpr) -> { - duplicatedSlotNames.add(((Slot) oldExpr).getName()); - return oldExpr; - })); - /* - GroupByKey binding priority: - 1. child.output - 2. agg.output - CASE 1 - k is not in agg.output - plan: - agg(group_by: k) - +---child(output t1.k, t2.k) - - group_by_key: k is ambiguous, t1.k and t2.k are candidate. - - CASE 2 - k is in agg.output - plan: - agg(group_by: k, output (k+1 as k) - +---child(output t1.k, t2.k) - - it is failed to bind group_by_key with child.output(ambiguous), but group_by_key can be bound with - agg.output - - CASE 3 - group by key cannot bind with agg func - plan: - agg(group_by v, output sum(k) as v) - throw AnalysisException - - CASE 4 - sql: - `select count(1) from t1 join t2 group by a` - we cannot bind `group by a`, because it is ambiguous (t1.a and t2.a) - - CASE 5 - following case 4, if t1.a is in agg.output, we can bind `group by a` to t1.a - sql - select t1.a - from t1 join t2 on t1.a = t2.a - group by a - group_by_key is bound on t1.a - */ - duplicatedSlotNames.forEach(childOutputsToExpr::remove); - for (int i = 0; i < output.size(); i++) { - if (!(output.get(i) instanceof Alias)) { - continue; - } - Alias alias = (Alias) output.get(i); - if (alias.child().anyMatch(expr -> expr instanceof AggregateFunction)) { - continue; - } - /* - Alias(x) has been bound by binding agg's output - we add x to childOutputsToExpr, so when binding group by exprs later, we can use x directly - and won't bind it again - select - p_cycle_time / (select max(p_cycle_time) from log_event_8) as 'x', - count(distinct case_id) as 'y' - from - log_event_8 - group by - x - */ - childOutputsToExpr.putIfAbsent(alias.getName(), output.get(i).child(0)); - } - - Set boundedGroupByExpressions = Sets.newHashSet(); - List replacedGroupBy = agg.getGroupByExpressions().stream() - .map(groupBy -> { - if (groupBy instanceof UnboundSlot) { - UnboundSlot unboundSlot = (UnboundSlot) groupBy; - if (unboundSlot.getNameParts().size() == 1) { - String name = unboundSlot.getNameParts().get(0); - if (childOutputsToExpr.containsKey(name)) { - Expression expression = childOutputsToExpr.get(name); - boundedGroupByExpressions.add(expression); - return expression; - } - } - } - return groupBy; - }).collect(Collectors.toList()); - /* - according to case 4 and case 5, we construct boundSlots - */ - Set outputSlotNames = Sets.newHashSet(); - Set outputSlots = output.stream() - .filter(SlotReference.class::isInstance) - .peek(slot -> outputSlotNames.add(slot.getName())) - .map(NamedExpression::toSlot) - .collect(Collectors.toSet()); - // suppose group by key is a. - // if both t1.a and t2.a are in agg.child.output, and t1.a in agg.output, - // bind group_by_key a with t1.a - // ` .filter(slot -> !outputSlotNames.contains(slot.getName()))` - // is used to avoid add t2.a into boundSlots - Set boundSlots = agg.child().getOutputSet().stream() - .filter(slot -> !outputSlotNames.contains(slot.getName())) - .collect(Collectors.toSet()); - - boundSlots.addAll(outputSlots); - SlotBinder binder = new SlotBinder( - toScope(ctx.cascadesContext, ImmutableList.copyOf(boundSlots)), ctx.cascadesContext); - SlotBinder childBinder = new SlotBinder( - toScope(ctx.cascadesContext, ImmutableList.copyOf(agg.child().getOutputSet())), - ctx.cascadesContext); - - List groupBy = replacedGroupBy.stream() - .map(expression -> { - if (boundedGroupByExpressions.contains(expression)) { - // expr has been bound by binding agg's output - return expression; - } else { - // bind slot for unbound exprs - Expression e = binder.bind(expression); - if (e instanceof UnboundSlot) { - return childBinder.bind(e); - } - return e; - } - }) - .collect(Collectors.toList()); - groupBy.forEach(expression -> checkBoundExceptLambda(expression, ctx.root)); - groupBy = groupBy.stream() - // bind function for unbound exprs or return old expr if it's bound by binding agg's output - .map(expr -> boundedGroupByExpressions.contains(expr) ? expr - : bindFunction(expr, ctx.root, ctx.cascadesContext)) - .collect(ImmutableList.toImmutableList()); - checkIfOutputAliasNameDuplicatedForGroupBy(groupBy, output); - return agg.withGroupByAndOutput(groupBy, output); - }) + logicalAggregate().thenApply(this::bindAggregate) ), RuleType.BINDING_REPEAT_SLOT.build( - logicalRepeat().thenApply(ctx -> { - LogicalRepeat repeat = ctx.root; - List output = repeat.getOutputExpressions().stream() - .map(expr -> bindSlot(expr, repeat.child(), ctx.cascadesContext)) - .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) - .collect(ImmutableList.toImmutableList()); - - // The columns referenced in group by are first obtained from the child's output, - // and then from the node's output - Map childOutputsToExpr = repeat.child().getOutput().stream() - .collect(Collectors.toMap(Slot::getName, Slot::toSlot, (oldExpr, newExpr) -> oldExpr)); - Map aliasNameToExpr = output.stream() - .filter(ne -> ne instanceof Alias) - .map(Alias.class::cast) - .collect(Collectors.toMap(Alias::getName, UnaryNode::child, (oldExpr, newExpr) -> oldExpr)); - aliasNameToExpr.forEach(childOutputsToExpr::putIfAbsent); - - List> replacedGroupingSets = repeat.getGroupingSets().stream() - .map(groupBy -> - groupBy.stream().map(expr -> { - if (expr instanceof UnboundSlot) { - UnboundSlot unboundSlot = (UnboundSlot) expr; - if (unboundSlot.getNameParts().size() == 1) { - String name = unboundSlot.getNameParts().get(0); - if (childOutputsToExpr.containsKey(name)) { - return childOutputsToExpr.get(name); - } - } - } - return expr; - }).collect(Collectors.toList()) - ).collect(Collectors.toList()); - - List> groupingSets = replacedGroupingSets - .stream() - .map(groupingSet -> groupingSet.stream() - .map(expr -> bindSlot(expr, repeat.child(), ctx.cascadesContext)) - .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) - .collect(ImmutableList.toImmutableList())) - .collect(ImmutableList.toImmutableList()); - List newOutput = adjustNullableForRepeat(groupingSets, output); - groupingSets.forEach(list -> checkIfOutputAliasNameDuplicatedForGroupBy(list, newOutput)); - - // check all GroupingScalarFunction inputSlots must be from groupingExprs - Set groupingExprs = groupingSets.stream() - .flatMap(Collection::stream).map(expr -> expr.getInputSlots()) - .flatMap(Collection::stream).collect(Collectors.toSet()); - Set groupingScalarFunctions = ExpressionUtils - .collect(newOutput, GroupingScalarFunction.class::isInstance); - for (GroupingScalarFunction function : groupingScalarFunctions) { - if (!groupingExprs.containsAll(function.getInputSlots())) { - throw new AnalysisException("Column in " + function.getName() - + " does not exist in GROUP BY clause."); - } - } - return repeat.withGroupSetsAndOutput(groupingSets, newOutput); - }) + logicalRepeat().thenApply(this::bindRepeat) ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(aggregate()).thenApply(ctx -> { - LogicalSort> sort = ctx.root; - Aggregate aggregate = sort.child(); - return bindSort(sort, aggregate, ctx.cascadesContext); - }) - ), - RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalHaving(aggregate())).thenApply(ctx -> { - LogicalSort>> sort = ctx.root; - Aggregate aggregate = sort.child().child(); - return bindSort(sort, aggregate, ctx.cascadesContext); - }) - ), - RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalHaving(logicalProject())).thenApply(ctx -> { - LogicalSort>> sort = ctx.root; - LogicalProject project = sort.child().child(); - return bindSort(sort, project, ctx.cascadesContext); - }) - ), - RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalProject()).thenApply(ctx -> { - LogicalSort> sort = ctx.root; - LogicalProject project = sort.child(); - return bindSort(sort, project, ctx.cascadesContext); - }) - ), - RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalCTEConsumer()).thenApply(ctx -> { - LogicalSort sort = ctx.root; - LogicalCTEConsumer cteConsumer = sort.child(); - return bindSort(sort, cteConsumer, ctx.cascadesContext); - }) - ), - RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalCTEAnchor()).thenApply(ctx -> { - LogicalSort> sort = ctx.root; - LogicalCTEAnchor cteAnchor = sort.child(); - return bindSort(sort, cteAnchor, ctx.cascadesContext); - }) - ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalOneRowRelation()).thenApply(ctx -> { - LogicalSort sort = ctx.root; - LogicalOneRowRelation oneRowRelation = sort.child(); - return bindSort(sort, oneRowRelation, ctx.cascadesContext); - }) + logicalSort(any().whenNot(SetOperation.class::isInstance)) + .thenApply(this::bindSortWithoutSetOperation) ), RuleType.BINDING_SORT_SET_OPERATION_SLOT.build( - logicalSort(logicalSetOperation()).thenApply(ctx -> { - LogicalSort sort = ctx.root; - List sortItemList = sort.getOrderKeys() - .stream() - .map(orderKey -> { - Expression item = bindSlot(orderKey.getExpr(), sort.child(), ctx.cascadesContext); - item = bindFunction(item, ctx.root, ctx.cascadesContext); - return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst()); - }).collect(Collectors.toList()); - return new LogicalSort<>(sortItemList, sort.child()); - }) + logicalSort(logicalSetOperation()).thenApply(this::bindSortWithSetOperation) ), RuleType.BINDING_HAVING_SLOT.build( - logicalHaving(aggregate()).when(Plan::canBind).thenApply(ctx -> { - LogicalHaving> having = ctx.root; - Aggregate childPlan = having.child(); - Set boundConjuncts = having.getConjuncts().stream() - .map(expr -> { - expr = bindSlot(expr, childPlan.child(), ctx.cascadesContext, false); - return bindSlot(expr, childPlan, ctx.cascadesContext, false); - }) - .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) - .map(expr -> TypeCoercionUtils.castIfNotSameType(expr, BooleanType.INSTANCE)) - .collect(Collectors.toSet()); - checkIfOutputAliasNameDuplicatedForGroupBy(ImmutableList.copyOf(boundConjuncts), - childPlan.getOutputExpressions()); - return new LogicalHaving<>(boundConjuncts, having.child()); - }) + logicalHaving(aggregate()).thenApply(this::bindHavingAggregate) ), RuleType.BINDING_HAVING_SLOT.build( - logicalHaving(any()).thenApply(ctx -> { - LogicalHaving having = ctx.root; - Plan childPlan = having.child(); - Set boundConjuncts = having.getConjuncts().stream() - .map(expr -> { - expr = bindSlot(expr, childPlan, ctx.cascadesContext, false); - return bindSlot(expr, childPlan.children(), ctx.cascadesContext, false); - }) - .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext)) - .map(expr -> TypeCoercionUtils.castIfNotSameType(expr, BooleanType.INSTANCE)) - .collect(Collectors.toSet()); - checkIfOutputAliasNameDuplicatedForGroupBy(ImmutableList.copyOf(boundConjuncts), - childPlan.getOutput().stream().map(NamedExpression.class::cast) - .collect(Collectors.toList())); - return new LogicalHaving<>(boundConjuncts, having.child()); - }) + logicalHaving(any().whenNot(Aggregate.class::isInstance)).thenApply(this::bindHaving) ), RuleType.BINDING_INLINE_TABLE_SLOT.build( - logicalInlineTable().thenApply(ctx -> { - LogicalInlineTable logicalInlineTable = ctx.root; - // ensure all expressions are valid. - List relations - = Lists.newArrayListWithCapacity(logicalInlineTable.getConstantExprsList().size()); - for (int i = 0; i < logicalInlineTable.getConstantExprsList().size(); i++) { - if (logicalInlineTable.getConstantExprsList().get(i).stream() - .anyMatch(DefaultValueSlot.class::isInstance)) { - throw new AnalysisException("Default expression" - + " can't exist in SELECT statement at row " + (i + 1)); - } - relations.add(new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), - logicalInlineTable.getConstantExprsList().get(i))); - } - // construct union all tree - return LogicalPlanBuilder.reduceToLogicalPlanTree(0, relations.size() - 1, - relations, Qualifier.ALL); - }) + logicalInlineTable().thenApply(this::bindInlineTable) ), RuleType.BINDING_ONE_ROW_RELATION_SLOT.build( // we should bind UnboundAlias in the UnboundOneRowRelation - unboundOneRowRelation().thenApply(ctx -> { - UnboundOneRowRelation oneRowRelation = ctx.root; - List projects = oneRowRelation.getProjects() - .stream() - .map(project -> bindSlot(project, ImmutableList.of(), ctx.cascadesContext)) - .map(project -> bindFunction(project, ctx.root, ctx.cascadesContext)) - .collect(Collectors.toList()); - return new LogicalOneRowRelation(oneRowRelation.getRelationId(), projects); - }) + unboundOneRowRelation().thenApply(this::bindOneRowRelation) ), RuleType.BINDING_SET_OPERATION_SLOT.build( // LogicalSetOperation don't bind again if LogicalSetOperation.outputs is not empty, this is special // we should not remove LogicalSetOperation::canBind, because in default case, the plan can run into // bind callback if not bound or **not run into bind callback yet**. - logicalSetOperation().when(LogicalSetOperation::canBind).then(setOperation -> { - // check whether the left and right child output columns are the same - if (setOperation.child(0).getOutput().size() != setOperation.child(1).getOutput().size()) { - throw new AnalysisException("Operands have unequal number of columns:\n" - + "'" + setOperation.child(0).getOutput() + "' has " - + setOperation.child(0).getOutput().size() + " column(s)\n" - + "'" + setOperation.child(1).getOutput() + "' has " - + setOperation.child(1).getOutput().size() + " column(s)"); - } - - // INTERSECT and EXCEPT does not support ALL qualified - if (setOperation.getQualifier() == Qualifier.ALL - && (setOperation instanceof LogicalExcept || setOperation instanceof LogicalIntersect)) { - throw new AnalysisException("INTERSECT and EXCEPT does not support ALL qualified"); - } - // we need to do cast before set operation, because we maybe use these slot to do shuffle - // so, we must cast it before shuffle to get correct hash code. - List> childrenProjections = setOperation.collectChildrenProjections(); - Builder> childrenOutputs = ImmutableList.builder(); - Builder newChildren = ImmutableList.builder(); - for (int i = 0; i < childrenProjections.size(); i++) { - Plan newChild; - if (childrenProjections.stream().allMatch(SlotReference.class::isInstance)) { - newChild = setOperation.child(i); - } else { - newChild = new LogicalProject<>(childrenProjections.get(i), setOperation.child(i)); - } - newChildren.add(newChild); - childrenOutputs.add(newChild.getOutput().stream() - .map(SlotReference.class::cast) - .collect(ImmutableList.toImmutableList())); - } - setOperation = setOperation.withChildrenAndTheirOutputs( - newChildren.build(), childrenOutputs.build()); - List newOutputs = setOperation.buildNewOutputs(); - return setOperation.withNewOutputs(newOutputs); - }) + logicalSetOperation().when(LogicalSetOperation::canBind).then(this::bindSetOperation) ), RuleType.BINDING_GENERATE_SLOT.build( - logicalGenerate().when(AbstractPlan::canBind).thenApply(ctx -> { - LogicalGenerate generate = ctx.root; - List boundSlotGenerators - = bindSlot(generate.getGenerators(), generate.child(), ctx.cascadesContext); - List boundFunctionGenerators = boundSlotGenerators.stream() - .map(f -> bindTableGeneratingFunction((UnboundFunction) f, ctx.root, ctx.cascadesContext)) - .collect(Collectors.toList()); - ImmutableList.Builder slotBuilder = ImmutableList.builder(); - List expandAlias = Lists.newArrayList(); - for (int i = 0; i < generate.getGeneratorOutput().size(); i++) { - Function generator = boundFunctionGenerators.get(i); - UnboundSlot slot = (UnboundSlot) generate.getGeneratorOutput().get(i); - Preconditions.checkState(slot.getNameParts().size() == 2, - "the size of nameParts of UnboundSlot in LogicalGenerate must be 2."); - Slot boundSlot = new SlotReference(slot.getNameParts().get(1), generator.getDataType(), - generator.nullable(), ImmutableList.of(slot.getNameParts().get(0))); - slotBuilder.add(boundSlot); - // the boundSlot may has two situation: - // 1. the expandColumnsAlias is not empty, we should use make boundSlot expand to multi alias - // 2. the expandColumnsAlias is empty, we should use origin boundSlot - if (generate.getExpandColumnAlias() != null && i < generate.getExpandColumnAlias().size() - && !CollectionUtils.isEmpty(generate.getExpandColumnAlias().get(i))) { - // if the alias is not empty, we should bind it with struct_element as child expr with alias - // struct_element(#expand_col#k, #k) as #k - // struct_element(#expand_col#v, #v) as #v - List fields = ((StructType) boundSlot.getDataType()).getFields(); - for (int idx = 0; idx < fields.size(); ++idx) { - expandAlias.add(new Alias(new StructElement( - boundSlot, new StringLiteral(fields.get(idx).getName())), - generate.getExpandColumnAlias().get(i).get(idx))); - } - } - } - LogicalGenerate ret = new LogicalGenerate<>( - boundFunctionGenerators, slotBuilder.build(), generate.child()); - if (expandAlias.size() > 0) { - // we need a project to deal with explode(map) to struct with field alias - // project should contains: generator.child slot + expandAlias - List allProjectSlots = generate.child().getOutput().stream() - .map(NamedExpression.class::cast) - .collect(Collectors.toList()); - allProjectSlots.addAll(expandAlias); - return new LogicalProject<>(allProjectSlots, ret); - } - return ret; - }) + logicalGenerate().when(AbstractPlan::canBind).thenApply(this::bindGenerate) ), RuleType.BINDING_UNBOUND_TVF_RELATION_FUNCTION.build( - unboundTVFRelation().thenApply(ctx -> { - UnboundTVFRelation relation = ctx.root; - return bindTableValuedFunction(relation, ctx.statementContext); - }) + unboundTVFRelation().thenApply(this::bindTableValuedFunction) ), RuleType.BINDING_SUBQUERY_ALIAS_SLOT.build( - logicalSubQueryAlias().thenApply(ctx -> { - LogicalSubQueryAlias subQueryAlias = ctx.root; - checkSameNameSlot(subQueryAlias.child(0).getOutput(), subQueryAlias.getAlias()); - return subQueryAlias; - }) + logicalSubQueryAlias().thenApply(this::bindSubqueryAlias) ), RuleType.BINDING_RESULT_SINK.build( - unboundResultSink().thenApply(ctx -> { - LogicalSink sink = ctx.root; - if (ctx.connectContext.getState().isQuery()) { - List outputExprs = sink.child().getOutput().stream() - .map(NamedExpression.class::cast) - .collect(ImmutableList.toImmutableList()); - return new LogicalResultSink<>(outputExprs, sink.child()); - } - // Should infer column name for expression when query command - final ImmutableListMultimap.Builder exprIdToIndexMapBuilder = - ImmutableListMultimap.builder(); - List childOutput = sink.child().getOutput(); - for (int index = 0; index < childOutput.size(); index++) { - exprIdToIndexMapBuilder.put(childOutput.get(index).getExprId(), index); - } - InferPlanOutputAlias aliasInfer = new InferPlanOutputAlias(childOutput); - sink.child().accept(aliasInfer, exprIdToIndexMapBuilder.build()); - return new LogicalResultSink<>(aliasInfer.getOutputs(), sink.child()); - }) + unboundResultSink().thenApply(this::bindResultSink) ) ).stream().map(ruleCondition).collect(ImmutableList.toImmutableList()); } - private Plan bindSort(LogicalSort sort, Plan plan, CascadesContext ctx) { + private LogicalResultSink bindResultSink(MatchingContext> ctx) { + LogicalSink sink = ctx.root; + if (ctx.connectContext.getState().isQuery()) { + List outputExprs = sink.child().getOutput().stream() + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()); + return new LogicalResultSink<>(outputExprs, sink.child()); + } + // Should infer column name for expression when query command + final ImmutableListMultimap.Builder exprIdToIndexMapBuilder = + ImmutableListMultimap.builder(); + List childOutput = sink.child().getOutput(); + for (int index = 0; index < childOutput.size(); index++) { + exprIdToIndexMapBuilder.put(childOutput.get(index).getExprId(), index); + } + InferPlanOutputAlias aliasInfer = new InferPlanOutputAlias(childOutput); + List output = aliasInfer.infer(sink.child(), exprIdToIndexMapBuilder.build()); + return new LogicalResultSink<>(output, sink.child()); + } + + private LogicalSubQueryAlias bindSubqueryAlias(MatchingContext> ctx) { + LogicalSubQueryAlias subQueryAlias = ctx.root; + checkSameNameSlot(subQueryAlias.child(0).getOutput(), subQueryAlias.getAlias()); + return subQueryAlias; + } + + private LogicalPlan bindGenerate(MatchingContext> ctx) { + LogicalGenerate generate = ctx.root; + CascadesContext cascadesContext = ctx.cascadesContext; + + Builder outputSlots = ImmutableList.builder(); + Builder boundGenerators = ImmutableList.builder(); + List expandAlias = Lists.newArrayList(); + SimpleExprAnalyzer analyzer = buildSimpleExprAnalyzer( + generate, cascadesContext, generate.children(), true, true); + for (int i = 0; i < generate.getGeneratorOutput().size(); i++) { + UnboundSlot slot = (UnboundSlot) generate.getGeneratorOutput().get(i); + Preconditions.checkState(slot.getNameParts().size() == 2, + "the size of nameParts of UnboundSlot in LogicalGenerate must be 2."); + + Expression boundGenerator = analyzer.analyze(generate.getGenerators().get(i)); + if (!(boundGenerator instanceof TableGeneratingFunction)) { + throw new AnalysisException(boundGenerator.toSql() + " is not a TableGeneratingFunction"); + } + Function generator = (Function) boundGenerator; + boundGenerators.add(generator); + + Slot boundSlot = new SlotReference(slot.getNameParts().get(1), generator.getDataType(), + generator.nullable(), ImmutableList.of(slot.getNameParts().get(0))); + outputSlots.add(boundSlot); + // the boundSlot may has two situation: + // 1. the expandColumnsAlias is not empty, we should use make boundSlot expand to multi alias + // 2. the expandColumnsAlias is empty, we should use origin boundSlot + if (generate.getExpandColumnAlias() != null && i < generate.getExpandColumnAlias().size() + && !CollectionUtils.isEmpty(generate.getExpandColumnAlias().get(i))) { + // if the alias is not empty, we should bind it with struct_element as child expr with alias + // struct_element(#expand_col#k, #k) as #k + // struct_element(#expand_col#v, #v) as #v + List fields = ((StructType) boundSlot.getDataType()).getFields(); + for (int idx = 0; idx < fields.size(); ++idx) { + expandAlias.add(new Alias(new StructElement( + boundSlot, new StringLiteral(fields.get(idx).getName())), + generate.getExpandColumnAlias().get(i).get(idx))); + } + } + } + LogicalGenerate ret = new LogicalGenerate<>( + boundGenerators.build(), outputSlots.build(), generate.child()); + if (!expandAlias.isEmpty()) { + // we need a project to deal with explode(map) to struct with field alias + // project should contains: generator.child slot + expandAlias + List allProjectSlots = generate.child().getOutput().stream() + .map(NamedExpression.class::cast) + .collect(Collectors.toList()); + allProjectSlots.addAll(expandAlias); + return new LogicalProject<>(allProjectSlots, ret); + } + return ret; + } + + private LogicalSetOperation bindSetOperation(LogicalSetOperation setOperation) { + // check whether the left and right child output columns are the same + if (setOperation.child(0).getOutput().size() != setOperation.child(1).getOutput().size()) { + throw new AnalysisException("Operands have unequal number of columns:\n" + + "'" + setOperation.child(0).getOutput() + "' has " + + setOperation.child(0).getOutput().size() + " column(s)\n" + + "'" + setOperation.child(1).getOutput() + "' has " + + setOperation.child(1).getOutput().size() + " column(s)"); + } + + // INTERSECT and EXCEPT does not support ALL qualified + if (setOperation.getQualifier() == Qualifier.ALL + && (setOperation instanceof LogicalExcept || setOperation instanceof LogicalIntersect)) { + throw new AnalysisException("INTERSECT and EXCEPT does not support ALL qualified"); + } + // we need to do cast before set operation, because we maybe use these slot to do shuffle + // so, we must cast it before shuffle to get correct hash code. + List> childrenProjections = setOperation.collectChildrenProjections(); + int childrenProjectionSize = childrenProjections.size(); + Builder> childrenOutputs = ImmutableList.builderWithExpectedSize(childrenProjectionSize); + Builder newChildren = ImmutableList.builderWithExpectedSize(childrenProjectionSize); + for (int i = 0; i < childrenProjectionSize; i++) { + Plan newChild; + if (childrenProjections.stream().allMatch(SlotReference.class::isInstance)) { + newChild = setOperation.child(i); + } else { + newChild = new LogicalProject<>(childrenProjections.get(i), setOperation.child(i)); + } + newChildren.add(newChild); + childrenOutputs.add((List) (List) newChild.getOutput()); + } + setOperation = setOperation.withChildrenAndTheirOutputs(newChildren.build(), childrenOutputs.build()); + List newOutputs = setOperation.buildNewOutputs(); + return setOperation.withNewOutputs(newOutputs); + } + + @NotNull + private LogicalOneRowRelation bindOneRowRelation(MatchingContext ctx) { + UnboundOneRowRelation oneRowRelation = ctx.root; + CascadesContext cascadesContext = ctx.cascadesContext; + SimpleExprAnalyzer analyzer = buildSimpleExprAnalyzer( + oneRowRelation, cascadesContext, ImmutableList.of(), true, true); + List projects = analyzer.analyzeToList(oneRowRelation.getProjects()); + return new LogicalOneRowRelation(oneRowRelation.getRelationId(), projects); + } + + private LogicalPlan bindInlineTable(MatchingContext ctx) { + LogicalInlineTable logicalInlineTable = ctx.root; + // ensure all expressions are valid. + List relations + = Lists.newArrayListWithCapacity(logicalInlineTable.getConstantExprsList().size()); + for (int i = 0; i < logicalInlineTable.getConstantExprsList().size(); i++) { + if (logicalInlineTable.getConstantExprsList().get(i).stream() + .anyMatch(DefaultValueSlot.class::isInstance)) { + throw new AnalysisException("Default expression" + + " can't exist in SELECT statement at row " + (i + 1)); + } + relations.add(new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), + logicalInlineTable.getConstantExprsList().get(i))); + } + // construct union all tree + return LogicalPlanBuilder.reduceToLogicalPlanTree(0, relations.size() - 1, + relations, Qualifier.ALL); + } + + private LogicalHaving bindHaving(MatchingContext> ctx) { + LogicalHaving having = ctx.root; + Plan childPlan = having.child(); + CascadesContext cascadesContext = ctx.cascadesContext; + + // bind slot by child.output first + Scope defaultScope = toScope(cascadesContext, childPlan.getOutput()); + // then bind slot by child.children.output + Supplier backupScope = Suppliers.memoize(() -> + toScope(cascadesContext, PlanUtils.fastGetChildrenOutputs(childPlan.children())) + ); + return bindHavingByScopes(having, cascadesContext, defaultScope, backupScope); + } + + private LogicalHaving bindHavingAggregate( + MatchingContext>> ctx) { + LogicalHaving> having = ctx.root; + Aggregate aggregate = having.child(); + CascadesContext cascadesContext = ctx.cascadesContext; + + // having(aggregate) should bind slot by aggregate.child.output first + Scope defaultScope = toScope(cascadesContext, PlanUtils.fastGetChildrenOutputs(aggregate.children())); + // then bind slot by aggregate.output + Supplier backupScope = Suppliers.memoize(() -> + toScope(cascadesContext, aggregate.getOutput()) + ); + return bindHavingByScopes(ctx.root, ctx.cascadesContext, defaultScope, backupScope); + } + + private LogicalHaving bindHavingByScopes( + LogicalHaving having, + CascadesContext cascadesContext, Scope defaultScope, Supplier backupScope) { + Plan child = having.child(); + + SimpleExprAnalyzer analyzer = buildCustomSlotBinderAnalyzer( + having, cascadesContext, defaultScope, false, true, + (self, unboundSlot) -> { + List slots = self.bindSlotByScope(unboundSlot, defaultScope); + if (!slots.isEmpty()) { + return slots; + } + return self.bindSlotByScope(unboundSlot, backupScope.get()); + }); + ImmutableSet.Builder boundConjuncts + = ImmutableSet.builderWithExpectedSize(having.getConjuncts().size()); + for (Expression conjunct : having.getConjuncts()) { + conjunct = analyzer.analyze(conjunct); + conjunct = TypeCoercionUtils.castIfNotSameType(conjunct, BooleanType.INSTANCE); + boundConjuncts.add(conjunct); + } + checkIfOutputAliasNameDuplicatedForGroupBy(boundConjuncts.build(), child.getOutput()); + return new LogicalHaving<>(boundConjuncts.build(), having.child()); + } + + private LogicalSort bindSortWithSetOperation( + MatchingContext> ctx) { + LogicalSort sort = ctx.root; + CascadesContext cascadesContext = ctx.cascadesContext; + + SimpleExprAnalyzer analyzer = buildSimpleExprAnalyzer( + sort, cascadesContext, sort.children(), true, true); + Builder boundKeys = ImmutableList.builderWithExpectedSize(sort.getOrderKeys().size()); + for (OrderKey orderKey : sort.getOrderKeys()) { + Expression boundKey = analyzer.analyze(orderKey.getExpr()); + boundKeys.add(orderKey.withExpression(boundKey)); + } + return new LogicalSort<>(boundKeys.build(), sort.child()); + } + + private LogicalJoin bindJoin(MatchingContext> ctx) { + LogicalJoin join = ctx.root; + CascadesContext cascadesContext = ctx.cascadesContext; + + SimpleExprAnalyzer analyzer = buildSimpleExprAnalyzer( + join, cascadesContext, join.children(), true, true); + + Builder hashJoinConjuncts = ImmutableList.builderWithExpectedSize( + join.getHashJoinConjuncts().size()); + for (Expression hashJoinConjunct : join.getHashJoinConjuncts()) { + hashJoinConjunct = analyzer.analyze(hashJoinConjunct); + hashJoinConjunct = TypeCoercionUtils.castIfNotSameType(hashJoinConjunct, BooleanType.INSTANCE); + hashJoinConjuncts.add(hashJoinConjunct); + } + Builder otherJoinConjuncts = ImmutableList.builderWithExpectedSize( + join.getOtherJoinConjuncts().size()); + for (Expression otherJoinConjunct : join.getOtherJoinConjuncts()) { + otherJoinConjunct = analyzer.analyze(otherJoinConjunct); + otherJoinConjunct = TypeCoercionUtils.castIfNotSameType(otherJoinConjunct, BooleanType.INSTANCE); + otherJoinConjuncts.add(otherJoinConjunct); + } + + return new LogicalJoin<>(join.getJoinType(), + hashJoinConjuncts.build(), otherJoinConjuncts.build(), + join.getDistributeHint(), join.getMarkJoinSlotReference(), + join.children(), null); + } + + private LogicalJoin bindUsingJoin(MatchingContext> ctx) { + UsingJoin using = ctx.root; + CascadesContext cascadesContext = ctx.cascadesContext; + List unboundHashJoinConjunct = using.getHashJoinConjuncts(); + + // Suppose A JOIN B USING(name) JOIN C USING(name), [A JOIN B] is the left node, in this case, + // C should combine with table B on C.name=B.name. so we reverse the output to make sure that + // the most right slot is matched with priority. + List leftOutput = Utils.reverseImmutableList(using.left().getOutput()); + Scope leftScope = toScope(cascadesContext, ExpressionUtils.distinctSlotByName(leftOutput)); + Scope rightScope = toScope(cascadesContext, ExpressionUtils.distinctSlotByName(using.right().getOutput())); + ExpressionRewriteContext rewriteContext = new ExpressionRewriteContext(cascadesContext); + + Builder hashEqExprs = ImmutableList.builderWithExpectedSize(unboundHashJoinConjunct.size()); + for (Expression usingColumn : unboundHashJoinConjunct) { + ExpressionAnalyzer leftExprAnalyzer = new ExpressionAnalyzer( + using, leftScope, cascadesContext, true, false); + Expression usingLeftSlot = leftExprAnalyzer.analyze(usingColumn, rewriteContext); + + ExpressionAnalyzer rightExprAnalyzer = new ExpressionAnalyzer( + using, rightScope, cascadesContext, true, false); + Expression usingRightSlot = rightExprAnalyzer.analyze(usingColumn, rewriteContext); + hashEqExprs.add(new EqualTo(usingLeftSlot, usingRightSlot)); + } + + return new LogicalJoin<>( + using.getJoinType() == JoinType.CROSS_JOIN ? JoinType.INNER_JOIN : using.getJoinType(), + hashEqExprs.build(), using.getOtherJoinConjuncts(), + using.getDistributeHint(), using.getMarkJoinSlotReference(), + using.children(), null); + } + + private Plan bindProject(MatchingContext> ctx) { + LogicalProject project = ctx.root; + CascadesContext cascadesContext = ctx.cascadesContext; + + SimpleExprAnalyzer analyzer = buildSimpleExprAnalyzer( + project, cascadesContext, project.children(), true, true); + + List excepts = project.getExcepts(); + Supplier> boundExcepts = Suppliers.memoize( + () -> analyzer.analyzeToSet(project.getExcepts())); + + Builder boundProjections = ImmutableList.builderWithExpectedSize(project.arity()); + for (Expression expression : project.getProjects()) { + Expression expr = analyzer.analyze(expression); + if (!(expr instanceof BoundStar)) { + boundProjections.add((NamedExpression) expr); + } else { + BoundStar boundStar = (BoundStar) expr; + List slots = boundStar.getSlots(); + if (!excepts.isEmpty()) { + slots = Utils.filterImmutableList(slots, slot -> !boundExcepts.get().contains(slot)); + } + boundProjections.addAll(slots); + } + } + return project.withProjects(boundProjections.build()); + } + + private Plan bindFilter(MatchingContext> ctx) { + LogicalFilter filter = ctx.root; + CascadesContext cascadesContext = ctx.cascadesContext; + + SimpleExprAnalyzer analyzer = buildSimpleExprAnalyzer( + filter, cascadesContext, filter.children(), true, true); + ImmutableSet.Builder boundConjuncts = ImmutableSet.builderWithExpectedSize( + filter.getConjuncts().size() * 2); + for (Expression conjunct : filter.getConjuncts()) { + Expression boundConjunct = analyzer.analyze(conjunct); + boundConjunct = TypeCoercionUtils.castIfNotSameType(boundConjunct, BooleanType.INSTANCE); + boundConjuncts.add(boundConjunct); + } + return new LogicalFilter<>(boundConjuncts.build(), filter.child()); + } + + private Plan bindAggregate(MatchingContext> ctx) { + LogicalAggregate agg = ctx.root; + CascadesContext cascadesContext = ctx.cascadesContext; + + SimpleExprAnalyzer aggOutputAnalyzer = buildSimpleExprAnalyzer( + agg, cascadesContext, agg.children(), true, true); + List boundAggOutput = aggOutputAnalyzer.analyzeToList(agg.getOutputExpressions()); + Supplier aggOutputScopeWithoutAggFun = buildAggOutputScopeWithoutAggFun(boundAggOutput, cascadesContext); + List boundGroupBy = bindGroupBy( + agg, agg.getGroupByExpressions(), boundAggOutput, aggOutputScopeWithoutAggFun, cascadesContext); + return agg.withGroupByAndOutput(boundGroupBy, boundAggOutput); + } + + private Plan bindRepeat(MatchingContext> ctx) { + LogicalRepeat repeat = ctx.root; + CascadesContext cascadesContext = ctx.cascadesContext; + + SimpleExprAnalyzer repeatOutputAnalyzer = buildSimpleExprAnalyzer( + repeat, cascadesContext, repeat.children(), true, true); + List boundRepeatOutput = repeatOutputAnalyzer.analyzeToList(repeat.getOutputExpressions()); + Supplier aggOutputScopeWithoutAggFun = + buildAggOutputScopeWithoutAggFun(boundRepeatOutput, cascadesContext); + + Builder> boundGroupingSetsBuilder = + ImmutableList.builderWithExpectedSize(repeat.getGroupingSets().size()); + for (List groupingSet : repeat.getGroupingSets()) { + List boundGroupingSet = bindGroupBy( + repeat, groupingSet, boundRepeatOutput, aggOutputScopeWithoutAggFun, cascadesContext); + boundGroupingSetsBuilder.add(boundGroupingSet); + } + List> boundGroupingSets = boundGroupingSetsBuilder.build(); + List nullableOutput = adjustNullableForRepeat(boundGroupingSets, boundRepeatOutput); + for (List groupingSet : boundGroupingSets) { + checkIfOutputAliasNameDuplicatedForGroupBy(groupingSet, nullableOutput); + } + + // check all GroupingScalarFunction inputSlots must be from groupingExprs + Set groupingExprs = boundGroupingSets.stream() + .flatMap(Collection::stream).map(expr -> expr.getInputSlots()) + .flatMap(Collection::stream).collect(Collectors.toSet()); + Set groupingScalarFunctions = ExpressionUtils + .collect(nullableOutput, GroupingScalarFunction.class::isInstance); + for (GroupingScalarFunction function : groupingScalarFunctions) { + if (!groupingExprs.containsAll(function.getInputSlots())) { + throw new AnalysisException("Column in " + function.getName() + + " does not exist in GROUP BY clause."); + } + } + return repeat.withGroupSetsAndOutput(boundGroupingSets, nullableOutput); + } + + private List bindGroupBy( + Aggregate agg, List groupBy, List boundAggOutput, + Supplier aggOutputScopeWithoutAggFun, CascadesContext cascadesContext) { + Scope childOutputScope = toScope(cascadesContext, agg.child().getOutput()); + + SimpleExprAnalyzer analyzer = buildCustomSlotBinderAnalyzer( + agg, cascadesContext, childOutputScope, true, true, + (self, unboundSlot) -> { + // see: https://github.com/apache/doris/pull/15240 + // + // first, try to bind by agg.child.output + List slotsInChildren = self.bindExactSlotsByThisScope(unboundSlot, childOutputScope); + if (slotsInChildren.size() == 1) { + // bind succeed + return slotsInChildren; + } + // second, bind failed: + // if the slot not found, or more than one candidate slots found in agg.child.output, + // then try to bind by agg.output + List slotsInOutput = self.bindExactSlotsByThisScope( + unboundSlot, aggOutputScopeWithoutAggFun.get()); + if (slotsInOutput.isEmpty()) { + // if slotsInChildren.size() > 1 && slotsInOutput.isEmpty(), + // we return slotsInChildren to throw an ambiguous slots exception + return slotsInChildren; + } + + Builder useOutputExpr = ImmutableList.builderWithExpectedSize(slotsInOutput.size()); + for (Slot slotInOutput : slotsInOutput) { + // mappingSlot is provided by aggOutputScopeWithoutAggFun + // and no non-MappingSlot slot exist in the Scope, so we + // can direct cast it safely + MappingSlot mappingSlot = (MappingSlot) slotInOutput; + + // groupBy can not direct use the slot in agg.output, because + // groupBy evaluate before generate agg.output, so we should + // replace the output to the real expr tree + // + // for example: + // select k + 1 as k1 from tbl group by k1 + // + // The k1 in groupBy use the slot in agg.output: Alias(k + 1).toSlot(), + // we should rewrite to: select k + 1 as k1 from tbl group by k + 1 + useOutputExpr.add(mappingSlot.getMappingExpression()); + } + return useOutputExpr.build(); + }); + + List boundGroupBy = analyzer.analyzeToList(groupBy); + checkIfOutputAliasNameDuplicatedForGroupBy(boundGroupBy, boundAggOutput); + return boundGroupBy; + } + + private Supplier buildAggOutputScopeWithoutAggFun( + List boundAggOutput, CascadesContext cascadesContext) { + return Suppliers.memoize(() -> { + Builder nonAggFunOutput = ImmutableList.builderWithExpectedSize(boundAggOutput.size()); + for (NamedExpression output : boundAggOutput) { + if (!output.containsType(AggregateFunction.class)) { + Slot outputSlot = output.toSlot(); + MappingSlot mappingSlot = new MappingSlot(outputSlot, + output instanceof Alias ? output.child(0) : output); + nonAggFunOutput.add(mappingSlot); + } + } + return toScope(cascadesContext, nonAggFunOutput.build()); + }); + } + + private Plan bindSortWithoutSetOperation(MatchingContext> ctx) { + LogicalSort sort = ctx.root; + Plan input = sort.child(); + // we should skip LogicalHaving to bind slot in LogicalSort; + if (input instanceof LogicalHaving) { + input = input.child(0); + } + CascadesContext cascadesContext = ctx.cascadesContext; + // 1. We should deduplicate the slots, otherwise the binding process will fail due to the // ambiguous slots exist. // 2. try to bound order-key with agg output, if failed, try to bound with output of agg.child @@ -714,83 +662,32 @@ public class BindExpression implements AnalysisRuleFactory { // group by col1 // order by col1; # order by order_col1 // bind order_col1 with alias_col1, then, bind it with inner_col1 - List sortItemList = sort.getOrderKeys() - .stream() - .map(orderKey -> { - Expression item = bindSlot(orderKey.getExpr(), plan, ctx, true, false); - item = bindSlot(item, plan.children(), ctx, true, false); - item = bindFunction(item, sort, ctx); - return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst()); - }).collect(Collectors.toList()); - return new LogicalSort<>(sortItemList, sort.child()); - } + Scope inputScope = toScope(cascadesContext, input.getOutput()); - private List flatBoundStar( - List boundSlots, - List boundExceptions) { - return boundSlots - .stream() - .flatMap(slot -> { - if (slot instanceof BoundStar) { - return ((BoundStar) slot).getSlots().stream(); - } else { - return Stream.of(slot); - } - }) - .filter(s -> !boundExceptions.contains(s)) - .collect(ImmutableList.toImmutableList()); - } + final Plan finalInput = input; + Supplier inputChildrenScope = Suppliers.memoize( + () -> toScope(cascadesContext, PlanUtils.fastGetChildrenOutputs(finalInput.children()))); + SimpleExprAnalyzer analyzer = buildCustomSlotBinderAnalyzer( + sort, cascadesContext, inputScope, true, false, + (self, unboundSlot) -> { + // first, try to bind slot in Scope(input.output) + List slotsInInput = self.bindExactSlotsByThisScope(unboundSlot, inputScope); + if (slotsInInput.size() == 1) { + // bind succeed + return slotsInInput; + } + // second, bind failed: + // if the slot not found, or more than one candidate slots found in input.output, + // then try to bind by input.children.output + return self.bindExactSlotsByThisScope(unboundSlot, inputChildrenScope.get()); + }); - private List bindSlot( - List exprList, Plan input, CascadesContext cascadesContext) { - List slots = new ArrayList<>(exprList.size()); - for (E expr : exprList) { - E result = bindSlot(expr, input, cascadesContext); - slots.add(result); + Builder boundOrderKeys = ImmutableList.builderWithExpectedSize(sort.getOrderKeys().size()); + for (OrderKey orderKey : sort.getOrderKeys()) { + Expression boundKey = analyzer.analyze(orderKey.getExpr()); + boundOrderKeys.add(orderKey.withExpression(boundKey)); } - return slots; - } - - private E bindSlot(E expr, Plan input, CascadesContext cascadesContext) { - return bindSlot(expr, input, cascadesContext, true, true); - } - - private E bindSlot(E expr, Plan input, CascadesContext cascadesContext, - boolean enableExactMatch) { - return bindSlot(expr, input, cascadesContext, enableExactMatch, true); - } - - private E bindSlot(E expr, Plan input, CascadesContext cascadesContext, - boolean enableExactMatch, boolean bindSlotInOuterScope) { - return (E) new SlotBinder(toScope(cascadesContext, input.getOutput()), cascadesContext, - enableExactMatch, bindSlotInOuterScope).bind(expr); - } - - @SuppressWarnings("unchecked") - private E bindSlot(E expr, List inputs, CascadesContext cascadesContext, - boolean enableExactMatch) { - return bindSlot(expr, inputs, cascadesContext, enableExactMatch, true); - } - - private E bindSlot(E expr, List inputs, CascadesContext cascadesContext, - boolean enableExactMatch, boolean bindSlotInOuterScope) { - List boundedSlots = new ArrayList<>(); - for (Plan input : inputs) { - boundedSlots.addAll(input.getOutput()); - } - return (E) new SlotBinder(toScope(cascadesContext, boundedSlots), cascadesContext, - enableExactMatch, bindSlotInOuterScope).bind(expr); - } - - @SuppressWarnings("unchecked") - private E bindSlot(E expr, List inputs, CascadesContext cascadesContext) { - return bindSlot(expr, inputs, cascadesContext, true); - } - - @SuppressWarnings("unchecked") - private E bindFunction(E expr, Plan plan, CascadesContext cascadesContext) { - return (E) FunctionBinder.INSTANCE.rewrite(checkBoundExceptLambda(expr, plan), - new ExpressionRewriteContext(cascadesContext)); + return new LogicalSort<>(boundOrderKeys.build(), sort.child()); } /** @@ -798,32 +695,28 @@ public class BindExpression implements AnalysisRuleFactory { */ private List adjustNullableForRepeat( List> groupingSets, - List output) { - Set groupingSetsSlots = groupingSets.stream() + List outputs) { + Set groupingSetsUsedSlots = groupingSets.stream() .flatMap(Collection::stream) .map(Expression::getInputSlots) .flatMap(Set::stream) .collect(Collectors.toSet()); - return output.stream() - .map(e -> e.accept(RewriteNullableToTrue.INSTANCE, groupingSetsSlots)) - .map(NamedExpression.class::cast) - .collect(ImmutableList.toImmutableList()); - } - - private static class RewriteNullableToTrue extends DefaultExpressionRewriter> { - public static RewriteNullableToTrue INSTANCE = new RewriteNullableToTrue(); - - @Override - public Expression visitSlotReference(SlotReference slotReference, Set childrenOutput) { - if (childrenOutput.contains(slotReference)) { - return slotReference.withNullable(true); - } - return slotReference; + Builder nullableOutputs = ImmutableList.builderWithExpectedSize(outputs.size()); + for (NamedExpression output : outputs) { + Expression nullableOutput = output.rewriteUp(expr -> { + if (expr instanceof Slot && groupingSetsUsedSlots.contains(expr)) { + return ((Slot) expr).withNullable(true); + } + return expr; + }); + nullableOutputs.add((NamedExpression) nullableOutput); } + return nullableOutputs.build(); } - private LogicalTVFRelation bindTableValuedFunction(UnboundTVFRelation unboundTVFRelation, - StatementContext statementContext) { + private LogicalTVFRelation bindTableValuedFunction(MatchingContext ctx) { + UnboundTVFRelation unboundTVFRelation = ctx.root; + StatementContext statementContext = ctx.statementContext; Env env = statementContext.getConnectContext().getEnv(); FunctionRegistry functionRegistry = env.getFunctionRegistry(); @@ -838,35 +731,17 @@ public class BindExpression implements AnalysisRuleFactory { } private void checkSameNameSlot(List childOutputs, String subQueryAlias) { - Set nameSlots = new HashSet<>(); + Set nameSlots = new HashSet<>(childOutputs.size() * 2); for (Slot s : childOutputs) { - if (nameSlots.contains(s.getInternalName())) { + if (!nameSlots.add(s.getInternalName())) { throw new AnalysisException("Duplicated inline view column alias: '" + s.getName() + "'" + " in inline view: '" + subQueryAlias + "'"); - } else { - nameSlots.add(s.getInternalName()); } } } - private BoundFunction bindTableGeneratingFunction(UnboundFunction unboundFunction, Plan plan, - CascadesContext cascadesContext) { - List boundArguments = unboundFunction.getArguments().stream() - .map(e -> bindFunction(e, plan, cascadesContext)) - .collect(Collectors.toList()); - FunctionRegistry functionRegistry = cascadesContext.getConnectContext().getEnv().getFunctionRegistry(); - - String functionName = unboundFunction.getName(); - FunctionBuilder functionBuilder = functionRegistry.findFunctionBuilder(functionName, boundArguments); - Expression function = functionBuilder.build(functionName, boundArguments); - if (!(function instanceof TableGeneratingFunction)) { - throw new AnalysisException(function.toSql() + " is not a TableGeneratingFunction"); - } - return (BoundFunction) TypeCoercionUtils.processBoundFunction((BoundFunction) function); - } - - private void checkIfOutputAliasNameDuplicatedForGroupBy(List expressions, - List output) { + private void checkIfOutputAliasNameDuplicatedForGroupBy(Collection expressions, + List output) { // if group_by_and_having_use_alias_first=true, we should fall back to original planner until we // support the session variable. if (output.stream().noneMatch(Alias.class::isInstance)) { @@ -875,10 +750,8 @@ public class BindExpression implements AnalysisRuleFactory { List aliasList = output.stream().filter(Alias.class::isInstance) .map(Alias.class::cast).collect(Collectors.toList()); - List exprAliasList = expressions.stream() - .map(expr -> (Set) expr.collect(NamedExpression.class::isInstance)) - .flatMap(Collection::stream) - .collect(Collectors.toList()); + List exprAliasList = + ExpressionUtils.collectAll(expressions, NamedExpression.class::isInstance); boolean isGroupByContainAlias = exprAliasList.stream().anyMatch(ne -> aliasList.stream().anyMatch(alias -> !alias.getExprId().equals(ne.getExprId()) @@ -909,4 +782,62 @@ public class BindExpression implements AnalysisRuleFactory { expression.children().forEach(e -> checkBoundExceptLambda(e, plan)); return expression; } + + private Scope toScope(CascadesContext cascadesContext, List slots) { + Optional outerScope = cascadesContext.getOuterScope(); + if (outerScope.isPresent()) { + return new Scope(outerScope, slots, outerScope.get().getSubquery()); + } else { + return new Scope(slots); + } + } + + private SimpleExprAnalyzer buildSimpleExprAnalyzer( + Plan currentPlan, CascadesContext cascadesContext, List children, + boolean enableExactMatch, boolean bindSlotInOuterScope) { + List childrenOutputs = PlanUtils.fastGetChildrenOutputs(children); + Scope scope = toScope(cascadesContext, childrenOutputs); + ExpressionRewriteContext rewriteContext = new ExpressionRewriteContext(cascadesContext); + ExpressionAnalyzer expressionAnalyzer = new ExpressionAnalyzer(currentPlan, + scope, cascadesContext, enableExactMatch, bindSlotInOuterScope); + return expr -> expressionAnalyzer.analyze(expr, rewriteContext); + } + + private SimpleExprAnalyzer buildCustomSlotBinderAnalyzer( + Plan currentPlan, CascadesContext cascadesContext, Scope defaultScope, + boolean enableExactMatch, boolean bindSlotInOuterScope, CustomSlotBinderAnalyzer customSlotBinder) { + ExpressionRewriteContext rewriteContext = new ExpressionRewriteContext(cascadesContext); + ExpressionAnalyzer expressionAnalyzer = new ExpressionAnalyzer(currentPlan, defaultScope, cascadesContext, + enableExactMatch, bindSlotInOuterScope) { + @Override + protected List bindSlotByThisScope(UnboundSlot unboundSlot) { + return customSlotBinder.bindSlot(this, unboundSlot); + } + }; + return expr -> expressionAnalyzer.analyze(expr, rewriteContext); + } + + private interface SimpleExprAnalyzer { + Expression analyze(Expression expr); + + default List analyzeToList(List exprs) { + ImmutableList.Builder result = ImmutableList.builderWithExpectedSize(exprs.size()); + for (E expr : exprs) { + result.add((E) analyze(expr)); + } + return result.build(); + } + + default Set analyzeToSet(List exprs) { + ImmutableSet.Builder result = ImmutableSet.builderWithExpectedSize(exprs.size() * 2); + for (E expr : exprs) { + result.add((E) analyze(expr)); + } + return result.build(); + } + } + + private interface CustomSlotBinderAnalyzer { + List bindSlot(ExpressionAnalyzer analyzer, UnboundSlot unboundSlot); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotWithPaths.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotWithPaths.java index 04eb8af073..114e4c1d12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotWithPaths.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotWithPaths.java @@ -79,7 +79,7 @@ public class BindSlotWithPaths implements AnalysisRuleFactory { return ctx.root; } newProjectsExpr.addAll(newExprs); - return new LogicalProject(newProjectsExpr, logicalOlapScan.withProjectPulledUp()); + return new LogicalProject<>(newProjectsExpr, logicalOlapScan.withProjectPulledUp()); })) ); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java index 49de801b78..5a310d697a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java @@ -128,13 +128,11 @@ public class CheckAnalysis implements AnalysisRuleFactory { } private void checkExpressionInputTypes(Plan plan) { - final Optional firstFailed = plan.getExpressions().stream() - .map(Expression::checkInputDataTypes) - .filter(TypeCheckResult::failed) - .findFirst(); - - if (firstFailed.isPresent()) { - throw new AnalysisException(firstFailed.get().getMessage()); + for (Expression expression : plan.getExpressions()) { + TypeCheckResult firstFailed = expression.checkInputDataTypes(); + if (firstFailed.failed()) { + throw new AnalysisException(firstFailed.getMessage()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java index 83eedb7ccd..29d6b5e4a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java @@ -91,6 +91,7 @@ public class EliminateLogicalSelectHint extends OneRewriteRuleFactory { if (value.isPresent()) { try { VariableMgr.setVar(sessionVariable, new SetVar(key, new StringLiteral(value.get()))); + context.invalidCache(key); } catch (Throwable t) { throw new AnalysisException("Can not set session variable '" + key + "' = '" + value.get() + "'", t); @@ -108,7 +109,6 @@ public class EliminateLogicalSelectHint extends OneRewriteRuleFactory { } throw new AnalysisException("The nereids is disabled in this sql, fallback to original planner"); } - context.invalidCache(selectHint.getHintName()); } private void extractLeading(SelectHintLeading selectHint, CascadesContext context, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java new file mode 100644 index 0000000000..dbdf817ef6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -0,0 +1,807 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.analysis.ArithmeticExpr.Operator; +import org.apache.doris.analysis.SetType; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.FunctionRegistry; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.util.Util; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.Scope; +import org.apache.doris.nereids.analyzer.UnboundAlias; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.analyzer.UnboundStar; +import org.apache.doris.nereids.analyzer.UnboundVariable; +import org.apache.doris.nereids.analyzer.UnboundVariable.VariableType; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ArrayItemReference; +import org.apache.doris.nereids.trees.expressions.BinaryArithmetic; +import org.apache.doris.nereids.trees.expressions.BitNot; +import org.apache.doris.nereids.trees.expressions.BoundStar; +import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.CompoundPredicate; +import org.apache.doris.nereids.trees.expressions.Divide; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.IntegralDivide; +import org.apache.doris.nereids.trees.expressions.ListQuery; +import org.apache.doris.nereids.trees.expressions.Match; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; +import org.apache.doris.nereids.trees.expressions.Variable; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl; +import org.apache.doris.nereids.trees.expressions.functions.scalar.PushDownToProjectionFunction; +import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdfBuilder; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.typecoercion.ImplicitCastInputTypes; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.VariableMgr; +import org.apache.doris.qe.VariableVarConverters; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.stream.Collectors; + +/** ExpressionAnalyzer */ +public class ExpressionAnalyzer extends SubExprAnalyzer { + + private final Plan currentPlan; + /* + bounded={table.a, a} + unbound=a + if enableExactMatch, 'a' is bound to bounded 'a', + if not enableExactMatch, 'a' is ambiguous + in order to be compatible to original planner, + exact match mode is not enabled for having clause + but enabled for order by clause + TODO after remove original planner, always enable exact match mode. + */ + private final boolean enableExactMatch; + private final boolean bindSlotInOuterScope; + private boolean currentInLambda; + + public ExpressionAnalyzer(Plan currentPlan, Scope scope, CascadesContext cascadesContext, + boolean enableExactMatch, boolean bindSlotInOuterScope) { + super(scope, cascadesContext); + this.currentPlan = currentPlan; + this.enableExactMatch = enableExactMatch; + this.bindSlotInOuterScope = bindSlotInOuterScope; + } + + public Expression analyze(Expression expression, ExpressionRewriteContext context) { + return expression.accept(this, context); + } + + @Override + public Expression visit(Expression expr, ExpressionRewriteContext context) { + expr = super.visit(expr, context); + + expr.checkLegalityBeforeTypeCoercion(); + // this cannot be removed, because some function already construct in parser. + if (expr instanceof ImplicitCastInputTypes) { + List expectedInputTypes = ((ImplicitCastInputTypes) expr).expectedInputTypes(); + if (!expectedInputTypes.isEmpty()) { + return TypeCoercionUtils.implicitCastInputTypes(expr, expectedInputTypes); + } + } + return expr; + } + + @Override + public Expression visitLambda(Lambda lambda, ExpressionRewriteContext context) { + boolean originInLambda = currentInLambda; + try { + currentInLambda = true; + return super.visitLambda(lambda, context); + } finally { + currentInLambda = originInLambda; + } + } + + /* ******************************************************************************************** + * bind slot + * ******************************************************************************************** */ + @Override + public Expression visitUnboundVariable(UnboundVariable unboundVariable, ExpressionRewriteContext context) { + String name = unboundVariable.getName(); + SessionVariable sessionVariable = ConnectContext.get().getSessionVariable(); + Literal literal = null; + if (unboundVariable.getType() == VariableType.DEFAULT) { + literal = VariableMgr.getLiteral(sessionVariable, name, SetType.DEFAULT); + } else if (unboundVariable.getType() == VariableType.SESSION) { + literal = VariableMgr.getLiteral(sessionVariable, name, SetType.SESSION); + } else if (unboundVariable.getType() == VariableType.GLOBAL) { + literal = VariableMgr.getLiteral(sessionVariable, name, SetType.GLOBAL); + } else if (unboundVariable.getType() == VariableType.USER) { + literal = ConnectContext.get().getLiteralForUserVar(name); + } + if (literal == null) { + throw new AnalysisException("Unsupported system variable: " + unboundVariable.getName()); + } + if (!Strings.isNullOrEmpty(name) && VariableVarConverters.hasConverter(name)) { + try { + Preconditions.checkArgument(literal instanceof IntegerLikeLiteral); + IntegerLikeLiteral integerLikeLiteral = (IntegerLikeLiteral) literal; + literal = new StringLiteral(VariableVarConverters.decode(name, integerLikeLiteral.getLongValue())); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage()); + } + } + return new Variable(unboundVariable.getName(), unboundVariable.getType(), literal); + } + + @Override + public Expression visitUnboundAlias(UnboundAlias unboundAlias, ExpressionRewriteContext context) { + Expression child = unboundAlias.child().accept(this, context); + if (unboundAlias.getAlias().isPresent()) { + return new Alias(child, unboundAlias.getAlias().get()); + // TODO: the variant bind element_at(slot, 'name') will return a slot, and we should + // assign an Alias to this function, this is trick and should refactor it + } else if (!(unboundAlias.child() instanceof ElementAt) && child instanceof NamedExpression) { + return new Alias(child, ((NamedExpression) child).getName()); + } else { + return new Alias(child); + } + } + + @Override + public Expression visitUnboundSlot(UnboundSlot unboundSlot, ExpressionRewriteContext context) { + Optional outerScope = getScope().getOuterScope(); + Optional> boundedOpt = Optional.of(bindSlotByThisScope(unboundSlot)); + boolean foundInThisScope = !boundedOpt.get().isEmpty(); + // Currently only looking for symbols on the previous level. + if (bindSlotInOuterScope && !foundInThisScope && outerScope.isPresent()) { + boundedOpt = Optional.of(bindSlotByScope(unboundSlot, outerScope.get())); + } + List bounded = boundedOpt.get(); + switch (bounded.size()) { + case 0: + if (!currentInLambda) { + String tableName = StringUtils.join(unboundSlot.getQualifier(), "."); + if (tableName.isEmpty()) { + tableName = "table list"; + } + throw new AnalysisException("Unknown column '" + + unboundSlot.getNameParts().get(unboundSlot.getNameParts().size() - 1) + + "' in '" + tableName + "' in " + + currentPlan.getType().toString().substring("LOGICAL_".length()) + " clause"); + } + return unboundSlot; + case 1: + Expression firstBound = bounded.get(0); + if (!foundInThisScope && firstBound instanceof Slot + && !outerScope.get().getCorrelatedSlots().contains(firstBound)) { + outerScope.get().getCorrelatedSlots().add((Slot) firstBound); + } + return firstBound; + default: + if (enableExactMatch) { + // select t1.k k, t2.k + // from t1 join t2 order by k + // + // 't1.k k' is denoted by alias_k, its full name is 'k' + // 'order by k' is denoted as order_k, it full name is 'k' + // 't2.k' in select list, its full name is 't2.k' + // + // order_k can be bound on alias_k and t2.k + // alias_k is exactly matched, since its full name is exactly match full name of order_k + // t2.k is not exactly matched, since t2.k's full name is larger than order_k + List exactMatch = bounded.stream() + .filter(Slot.class::isInstance) + .map(Slot.class::cast) + .filter(bound -> unboundSlot.getNameParts().size() == bound.getQualifier().size() + 1) + .collect(Collectors.toList()); + if (exactMatch.size() == 1) { + return exactMatch.get(0); + } + } + throw new AnalysisException(String.format("%s is ambiguous: %s.", + unboundSlot.toSql(), + bounded.stream() + .map(Expression::toString) + .collect(Collectors.joining(", ")))); + } + } + + @Override + public Expression visitUnboundStar(UnboundStar unboundStar, ExpressionRewriteContext context) { + List qualifier = unboundStar.getQualifier(); + boolean showHidden = Util.showHiddenColumns(); + List slots = getScope().getSlots() + .stream() + .filter(slot -> !(slot instanceof SlotReference) + || (((SlotReference) slot).isVisible()) || showHidden) + .filter(slot -> !(((SlotReference) slot).hasSubColPath())) + .collect(Collectors.toList()); + switch (qualifier.size()) { + case 0: // select * + return new BoundStar(slots); + case 1: // select table.* + case 2: // select db.table.* + case 3: // select catalog.db.table.* + return bindQualifiedStar(qualifier, slots); + default: + throw new AnalysisException("Not supported qualifier: " + + StringUtils.join(qualifier, ".")); + } + } + + /* ******************************************************************************************** + * bind function + * ******************************************************************************************** */ + @Override + public Expression visitUnboundFunction(UnboundFunction unboundFunction, ExpressionRewriteContext context) { + if (unboundFunction.isHighOrder()) { + unboundFunction = bindHighOrderFunction(unboundFunction, context); + } else { + unboundFunction = (UnboundFunction) rewriteChildren(this, unboundFunction, context); + } + + // bind function + FunctionRegistry functionRegistry = Env.getCurrentEnv().getFunctionRegistry(); + List arguments = unboundFunction.isDistinct() + ? ImmutableList.builderWithExpectedSize(unboundFunction.arity() + 1) + .add(unboundFunction.isDistinct()) + .addAll(unboundFunction.getArguments()) + .build() + : (List) unboundFunction.getArguments(); + + if (StringUtils.isEmpty(unboundFunction.getDbName())) { + // we will change arithmetic function like add(), subtract(), bitnot() + // to the corresponding objects rather than BoundFunction. + ArithmeticFunctionBinder functionBinder = new ArithmeticFunctionBinder(); + if (functionBinder.isBinaryArithmetic(unboundFunction.getName())) { + return functionBinder.bindBinaryArithmetic(unboundFunction.getName(), unboundFunction.children()) + .accept(this, context); + } + } + + String functionName = unboundFunction.getName(); + FunctionBuilder builder = functionRegistry.findFunctionBuilder( + unboundFunction.getDbName(), functionName, arguments); + if (builder instanceof AliasUdfBuilder) { + // we do type coercion in build function in alias function, so it's ok to return directly. + return builder.build(functionName, arguments); + } else { + Expression boundFunction = TypeCoercionUtils + .processBoundFunction((BoundFunction) builder.build(functionName, arguments)); + if (boundFunction instanceof Count + && context.cascadesContext.getOuterScope().isPresent() + && !context.cascadesContext.getOuterScope().get().getCorrelatedSlots() + .isEmpty()) { + // consider sql: SELECT * FROM t1 WHERE t1.a <= (SELECT COUNT(t2.a) FROM t2 WHERE (t1.b = t2.b)); + // when unnest correlated subquery, we create a left join node. + // outer query is left table and subquery is right one + // if there is no match, the row from right table is filled with nulls + // but COUNT function is always not nullable. + // so wrap COUNT with Nvl to ensure it's result is 0 instead of null to get the correct result + boundFunction = new Nvl(boundFunction, new BigIntLiteral(0)); + } + return boundFunction; + } + } + + @Override + public Expression visitBoundFunction(BoundFunction boundFunction, ExpressionRewriteContext context) { + boundFunction = (BoundFunction) super.visitBoundFunction(boundFunction, context); + return TypeCoercionUtils.processBoundFunction(boundFunction); + } + + @Override + public Expression visitElementAt(ElementAt elementAt, ExpressionRewriteContext context) { + ElementAt boundFunction = (ElementAt) visitBoundFunction(elementAt, context); + if (PushDownToProjectionFunction.validToPushDown(boundFunction)) { + if (ConnectContext.get() != null + && ConnectContext.get().getSessionVariable() != null + && !ConnectContext.get().getSessionVariable().isEnableRewriteElementAtToSlot()) { + return boundFunction; + } + Slot slot = boundFunction.getInputSlots().stream().findFirst().get(); + if (slot.hasUnbound()) { + slot = (Slot) slot.accept(this, context); + } + StatementContext statementContext = context.cascadesContext.getStatementContext(); + Expression originBoundFunction = boundFunction.rewriteUp(expr -> { + if (expr instanceof SlotReference) { + Expression originalExpr = statementContext.getOriginalExpr((SlotReference) expr); + return originalExpr == null ? expr : originalExpr; + } + return expr; + }); + // rewrite to slot and bound this slot + return PushDownToProjectionFunction.rewriteToSlot( + (PushDownToProjectionFunction) originBoundFunction, (SlotReference) slot); + } + return boundFunction; + } + + /** + * gets the method for calculating the time. + * e.g. YEARS_ADD、YEARS_SUB、DAYS_ADD 、DAYS_SUB + */ + @Override + public Expression visitTimestampArithmetic(TimestampArithmetic arithmetic, ExpressionRewriteContext context) { + Expression left = arithmetic.left().accept(this, context); + Expression right = arithmetic.right().accept(this, context); + + arithmetic = (TimestampArithmetic) arithmetic.withChildren(left, right); + // bind function + String funcOpName; + if (arithmetic.getFuncName() == null) { + // e.g. YEARS_ADD, MONTHS_SUB + funcOpName = String.format("%sS_%s", arithmetic.getTimeUnit(), + (arithmetic.getOp() == Operator.ADD) ? "ADD" : "SUB"); + } else { + funcOpName = arithmetic.getFuncName(); + } + arithmetic = (TimestampArithmetic) arithmetic.withFuncName(funcOpName.toLowerCase(Locale.ROOT)); + + // type coercion + return TypeCoercionUtils.processTimestampArithmetic(arithmetic); + } + + /* ******************************************************************************************** + * type coercion + * ******************************************************************************************** */ + + @Override + public Expression visitBitNot(BitNot bitNot, ExpressionRewriteContext context) { + Expression child = bitNot.child().accept(this, context); + // type coercion + if (!(child.getDataType().isIntegralType() || child.getDataType().isBooleanType())) { + child = new Cast(child, BigIntType.INSTANCE); + } + return bitNot.withChildren(child); + } + + @Override + public Expression visitDivide(Divide divide, ExpressionRewriteContext context) { + Expression left = divide.left().accept(this, context); + Expression right = divide.right().accept(this, context); + divide = (Divide) divide.withChildren(left, right); + // type coercion + return TypeCoercionUtils.processDivide(divide); + } + + @Override + public Expression visitIntegralDivide(IntegralDivide integralDivide, ExpressionRewriteContext context) { + Expression left = integralDivide.left().accept(this, context); + Expression right = integralDivide.right().accept(this, context); + integralDivide = (IntegralDivide) integralDivide.withChildren(left, right); + // type coercion + return TypeCoercionUtils.processIntegralDivide(integralDivide); + } + + @Override + public Expression visitBinaryArithmetic(BinaryArithmetic binaryArithmetic, ExpressionRewriteContext context) { + Expression left = binaryArithmetic.left().accept(this, context); + Expression right = binaryArithmetic.right().accept(this, context); + binaryArithmetic = (BinaryArithmetic) binaryArithmetic.withChildren(left, right); + return TypeCoercionUtils.processBinaryArithmetic(binaryArithmetic); + } + + @Override + public Expression visitCompoundPredicate(CompoundPredicate compoundPredicate, ExpressionRewriteContext context) { + Expression left = compoundPredicate.left().accept(this, context); + Expression right = compoundPredicate.right().accept(this, context); + CompoundPredicate ret = (CompoundPredicate) compoundPredicate.withChildren(left, right); + return TypeCoercionUtils.processCompoundPredicate(ret); + } + + @Override + public Expression visitNot(Not not, ExpressionRewriteContext context) { + // maybe is `not subquery`, we should bind it first + Expression expr = super.visitNot(not, context); + + // expression is not subquery + if (expr instanceof Not) { + Expression child = not.child().accept(this, context); + Expression newChild = TypeCoercionUtils.castIfNotSameType(child, BooleanType.INSTANCE); + if (child != newChild) { + return expr.withChildren(newChild); + } + } + return expr; + } + + @Override + public Expression visitComparisonPredicate(ComparisonPredicate cp, ExpressionRewriteContext context) { + Expression left = cp.left().accept(this, context); + Expression right = cp.right().accept(this, context); + cp = (ComparisonPredicate) cp.withChildren(left, right); + return TypeCoercionUtils.processComparisonPredicate(cp); + } + + @Override + public Expression visitCaseWhen(CaseWhen caseWhen, ExpressionRewriteContext context) { + Builder rewrittenChildren = ImmutableList.builderWithExpectedSize(caseWhen.arity()); + for (Expression child : caseWhen.children()) { + rewrittenChildren.add(child.accept(this, context)); + } + CaseWhen newCaseWhen = caseWhen.withChildren(rewrittenChildren.build()); + newCaseWhen.checkLegalityBeforeTypeCoercion(); + return TypeCoercionUtils.processCaseWhen(newCaseWhen); + } + + @Override + public Expression visitWhenClause(WhenClause whenClause, ExpressionRewriteContext context) { + return whenClause.withChildren(TypeCoercionUtils.castIfNotSameType( + whenClause.getOperand().accept(this, context), BooleanType.INSTANCE), + whenClause.getResult().accept(this, context)); + } + + @Override + public Expression visitInPredicate(InPredicate inPredicate, ExpressionRewriteContext context) { + List rewrittenChildren = inPredicate.children().stream() + .map(e -> e.accept(this, context)).collect(Collectors.toList()); + InPredicate newInPredicate = inPredicate.withChildren(rewrittenChildren); + return TypeCoercionUtils.processInPredicate(newInPredicate); + } + + @Override + public Expression visitInSubquery(InSubquery inSubquery, ExpressionRewriteContext context) { + // analyze subquery + inSubquery = (InSubquery) super.visitInSubquery(inSubquery, context); + + // compareExpr already analyze when invoke super.visitInSubquery + Expression newCompareExpr = inSubquery.getCompareExpr(); + // but ListQuery does not analyze + Expression newListQuery = inSubquery.getListQuery().accept(this, context); + + ComparisonPredicate afterTypeCoercion = (ComparisonPredicate) TypeCoercionUtils.processComparisonPredicate( + new EqualTo(newCompareExpr, newListQuery)); + if (newListQuery.getDataType().isBitmapType()) { + if (!newCompareExpr.getDataType().isBigIntType()) { + newCompareExpr = new Cast(newCompareExpr, BigIntType.INSTANCE); + } + } else { + newCompareExpr = afterTypeCoercion.left(); + } + return new InSubquery(newCompareExpr, (ListQuery) afterTypeCoercion.right(), + inSubquery.getCorrelateSlots(), ((ListQuery) afterTypeCoercion.right()).getTypeCoercionExpr(), + inSubquery.isNot()); + } + + @Override + public Expression visitMatch(Match match, ExpressionRewriteContext context) { + Expression left = match.left().accept(this, context); + Expression right = match.right().accept(this, context); + // check child type + if (!left.getDataType().isStringLikeType() + && !(left.getDataType() instanceof ArrayType + && ((ArrayType) left.getDataType()).getItemType().isStringLikeType()) + && !left.getDataType().isVariantType()) { + throw new AnalysisException(String.format( + "left operand '%s' part of predicate " + + "'%s' should return type 'STRING', 'ARRAY or VARIANT' but " + + "returns type '%s'.", + left.toSql(), match.toSql(), left.getDataType())); + } + + if (!right.getDataType().isStringLikeType() && !right.getDataType().isNullType()) { + throw new AnalysisException(String.format( + "right operand '%s' part of predicate " + "'%s' should return type 'STRING' but " + + "returns type '%s'.", + right.toSql(), match.toSql(), right.getDataType())); + } + + if (left.getDataType().isVariantType()) { + left = new Cast(left, right.getDataType()); + } + return match.withChildren(left, right); + } + + @Override + public Expression visitCast(Cast cast, ExpressionRewriteContext context) { + cast = (Cast) super.visitCast(cast, context); + // NOTICE: just for compatibility with legacy planner. + if (cast.child().getDataType().isComplexType() || cast.getDataType().isComplexType()) { + TypeCoercionUtils.checkCanCastTo(cast.child().getDataType(), cast.getDataType()); + } + return cast; + } + + private BoundStar bindQualifiedStar(List qualifierStar, List boundSlots) { + // FIXME: compatible with previous behavior: + // https://github.com/apache/doris/pull/10415/files/3fe9cb0c3f805ab3a9678033b281b16ad93ec60a#r910239452 + List slots = boundSlots.stream().filter(boundSlot -> { + switch (qualifierStar.size()) { + // table.* + case 1: + List boundSlotQualifier = boundSlot.getQualifier(); + switch (boundSlotQualifier.size()) { + // bound slot is `column` and no qualified + case 0: + return false; + case 1: // bound slot is `table`.`column` + return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)); + case 2:// bound slot is `db`.`table`.`column` + return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(1)); + case 3:// bound slot is `catalog`.`db`.`table`.`column` + return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(2)); + default: + throw new AnalysisException("Not supported qualifier: " + + StringUtils.join(qualifierStar, ".")); + } + case 2: // db.table.* + boundSlotQualifier = boundSlot.getQualifier(); + switch (boundSlotQualifier.size()) { + // bound slot is `column` and no qualified + case 0: + case 1: // bound slot is `table`.`column` + return false; + case 2:// bound slot is `db`.`table`.`column` + return compareDbName(qualifierStar.get(0), boundSlotQualifier.get(0)) + && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(1)); + case 3:// bound slot is `catalog`.`db`.`table`.`column` + return compareDbName(qualifierStar.get(0), boundSlotQualifier.get(1)) + && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(2)); + default: + throw new AnalysisException("Not supported qualifier: " + + StringUtils.join(qualifierStar, ".") + ".*"); + } + case 3: // catalog.db.table.* + boundSlotQualifier = boundSlot.getQualifier(); + switch (boundSlotQualifier.size()) { + // bound slot is `column` and no qualified + case 0: + case 1: // bound slot is `table`.`column` + case 2: // bound slot is `db`.`table`.`column` + return false; + case 3:// bound slot is `catalog`.`db`.`table`.`column` + return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)) + && compareDbName(qualifierStar.get(1), boundSlotQualifier.get(1)) + && qualifierStar.get(2).equalsIgnoreCase(boundSlotQualifier.get(2)); + default: + throw new AnalysisException("Not supported qualifier: " + + StringUtils.join(qualifierStar, ".") + ".*"); + } + default: + throw new AnalysisException("Not supported name: " + + StringUtils.join(qualifierStar, ".") + ".*"); + } + }).collect(Collectors.toList()); + + if (slots.isEmpty()) { + throw new AnalysisException("unknown qualifier: " + StringUtils.join(qualifierStar, ".") + ".*"); + } + return new BoundStar(slots); + } + + protected List bindSlotByThisScope(UnboundSlot unboundSlot) { + return bindSlotByScope(unboundSlot, getScope()); + } + + protected List bindExactSlotsByThisScope(UnboundSlot unboundSlot, Scope scope) { + List candidates = bindSlotByScope(unboundSlot, scope); + if (candidates.size() == 1) { + return candidates; + } + List extractSlots = Utils.filterImmutableList(candidates, bound -> + unboundSlot.getNameParts().size() == bound.getQualifier().size() + 1 + ); + // we should return origin candidates slots if extract slots is empty, + // and then throw an ambiguous exception + return !extractSlots.isEmpty() ? extractSlots : candidates; + } + + /** bindSlotByScope */ + public List bindSlotByScope(UnboundSlot unboundSlot, Scope scope) { + List nameParts = unboundSlot.getNameParts(); + int namePartSize = nameParts.size(); + switch (namePartSize) { + // column + case 1: { + return bindSingleSlotByName(nameParts.get(0), scope); + } + // table.column + case 2: { + return bindSingleSlotByTable(nameParts.get(0), nameParts.get(1), scope); + } + // db.table.column + case 3: { + return bindSingleSlotByDb(nameParts.get(0), nameParts.get(1), nameParts.get(2), scope); + } + // catalog.db.table.column + case 4: { + return bindSingleSlotByCatalog( + nameParts.get(0), nameParts.get(1), nameParts.get(2), nameParts.get(3), scope); + } + default: { + throw new AnalysisException("Not supported name: " + StringUtils.join(nameParts, ".")); + } + } + } + + public static boolean compareDbName(String boundedDbName, String unBoundDbName) { + return unBoundDbName.equalsIgnoreCase(boundedDbName); + } + + public static boolean sameTableName(String boundSlot, String unboundSlot) { + if (GlobalVariable.lowerCaseTableNames != 1) { + return boundSlot.equals(unboundSlot); + } else { + return boundSlot.equalsIgnoreCase(unboundSlot); + } + } + + private void checkBoundLambda(Expression lambdaFunction, List argumentNames) { + lambdaFunction.foreachUp(e -> { + if (e instanceof UnboundSlot) { + UnboundSlot unboundSlot = (UnboundSlot) e; + throw new AnalysisException("Unknown lambda slot '" + + unboundSlot.getNameParts().get(unboundSlot.getNameParts().size() - 1) + + " in lambda arguments" + argumentNames); + } + }); + } + + private UnboundFunction bindHighOrderFunction(UnboundFunction unboundFunction, ExpressionRewriteContext context) { + int childrenSize = unboundFunction.children().size(); + List subChildren = new ArrayList<>(); + for (int i = 1; i < childrenSize; i++) { + subChildren.add(unboundFunction.child(i).accept(this, context)); + } + + // bindLambdaFunction + Lambda lambda = (Lambda) unboundFunction.children().get(0); + Expression lambdaFunction = lambda.getLambdaFunction(); + List arrayItemReferences = lambda.makeArguments(subChildren); + + // 1.bindSlot + List boundedSlots = arrayItemReferences.stream() + .map(ArrayItemReference::toSlot) + .collect(ImmutableList.toImmutableList()); + lambdaFunction = new SlotBinder(new Scope(boundedSlots), context.cascadesContext, + true, false).bind(lambdaFunction); + checkBoundLambda(lambdaFunction, lambda.getLambdaArgumentNames()); + + // 2.bindFunction + lambdaFunction = lambdaFunction.accept(this, context); + + Lambda lambdaClosure = lambda.withLambdaFunctionArguments(lambdaFunction, arrayItemReferences); + + // We don't add the ArrayExpression in high order function at all + return unboundFunction.withChildren(ImmutableList.builder() + .add(lambdaClosure) + .build()); + } + + private boolean shouldBindSlotBy(int namePartSize, Slot boundSlot) { + if (boundSlot instanceof SlotReference + && ((SlotReference) boundSlot).hasSubColPath()) { + // already bounded + return false; + } + if (namePartSize > boundSlot.getQualifier().size() + 1) { + return false; + } + return true; + } + + private List bindSingleSlotByName(String name, Scope scope) { + int namePartSize = 1; + Builder usedSlots = ImmutableList.builderWithExpectedSize(1); + for (Slot boundSlot : scope.findSlotIgnoreCase(name)) { + if (!shouldBindSlotBy(namePartSize, boundSlot)) { + continue; + } + // set sql case as alias + usedSlots.add(boundSlot.withName(name)); + } + return usedSlots.build(); + } + + private List bindSingleSlotByTable(String table, String name, Scope scope) { + int namePartSize = 2; + Builder usedSlots = ImmutableList.builderWithExpectedSize(1); + for (Slot boundSlot : scope.findSlotIgnoreCase(name)) { + if (!shouldBindSlotBy(namePartSize, boundSlot)) { + continue; + } + List boundSlotQualifier = boundSlot.getQualifier(); + String boundSlotTable = boundSlotQualifier.get(boundSlotQualifier.size() - 1); + if (!sameTableName(boundSlotTable, table)) { + continue; + } + // set sql case as alias + usedSlots.add(boundSlot.withName(name)); + } + return usedSlots.build(); + } + + private List bindSingleSlotByDb(String db, String table, String name, Scope scope) { + int namePartSize = 3; + Builder usedSlots = ImmutableList.builderWithExpectedSize(1); + for (Slot boundSlot : scope.findSlotIgnoreCase(name)) { + if (!shouldBindSlotBy(namePartSize, boundSlot)) { + continue; + } + List boundSlotQualifier = boundSlot.getQualifier(); + String boundSlotDb = boundSlotQualifier.get(boundSlotQualifier.size() - 2); + String boundSlotTable = boundSlotQualifier.get(boundSlotQualifier.size() - 1); + if (!compareDbName(boundSlotDb, db) || !sameTableName(boundSlotTable, table)) { + continue; + } + // set sql case as alias + usedSlots.add(boundSlot.withName(name)); + } + return usedSlots.build(); + } + + private List bindSingleSlotByCatalog(String catalog, String db, String table, String name, Scope scope) { + int namePartSize = 4; + Builder usedSlots = ImmutableList.builderWithExpectedSize(1); + for (Slot boundSlot : scope.findSlotIgnoreCase(name)) { + if (!shouldBindSlotBy(namePartSize, boundSlot)) { + continue; + } + List boundSlotQualifier = boundSlot.getQualifier(); + String boundSlotCatalog = boundSlotQualifier.get(boundSlotQualifier.size() - 3); + String boundSlotDb = boundSlotQualifier.get(boundSlotQualifier.size() - 2); + String boundSlotTable = boundSlotQualifier.get(boundSlotQualifier.size() - 1); + if (!boundSlotCatalog.equalsIgnoreCase(catalog) + || !compareDbName(boundSlotDb, db) + || !sameTableName(boundSlotTable, table)) { + continue; + } + // set sql case as alias + usedSlots.add(boundSlot.withName(name)); + } + return usedSlots.build(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java index 8ae79b9c21..8b6d82087d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java @@ -55,7 +55,7 @@ import java.util.stream.Collectors; /** * SlotBinder is used to bind slot */ -public class SlotBinder extends SubExprAnalyzer { +public class SlotBinder extends SubExprAnalyzer { /* bounded={table.a, a} unbound=a diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java index 4513d2f797..047b0efbd2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java @@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.BinaryOperator; import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InSubquery; @@ -49,8 +48,7 @@ import java.util.Optional; /** * Use the visitor to iterate sub expression. */ -class SubExprAnalyzer extends DefaultExpressionRewriter { - +class SubExprAnalyzer extends DefaultExpressionRewriter { private final Scope scope; private final CascadesContext cascadesContext; @@ -60,7 +58,7 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { } @Override - public Expression visitNot(Not not, CascadesContext context) { + public Expression visitNot(Not not, T context) { Expression child = not.child(); if (child instanceof Exists) { return visitExistsSubquery( @@ -73,7 +71,7 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { } @Override - public Expression visitExistsSubquery(Exists exists, CascadesContext context) { + public Expression visitExistsSubquery(Exists exists, T context) { AnalyzedResult analyzedResult = analyzeSubquery(exists); if (analyzedResult.rootIsLimitZero()) { return BooleanLiteral.of(exists.isNot()); @@ -87,7 +85,7 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { } @Override - public Expression visitInSubquery(InSubquery expr, CascadesContext context) { + public Expression visitInSubquery(InSubquery expr, T context) { AnalyzedResult analyzedResult = analyzeSubquery(expr); checkOutputColumn(analyzedResult.getLogicalPlan()); @@ -101,7 +99,7 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { } @Override - public Expression visitScalarSubquery(ScalarSubquery scalar, CascadesContext context) { + public Expression visitScalarSubquery(ScalarSubquery scalar, T context) { AnalyzedResult analyzedResult = analyzeSubquery(scalar); checkOutputColumn(analyzedResult.getLogicalPlan()); @@ -111,13 +109,6 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { return new ScalarSubquery(analyzedResult.getLogicalPlan(), analyzedResult.getCorrelatedSlots()); } - private boolean childrenAtLeastOneInOrExistsSub(BinaryOperator binaryOperator) { - return binaryOperator.left().anyMatch(InSubquery.class::isInstance) - || binaryOperator.left().anyMatch(Exists.class::isInstance) - || binaryOperator.right().anyMatch(InSubquery.class::isInstance) - || binaryOperator.right().anyMatch(Exists.class::isInstance); - } - private void checkOutputColumn(LogicalPlan plan) { if (plan.getOutput().size() != 1) { throw new AnalysisException("Multiple columns returned by subquery are not yet supported. Found " diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughUnion.java index b685f680c9..df3069105d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughUnion.java @@ -79,7 +79,7 @@ public class PushDownLimitDistinctThroughUnion implements RewriteRuleFactory { .map(expr -> ExpressionUtils.replace(expr, replaceMap)) .collect(Collectors.toList()); List newOutputs = agg.getOutputs().stream() - .map(expr -> ExpressionUtils.replace(expr, replaceMap)) + .map(expr -> ExpressionUtils.replaceNameExpression(expr, replaceMap)) .collect(Collectors.toList()); LogicalAggregate newAgg = new LogicalAggregate<>(newGroupBy, newOutputs, child); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectIntoUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectIntoUnion.java index da6aad5f69..971071d1a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectIntoUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectIntoUnion.java @@ -62,7 +62,7 @@ public class PushProjectIntoUnion extends OneRewriteRuleFactory { if (old instanceof SlotReference) { newProjections.add(replaceRootMap.get(old)); } else { - newProjections.add(ExpressionUtils.replace(old, replaceMap)); + newProjections.add(ExpressionUtils.replaceNameExpression(old, replaceMap)); } } newConstExprs.add(newProjections.build()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectThroughUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectThroughUnion.java index 9d997ba1d0..016641f60a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectThroughUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushProjectThroughUnion.java @@ -65,7 +65,7 @@ public class PushProjectThroughUnion extends OneRewriteRuleFactory { replaceMap.put(union.getOutput().get(j), union.getRegularChildOutput(i).get(j)); } List childProjections = project.getProjects().stream() - .map(e -> (NamedExpression) ExpressionUtils.replace(e, replaceMap)) + .map(e -> (NamedExpression) ExpressionUtils.replaceNameExpression(e, replaceMap)) .map(e -> { if (e instanceof Alias) { return new Alias(((Alias) e).child(), e.getName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index 570404c6ea..3857a04ba1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -1572,7 +1572,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial LogicalProject project, Map projectMap) { return project.getProjects().stream() - .map(expr -> (NamedExpression) ExpressionUtils.replace(expr, projectMap)) + .map(expr -> (NamedExpression) ExpressionUtils.replaceNameExpression(expr, projectMap)) .collect(Collectors.toList()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java index 7a545ec17b..59d2acbe22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/AbstractTreeNode.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.trees; -import com.google.common.collect.ImmutableList; +import org.apache.doris.nereids.util.Utils; import java.util.List; @@ -34,11 +34,15 @@ public abstract class AbstractTreeNode> // https://github.com/apache/doris/pull/9807#discussion_r884829067 protected AbstractTreeNode(NODE_TYPE... children) { - this.children = ImmutableList.copyOf(children); + // NOTE: ImmutableList.copyOf has additional clone of the list, so here we + // direct generate a ImmutableList + this.children = Utils.fastToImmutableList(children); } protected AbstractTreeNode(List children) { - this.children = ImmutableList.copyOf(children); + // NOTE: ImmutableList.copyOf has additional clone of the list, so here we + // direct generate a ImmutableList + this.children = Utils.fastToImmutableList(children); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 3519a983fd..fb41384b5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees; +import org.apache.doris.nereids.util.Utils; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; @@ -24,6 +26,7 @@ import com.google.common.collect.ImmutableSet; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Deque; +import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.function.BiFunction; @@ -46,7 +49,7 @@ public interface TreeNode> { int arity(); default NODE_TYPE withChildren(NODE_TYPE... children) { - return withChildren(ImmutableList.copyOf(children)); + return withChildren(Utils.fastToImmutableList(children)); } NODE_TYPE withChildren(List children); @@ -175,6 +178,18 @@ public interface TreeNode> { } } + /** foreachBreath */ + default void foreachBreath(Predicate> func) { + LinkedList> queue = new LinkedList<>(); + queue.add(this); + while (!queue.isEmpty()) { + TreeNode current = queue.pollFirst(); + if (!func.test(current)) { + queue.addAll(current.children()); + } + } + } + default void foreachUp(Consumer> func) { for (NODE_TYPE child : children()) { child.foreach(func); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java index 11456e8f94..bd48b648a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CaseWhen.java @@ -19,18 +19,19 @@ package org.apache.doris.nereids.trees.expressions; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.UnboundException; -import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import com.google.common.base.Preconditions; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.stream.Stream; +import java.util.function.Supplier; /** * The internal representation of @@ -43,17 +44,24 @@ public class CaseWhen extends Expression { private final List whenClauses; private final Optional defaultValue; + private Supplier> dataTypesForCoercion; public CaseWhen(List whenClauses) { super((List) whenClauses); this.whenClauses = ImmutableList.copyOf(Objects.requireNonNull(whenClauses)); defaultValue = Optional.empty(); + this.dataTypesForCoercion = computeDataTypesForCoercion(); } + /** CaseWhen */ public CaseWhen(List whenClauses, Expression defaultValue) { - super(ImmutableList.builder().addAll(whenClauses).add(defaultValue).build()); + super(ImmutableList.builderWithExpectedSize(whenClauses.size() + 1) + .addAll(whenClauses) + .add(defaultValue) + .build()); this.whenClauses = ImmutableList.copyOf(Objects.requireNonNull(whenClauses)); this.defaultValue = Optional.of(Objects.requireNonNull(defaultValue)); + this.dataTypesForCoercion = computeDataTypesForCoercion(); } public List getWhenClauses() { @@ -64,10 +72,9 @@ public class CaseWhen extends Expression { return defaultValue; } + /** dataTypesForCoercion */ public List dataTypesForCoercion() { - return Stream.concat(whenClauses.stream(), defaultValue.map(Stream::of).orElseGet(Stream::empty)) - .map(ExpressionTrait::getDataType) - .collect(ImmutableList.toImmutableList()); + return this.dataTypesForCoercion.get(); } public R accept(ExpressionVisitor visitor, C context) { @@ -136,4 +143,16 @@ public class CaseWhen extends Expression { } return new CaseWhen(whenClauseList, defaultValue); } + + private Supplier> computeDataTypesForCoercion() { + return Suppliers.memoize(() -> { + Builder dataTypes = ImmutableList.builderWithExpectedSize( + whenClauses.size() + (defaultValue.isPresent() ? 1 : 0)); + for (WhenClause whenClause : whenClauses) { + dataTypes.add(whenClause.getDataType()); + } + defaultValue.ifPresent(expression -> dataTypes.add(expression.getDataType())); + return dataTypes.build(); + }); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index 6909642113..1660efa3a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -39,7 +39,6 @@ import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; import org.apache.doris.nereids.types.StructField; import org.apache.doris.nereids.types.StructType; -import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; @@ -47,7 +46,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; -import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -67,36 +65,36 @@ public abstract class Expression extends AbstractTreeNode implements protected Expression(Expression... children) { super(children); - depth = Arrays.stream(children) - .mapToInt(e -> e.depth) - .max().orElse(0) + 1; - width = Arrays.stream(children) - .mapToInt(e -> e.width) - .sum() + (children.length == 0 ? 1 : 0); + int maxChildDepth = 0; + int sumChildWidth = 0; + for (int i = 0; i < children.length; ++i) { + Expression child = children[i]; + maxChildDepth = Math.max(child.depth, maxChildDepth); + sumChildWidth += child.width; + } + this.depth = maxChildDepth + 1; + this.width = sumChildWidth + ((children.length == 0) ? 1 : 0); + checkLimit(); this.inferred = false; } protected Expression(List children) { - super(children); - depth = children.stream() - .mapToInt(e -> e.depth) - .max().orElse(0) + 1; - width = children.stream() - .mapToInt(e -> e.width) - .sum() + (children.isEmpty() ? 1 : 0); - checkLimit(); - this.inferred = false; + this(children, false); } protected Expression(List children, boolean inferred) { super(children); - depth = children.stream() - .mapToInt(e -> e.depth) - .max().orElse(0) + 1; - width = children.stream() - .mapToInt(e -> e.width) - .sum() + (children.isEmpty() ? 1 : 0); + int maxChildDepth = 0; + int sumChildWidth = 0; + for (int i = 0; i < children.size(); ++i) { + Expression child = children.get(i); + maxChildDepth = Math.max(child.depth, maxChildDepth); + sumChildWidth += child.width; + } + this.depth = maxChildDepth + 1; + this.width = sumChildWidth + ((children.isEmpty()) ? 1 : 0); + checkLimit(); this.inferred = inferred; } @@ -130,8 +128,8 @@ public abstract class Expression extends AbstractTreeNode implements */ public TypeCheckResult checkInputDataTypes() { // check all of its children recursively. - for (Expression expression : this.children) { - TypeCheckResult childResult = expression.checkInputDataTypes(); + for (Expression child : this.children) { + TypeCheckResult childResult = child.checkInputDataTypes(); if (childResult.failed()) { return childResult; } @@ -180,21 +178,20 @@ public abstract class Expression extends AbstractTreeNode implements } private boolean checkPrimitiveInputDataTypesWithExpectType(DataType input, DataType expected) { - // These type will throw exception when invoke toCatalogDataType() - if (expected instanceof AnyDataType) { - return expected.acceptsType(input); + // support fast check the case: input=TinyIntType, expected=NumericType, for example: `1 + 1`. + // if no this check, there will have an exception when invoke NumericType.toCatalogDataType, + // when there has lots of expression, the exception become the bottleneck, because an exception + // need to record the whole StackFrame. + if (expected.acceptsType(input)) { + return true; } + // TODO: complete the cast logic like FunctionCallExpr.analyzeImpl - boolean legacyCastCompatible = false; try { - legacyCastCompatible = input.toCatalogDataType().matchesType(expected.toCatalogDataType()); + return input.toCatalogDataType().matchesType(expected.toCatalogDataType()); } catch (Throwable t) { - // ignore. - } - if (!legacyCastCompatible && !expected.acceptsType(input)) { return false; } - return true; } private TypeCheckResult checkInputDataTypesWithExpectTypes( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 6917a5648f..3be5a56447 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; @@ -101,7 +102,8 @@ public class SlotReference extends Slot { this.exprId = exprId; this.name = name; this.dataType = dataType; - this.qualifier = ImmutableList.copyOf(Objects.requireNonNull(qualifier, "qualifier can not be null")); + this.qualifier = Utils.fastToImmutableList( + Objects.requireNonNull(qualifier, "qualifier can not be null")); this.nullable = nullable; this.table = table; this.column = column; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java index 075d49763a..6d0a5d85de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java @@ -38,9 +38,9 @@ import org.apache.doris.nereids.util.ResponsibilityChain; import org.apache.doris.nereids.util.TypeCoercionUtils; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import java.math.BigDecimal; import java.util.List; @@ -233,7 +233,7 @@ public class ComputeSignatureHelper { public static FunctionSignature implementAnyDataTypeWithOutIndex( FunctionSignature signature, List arguments) { // collect all any data type with index - List newArgTypes = Lists.newArrayList(); + List newArgTypes = Lists.newArrayListWithCapacity(arguments.size()); for (int i = 0; i < arguments.size(); i++) { DataType sigType; if (i >= signature.argumentsTypes.size()) { @@ -267,10 +267,19 @@ public class ComputeSignatureHelper { collectAnyDataType(sigType, expressionType, indexToArgumentTypes); } // if all any data type's expression is NULL, we should use follow to any data type to do type coercion - Set allNullTypeIndex = indexToArgumentTypes.entrySet().stream() - .filter(entry -> entry.getValue().stream().allMatch(NullType.class::isInstance)) - .map(Entry::getKey) - .collect(ImmutableSet.toImmutableSet()); + Set allNullTypeIndex = Sets.newHashSetWithExpectedSize(indexToArgumentTypes.size()); + for (Entry> entry : indexToArgumentTypes.entrySet()) { + boolean allIsNullType = true; + for (DataType dataType : entry.getValue()) { + if (!(dataType instanceof NullType)) { + allIsNullType = false; + break; + } + } + if (allIsNullType) { + allNullTypeIndex.add(entry.getKey()); + } + } if (!allNullTypeIndex.isEmpty()) { for (int i = 0; i < arguments.size(); i++) { DataType sigType; @@ -297,7 +306,7 @@ public class ComputeSignatureHelper { } // replace any data type and follow to any data type with real data type - List newArgTypes = Lists.newArrayList(); + List newArgTypes = Lists.newArrayListWithCapacity(signature.argumentsTypes.size()); for (DataType sigType : signature.argumentsTypes) { newArgTypes.add(replaceAnyDataType(sigType, indexToCommonTypes)); } @@ -324,10 +333,18 @@ public class ComputeSignatureHelper { if (computeSignature instanceof ComputePrecision) { return ((ComputePrecision) computeSignature).computePrecision(signature); } - if (signature.argumentsTypes.stream().anyMatch(TypeCoercionUtils::hasDateTimeV2Type)) { + + boolean hasDateTimeV2Type = false; + boolean hasDecimalV3Type = false; + for (DataType argumentsType : signature.argumentsTypes) { + hasDateTimeV2Type |= TypeCoercionUtils.hasDateTimeV2Type(argumentsType); + hasDecimalV3Type |= TypeCoercionUtils.hasDecimalV3Type(argumentsType); + } + + if (hasDateTimeV2Type) { signature = defaultDateTimeV2PrecisionPromotion(signature, arguments); } - if (signature.argumentsTypes.stream().anyMatch(TypeCoercionUtils::hasDecimalV3Type)) { + if (hasDecimalV3Type) { // do decimal v3 precision signature = defaultDecimalV3PrecisionPromotion(signature, arguments); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java index 054e2f947e..a8a831ae1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExplicitlyCastableSignature.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.analyzer.ComplexDataType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.coercion.AnyDataType; @@ -49,6 +50,9 @@ public interface ExplicitlyCastableSignature extends ComputeSignature { if (realType instanceof NullType) { return true; } + if (signatureType instanceof ComplexDataType && !(realType instanceof ComplexDataType)) { + return false; + } try { // TODO: copy canCastTo method to DataType return Type.canCastTo(realType.toCatalogDataType(), signatureType.toCatalogDataType()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/IdenticalSignature.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/IdenticalSignature.java index 410074124e..a81065e95f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/IdenticalSignature.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/IdenticalSignature.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions.functions; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.analyzer.ComplexDataType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.types.coercion.FollowToAnyDataType; @@ -45,6 +46,9 @@ public interface IdenticalSignature extends ComputeSignature { if (signatureType instanceof AnyDataType || signatureType instanceof FollowToAnyDataType) { return false; } + if (signatureType instanceof ComplexDataType && !(realType instanceof ComplexDataType)) { + return false; + } return realType.toCatalogDataType().matchesType(signatureType.toCatalogDataType()); } catch (Throwable t) { // the signatureType maybe DataType and can not cast to catalog data type. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ImplicitlyCastableSignature.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ImplicitlyCastableSignature.java index eb3c4c03f1..c189dd9bf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ImplicitlyCastableSignature.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ImplicitlyCastableSignature.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions; import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.analyzer.ComplexDataType; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.NullType; @@ -51,6 +52,9 @@ public interface ImplicitlyCastableSignature extends ComputeSignature { if (realType instanceof NullType) { return true; } + if (signatureType instanceof ComplexDataType && !(realType instanceof ComplexDataType)) { + return false; + } try { // TODO: copy isImplicitlyCastable method to DataType // TODO: resolve AnyDataType invoke toCatalogDataType @@ -68,8 +72,10 @@ public interface ImplicitlyCastableSignature extends ComputeSignature { } try { List allPromotions = realType.getAllPromotions(); - if (allPromotions.stream().anyMatch(promotion -> isImplicitlyCastable(signatureType, promotion))) { - return true; + for (DataType promotion : allPromotions) { + if (isImplicitlyCastable(signatureType, promotion)) { + return true; + } } } catch (Throwable t) { // the signatureType maybe DataType and can not cast to catalog data type. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/NullOrIdenticalSignature.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/NullOrIdenticalSignature.java index 37530bc8b1..8cc4eef249 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/NullOrIdenticalSignature.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/NullOrIdenticalSignature.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions.functions; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.analyzer.ComplexDataType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.coercion.AnyDataType; @@ -48,6 +49,9 @@ public interface NullOrIdenticalSignature extends ComputeSignature { if (signatureType instanceof AnyDataType) { return false; } + if (signatureType instanceof ComplexDataType && !(realType instanceof ComplexDataType)) { + return false; + } return realType.toCatalogDataType().matchesType(signatureType.toCatalogDataType()); } catch (Throwable t) { // the signatureType maybe DataType and can not cast to catalog data type. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/DefaultExpressionRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/DefaultExpressionRewriter.java index f6e3fc6464..2248666dbc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/DefaultExpressionRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/DefaultExpressionRewriter.java @@ -19,8 +19,8 @@ package org.apache.doris.nereids.trees.expressions.visitor; import org.apache.doris.nereids.trees.expressions.Expression; -import java.util.ArrayList; -import java.util.List; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; /** * Default implementation for expression rewriting, delegating to child expressions and rewrite current root @@ -30,20 +30,42 @@ public abstract class DefaultExpressionRewriter extends ExpressionVisitor Expression rewrite(ExpressionVisitor rewriter, Expression expr, C context) { - List newChildren = new ArrayList<>(expr.arity()); - boolean hasNewChildren = false; - for (Expression child : expr.children()) { - Expression newChild = child.accept(rewriter, context); - if (newChild != child) { - hasNewChildren = true; + /** rewriteChildren */ + public static final Expression rewriteChildren( + ExpressionVisitor rewriter, Expression expr, C context) { + switch (expr.arity()) { + case 1: { + Expression originChild = expr.child(0); + Expression newChild = originChild.accept(rewriter, context); + return (originChild != newChild) ? expr.withChildren(ImmutableList.of(newChild)) : expr; + } + case 2: { + Expression originLeft = expr.child(0); + Expression newLeft = originLeft.accept(rewriter, context); + Expression originRight = expr.child(1); + Expression newRight = originRight.accept(rewriter, context); + return (originLeft != newLeft || originRight != newRight) + ? expr.withChildren(ImmutableList.of(newLeft, newRight)) + : expr; + } + case 0: { + return expr; + } + default: { + boolean hasNewChildren = false; + Builder newChildren = ImmutableList.builderWithExpectedSize(expr.arity()); + for (Expression child : expr.children()) { + Expression newChild = child.accept(rewriter, context); + if (newChild != child) { + hasNewChildren = true; + } + newChildren.add(newChild); + } + return hasNewChildren ? expr.withChildren(newChildren.build()) : expr; } - newChildren.add(newChild); } - return hasNewChildren ? expr.withChildren(newChildren) : expr; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index 0c38bd9a06..0fb3964e51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -63,9 +63,9 @@ public class LogicalGenerate extends LogicalUnary groupExpression, Optional logicalProperties, CHILD_TYPE child) { super(PlanType.LOGICAL_GENERATE, groupExpression, logicalProperties, child); - this.generators = ImmutableList.copyOf(generators); - this.generatorOutput = ImmutableList.copyOf(generatorOutput); - this.expandColumnAlias = ImmutableList.copyOf(expandColumnAlias); + this.generators = Utils.fastToImmutableList(generators); + this.generatorOutput = Utils.fastToImmutableList(generatorOutput); + this.expandColumnAlias = Utils.fastToImmutableList(expandColumnAlias); } public List getGenerators() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index c8db68081a..c440670e21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -158,9 +158,9 @@ public class LogicalJoin { - if (cacheSlotWithSlotName.containsKey(Pair.of(selectedIndexId, col.getName()))) { - return cacheSlotWithSlotName.get(Pair.of(selectedIndexId, col.getName())); + List baseSchema = table.getBaseSchema(true); + Builder slots = ImmutableList.builder(); + for (Column col : baseSchema) { + Pair key = Pair.of(selectedIndexId, col.getName()); + Slot slot = cacheSlotWithSlotName.get(key); + if (slot != null) { + slots.add(slot); + } else { + slot = SlotReference.fromColumn(table, col, qualified(), this); + cacheSlotWithSlotName.put(key, slot); + slots.add(slot); } - Slot slot = SlotReference.fromColumn(table, col, qualified(), this); - cacheSlotWithSlotName.put(Pair.of(selectedIndexId, col.getName()), slot); - return slot; - }).collect(ImmutableList.toImmutableList()); + } + return slots.build(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index f3a4d050b8..d899d228fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import org.json.JSONObject; @@ -94,7 +95,7 @@ public class LogicalProject extends LogicalUnary extends LogicalUnary computeOutput() { - return projects.stream() - .map(NamedExpression::toSlot) - .collect(ImmutableList.toImmutableList()); + Builder slots = ImmutableList.builderWithExpectedSize(projects.size()); + for (NamedExpression project : projects) { + slots.add(project.toSlot()); + } + return slots.build(); } @Override @@ -170,7 +173,7 @@ public class LogicalProject extends LogicalUnary withChildren(List children) { Preconditions.checkArgument(children.size() == 1); - return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, ImmutableList.copyOf(children)); + return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, Utils.fastToImmutableList(children)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index 17297ee8ce..2e4ddb55ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -41,9 +41,7 @@ import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableList.Builder; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -115,8 +113,9 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements * Generate new output for SetOperation. */ public List buildNewOutputs() { - ImmutableList.Builder newOutputs = new Builder<>(); - for (Slot slot : resetNullableForLeftOutputs()) { + List slots = resetNullableForLeftOutputs(); + ImmutableList.Builder newOutputs = ImmutableList.builderWithExpectedSize(slots.size()); + for (Slot slot : slots) { newOutputs.add(new SlotReference(slot.toSql(), slot.getDataType(), slot.nullable())); } return newOutputs.build(); @@ -124,22 +123,28 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements // If the right child is nullable, need to ensure that the left child is also nullable private List resetNullableForLeftOutputs() { - List resetNullableForLeftOutputs = new ArrayList<>(); - for (int i = 0; i < child(1).getOutput().size(); ++i) { + int rightChildOutputSize = child(1).getOutput().size(); + ImmutableList.Builder resetNullableForLeftOutputs + = ImmutableList.builderWithExpectedSize(rightChildOutputSize); + for (int i = 0; i < rightChildOutputSize; ++i) { if (child(1).getOutput().get(i).nullable() && !child(0).getOutput().get(i).nullable()) { resetNullableForLeftOutputs.add(child(0).getOutput().get(i).withNullable(true)); } else { resetNullableForLeftOutputs.add(child(0).getOutput().get(i)); } } - return ImmutableList.copyOf(resetNullableForLeftOutputs); + return resetNullableForLeftOutputs.build(); } private List> castCommonDataTypeOutputs() { - List newLeftOutputs = new ArrayList<>(); - List newRightOutputs = new ArrayList<>(); + int childOutputSize = child(0).getOutput().size(); + ImmutableList.Builder newLeftOutputs = ImmutableList.builderWithExpectedSize( + childOutputSize); + ImmutableList.Builder newRightOutputs = ImmutableList.builderWithExpectedSize( + childOutputSize + ); // Ensure that the output types of the left and right children are consistent and expand upward. - for (int i = 0; i < child(0).getOutput().size(); ++i) { + for (int i = 0; i < childOutputSize; ++i) { Slot left = child(0).getOutput().get(i); Slot right = child(1).getOutput().get(i); DataType compatibleType = getAssignmentCompatibleType(left.getDataType(), right.getDataType()); @@ -155,10 +160,7 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements newRightOutputs.add((NamedExpression) newRight); } - List> resultExpressions = new ArrayList<>(); - resultExpressions.add(newLeftOutputs); - resultExpressions.add(newRightOutputs); - return ImmutableList.copyOf(resultExpressions); + return ImmutableList.of(newLeftOutputs.build(), newRightOutputs.build()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index 9b35f36465..3a88020ac9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -70,7 +70,7 @@ public class LogicalUnion extends LogicalSetOperation implements Union, OutputPr List> constantExprsList, boolean hasPushedFilter, List children) { super(PlanType.LOGICAL_UNION, qualifier, outputs, childrenOutputs, children); this.hasPushedFilter = hasPushedFilter; - this.constantExprsList = ImmutableList.copyOf( + this.constantExprsList = Utils.fastToImmutableList( Objects.requireNonNull(constantExprsList, "constantExprsList should not be null")); } @@ -81,7 +81,7 @@ public class LogicalUnion extends LogicalSetOperation implements Union, OutputPr super(PlanType.LOGICAL_UNION, qualifier, outputs, childrenOutputs, groupExpression, logicalProperties, children); this.hasPushedFilter = hasPushedFilter; - this.constantExprsList = ImmutableList.copyOf( + this.constantExprsList = Utils.fastToImmutableList( Objects.requireNonNull(constantExprsList, "constantExprsList should not be null")); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/InferPlanOutputAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/InferPlanOutputAlias.java index 48248e1c58..88f3027f7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/InferPlanOutputAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/InferPlanOutputAlias.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.plans.visitor; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; @@ -28,49 +29,64 @@ import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; -import java.util.stream.Collectors; +import java.util.Set; /** * Infer output column name when it refers an expression and not has an alias manually. */ -public class InferPlanOutputAlias extends DefaultPlanVisitor> { +public class InferPlanOutputAlias { private final List currentOutputs; private final List finalOutputs; + private final Set shouldProcessOutputIndex; + /** InferPlanOutputAlias */ public InferPlanOutputAlias(List currentOutputs) { this.currentOutputs = currentOutputs; this.finalOutputs = new ArrayList<>(currentOutputs); - } - - @Override - public Void visit(Plan plan, ImmutableMultimap currentExprIdAndIndexMap) { - - List aliasProjects = plan.getExpressions().stream() - .filter(expression -> expression instanceof Alias) - .map(Alias.class::cast) - .collect(Collectors.toList()); - - ImmutableSet currentOutputExprIdSet = currentExprIdAndIndexMap.keySet(); - for (Alias projectItem : aliasProjects) { - ExprId exprId = projectItem.getExprId(); - // Infer name when alias child is expression and alias's name is from child - if (currentOutputExprIdSet.contains(projectItem.getExprId()) - && projectItem.isNameFromChild()) { - String inferredAliasName = projectItem.child().getExpressionName(); - ImmutableCollection outPutExprIndexes = currentExprIdAndIndexMap.get(exprId); - // replace output name by inferred name - outPutExprIndexes.forEach(index -> { - Slot slot = currentOutputs.get(index); - finalOutputs.set(index, slot.withName("__" + inferredAliasName + "_" + index)); - }); - } + this.shouldProcessOutputIndex = new HashSet<>(); + for (int i = 0; i < currentOutputs.size(); i++) { + shouldProcessOutputIndex.add(i); } - return super.visit(plan, currentExprIdAndIndexMap); } - public List getOutputs() { + /** infer */ + public List infer(Plan plan, ImmutableMultimap currentExprIdAndIndexMap) { + ImmutableSet currentOutputExprIdSet = currentExprIdAndIndexMap.keySet(); + // Breath First Search + plan.foreachBreath(childPlan -> { + if (shouldProcessOutputIndex.isEmpty()) { + return true; + } + for (Expression expression : ((Plan) childPlan).getExpressions()) { + if (!(expression instanceof Alias)) { + continue; + } + Alias projectItem = (Alias) expression; + ExprId exprId = projectItem.getExprId(); + // Infer name when alias child is expression and alias's name is from child + if (currentOutputExprIdSet.contains(projectItem.getExprId()) + && projectItem.isNameFromChild()) { + String inferredAliasName = projectItem.child().getExpressionName(); + ImmutableCollection outputExprIndexes = currentExprIdAndIndexMap.get(exprId); + // replace output name by inferred name + for (Integer index : outputExprIndexes) { + Slot slot = currentOutputs.get(index); + finalOutputs.set(index, slot.withName("__" + inferredAliasName + "_" + index)); + shouldProcessOutputIndex.remove(index); + + if (shouldProcessOutputIndex.isEmpty()) { + // replace finished + return true; + } + } + } + } + // continue replace + return false; + }); return finalOutputs; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java index dd97e36da3..4acdb50013 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java @@ -18,13 +18,14 @@ package org.apache.doris.nereids.types; import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.analyzer.ComplexDataType; import java.util.Objects; /** * Array type in Nereids. */ -public class ArrayType extends DataType { +public class ArrayType extends DataType implements ComplexDataType { public static final ArrayType SYSTEM_DEFAULT = new ArrayType(NullType.INSTANCE, true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java index 2f06c1d7f7..2fe81f048e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.types; import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.analyzer.ComplexDataType; import org.apache.doris.nereids.annotation.Developing; import java.util.Objects; @@ -26,7 +27,7 @@ import java.util.Objects; * Struct type in Nereids. */ @Developing -public class MapType extends DataType { +public class MapType extends DataType implements ComplexDataType { public static final MapType SYSTEM_DEFAULT = new MapType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java index ef7dd06165..b9ad199970 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.types; import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.analyzer.ComplexDataType; import org.apache.doris.nereids.annotation.Developing; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -36,7 +37,7 @@ import java.util.stream.Collectors; * Struct type in Nereids. */ @Developing -public class StructType extends DataType { +public class StructType extends DataType implements ComplexDataType { public static final StructType SYSTEM_DEFAULT = new StructType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 00a6ff2ec0..f35ae1a75d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -59,6 +59,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -67,6 +68,7 @@ import com.google.common.collect.Sets; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -188,9 +190,9 @@ public class ExpressionUtils { */ public static Expression combine(Class type, Collection expressions) { /* - * (AB) (CD) E ((AB)(CD)) E (((AB)(CD))E) - * ▲ ▲ ▲ ▲ ▲ ▲ - * │ │ │ │ │ │ + * (AB) (CD) E ((AB)(CD)) E (((AB)(CD))E) + * ▲ ▲ ▲ ▲ ▲ ▲ + * │ │ │ │ │ │ * A B C D E ──► A B C D E ──► (AB) (CD) E ──► ((AB)(CD)) E ──► (((AB)(CD))E) */ Preconditions.checkArgument(type == And.class || type == Or.class); @@ -223,8 +225,7 @@ public class ExpressionUtils { } /** - * Replace the slot in expressions with the lineage identifier from - * specifiedbaseTable sets or target table types + * Replace the slot in expressions with the lineage identifier from specifiedbaseTable sets or target table types * example as following: * select a + 10 as a1, d from ( * select b - 5 as a, d from table @@ -241,9 +242,9 @@ public class ExpressionUtils { } ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = new ExpressionLineageReplacer.ExpressionReplaceContext( - expressions.stream().map(Expression.class::cast).collect(Collectors.toList()), - targetTypes, - tableIdentifiers); + expressions.stream().map(Expression.class::cast).collect(Collectors.toList()), + targetTypes, + tableIdentifiers); plan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map @@ -277,10 +278,8 @@ public class ExpressionUtils { } /** - * Check whether the input expression is a - * {@link org.apache.doris.nereids.trees.expressions.Slot} - * or at least one {@link Cast} on a - * {@link org.apache.doris.nereids.trees.expressions.Slot} + * Check whether the input expression is a {@link org.apache.doris.nereids.trees.expressions.Slot} + * or at least one {@link Cast} on a {@link org.apache.doris.nereids.trees.expressions.Slot} *

* for example: * - SlotReference to a column: @@ -290,8 +289,7 @@ public class ExpressionUtils { * cast(cast(int_col as long) as string) * * @param expr input expression - * @return Return Optional[ExprId] of underlying slot reference if input - * expression is a slot or cast on slot. + * @return Return Optional[ExprId] of underlying slot reference if input expression is a slot or cast on slot. * Otherwise, return empty optional result. */ public static Optional isSlotOrCastOnSlot(Expression expr) { @@ -299,10 +297,8 @@ public class ExpressionUtils { } /** - * Check whether the input expression is a - * {@link org.apache.doris.nereids.trees.expressions.Slot} - * or at least one {@link Cast} on a - * {@link org.apache.doris.nereids.trees.expressions.Slot} + * Check whether the input expression is a {@link org.apache.doris.nereids.trees.expressions.Slot} + * or at least one {@link Cast} on a {@link org.apache.doris.nereids.trees.expressions.Slot} */ public static Optional extractSlotOrCastOnSlot(Expression expr) { while (expr instanceof Cast) { @@ -317,23 +313,35 @@ public class ExpressionUtils { } /** - * Generate replaceMap Slot -> Expression from NamedExpression[Expression as - * name] + * Generate replaceMap Slot -> Expression from NamedExpression[Expression as name] */ public static Map generateReplaceMap(List namedExpressions) { - return namedExpressions - .stream() - .filter(Alias.class::isInstance) - .collect( - Collectors.toMap( - NamedExpression::toSlot, - // Avoid cast to alias, retrieving the first child expression. - alias -> alias.child(0))); + ImmutableMap.Builder replaceMap = ImmutableMap.builderWithExpectedSize( + namedExpressions.size() * 2); + for (NamedExpression namedExpression : namedExpressions) { + if (namedExpression instanceof Alias) { + // Avoid cast to alias, retrieving the first child expression. + replaceMap.put(namedExpression.toSlot(), namedExpression.child(0)); + } + } + return replaceMap.build(); } /** - * Replace expression node in the expression tree by `replaceMap` in top-down - * manner. + * replace NameExpression. + */ + public static NamedExpression replaceNameExpression(NamedExpression expr, + Map replaceMap) { + Expression newExpr = replace(expr, replaceMap); + if (newExpr instanceof NamedExpression) { + return (NamedExpression) newExpr; + } else { + return new Alias(expr.getExprId(), newExpr, expr.getName()); + } + } + + /** + * Replace expression node in the expression tree by `replaceMap` in top-down manner. * For example. * 

      * input expression: a > 1
@@ -344,20 +352,10 @@ public class ExpressionUtils {
      *
*/ public static Expression replace(Expression expr, Map replaceMap) { - return expr.accept(ExpressionReplacer.INSTANCE, replaceMap); - } - - /** - * replace NameExpression. - */ - public static NamedExpression replace(NamedExpression expr, - Map replaceMap) { - Expression newExpr = expr.accept(ExpressionReplacer.INSTANCE, replaceMap); - if (newExpr instanceof NamedExpression) { - return (NamedExpression) newExpr; - } else { - return new Alias(expr.getExprId(), newExpr, expr.getName()); - } + return expr.rewriteDownShortCircuit(e -> { + Expression replacedExpr = replaceMap.get(e); + return replacedExpr == null ? e : replacedExpr; + }); } public static List replace(List exprs, @@ -375,21 +373,20 @@ public class ExpressionUtils { } /** - * Replace expression node in the expression tree by `replaceMap` in top-down - * manner. + * Replace expression node in the expression tree by `replaceMap` in top-down manner. */ public static List replaceNamedExpressions(List namedExpressions, Map replaceMap) { - return namedExpressions.stream() - .map(namedExpression -> { - NamedExpression newExpr = replace(namedExpression, replaceMap); - if (newExpr.getExprId().equals(namedExpression.getExprId())) { - return newExpr; - } else { - return new Alias(namedExpression.getExprId(), newExpr, namedExpression.getName()); - } - }) - .collect(ImmutableList.toImmutableList()); + Builder replaceExprs = ImmutableList.builderWithExpectedSize(namedExpressions.size()); + for (NamedExpression namedExpression : namedExpressions) { + NamedExpression newExpr = replaceNameExpression(namedExpression, replaceMap); + if (newExpr.getExprId().equals(namedExpression.getExprId())) { + replaceExprs.add(newExpr); + } else { + replaceExprs.add(new Alias(namedExpression.getExprId(), newExpr, namedExpression.getName())); + } + } + return replaceExprs.build(); } public static List rewriteDownShortCircuit( @@ -489,19 +486,19 @@ public class ExpressionUtils { public static boolean canInferNotNullForMarkSlot(Expression predicate) { /* * assume predicate is from LogicalFilter - * the idea is replacing each mark join slot with null and false literal then - * run FoldConstant rule + * the idea is replacing each mark join slot with null and false literal then run FoldConstant rule * if the evaluate result are: * 1. all true - * 2. all null and false (in logicalFilter, we discard both null and false - * values) + * 2. all null and false (in logicalFilter, we discard both null and false values) * the mark slot can be non-nullable boolean * and in semi join, we can safely change the mark conjunct to hash conjunct */ - ImmutableList literals = ImmutableList.of(new NullLiteral(BooleanType.INSTANCE), BooleanLiteral.FALSE); - List markJoinSlotReferenceList = ((Set) predicate - .collect(MarkJoinSlotReference.class::isInstance)).stream() - .collect(Collectors.toList()); + ImmutableList literals = + ImmutableList.of(new NullLiteral(BooleanType.INSTANCE), BooleanLiteral.FALSE); + List markJoinSlotReferenceList = + ((Set) predicate + .collect(MarkJoinSlotReference.class::isInstance)).stream() + .collect(Collectors.toList()); int markSlotSize = markJoinSlotReferenceList.size(); int maxMarkSlotCount = 4; // if the conjunct has mark slot, and maximum 4 mark slots(for performance) @@ -510,9 +507,9 @@ public class ExpressionUtils { boolean meetTrue = false; boolean meetNullOrFalse = false; /* - * markSlotSize = 1 -> loopCount = 2 ---- 0, 1 - * markSlotSize = 2 -> loopCount = 4 ---- 00, 01, 10, 11 - * markSlotSize = 3 -> loopCount = 8 ---- 000, 001, 010, 011, 100, 101, 110, 111 + * markSlotSize = 1 -> loopCount = 2 ---- 0, 1 + * markSlotSize = 2 -> loopCount = 4 ---- 00, 01, 10, 11 + * markSlotSize = 3 -> loopCount = 8 ---- 000, 001, 010, 011, 100, 101, 110, 111 * markSlotSize = 4 -> loopCount = 16 ---- 0000, 0001, ... 1111 */ int loopCount = 2 << markSlotSize; @@ -583,8 +580,7 @@ public class ExpressionUtils { } /** - * infer notNulls slot from predicate but these slots must be in the given - * slots. + * infer notNulls slot from predicate but these slots must be in the given slots. */ public static Set inferNotNull(Set predicates, Set slots, CascadesContext cascadesContext) { @@ -614,7 +610,7 @@ public class ExpressionUtils { return anyMatch(expressions, type::isInstance); } - public static Set collect(List expressions, + public static Set collect(Collection expressions, Predicate> predicate) { return expressions.stream() .flatMap(expr -> expr.>collect(predicate).stream()) @@ -654,7 +650,7 @@ public class ExpressionUtils { .collect(Collectors.toSet()); } - public static List collectAll(List expressions, + public static List collectAll(Collection expressions, Predicate> predicate) { return expressions.stream() .flatMap(expr -> expr.>collect(predicate).stream()) @@ -764,18 +760,18 @@ public class ExpressionUtils { */ public static boolean checkSlotConstant(Slot slot, Set predicates) { return predicates.stream().anyMatch(predicate -> { - if (predicate instanceof EqualTo) { - EqualTo equalTo = (EqualTo) predicate; - return (equalTo.left() instanceof Literal && equalTo.right().equals(slot)) - || (equalTo.right() instanceof Literal && equalTo.left().equals(slot)); - } - return false; - }); + if (predicate instanceof EqualTo) { + EqualTo equalTo = (EqualTo) predicate; + return (equalTo.left() instanceof Literal && equalTo.right().equals(slot)) + || (equalTo.right() instanceof Literal && equalTo.left().equals(slot)); + } + return false; + } + ); } /** - * Check the expression is inferred or not, if inferred return true, nor return - * false + * Check the expression is inferred or not, if inferred return true, nor return false */ public static boolean isInferred(Expression expression) { return expression.accept(new DefaultExpressionVisitor() { @@ -794,4 +790,17 @@ public class ExpressionUtils { } }, null); } + + /** distinctSlotByName */ + public static List distinctSlotByName(List slots) { + Set existSlotNames = new HashSet<>(slots.size() * 2); + Builder distinctSlots = ImmutableList.builderWithExpectedSize(slots.size()); + for (Slot slot : slots) { + String name = slot.getName(); + if (existSlotNames.add(name)) { + distinctSlots.add(slot); + } + } + return distinctSlots.build(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 07274cc27c..a73c183e4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -45,6 +45,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -290,8 +291,11 @@ public class JoinUtils { } private static List applyNullable(List slots, boolean nullable) { - return slots.stream().map(o -> o.withNullable(nullable)) - .collect(ImmutableList.toImmutableList()); + Builder newSlots = ImmutableList.builderWithExpectedSize(slots.size()); + for (Slot slot : slots) { + newSlots.add(slot.withNullable(nullable)); + } + return newSlots.build(); } private static Map mapPrimaryToForeign(ImmutableEqualSet equivalenceSet, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java index c2ac7e4314..944148cd34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -125,6 +126,23 @@ public class PlanUtils { return resultSet; } + /** fastGetChildrenOutput */ + public static List fastGetChildrenOutputs(List children) { + int outputNum = 0; + // child.output is cached by AbstractPlan.logicalProperties, + // we can compute output num without the overhead of re-compute output + for (Plan child : children) { + List output = child.getOutput(); + outputNum += output.size(); + } + // generate output list only copy once and without resize the list + Builder output = ImmutableList.builderWithExpectedSize(outputNum); + for (Plan child : children) { + output.addAll(child.getOutput()); + } + return output.build(); + } + /** * collect non_window_agg_func */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 34d23aecf5..a953122d46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; +import org.apache.doris.nereids.analyzer.ComplexDataType; import org.apache.doris.nereids.annotation.Developing; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Add; @@ -119,9 +120,7 @@ import java.math.BigInteger; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Utils for type coercion. @@ -595,7 +594,7 @@ public class TypeCoercionUtils { private static List> getInputImplicitCastTypes( List inputs, List expectedTypes) { - Builder> implicitCastTypes = ImmutableList.builder(); + Builder> implicitCastTypes = ImmutableList.builderWithExpectedSize(inputs.size()); for (int i = 0; i < inputs.size(); i++) { DataType argType = inputs.get(i).getDataType(); DataType expectedType = expectedTypes.get(i); @@ -765,15 +764,16 @@ public class TypeCoercionUtils { binaryArithmetic = TypeCoercionUtils.processCharacterLiteralInBinaryOperator(binaryArithmetic, left, right); // check string literal can cast to double - binaryArithmetic.children().stream().filter(e -> e instanceof StringLikeLiteral) - .forEach(expr -> { - try { - new BigDecimal(((StringLikeLiteral) expr).getStringValue()); - } catch (NumberFormatException e) { - throw new IllegalStateException(String.format( - "string literal %s cannot be cast to double", expr.toSql())); - } - }); + for (Expression expr : binaryArithmetic.children()) { + if (expr instanceof StringLikeLiteral) { + try { + new BigDecimal(((StringLikeLiteral) expr).getStringValue()); + } catch (NumberFormatException e) { + throw new IllegalStateException(String.format( + "string literal %s cannot be cast to double", expr.toSql())); + } + } + } // 1. choose default numeric type for left and right DataType t1 = TypeCoercionUtils.getNumResultType(left.getDataType()); @@ -1106,14 +1106,13 @@ public class TypeCoercionUtils { private static Optional findWiderTypeForTwoForComparison( DataType left, DataType right, boolean intStringToString) { // TODO: need to rethink how to handle char and varchar to return char or varchar as much as possible. - return Stream - .>>of( - () -> findCommonComplexTypeForComparison(left, right, intStringToString), - () -> findCommonPrimitiveTypeForComparison(left, right, intStringToString)) - .map(Supplier::get) - .filter(Optional::isPresent) - .map(Optional::get) - .findFirst(); + if (left instanceof ComplexDataType) { + Optional commonType = findCommonComplexTypeForComparison(left, right, intStringToString); + if (commonType.isPresent()) { + return commonType; + } + } + return findCommonPrimitiveTypeForComparison(left, right, intStringToString); } /** @@ -1310,20 +1309,22 @@ public class TypeCoercionUtils { Map> partitioned = dataTypes.stream() .collect(Collectors.partitioningBy(TypeCoercionUtils::hasCharacterType)); List needTypeCoercion = Lists.newArrayList(Sets.newHashSet(partitioned.get(true))); - if (needTypeCoercion.size() > 1 || !partitioned.get(false).isEmpty()) { - needTypeCoercion = needTypeCoercion.stream() - .map(TypeCoercionUtils::replaceCharacterToString) - .collect(Collectors.toList()); + List nonCharTypes = partitioned.get(false); + if (needTypeCoercion.size() > 1 || !nonCharTypes.isEmpty()) { + needTypeCoercion = Utils.fastMapList( + needTypeCoercion, nonCharTypes.size(), TypeCoercionUtils::replaceCharacterToString); } - needTypeCoercion.addAll(partitioned.get(false)); - return needTypeCoercion.stream().map(Optional::of).reduce(Optional.of(NullType.INSTANCE), - (r, c) -> { - if (r.isPresent() && c.isPresent()) { - return findWiderTypeForTwoForCaseWhen(r.get(), c.get()); - } else { - return Optional.empty(); - } - }); + needTypeCoercion.addAll(nonCharTypes); + + DataType commonType = NullType.INSTANCE; + for (DataType dataType : needTypeCoercion) { + Optional newCommonType = findWiderTypeForTwoForCaseWhen(commonType, dataType); + if (!newCommonType.isPresent()) { + return Optional.empty(); + } + commonType = newCommonType.get(); + } + return Optional.of(commonType); } /** @@ -1332,14 +1333,11 @@ public class TypeCoercionUtils { @Developing private static Optional findWiderTypeForTwoForCaseWhen(DataType left, DataType right) { // TODO: need to rethink how to handle char and varchar to return char or varchar as much as possible. - return Stream - .>>of( - () -> findCommonComplexTypeForCaseWhen(left, right), - () -> findCommonPrimitiveTypeForCaseWhen(left, right)) - .map(Supplier::get) - .filter(Optional::isPresent) - .map(Optional::get) - .findFirst(); + Optional commonType = findCommonComplexTypeForCaseWhen(left, right); + if (commonType.isPresent()) { + return commonType; + } + return findCommonPrimitiveTypeForCaseWhen(left, right); } /** @@ -1585,7 +1583,7 @@ public class TypeCoercionUtils { */ public static BoundFunction fillJsonValueModifyTypeArgument(BoundFunction function) { List arguments = function.getArguments(); - List newArguments = Lists.newArrayList(); + List newArguments = Lists.newArrayListWithCapacity(arguments.size() + 1); StringBuilder jsonTypeStr = new StringBuilder(); for (int i = 0; i < arguments.size(); i++) { Expression argument = arguments.get(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index 08a606cd64..df9528cc49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import com.google.common.base.CaseFormat; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; @@ -34,6 +35,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -294,4 +297,70 @@ public class Utils { } return false; } + + public static List fastMapList(List list, int additionSize, Function transformer) { + List newList = Lists.newArrayListWithCapacity(list.size() + additionSize); + for (I input : list) { + newList.add(transformer.apply(input)); + } + return newList; + } + + /** fastToImmutableList */ + public static ImmutableList fastToImmutableList(E[] array) { + switch (array.length) { + case 0: + return ImmutableList.of(); + case 1: + return ImmutableList.of(array[0]); + default: + // NOTE: ImmutableList.copyOf(array) has additional clone of the array, so here we + // direct generate a ImmutableList + Builder copyChildren = ImmutableList.builderWithExpectedSize(array.length); + for (E child : array) { + copyChildren.add(child); + } + return copyChildren.build(); + } + } + + /** fastToImmutableList */ + public static ImmutableList fastToImmutableList(List originList) { + if (originList instanceof ImmutableList) { + return (ImmutableList) originList; + } + + switch (originList.size()) { + case 0: return ImmutableList.of(); + case 1: return ImmutableList.of(originList.get(0)); + default: { + // NOTE: ImmutableList.copyOf(list) has additional clone of the list, so here we + // direct generate a ImmutableList + Builder copyChildren = ImmutableList.builderWithExpectedSize(originList.size()); + copyChildren.addAll(originList); + return copyChildren.build(); + } + } + } + + /** reverseImmutableList */ + public static ImmutableList reverseImmutableList(List list) { + Builder reverseList = ImmutableList.builderWithExpectedSize(list.size()); + for (int i = list.size() - 1; i >= 0; i--) { + reverseList.add(list.get(i)); + } + return reverseList.build(); + } + + /** filterImmutableList */ + public static ImmutableList filterImmutableList(List list, Predicate filter) { + Builder newList = ImmutableList.builderWithExpectedSize(list.size()); + for (int i = 0; i < list.size(); i++) { + E item = list.get(i); + if (filter.test(item)) { + newList.add(item); + } + } + return newList.build(); + } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy index ec43e058e6..4f0515c382 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy @@ -17,9 +17,12 @@ package org.apache.doris.regression +import ch.qos.logback.classic.PatternLayout +import ch.qos.logback.core.OutputStreamAppender import com.google.common.collect.Lists import groovy.transform.CompileStatic import jodd.util.Wildcard +import org.apache.doris.regression.logger.TeamcityServiceMessageEncoder import org.apache.doris.regression.suite.Suite import org.apache.doris.regression.suite.event.EventListener import org.apache.doris.regression.suite.GroovyFileSource @@ -64,7 +67,13 @@ class RegressionTest { ch.qos.logback.classic.Logger loggerOfSuite = LoggerFactory.getLogger(Suite.class) as ch.qos.logback.classic.Logger def context = loggerOfSuite.getLoggerContext() - context.getFrameworkPackages().add(IndyInterface.class.getPackage().getName()) + def frameworkPackages = context.getFrameworkPackages() + + // don't print this class name as the log class name + frameworkPackages.add(TeamcityServiceMessageEncoder.class.getPackage().getName()) + frameworkPackages.add(IndyInterface.class.getPackage().getName()) + frameworkPackages.add(OutputStreamAppender.class.getPackage().getName()) + frameworkPackages.add(PatternLayout.class.getPackage().getName()) } static void main(String[] args) { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy index b140673e7a..95f6f615c6 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy @@ -141,7 +141,7 @@ class OutputUtils { def res = checkCell(info, line, expectCell, realCell, dataType) if(res != null) { - res += "line ${line} mismatch\nExpectRow: ${expectRaw}\nRealRow: ${realRaw}"; + res += "\nline ${line} mismatch\nExpectRow: ${expectRaw}\nRealRow: ${realRaw}"; return res } }