From 5c00caa259a5017dea327ecd453f9eb3d2286f0a Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Mon, 30 Jan 2023 10:57:39 +0800 Subject: [PATCH] [refactor](Nereids) refactor BindSlotReference for easy merge all bind process in one rule (#16156) --- .../apache/doris/nereids/CascadesContext.java | 4 +- .../analysis => analyzer}/CTEContext.java | 2 +- .../nereids/analyzer/NereidsAnalyzer.java | 2 +- .../{rules/analysis => analyzer}/Scope.java | 2 +- .../analyzer/UnboundOneRowRelation.java | 1 + .../nereids/analyzer/UnboundRelation.java | 1 + .../doris/nereids/analyzer/UnboundSlot.java | 3 +- .../doris/nereids/analyzer/UnboundStar.java | 1 + .../nereids/analyzer/UnboundTVFRelation.java | 12 +- .../nereids/jobs/batch/AnalyzeRulesJob.java | 2 +- .../org/apache/doris/nereids/memo/Memo.java | 2 +- .../nereids/parser/LogicalPlanBuilder.java | 14 +- .../nereids/pattern/MatchingContext.java | 2 +- .../nereids/rules/analysis/BindRelation.java | 1 + .../rules/analysis/BindSlotReference.java | 676 ++++-------------- .../doris/nereids/rules/analysis/Binder.java | 201 ++++++ .../nereids/rules/analysis/RegisterCTE.java | 1 + .../rules/analysis/SubExprAnalyzer.java | 188 +++++ .../nereids/trees/expressions/BoundStar.java | 51 ++ .../trees/expressions/SubqueryExpr.java | 2 +- .../visitor/ExpressionVisitor.java | 2 +- .../doris/nereids/trees/plans/JoinType.java | 1 - .../trees/plans/algebra/CatalogRelation.java | 1 - .../trees/plans/algebra/EmptyRelation.java | 1 - .../trees/plans/algebra/OneRowRelation.java | 1 - .../plans/algebra}/Relation.java | 2 +- .../nereids/trees/plans/algebra/Scan.java | 1 - .../trees/plans/algebra/TVFRelation.java | 1 - .../trees/plans/logical/LogicalProject.java | 2 +- .../nereids/parser/NereidsParserTest.java | 2 +- .../rules/analysis/AnalyzeSubQueryTest.java | 4 +- .../rules/analysis/RegisterCTETest.java | 3 +- .../test_string_function_regexp.out | 4 +- 33 files changed, 607 insertions(+), 586 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/nereids/{rules/analysis => analyzer}/CTEContext.java (98%) rename fe/fe-core/src/main/java/org/apache/doris/nereids/{rules/analysis => analyzer}/Scope.java (98%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/Binder.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/{analyzer => trees/plans/algebra}/Relation.java (94%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 1a9af3126e..413aa56e0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -20,7 +20,9 @@ package org.apache.doris.nereids; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Table; +import org.apache.doris.nereids.analyzer.CTEContext; import org.apache.doris.nereids.analyzer.NereidsAnalyzer; +import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.jobs.Job; import org.apache.doris.nereids.jobs.JobContext; @@ -36,8 +38,6 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.analysis.CTEContext; -import org.apache.doris.nereids.rules.analysis.Scope; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CTEContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/CTEContext.java similarity index 98% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CTEContext.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/CTEContext.java index 7bddf9d57b..fada3e4b04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CTEContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/CTEContext.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.analysis; +package org.apache.doris.nereids.analyzer; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.Plan; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java index 2924d2cec5..48be0c6978 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.jobs.batch.AnalyzeRulesJob; import org.apache.doris.nereids.jobs.batch.AnalyzeSubqueryRulesJob; import org.apache.doris.nereids.jobs.batch.CheckAnalysisJob; import org.apache.doris.nereids.jobs.batch.TypeCoercionJob; -import org.apache.doris.nereids.rules.analysis.Scope; import java.util.Objects; import java.util.Optional; @@ -33,6 +32,7 @@ import java.util.Optional; * TODO: revisit the interface after subquery analysis is supported. */ public class NereidsAnalyzer { + private final CascadesContext cascadesContext; private final Optional outerScope; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/Scope.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java similarity index 98% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/Scope.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java index 1c0db90f8a..5e32f1824e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/Scope.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.analysis; +package org.apache.doris.nereids.analyzer; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java index 0021663cff..705238aee7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundOneRowRelation.java @@ -44,6 +44,7 @@ import java.util.Optional; * e.g. select 100, 'value' */ public class UnboundOneRowRelation extends LogicalLeaf implements Unbound, OneRowRelation { + private final RelationId id; private final List projects; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index f33009b982..acef46f170 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -42,6 +42,7 @@ import java.util.Optional; * Represent a relation plan node that has not been bound. */ public class UnboundRelation extends LogicalRelation implements Unbound { + private final List nameParts; private final List partNames; private final boolean isTempPart; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java index bb1a07c36a..09eb1c94f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundSlot.java @@ -32,6 +32,7 @@ import java.util.Objects; * Slot has not been bound. */ public class UnboundSlot extends Slot implements Unbound, PropagateNullable { + private final List nameParts; public UnboundSlot(String... nameParts) { @@ -39,7 +40,7 @@ public class UnboundSlot extends Slot implements Unbound, PropagateNullable { } public UnboundSlot(List nameParts) { - this.nameParts = Objects.requireNonNull(nameParts, "nameParts can not be null"); + this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts can not be null")); } public List getNameParts() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index 2ba1d593d0..d6c54dffce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -33,6 +33,7 @@ import java.util.Objects; * Star expression. */ public class UnboundStar extends NamedExpression implements LeafExpression, Unbound, PropagateNullable { + private final List qualifier; public UnboundStar(List qualifier) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java index d484202e2b..fa63fa7d5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTVFRelation.java @@ -24,9 +24,11 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.TVFProperties; +import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -36,10 +38,11 @@ import java.util.Objects; import java.util.Optional; /** UnboundTVFRelation */ -public class UnboundTVFRelation extends LogicalLeaf implements Relation, Unbound { +public class UnboundTVFRelation extends LogicalLeaf implements TVFRelation, Unbound { + + private final RelationId id; private final String functionName; private final TVFProperties properties; - private final RelationId id; public UnboundTVFRelation(RelationId id, String functionName, TVFProperties properties) { this(id, functionName, properties, Optional.empty(), Optional.empty()); @@ -65,6 +68,11 @@ public class UnboundTVFRelation extends LogicalLeaf implements Relation, Unbound return id; } + @Override + public TableValuedFunction getFunction() { + throw new UnboundException("getFunction"); + } + @Override public LogicalProperties computeLogicalProperties() { return UnboundLogicalProperties.INSTANCE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java index 3dcd7f7682..0d7443110f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.jobs.batch; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.rules.analysis.AvgDistinctToSumDivCount; import org.apache.doris.nereids.rules.analysis.BindFunction; import org.apache.doris.nereids.rules.analysis.BindRelation; @@ -30,7 +31,6 @@ import org.apache.doris.nereids.rules.analysis.ProjectWithDistinctToAggregate; import org.apache.doris.nereids.rules.analysis.RegisterCTE; import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput; import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy; -import org.apache.doris.nereids.rules.analysis.Scope; import org.apache.doris.nereids.rules.analysis.UserAuthentication; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.rewrite.rules.CharacterLiteralTypeCoercion; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index c4e7465a88..09f81a4574 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -20,13 +20,13 @@ package org.apache.doris.nereids.memo; import org.apache.doris.common.IdGenerator; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.CTEContext; import org.apache.doris.nereids.metrics.EventChannel; import org.apache.doris.nereids.metrics.EventProducer; import org.apache.doris.nereids.metrics.consumer.LogConsumer; import org.apache.doris.nereids.metrics.event.GroupMergeEvent; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.rules.analysis.CTEContext; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index db976292b1..0a8e313bbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -444,7 +444,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { } private LogicalPlan withCheckPolicy(LogicalPlan plan) { - return new LogicalCheckPolicy(plan); + return new LogicalCheckPolicy<>(plan); } @Override @@ -1037,7 +1037,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { public Literal visitStringLiteral(StringLiteralContext ctx) { // TODO: add unescapeSQLString. String txt = ctx.STRING().getText(); - String s = txt.substring(1, txt.length() - 1); + String s = escapeBackSlash(txt.substring(1, txt.length() - 1)); return new VarcharLiteral(s); } @@ -1334,6 +1334,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { joinType = JoinType.RIGHT_OUTER_JOIN; } else if (join.joinType().INNER() != null) { joinType = JoinType.INNER_JOIN; + } else if (join.joinCriteria() != null) { + joinType = JoinType.INNER_JOIN; } else { joinType = JoinType.CROSS_JOIN; } @@ -1347,10 +1349,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { throw new ParseException("Invalid join hint: " + hint, hintCtx); } }).orElse(JoinHint.NONE); - // TODO: natural join, lateral join, using join, union join + // TODO: natural join, lateral join, union join JoinCriteriaContext joinCriteria = join.joinCriteria(); Optional condition = Optional.empty(); - List ids = null; + List ids = null; if (joinCriteria != null) { if (join.joinType().CROSS() != null) { throw new ParseException("Cross join can't be used with ON clause", joinCriteria); @@ -1376,7 +1378,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { last, plan(join.relationPrimary())); } else { - last = new UsingJoin(joinType, last, plan(join.relationPrimary()), ImmutableList.of(), ids, joinHint); + last = new UsingJoin<>(joinType, last, + plan(join.relationPrimary()), ImmutableList.of(), ids, joinHint); + } } return last; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java index de9205d2f8..de067a427d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java @@ -19,7 +19,7 @@ package org.apache.doris.nereids.pattern; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.rules.analysis.CTEContext; +import org.apache.doris.nereids.analyzer.CTEContext; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.qe.ConnectContext; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index fce1228a6c..7d21e2895f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -29,6 +29,7 @@ import org.apache.doris.catalog.external.HMSExternalTable; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.CTEContext; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.Memo; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java index d1ed45f4ba..18f9203cc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java @@ -19,35 +19,25 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.catalog.BuiltinAggregateFunctions; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.analyzer.UnboundAlias; +import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; -import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.BindFunction.FunctionBinder; import org.apache.doris.nereids.trees.UnaryNode; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.EqualTo; -import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.InSubquery; -import org.apache.doris.nereids.trees.expressions.ListQuery; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Not; -import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.SubqueryExpr; -import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.functions.Function; -import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.LeafPlan; @@ -68,13 +58,11 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.UsingJoin; -import org.apache.doris.planner.PlannerContext; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.commons.lang.StringUtils; import java.util.ArrayList; import java.util.Collection; @@ -115,85 +103,83 @@ public class BindSlotReference implements AnalysisRuleFactory { RuleType.BINDING_PROJECT_SLOT.build( logicalProject().when(Plan::canBind).thenApply(ctx -> { LogicalProject project = ctx.root; - List boundSlots = - bind(project.getProjects(), project.children(), project, ctx.cascadesContext); - List exceptSlots = bind(project.getExcepts(), project.children(), project, + List boundProjections = + bind(project.getProjects(), project.children(), ctx.cascadesContext); + List boundExceptions = bind(project.getExcepts(), project.children(), ctx.cascadesContext); - List newOutput = flatBoundStar(adjustNullableForProjects(project, boundSlots)); - newOutput.removeAll(exceptSlots); - return new LogicalProject<>(newOutput, project.child(), project.isDistinct()); + boundProjections = flatBoundStar(boundProjections, boundExceptions); + return new LogicalProject<>(boundProjections, project.child(), project.isDistinct()); }) ), RuleType.BINDING_FILTER_SLOT.build( logicalFilter().when(Plan::canBind).thenApply(ctx -> { LogicalFilter filter = ctx.root; Set boundConjuncts - = bind(filter.getConjuncts(), filter.children(), filter, ctx.cascadesContext); + = bind(filter.getConjuncts(), filter.children(), ctx.cascadesContext); return new LogicalFilter<>(boundConjuncts, filter.child()); }) ), RuleType.BINDING_USING_JOIN_SLOT.build( - usingJoin().thenApply(ctx -> { - UsingJoin using = ctx.root; - LogicalJoin lj = new LogicalJoin(using.getJoinType() == JoinType.CROSS_JOIN - ? JoinType.INNER_JOIN : using.getJoinType(), - using.getHashJoinConjuncts(), - using.getOtherJoinConjuncts(), using.getHint(), using.left(), - using.right()); - List unboundSlots = lj.getHashJoinConjuncts(); - Set slotNames = new HashSet<>(); - List leftOutput = new ArrayList<>(lj.left().getOutput()); - // Suppose A JOIN B USING(name) JOIN C USING(name), [A JOIN B] is the left node, in this case, - // C should combine with table B on C.name=B.name. so we reverse the output to make sure that - // the most right slot is matched with priority. - Collections.reverse(leftOutput); - List leftSlots = new ArrayList<>(); - Scope scope = toScope(leftOutput.stream() - .filter(s -> !slotNames.contains(s.getName())) - .peek(s -> slotNames.add(s.getName())).collect( - Collectors.toList())); - for (Expression unboundSlot : unboundSlots) { - Expression expression = new SlotBinder(scope, ctx.cascadesContext).bind(unboundSlot); - leftSlots.add(expression); - } - slotNames.clear(); - scope = toScope(lj.right().getOutput().stream() - .filter(s -> !slotNames.contains(s.getName())) - .peek(s -> slotNames.add(s.getName())).collect( - Collectors.toList())); - List rightSlots = new ArrayList<>(); - for (Expression unboundSlot : unboundSlots) { - Expression expression = new SlotBinder(scope, ctx.cascadesContext).bind(unboundSlot); - rightSlots.add(expression); - } - int size = leftSlots.size(); - List hashEqExpr = new ArrayList<>(); - for (int i = 0; i < size; i++) { - hashEqExpr.add(new EqualTo(leftSlots.get(i), rightSlots.get(i))); - } - return lj.withHashJoinConjuncts(hashEqExpr); - }) - ), - RuleType.BINDING_JOIN_SLOT.build( - logicalJoin().when(Plan::canBind) - .whenNot(j -> j.getJoinType().equals(JoinType.USING_JOIN)).thenApply(ctx -> { - LogicalJoin join = ctx.root; - List cond = join.getOtherJoinConjuncts().stream() - .map(expr -> bind(expr, join.children(), ctx.cascadesContext)) - .collect(Collectors.toList()); - List hashJoinConjuncts = join.getHashJoinConjuncts().stream() - .map(expr -> bind(expr, join.children(), ctx.cascadesContext)) - .collect(Collectors.toList()); - return new LogicalJoin<>(join.getJoinType(), - hashJoinConjuncts, cond, join.getHint(), join.left(), join.right()); - }) - ), + usingJoin().thenApply(ctx -> { + UsingJoin using = ctx.root; + LogicalJoin lj = new LogicalJoin<>(using.getJoinType() == JoinType.CROSS_JOIN + ? JoinType.INNER_JOIN : using.getJoinType(), + using.getHashJoinConjuncts(), + using.getOtherJoinConjuncts(), using.getHint(), using.left(), + using.right()); + List unboundSlots = lj.getHashJoinConjuncts(); + Set slotNames = new HashSet<>(); + List leftOutput = new ArrayList<>(lj.left().getOutput()); + // Suppose A JOIN B USING(name) JOIN C USING(name), [A JOIN B] is the left node, in this case, + // C should combine with table B on C.name=B.name. so we reverse the output to make sure that + // the most right slot is matched with priority. + Collections.reverse(leftOutput); + List leftSlots = new ArrayList<>(); + Scope scope = toScope(leftOutput.stream() + .filter(s -> !slotNames.contains(s.getName())) + .peek(s -> slotNames.add(s.getName())) + .collect(Collectors.toList())); + for (Expression unboundSlot : unboundSlots) { + Expression expression = new Binder(scope, ctx.cascadesContext).bind(unboundSlot); + leftSlots.add(expression); + } + slotNames.clear(); + scope = toScope(lj.right().getOutput().stream() + .filter(s -> !slotNames.contains(s.getName())) + .peek(s -> slotNames.add(s.getName())) + .collect(Collectors.toList())); + List rightSlots = new ArrayList<>(); + for (Expression unboundSlot : unboundSlots) { + Expression expression = new Binder(scope, ctx.cascadesContext).bind(unboundSlot); + rightSlots.add(expression); + } + int size = leftSlots.size(); + List hashEqExpr = new ArrayList<>(); + for (int i = 0; i < size; i++) { + hashEqExpr.add(new EqualTo(leftSlots.get(i), rightSlots.get(i))); + } + return lj.withHashJoinConjuncts(hashEqExpr); + }) + ), + RuleType.BINDING_JOIN_SLOT.build( + logicalJoin().when(Plan::canBind).thenApply(ctx -> { + LogicalJoin join = ctx.root; + List cond = join.getOtherJoinConjuncts().stream() + .map(expr -> bind(expr, join.children(), ctx.cascadesContext)) + .collect(Collectors.toList()); + List hashJoinConjuncts = join.getHashJoinConjuncts().stream() + .map(expr -> bind(expr, join.children(), ctx.cascadesContext)) + .collect(Collectors.toList()); + return new LogicalJoin<>(join.getJoinType(), + hashJoinConjuncts, cond, join.getHint(), join.left(), join.right()); + }) + ), RuleType.BINDING_AGGREGATE_SLOT.build( logicalAggregate().when(Plan::canBind).thenApply(ctx -> { LogicalAggregate agg = ctx.root; List output = - bind(agg.getOutputExpressions(), agg.children(), agg, ctx.cascadesContext); + bind(agg.getOutputExpressions(), agg.children(), ctx.cascadesContext); // The columns referenced in group by are first obtained from the child's output, // and then from the node's output @@ -244,12 +230,14 @@ public class BindSlotReference implements AnalysisRuleFactory { group by a group_by_key is bound on t1.a */ - duplicatedSlotNames.stream().forEach(dup -> childOutputsToExpr.remove(dup)); - Map aliasNameToExpr = output.stream() + duplicatedSlotNames.forEach(childOutputsToExpr::remove); + output.stream() .filter(ne -> ne instanceof Alias) .map(Alias.class::cast) - //agg function cannot be bound with group_by_key - .filter(alias -> ! alias.child().anyMatch(expr -> { + // agg function cannot be bound with group_by_key + // TODO(morrySnow): after bind function moved here, + // we could just use instanceof AggregateFunction + .filter(alias -> !alias.child().anyMatch(expr -> { if (expr instanceof UnboundFunction) { UnboundFunction unboundFunction = (UnboundFunction) expr; return BuiltinAggregateFunctions.INSTANCE.aggFuncNames.contains( @@ -258,9 +246,7 @@ public class BindSlotReference implements AnalysisRuleFactory { return false; } )) - .collect(Collectors.toMap(Alias::getName, UnaryNode::child, (oldExpr, newExpr) -> oldExpr)); - aliasNameToExpr.entrySet().stream() - .forEach(e -> childOutputsToExpr.putIfAbsent(e.getKey(), e.getValue())); + .forEach(alias -> childOutputsToExpr.putIfAbsent(alias.getName(), alias.child())); List replacedGroupBy = agg.getGroupByExpressions().stream() .map(groupBy -> { @@ -282,9 +268,9 @@ public class BindSlotReference implements AnalysisRuleFactory { Set outputSlots = output.stream() .filter(SlotReference.class::isInstance) .peek(slot -> outputSlotNames.add(slot.getName())) - .map(NamedExpression::toSlot).collect( - Collectors.toSet()); - //suppose group by key is a. + .map(NamedExpression::toSlot) + .collect(Collectors.toSet()); + // suppose group by key is a. // if both t1.a and t2.a are in agg.child.output, and t1.a in agg.output, // bind group_by_key a with t1.a // ` .filter(slot -> !outputSlotNames.contains(slot.getName()))` @@ -294,9 +280,9 @@ public class BindSlotReference implements AnalysisRuleFactory { .collect(Collectors.toSet()); boundSlots.addAll(outputSlots); - SlotBinder binder = new SlotBinder(toScope(Lists.newArrayList(boundSlots)), ctx.cascadesContext); - SlotBinder childBinder - = new SlotBinder(toScope(new ArrayList<>(agg.child().getOutputSet())), ctx.cascadesContext); + Binder binder = new Binder(toScope(Lists.newArrayList(boundSlots)), ctx.cascadesContext); + Binder childBinder + = new Binder(toScope(new ArrayList<>(agg.child().getOutputSet())), ctx.cascadesContext); List groupBy = replacedGroupBy.stream() .map(expression -> { @@ -308,21 +294,18 @@ public class BindSlotReference implements AnalysisRuleFactory { }) .collect(Collectors.toList()); List unboundGroupBys = Lists.newArrayList(); - Predicate> hasUnBound = (exprs) -> { - return exprs.stream().anyMatch( - expression -> { - if (expression.anyMatch(UnboundSlot.class::isInstance)) { - unboundGroupBys.add(expression); - return true; - } - return false; - }); - }; + Predicate> hasUnBound = (exprs) -> exprs.stream().anyMatch( + expression -> { + if (expression.anyMatch(UnboundSlot.class::isInstance)) { + unboundGroupBys.add(expression); + return true; + } + return false; + }); if (hasUnBound.test(groupBy)) { throw new AnalysisException("cannot bind GROUP BY KEY: " + unboundGroupBys.get(0).toSql()); } - List newOutput = adjustNullableForAgg(agg, output); - return agg.withGroupByAndOutput(groupBy, newOutput); + return agg.withGroupByAndOutput(groupBy, output); }) ), RuleType.BINDING_REPEAT_SLOT.build( @@ -330,7 +313,7 @@ public class BindSlotReference implements AnalysisRuleFactory { LogicalRepeat repeat = ctx.root; List output = - bind(repeat.getOutputExpressions(), repeat.children(), repeat, ctx.cascadesContext); + bind(repeat.getOutputExpressions(), repeat.children(), ctx.cascadesContext); // The columns referenced in group by are first obtained from the child's output, // and then from the node's output @@ -340,8 +323,7 @@ public class BindSlotReference implements AnalysisRuleFactory { .filter(ne -> ne instanceof Alias) .map(Alias.class::cast) .collect(Collectors.toMap(Alias::getName, UnaryNode::child, (oldExpr, newExpr) -> oldExpr)); - aliasNameToExpr.entrySet().stream() - .forEach(e -> childOutputsToExpr.putIfAbsent(e.getKey(), e.getValue())); + aliasNameToExpr.forEach(childOutputsToExpr::putIfAbsent); List> replacedGroupingSets = repeat.getGroupingSets().stream() .map(groupBy -> @@ -361,7 +343,7 @@ public class BindSlotReference implements AnalysisRuleFactory { List> groupingSets = replacedGroupingSets .stream() - .map(groupingSet -> bind(groupingSet, repeat.children(), repeat, ctx.cascadesContext)) + .map(groupingSet -> bind(groupingSet, repeat.children(), ctx.cascadesContext)) .collect(ImmutableList.toImmutableList()); List newOutput = adjustNullableForRepeat(groupingSets, output); return repeat.withGroupSetsAndOutput(groupingSets, newOutput); @@ -371,53 +353,28 @@ public class BindSlotReference implements AnalysisRuleFactory { logicalSort(aggregate()).when(Plan::canBind).thenApply(ctx -> { LogicalSort> sort = ctx.root; Aggregate aggregate = sort.child(); - return bindSortWithAggregateFunction(sort, aggregate, ctx.cascadesContext); + return bindSort(sort, aggregate, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SLOT.build( logicalSort(logicalHaving(aggregate())).when(Plan::canBind).thenApply(ctx -> { LogicalSort>> sort = ctx.root; Aggregate aggregate = sort.child().child(); - return bindSortWithAggregateFunction(sort, aggregate, ctx.cascadesContext); + return bindSort(sort, aggregate, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SLOT.build( logicalSort(logicalHaving(logicalProject())).when(Plan::canBind).thenApply(ctx -> { LogicalSort>> sort = ctx.root; - List sortItemList = sort.getOrderKeys() - .stream() - .map(orderKey -> { - Expression item = bind(orderKey.getExpr(), sort.children(), ctx.cascadesContext); - if (item.containsType(UnboundSlot.class)) { - item = bind(item, sort.child().children(), ctx.cascadesContext); - } - return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst()); - }).collect(Collectors.toList()); - - return new LogicalSort<>(sortItemList, sort.child()); + LogicalProject project = sort.child().child(); + return bindSort(sort, project, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SLOT.build( logicalSort(logicalProject()).when(Plan::canBind).thenApply(ctx -> { LogicalSort> sort = ctx.root; - Set projectOutput = sort.child().getOutputSet(); - SlotBinder binderOnProject = new SlotBinder(toScope(Lists.newArrayList(projectOutput)), - ctx.cascadesContext); - Set projectChildrenOutput = sort.child().children().stream() - .flatMap(plan -> plan.getOutputSet().stream()) - .collect(Collectors.toSet()); - SlotBinder binderOnProjectChild = new SlotBinder( - toScope(Lists.newArrayList(projectChildrenOutput)), - ctx.cascadesContext); - List sortItemList = sort.getOrderKeys() - .stream() - .map(orderKey -> { - Expression item = binderOnProject.bind(orderKey.getExpr()); - item = binderOnProjectChild.bind(item); - return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst()); - }).collect(Collectors.toList()); - - return new LogicalSort<>(sortItemList, sort.child()); + LogicalProject project = sort.child(); + return bindSort(sort, project, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SET_OPERATION_SLOT.build( @@ -426,13 +383,9 @@ public class BindSlotReference implements AnalysisRuleFactory { List sortItemList = sort.getOrderKeys() .stream() .map(orderKey -> { - Expression item = bind(orderKey.getExpr(), sort.children(), ctx.cascadesContext); - if (item.containsType(UnboundSlot.class)) { - item = bind(item, sort.child().children(), ctx.cascadesContext); - } + Expression item = bind(orderKey.getExpr(), sort.child(), ctx.cascadesContext); return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst()); }).collect(Collectors.toList()); - return new LogicalSort<>(sortItemList, sort.child()); }) ), @@ -440,21 +393,10 @@ public class BindSlotReference implements AnalysisRuleFactory { logicalHaving(aggregate()).when(Plan::canBind).thenApply(ctx -> { LogicalHaving> having = ctx.root; Plan childPlan = having.child(); - // We should deduplicate the slots, otherwise the binding process will fail due to the - // ambiguous slots exist. - List childChildSlots = childPlan.children().stream() - .flatMap(plan -> plan.getOutputSet().stream()) - .collect(Collectors.toList()); - SlotBinder childChildBinder = new SlotBinder(toScope(childChildSlots), - ctx.cascadesContext); - List childSlots = childPlan.getOutputSet().stream() - .collect(Collectors.toList()); - SlotBinder childBinder = new SlotBinder(toScope(childSlots), - ctx.cascadesContext); Set boundConjuncts = having.getConjuncts().stream().map( expr -> { - expr = childChildBinder.bind(expr); - return childBinder.bind(expr); + expr = bind(expr, childPlan.children(), ctx.cascadesContext); + return bind(expr, childPlan, ctx.cascadesContext); }) .collect(Collectors.toSet()); return new LogicalHaving<>(boundConjuncts, having.child()); @@ -464,21 +406,10 @@ public class BindSlotReference implements AnalysisRuleFactory { logicalHaving(any()).when(Plan::canBind).thenApply(ctx -> { LogicalHaving having = ctx.root; Plan childPlan = having.child(); - // We should deduplicate the slots, otherwise the binding process will fail due to the - // ambiguous slots exist. - List childChildSlots = childPlan.children().stream() - .flatMap(plan -> plan.getOutputSet().stream()) - .collect(Collectors.toList()); - SlotBinder childChildBinder = new SlotBinder(toScope(childChildSlots), - ctx.cascadesContext); - List childSlots = childPlan.getOutputSet().stream() - .collect(Collectors.toList()); - SlotBinder childBinder = new SlotBinder(toScope(childSlots), - ctx.cascadesContext); Set boundConjuncts = having.getConjuncts().stream().map( expr -> { - expr = childBinder.bind(expr); - return childChildBinder.bind(expr); + expr = bind(expr, childPlan, ctx.cascadesContext); + return bind(expr, childPlan.children(), ctx.cascadesContext); }) .collect(Collectors.toSet()); return new LogicalHaving<>(boundConjuncts, having.child()); @@ -506,10 +437,10 @@ public class BindSlotReference implements AnalysisRuleFactory { + setOperation.child(1).getOutput().size() + " column(s)"); } - // INTERSECT and EXCEPT does not support ALL qualifie + // INTERSECT and EXCEPT does not support ALL qualified if (setOperation.getQualifier() == Qualifier.ALL && (setOperation instanceof LogicalExcept || setOperation instanceof LogicalIntersect)) { - throw new AnalysisException("INTERSECT and EXCEPT does not support ALL qualifie"); + throw new AnalysisException("INTERSECT and EXCEPT does not support ALL qualified"); } List> castExpressions = setOperation.collectCastExpressions(); @@ -521,8 +452,8 @@ public class BindSlotReference implements AnalysisRuleFactory { RuleType.BINDING_GENERATE_SLOT.build( logicalGenerate().when(Plan::canBind).thenApply(ctx -> { LogicalGenerate generate = ctx.root; - List boundSlotGenerators = bind(generate.getGenerators(), generate.children(), - generate, ctx.cascadesContext); + List boundSlotGenerators + = bind(generate.getGenerators(), generate.children(), ctx.cascadesContext); List boundFunctionGenerators = boundSlotGenerators.stream() .map(f -> FunctionBinder.INSTANCE.bindTableGeneratingFunction( (UnboundFunction) f, ctx.statementContext)) @@ -551,8 +482,8 @@ public class BindSlotReference implements AnalysisRuleFactory { ); } - private Plan bindSortWithAggregateFunction( - LogicalSort sort, Aggregate aggregate, CascadesContext ctx) { + private Plan bindSort( + LogicalSort sort, Plan plan, CascadesContext ctx) { // 1. We should deduplicate the slots, otherwise the binding process will fail due to the // ambiguous slots exist. // 2. try to bound order-key with agg output, if failed, try to bound with output of agg.child @@ -568,21 +499,19 @@ public class BindSlotReference implements AnalysisRuleFactory { // group by col1 // order by col1; # order by order_col1 // bind order_col1 with alias_col1, then, bind it with inner_col1 - SlotBinder outputBinder = new SlotBinder( - toScope(aggregate.getOutputSet().stream().collect(Collectors.toList())), ctx); - List childOutputSlots = aggregate.child().getOutputSet().stream().collect(Collectors.toList()); - SlotBinder childOutputBinder = new SlotBinder(toScope(childOutputSlots), ctx); List sortItemList = sort.getOrderKeys() .stream() .map(orderKey -> { - Expression item = outputBinder.bind(orderKey.getExpr()); - item = childOutputBinder.bind(item); + Expression item = bind(orderKey.getExpr(), plan, ctx); + item = bind(item, plan.children(), ctx); return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst()); }).collect(Collectors.toList()); return new LogicalSort<>(sortItemList, sort.child()); } - private List flatBoundStar(List boundSlots) { + private List flatBoundStar( + List boundSlots, + List boundExceptions) { return boundSlots .stream() .flatMap(slot -> { @@ -591,395 +520,34 @@ public class BindSlotReference implements AnalysisRuleFactory { } else { return Stream.of(slot); } - }).collect(Collectors.toList()); + }) + .filter(s -> !boundExceptions.contains(s)) + .collect(ImmutableList.toImmutableList()); } - private List bind(List exprList, List inputs, Plan plan, - CascadesContext cascadesContext) { + private List bind(List exprList, List inputs, CascadesContext cascadesContext) { return exprList.stream() .map(expr -> bind(expr, inputs, cascadesContext)) .collect(Collectors.toList()); } - private Set bind(Set exprList, List inputs, Plan plan, - CascadesContext cascadesContext) { + private Set bind(Set exprList, List inputs, CascadesContext cascadesContext) { return exprList.stream() .map(expr -> bind(expr, inputs, cascadesContext)) .collect(Collectors.toSet()); } + @SuppressWarnings("unchecked") + private E bind(E expr, Plan input, CascadesContext cascadesContext) { + return (E) new Binder(toScope(input.getOutput()), cascadesContext).bind(expr); + } + + @SuppressWarnings("unchecked") private E bind(E expr, List inputs, CascadesContext cascadesContext) { List boundedSlots = inputs.stream() .flatMap(input -> input.getOutput().stream()) .collect(Collectors.toList()); - return (E) new SlotBinder(toScope(boundedSlots), cascadesContext).bind(expr); - } - - private class SlotBinder extends SubExprAnalyzer { - - public SlotBinder(Scope scope, CascadesContext cascadesContext) { - super(scope, cascadesContext); - } - - public Expression bind(Expression expression) { - return expression.accept(this, null); - } - - @Override - public Expression visitUnboundAlias(UnboundAlias unboundAlias, PlannerContext context) { - Expression child = unboundAlias.child().accept(this, context); - if (unboundAlias.getAlias().isPresent()) { - return new Alias(child, unboundAlias.getAlias().get()); - } - if (child instanceof NamedExpression) { - return new Alias(child, ((NamedExpression) child).getName()); - } else { - // TODO: resolve aliases - return new Alias(child, child.toSql()); - } - } - - @Override - public Slot visitUnboundSlot(UnboundSlot unboundSlot, PlannerContext context) { - Optional> boundedOpt = Optional.of(bindSlot(unboundSlot, getScope().getSlots())); - boolean foundInThisScope = !boundedOpt.get().isEmpty(); - // Currently only looking for symbols on the previous level. - if (!foundInThisScope && getScope().getOuterScope().isPresent()) { - boundedOpt = Optional.of(bindSlot(unboundSlot, - getScope() - .getOuterScope() - .get() - .getSlots())); - } - List bounded = boundedOpt.get(); - switch (bounded.size()) { - case 0: - // just return, give a chance to bind on another slot. - // if unbound finally, check will throw exception - return unboundSlot; - case 1: - if (!foundInThisScope) { - getScope().getOuterScope().get().getCorrelatedSlots().add(bounded.get(0)); - } - return bounded.get(0); - default: - /* - select t1.k k, t2.k - from t1 join t2 order by k - - 't1.k k' is denoted by alias_k, its full name is 'k' - 'order by k' is denoted as order_k, it full name is 'k' - 't2.k' in select list, its full name is 't2.k' - - order_k can be bound on alias_k and t2.k - alias_k is exactly matched, since its full name is exactly match full name of order_k - t2.k is not exactly matched, since t2.k's full name is larger than order_k - */ - List exactMatch = bounded.stream() - .filter(bound -> unboundSlot.getNameParts().size() == bound.getQualifier().size() + 1) - .collect(Collectors.toList()); - if (exactMatch.size() == 1) { - return exactMatch.get(0); - } - throw new AnalysisException(String.format("%s is ambiguous: %s.", - unboundSlot.toSql(), - bounded.stream() - .map(Slot::toString) - .collect(Collectors.joining(", ")))); - } - } - - @Override - public Expression visitUnboundStar(UnboundStar unboundStar, PlannerContext context) { - List qualifier = unboundStar.getQualifier(); - List slots = getScope().getSlots() - .stream() - .filter(slot -> !(slot instanceof SlotReference) || ((SlotReference) slot).isVisible()) - .collect(Collectors.toList()); - switch (qualifier.size()) { - case 0: // select * - return new BoundStar(slots); - case 1: // select table.* - case 2: // select db.table.* - return bindQualifiedStar(qualifier, slots); - default: - throw new AnalysisException("Not supported qualifier: " - + StringUtils.join(qualifier, ".")); - } - } - - private BoundStar bindQualifiedStar(List qualifierStar, List boundSlots) { - // FIXME: compatible with previous behavior: - // https://github.com/apache/doris/pull/10415/files/3fe9cb0c3f805ab3a9678033b281b16ad93ec60a#r910239452 - List slots = boundSlots.stream().filter(boundSlot -> { - switch (qualifierStar.size()) { - // table.* - case 1: - List boundSlotQualifier = boundSlot.getQualifier(); - switch (boundSlotQualifier.size()) { - // bound slot is `column` and no qualified - case 0: - return false; - case 1: // bound slot is `table`.`column` - return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)); - case 2:// bound slot is `db`.`table`.`column` - return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(1)); - default: - throw new AnalysisException("Not supported qualifier: " - + StringUtils.join(qualifierStar, ".")); - } - case 2: // db.table.* - boundSlotQualifier = boundSlot.getQualifier(); - switch (boundSlotQualifier.size()) { - // bound slot is `column` and no qualified - case 0: - case 1: // bound slot is `table`.`column` - return false; - case 2:// bound slot is `db`.`table`.`column` - return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)) - && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(1)); - default: - throw new AnalysisException("Not supported qualifier: " - + StringUtils.join(qualifierStar, ".") + ".*"); - } - default: - throw new AnalysisException("Not supported name: " - + StringUtils.join(qualifierStar, ".") + ".*"); - } - }).collect(Collectors.toList()); - - return new BoundStar(slots); - } - - private List bindSlot(UnboundSlot unboundSlot, List boundSlots) { - return boundSlots.stream().filter(boundSlot -> { - List nameParts = unboundSlot.getNameParts(); - int qualifierSize = boundSlot.getQualifier().size(); - int namePartsSize = nameParts.size(); - if (namePartsSize > qualifierSize + 1) { - return false; - } - if (namePartsSize == 1) { - return nameParts.get(0).equalsIgnoreCase(boundSlot.getName()); - } - if (namePartsSize == 2) { - String qualifierDbName = boundSlot.getQualifier().get(qualifierSize - 1); - return qualifierDbName.equalsIgnoreCase(nameParts.get(0)) - && boundSlot.getName().equalsIgnoreCase(nameParts.get(1)); - } else if (nameParts.size() == 3) { - String qualifierDbName = boundSlot.getQualifier().get(qualifierSize - 1); - String qualifierClusterName = boundSlot.getQualifier().get(qualifierSize - 2); - return qualifierClusterName.equalsIgnoreCase(nameParts.get(0)) - && qualifierDbName.equalsIgnoreCase(nameParts.get(1)) - && boundSlot.getName().equalsIgnoreCase(nameParts.get(2)); - } - //TODO: handle name parts more than three. - throw new AnalysisException("Not supported name: " - + StringUtils.join(nameParts, ".")); - }).collect(Collectors.toList()); - } - } - - /** BoundStar is used to wrap list of slots for temporary. */ - public static class BoundStar extends NamedExpression implements PropagateNullable { - public BoundStar(List children) { - super(children.toArray(new Slot[0])); - Preconditions.checkArgument(children.stream().noneMatch(slot -> slot instanceof UnboundSlot), - "BoundStar can not wrap UnboundSlot" - ); - } - - public String toSql() { - return children.stream().map(Expression::toSql).collect(Collectors.joining(", ")); - } - - public List getSlots() { - return (List) children(); - } - - @Override - public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitBoundStar(this, context); - } - } - - /** - * Use the visitor to iterate sub expression. - */ - private static class SubExprAnalyzer extends DefaultExpressionRewriter { - private final Scope scope; - private final CascadesContext cascadesContext; - - public SubExprAnalyzer(Scope scope, CascadesContext cascadesContext) { - this.scope = scope; - this.cascadesContext = cascadesContext; - } - - @Override - public Expression visitNot(Not not, PlannerContext context) { - Expression child = not.child(); - if (child instanceof Exists) { - return visitExistsSubquery( - new Exists(((Exists) child).getQueryPlan(), true), context); - } else if (child instanceof InSubquery) { - return visitInSubquery(new InSubquery(((InSubquery) child).getCompareExpr(), - ((InSubquery) child).getListQuery(), true), context); - } - return visit(not, context); - } - - @Override - public Expression visitExistsSubquery(Exists exists, PlannerContext context) { - AnalyzedResult analyzedResult = analyzeSubquery(exists); - - return new Exists(analyzedResult.getLogicalPlan(), - analyzedResult.getCorrelatedSlots(), exists.isNot()); - } - - @Override - public Expression visitInSubquery(InSubquery expr, PlannerContext context) { - AnalyzedResult analyzedResult = analyzeSubquery(expr); - - checkOutputColumn(analyzedResult.getLogicalPlan()); - checkHasGroupBy(analyzedResult); - - return new InSubquery( - expr.getCompareExpr().accept(this, context), - new ListQuery(analyzedResult.getLogicalPlan()), - analyzedResult.getCorrelatedSlots(), expr.isNot()); - } - - @Override - public Expression visitScalarSubquery(ScalarSubquery scalar, PlannerContext context) { - AnalyzedResult analyzedResult = analyzeSubquery(scalar); - - checkOutputColumn(analyzedResult.getLogicalPlan()); - checkRootIsAgg(analyzedResult); - checkHasGroupBy(analyzedResult); - - return new ScalarSubquery(analyzedResult.getLogicalPlan(), analyzedResult.getCorrelatedSlots()); - } - - private void checkOutputColumn(LogicalPlan plan) { - if (plan.getOutput().size() != 1) { - throw new AnalysisException("Multiple columns returned by subquery are not yet supported. Found " - + plan.getOutput().size()); - } - } - - private void checkRootIsAgg(AnalyzedResult analyzedResult) { - if (!analyzedResult.isCorrelated()) { - return; - } - if (!analyzedResult.rootIsAgg()) { - throw new AnalysisException("The select item in correlated subquery of binary predicate " - + "should only be sum, min, max, avg and count. Current subquery: " - + analyzedResult.getLogicalPlan()); - } - } - - private void checkHasGroupBy(AnalyzedResult analyzedResult) { - if (!analyzedResult.isCorrelated()) { - return; - } - if (analyzedResult.hasGroupBy()) { - throw new AnalysisException("Unsupported correlated subquery with grouping and/or aggregation " - + analyzedResult.getLogicalPlan()); - } - } - - private AnalyzedResult analyzeSubquery(SubqueryExpr expr) { - CascadesContext subqueryContext = new Memo(expr.getQueryPlan()) - .newCascadesContext((cascadesContext.getStatementContext()), cascadesContext.getCteContext()); - Scope subqueryScope = genScopeWithSubquery(expr); - subqueryContext - .newAnalyzer(Optional.of(subqueryScope)) - .analyze(); - return new AnalyzedResult((LogicalPlan) subqueryContext.getMemo().copyOut(false), - subqueryScope.getCorrelatedSlots()); - } - - private Scope genScopeWithSubquery(SubqueryExpr expr) { - return new Scope(getScope().getOuterScope(), - getScope().getSlots(), - Optional.ofNullable(expr)); - } - - public Scope getScope() { - return scope; - } - - public CascadesContext getCascadesContext() { - return cascadesContext; - } - } - - private static class AnalyzedResult { - private final LogicalPlan logicalPlan; - private final List correlatedSlots; - - public AnalyzedResult(LogicalPlan logicalPlan, List correlatedSlots) { - this.logicalPlan = Objects.requireNonNull(logicalPlan, "logicalPlan can not be null"); - this.correlatedSlots = correlatedSlots == null ? new ArrayList<>() : ImmutableList.copyOf(correlatedSlots); - } - - public LogicalPlan getLogicalPlan() { - return logicalPlan; - } - - public List getCorrelatedSlots() { - return correlatedSlots; - } - - public boolean isCorrelated() { - return !correlatedSlots.isEmpty(); - } - - public boolean rootIsAgg() { - return logicalPlan instanceof LogicalAggregate; - } - - public boolean hasGroupBy() { - if (rootIsAgg()) { - return !((LogicalAggregate) logicalPlan).getGroupByExpressions().isEmpty(); - } - return false; - } - } - - /** - * When there is a repeat operator in the query, - * it will change the nullable information of the output column, which will be changed uniformly here. - * - * adjust the project with the children output - */ - private List adjustNullableForProjects( - LogicalProject project, List projects) { - Set childrenOutput = project.children().stream() - .map(Plan::getOutput) - .flatMap(List::stream) - .filter(ExpressionTrait::nullable) - .collect(Collectors.toSet()); - return projects.stream() - .map(e -> e.accept(RewriteNullableToTrue.INSTANCE, childrenOutput)) - .map(NamedExpression.class::cast) - .collect(ImmutableList.toImmutableList()); - } - - /** - * same as project, adjust the agg with the children output. - */ - private List adjustNullableForAgg( - LogicalAggregate aggregate, List output) { - Set childrenOutput = aggregate.children().stream() - .map(Plan::getOutput) - .flatMap(List::stream) - .filter(ExpressionTrait::nullable) - .collect(Collectors.toSet()); - return output.stream() - .map(e -> e.accept(RewriteNullableToTrue.INSTANCE, childrenOutput)) - .map(NamedExpression.class::cast) - .collect(ImmutableList.toImmutableList()); + return (E) new Binder(toScope(boundedSlots), cascadesContext).bind(expr); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/Binder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/Binder.java new file mode 100644 index 0000000000..b6f5be8568 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/Binder.java @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; +import org.apache.doris.nereids.analyzer.UnboundAlias; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.analyzer.UnboundStar; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.BoundStar; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.planner.PlannerContext; + +import org.apache.commons.lang.StringUtils; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +class Binder extends SubExprAnalyzer { + + public Binder(Scope scope, CascadesContext cascadesContext) { + super(scope, cascadesContext); + } + + public Expression bind(Expression expression) { + return expression.accept(this, null); + } + + @Override + public Expression visitUnboundAlias(UnboundAlias unboundAlias, PlannerContext context) { + Expression child = unboundAlias.child().accept(this, context); + if (unboundAlias.getAlias().isPresent()) { + return new Alias(child, unboundAlias.getAlias().get()); + } + if (child instanceof NamedExpression) { + return new Alias(child, ((NamedExpression) child).getName()); + } else { + // TODO: resolve aliases + return new Alias(child, child.toSql()); + } + } + + @Override + public Slot visitUnboundSlot(UnboundSlot unboundSlot, PlannerContext context) { + Optional> boundedOpt = Optional.of(bindSlot(unboundSlot, getScope().getSlots())); + boolean foundInThisScope = !boundedOpt.get().isEmpty(); + // Currently only looking for symbols on the previous level. + if (!foundInThisScope && getScope().getOuterScope().isPresent()) { + boundedOpt = Optional.of(bindSlot(unboundSlot, + getScope() + .getOuterScope() + .get() + .getSlots())); + } + List bounded = boundedOpt.get(); + switch (bounded.size()) { + case 0: + // just return, give a chance to bind on another slot. + // if unbound finally, check will throw exception + return unboundSlot; + case 1: + if (!foundInThisScope) { + getScope().getOuterScope().get().getCorrelatedSlots().add(bounded.get(0)); + } + return bounded.get(0); + default: + // select t1.k k, t2.k + // from t1 join t2 order by k + // + // 't1.k k' is denoted by alias_k, its full name is 'k' + // 'order by k' is denoted as order_k, it full name is 'k' + // 't2.k' in select list, its full name is 't2.k' + // + // order_k can be bound on alias_k and t2.k + // alias_k is exactly matched, since its full name is exactly match full name of order_k + // t2.k is not exactly matched, since t2.k's full name is larger than order_k + List exactMatch = bounded.stream() + .filter(bound -> unboundSlot.getNameParts().size() == bound.getQualifier().size() + 1) + .collect(Collectors.toList()); + if (exactMatch.size() == 1) { + return exactMatch.get(0); + } + throw new AnalysisException(String.format("%s is ambiguous: %s.", + unboundSlot.toSql(), + bounded.stream() + .map(Slot::toString) + .collect(Collectors.joining(", ")))); + } + } + + @Override + public Expression visitUnboundStar(UnboundStar unboundStar, PlannerContext context) { + List qualifier = unboundStar.getQualifier(); + List slots = getScope().getSlots() + .stream() + .filter(slot -> !(slot instanceof SlotReference) || ((SlotReference) slot).isVisible()) + .collect(Collectors.toList()); + switch (qualifier.size()) { + case 0: // select * + return new BoundStar(slots); + case 1: // select table.* + case 2: // select db.table.* + return bindQualifiedStar(qualifier, slots); + default: + throw new AnalysisException("Not supported qualifier: " + + StringUtils.join(qualifier, ".")); + } + } + + private BoundStar bindQualifiedStar(List qualifierStar, List boundSlots) { + // FIXME: compatible with previous behavior: + // https://github.com/apache/doris/pull/10415/files/3fe9cb0c3f805ab3a9678033b281b16ad93ec60a#r910239452 + List slots = boundSlots.stream().filter(boundSlot -> { + switch (qualifierStar.size()) { + // table.* + case 1: + List boundSlotQualifier = boundSlot.getQualifier(); + switch (boundSlotQualifier.size()) { + // bound slot is `column` and no qualified + case 0: + return false; + case 1: // bound slot is `table`.`column` + return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)); + case 2:// bound slot is `db`.`table`.`column` + return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(1)); + default: + throw new AnalysisException("Not supported qualifier: " + + StringUtils.join(qualifierStar, ".")); + } + case 2: // db.table.* + boundSlotQualifier = boundSlot.getQualifier(); + switch (boundSlotQualifier.size()) { + // bound slot is `column` and no qualified + case 0: + case 1: // bound slot is `table`.`column` + return false; + case 2:// bound slot is `db`.`table`.`column` + return qualifierStar.get(0).equalsIgnoreCase(boundSlotQualifier.get(0)) + && qualifierStar.get(1).equalsIgnoreCase(boundSlotQualifier.get(1)); + default: + throw new AnalysisException("Not supported qualifier: " + + StringUtils.join(qualifierStar, ".") + ".*"); + } + default: + throw new AnalysisException("Not supported name: " + + StringUtils.join(qualifierStar, ".") + ".*"); + } + }).collect(Collectors.toList()); + + return new BoundStar(slots); + } + + private List bindSlot(UnboundSlot unboundSlot, List boundSlots) { + return boundSlots.stream().filter(boundSlot -> { + List nameParts = unboundSlot.getNameParts(); + int qualifierSize = boundSlot.getQualifier().size(); + int namePartsSize = nameParts.size(); + if (namePartsSize > qualifierSize + 1) { + return false; + } + if (namePartsSize == 1) { + return nameParts.get(0).equalsIgnoreCase(boundSlot.getName()); + } + if (namePartsSize == 2) { + String qualifierDbName = boundSlot.getQualifier().get(qualifierSize - 1); + return qualifierDbName.equalsIgnoreCase(nameParts.get(0)) + && boundSlot.getName().equalsIgnoreCase(nameParts.get(1)); + } else if (nameParts.size() == 3) { + String qualifierDbName = boundSlot.getQualifier().get(qualifierSize - 1); + String qualifierClusterName = boundSlot.getQualifier().get(qualifierSize - 2); + return qualifierClusterName.equalsIgnoreCase(nameParts.get(0)) + && qualifierDbName.equalsIgnoreCase(nameParts.get(1)) + && boundSlot.getName().equalsIgnoreCase(nameParts.get(2)); + } + //TODO: handle name parts more than three. + throw new AnalysisException("Not supported name: " + + StringUtils.join(nameParts, ".")); + }).collect(Collectors.toList()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java index 39e5701e0c..a8b25c2577 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.CTEContext; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.rules.Rule; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java new file mode 100644 index 0000000000..a4b57fa851 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.memo.Memo; +import org.apache.doris.nereids.trees.expressions.Exists; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.ListQuery; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.ScalarSubquery; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.planner.PlannerContext; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Use the visitor to iterate sub expression. + */ +public class SubExprAnalyzer extends DefaultExpressionRewriter { + + private final Scope scope; + private final CascadesContext cascadesContext; + + public SubExprAnalyzer(Scope scope, CascadesContext cascadesContext) { + this.scope = scope; + this.cascadesContext = cascadesContext; + } + + @Override + public Expression visitNot(Not not, PlannerContext context) { + Expression child = not.child(); + if (child instanceof Exists) { + return visitExistsSubquery( + new Exists(((Exists) child).getQueryPlan(), true), context); + } else if (child instanceof InSubquery) { + return visitInSubquery(new InSubquery(((InSubquery) child).getCompareExpr(), + ((InSubquery) child).getListQuery(), true), context); + } + return visit(not, context); + } + + @Override + public Expression visitExistsSubquery(Exists exists, PlannerContext context) { + AnalyzedResult analyzedResult = analyzeSubquery(exists); + + return new Exists(analyzedResult.getLogicalPlan(), + analyzedResult.getCorrelatedSlots(), exists.isNot()); + } + + @Override + public Expression visitInSubquery(InSubquery expr, PlannerContext context) { + AnalyzedResult analyzedResult = analyzeSubquery(expr); + + checkOutputColumn(analyzedResult.getLogicalPlan()); + checkHasGroupBy(analyzedResult); + + return new InSubquery( + expr.getCompareExpr().accept(this, context), + new ListQuery(analyzedResult.getLogicalPlan()), + analyzedResult.getCorrelatedSlots(), expr.isNot()); + } + + @Override + public Expression visitScalarSubquery(ScalarSubquery scalar, PlannerContext context) { + AnalyzedResult analyzedResult = analyzeSubquery(scalar); + + checkOutputColumn(analyzedResult.getLogicalPlan()); + checkRootIsAgg(analyzedResult); + checkHasGroupBy(analyzedResult); + + return new ScalarSubquery(analyzedResult.getLogicalPlan(), analyzedResult.getCorrelatedSlots()); + } + + private void checkOutputColumn(LogicalPlan plan) { + if (plan.getOutput().size() != 1) { + throw new AnalysisException("Multiple columns returned by subquery are not yet supported. Found " + + plan.getOutput().size()); + } + } + + private void checkRootIsAgg(AnalyzedResult analyzedResult) { + if (!analyzedResult.isCorrelated()) { + return; + } + if (!analyzedResult.rootIsAgg()) { + throw new AnalysisException("The select item in correlated subquery of binary predicate " + + "should only be sum, min, max, avg and count. Current subquery: " + + analyzedResult.getLogicalPlan()); + } + } + + private void checkHasGroupBy(AnalyzedResult analyzedResult) { + if (!analyzedResult.isCorrelated()) { + return; + } + if (analyzedResult.hasGroupBy()) { + throw new AnalysisException("Unsupported correlated subquery with grouping and/or aggregation " + + analyzedResult.getLogicalPlan()); + } + } + + private AnalyzedResult analyzeSubquery(SubqueryExpr expr) { + CascadesContext subqueryContext = new Memo(expr.getQueryPlan()) + .newCascadesContext((cascadesContext.getStatementContext()), cascadesContext.getCteContext()); + Scope subqueryScope = genScopeWithSubquery(expr); + subqueryContext + .newAnalyzer(Optional.of(subqueryScope)) + .analyze(); + return new AnalyzedResult((LogicalPlan) subqueryContext.getMemo().copyOut(false), + subqueryScope.getCorrelatedSlots()); + } + + private Scope genScopeWithSubquery(SubqueryExpr expr) { + return new Scope(getScope().getOuterScope(), + getScope().getSlots(), + Optional.ofNullable(expr)); + } + + public Scope getScope() { + return scope; + } + + public CascadesContext getCascadesContext() { + return cascadesContext; + } + + private static class AnalyzedResult { + private final LogicalPlan logicalPlan; + private final List correlatedSlots; + + public AnalyzedResult(LogicalPlan logicalPlan, List correlatedSlots) { + this.logicalPlan = Objects.requireNonNull(logicalPlan, "logicalPlan can not be null"); + this.correlatedSlots = correlatedSlots == null ? new ArrayList<>() : ImmutableList.copyOf(correlatedSlots); + } + + public LogicalPlan getLogicalPlan() { + return logicalPlan; + } + + public List getCorrelatedSlots() { + return correlatedSlots; + } + + public boolean isCorrelated() { + return !correlatedSlots.isEmpty(); + } + + public boolean rootIsAgg() { + return logicalPlan instanceof LogicalAggregate; + } + + public boolean hasGroupBy() { + if (rootIsAgg()) { + return !((LogicalAggregate) logicalPlan).getGroupByExpressions().isEmpty(); + } + return false; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java new file mode 100644 index 0000000000..02a2d8338c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; + +import com.google.common.base.Preconditions; + +import java.util.List; +import java.util.stream.Collectors; + +/** BoundStar is used to wrap list of slots for temporary. */ +public class BoundStar extends NamedExpression implements PropagateNullable { + public BoundStar(List children) { + super(children.toArray(new Slot[0])); + Preconditions.checkArgument(children.stream().noneMatch(slot -> slot instanceof UnboundSlot), + "BoundStar can not wrap UnboundSlot" + ); + } + + public String toSql() { + return children.stream().map(Expression::toSql).collect(Collectors.joining(", ")); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + public List getSlots() { + return (List) children(); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitBoundStar(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java index 37e3f4c0e5..44146728c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java @@ -50,7 +50,7 @@ public abstract class SubqueryExpr extends Expression { @Override public DataType getDataType() throws UnboundException { - throw new UnboundException("not support"); + throw new UnboundException("getDataType"); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index 204375dfac..cd5c615285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundStar; -import org.apache.doris.nereids.rules.analysis.BindSlotReference.BoundStar; import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.AggregateExpression; import org.apache.doris.nereids.trees.expressions.Alias; @@ -34,6 +33,7 @@ import org.apache.doris.nereids.trees.expressions.BitAnd; import org.apache.doris.nereids.trees.expressions.BitNot; import org.apache.doris.nereids.trees.expressions.BitOr; import org.apache.doris.nereids.trees.expressions.BitXor; +import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.CaseWhen; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java index a6f5cd2cb0..a01f650d63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java @@ -38,7 +38,6 @@ public enum JoinType { RIGHT_ANTI_JOIN, CROSS_JOIN, NULL_AWARE_LEFT_ANTI_JOIN, - USING_JOIN, ; private static final Map joinSwapMap = ImmutableMap diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/CatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/CatalogRelation.java index cd7124187f..cef0b3fce4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/CatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/CatalogRelation.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.trees.plans.algebra; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Table; -import org.apache.doris.nereids.analyzer.Relation; import org.apache.doris.nereids.exceptions.AnalysisException; /** CatalogRelation */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/EmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/EmptyRelation.java index f6ad9a015e..ec57ff6044 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/EmptyRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/EmptyRelation.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.trees.plans.algebra; -import org.apache.doris.nereids.analyzer.Relation; import org.apache.doris.nereids.trees.expressions.NamedExpression; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OneRowRelation.java index 2c63d4e856..3ae4a207ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OneRowRelation.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.trees.plans.algebra; -import org.apache.doris.nereids.analyzer.Relation; import org.apache.doris.nereids.trees.expressions.NamedExpression; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Relation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Relation.java similarity index 94% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Relation.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Relation.java index 349179ba6c..f0f0ab844e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Relation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Relation.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.analyzer; +package org.apache.doris.nereids.trees.plans.algebra; import org.apache.doris.nereids.trees.expressions.Slot; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java index c221391e3f..5f0baf2232 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Scan.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.plans.algebra; import org.apache.doris.catalog.TableIf; -import org.apache.doris.nereids.analyzer.Relation; /** * Common interface for logical/physical scan. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/TVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/TVFRelation.java index 0e57f51058..829a821d98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/TVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/TVFRelation.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.trees.plans.algebra; -import org.apache.doris.nereids.analyzer.Relation; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; /** TVFRelation */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index e643b640e4..12037f81b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -20,7 +20,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.rules.analysis.BindSlotReference.BoundStar; +import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 1a91837836..cf120e210f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -173,7 +173,7 @@ public class NereidsParserTest extends ParserTestBase { String innerJoin2 = "SELECT t1.a FROM t1 JOIN t2 ON t1.id = t2.id;"; logicalPlan = nereidsParser.parseSingle(innerJoin2); logicalJoin = (LogicalJoin) logicalPlan.child(0); - Assertions.assertEquals(JoinType.CROSS_JOIN, logicalJoin.getJoinType()); + Assertions.assertEquals(JoinType.INNER_JOIN, logicalJoin.getJoinType()); String leftJoin1 = "SELECT t1.a FROM t1 LEFT JOIN t2 ON t1.id = t2.id;"; logicalPlan = nereidsParser.parseSingle(leftJoin1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java index 899d91ef18..d55083b718 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java @@ -131,7 +131,7 @@ public class AnalyzeSubQueryTest extends TestWithFeService implements PatternMat .applyTopDown(new LogicalSubQueryAliasToLogicalProject()) .matchesFromRoot( logicalProject( - crossLogicalJoin( + innerLogicalJoin( logicalProject( logicalOlapScan() ), @@ -167,7 +167,7 @@ public class AnalyzeSubQueryTest extends TestWithFeService implements PatternMat .applyTopDown(new LogicalSubQueryAliasToLogicalProject()) .matchesFromRoot( logicalProject( - crossLogicalJoin( + innerLogicalJoin( logicalOlapScan(), logicalProject( logicalOlapScan() diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java index 542e28f1a4..15b0349e4c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.common.NereidsException; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.CTEContext; import org.apache.doris.nereids.datasets.ssb.SSBUtils; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; @@ -236,7 +237,7 @@ public class RegisterCTETest extends TestWithFeService implements PatternMatchSu .analyze(sql4) .matchesFromRoot( logicalProject( - crossLogicalJoin( + innerLogicalJoin( logicalSubQueryAlias( logicalProject().when(p -> p.getProjects().equals(ImmutableList.of(skAlias))) ).when(a -> a.getAlias().equals("cte1")), diff --git a/regression-test/data/nereids_p0/sql_functions/string_functions/test_string_function_regexp.out b/regression-test/data/nereids_p0/sql_functions/string_functions/test_string_function_regexp.out index 47cf5dfd8c..5e7fac9d56 100644 --- a/regression-test/data/nereids_p0/sql_functions/string_functions/test_string_function_regexp.out +++ b/regression-test/data/nereids_p0/sql_functions/string_functions/test_string_function_regexp.out @@ -19,13 +19,13 @@ billie eillish a-b-c -- !sql -- -a <\\1> c +a c -- !sql -- a-b c -- !sql -- -a <\\1> b +a b -- !sql -- false 1 1989 1001 11011902 123.123 true 1989-03-21 1989-03-21T13:00 wangjuoo4 0.1 6.333 string12345 170141183460469231731687303715884105727