diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 413aa56e0a..2ed1123bb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -26,11 +26,14 @@ import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.jobs.Job; import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob; import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob; import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob; +import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob.RootRewriteJobContext; import org.apache.doris.nereids.jobs.scheduler.JobPool; import org.apache.doris.nereids.jobs.scheduler.JobScheduler; import org.apache.doris.nereids.jobs.scheduler.JobStack; +import org.apache.doris.nereids.jobs.scheduler.ScheduleContext; import org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler; import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.processor.post.RuntimeFilterContext; @@ -38,12 +41,14 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -57,12 +62,20 @@ import java.util.Optional; import java.util.Set; import java.util.Stack; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; /** * Context used in memo. */ -public class CascadesContext { - private final Memo memo; +public class CascadesContext implements ScheduleContext, PlanSource { + // in analyze/rewrite stage, the plan will storage in this field + private Plan plan; + + private Optional currentRootRewriteJobContext; + + // in optimize stage, the plan will storage in the memo + private Memo memo; + private final StatementContext statementContext; private CTEContext cteContext; @@ -76,8 +89,13 @@ public class CascadesContext { private List tables = null; - public CascadesContext(Memo memo, StatementContext statementContext, PhysicalProperties requestProperties) { - this(memo, statementContext, new CTEContext(), requestProperties); + private boolean isRewriteRoot; + + private Optional outerScope = Optional.empty(); + + public CascadesContext(Plan plan, Memo memo, StatementContext statementContext, + PhysicalProperties requestProperties) { + this(plan, memo, statementContext, new CTEContext(), requestProperties); } /** @@ -86,8 +104,9 @@ public class CascadesContext { * @param memo {@link Memo} reference * @param statementContext {@link StatementContext} reference */ - public CascadesContext(Memo memo, StatementContext statementContext, + public CascadesContext(Plan plan, Memo memo, StatementContext statementContext, CTEContext cteContext, PhysicalProperties requireProperties) { + this.plan = plan; this.memo = memo; this.statementContext = statementContext; this.ruleSet = new RuleSet(); @@ -99,19 +118,30 @@ public class CascadesContext { this.cteContext = cteContext; } - public static CascadesContext newContext(StatementContext statementContext, + public static CascadesContext newMemoContext(StatementContext statementContext, Plan initPlan, PhysicalProperties requireProperties) { - return new CascadesContext(new Memo(initPlan), statementContext, requireProperties); + return new CascadesContext(initPlan, new Memo(initPlan), statementContext, requireProperties); + } + + public static CascadesContext newRewriteContext(StatementContext statementContext, + Plan initPlan, PhysicalProperties requireProperties) { + return new CascadesContext(initPlan, null, statementContext, requireProperties); + } + + public static CascadesContext newRewriteContext(StatementContext statementContext, + Plan initPlan, CTEContext cteContext) { + return new CascadesContext(initPlan, null, statementContext, cteContext, PhysicalProperties.ANY); + } + + public void toMemo() { + this.memo = new Memo(plan); } public NereidsAnalyzer newAnalyzer() { return new NereidsAnalyzer(this); } - public NereidsAnalyzer newAnalyzer(Optional outerScope) { - return new NereidsAnalyzer(this, outerScope); - } - + @Override public void pushJob(Job job) { jobPool.push(job); } @@ -136,6 +166,7 @@ public class CascadesContext { this.ruleSet = ruleSet; } + @Override public JobPool getJobPool() { return jobPool; } @@ -165,6 +196,23 @@ public class CascadesContext { return this; } + public Plan getRewritePlan() { + return plan; + } + + public void setRewritePlan(Plan plan) { + this.plan = plan; + } + + public Optional getCurrentRootRewriteJobContext() { + return currentRootRewriteJobContext; + } + + public void setCurrentRootRewriteJobContext( + RootRewriteJobContext currentRootRewriteJobContext) { + this.currentRootRewriteJobContext = Optional.ofNullable(currentRootRewriteJobContext); + } + public void setSubqueryExprIsAnalyzed(SubqueryExpr subqueryExpr, boolean isAnalyzed) { subqueryExprIsAnalyzed.put(subqueryExpr, isAnalyzed); } @@ -201,6 +249,13 @@ public class CascadesContext { return execute(new RewriteTopDownJob(memo.getRoot(), rules, currentJobContext)); } + public CascadesContext topDownRewrite(CustomRewriter customRewriter) { + CustomRewriteJob customRewriteJob = new CustomRewriteJob(() -> customRewriter, RuleType.TEST_REWRITE); + customRewriteJob.execute(currentJobContext); + toMemo(); + return this; + } + public CTEContext getCteContext() { return cteContext; } @@ -209,6 +264,22 @@ public class CascadesContext { this.cteContext = cteContext; } + public void setIsRewriteRoot(boolean isRewriteRoot) { + this.isRewriteRoot = isRewriteRoot; + } + + public boolean isRewriteRoot() { + return isRewriteRoot; + } + + public Optional getOuterScope() { + return outerScope; + } + + public void setOuterScope(@Nullable Scope outerScope) { + this.outerScope = Optional.ofNullable(outerScope); + } + private CascadesContext execute(Job job) { pushJob(job); jobScheduler.executeJobPool(this); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 9d8db1b127..d19c176228 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; -import org.apache.doris.nereids.jobs.batch.NereidsRewriteJobExecutor; -import org.apache.doris.nereids.jobs.batch.OptimizeRulesJob; +import org.apache.doris.nereids.jobs.batch.CascadesOptimizer; +import org.apache.doris.nereids.jobs.batch.NereidsRewriter; import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob; import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; import org.apache.doris.nereids.memo.CopyInResult; @@ -156,7 +156,7 @@ public class NereidsPlanner extends Planner { // resolve column, table and function analyze(); if (explainLevel == ExplainLevel.ANALYZED_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { - analyzedPlan = cascadesContext.getMemo().copyOut(false); + analyzedPlan = cascadesContext.getRewritePlan(); if (explainLevel == ExplainLevel.ANALYZED_PLAN) { return analyzedPlan; } @@ -164,11 +164,14 @@ public class NereidsPlanner extends Planner { // rule-based optimize rewrite(); if (explainLevel == ExplainLevel.REWRITTEN_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { - rewrittenPlan = cascadesContext.getMemo().copyOut(false); + rewrittenPlan = cascadesContext.getRewritePlan(); if (explainLevel == ExplainLevel.REWRITTEN_PLAN) { return rewrittenPlan; } } + + initMemo(); + deriveStats(); optimize(); @@ -190,7 +193,7 @@ public class NereidsPlanner extends Planner { } private void initCascadesContext(LogicalPlan plan, PhysicalProperties requireProperties) { - cascadesContext = CascadesContext.newContext(statementContext, plan, requireProperties); + cascadesContext = CascadesContext.newRewriteContext(statementContext, plan, requireProperties); } private void analyze() { @@ -201,7 +204,11 @@ public class NereidsPlanner extends Planner { * Logical plan rewrite based on a series of heuristic rules. */ private void rewrite() { - new NereidsRewriteJobExecutor(cascadesContext).execute(); + new NereidsRewriter(cascadesContext).execute(); + } + + private void initMemo() { + cascadesContext.toMemo(); } private void deriveStats() { @@ -236,7 +243,7 @@ public class NereidsPlanner extends Planner { .getSessionVariable().getMaxTableCountUseCascadesJoinReorder()) { dpHypOptimize(); } - new OptimizeRulesJob(cascadesContext).execute(); + new CascadesOptimizer(cascadesContext).execute(); } private PhysicalPlan postProcess(PhysicalPlan physicalPlan) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanSource.java new file mode 100644 index 0000000000..317b03d295 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlanSource.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids; + +/** PlanSource */ +public interface PlanSource { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java index 99c2eb7604..f470f88244 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java @@ -18,41 +18,80 @@ package org.apache.doris.nereids.analyzer; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.batch.AdjustAggregateNullableForEmptySetJob; -import org.apache.doris.nereids.jobs.batch.AnalyzeRulesJob; -import org.apache.doris.nereids.jobs.batch.AnalyzeSubqueryRulesJob; -import org.apache.doris.nereids.jobs.batch.CheckAnalysisJob; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.jobs.batch.BatchRewriteJob; +import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; +import org.apache.doris.nereids.rules.analysis.BindExpression; +import org.apache.doris.nereids.rules.analysis.BindRelation; +import org.apache.doris.nereids.rules.analysis.CheckAnalysis; +import org.apache.doris.nereids.rules.analysis.CheckPolicy; +import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots; +import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; +import org.apache.doris.nereids.rules.analysis.ProjectToGlobalAggregate; +import org.apache.doris.nereids.rules.analysis.ProjectWithDistinctToAggregate; +import org.apache.doris.nereids.rules.analysis.RegisterCTE; +import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput; +import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy; +import org.apache.doris.nereids.rules.analysis.SubqueryToApply; +import org.apache.doris.nereids.rules.analysis.UserAuthentication; +import org.apache.doris.nereids.rules.rewrite.logical.HideOneRowRelationUnderUnion; -import java.util.Objects; -import java.util.Optional; +import java.util.List; /** * Bind symbols according to metadata in the catalog, perform semantic analysis, etc. * TODO: revisit the interface after subquery analysis is supported. */ -public class NereidsAnalyzer { - - private final CascadesContext cascadesContext; - private final Optional outerScope; +public class NereidsAnalyzer extends BatchRewriteJob { + public static final List ANALYZE_JOBS = jobs( + topDown( + new RegisterCTE() + ), + bottomUp( + new BindRelation(), + new CheckPolicy(), + new UserAuthentication(), + new BindExpression(), + new ProjectToGlobalAggregate(), + // this rule check's the logicalProject node's isDisinct property + // and replace the logicalProject node with a LogicalAggregate node + // so any rule before this, if create a new logicalProject node + // should make sure isDistinct property is correctly passed around. + // please see rule BindSlotReference or BindFunction for example + new ProjectWithDistinctToAggregate(), + new ResolveOrdinalInOrderByAndGroupBy(), + new ReplaceExpressionByChildOutput(), + new HideOneRowRelationUnderUnion() + ), + topDown( + new FillUpMissingSlots(), + // We should use NormalizeRepeat to compute nullable properties for LogicalRepeat in the analysis + // stage. NormalizeRepeat will compute nullable property, add virtual slot, LogicalAggregate and + // LogicalProject for normalize. This rule depends on FillUpMissingSlots to fill up slots. + new NormalizeRepeat() + ), + bottomUp(new SubqueryToApply()), + bottomUp(new AdjustAggregateNullableForEmptySet()), + bottomUp(new CheckAnalysis()) + ); + /** + * Execute the analysis job with scope. + * @param cascadesContext planner context for execute job + */ public NereidsAnalyzer(CascadesContext cascadesContext) { - this(cascadesContext, Optional.empty()); + super(cascadesContext); } - public NereidsAnalyzer(CascadesContext cascadesContext, Optional outerScope) { - this.cascadesContext = Objects.requireNonNull(cascadesContext, "cascadesContext cannot be null"); - this.outerScope = Objects.requireNonNull(outerScope, "outerScope cannot be null"); + @Override + public List getJobs() { + return ANALYZE_JOBS; } /** * nereids analyze sql. */ public void analyze() { - new AnalyzeRulesJob(cascadesContext, outerScope).execute(); - new AnalyzeSubqueryRulesJob(cascadesContext).execute(); - new AdjustAggregateNullableForEmptySetJob(cascadesContext).execute(); - // check whether analyze result is meaningful - new CheckAnalysisJob(cascadesContext).execute(); + execute(); } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java index 5e32f1824e..dcdb79b2c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java @@ -22,11 +22,12 @@ import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.Sets; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * The slot range required for expression analyze. @@ -57,13 +58,13 @@ public class Scope { private final List slots; private final Optional ownerSubquery; - private List correlatedSlots; + private Set correlatedSlots; public Scope(Optional outerScope, List slots, Optional subqueryExpr) { this.outerScope = outerScope; this.slots = ImmutableList.copyOf(Objects.requireNonNull(slots, "slots can not be null")); this.ownerSubquery = subqueryExpr; - this.correlatedSlots = new ArrayList<>(); + this.correlatedSlots = Sets.newLinkedHashSet(); } public Scope(List slots) { @@ -82,7 +83,7 @@ public class Scope { return ownerSubquery; } - public List getCorrelatedSlots() { + public Set getCorrelatedSlots() { return correlatedSlots; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java index c16a654490..f017dfbacb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.jobs; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.CopyInResult; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -39,17 +38,14 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.util.List; -import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; /** * Abstract class for all job using for analyze and optimize query plan in Nereids. @@ -60,8 +56,6 @@ public abstract class Job implements TracerSupplier { EventChannel.getDefaultChannel() .addEnhancers(new AddCounterEventEnhancer()) .addConsumers(new LogConsumer(CounterEvent.class, EventChannel.LOG))); - public final Logger logger = LogManager.getLogger(getClass()); - protected JobType type; protected JobContext context; protected boolean once; @@ -76,12 +70,11 @@ public abstract class Job implements TracerSupplier { this.type = type; this.context = context; this.once = once; - this.disableRules = getAndCacheSessionVariable(context, "disableNereidsRules", - ImmutableSet.of(), SessionVariable::getDisableNereidsRules); + this.disableRules = getDisableRules(context); } public void pushJob(Job job) { - context.getCascadesContext().pushJob(job); + context.getScheduleContext().pushJob(job); } public RuleSet getRuleSet() { @@ -101,12 +94,21 @@ public abstract class Job implements TracerSupplier { */ public List getValidRules(GroupExpression groupExpression, List candidateRules) { return candidateRules.stream() - .filter(rule -> !disableRules.contains(rule.getRuleType().name().toUpperCase(Locale.ROOT))) - .filter(rule -> Objects.nonNull(rule) && rule.getPattern().matchRoot(groupExpression.getPlan()) - && groupExpression.notApplied(rule)).collect(Collectors.toList()); + .filter(rule -> Objects.nonNull(rule) + && !disableRules.contains(rule.getRuleType().name()) + && rule.getPattern().matchRoot(groupExpression.getPlan()) + && groupExpression.notApplied(rule)) + .collect(ImmutableList.toImmutableList()); } - public abstract void execute() throws AnalysisException; + public List getValidRules(List candidateRules) { + return candidateRules.stream() + .filter(rule -> Objects.nonNull(rule) + && !disableRules.contains(rule.getRuleType().name())) + .collect(ImmutableList.toImmutableList()); + } + + public abstract void execute(); public EventProducer getEventTracer() { throw new UnsupportedOperationException("get_event_tracer is unsupported"); @@ -146,7 +148,17 @@ public abstract class Job implements TracerSupplier { groupExpression.getOwnerGroup(), groupExpression, groupExpression.getPlan())); } - private T getAndCacheSessionVariable(JobContext context, String cacheName, + public static Set getDisableRules(JobContext context) { + return getAndCacheSessionVariable(context, "disableNereidsRules", + ImmutableSet.of(), SessionVariable::getDisableNereidsRules); + } + + public static boolean isTraceEnable(JobContext context) { + return getAndCacheSessionVariable(context, "isTraceEnable", + false, SessionVariable::isEnableNereidsTrace); + } + + private static T getAndCacheSessionVariable(JobContext context, String cacheName, T defaultValue, Function variableSupplier) { CascadesContext cascadesContext = context.getCascadesContext(); ConnectContext connectContext = cascadesContext.getConnectContext(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java index 04fb4e87a4..626b8f24cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.jobs; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.scheduler.ScheduleContext; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.RuleType; @@ -29,21 +30,25 @@ import java.util.Map; * Context for one job in Nereids' cascades framework. */ public class JobContext { - protected final CascadesContext cascadesContext; + protected final ScheduleContext scheduleContext; protected final PhysicalProperties requiredProperties; protected double costUpperBound; protected boolean rewritten = false; protected Map ruleInvokeTimes = Maps.newLinkedHashMap(); - public JobContext(CascadesContext cascadesContext, PhysicalProperties requiredProperties, double costUpperBound) { - this.cascadesContext = cascadesContext; + public JobContext(ScheduleContext scheduleContext, PhysicalProperties requiredProperties, double costUpperBound) { + this.scheduleContext = scheduleContext; this.requiredProperties = requiredProperties; this.costUpperBound = costUpperBound; } + public ScheduleContext getScheduleContext() { + return scheduleContext; + } + public CascadesContext getCascadesContext() { - return cascadesContext; + return (CascadesContext) scheduleContext; } public PhysicalProperties getRequiredProperties() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobType.java index 03681c58ee..d7a1517af2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobType.java @@ -31,5 +31,6 @@ public enum JobType { TOP_DOWN_REWRITE, VISITOR_REWRITE, BOTTOM_UP_REWRITE, - JOIN_ORDER; + JOIN_ORDER, + LINK_PLAN; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/RewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/RewriteJob.java new file mode 100644 index 0000000000..cdaeefb086 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/RewriteJob.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs; + +/** RewriteJob */ +public interface RewriteJob { + void execute(JobContext jobContext); + + boolean isOnce(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/TopicRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/TopicRewriteJob.java new file mode 100644 index 0000000000..3fb026c4fb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/TopicRewriteJob.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.stream.Stream; + +/** TopicRewriteJob */ +public class TopicRewriteJob implements RewriteJob { + public final String topicName; + public final List jobs; + + /** constructor */ + public TopicRewriteJob(String topicName, List jobs) { + this.topicName = topicName; + this.jobs = jobs.stream() + .flatMap(job -> job instanceof TopicRewriteJob + ? ((TopicRewriteJob) job).jobs.stream() + : Stream.of(job) + ) + .collect(ImmutableList.toImmutableList()); + } + + @Override + public void execute(JobContext jobContext) { + for (RewriteJob job : jobs) { + job.execute(jobContext); + } + } + + @Override + public boolean isOnce() { + return true; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java deleted file mode 100644 index c68e096baf..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.batch; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.analyzer.Scope; -import org.apache.doris.nereids.rules.analysis.AvgDistinctToSumDivCount; -import org.apache.doris.nereids.rules.analysis.BindExpression; -import org.apache.doris.nereids.rules.analysis.BindRelation; -import org.apache.doris.nereids.rules.analysis.CheckPolicy; -import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots; -import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; -import org.apache.doris.nereids.rules.analysis.ProjectToGlobalAggregate; -import org.apache.doris.nereids.rules.analysis.ProjectWithDistinctToAggregate; -import org.apache.doris.nereids.rules.analysis.RegisterCTE; -import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput; -import org.apache.doris.nereids.rules.analysis.ResolveOrdinalInOrderByAndGroupBy; -import org.apache.doris.nereids.rules.analysis.UserAuthentication; -import org.apache.doris.nereids.rules.rewrite.logical.HideOneRowRelationUnderUnion; - -import com.google.common.collect.ImmutableList; - -import java.util.Optional; - -/** - * Execute the analysis rules. - */ -public class AnalyzeRulesJob extends BatchRulesJob { - - /** - * Execute the analysis job with scope. - * @param cascadesContext planner context for execute job - * @param scope Parse the symbolic scope of the field - */ - public AnalyzeRulesJob(CascadesContext cascadesContext, Optional scope) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - bottomUpBatch( - new RegisterCTE() - ), - bottomUpBatch( - new BindRelation(), - new CheckPolicy(), - new UserAuthentication(), - new BindExpression(scope), - new ProjectToGlobalAggregate(), - // this rule check's the logicalProject node's isDisinct property - // and replace the logicalProject node with a LogicalAggregate node - // so any rule before this, if create a new logicalProject node - // should make sure isDisinct property is correctly passed around. - // please see rule BindSlotReference or BindFunction for example - new ProjectWithDistinctToAggregate(), - new AvgDistinctToSumDivCount(), - new ResolveOrdinalInOrderByAndGroupBy(), - new ReplaceExpressionByChildOutput(), - new HideOneRowRelationUnderUnion() - ), - topDownBatch( - new FillUpMissingSlots(), - // We should use NormalizeRepeat to compute nullable properties for LogicalRepeat in the analysis - // stage. NormalizeRepeat will compute nullable property, add virtual slot, LogicalAggregate and - // LogicalProject for normalize. This rule depends on FillUpMissingSlots to fill up slots. - new NormalizeRepeat() - ) - )); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ConvertApplyToJoinJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ApplyToJoin.java similarity index 70% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ConvertApplyToJoinJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ApplyToJoin.java index d086c4fb82..a3597ac1c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ConvertApplyToJoinJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/ApplyToJoin.java @@ -17,27 +17,28 @@ package org.apache.doris.nereids.jobs.batch; -import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.rewrite.BatchRewriteRuleFactory; import org.apache.doris.nereids.rules.rewrite.logical.ExistsApplyToJoin; import org.apache.doris.nereids.rules.rewrite.logical.InApplyToJoin; import org.apache.doris.nereids.rules.rewrite.logical.ScalarApplyToJoin; import com.google.common.collect.ImmutableList; +import java.util.List; + /** * Convert logicalApply without a correlated to a logicalJoin. */ -public class ConvertApplyToJoinJob extends BatchRulesJob { - /** - * Constructor. - */ - public ConvertApplyToJoinJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - topDownBatch(ImmutableList.of( - new ScalarApplyToJoin(), - new InApplyToJoin(), - new ExistsApplyToJoin()) - ))); +public class ApplyToJoin implements BatchRewriteRuleFactory { + public static final List RULES = ImmutableList.of( + new ScalarApplyToJoin(), + new InApplyToJoin(), + new ExistsApplyToJoin() + ); + + @Override + public List getRuleFactories() { + return RULES; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRewriteJob.java new file mode 100644 index 0000000000..f4d5419d81 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRewriteJob.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.batch; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.jobs.TopicRewriteJob; +import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob; +import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob; +import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteTopDownJob; +import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; +import java.util.stream.Stream; + +/** + * Base class for executing all jobs. + * + * Each batch of rules will be uniformly executed. + */ +public abstract class BatchRewriteJob { + protected CascadesContext cascadesContext; + + public BatchRewriteJob(CascadesContext cascadesContext) { + this.cascadesContext = Objects.requireNonNull(cascadesContext, "cascadesContext can not null"); + } + + public static List jobs(RewriteJob... jobs) { + return Arrays.stream(jobs) + .flatMap(job -> job instanceof TopicRewriteJob + ? ((TopicRewriteJob) job).jobs.stream() + : Stream.of(job) + ).collect(ImmutableList.toImmutableList()); + } + + public static TopicRewriteJob topic(String topicName, RewriteJob... jobs) { + return new TopicRewriteJob(topicName, Arrays.asList(jobs)); + } + + public static RewriteJob bottomUp(String batchName, RuleFactory... ruleFactories) { + return bottomUp(Arrays.asList(ruleFactories)); + } + + public static RewriteJob bottomUp(RuleFactory... ruleFactories) { + return bottomUp(Arrays.asList(ruleFactories)); + } + + public static RewriteJob bottomUp(List ruleFactories) { + List rules = new ArrayList<>(); + for (RuleFactory ruleFactory : ruleFactories) { + rules.addAll(ruleFactory.buildRules()); + } + return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteBottomUpJob::new, true); + } + + public static RewriteJob topDown(RuleFactory... ruleFactories) { + return topDown(Arrays.asList(ruleFactories)); + } + + public static RewriteJob topDown(List ruleFactories) { + return topDown(ruleFactories, true); + } + + public static RewriteJob topDown(List ruleFactories, boolean once) { + List rules = new ArrayList<>(); + for (RuleFactory ruleFactory : ruleFactories) { + rules.addAll(ruleFactory.buildRules()); + } + return new RootPlanTreeRewriteJob(rules, PlanTreeRewriteTopDownJob::new, once); + } + + public static RewriteJob custom(RuleType ruleType, Supplier planRewriter) { + return new CustomRewriteJob(planRewriter, ruleType); + } + + /** + * execute. + */ + public void execute() { + for (RewriteJob job : getJobs()) { + JobContext jobContext = cascadesContext.getCurrentJobContext(); + do { + jobContext.setRewritten(false); + job.execute(jobContext); + } while (!job.isOnce() && jobContext.isRewritten()); + } + } + + public abstract List getJobs(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java deleted file mode 100644 index 036437eff6..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java +++ /dev/null @@ -1,109 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.batch; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.Job; -import org.apache.doris.nereids.jobs.JobContext; -import org.apache.doris.nereids.jobs.cascades.OptimizeGroupJob; -import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob; -import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob; -import org.apache.doris.nereids.jobs.rewrite.VisitorRewriteJob; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleFactory; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; - -/** - * Base class for executing all jobs. - * - * Each batch of rules will be uniformly executed. - */ -public abstract class BatchRulesJob { - protected CascadesContext cascadesContext; - protected List rulesJob = new ArrayList<>(); - - BatchRulesJob(CascadesContext cascadesContext) { - this.cascadesContext = Objects.requireNonNull(cascadesContext, "cascadesContext can not null"); - } - - protected Job bottomUpBatch(RuleFactory... ruleFactories) { - return bottomUpBatch(Arrays.asList(ruleFactories)); - } - - protected Job bottomUpBatch(List ruleFactories) { - List rules = new ArrayList<>(); - for (RuleFactory ruleFactory : ruleFactories) { - rules.addAll(ruleFactory.buildRules()); - } - return new RewriteBottomUpJob( - cascadesContext.getMemo().getRoot(), - rules, - cascadesContext.getCurrentJobContext()); - } - - protected Job topDownBatch(RuleFactory... ruleFactories) { - return topDownBatch(Arrays.asList(ruleFactories)); - } - - protected Job topDownBatch(List ruleFactories) { - List rules = new ArrayList<>(); - for (RuleFactory ruleFactory : ruleFactories) { - rules.addAll(ruleFactory.buildRules()); - } - return new RewriteTopDownJob(cascadesContext.getMemo().getRoot(), rules, - cascadesContext.getCurrentJobContext()); - } - - protected Job topDownBatch(List ruleFactories, boolean once) { - List rules = new ArrayList<>(); - for (RuleFactory ruleFactory : ruleFactories) { - rules.addAll(ruleFactory.buildRules()); - } - return new RewriteTopDownJob(cascadesContext.getMemo().getRoot(), rules, - cascadesContext.getCurrentJobContext(), once); - } - - protected Job visitorJob(RuleType ruleType, DefaultPlanRewriter planRewriter) { - return new VisitorRewriteJob(cascadesContext, planRewriter, ruleType); - } - - protected Job optimize() { - return new OptimizeGroupJob( - cascadesContext.getMemo().getRoot(), - cascadesContext.getCurrentJobContext()); - } - - /** - * execute. - */ - public void execute() { - for (Job job : rulesJob) { - do { - cascadesContext.getCurrentJobContext().setRewritten(false); - cascadesContext.pushJob(job); - cascadesContext.getJobScheduler().executeJobPool(cascadesContext); - } while (!job.isOnce() && cascadesContext.getCurrentJobContext().isRewritten()); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustAggregateNullableForEmptySetJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CascadesOptimizer.java similarity index 58% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustAggregateNullableForEmptySetJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CascadesOptimizer.java index 403f1b8ff9..9b7d15c9fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustAggregateNullableForEmptySetJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CascadesOptimizer.java @@ -18,17 +18,25 @@ package org.apache.doris.nereids.jobs.batch; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; +import org.apache.doris.nereids.jobs.cascades.OptimizeGroupJob; -import com.google.common.collect.ImmutableList; +import java.util.Objects; /** - * Analyze subquery. + * cascade optimizer. */ -public class AdjustAggregateNullableForEmptySetJob extends BatchRulesJob { - public AdjustAggregateNullableForEmptySetJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - bottomUpBatch(ImmutableList.of(new AdjustAggregateNullableForEmptySet())))); +public class CascadesOptimizer { + private CascadesContext cascadesContext; + + public CascadesOptimizer(CascadesContext cascadesContext) { + this.cascadesContext = Objects.requireNonNull(cascadesContext, "cascadesContext cannot be null"); + } + + public void execute() { + cascadesContext.pushJob(new OptimizeGroupJob( + cascadesContext.getMemo().getRoot(), + cascadesContext.getCurrentJobContext()) + ); + cascadesContext.getJobScheduler().executeJobPool(cascadesContext); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustApplyFromCorrelateToUnCorrelateJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CorrelateApplyToUnCorrelateApply.java similarity index 54% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustApplyFromCorrelateToUnCorrelateJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CorrelateApplyToUnCorrelateApply.java index 856d857f11..45e51ea969 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustApplyFromCorrelateToUnCorrelateJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CorrelateApplyToUnCorrelateApply.java @@ -17,15 +17,18 @@ package org.apache.doris.nereids.jobs.batch; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.rewrite.logical.ApplyPullFilterOnAgg; -import org.apache.doris.nereids.rules.rewrite.logical.ApplyPullFilterOnProjectUnderAgg; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilterUnderApplyProject; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderFilter; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderProject; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.rewrite.BatchRewriteRuleFactory; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpCorrelatedFilterUnderApplyAggregateProject; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpProjectUnderApply; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyAggregateFilter; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyFilter; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyProjectFilter; import com.google.common.collect.ImmutableList; +import java.util.List; + /** * Adjust the plan in logicalApply so that there are no correlated columns in the subquery. * Adjust the positions of apply and sub query nodes and apply, @@ -34,19 +37,17 @@ import com.google.common.collect.ImmutableList; * For the project and filter on AGG, try to adjust them to apply. * For the project and filter under AGG, bring the filter under AGG and merge it with agg. */ -public class AdjustApplyFromCorrelateToUnCorrelateJob extends BatchRulesJob { - /** - * Constructor. - */ - public AdjustApplyFromCorrelateToUnCorrelateJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - topDownBatch(ImmutableList.of( - new PushApplyUnderProject(), - new PushApplyUnderFilter(), - new EliminateFilterUnderApplyProject(), - new ApplyPullFilterOnAgg(), - new ApplyPullFilterOnProjectUnderAgg() - )))); +public class CorrelateApplyToUnCorrelateApply implements BatchRewriteRuleFactory { + public static final List RULES = ImmutableList.of( + new PullUpProjectUnderApply(), + new UnCorrelatedApplyFilter(), + new UnCorrelatedApplyProjectFilter(), + new UnCorrelatedApplyAggregateFilter(), + new PullUpCorrelatedFilterUnderApplyAggregateProject() + ); + + @Override + public List getRuleFactories() { + return RULES; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateSpecificPlanUnderApplyJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateUselessPlanUnderApply.java similarity index 72% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateSpecificPlanUnderApplyJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateUselessPlanUnderApply.java index 2b8f7b25e0..4878e13f25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateSpecificPlanUnderApplyJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/EliminateUselessPlanUnderApply.java @@ -17,26 +17,27 @@ package org.apache.doris.nereids.jobs.batch; -import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.rewrite.BatchRewriteRuleFactory; import org.apache.doris.nereids.rules.rewrite.logical.EliminateLimitUnderApply; import org.apache.doris.nereids.rules.rewrite.logical.EliminateSortUnderApply; import com.google.common.collect.ImmutableList; +import java.util.List; + /** * Eliminate useless operators in the subquery, including limit and sort. * Compatible with the old optimizer, the sort and limit in the subquery will not take effect, just delete it directly. */ -public class EliminateSpecificPlanUnderApplyJob extends BatchRulesJob { - /** - * Constructor. - */ - public EliminateSpecificPlanUnderApplyJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - topDownBatch(ImmutableList.of( - new EliminateLimitUnderApply(), - new EliminateSortUnderApply() - )))); +public class EliminateUselessPlanUnderApply implements BatchRewriteRuleFactory { + public static final List RULES = ImmutableList.of( + new EliminateLimitUnderApply(), + new EliminateSortUnderApply() + ); + + @Override + public List getRuleFactories() { + return RULES; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java deleted file mode 100644 index f86fa0abb0..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java +++ /dev/null @@ -1,150 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.batch; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.Job; -import org.apache.doris.nereids.rules.RuleSet; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; -import org.apache.doris.nereids.rules.analysis.CheckAfterRewrite; -import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; -import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization; -import org.apache.doris.nereids.rules.expression.rewrite.ExpressionOptimization; -import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewrite; -import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate; -import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithoutAggregate; -import org.apache.doris.nereids.rules.rewrite.logical.AdjustNullable; -import org.apache.doris.nereids.rules.rewrite.logical.BuildAggForUnion; -import org.apache.doris.nereids.rules.rewrite.logical.CheckAndStandardizeWindowFunctionAndFrame; -import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning; -import org.apache.doris.nereids.rules.rewrite.logical.CountDistinctRewrite; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateDedupJoinCondition; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateGroupByConstant; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateLimit; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateNotNull; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateNullAwareLeftAntiJoin; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateOrderByConstant; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateUnnecessaryProject; -import org.apache.doris.nereids.rules.rewrite.logical.ExtractAndNormalizeWindowExpression; -import org.apache.doris.nereids.rules.rewrite.logical.ExtractFilterFromCrossJoin; -import org.apache.doris.nereids.rules.rewrite.logical.ExtractSingleTableExpressionFromDisjunction; -import org.apache.doris.nereids.rules.rewrite.logical.FindHashConditionForJoin; -import org.apache.doris.nereids.rules.rewrite.logical.InferFilterNotNull; -import org.apache.doris.nereids.rules.rewrite.logical.InferJoinNotNull; -import org.apache.doris.nereids.rules.rewrite.logical.InferPredicates; -import org.apache.doris.nereids.rules.rewrite.logical.InnerToCrossJoin; -import org.apache.doris.nereids.rules.rewrite.logical.LimitPushDown; -import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters; -import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; -import org.apache.doris.nereids.rules.rewrite.logical.MergeSetOperations; -import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate; -import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition; -import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet; -import org.apache.doris.nereids.rules.rewrite.logical.PushFilterInsideJoin; -import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin; - -import com.google.common.collect.ImmutableList; - -/** - * Apply rules to optimize logical plan. - */ -public class NereidsRewriteJobExecutor extends BatchRulesJob { - - /** - * Constructor. - * - * @param cascadesContext context for applying rules. - */ - public NereidsRewriteJobExecutor(CascadesContext cascadesContext) { - super(cascadesContext); - ImmutableList jobs = new ImmutableList.Builder() - .addAll(new EliminateSpecificPlanUnderApplyJob(cascadesContext).rulesJob) - // MergeProjects depends on this rule - .add(bottomUpBatch(ImmutableList.of(new LogicalSubQueryAliasToLogicalProject()))) - // AdjustApplyFromCorrelateToUnCorrelateJob and ConvertApplyToJoinJob - // and SelectMaterializedIndexWithAggregate depends on this rule - .add(topDownBatch(ImmutableList.of(new MergeProjects()))) - .add(topDownBatch(ImmutableList.of(new ExpressionNormalization(cascadesContext.getConnectContext())))) - .add(topDownBatch(ImmutableList.of(new ExpressionOptimization()))) - .add(topDownBatch(ImmutableList.of(new ExtractSingleTableExpressionFromDisjunction()))) - /* - * Subquery unnesting. - * 1. Adjust the plan in correlated logicalApply - * so that there are no correlated columns in the subquery. - * 2. Convert logicalApply to a logicalJoin. - * TODO: group these rules to make sure the result plan is what we expected. - */ - .addAll(new AdjustApplyFromCorrelateToUnCorrelateJob(cascadesContext).rulesJob) - .addAll(new ConvertApplyToJoinJob(cascadesContext).rulesJob) - .add(bottomUpBatch(ImmutableList.of(new AdjustAggregateNullableForEmptySet()))) - .add(topDownBatch(ImmutableList.of(new EliminateGroupByConstant()))) - .add(topDownBatch(ImmutableList.of(new NormalizeAggregate()))) - .add(topDownBatch(ImmutableList.of(new ExtractAndNormalizeWindowExpression()))) - // execute NormalizeAggregate() again to resolve nested AggregateFunctions in WindowExpression, - // e.g. sum(sum(c1)) over(partition by avg(c1)) - .add(topDownBatch(ImmutableList.of(new NormalizeAggregate()))) - .add(topDownBatch(ImmutableList.of(new CheckAndStandardizeWindowFunctionAndFrame()))) - .add(topDownBatch(ImmutableList.of(new InferFilterNotNull()))) - .add(topDownBatch(ImmutableList.of(new InferJoinNotNull()))) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(visitorJob(RuleType.INFER_PREDICATES, new InferPredicates())) - .add(topDownBatch(ImmutableList.of(new ExtractFilterFromCrossJoin()))) - .add(topDownBatch(ImmutableList.of(new MergeFilters()))) - .add(topDownBatch(ImmutableList.of(new ReorderJoin()))) - .add(topDownBatch(ImmutableList.of(new EliminateDedupJoinCondition()))) - .add(topDownBatch(ImmutableList.of(new ColumnPruning()))) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(visitorJob(RuleType.INFER_PREDICATES, new InferPredicates())) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(visitorJob(RuleType.INFER_PREDICATES, new InferPredicates())) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(topDownBatch(ImmutableList.of(new PushFilterInsideJoin()))) - .add(topDownBatch(ImmutableList.of(new FindHashConditionForJoin()))) - .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false)) - .add(topDownBatch(ImmutableList.of(new InnerToCrossJoin()))) - .add(topDownBatch(ImmutableList.of(new EliminateNotNull()))) - .add(topDownBatch(ImmutableList.of(new EliminateLimit()))) - .add(topDownBatch(ImmutableList.of(new EliminateFilter()))) - .add(topDownBatch(ImmutableList.of(new PruneOlapScanPartition()))) - .add(topDownBatch(ImmutableList.of(new CountDistinctRewrite()))) - // we need to execute this rule at the end of rewrite - // to avoid two consecutive same project appear when we do optimization. - .add(topDownBatch(ImmutableList.of(new EliminateOrderByConstant()))) - .add(topDownBatch(ImmutableList.of(new EliminateUnnecessaryProject()))) - .add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithAggregate()))) - .add(topDownBatch(ImmutableList.of(new SelectMaterializedIndexWithoutAggregate()))) - .add(topDownBatch(ImmutableList.of(new PruneOlapScanTablet()))) - .add(topDownBatch(ImmutableList.of(new EliminateAggregate()))) - .add(bottomUpBatch(ImmutableList.of(new MergeSetOperations()))) - .add(topDownBatch(ImmutableList.of(new LimitPushDown()))) - .add(topDownBatch(ImmutableList.of(new BuildAggForUnion()))) - .add(topDownBatch(ImmutableList.of(new EliminateNullAwareLeftAntiJoin()))) - // this rule batch must keep at the end of rewrite to do some plan check - .add(bottomUpBatch(ImmutableList.of( - new AdjustNullable(), - new ExpressionRewrite(CheckLegalityAfterRewrite.INSTANCE), - new CheckAfterRewrite())) - ) - .build(); - - rulesJob.addAll(jobs); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java new file mode 100644 index 0000000000..28571b1724 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java @@ -0,0 +1,214 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.batch; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.rules.RuleSet; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.analysis.AdjustAggregateNullableForEmptySet; +import org.apache.doris.nereids.rules.analysis.AvgDistinctToSumDivCount; +import org.apache.doris.nereids.rules.analysis.CheckAfterRewrite; +import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionOptimization; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewrite; +import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithAggregate; +import org.apache.doris.nereids.rules.mv.SelectMaterializedIndexWithoutAggregate; +import org.apache.doris.nereids.rules.rewrite.logical.AdjustNullable; +import org.apache.doris.nereids.rules.rewrite.logical.BuildAggForUnion; +import org.apache.doris.nereids.rules.rewrite.logical.CheckAndStandardizeWindowFunctionAndFrame; +import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning; +import org.apache.doris.nereids.rules.rewrite.logical.CountDistinctRewrite; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateDedupJoinCondition; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateGroupByConstant; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateLimit; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateNotNull; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateNullAwareLeftAntiJoin; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateOrderByConstant; +import org.apache.doris.nereids.rules.rewrite.logical.EliminateUnnecessaryProject; +import org.apache.doris.nereids.rules.rewrite.logical.ExtractAndNormalizeWindowExpression; +import org.apache.doris.nereids.rules.rewrite.logical.ExtractFilterFromCrossJoin; +import org.apache.doris.nereids.rules.rewrite.logical.ExtractSingleTableExpressionFromDisjunction; +import org.apache.doris.nereids.rules.rewrite.logical.FindHashConditionForJoin; +import org.apache.doris.nereids.rules.rewrite.logical.InferFilterNotNull; +import org.apache.doris.nereids.rules.rewrite.logical.InferJoinNotNull; +import org.apache.doris.nereids.rules.rewrite.logical.InferPredicates; +import org.apache.doris.nereids.rules.rewrite.logical.InnerToCrossJoin; +import org.apache.doris.nereids.rules.rewrite.logical.LimitPushDown; +import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters; +import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; +import org.apache.doris.nereids.rules.rewrite.logical.MergeSetOperations; +import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate; +import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition; +import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet; +import org.apache.doris.nereids.rules.rewrite.logical.PushFilterInsideJoin; +import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin; + +import java.util.List; + +/** + * Apply rules to optimize logical plan. + */ +public class NereidsRewriter extends BatchRewriteJob { + private static final List REWRITE_JOBS = jobs( + topic("Normalization", + topDown( + new EliminateOrderByConstant(), + new EliminateGroupByConstant(), + + // MergeProjects depends on this rule + new LogicalSubQueryAliasToLogicalProject(), + + // rewrite expressions, no depends + new ExpressionNormalization(), + new ExpressionOptimization(), + new AvgDistinctToSumDivCount(), + new CountDistinctRewrite(), + + new NormalizeAggregate(), + new ExtractFilterFromCrossJoin() + ), + + // ExtractSingleTableExpressionFromDisjunction conflict to InPredicateToEqualToRule + // in the ExpressionNormalization, so must invoke in another job, or else run into + // deep loop + topDown( + new ExtractSingleTableExpressionFromDisjunction() + ) + ), + + topic("Subquery unnesting", + bottomUp( + new EliminateUselessPlanUnderApply(), + + // CorrelateApplyToUnCorrelateApply and ApplyToJoin + // and SelectMaterializedIndexWithAggregate depends on this rule + new MergeProjects(), + + /* + * Subquery unnesting. + * 1. Adjust the plan in correlated logicalApply + * so that there are no correlated columns in the subquery. + * 2. Convert logicalApply to a logicalJoin. + * TODO: group these rules to make sure the result plan is what we expected. + */ + new CorrelateApplyToUnCorrelateApply(), + new ApplyToJoin() + ) + ), + + topDown( + new AdjustAggregateNullableForEmptySet() + ), + + topic("Window analysis", + topDown( + new ExtractAndNormalizeWindowExpression(), + // execute NormalizeAggregate() again to resolve nested AggregateFunctions in WindowExpression, + // e.g. sum(sum(c1)) over(partition by avg(c1)) + new NormalizeAggregate(), + new CheckAndStandardizeWindowFunctionAndFrame() + ) + ), + + topic("Rewrite join", + // infer not null filter, then push down filter, and then reorder join(cross join to inner join) + topDown( + new InferFilterNotNull(), + new InferJoinNotNull() + ), + // ReorderJoin depends PUSH_DOWN_FILTERS + // the PUSH_DOWN_FILTERS depends on lots of rules, e.g. merge project, eliminate outer, + // sometimes transform the bottom plan make some rules usable which can apply to the top plan, + // but top-down traverse can not cover this case in one iteration, so bottom-up is more + // efficient because it can find the new plans and apply transform wherever it is + bottomUp(RuleSet.PUSH_DOWN_FILTERS), + + topDown( + new MergeFilters(), + new ReorderJoin(), + new PushFilterInsideJoin(), + new FindHashConditionForJoin(), + new InnerToCrossJoin(), + new EliminateNullAwareLeftAntiJoin() + ), + topDown( + new EliminateDedupJoinCondition() + ) + ), + + topic("Column pruning and infer predicate", + topDown(new ColumnPruning()), + + custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()), + + // column pruning create new project, so we should use PUSH_DOWN_FILTERS + // to change filter-project to project-filter + bottomUp(RuleSet.PUSH_DOWN_FILTERS), + + // after eliminate outer join in the PUSH_DOWN_FILTERS, we can infer more predicate and push down + custom(RuleType.INFER_PREDICATES, () -> new InferPredicates()), + + bottomUp(RuleSet.PUSH_DOWN_FILTERS), + + // after eliminate outer join, we can move some filters to join.otherJoinConjuncts, + // this can help to translate plan to backend + topDown( + new PushFilterInsideJoin() + ) + ), + + // this rule should invoke after ColumnPruning + custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, () -> new EliminateUnnecessaryProject()), + + // we need to execute this rule at the end of rewrite + // to avoid two consecutive same project appear when we do optimization. + topic("Others optimization", topDown( + new EliminateNotNull(), + new EliminateLimit(), + new EliminateFilter(), + new PruneOlapScanPartition(), + new SelectMaterializedIndexWithAggregate(), + new SelectMaterializedIndexWithoutAggregate(), + new PruneOlapScanTablet(), + new EliminateAggregate(), + new MergeSetOperations(), + new LimitPushDown(), + new BuildAggForUnion() + )), + + // this rule batch must keep at the end of rewrite to do some plan check + topic("Final rewrite and check", bottomUp( + new AdjustNullable(), + new ExpressionRewrite(CheckLegalityAfterRewrite.INSTANCE), + new CheckAfterRewrite() + )) + ); + + public NereidsRewriter(CascadesContext cascadesContext) { + super(cascadesContext); + } + + @Override + public List getJobs() { + return REWRITE_JOBS; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java new file mode 100644 index 0000000000..a05ad215a3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; + +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; + +/** + * Custom rewrite the plan. + */ +public class CustomRewriteJob implements RewriteJob { + private final RuleType ruleType; + + private final Supplier customRewriter; + + /** + * Constructor. + */ + public CustomRewriteJob(Supplier rewriter, RuleType ruleType) { + this.ruleType = Objects.requireNonNull(ruleType, "ruleType cannot be null"); + this.customRewriter = Objects.requireNonNull(rewriter, "customRewriter cannot be null"); + } + + @Override + public void execute(JobContext context) { + Set disableRules = Job.getDisableRules(context); + if (disableRules.contains(ruleType.name().toUpperCase(Locale.ROOT))) { + return; + } + Plan root = context.getCascadesContext().getRewritePlan(); + // COUNTER_TRACER.log(CounterEvent.of(Memo.get=-StateId(), CounterType.JOB_EXECUTION, group, logicalExpression, + // root)); + Plan rewrittenRoot = customRewriter.get().rewriteRoot(root, context); + + // don't remove this comment, it can help us to trace some bug when developing. + + // if (!root.deepEquals(rewrittenRoot)) { + // String traceBefore = root.treeString(); + // String traceAfter = root.treeString(); + // printTraceLog(ruleType, traceBefore, traceAfter); + // } + context.getCascadesContext().setRewritePlan(rewrittenRoot); + } + + @Override + public boolean isOnce() { + return false; + } + + private void printTraceLog(RuleType ruleType, String traceBefore, String traceAfter) { + System.out.println("========== " + getClass().getSimpleName() + " " + ruleType + + " ==========\nbefore:\n" + traceBefore + "\n\nafter:\n" + traceAfter + "\n"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java new file mode 100644 index 0000000000..8e625b638d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.JobType; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** PlanTreeRewriteBottomUpJob */ +public class PlanTreeRewriteBottomUpJob extends PlanTreeRewriteJob { + private static final String REWRITE_STATE_KEY = "rewrite_state"; + private RewriteJobContext rewriteJobContext; + private List rules; + + enum RewriteState { + ENSURE_CHILDREN_REWRITTEN, REWRITE_THIS, REWRITTEN + } + + public PlanTreeRewriteBottomUpJob(RewriteJobContext rewriteJobContext, JobContext context, List rules) { + super(JobType.TOP_DOWN_REWRITE, context); + this.rewriteJobContext = Objects.requireNonNull(rewriteJobContext, "rewriteContext cannot be null"); + this.rules = Objects.requireNonNull(rules, "rules cannot be null"); + } + + @Override + public void execute() { + // use childrenVisited to judge whether clear the state in the previous batch + boolean clearStatePhase = !rewriteJobContext.childrenVisited; + if (clearStatePhase) { + traverseClearState(); + return; + } + + Plan plan = rewriteJobContext.plan; + RewriteState state = getState(plan); + switch (state) { + case REWRITE_THIS: + rewriteThis(); + return; + case ENSURE_CHILDREN_REWRITTEN: + ensureChildrenRewritten(); + return; + case REWRITTEN: + rewriteJobContext.result = plan; + return; + default: + throw new IllegalStateException("Unknown rewrite state: " + state); + } + } + + private void traverseClearState() { + RewriteJobContext clearedStateContext = rewriteJobContext.withChildrenVisited(true); + setState(clearedStateContext.plan, RewriteState.REWRITE_THIS); + pushJob(new PlanTreeRewriteBottomUpJob(clearedStateContext, context, rules)); + + List children = clearedStateContext.plan.children(); + for (int i = children.size() - 1; i >= 0; i--) { + Plan child = children.get(i); + RewriteJobContext childRewriteJobContext = new RewriteJobContext( + child, clearedStateContext, i, false); + pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, rules)); + } + } + + private void rewriteThis() { + Plan plan = linkChildren(rewriteJobContext.plan, rewriteJobContext.childrenContext); + RewriteResult rewriteResult = rewrite(plan, rules, rewriteJobContext); + if (rewriteResult.hasNewPlan) { + RewriteJobContext newJobContext = rewriteJobContext.withPlan(rewriteResult.plan); + RewriteState state = getState(rewriteResult.plan); + // some eliminate rule will return a rewritten plan + if (state == RewriteState.REWRITTEN) { + newJobContext.setResult(rewriteResult.plan); + return; + } + pushJob(new PlanTreeRewriteBottomUpJob(newJobContext, context, rules)); + setState(rewriteResult.plan, RewriteState.ENSURE_CHILDREN_REWRITTEN); + } else { + setState(rewriteResult.plan, RewriteState.REWRITTEN); + rewriteJobContext.setResult(rewriteResult.plan); + } + } + + private void ensureChildrenRewritten() { + Plan plan = rewriteJobContext.plan; + setState(plan, RewriteState.REWRITE_THIS); + pushJob(new PlanTreeRewriteBottomUpJob(rewriteJobContext, context, rules)); + + List children = plan.children(); + for (int i = children.size() - 1; i >= 0; i--) { + Plan child = children.get(i); + // some rule return new plan tree, which the number of new plan node > 1, + // we should transform this new plan nodes too. + RewriteJobContext childRewriteJobContext = new RewriteJobContext( + child, rewriteJobContext, i, false); + pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, rules)); + } + } + + private static final RewriteState getState(Plan plan) { + Optional state = plan.getMutableState(REWRITE_STATE_KEY); + return state.orElse(RewriteState.ENSURE_CHILDREN_REWRITTEN); + } + + private static final void setState(Plan plan, RewriteState state) { + plan.setMutableState(REWRITE_STATE_KEY, state); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java new file mode 100644 index 0000000000..b54e6197d9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.JobType; +import org.apache.doris.nereids.pattern.Pattern; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.plans.Plan; + +import com.google.common.base.Preconditions; + +import java.util.List; + +/** PlanTreeRewriteJob */ +public abstract class PlanTreeRewriteJob extends Job { + public PlanTreeRewriteJob(JobType type, JobContext context) { + super(type, context); + } + + protected RewriteResult rewrite(Plan plan, List rules, RewriteJobContext rewriteJobContext) { + // boolean traceEnable = isTraceEnable(context); + boolean isRewriteRoot = rewriteJobContext.isRewriteRoot(); + CascadesContext cascadesContext = context.getCascadesContext(); + cascadesContext.setIsRewriteRoot(isRewriteRoot); + List validRules = getValidRules(rules); + for (Rule rule : validRules) { + Pattern pattern = (Pattern) rule.getPattern(); + if (pattern.matchPlanTree(plan)) { + List newPlans = rule.transform(plan, cascadesContext); + Preconditions.checkState(newPlans.size() == 1, + "Rewrite rule should generate one plan: " + rule.getRuleType()); + Plan newPlan = newPlans.get(0); + if (!newPlan.deepEquals(plan)) { + // don't remove this comment, it can help us to trace some bug when developing. + + // String traceBefore = null; + // if (traceEnable) { + // traceBefore = getCurrentPlanTreeString(); + // } + rewriteJobContext.result = newPlan; + context.setRewritten(true); + rule.acceptPlan(newPlan); + // if (traceEnable) { + // String traceAfter = getCurrentPlanTreeString(); + // printTraceLog(rule, traceBefore, traceAfter); + // } + return new RewriteResult(true, newPlan); + } + } + } + return new RewriteResult(false, plan); + } + + protected Plan linkChildrenAndParent(Plan plan, RewriteJobContext rewriteJobContext) { + Plan newPlan = linkChildren(plan, rewriteJobContext.childrenContext); + rewriteJobContext.setResult(newPlan); + return newPlan; + } + + protected Plan linkChildren(Plan plan, RewriteJobContext[] childrenContext) { + boolean changed = false; + Plan[] newChildren = new Plan[childrenContext.length]; + for (int i = 0; i < childrenContext.length; ++i) { + Plan result = childrenContext[i].result; + Plan oldChild = plan.child(i); + if (result != null && result != oldChild) { + newChildren[i] = result; + changed = true; + } else { + newChildren[i] = oldChild; + } + } + return changed ? plan.withChildren(newChildren) : plan; + } + + private String getCurrentPlanTreeString() { + return context.getCascadesContext() + .getCurrentRootRewriteJobContext().get() + .getNewestPlan() + .treeString(); + } + + private void printTraceLog(Rule rule, String traceBefore, String traceAfter) { + System.out.println("========== " + getClass().getSimpleName() + " " + rule.getRuleType() + + " ==========\nbefore:\n" + traceBefore + "\n\nafter:\n" + traceAfter + "\n"); + // LOGGER.info("========== {} {} ==========\nbefore:\n{}\n\nafter:\n{}\n", + // getClass().getSimpleName(), rule.getRuleType(), traceBefore, traceAfter); + } + + static class RewriteResult { + final boolean hasNewPlan; + final Plan plan; + + public RewriteResult(boolean hasNewPlan, Plan plan) { + this.hasNewPlan = hasNewPlan; + this.plan = plan; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java new file mode 100644 index 0000000000..e92dd15f27 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.JobType; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; +import java.util.Objects; + +/** PlanTreeRewriteTopDownJob */ +public class PlanTreeRewriteTopDownJob extends PlanTreeRewriteJob { + private RewriteJobContext rewriteJobContext; + private List rules; + + public PlanTreeRewriteTopDownJob(RewriteJobContext rewriteJobContext, JobContext context, List rules) { + super(JobType.TOP_DOWN_REWRITE, context); + this.rewriteJobContext = Objects.requireNonNull(rewriteJobContext, "rewriteContext cannot be null"); + this.rules = Objects.requireNonNull(rules, "rules cannot be null"); + } + + @Override + public void execute() { + if (!rewriteJobContext.childrenVisited) { + RewriteResult rewriteResult = rewrite(rewriteJobContext.plan, rules, rewriteJobContext); + if (rewriteResult.hasNewPlan) { + RewriteJobContext newContext = rewriteJobContext + .withPlanAndChildrenVisited(rewriteResult.plan, false); + pushJob(new PlanTreeRewriteTopDownJob(newContext, context, rules)); + return; + } + + RewriteJobContext newRewriteJobContext = rewriteJobContext.withChildrenVisited(true); + pushJob(new PlanTreeRewriteTopDownJob(newRewriteJobContext, context, rules)); + + List children = newRewriteJobContext.plan.children(); + for (int i = children.size() - 1; i >= 0; i--) { + RewriteJobContext childRewriteJobContext = new RewriteJobContext( + children.get(i), newRewriteJobContext, i, false); + pushJob(new PlanTreeRewriteTopDownJob(childRewriteJobContext, context, rules)); + } + } else { + Plan result = linkChildrenAndParent(rewriteJobContext.plan, rewriteJobContext); + if (rewriteJobContext.parentContext == null) { + context.getCascadesContext().setRewritePlan(result); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteJobContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteJobContext.java new file mode 100644 index 0000000000..32af07573d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteJobContext.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.trees.plans.Plan; + +import javax.annotation.Nullable; + +/** RewriteJobContext */ +public class RewriteJobContext { + final boolean childrenVisited; + final RewriteJobContext parentContext; + final int childIndexInParentContext; + final Plan plan; + final RewriteJobContext[] childrenContext; + Plan result; + + /** RewriteJobContext */ + public RewriteJobContext(Plan plan, @Nullable RewriteJobContext parentContext, int childIndexInParentContext, + boolean childrenVisited) { + this.plan = plan; + this.parentContext = parentContext; + this.childIndexInParentContext = childIndexInParentContext; + this.childrenVisited = childrenVisited; + this.childrenContext = new RewriteJobContext[plan.arity()]; + if (parentContext != null) { + parentContext.childrenContext[childIndexInParentContext] = this; + } + } + + public void setResult(Plan result) { + this.result = result; + } + + public RewriteJobContext withChildrenVisited(boolean childrenVisited) { + return new RewriteJobContext(plan, parentContext, childIndexInParentContext, childrenVisited); + } + + public RewriteJobContext withPlan(Plan plan) { + return new RewriteJobContext(plan, parentContext, childIndexInParentContext, childrenVisited); + } + + public RewriteJobContext withPlanAndChildrenVisited(Plan plan, boolean childrenVisited) { + return new RewriteJobContext(plan, parentContext, childIndexInParentContext, childrenVisited); + } + + public boolean isRewriteRoot() { + return false; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java new file mode 100644 index 0000000000..06a9904799 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.rewrite; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.JobType; +import org.apache.doris.nereids.jobs.RewriteJob; +import org.apache.doris.nereids.jobs.scheduler.JobStack; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; +import java.util.Objects; + +/** RootPlanTreeRewriteJob */ +public class RootPlanTreeRewriteJob implements RewriteJob { + private final List rules; + private final RewriteJobBuilder rewriteJobBuilder; + private final boolean once; + + public RootPlanTreeRewriteJob(List rules, RewriteJobBuilder rewriteJobBuilder, boolean once) { + this.rules = Objects.requireNonNull(rules, "rules cannot be null"); + this.rewriteJobBuilder = Objects.requireNonNull(rewriteJobBuilder, "rewriteJobBuilder cannot be null"); + this.once = once; + } + + @Override + public void execute(JobContext context) { + CascadesContext cascadesContext = context.getCascadesContext(); + // get plan from the cascades context + Plan root = cascadesContext.getRewritePlan(); + // write rewritten root plan to cascades context by the RootRewriteJobContext + RootRewriteJobContext rewriteJobContext = new RootRewriteJobContext(root, false, context); + Job rewriteJob = rewriteJobBuilder.build(rewriteJobContext, context, rules); + + context.getScheduleContext().pushJob(rewriteJob); + cascadesContext.getJobScheduler().executeJobPool(cascadesContext); + + cascadesContext.setCurrentRootRewriteJobContext(null); + } + + @Override + public boolean isOnce() { + return once; + } + + /** RewriteJobBuilder */ + public interface RewriteJobBuilder { + Job build(RewriteJobContext rewriteJobContext, JobContext jobContext, List rules); + } + + /** RootRewriteJobContext */ + public static class RootRewriteJobContext extends RewriteJobContext { + private JobContext jobContext; + + RootRewriteJobContext(Plan plan, boolean childrenVisited, JobContext jobContext) { + super(plan, null, -1, childrenVisited); + this.jobContext = Objects.requireNonNull(jobContext, "jobContext cannot be null"); + jobContext.getCascadesContext().setCurrentRootRewriteJobContext(this); + } + + @Override + public boolean isRewriteRoot() { + return true; + } + + @Override + public void setResult(Plan result) { + jobContext.getCascadesContext().setRewritePlan(result); + } + + @Override + public RewriteJobContext withChildrenVisited(boolean childrenVisited) { + return new RootRewriteJobContext(plan, childrenVisited, jobContext); + } + + @Override + public RewriteJobContext withPlan(Plan plan) { + return new RootRewriteJobContext(plan, childrenVisited, jobContext); + } + + @Override + public RewriteJobContext withPlanAndChildrenVisited(Plan plan, boolean childrenVisited) { + return new RootRewriteJobContext(plan, childrenVisited, jobContext); + } + + /** linkChildren */ + public Plan getNewestPlan() { + JobStack jobStack = new JobStack(); + LinkPlanJob linkPlanJob = new LinkPlanJob( + jobContext, this, null, false, jobStack); + jobStack.push(linkPlanJob); + while (!jobStack.isEmpty()) { + Job job = jobStack.pop(); + job.execute(); + } + return linkPlanJob.result; + } + } + + /** use to assemble the rewriting plan */ + private static class LinkPlanJob extends Job { + LinkPlanJob parentJob; + RewriteJobContext rewriteJobContext; + Plan[] childrenResult; + Plan result; + boolean linked; + JobStack jobStack; + + private LinkPlanJob(JobContext context, RewriteJobContext rewriteJobContext, + LinkPlanJob parentJob, boolean linked, JobStack jobStack) { + super(JobType.LINK_PLAN, context); + this.rewriteJobContext = rewriteJobContext; + this.parentJob = parentJob; + this.linked = linked; + this.childrenResult = new Plan[rewriteJobContext.plan.arity()]; + this.jobStack = jobStack; + } + + @Override + public void execute() { + if (!linked) { + linked = true; + jobStack.push(this); + for (int i = rewriteJobContext.childrenContext.length - 1; i >= 0; i--) { + RewriteJobContext childContext = rewriteJobContext.childrenContext[i]; + if (childContext != null) { + jobStack.push(new LinkPlanJob(context, childContext, this, false, jobStack)); + } + } + } else if (rewriteJobContext.result != null) { + linkResult(rewriteJobContext.result); + } else { + Plan[] newChildren = new Plan[childrenResult.length]; + for (int i = 0; i < newChildren.length; i++) { + Plan childResult = childrenResult[i]; + if (childResult == null) { + childResult = rewriteJobContext.plan.child(i); + } + newChildren[i] = childResult; + } + linkResult(rewriteJobContext.plan.withChildren(newChildren)); + } + } + + private void linkResult(Plan result) { + if (parentJob != null) { + parentJob.childrenResult[rewriteJobContext.childIndexInParentContext] = result; + } else { + this.result = result; + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/VisitorRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/VisitorRewriteJob.java deleted file mode 100644 index 4836803be1..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/VisitorRewriteJob.java +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.rewrite; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.Job; -import org.apache.doris.nereids.jobs.JobContext; -import org.apache.doris.nereids.jobs.JobType; -import org.apache.doris.nereids.memo.Group; -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.memo.Memo; -import org.apache.doris.nereids.metrics.CounterType; -import org.apache.doris.nereids.metrics.event.CounterEvent; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; - -import java.util.Locale; -import java.util.Objects; - -/** - * Use visitor to rewrite the plan. - */ -public class VisitorRewriteJob extends Job { - private final RuleType ruleType; - private final Group group; - - private final DefaultPlanRewriter planRewriter; - - /** - * Constructor. - */ - public VisitorRewriteJob(CascadesContext cascadesContext, - DefaultPlanRewriter rewriter, RuleType ruleType) { - super(JobType.VISITOR_REWRITE, cascadesContext.getCurrentJobContext(), true); - this.ruleType = Objects.requireNonNull(ruleType, "ruleType cannot be null"); - this.group = Objects.requireNonNull(cascadesContext.getMemo().getRoot(), "group cannot be null"); - this.planRewriter = Objects.requireNonNull(rewriter, "planRewriter cannot be null"); - } - - @Override - public void execute() { - if (disableRules.contains(ruleType.name().toUpperCase(Locale.ROOT))) { - return; - } - GroupExpression logicalExpression = group.getLogicalExpression(); - Plan root = context.getCascadesContext().getMemo().copyOut(logicalExpression, true); - COUNTER_TRACER.log(CounterEvent.of(Memo.getStateId(), CounterType.JOB_EXECUTION, group, logicalExpression, - root)); - Plan rewrittenRoot = root.accept(planRewriter, context); - context.getCascadesContext().getMemo().copyIn(rewrittenRoot, group, true); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/JobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/JobScheduler.java index 97dcd96d52..4b26c0decf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/JobScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/JobScheduler.java @@ -17,15 +17,9 @@ package org.apache.doris.nereids.jobs.scheduler; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.jobs.Job; - /** * Scheduler to schedule jobs in Nereids. */ public interface JobScheduler { - void executeJob(Job job, CascadesContext context); - - void executeJobPool(CascadesContext cascadesContext) throws AnalysisException; + void executeJobPool(ScheduleContext scheduleContext); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeSubqueryRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/ScheduleContext.java similarity index 60% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeSubqueryRulesJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/ScheduleContext.java index 1e41652e2b..f7f60119f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeSubqueryRulesJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/ScheduleContext.java @@ -15,20 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.jobs.batch; +package org.apache.doris.nereids.jobs.scheduler; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.analysis.AnalyzeSubquery; +import org.apache.doris.nereids.jobs.Job; -import com.google.common.collect.ImmutableList; +/** ScheduleContext */ +public interface ScheduleContext { -/** - * Analyze subquery. - */ -public class AnalyzeSubqueryRulesJob extends BatchRulesJob { - public AnalyzeSubqueryRulesJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - bottomUpBatch(ImmutableList.of(new AnalyzeSubquery())))); - } + JobPool getJobPool(); + + void pushJob(Job job); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java index 8bd104c010..a93270d48e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java @@ -17,8 +17,6 @@ package org.apache.doris.nereids.jobs.scheduler; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.jobs.Job; /** @@ -26,13 +24,8 @@ import org.apache.doris.nereids.jobs.Job; */ public class SimpleJobScheduler implements JobScheduler { @Override - public void executeJob(Job job, CascadesContext context) { - - } - - @Override - public void executeJobPool(CascadesContext cascadesContext) throws AnalysisException { - JobPool pool = cascadesContext.getJobPool(); + public void executeJobPool(ScheduleContext scheduleContext) { + JobPool pool = scheduleContext.getJobPool(); while (!pool.isEmpty()) { Job job = pool.pop(); job.execute(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index bd53cb86df..88e20f34e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -393,7 +393,16 @@ public class Group { @Override public String toString() { - return "Group[" + groupId + "]"; + StringBuilder str = new StringBuilder("Group[" + groupId + "]\n"); + str.append("logical expressions:\n"); + for (GroupExpression logicalExpression : logicalExpressions) { + str.append(" ").append(logicalExpression).append("\n"); + } + str.append("physical expressions:\n"); + for (GroupExpression physicalExpression : physicalExpressions) { + str.append(" ").append(physicalExpression).append("\n"); + } + return str.toString(); } /** @@ -404,7 +413,7 @@ public class Group { public String treeString() { Function toString = obj -> { if (obj instanceof Group) { - return obj.toString(); + return "Group[" + ((Group) obj).groupId + "]"; } else if (obj instanceof GroupExpression) { return ((GroupExpression) obj).getPlan().toString(); } else if (obj instanceof Pair) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index a64db02847..304a2da025 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -216,11 +216,11 @@ public class Memo { * Utility function to create a new {@link CascadesContext} with this Memo. */ public CascadesContext newCascadesContext(StatementContext statementContext) { - return new CascadesContext(this, statementContext, PhysicalProperties.ANY); + return new CascadesContext(null, this, statementContext, PhysicalProperties.ANY); } public CascadesContext newCascadesContext(StatementContext statementContext, CTEContext cteContext) { - return new CascadesContext(this, statementContext, cteContext, PhysicalProperties.ANY); + return new CascadesContext(null, this, statementContext, cteContext, PhysicalProperties.ANY); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java index de067a427d..e889159d63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MatchingContext.java @@ -49,4 +49,8 @@ public class MatchingContext { this.connectContext = cascadesContext.getConnectContext(); this.cteContext = cascadesContext.getCteContext(); } + + public boolean isRewriteRoot() { + return cascadesContext.isRewriteRoot(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java new file mode 100644 index 0000000000..fa9d191f5e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/MemoPatterns.java @@ -0,0 +1,322 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.pattern; + +import org.apache.doris.nereids.trees.plans.BinaryPlan; +import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.LeafPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.UnaryPlan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalBinary; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; +import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnary; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalBinary; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnary; + +import java.util.Arrays; + +/** MemoPatterns */ +public interface MemoPatterns extends Patterns { + + default PatternDescriptor group() { + return new PatternDescriptor<>(Pattern.GROUP, defaultPromise()); + } + + default PatternDescriptor multiGroup() { + return new PatternDescriptor<>(Pattern.MULTI_GROUP, defaultPromise()); + } + + /* abstract plan operator patterns */ + + /** + * create a leafPlan pattern. + */ + default PatternDescriptor leafPlan() { + return new PatternDescriptor(new TypePattern(LeafPlan.class), defaultPromise()); + } + + /** + * create a unaryPlan pattern. + */ + default PatternDescriptor> unaryPlan() { + return new PatternDescriptor(new TypePattern(UnaryPlan.class, Pattern.GROUP), defaultPromise()); + } + + /** + * create a unaryPlan pattern. + */ + default PatternDescriptor> + unaryPlan(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(UnaryPlan.class, child.pattern), defaultPromise()); + } + + /** + * create a binaryPlan pattern. + */ + default PatternDescriptor> binaryPlan() { + return new PatternDescriptor( + new TypePattern(BinaryPlan.class, Pattern.GROUP, Pattern.GROUP), + defaultPromise() + ); + } + + /** + * create a binaryPlan pattern. + */ + default + PatternDescriptor> binaryPlan( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(BinaryPlan.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /* abstract logical plan patterns */ + + /** + * create a logicalPlan pattern. + */ + default PatternDescriptor logicalPlan() { + return new PatternDescriptor(new TypePattern(LogicalPlan.class, multiGroup().pattern), defaultPromise()); + } + + /** + * create a logicalLeaf pattern. + */ + default PatternDescriptor logicalLeaf() { + return new PatternDescriptor(new TypePattern(LogicalLeaf.class), defaultPromise()); + } + + /** + * create a logicalUnary pattern. + */ + default PatternDescriptor> logicalUnary() { + return new PatternDescriptor(new TypePattern(LogicalUnary.class, Pattern.GROUP), defaultPromise()); + } + + /** + * create a logicalUnary pattern. + */ + default PatternDescriptor> + logicalUnary(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(LogicalUnary.class, child.pattern), defaultPromise()); + } + + /** + * create a logicalBinary pattern. + */ + default PatternDescriptor> logicalBinary() { + return new PatternDescriptor( + new TypePattern(LogicalBinary.class, Pattern.GROUP, Pattern.GROUP), + defaultPromise() + ); + } + + /** + * create a logicalBinary pattern. + */ + default + PatternDescriptor> + logicalBinary( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(LogicalBinary.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /** + * create a logicalRelation pattern. + */ + default PatternDescriptor logicalRelation() { + return new PatternDescriptor(new TypePattern(LogicalRelation.class), defaultPromise()); + } + + /** + * create a logicalSetOperation pattern. + */ + default PatternDescriptor + logicalSetOperation( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalSetOperation group. + */ + default PatternDescriptor logicalSetOperation() { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, multiGroup().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalUnion( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion group. + */ + default PatternDescriptor logicalUnion() { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, multiGroup().pattern), + defaultPromise()); + } + + /** + * create a logicalExcept pattern. + */ + default PatternDescriptor + logicalExcept( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalExcept group. + */ + default PatternDescriptor logicalExcept() { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, multiGroup().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalIntersect( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion group. + */ + default PatternDescriptor logicalIntersect() { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, multiGroup().pattern), + defaultPromise()); + } + + /* abstract physical plan patterns */ + + /** + * create a physicalLeaf pattern. + */ + default PatternDescriptor physicalLeaf() { + return new PatternDescriptor(new TypePattern(PhysicalLeaf.class), defaultPromise()); + } + + /** + * create a physicalUnary pattern. + */ + default PatternDescriptor> physicalUnary() { + return new PatternDescriptor(new TypePattern(PhysicalUnary.class, Pattern.GROUP), defaultPromise()); + } + + /** + * create a physicalUnary pattern. + */ + default PatternDescriptor> + physicalUnary(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(PhysicalUnary.class, child.pattern), defaultPromise()); + } + + /** + * create a physicalBinary pattern. + */ + default PatternDescriptor> physicalBinary() { + return new PatternDescriptor( + new TypePattern(PhysicalBinary.class, Pattern.GROUP, Pattern.GROUP), + defaultPromise() + ); + } + + /** + * create a physicalBinary pattern. + */ + default + PatternDescriptor> + physicalBinary( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(PhysicalBinary.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /** + * create a physicalRelation pattern. + */ + default PatternDescriptor physicalRelation() { + return new PatternDescriptor(new TypePattern(PhysicalRelation.class), defaultPromise()); + } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor> aggregate() { + return new PatternDescriptor(new TypePattern(Aggregate.class, Pattern.GROUP), defaultPromise()); + } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor> aggregate(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(Aggregate.class, child.pattern), defaultPromise()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java index 31c2713aa3..c47dcd6a72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Pattern.java @@ -75,7 +75,7 @@ public class Pattern * @param predicates custom matching predicate * @param children sub pattern */ - private Pattern(PatternType patternType, PlanType planType, + protected Pattern(PatternType patternType, PlanType planType, List> predicates, Pattern... children) { super(children); this.patternType = patternType; @@ -134,6 +134,35 @@ public class Pattern return patternType == PatternType.MULTI; } + /** matchPlan */ + public boolean matchPlanTree(Plan plan) { + if (!matchRoot(plan)) { + return false; + } + int childPatternNum = arity(); + if (childPatternNum != plan.arity() && childPatternNum > 0 && child(childPatternNum - 1) != MULTI) { + return false; + } + switch (patternType) { + case ANY: + case MULTI: + return matchPredicates((TYPE) plan); + default: + } + if (this instanceof SubTreePattern) { + return matchPredicates((TYPE) plan); + } + List childrenPlan = plan.children(); + for (int i = 0; i < childrenPlan.size(); i++) { + Plan child = childrenPlan.get(i); + Pattern childPattern = child(Math.min(i, childPatternNum - 1)); + if (!childPattern.matchPlanTree(child)) { + return false; + } + } + return matchPredicates((TYPE) plan); + } + /** * Return ture if current Pattern match Plan in params. * @@ -161,7 +190,13 @@ public class Pattern * @return true if all predicates matched */ public boolean matchPredicates(TYPE root) { - return predicates.stream().allMatch(predicate -> predicate.test(root)); + // use loop to speed up + for (Predicate predicate : predicates) { + if (!predicate.test(root)) { + return false; + } + } + return true; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java index 765c087f19..149677c617 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java @@ -18,27 +18,7 @@ package org.apache.doris.nereids.pattern; import org.apache.doris.nereids.rules.RulePromise; -import org.apache.doris.nereids.trees.plans.BinaryPlan; -import org.apache.doris.nereids.trees.plans.GroupPlan; -import org.apache.doris.nereids.trees.plans.LeafPlan; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.UnaryPlan; -import org.apache.doris.nereids.trees.plans.algebra.Aggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalBinary; -import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; -import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; -import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; -import org.apache.doris.nereids.trees.plans.logical.LogicalUnary; -import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; -import org.apache.doris.nereids.trees.plans.physical.PhysicalBinary; -import org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf; -import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; -import org.apache.doris.nereids.trees.plans.physical.PhysicalUnary; - -import java.util.Arrays; /** * An interface provided some PatternDescriptor. @@ -59,285 +39,7 @@ public interface Patterns { return new PatternDescriptor<>(Pattern.MULTI, defaultPromise()); } - default PatternDescriptor group() { - return new PatternDescriptor<>(Pattern.GROUP, defaultPromise()); - } - - default PatternDescriptor multiGroup() { - return new PatternDescriptor<>(Pattern.MULTI_GROUP, defaultPromise()); - } - default PatternDescriptor subTree(Class... subTreeNodeTypes) { return new PatternDescriptor<>(new SubTreePattern(subTreeNodeTypes), defaultPromise()); } - - /* abstract plan operator patterns */ - - /** - * create a leafPlan pattern. - */ - default PatternDescriptor leafPlan() { - return new PatternDescriptor(new TypePattern(LeafPlan.class), defaultPromise()); - } - - /** - * create a unaryPlan pattern. - */ - default PatternDescriptor> unaryPlan() { - return new PatternDescriptor(new TypePattern(UnaryPlan.class, Pattern.GROUP), defaultPromise()); - } - - /** - * create a unaryPlan pattern. - */ - default PatternDescriptor> - unaryPlan(PatternDescriptor child) { - return new PatternDescriptor(new TypePattern(UnaryPlan.class, child.pattern), defaultPromise()); - } - - /** - * create a binaryPlan pattern. - */ - default PatternDescriptor> binaryPlan() { - return new PatternDescriptor( - new TypePattern(BinaryPlan.class, Pattern.GROUP, Pattern.GROUP), - defaultPromise() - ); - } - - /** - * create a binaryPlan pattern. - */ - default - PatternDescriptor> binaryPlan( - PatternDescriptor leftChild, - PatternDescriptor rightChild) { - return new PatternDescriptor( - new TypePattern(BinaryPlan.class, leftChild.pattern, rightChild.pattern), - defaultPromise() - ); - } - - /* abstract logical plan patterns */ - - /** - * create a logicalPlan pattern. - */ - default PatternDescriptor logicalPlan() { - return new PatternDescriptor(new TypePattern(LogicalPlan.class, multiGroup().pattern), defaultPromise()); - } - - /** - * create a logicalLeaf pattern. - */ - default PatternDescriptor logicalLeaf() { - return new PatternDescriptor(new TypePattern(LogicalLeaf.class), defaultPromise()); - } - - /** - * create a logicalUnary pattern. - */ - default PatternDescriptor> logicalUnary() { - return new PatternDescriptor(new TypePattern(LogicalUnary.class, Pattern.GROUP), defaultPromise()); - } - - /** - * create a logicalUnary pattern. - */ - default PatternDescriptor> - logicalUnary(PatternDescriptor child) { - return new PatternDescriptor(new TypePattern(LogicalUnary.class, child.pattern), defaultPromise()); - } - - /** - * create a logicalBinary pattern. - */ - default PatternDescriptor> logicalBinary() { - return new PatternDescriptor( - new TypePattern(LogicalBinary.class, Pattern.GROUP, Pattern.GROUP), - defaultPromise() - ); - } - - /** - * create a logicalBinary pattern. - */ - default - PatternDescriptor> - logicalBinary( - PatternDescriptor leftChild, - PatternDescriptor rightChild) { - return new PatternDescriptor( - new TypePattern(LogicalBinary.class, leftChild.pattern, rightChild.pattern), - defaultPromise() - ); - } - - /** - * create a logicalRelation pattern. - */ - default PatternDescriptor logicalRelation() { - return new PatternDescriptor(new TypePattern(LogicalRelation.class), defaultPromise()); - } - - /** - * create a logicalSetOperation pattern. - */ - default PatternDescriptor - logicalSetOperation( - PatternDescriptor... children) { - return new PatternDescriptor( - new TypePattern(LogicalSetOperation.class, - Arrays.stream(children) - .map(PatternDescriptor::getPattern) - .toArray(Pattern[]::new)), - defaultPromise()); - } - - /** - * create a logicalSetOperation group. - */ - default PatternDescriptor logicalSetOperation() { - return new PatternDescriptor( - new TypePattern(LogicalSetOperation.class, multiGroup().pattern), - defaultPromise()); - } - - /** - * create a logicalUnion pattern. - */ - default PatternDescriptor - logicalUnion( - PatternDescriptor... children) { - return new PatternDescriptor( - new TypePattern(LogicalUnion.class, - Arrays.stream(children) - .map(PatternDescriptor::getPattern) - .toArray(Pattern[]::new)), - defaultPromise()); - } - - /** - * create a logicalUnion group. - */ - default PatternDescriptor logicalUnion() { - return new PatternDescriptor( - new TypePattern(LogicalUnion.class, multiGroup().pattern), - defaultPromise()); - } - - /** - * create a logicalExcept pattern. - */ - default PatternDescriptor - logicalExcept( - PatternDescriptor... children) { - return new PatternDescriptor( - new TypePattern(LogicalExcept.class, - Arrays.stream(children) - .map(PatternDescriptor::getPattern) - .toArray(Pattern[]::new)), - defaultPromise()); - } - - /** - * create a logicalExcept group. - */ - default PatternDescriptor logicalExcept() { - return new PatternDescriptor( - new TypePattern(LogicalExcept.class, multiGroup().pattern), - defaultPromise()); - } - - /** - * create a logicalUnion pattern. - */ - default PatternDescriptor - logicalIntersect( - PatternDescriptor... children) { - return new PatternDescriptor( - new TypePattern(LogicalIntersect.class, - Arrays.stream(children) - .map(PatternDescriptor::getPattern) - .toArray(Pattern[]::new)), - defaultPromise()); - } - - /** - * create a logicalUnion group. - */ - default PatternDescriptor logicalIntersect() { - return new PatternDescriptor( - new TypePattern(LogicalIntersect.class, multiGroup().pattern), - defaultPromise()); - } - - /* abstract physical plan patterns */ - - /** - * create a physicalLeaf pattern. - */ - default PatternDescriptor physicalLeaf() { - return new PatternDescriptor(new TypePattern(PhysicalLeaf.class), defaultPromise()); - } - - /** - * create a physicalUnary pattern. - */ - default PatternDescriptor> physicalUnary() { - return new PatternDescriptor(new TypePattern(PhysicalUnary.class, Pattern.GROUP), defaultPromise()); - } - - /** - * create a physicalUnary pattern. - */ - default PatternDescriptor> - physicalUnary(PatternDescriptor child) { - return new PatternDescriptor(new TypePattern(PhysicalUnary.class, child.pattern), defaultPromise()); - } - - /** - * create a physicalBinary pattern. - */ - default PatternDescriptor> physicalBinary() { - return new PatternDescriptor( - new TypePattern(PhysicalBinary.class, Pattern.GROUP, Pattern.GROUP), - defaultPromise() - ); - } - - /** - * create a physicalBinary pattern. - */ - default - PatternDescriptor> - physicalBinary( - PatternDescriptor leftChild, - PatternDescriptor rightChild) { - return new PatternDescriptor( - new TypePattern(PhysicalBinary.class, leftChild.pattern, rightChild.pattern), - defaultPromise() - ); - } - - /** - * create a physicalRelation pattern. - */ - default PatternDescriptor physicalRelation() { - return new PatternDescriptor(new TypePattern(PhysicalRelation.class), defaultPromise()); - } - - /** - * create a aggregate pattern. - */ - default PatternDescriptor> aggregate() { - return new PatternDescriptor(new TypePattern(Aggregate.class, Pattern.GROUP), defaultPromise()); - } - - /** - * create a aggregate pattern. - */ - default PatternDescriptor> aggregate(PatternDescriptor child) { - return new PatternDescriptor(new TypePattern(Aggregate.class, child.pattern), defaultPromise()); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PlanPatterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PlanPatterns.java new file mode 100644 index 0000000000..ffe096890a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PlanPatterns.java @@ -0,0 +1,314 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.pattern; + +import org.apache.doris.nereids.trees.plans.BinaryPlan; +import org.apache.doris.nereids.trees.plans.LeafPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.UnaryPlan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalBinary; +import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; +import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnary; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalBinary; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnary; + +import java.util.Arrays; + +/** PlanPatterns */ +public interface PlanPatterns extends Patterns { + + /* abstract plan operator patterns */ + + /** + * create a leafPlan pattern. + */ + default PatternDescriptor leafPlan() { + return new PatternDescriptor(new TypePattern(LeafPlan.class), defaultPromise()); + } + + /** + * create a unaryPlan pattern. + */ + default PatternDescriptor> unaryPlan() { + return new PatternDescriptor(new TypePattern(UnaryPlan.class, Pattern.ANY), defaultPromise()); + } + + /** + * create a unaryPlan pattern. + */ + default PatternDescriptor> + unaryPlan(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(UnaryPlan.class, child.pattern), defaultPromise()); + } + + /** + * create a binaryPlan pattern. + */ + default PatternDescriptor> binaryPlan() { + return new PatternDescriptor( + new TypePattern(BinaryPlan.class, Pattern.ANY, Pattern.ANY), + defaultPromise() + ); + } + + /** + * create a binaryPlan pattern. + */ + default + PatternDescriptor> binaryPlan( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(BinaryPlan.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /* abstract logical plan patterns */ + + /** + * create a logicalPlan pattern. + */ + default PatternDescriptor logicalPlan() { + return new PatternDescriptor(new TypePattern(LogicalPlan.class, multi().pattern), defaultPromise()); + } + + /** + * create a logicalLeaf pattern. + */ + default PatternDescriptor logicalLeaf() { + return new PatternDescriptor(new TypePattern(LogicalLeaf.class), defaultPromise()); + } + + /** + * create a logicalUnary pattern. + */ + default PatternDescriptor> logicalUnary() { + return new PatternDescriptor(new TypePattern(LogicalUnary.class, Pattern.ANY), defaultPromise()); + } + + /** + * create a logicalUnary pattern. + */ + default PatternDescriptor> + logicalUnary(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(LogicalUnary.class, child.pattern), defaultPromise()); + } + + /** + * create a logicalBinary pattern. + */ + default PatternDescriptor> logicalBinary() { + return new PatternDescriptor( + new TypePattern(LogicalBinary.class, Pattern.ANY, Pattern.ANY), + defaultPromise() + ); + } + + /** + * create a logicalBinary pattern. + */ + default + PatternDescriptor> + logicalBinary( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(LogicalBinary.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /** + * create a logicalRelation pattern. + */ + default PatternDescriptor logicalRelation() { + return new PatternDescriptor(new TypePattern(LogicalRelation.class), defaultPromise()); + } + + /** + * create a logicalSetOperation pattern. + */ + default PatternDescriptor + logicalSetOperation( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalSetOperation multi. + */ + default PatternDescriptor logicalSetOperation() { + return new PatternDescriptor( + new TypePattern(LogicalSetOperation.class, multi().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalUnion( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion multi. + */ + default PatternDescriptor logicalUnion() { + return new PatternDescriptor( + new TypePattern(LogicalUnion.class, multi().pattern), + defaultPromise()); + } + + /** + * create a logicalExcept pattern. + */ + default PatternDescriptor + logicalExcept( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalExcept multi. + */ + default PatternDescriptor logicalExcept() { + return new PatternDescriptor( + new TypePattern(LogicalExcept.class, multi().pattern), + defaultPromise()); + } + + /** + * create a logicalUnion pattern. + */ + default PatternDescriptor + logicalIntersect( + PatternDescriptor... children) { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, + Arrays.stream(children) + .map(PatternDescriptor::getPattern) + .toArray(Pattern[]::new)), + defaultPromise()); + } + + /** + * create a logicalUnion multi. + */ + default PatternDescriptor logicalIntersect() { + return new PatternDescriptor( + new TypePattern(LogicalIntersect.class, multi().pattern), + defaultPromise()); + } + + /* abstract physical plan patterns */ + + /** + * create a physicalLeaf pattern. + */ + default PatternDescriptor physicalLeaf() { + return new PatternDescriptor(new TypePattern(PhysicalLeaf.class), defaultPromise()); + } + + /** + * create a physicalUnary pattern. + */ + default PatternDescriptor> physicalUnary() { + return new PatternDescriptor(new TypePattern(PhysicalUnary.class, Pattern.ANY), defaultPromise()); + } + + /** + * create a physicalUnary pattern. + */ + default PatternDescriptor> + physicalUnary(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(PhysicalUnary.class, child.pattern), defaultPromise()); + } + + /** + * create a physicalBinary pattern. + */ + default PatternDescriptor> physicalBinary() { + return new PatternDescriptor( + new TypePattern(PhysicalBinary.class, Pattern.ANY, Pattern.ANY), + defaultPromise() + ); + } + + /** + * create a physicalBinary pattern. + */ + default + PatternDescriptor> + physicalBinary( + PatternDescriptor leftChild, + PatternDescriptor rightChild) { + return new PatternDescriptor( + new TypePattern(PhysicalBinary.class, leftChild.pattern, rightChild.pattern), + defaultPromise() + ); + } + + /** + * create a physicalRelation pattern. + */ + default PatternDescriptor physicalRelation() { + return new PatternDescriptor(new TypePattern(PhysicalRelation.class), defaultPromise()); + } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor> aggregate() { + return new PatternDescriptor(new TypePattern(Aggregate.class, Pattern.ANY), defaultPromise()); + } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor> aggregate(PatternDescriptor child) { + return new PatternDescriptor(new TypePattern(Aggregate.class, child.pattern), defaultPromise()); + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ProxyPattern.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ProxyPattern.java new file mode 100644 index 0000000000..9f6f009371 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ProxyPattern.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.pattern; + +import org.apache.doris.nereids.trees.plans.Plan; + +/** ProxyPattern */ +public class ProxyPattern extends Pattern { + protected final Pattern pattern; + + public ProxyPattern(Pattern pattern) { + super(pattern.getPlanType(), pattern.children()); + this.pattern = pattern; + } + + @Override + public boolean matchPlanTree(Plan plan) { + return pattern.matchPlanTree(plan); + } + + @Override + public boolean matchRoot(Plan plan) { + return pattern.matchRoot(plan); + } + + @Override + public boolean matchPredicates(TYPE root) { + return pattern.matchPredicates(root); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalBinaryPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalBinaryPatternGenerator.java index ff47009f6c..bec3efa270 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalBinaryPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalBinaryPatternGenerator.java @@ -26,13 +26,13 @@ import java.util.TreeSet; public class LogicalBinaryPatternGenerator extends PatternGenerator { public LogicalBinaryPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override public String genericType() { - return "<" + opType.name + ">"; + return "<" + opType.name + "<" + childType() + ", " + childType() + ">>"; } @Override @@ -44,7 +44,9 @@ public class LogicalBinaryPatternGenerator extends PatternGenerator { public Set getImports() { Set imports = new TreeSet<>(); imports.add(opType.getFullQualifiedName()); - imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + if (isMemoPattern) { + imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + } imports.add("org.apache.doris.nereids.trees.plans.Plan"); enumFieldPatternInfos.stream() .map(info -> info.enumFullName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalLeafPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalLeafPatternGenerator.java index 19580c6e37..fd7b30a8e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalLeafPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalLeafPatternGenerator.java @@ -26,8 +26,8 @@ import java.util.TreeSet; public class LogicalLeafPatternGenerator extends PatternGenerator { public LogicalLeafPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalUnaryPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalUnaryPatternGenerator.java index 7f23098e1e..8ecb7c14e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalUnaryPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/LogicalUnaryPatternGenerator.java @@ -26,13 +26,13 @@ import java.util.TreeSet; public class LogicalUnaryPatternGenerator extends PatternGenerator { public LogicalUnaryPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override public String genericType() { - return "<" + opType.name + ">"; + return "<" + opType.name + "<" + childType() + ">>"; } @Override @@ -44,7 +44,9 @@ public class LogicalUnaryPatternGenerator extends PatternGenerator { public Set getImports() { Set imports = new TreeSet<>(); imports.add(opType.getFullQualifiedName()); - imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + if (isMemoPattern) { + imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + } imports.add("org.apache.doris.nereids.trees.plans.Plan"); enumFieldPatternInfos.stream() .map(info -> info.enumFullName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternDescribableProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternDescribableProcessor.java index 504f3ed1ae..42cf82e3c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternDescribableProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternDescribableProcessor.java @@ -86,21 +86,9 @@ public class PatternDescribableProcessor extends AbstractProcessor { List asts = parseJavaFile(file); patternGeneratorAnalyzer.addAsts(asts); } - String generatePatternCode = patternGeneratorAnalyzer.generatePatterns(); - File generatePatternFile = new File(processingEnv.getFiler() - .getResource(StandardLocation.SOURCE_OUTPUT, "org.apache.doris.nereids.pattern", - "GeneratedPatterns.java").toUri()); - if (generatePatternFile.exists()) { - generatePatternFile.delete(); - } - if (!generatePatternFile.getParentFile().exists()) { - generatePatternFile.getParentFile().mkdirs(); - } - // bypass create file for processingEnv.getFiler(), compile GeneratePatterns in next compile term - try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(generatePatternFile))) { - bufferedWriter.write(generatePatternCode); - } + doGenerate("GeneratedMemoPatterns", "MemoPatterns", true, patternGeneratorAnalyzer); + doGenerate("GeneratedPlanPatterns", "PlanPatterns", false, patternGeneratorAnalyzer); } catch (Throwable t) { String exceptionMsg = Throwables.getStackTraceAsString(t); processingEnv.getMessager().printMessage(Kind.ERROR, @@ -109,6 +97,26 @@ public class PatternDescribableProcessor extends AbstractProcessor { return false; } + private void doGenerate(String className, String parentClassName, boolean isMemoPattern, + PatternGeneratorAnalyzer patternGeneratorAnalyzer) throws IOException { + String generatePatternCode = patternGeneratorAnalyzer.generatePatterns( + className, parentClassName, isMemoPattern); + File generatePatternFile = new File(processingEnv.getFiler() + .getResource(StandardLocation.SOURCE_OUTPUT, "org.apache.doris.nereids.pattern", + className + ".java").toUri()); + if (generatePatternFile.exists()) { + generatePatternFile.delete(); + } + if (!generatePatternFile.getParentFile().exists()) { + generatePatternFile.getParentFile().mkdirs(); + } + + // bypass create file for processingEnv.getFiler(), compile GeneratePatterns in next compile term + try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(generatePatternFile))) { + bufferedWriter.write(generatePatternCode); + } + } + private List findJavaFiles(List dirs) { List files = new ArrayList<>(); for (File dir : dirs) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGenerator.java index 69d34b967d..e916e6bee0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGenerator.java @@ -50,13 +50,16 @@ public abstract class PatternGenerator { protected final Set parentClass; protected final List enumFieldPatternInfos; protected final List generatePatterns = new ArrayList<>(); + protected final boolean isMemoPattern; /** constructor. */ - public PatternGenerator(PatternGeneratorAnalyzer analyzer, ClassDeclaration opType, Set parentClass) { + public PatternGenerator(PatternGeneratorAnalyzer analyzer, ClassDeclaration opType, + Set parentClass, boolean isMemoPattern) { this.analyzer = analyzer; this.opType = opType; this.parentClass = parentClass; this.enumFieldPatternInfos = getEnumFieldPatternInfos(); + this.isMemoPattern = isMemoPattern; } public abstract String genericType(); @@ -74,7 +77,8 @@ public abstract class PatternGenerator { } /** generate code by generators and analyzer. */ - public static String generateCode(List generators, PatternGeneratorAnalyzer analyzer) { + public static String generateCode(String className, String parentClassName, List generators, + PatternGeneratorAnalyzer analyzer, boolean isMemoPattern) { String generateCode = "// Licensed to the Apache Software Foundation (ASF) under one\n" + "// or more contributor license agreements. See the NOTICE file\n" @@ -97,11 +101,10 @@ public abstract class PatternGenerator { + "\n" + generateImports(generators) + "\n"; - - generateCode += "public interface GeneratedPatterns extends Patterns {\n"; + generateCode += "public interface " + className + " extends " + parentClassName + " {\n"; generateCode += generators.stream() .map(generator -> { - String patternMethods = generator.generate(); + String patternMethods = generator.generate(isMemoPattern); // add indent return Arrays.stream(patternMethods.split("\n")) .map(line -> " " + line + "\n") @@ -199,21 +202,25 @@ public abstract class PatternGenerator { return parts; } + protected String childType() { + return isMemoPattern ? "GroupPlan" : "Plan"; + } + /** create generator by plan's type. */ public static Optional create(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { if (parentClass.contains("org.apache.doris.nereids.trees.plans.logical.LogicalLeaf")) { - return Optional.of(new LogicalLeafPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new LogicalLeafPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.logical.LogicalUnary")) { - return Optional.of(new LogicalUnaryPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new LogicalUnaryPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.logical.LogicalBinary")) { - return Optional.of(new LogicalBinaryPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new LogicalBinaryPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.physical.PhysicalLeaf")) { - return Optional.of(new PhysicalLeafPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new PhysicalLeafPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.physical.PhysicalUnary")) { - return Optional.of(new PhysicalUnaryPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new PhysicalUnaryPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else if (parentClass.contains("org.apache.doris.nereids.trees.plans.physical.PhysicalBinary")) { - return Optional.of(new PhysicalBinaryPatternGenerator(analyzer, opType, parentClass)); + return Optional.of(new PhysicalBinaryPatternGenerator(analyzer, opType, parentClass, isMemoPattern)); } else { return Optional.empty(); } @@ -233,21 +240,24 @@ public abstract class PatternGenerator { } /** generate some pattern method code. */ - public String generate() { + public String generate(boolean isMemoPattern) { String opClassName = opType.name; String methodName = getPatternMethodName(); - generateTypePattern(methodName, opClassName, genericType(), "", false); + generateTypePattern(methodName, opClassName, genericType(), "", false, isMemoPattern); if (childrenNum() > 0) { - generateTypePattern(methodName, opClassName, genericTypeWithChildren(), "", true); + generateTypePattern(methodName, opClassName, genericTypeWithChildren(), + "", true, isMemoPattern); } for (EnumFieldPatternInfo info : enumFieldPatternInfos) { String predicate = ".when(p -> p." + info.enumInstanceGetter + "() == " + info.enumType + "." + info.enumInstance + ")"; - generateTypePattern(info.patternName, opClassName, genericType(), predicate, false); + generateTypePattern(info.patternName, opClassName, genericType(), + predicate, false, isMemoPattern); if (childrenNum() > 0) { - generateTypePattern(info.patternName, opClassName, genericTypeWithChildren(), predicate, true); + generateTypePattern(info.patternName, opClassName, genericTypeWithChildren(), + predicate, true, isMemoPattern); } } return generatePatterns(); @@ -255,7 +265,7 @@ public abstract class PatternGenerator { /** generate a pattern method code. */ public String generateTypePattern(String patterName, String className, - String genericParam, String predicate, boolean specifyChildren) { + String genericParam, String predicate, boolean specifyChildren, boolean isMemoPattern) { int childrenNum = childrenNum(); @@ -286,7 +296,8 @@ public abstract class PatternGenerator { generatePatterns.add(pattern); return pattern; } else { - String childrenPattern = StringUtils.repeat("Pattern.GROUP", ", ", childrenNum); + String childrenPattern = StringUtils.repeat( + isMemoPattern ? "Pattern.GROUP" : "Pattern.ANY", ", ", childrenNum); if (childrenNum > 0) { childrenPattern = ", " + childrenPattern; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGeneratorAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGeneratorAnalyzer.java index 6c5343fccb..f4a9d12808 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGeneratorAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PatternGeneratorAnalyzer.java @@ -57,10 +57,10 @@ public class PatternGeneratorAnalyzer { } /** generate pattern methods. */ - public String generatePatterns() { + public String generatePatterns(String className, String parentClassName, boolean isMemoPattern) { analyzeImport(); analyzeParentClass(); - return doGenerate(); + return doGenerate(className, parentClassName, isMemoPattern); } Optional getType(TypeDeclaration typeDeclaration, TypeType type) { @@ -73,7 +73,7 @@ public class PatternGeneratorAnalyzer { return Optional.empty(); } - private String doGenerate() { + private String doGenerate(String className, String parentClassName, boolean isMemoPattern) { Map> planClassMap = parentClassMap.entrySet().stream() .filter(kv -> kv.getValue().contains("org.apache.doris.nereids.trees.plans.Plan")) .filter(kv -> !kv.getKey().name.equals("GroupPlan")) @@ -83,7 +83,7 @@ public class PatternGeneratorAnalyzer { List generators = planClassMap.entrySet() .stream() - .map(kv -> PatternGenerator.create(this, kv.getKey(), kv.getValue())) + .map(kv -> PatternGenerator.create(this, kv.getKey(), kv.getValue(), isMemoPattern)) .filter(Optional::isPresent) .map(Optional::get) .sorted((g1, g2) -> { @@ -100,7 +100,7 @@ public class PatternGeneratorAnalyzer { }) .collect(Collectors.toList()); - return PatternGenerator.generateCode(generators, this); + return PatternGenerator.generateCode(className, parentClassName, generators, this, isMemoPattern); } private void analyzeImport() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalBinaryPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalBinaryPatternGenerator.java index f148912d99..72a3155749 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalBinaryPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalBinaryPatternGenerator.java @@ -26,13 +26,13 @@ import java.util.TreeSet; public class PhysicalBinaryPatternGenerator extends PatternGenerator { public PhysicalBinaryPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override public String genericType() { - return "<" + opType.name + ">"; + return "<" + opType.name + "<" + childType() + ", " + childType() + ">>"; } @Override @@ -44,7 +44,9 @@ public class PhysicalBinaryPatternGenerator extends PatternGenerator { public Set getImports() { Set imports = new TreeSet<>(); imports.add(opType.getFullQualifiedName()); - imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + if (isMemoPattern) { + imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + } imports.add("org.apache.doris.nereids.trees.plans.Plan"); enumFieldPatternInfos.stream() .map(info -> info.enumFullName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalLeafPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalLeafPatternGenerator.java index 68c7154999..f75746b514 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalLeafPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalLeafPatternGenerator.java @@ -26,8 +26,8 @@ import java.util.TreeSet; public class PhysicalLeafPatternGenerator extends PatternGenerator { public PhysicalLeafPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalUnaryPatternGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalUnaryPatternGenerator.java index ea01341039..4254e28ee4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalUnaryPatternGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PhysicalUnaryPatternGenerator.java @@ -26,13 +26,13 @@ import java.util.TreeSet; public class PhysicalUnaryPatternGenerator extends PatternGenerator { public PhysicalUnaryPatternGenerator(PatternGeneratorAnalyzer analyzer, - ClassDeclaration opType, Set parentClass) { - super(analyzer, opType, parentClass); + ClassDeclaration opType, Set parentClass, boolean isMemoPattern) { + super(analyzer, opType, parentClass, isMemoPattern); } @Override public String genericType() { - return "<" + opType.name + ">"; + return "<" + opType.name + "<" + childType() + ">>"; } @Override @@ -44,7 +44,9 @@ public class PhysicalUnaryPatternGenerator extends PatternGenerator { public Set getImports() { Set imports = new TreeSet<>(); imports.add(opType.getFullQualifiedName()); - imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + if (isMemoPattern) { + imports.add("org.apache.doris.nereids.trees.plans.GroupPlan"); + } imports.add("org.apache.doris.nereids.trees.plans.Plan"); enumFieldPatternInfos.stream() .map(info -> info.enumFullName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java index adefdad45a..3bc32c8c6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.processor.post; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.plans.Plan; @@ -56,7 +57,10 @@ public class Validator extends PlanPostProcessor { Plan child = filter.child(); // Forbidden filter-project, we must make filter-project -> project-filter. - Preconditions.checkState(!(child instanceof PhysicalProject)); + if (child instanceof PhysicalProject) { + throw new AnalysisException( + "Nereids generate a filter-project plan, but backend not support:\n" + filter.treeString()); + } // Check filter is from child output. Set childOutputSet = child.getOutputSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java new file mode 100644 index 0000000000..d7f6983c3f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.TransformException; +import org.apache.doris.nereids.pattern.Pattern; +import org.apache.doris.nereids.pattern.ProxyPattern; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.BitSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.BiPredicate; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** AppliedAwareRule */ +public class AppliedAwareRule extends Rule { + private static final String APPLIED_RULES_KEY = "applied_rules"; + + private static final Supplier CREATE_APPLIED_RULES = () -> new BitSet(RuleType.SENTINEL.ordinal()); + + protected final Rule rule; + protected final RuleType ruleType; + protected int ruleTypeIndex; + + private AppliedAwareRule(Rule rule, BiPredicate matchRootPredicate) { + super(rule.getRuleType(), + new ExtendPattern(rule.getPattern(), (Predicate) (plan -> matchRootPredicate.test(rule, plan))), + rule.getRulePromise()); + this.rule = rule; + this.ruleType = rule.getRuleType(); + this.ruleTypeIndex = rule.getRuleType().ordinal(); + } + + @Override + public List transform(Plan plan, CascadesContext context) throws TransformException { + return rule.transform(plan, context); + } + + @Override + public void acceptPlan(Plan plan) { + BitSet appliedRules = plan.getOrInitMutableState(APPLIED_RULES_KEY, CREATE_APPLIED_RULES); + appliedRules.set(ruleTypeIndex); + } + + /** + * AppliedAwareRuleCondition: convert one rule to AppliedAwareRule, so that the rule can add + * some condition depends on whether this rule is applied to some plan + */ + public static class AppliedAwareRuleCondition implements Function { + public Rule apply(Rule rule) { + return new AppliedAwareRule(rule, this::condition); + } + + /** provide this method for the child class get the applied state */ + public final boolean isAppliedRule(Rule rule, Plan plan) { + Optional appliedRules = plan.getMutableState("applied_rules"); + if (!appliedRules.isPresent()) { + return false; + } + return appliedRules.get().get(rule.getRuleType().ordinal()); + } + + /** + * the default condition is whether this rule already applied to a plan, + * this means one plan only apply for a rule only once. child class can + * override this method. + */ + protected boolean condition(Rule rule, Plan plan) { + return isAppliedRule(rule, plan); + } + } + + private static class ExtendPattern extends ProxyPattern { + private final Predicate matchRootPredicate; + + public ExtendPattern(Pattern pattern, Predicate matchRootPredicate) { + super(pattern); + this.matchRootPredicate = Objects.requireNonNull(matchRootPredicate, "matchRootPredicate cannot be null"); + } + + @Override + public boolean matchPlanTree(Plan plan) { + return matchRootPredicate.test(plan) && super.matchPlanTree(plan); + } + + @Override + public boolean matchRoot(Plan plan) { + return matchRootPredicate.test(plan) && super.matchRoot(plan); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/ProxyRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/ProxyRule.java new file mode 100644 index 0000000000..7ea13662ee --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/ProxyRule.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.TransformException; +import org.apache.doris.nereids.pattern.Pattern; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; +import java.util.Objects; + +/** ProxyRule */ +public class ProxyRule extends Rule { + protected final Rule rule; + + public ProxyRule(Rule rule) { + this(rule, rule.getRuleType(), rule.getPattern(), rule.getRulePromise()); + } + + public ProxyRule(Rule rule, RuleType ruleType, Pattern pattern, RulePromise rulePromise) { + super(ruleType, pattern, rulePromise); + this.rule = Objects.requireNonNull(rule, "rule cannot be null"); + } + + @Override + public List transform(Plan node, CascadesContext context) throws TransformException { + return rule.transform(node, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java index 1d54737496..25cba46002 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/Rule.java @@ -68,4 +68,9 @@ public abstract class Rule { } public abstract List transform(Plan node, CascadesContext context) throws TransformException; + + /** callback this function when the traverse framework accept a new plan which produce by this rule */ + public void acceptPlan(Plan plan) { + + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleFactory.java index d2fb2cf872..a4c3838716 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleFactory.java @@ -17,14 +17,14 @@ package org.apache.doris.nereids.rules; -import org.apache.doris.nereids.pattern.GeneratedPatterns; +import org.apache.doris.nereids.pattern.Patterns; import java.util.List; /** * interface for all rule factories for build some rules. */ -public interface RuleFactory extends GeneratedPatterns { +public interface RuleFactory extends Patterns { // need implement List buildRules(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 5409ea0799..99d33a16d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -93,7 +93,6 @@ public enum RuleType { CHECK_AND_STANDARDIZE_WINDOW_FUNCTION_AND_FRAME(RuleTypeClass.REWRITE), AGGREGATE_DISASSEMBLE(RuleTypeClass.REWRITE), DISTINCT_AGGREGATE_DISASSEMBLE(RuleTypeClass.REWRITE), - MARK_NECESSARY_PROJECT(RuleTypeClass.REWRITE), LOGICAL_SUB_QUERY_ALIAS_TO_LOGICAL_PROJECT(RuleTypeClass.REWRITE), ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE), ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE), @@ -101,16 +100,17 @@ public enum RuleType { INFER_FILTER_NOT_NULL(RuleTypeClass.REWRITE), INFER_JOIN_NOT_NULL(RuleTypeClass.REWRITE), // subquery analyze - ANALYZE_FILTER_SUBQUERY(RuleTypeClass.REWRITE), - ANALYZE_PROJECT_SUBQUERY(RuleTypeClass.REWRITE), + FILTER_SUBQUERY_TO_APPLY(RuleTypeClass.REWRITE), + PROJECT_SUBQUERY_TO_APPLY(RuleTypeClass.REWRITE), // subquery rewrite rule ELIMINATE_LIMIT_UNDER_APPLY(RuleTypeClass.REWRITE), ELIMINATE_SORT_UNDER_APPLY(RuleTypeClass.REWRITE), - PUSH_APPLY_UNDER_PROJECT(RuleTypeClass.REWRITE), - PUSH_APPLY_UNDER_FILTER(RuleTypeClass.REWRITE), - ELIMINATE_FILTER_UNDER_APPLY_PROJECT(RuleTypeClass.REWRITE), - APPLY_PULL_FILTER_ON_AGG(RuleTypeClass.REWRITE), - APPLY_PULL_FILTER_ON_PROJECT_UNDER_AGG(RuleTypeClass.REWRITE), + ELIMINATE_SORT_UNDER_APPLY_PROJECT(RuleTypeClass.REWRITE), + PULL_UP_PROJECT_UNDER_APPLY(RuleTypeClass.REWRITE), + UN_CORRELATED_APPLY_FILTER(RuleTypeClass.REWRITE), + UN_CORRELATED_APPLY_PROJECT_FILTER(RuleTypeClass.REWRITE), + UN_CORRELATED_APPLY_AGGREGATE_FILTER(RuleTypeClass.REWRITE), + PULL_UP_CORRELATED_FILTER_UNDER_APPLY_AGGREGATE_PROJECT(RuleTypeClass.REWRITE), SCALAR_APPLY_TO_JOIN(RuleTypeClass.REWRITE), IN_APPLY_TO_JOIN(RuleTypeClass.REWRITE), EXISTS_APPLY_TO_JOIN(RuleTypeClass.REWRITE), @@ -124,6 +124,7 @@ public enum RuleType { PUSHDOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE), PUSH_FILTER_INSIDE_JOIN(RuleTypeClass.REWRITE), PUSHDOWN_FILTER_THROUGH_PROJECT(RuleTypeClass.REWRITE), + PUSHDOWN_FILTER_THROUGH_PROJECT_UNDER_LIMIT(RuleTypeClass.REWRITE), PUSHDOWN_PROJECT_THROUGH_LIMIT(RuleTypeClass.REWRITE), PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE), // column prune rules, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalysisRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalysisRuleFactory.java index 0938297883..10c207eb8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalysisRuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalysisRuleFactory.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; import org.apache.doris.nereids.rules.PlanRuleFactory; import org.apache.doris.nereids.rules.RulePromise; /** * interface for all rule factories used in analysis stage. */ -public interface AnalysisRuleFactory extends PlanRuleFactory { +public interface AnalysisRuleFactory extends PlanRuleFactory, GeneratedPlanPatterns { @Override default RulePromise defaultPromise() { return RulePromise.ANALYSIS; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java index f72e4bc5cd..360462d48c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AvgDistinctToSumDivCount.java @@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -40,7 +41,7 @@ import java.util.Map; * * change avg( distinct a ) into sum( distinct a ) / count( distinct a ) if there are more than 1 distinct arguments */ -public class AvgDistinctToSumDivCount extends OneAnalysisRuleFactory { +public class AvgDistinctToSumDivCount extends OneRewriteRuleFactory { @Override public Rule build() { return RuleType.AVG_DISTINCT_TO_SUM_DIV_COUNT.build( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index fc7e38ea00..8ceecd86db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.rules.AppliedAwareRule.AppliedAwareRuleCondition; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.expression.rewrite.rules.TypeCoercion; @@ -47,9 +48,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunctio import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; -import org.apache.doris.nereids.trees.plans.LeafPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; @@ -61,7 +60,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; @@ -80,7 +78,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; @@ -93,13 +90,8 @@ import java.util.stream.Stream; @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class BindExpression implements AnalysisRuleFactory { - private final Optional outerScope; - - public BindExpression(Optional outerScope) { - this.outerScope = Objects.requireNonNull(outerScope, "outerScope cannot be null"); - } - - private Scope toScope(List slots) { + private Scope toScope(CascadesContext cascadesContext, List slots) { + Optional outerScope = cascadesContext.getOuterScope(); if (outerScope.isPresent()) { return new Scope(outerScope, slots, outerScope.get().getSubquery()); } else { @@ -109,10 +101,29 @@ public class BindExpression implements AnalysisRuleFactory { @Override public List buildRules() { + /* + * some rules not only depends on the condition Plan::canBind, for example, + * BINDING_FILTER_SLOT need transform 'filter(unix_timestamp() > 100)' to + * 'filter(unix_timestamp() > cast(100 as int))'. there is no any unbound expression + * in the filter, so the Plan::canBind return false. + * + * we need `isAppliedRule` to judge whether a plan is applied to a rule, so need convert + * the normal rule to `AppliedAwareRule` to read and write the mutable state. + */ + AppliedAwareRuleCondition ruleCondition = new AppliedAwareRuleCondition() { + @Override + protected boolean condition(Rule rule, Plan plan) { + if (!rule.getPattern().matchRoot(plan)) { + return false; + } + return plan.canBind() || (plan.bound() && !isAppliedRule(rule, plan)); + } + }; + return ImmutableList.of( RuleType.BINDING_PROJECT_SLOT.build( - logicalProject().when(Plan::canBind).thenApply(ctx -> { - LogicalProject project = ctx.root; + logicalProject().thenApply(ctx -> { + LogicalProject project = ctx.root; List boundProjections = bindSlot(project.getProjects(), project.children(), ctx.cascadesContext); List boundExceptions = bindSlot(project.getExcepts(), project.children(), @@ -125,8 +136,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_FILTER_SLOT.build( - logicalFilter().when(Plan::canBind).thenApply(ctx -> { - LogicalFilter filter = ctx.root; + logicalFilter().thenApply(ctx -> { + LogicalFilter filter = ctx.root; Set boundConjuncts = filter.getConjuncts().stream() .map(expr -> bindSlot(expr, filter.children(), ctx.cascadesContext)) .map(expr -> bindFunction(expr, ctx.cascadesContext)) @@ -137,7 +148,7 @@ public class BindExpression implements AnalysisRuleFactory { RuleType.BINDING_USING_JOIN_SLOT.build( usingJoin().thenApply(ctx -> { - UsingJoin using = ctx.root; + UsingJoin using = ctx.root; LogicalJoin lj = new LogicalJoin<>(using.getJoinType() == JoinType.CROSS_JOIN ? JoinType.INNER_JOIN : using.getJoinType(), using.getHashJoinConjuncts(), @@ -151,7 +162,7 @@ public class BindExpression implements AnalysisRuleFactory { // the most right slot is matched with priority. Collections.reverse(leftOutput); List leftSlots = new ArrayList<>(); - Scope scope = toScope(leftOutput.stream() + Scope scope = toScope(ctx.cascadesContext, leftOutput.stream() .filter(s -> !slotNames.contains(s.getName())) .peek(s -> slotNames.add(s.getName())) .collect(Collectors.toList())); @@ -160,7 +171,7 @@ public class BindExpression implements AnalysisRuleFactory { leftSlots.add(expression); } slotNames.clear(); - scope = toScope(lj.right().getOutput().stream() + scope = toScope(ctx.cascadesContext, lj.right().getOutput().stream() .filter(s -> !slotNames.contains(s.getName())) .peek(s -> slotNames.add(s.getName())) .collect(Collectors.toList())); @@ -178,8 +189,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_JOIN_SLOT.build( - logicalJoin().when(Plan::canBind).thenApply(ctx -> { - LogicalJoin join = ctx.root; + logicalJoin().thenApply(ctx -> { + LogicalJoin join = ctx.root; List cond = join.getOtherJoinConjuncts().stream() .map(expr -> bindSlot(expr, join.children(), ctx.cascadesContext)) .map(expr -> bindFunction(expr, ctx.cascadesContext)) @@ -193,8 +204,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_AGGREGATE_SLOT.build( - logicalAggregate().when(Plan::canBind).thenApply(ctx -> { - LogicalAggregate agg = ctx.root; + logicalAggregate().thenApply(ctx -> { + LogicalAggregate agg = ctx.root; List output = agg.getOutputExpressions().stream() .map(expr -> bindSlot(expr, agg.children(), ctx.cascadesContext)) .map(expr -> bindFunction(expr, ctx.cascadesContext)) @@ -291,9 +302,10 @@ public class BindExpression implements AnalysisRuleFactory { boundSlots.addAll(outputSlots); SlotBinder binder = new SlotBinder( - toScope(Lists.newArrayList(boundSlots)), ctx.cascadesContext); + toScope(ctx.cascadesContext, ImmutableList.copyOf(boundSlots)), ctx.cascadesContext); SlotBinder childBinder = new SlotBinder( - toScope(new ArrayList<>(agg.child().getOutputSet())), ctx.cascadesContext); + toScope(ctx.cascadesContext, ImmutableList.copyOf(agg.child().getOutputSet())), + ctx.cascadesContext); List groupBy = replacedGroupBy.stream() .map(expression -> { @@ -323,8 +335,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_REPEAT_SLOT.build( - logicalRepeat().when(Plan::canBind).thenApply(ctx -> { - LogicalRepeat repeat = ctx.root; + logicalRepeat().thenApply(ctx -> { + LogicalRepeat repeat = ctx.root; List output = repeat.getOutputExpressions().stream() .map(expr -> bindSlot(expr, repeat.children(), ctx.cascadesContext)) .map(expr -> bindFunction(expr, ctx.cascadesContext)) @@ -368,35 +380,35 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(aggregate()).when(Plan::canBind).thenApply(ctx -> { - LogicalSort> sort = ctx.root; - Aggregate aggregate = sort.child(); + logicalSort(aggregate()).thenApply(ctx -> { + LogicalSort> sort = ctx.root; + Aggregate aggregate = sort.child(); return bindSort(sort, aggregate, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalHaving(aggregate())).when(Plan::canBind).thenApply(ctx -> { - LogicalSort>> sort = ctx.root; - Aggregate aggregate = sort.child().child(); + logicalSort(logicalHaving(aggregate())).thenApply(ctx -> { + LogicalSort>> sort = ctx.root; + Aggregate aggregate = sort.child().child(); return bindSort(sort, aggregate, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalHaving(logicalProject())).when(Plan::canBind).thenApply(ctx -> { - LogicalSort>> sort = ctx.root; - LogicalProject project = sort.child().child(); + logicalSort(logicalHaving(logicalProject())).thenApply(ctx -> { + LogicalSort>> sort = ctx.root; + LogicalProject project = sort.child().child(); return bindSort(sort, project, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SLOT.build( - logicalSort(logicalProject()).when(Plan::canBind).thenApply(ctx -> { - LogicalSort> sort = ctx.root; - LogicalProject project = sort.child(); + logicalSort(logicalProject()).thenApply(ctx -> { + LogicalSort> sort = ctx.root; + LogicalProject project = sort.child(); return bindSort(sort, project, ctx.cascadesContext); }) ), RuleType.BINDING_SORT_SET_OPERATION_SLOT.build( - logicalSort(logicalSetOperation()).when(Plan::canBind).thenApply(ctx -> { + logicalSort(logicalSetOperation()).thenApply(ctx -> { LogicalSort sort = ctx.root; List sortItemList = sort.getOrderKeys() .stream() @@ -409,8 +421,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_HAVING_SLOT.build( - logicalHaving(aggregate()).when(Plan::canBind).thenApply(ctx -> { - LogicalHaving> having = ctx.root; + logicalHaving(aggregate()).thenApply(ctx -> { + LogicalHaving> having = ctx.root; Plan childPlan = having.child(); Set boundConjuncts = having.getConjuncts().stream() .map(expr -> { @@ -423,7 +435,7 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_HAVING_SLOT.build( - logicalHaving(any()).when(Plan::canBind).thenApply(ctx -> { + logicalHaving(any()).thenApply(ctx -> { LogicalHaving having = ctx.root; Plan childPlan = having.child(); Set boundConjuncts = having.getConjuncts().stream() @@ -449,7 +461,7 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_SET_OPERATION_SLOT.build( - logicalSetOperation().when(Plan::canBind).then(setOperation -> { + logicalSetOperation().then(setOperation -> { // check whether the left and right child output columns are the same if (setOperation.child(0).getOutput().size() != setOperation.child(1).getOutput().size()) { throw new AnalysisException("Operands have unequal number of columns:\n" @@ -472,8 +484,8 @@ public class BindExpression implements AnalysisRuleFactory { }) ), RuleType.BINDING_GENERATE_SLOT.build( - logicalGenerate().when(Plan::canBind).thenApply(ctx -> { - LogicalGenerate generate = ctx.root; + logicalGenerate().thenApply(ctx -> { + LogicalGenerate generate = ctx.root; List boundSlotGenerators = bindSlot(generate.getGenerators(), generate.children(), ctx.cascadesContext); List boundFunctionGenerators = boundSlotGenerators.stream() @@ -497,16 +509,8 @@ public class BindExpression implements AnalysisRuleFactory { UnboundTVFRelation relation = ctx.root; return bindTableValuedFunction(relation, ctx.statementContext); }) - ), - - // when child update, we need update current plan's logical properties, - // since we use cache to avoid compute more than once. - RuleType.BINDING_NON_LEAF_LOGICAL_PLAN.build( - logicalPlan() - .when(plan -> plan.canBind() && !(plan instanceof LeafPlan)) - .then(LogicalPlan::recomputeLogicalProperties) ) - ); + ).stream().map(ruleCondition).collect(ImmutableList.toImmutableList()); } private Plan bindSort(LogicalSort sort, Plan plan, CascadesContext ctx) { @@ -528,8 +532,8 @@ public class BindExpression implements AnalysisRuleFactory { List sortItemList = sort.getOrderKeys() .stream() .map(orderKey -> { - Expression item = bindSlot(orderKey.getExpr(), plan, ctx); - item = bindSlot(item, plan.children(), ctx); + Expression item = bindSlot(orderKey.getExpr(), plan, ctx, true, false); + item = bindSlot(item, plan.children(), ctx, true, false); item = bindFunction(item, ctx); return new OrderKey(item, orderKey.isAsc(), orderKey.isNullFirst()); }).collect(Collectors.toList()); @@ -561,21 +565,33 @@ public class BindExpression implements AnalysisRuleFactory { @SuppressWarnings("unchecked") private E bindSlot(E expr, Plan input, CascadesContext cascadesContext) { - return bindSlot(expr, input, cascadesContext, true); + return bindSlot(expr, input, cascadesContext, true, true); } private E bindSlot(E expr, Plan input, CascadesContext cascadesContext, boolean enableExactMatch) { - return (E) new SlotBinder(toScope(input.getOutput()), cascadesContext, enableExactMatch).bind(expr); + return bindSlot(expr, input, cascadesContext, enableExactMatch, true); + } + + private E bindSlot(E expr, Plan input, CascadesContext cascadesContext, + boolean enableExactMatch, boolean bindSlotInOuterScope) { + return (E) new SlotBinder(toScope(cascadesContext, input.getOutput()), cascadesContext, + enableExactMatch, bindSlotInOuterScope).bind(expr); } @SuppressWarnings("unchecked") private E bindSlot(E expr, List inputs, CascadesContext cascadesContext, boolean enableExactMatch) { + return bindSlot(expr, inputs, cascadesContext, enableExactMatch, true); + } + + private E bindSlot(E expr, List inputs, CascadesContext cascadesContext, + boolean enableExactMatch, boolean bindSlotInOuterScope) { List boundedSlots = inputs.stream() .flatMap(input -> input.getOutput().stream()) .collect(Collectors.toList()); - return (E) new SlotBinder(toScope(boundedSlots), cascadesContext, enableExactMatch).bind(expr); + return (E) new SlotBinder(toScope(cascadesContext, boundedSlots), cascadesContext, + enableExactMatch, bindSlotInOuterScope).bind(expr); } @SuppressWarnings("unchecked") @@ -648,4 +664,8 @@ public class BindExpression implements AnalysisRuleFactory { function = (BoundFunction) TypeCoercion.INSTANCE.rewrite(function, null); return function; } + + public boolean canBind(Plan plan) { + return !plan.hasUnboundExpression() || plan.canBind(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 8ba0997572..b30cd4ad5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -30,10 +30,13 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.analyzer.CTEContext; +import org.apache.doris.nereids.analyzer.Unbound; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.pattern.MatchingContext; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.EqualTo; @@ -70,26 +73,37 @@ public class BindRelation extends OneAnalysisRuleFactory { @Override public Rule build() { return unboundRelation().thenApply(ctx -> { - List nameParts = ctx.root.getNameParts(); - switch (nameParts.size()) { - case 1: { // table - // Use current database name from catalog. - return bindWithCurrentDb(ctx.cascadesContext, ctx.root); - } - case 2: { // db.table - // Use database name from table name parts. - return bindWithDbNameFromNamePart(ctx.cascadesContext, ctx.root); - } - case 3: { // catalog.db.table - // Use catalog and database name from name parts. - return bindWithCatalogNameFromNamePart(ctx.cascadesContext, ctx.root); - } - default: - throw new IllegalStateException("Table name [" + ctx.root.getTableName() + "] is invalid."); + Plan plan = doBindRelation(ctx); + if (!(plan instanceof Unbound)) { + // init output and allocate slot id immediately, so that the slot id increase + // in the order in which the table appears. + LogicalProperties logicalProperties = plan.getLogicalProperties(); + logicalProperties.getOutput(); } + return plan; }).toRule(RuleType.BINDING_RELATION); } + private Plan doBindRelation(MatchingContext ctx) { + List nameParts = ctx.root.getNameParts(); + switch (nameParts.size()) { + case 1: { // table + // Use current database name from catalog. + return bindWithCurrentDb(ctx.cascadesContext, ctx.root); + } + case 2: { // db.table + // Use database name from table name parts. + return bindWithDbNameFromNamePart(ctx.cascadesContext, ctx.root); + } + case 3: { // catalog.db.table + // Use catalog and database name from name parts. + return bindWithCatalogNameFromNamePart(ctx.cascadesContext, ctx.root); + } + default: + throw new IllegalStateException("Table name [" + ctx.root.getTableName() + "] is invalid."); + } + } + private TableIf getTable(String catalogName, String dbName, String tableName, Env env) { CatalogIf catalog = env.getCatalogMgr().getCatalog(catalogName); if (catalog == null) { @@ -209,12 +223,12 @@ public class BindRelation extends OneAnalysisRuleFactory { private Plan parseAndAnalyzeView(String viewSql, CascadesContext parentContext) { LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(viewSql); - CascadesContext viewContext = new Memo(parsedViewPlan) - .newCascadesContext(parentContext.getStatementContext()); + CascadesContext viewContext = CascadesContext.newRewriteContext( + parentContext.getStatementContext(), parsedViewPlan, PhysicalProperties.ANY); viewContext.newAnalyzer().analyze(); // we should remove all group expression of the plan which in other memo, so the groupId would not conflict - return viewContext.getMemo().copyOut(false); + return viewContext.getRewritePlan(); } private List getPartitionIds(TableIf t, UnboundRelation unboundRelation) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckPolicy.java index 65df581bbf..e3c2ca5c68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckPolicy.java @@ -17,9 +17,11 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCheckPolicy; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; @@ -39,15 +41,13 @@ public class CheckPolicy implements AnalysisRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.CHECK_ROW_POLICY.build( - logicalCheckPolicy(logicalSubQueryAlias()).then(checkPolicy -> checkPolicy.child()) - ), - RuleType.CHECK_ROW_POLICY.build( - logicalCheckPolicy(logicalFilter()).then(checkPolicy -> checkPolicy.child()) - ), - RuleType.CHECK_ROW_POLICY.build( - logicalCheckPolicy(logicalRelation()).thenApply(ctx -> { - LogicalCheckPolicy checkPolicy = ctx.root; - LogicalRelation relation = checkPolicy.child(); + logicalCheckPolicy(any().when(child -> !(child instanceof UnboundRelation))).thenApply(ctx -> { + LogicalCheckPolicy checkPolicy = ctx.root; + Plan child = checkPolicy.child(); + if (!(child instanceof LogicalRelation)) { + return child; + } + LogicalRelation relation = (LogicalRelation) child; Optional filter = checkPolicy.getFilter(relation, ctx.connectContext); if (!filter.isPresent()) { return relation; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java index a8b25c2577..1fd829f5f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/RegisterCTE.java @@ -20,11 +20,9 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.analyzer.CTEContext; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -48,7 +46,7 @@ public class RegisterCTE extends OneAnalysisRuleFactory { @Override public Rule build() { return logicalCTE().thenApply(ctx -> { - LogicalCTE logicalCTE = ctx.root; + LogicalCTE logicalCTE = ctx.root; register(logicalCTE.getAliasQueries(), ctx.cascadesContext); return logicalCTE.child(); }).toRule(RuleType.REGISTER_CTE); @@ -69,10 +67,10 @@ public class RegisterCTE extends OneAnalysisRuleFactory { CTEContext localCteContext = cteCtx; Function analyzeCte = parsePlan -> { - CascadesContext localCascadesContext = new Memo(parsePlan) - .newCascadesContext(cascadesContext.getStatementContext(), localCteContext); + CascadesContext localCascadesContext = CascadesContext.newRewriteContext( + cascadesContext.getStatementContext(), parsePlan, localCteContext); localCascadesContext.newAnalyzer().analyze(); - return (LogicalPlan) localCascadesContext.getMemo().copyOut(false); + return (LogicalPlan) localCascadesContext.getRewritePlan(); }; LogicalPlan analyzedCteBody = analyzeCte.apply(aliasQuery.child()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutput.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutput.java index ba953307eb..b52e2f0218 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutput.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutput.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -46,7 +46,7 @@ public class ReplaceExpressionByChildOutput implements AnalysisRuleFactory { return ImmutableList.builder() .add(RuleType.REPLACE_SORT_EXPRESSION_BY_CHILD_OUTPUT.build( logicalSort(logicalProject()).then(sort -> { - LogicalProject project = sort.child(); + LogicalProject project = sort.child(); Map sMap = Maps.newHashMap(); project.getProjects().stream() .filter(Alias.class::isInstance) @@ -57,7 +57,7 @@ public class ReplaceExpressionByChildOutput implements AnalysisRuleFactory { )) .add(RuleType.REPLACE_SORT_EXPRESSION_BY_CHILD_OUTPUT.build( logicalSort(logicalAggregate()).then(sort -> { - LogicalAggregate aggregate = sort.child(); + LogicalAggregate aggregate = sort.child(); Map sMap = Maps.newHashMap(); aggregate.getOutputExpressions().stream() .filter(Alias.class::isInstance) @@ -67,7 +67,7 @@ public class ReplaceExpressionByChildOutput implements AnalysisRuleFactory { }) )).add(RuleType.REPLACE_SORT_EXPRESSION_BY_CHILD_OUTPUT.build( logicalSort(logicalHaving(logicalAggregate())).then(sort -> { - LogicalAggregate aggregate = sort.child().child(); + LogicalAggregate aggregate = sort.child().child(); Map sMap = Maps.newHashMap(); aggregate.getOutputExpressions().stream() .filter(Alias.class::isInstance) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java index c826cc98b0..9a3a2d675e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java @@ -20,13 +20,16 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import com.google.common.collect.ImmutableList; @@ -43,13 +46,15 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { public List buildRules() { return ImmutableList.builder() .add(RuleType.RESOLVE_ORDINAL_IN_ORDER_BY.build( - logicalSort().then(sort -> { + logicalSort().thenApply(ctx -> { + LogicalSort sort = ctx.root; List childOutput = sort.child().getOutput(); List orderKeys = sort.getOrderKeys(); List orderKeysWithoutOrd = new ArrayList<>(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); for (OrderKey k : orderKeys) { Expression expression = k.getExpr(); - expression = FoldConstantRule.INSTANCE.rewrite(expression); + expression = FoldConstantRule.INSTANCE.rewrite(expression, context); if (expression instanceof IntegerLikeLiteral) { IntegerLikeLiteral i = (IntegerLikeLiteral) expression; int ord = i.getIntValue(); @@ -64,12 +69,14 @@ public class ResolveOrdinalInOrderByAndGroupBy implements AnalysisRuleFactory { }) )) .add(RuleType.RESOLVE_ORDINAL_IN_GROUP_BY.build( - logicalAggregate().whenNot(agg -> agg.isOrdinalIsResolved()).then(agg -> { + logicalAggregate().whenNot(agg -> agg.isOrdinalIsResolved()).thenApply(ctx -> { + LogicalAggregate agg = ctx.root; List aggOutput = agg.getOutputExpressions(); List groupByWithoutOrd = new ArrayList<>(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); boolean ordExists = false; for (Expression groupByExpr : agg.getGroupByExpressions()) { - groupByExpr = FoldConstantRule.INSTANCE.rewrite(groupByExpr); + groupByExpr = FoldConstantRule.INSTANCE.rewrite(groupByExpr, context); if (groupByExpr instanceof IntegerLikeLiteral) { IntegerLikeLiteral i = (IntegerLikeLiteral) groupByExpr; int ord = i.getIntValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java index b2fcd34918..0d1785eafb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java @@ -47,15 +47,18 @@ class SlotBinder extends SubExprAnalyzer { but enabled for order by clause TODO after remove original planner, always enable exact match mode. */ - private boolean enableExactMatch = true; + private boolean enableExactMatch; + private final boolean bindSlotInOuterScope; public SlotBinder(Scope scope, CascadesContext cascadesContext) { - this(scope, cascadesContext, true); + this(scope, cascadesContext, true, true); } - public SlotBinder(Scope scope, CascadesContext cascadesContext, boolean enableExactMatch) { + public SlotBinder(Scope scope, CascadesContext cascadesContext, + boolean enableExactMatch, boolean bindSlotInOuterScope) { super(scope, cascadesContext); this.enableExactMatch = enableExactMatch; + this.bindSlotInOuterScope = bindSlotInOuterScope; } public Expression bind(Expression expression) { @@ -81,7 +84,7 @@ class SlotBinder extends SubExprAnalyzer { Optional> boundedOpt = Optional.of(bindSlot(unboundSlot, getScope().getSlots())); boolean foundInThisScope = !boundedOpt.get().isEmpty(); // Currently only looking for symbols on the previous level. - if (!foundInThisScope && getScope().getOuterScope().isPresent()) { + if (bindSlotInOuterScope && !foundInThisScope && getScope().getOuterScope().isPresent()) { boundedOpt = Optional.of(bindSlot(unboundSlot, getScope() .getOuterScope() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java index 26d281fe44..a51e049999 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java @@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InSubquery; @@ -38,6 +37,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -140,13 +140,12 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { } private AnalyzedResult analyzeSubquery(SubqueryExpr expr) { - CascadesContext subqueryContext = new Memo(expr.getQueryPlan()) - .newCascadesContext((cascadesContext.getStatementContext()), cascadesContext.getCteContext()); + CascadesContext subqueryContext = CascadesContext.newRewriteContext( + cascadesContext.getStatementContext(), expr.getQueryPlan(), cascadesContext.getCteContext()); Scope subqueryScope = genScopeWithSubquery(expr); - subqueryContext - .newAnalyzer(Optional.of(subqueryScope)) - .analyze(); - return new AnalyzedResult((LogicalPlan) subqueryContext.getMemo().copyOut(false), + subqueryContext.setOuterScope(subqueryScope); + subqueryContext.newAnalyzer().analyze(); + return new AnalyzedResult((LogicalPlan) subqueryContext.getRewritePlan(), subqueryScope.getCorrelatedSlots()); } @@ -168,7 +167,7 @@ class SubExprAnalyzer extends DefaultExpressionRewriter { private final LogicalPlan logicalPlan; private final List correlatedSlots; - public AnalyzedResult(LogicalPlan logicalPlan, List correlatedSlots) { + public AnalyzedResult(LogicalPlan logicalPlan, Collection correlatedSlots) { this.logicalPlan = Objects.requireNonNull(logicalPlan, "logicalPlan can not be null"); this.correlatedSlots = correlatedSlots == null ? new ArrayList<>() : ImmutableList.copyOf(correlatedSlots); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubquery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java similarity index 88% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubquery.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java index 1e85e87797..995a095a34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java @@ -30,7 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -46,18 +46,18 @@ import java.util.Optional; import java.util.Set; /** - * AnalyzeSubquery. translate from subquery to LogicalApply. + * SubqueryToApply. translate from subquery to LogicalApply. * In two steps * The first step is to replace the predicate corresponding to the filter where the subquery is located. * The second step converts the subquery into an apply node. */ -public class AnalyzeSubquery implements AnalysisRuleFactory { +public class SubqueryToApply implements AnalysisRuleFactory { @Override public List buildRules() { return ImmutableList.of( - RuleType.ANALYZE_FILTER_SUBQUERY.build( + RuleType.FILTER_SUBQUERY_TO_APPLY.build( logicalFilter().thenApply(ctx -> { - LogicalFilter filter = ctx.root; + LogicalFilter filter = ctx.root; Set subqueryExprs = filter.getPredicate().collect(SubqueryExpr.class::isInstance); if (subqueryExprs.isEmpty()) { return filter; @@ -66,14 +66,14 @@ public class AnalyzeSubquery implements AnalysisRuleFactory { // first step: Replace the subquery of predicate in LogicalFilter // second step: Replace subquery with LogicalApply return new LogicalFilter<>(new ReplaceSubquery().replace(filter.getConjuncts()), - analyzedSubquery( + subqueryToApply( subqueryExprs, filter.child(), ctx.cascadesContext )); }) ), - RuleType.ANALYZE_PROJECT_SUBQUERY.build( + RuleType.PROJECT_SUBQUERY_TO_APPLY.build( logicalProject().thenApply(ctx -> { - LogicalProject project = ctx.root; + LogicalProject project = ctx.root; Set subqueryExprs = new HashSet<>(); project.getProjects().stream() .filter(Alias.class::isInstance) @@ -89,17 +89,17 @@ public class AnalyzeSubquery implements AnalysisRuleFactory { return new LogicalProject(project.getProjects().stream() .map(p -> p.withChildren(new ReplaceSubquery().replace(p))) .collect(ImmutableList.toImmutableList()), - analyzedSubquery( + subqueryToApply( subqueryExprs, project.child(), ctx.cascadesContext )); }) - ) + ) ); } - private LogicalPlan analyzedSubquery(Set subqueryExprs, - LogicalPlan childPlan, CascadesContext ctx) { - LogicalPlan tmpPlan = childPlan; + private Plan subqueryToApply(Set subqueryExprs, + Plan childPlan, CascadesContext ctx) { + Plan tmpPlan = childPlan; for (SubqueryExpr subqueryExpr : subqueryExprs) { if (!ctx.subqueryIsAnalyzed(subqueryExpr)) { tmpPlan = addApply(subqueryExpr, tmpPlan, ctx); @@ -108,8 +108,7 @@ public class AnalyzeSubquery implements AnalysisRuleFactory { return tmpPlan; } - private LogicalPlan addApply(SubqueryExpr subquery, - LogicalPlan childPlan, CascadesContext ctx) { + private LogicalPlan addApply(SubqueryExpr subquery, Plan childPlan, CascadesContext ctx) { ctx.setSubqueryExprIsAnalyzed(subquery, true); LogicalApply newApply = new LogicalApply( subquery.getCorrelateSlots(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java index ef1158ae2f..e51f203932 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/UserAuthentication.java @@ -33,7 +33,8 @@ public class UserAuthentication extends OneAnalysisRuleFactory { @Override public Rule build() { - return logicalRelation().thenApply(ctx -> checkPermission(ctx.root, ctx.connectContext)) + return logicalRelation() + .thenApply(ctx -> checkPermission(ctx.root, ctx.connectContext)) .toRule(RuleType.RELATION_AUTHENTICATION); } @@ -46,7 +47,6 @@ public class UserAuthentication extends OneAnalysisRuleFactory { ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), dbName + ": " + tableName); throw new AnalysisException(message); - } return relation; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/ExplorationRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/ExplorationRuleFactory.java index 4e449a5146..a8b9f9ce53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/ExplorationRuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/ExplorationRuleFactory.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.rules.exploration; +import org.apache.doris.nereids.pattern.GeneratedMemoPatterns; import org.apache.doris.nereids.rules.PlanRuleFactory; import org.apache.doris.nereids.rules.RulePromise; /** * interface for all exploration rule factories. */ -public interface ExplorationRuleFactory extends PlanRuleFactory { +public interface ExplorationRuleFactory extends PlanRuleFactory, GeneratedMemoPatterns { @Override default RulePromise defaultPromise() { return RulePromise.EXPLORE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionNormalization.java index 5d869f662e..360cdbf87d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionNormalization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionNormalization.java @@ -29,7 +29,7 @@ import org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyCastRule; import org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyNotExprRule; import org.apache.doris.nereids.rules.expression.rewrite.rules.SupportJavaDateFormatter; import org.apache.doris.nereids.rules.expression.rewrite.rules.TypeCoercion; -import org.apache.doris.qe.ConnectContext; +import org.apache.doris.nereids.trees.expressions.Expression; import com.google.common.collect.ImmutableList; @@ -56,8 +56,13 @@ public class ExpressionNormalization extends ExpressionRewrite { SupportJavaDateFormatter.INSTANCE ); - public ExpressionNormalization(ConnectContext context) { - super(new ExpressionRuleExecutor(NORMALIZE_REWRITE_RULES, context)); + public ExpressionNormalization() { + super(new ExpressionRuleExecutor(NORMALIZE_REWRITE_RULES)); + } + + @Override + public Expression rewrite(Expression expression, ExpressionRewriteContext context) { + return super.rewrite(expression, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java index d4e033cb35..50b83de095 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java @@ -25,11 +25,16 @@ import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.functions.Function; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; +import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; @@ -56,8 +61,8 @@ public class ExpressionRewrite implements RewriteRuleFactory { this.rewriter = Objects.requireNonNull(rewriter, "rewriter is null"); } - public Expression rewrite(Expression expression) { - return rewriter.rewrite(expression); + public Expression rewrite(Expression expression, ExpressionRewriteContext expressionRewriteContext) { + return rewriter.rewrite(expression, expressionRewriteContext); } @Override @@ -77,10 +82,12 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class GenerateExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalGenerate().then(generate -> { + return logicalGenerate().thenApply(ctx -> { + LogicalGenerate generate = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); List generators = generate.getGenerators(); List newGenerators = generators.stream() - .map(func -> (Function) rewriter.rewrite(func)) + .map(func -> (Function) rewriter.rewrite(func, context)) .collect(ImmutableList.toImmutableList()); if (generators.equals(newGenerators)) { return generate; @@ -93,11 +100,14 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class OneRowRelationExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalOneRowRelation().then(oneRowRelation -> { + return logicalOneRowRelation().thenApply(ctx -> { + LogicalOneRowRelation oneRowRelation = ctx.root; List projects = oneRowRelation.getProjects(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + List newProjects = projects .stream() - .map(expr -> (NamedExpression) rewriter.rewrite(expr)) + .map(expr -> (NamedExpression) rewriter.rewrite(expr, context)) .collect(ImmutableList.toImmutableList()); if (projects.equals(newProjects)) { return oneRowRelation; @@ -110,10 +120,13 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class ProjectExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalProject().then(project -> { + return logicalProject().thenApply(ctx -> { + LogicalProject project = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); List projects = project.getProjects(); List newProjects = projects.stream() - .map(expr -> (NamedExpression) rewriter.rewrite(expr)).collect(ImmutableList.toImmutableList()); + .map(expr -> (NamedExpression) rewriter.rewrite(expr, context)) + .collect(ImmutableList.toImmutableList()); if (projects.equals(newProjects)) { return project; } @@ -125,9 +138,11 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class FilterExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalFilter().then(filter -> { + return logicalFilter().thenApply(ctx -> { + LogicalFilter filter = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); Set newConjuncts = ImmutableSet.copyOf(ExpressionUtils.extractConjunction( - rewriter.rewrite(filter.getPredicate()))); + rewriter.rewrite(filter.getPredicate(), context))); if (newConjuncts.equals(filter.getConjuncts())) { return filter; } @@ -139,13 +154,16 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class AggExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalAggregate().then(agg -> { + return logicalAggregate().thenApply(ctx -> { + LogicalAggregate agg = ctx.root; List groupByExprs = agg.getGroupByExpressions(); - List newGroupByExprs = rewriter.rewrite(groupByExprs); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + List newGroupByExprs = rewriter.rewrite(groupByExprs, context); List outputExpressions = agg.getOutputExpressions(); List newOutputExpressions = outputExpressions.stream() - .map(expr -> (NamedExpression) rewriter.rewrite(expr)).collect(ImmutableList.toImmutableList()); + .map(expr -> (NamedExpression) rewriter.rewrite(expr, context)) + .collect(ImmutableList.toImmutableList()); if (outputExpressions.equals(newOutputExpressions)) { return agg; } @@ -158,16 +176,18 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class JoinExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalJoin().then(join -> { + return logicalJoin().thenApply(ctx -> { + LogicalJoin join = ctx.root; List hashJoinConjuncts = join.getHashJoinConjuncts(); List otherJoinConjuncts = join.getOtherJoinConjuncts(); if (otherJoinConjuncts.isEmpty() && hashJoinConjuncts.isEmpty()) { return join; } + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); List rewriteHashJoinConjuncts = Lists.newArrayList(); boolean hashJoinConjunctsChanged = false; for (Expression expr : hashJoinConjuncts) { - Expression newExpr = rewriter.rewrite(expr); + Expression newExpr = rewriter.rewrite(expr, context); hashJoinConjunctsChanged = hashJoinConjunctsChanged || !newExpr.equals(expr); rewriteHashJoinConjuncts.add(newExpr); } @@ -175,7 +195,7 @@ public class ExpressionRewrite implements RewriteRuleFactory { List rewriteOtherJoinConjuncts = Lists.newArrayList(); boolean otherJoinConjunctsChanged = false; for (Expression expr : otherJoinConjuncts) { - Expression newExpr = rewriter.rewrite(expr); + Expression newExpr = rewriter.rewrite(expr, context); otherJoinConjunctsChanged = otherJoinConjunctsChanged || !newExpr.equals(expr); rewriteOtherJoinConjuncts.add(newExpr); } @@ -193,11 +213,13 @@ public class ExpressionRewrite implements RewriteRuleFactory { @Override public Rule build() { - return logicalSort().then(sort -> { + return logicalSort().thenApply(ctx -> { + LogicalSort sort = ctx.root; List orderKeys = sort.getOrderKeys(); List rewrittenOrderKeys = new ArrayList<>(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); for (OrderKey k : orderKeys) { - Expression expression = rewriter.rewrite(k.getExpr()); + Expression expression = rewriter.rewrite(k.getExpr(), context); rewrittenOrderKeys.add(new OrderKey(expression, k.isAsc(), k.isNullFirst())); } return sort.withOrderKeys(rewrittenOrderKeys); @@ -208,10 +230,12 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class HavingExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalHaving().then(having -> { + return logicalHaving().thenApply(ctx -> { + LogicalHaving having = ctx.root; Set rewrittenExpr = new HashSet<>(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); for (Expression e : having.getExpressions()) { - rewrittenExpr.add(rewriter.rewrite(e)); + rewrittenExpr.add(rewriter.rewrite(e, context)); } return having.withExpressions(rewrittenExpr); }).toRule(RuleType.REWRITE_HAVING_EXPRESSION); @@ -221,15 +245,19 @@ public class ExpressionRewrite implements RewriteRuleFactory { private class LogicalRepeatRewrite extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalRepeat().then(r -> { + return logicalRepeat().thenApply(ctx -> { + LogicalRepeat repeat = ctx.root; ImmutableList.Builder> groupingExprs = ImmutableList.builder(); - for (List expressions : r.getGroupingSets()) { - groupingExprs.add(expressions.stream().map(rewriter::rewrite) - .collect(ImmutableList.toImmutableList())); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + for (List expressions : repeat.getGroupingSets()) { + groupingExprs.add(expressions.stream() + .map(expr -> rewriter.rewrite(expr, context)) + .collect(ImmutableList.toImmutableList()) + ); } - return r.withGroupSetsAndOutput(groupingExprs.build(), - r.getOutputExpressions().stream() - .map(rewriter::rewrite) + return repeat.withGroupSetsAndOutput(groupingExprs.build(), + repeat.getOutputExpressions().stream() + .map(output -> rewriter.rewrite(output, context)) .map(e -> (NamedExpression) e) .collect(ImmutableList.toImmutableList())); }).toRule(RuleType.REWRITE_REPEAT_EXPRESSION); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteContext.java index dd32382810..37e571a0e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteContext.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.expression.rewrite; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.qe.ConnectContext; /** @@ -25,7 +26,7 @@ import org.apache.doris.qe.ConnectContext; public class ExpressionRewriteContext { public final ConnectContext connectContext; - public ExpressionRewriteContext(ConnectContext connectContext) { - this.connectContext = connectContext; + public ExpressionRewriteContext(CascadesContext cascadesContext) { + this.connectContext = cascadesContext.getConnectContext(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRuleExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRuleExecutor.java index fdcb0ba497..4058a566d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRuleExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRuleExecutor.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.expression.rewrite; import org.apache.doris.nereids.rules.expression.rewrite.rules.NormalizeBinaryPredicatesRule; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -30,30 +29,23 @@ import java.util.Optional; * Expression rewrite entry, which contains all rewrite rules. */ public class ExpressionRuleExecutor { - - private final ExpressionRewriteContext ctx; private final List rules; - public ExpressionRuleExecutor(List rules, ConnectContext context) { - this.rules = rules; - this.ctx = new ExpressionRewriteContext(context); - } - public ExpressionRuleExecutor(List rules) { - this(rules, null); + this.rules = rules; } - public List rewrite(List exprs) { - return exprs.stream().map(this::rewrite).collect(ImmutableList.toImmutableList()); + public List rewrite(List exprs, ExpressionRewriteContext ctx) { + return exprs.stream().map(expr -> rewrite(expr, ctx)).collect(ImmutableList.toImmutableList()); } /** * Given an expression, returns a rewritten expression. */ - public Expression rewrite(Expression root) { + public Expression rewrite(Expression root, ExpressionRewriteContext ctx) { Expression result = root; for (ExpressionRewriteRule rule : rules) { - result = applyRule(result, rule); + result = applyRule(result, rule, ctx); } return result; } @@ -61,11 +53,11 @@ public class ExpressionRuleExecutor { /** * Given an expression, returns a rewritten expression. */ - public Optional rewrite(Optional root) { - return root.map(this::rewrite); + public Optional rewrite(Optional root, ExpressionRewriteContext ctx) { + return root.map(r -> this.rewrite(r, ctx)); } - private Expression applyRule(Expression expr, ExpressionRewriteRule rule) { + private Expression applyRule(Expression expr, ExpressionRewriteRule rule, ExpressionRewriteContext ctx) { return rule.rewrite(expr, ctx); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRule.java index fcd8a7cff2..175e40b0aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRule.java @@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.expression.rewrite.rules; import org.apache.doris.nereids.rules.expression.rewrite.AbstractExpressionRewriteRule; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.qe.ConnectContext; /** * Constant evaluation of an expression. @@ -36,11 +35,5 @@ public class FoldConstantRule extends AbstractExpressionRewriteRule { } return FoldConstantRuleOnFE.INSTANCE.rewrite(expr, ctx); } - - public Expression rewrite(Expression expr) { - ExpressionRewriteContext ctx = new ExpressionRewriteContext(ConnectContext.get()); - return rewrite(expr, ctx); - } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index 7187a9cf52..5cf69e5f63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -87,8 +87,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { @Override public List buildRules() { - PatternDescriptor> basePattern = logicalAggregate() - .when(LogicalAggregate::isNormalized); + PatternDescriptor> basePattern = logicalAggregate(); return ImmutableList.of( RuleType.STORAGE_LAYER_AGGREGATE_WITHOUT_PROJECT.build( @@ -125,12 +124,12 @@ public class AggregateStrategies implements ImplementationRuleFactory { RuleType.TWO_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI.build( basePattern .when(this::containsCountDistinctMultiExpr) - .thenApplyMulti(ctx -> twoPhaseAggregateWithCountDistinctMulti(ctx.root, ctx.connectContext)) + .thenApplyMulti(ctx -> twoPhaseAggregateWithCountDistinctMulti(ctx.root, ctx.cascadesContext)) ), RuleType.THREE_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI.build( basePattern .when(this::containsCountDistinctMultiExpr) - .thenApplyMulti(ctx -> threePhaseAggregateWithCountDistinctMulti(ctx.root, ctx.connectContext)) + .thenApplyMulti(ctx -> threePhaseAggregateWithCountDistinctMulti(ctx.root, ctx.cascadesContext)) ), RuleType.TWO_PHASE_AGGREGATE_WITH_DISTINCT.build( basePattern @@ -393,7 +392,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { * */ private List> twoPhaseAggregateWithCountDistinctMulti( - LogicalAggregate logicalAgg, ConnectContext connectContext) { + LogicalAggregate logicalAgg, CascadesContext cascadesContext) { AggregateParam inputToBufferParam = new AggregateParam(AggPhase.LOCAL, AggMode.INPUT_TO_BUFFER); Set countDistinctArguments = logicalAgg.getDistinctArguments(); @@ -423,13 +422,14 @@ public class AggregateStrategies implements ImplementationRuleFactory { PhysicalHashAggregate gatherLocalAgg = new PhysicalHashAggregate<>( localAggGroupBy, localOutput, Optional.of(partitionExpressions), new AggregateParam(AggPhase.LOCAL, AggMode.INPUT_TO_BUFFER), - maybeUsingStreamAgg(connectContext, logicalAgg), + maybeUsingStreamAgg(cascadesContext.getConnectContext(), logicalAgg), logicalAgg.getLogicalProperties(), requireGather, logicalAgg.child() ); List distinctGroupBy = logicalAgg.getGroupByExpressions(); - LogicalAggregate countIfAgg = countDistinctMultiExprToCountIf(logicalAgg, connectContext).first; + LogicalAggregate countIfAgg = countDistinctMultiExprToCountIf( + logicalAgg, cascadesContext).first; AggregateParam distinctInputToResultParam = new AggregateParam(AggPhase.DISTINCT_LOCAL, AggMode.INPUT_TO_RESULT); @@ -509,7 +509,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { * */ private List> threePhaseAggregateWithCountDistinctMulti( - LogicalAggregate logicalAgg, ConnectContext connectContext) { + LogicalAggregate logicalAgg, CascadesContext cascadesContext) { AggregateParam inputToBufferParam = new AggregateParam(AggPhase.LOCAL, AggMode.INPUT_TO_BUFFER); Set countDistinctArguments = logicalAgg.getDistinctArguments(); @@ -539,7 +539,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { PhysicalHashAggregate anyLocalAgg = new PhysicalHashAggregate<>( localAggGroupBy, localOutput, Optional.of(partitionExpressions), new AggregateParam(AggPhase.LOCAL, AggMode.INPUT_TO_BUFFER), - maybeUsingStreamAgg(connectContext, logicalAgg), + maybeUsingStreamAgg(cascadesContext.getConnectContext(), logicalAgg), logicalAgg.getLogicalProperties(), requireAny, logicalAgg.child() ); @@ -571,7 +571,8 @@ public class AggregateStrategies implements ImplementationRuleFactory { bufferToBufferParam, false, logicalAgg.getLogicalProperties(), requireGather, anyLocalAgg); - LogicalAggregate countIfAgg = countDistinctMultiExprToCountIf(logicalAgg, connectContext).first; + LogicalAggregate countIfAgg = countDistinctMultiExprToCountIf( + logicalAgg, cascadesContext).first; AggregateParam distinctInputToResultParam = new AggregateParam(AggPhase.DISTINCT_LOCAL, AggMode.INPUT_TO_RESULT); @@ -1194,7 +1195,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { * phase of aggregate, please normalize to slot and create a bottom project like NormalizeAggregate. */ private Pair, List> countDistinctMultiExprToCountIf( - LogicalAggregate aggregate, ConnectContext connectContext) { + LogicalAggregate aggregate, CascadesContext cascadesContext) { ImmutableList.Builder countIfList = ImmutableList.builder(); List newOutput = ExpressionUtils.rewriteDownShortCircuit( aggregate.getOutputExpressions(), outputChild -> { @@ -1206,7 +1207,7 @@ public class AggregateStrategies implements ImplementationRuleFactory { for (int i = arguments.size() - 2; i >= 0; --i) { Expression argument = count.getArgument(i); If ifNull = new If(new IsNull(argument), NullLiteral.INSTANCE, countExpr); - countExpr = assignNullType(ifNull, connectContext); + countExpr = assignNullType(ifNull, cascadesContext); } Count countIf = new Count(countExpr); countIfList.add(countIf); @@ -1224,8 +1225,9 @@ public class AggregateStrategies implements ImplementationRuleFactory { } // don't invoke the ExpressionNormalization, because the expression maybe simplified and get rid of some slots - private If assignNullType(If ifExpr, ConnectContext context) { - If ifWithCoercion = (If) TypeCoercion.INSTANCE.rewrite(ifExpr, new ExpressionRewriteContext(context)); + private If assignNullType(If ifExpr, CascadesContext cascadesContext) { + ExpressionRewriteContext context = new ExpressionRewriteContext(cascadesContext); + If ifWithCoercion = (If) TypeCoercion.INSTANCE.rewrite(ifExpr, context); Expression trueValue = ifWithCoercion.getArgument(1); if (trueValue instanceof Cast && trueValue.child(0) instanceof NullLiteral) { List newArgs = Lists.newArrayList(ifWithCoercion.getArguments()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/ImplementationRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/ImplementationRuleFactory.java index ae15eed50e..5e71010c74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/ImplementationRuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/ImplementationRuleFactory.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.rules.implementation; +import org.apache.doris.nereids.pattern.GeneratedMemoPatterns; import org.apache.doris.nereids.rules.PlanRuleFactory; import org.apache.doris.nereids.rules.RulePromise; /** * interface for all implementation rule factories. */ -public interface ImplementationRuleFactory extends PlanRuleFactory { +public interface ImplementationRuleFactory extends PlanRuleFactory, GeneratedMemoPatterns { @Override default RulePromise defaultPromise() { return RulePromise.IMPLEMENT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java index 78dc66aa7c..1a486c7161 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.common.Pair; import org.apache.doris.nereids.annotation.Developing; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; @@ -56,7 +57,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.util.ExpressionUtils; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -484,9 +484,14 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial Set nonVirtualRequiredScanOutput = requiredScanOutput.stream() .filter(slot -> !(slot instanceof VirtualSlotReference)) .collect(ImmutableSet.toImmutableSet()); - Preconditions.checkArgument(scan.getOutputSet().containsAll(nonVirtualRequiredScanOutput), - String.format("Scan's output (%s) should contains all the input required scan output (%s).", - scan.getOutput(), nonVirtualRequiredScanOutput)); + + // use if condition to skip String.format() and speed up + if (!scan.getOutputSet().containsAll(nonVirtualRequiredScanOutput)) { + throw new AnalysisException( + String.format("Scan's output (%s) should contains all the input required scan output (%s).", + scan.getOutput(), nonVirtualRequiredScanOutput)); + } + OlapTable table = scan.getTable(); switch (scan.getTable().getKeysType()) { case AGG_KEYS: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java index 7134bd264a..9d004ca5f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.mv; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.rules.Rule; @@ -114,13 +115,20 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater LogicalOlapScan scan, Supplier> requiredScanOutputSupplier, Supplier> predicatesSupplier) { - switch (scan.getTable().getKeysType()) { + OlapTable table = scan.getTable(); + long baseIndexId = table.getBaseIndexId(); + KeysType keysType = scan.getTable().getKeysType(); + switch (keysType) { case AGG_KEYS: case UNIQUE_KEYS: + break; case DUP_KEYS: + if (table.getIndexIdToMeta().size() == 1) { + return scan.withMaterializedIndexSelected(PreAggStatus.on(), baseIndexId); + } break; default: - throw new RuntimeException("Not supported keys type: " + scan.getTable().getKeysType()); + throw new RuntimeException("Not supported keys type: " + keysType); } if (scan.getTable().isDupKeysOrMergeOnWrite()) { // Set pre-aggregation to `on` to keep consistency with legacy logic. @@ -132,8 +140,16 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater return scan.withMaterializedIndexSelected(PreAggStatus.on(), selectBestIndex(candidate, scan, predicatesSupplier.get())); } else { - OlapTable table = scan.getTable(); - long baseIndexId = table.getBaseIndexId(); + final PreAggStatus preAggStatus; + if (preAggEnabledByHint(scan)) { + // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys. + preAggStatus = PreAggStatus.on(); + } else { + preAggStatus = PreAggStatus.off("No aggregate on scan."); + } + if (table.getIndexIdToMeta().size() == 1) { + return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId); + } int baseIndexKeySize = table.getKeyColumnsByIndexId(table.getBaseIndexId()).size(); // No aggregate on scan. // So only base index and indexes that have all the keys could be used. @@ -143,13 +159,6 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater .filter(index -> containAllRequiredColumns(index, scan, requiredScanOutputSupplier.get())) .collect(Collectors.toList()); - final PreAggStatus preAggStatus; - if (preAggEnabledByHint(scan)) { - // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys. - preAggStatus = PreAggStatus.on(); - } else { - preAggStatus = PreAggStatus.off("No aggregate on scan."); - } if (candidates.size() == 1) { // `candidates` only have base index. return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BatchRewriteRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BatchRewriteRuleFactory.java new file mode 100644 index 0000000000..4dcd1998b0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/BatchRewriteRuleFactory.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.rules.PlanRuleFactory; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleFactory; +import org.apache.doris.nereids.rules.RulePromise; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; + +import java.util.List; + +/** BatchRewriteRuleFactory */ +public interface BatchRewriteRuleFactory extends PlanRuleFactory { + @Override + default RulePromise defaultPromise() { + return RulePromise.REWRITE; + } + + @Override + default List buildRules() { + Builder rules = ImmutableList.builder(); + for (RuleFactory ruleFactory : getRuleFactories()) { + rules.addAll(ruleFactory.buildRules()); + } + return rules.build(); + } + + List getRuleFactories(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleFactory.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleFactory.java index 03c220f03b..c07d5ceb5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/RewriteRuleFactory.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; import org.apache.doris.nereids.rules.PlanRuleFactory; import org.apache.doris.nereids.rules.RulePromise; /** * interface for all rewrite rule factories. */ -public interface RewriteRuleFactory extends PlanRuleFactory { +public interface RewriteRuleFactory extends PlanRuleFactory, GeneratedPlanPatterns { @Override default RulePromise defaultPromise() { return RulePromise.REWRITE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionAndFrame.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionAndFrame.java index 2855f6376f..42195eb0dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionAndFrame.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionAndFrame.java @@ -24,7 +24,7 @@ import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.WindowExpression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import com.google.common.base.Preconditions; @@ -45,7 +45,7 @@ public class CheckAndStandardizeWindowFunctionAndFrame extends OneRewriteRuleFac ); } - private LogicalWindow checkAndStandardize(LogicalWindow logicalWindow) { + private LogicalWindow checkAndStandardize(LogicalWindow logicalWindow) { List newOutputExpressions = logicalWindow.getWindowExpressions().stream() .map(expr -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateAggregate.java index 8e3d4977af..3d89320eaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateAggregate.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Project; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -34,7 +34,7 @@ public class EliminateAggregate extends OneRewriteRuleFactory { @Override public Rule build() { return logicalAggregate(logicalAggregate()).then(outerAgg -> { - LogicalAggregate innerAgg = outerAgg.child(); + LogicalAggregate innerAgg = outerAgg.child(); if (!isSame(outerAgg.getGroupByExpressions(), innerAgg.getGroupByExpressions())) { return outerAgg; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java index 1d52871a77..7598352e1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateGroupByConstant.java @@ -19,11 +19,14 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; @@ -46,13 +49,15 @@ import java.util.Set; public class EliminateGroupByConstant extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalAggregate().then(aggregate -> { + return logicalAggregate().thenApply(ctx -> { + LogicalAggregate aggregate = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); List groupByExprs = aggregate.getGroupByExpressions(); List outputExprs = aggregate.getOutputExpressions(); Set slotGroupByExprs = Sets.newLinkedHashSet(); Expression lit = null; for (Expression expression : groupByExprs) { - expression = FoldConstantRule.INSTANCE.rewrite(expression); + expression = FoldConstantRule.INSTANCE.rewrite(expression, context); if (!(expression instanceof Literal)) { slotGroupByExprs.add(expression); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateLimitUnderApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateLimitUnderApply.java index 2b66e52cd9..1f56660d9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateLimitUnderApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateLimitUnderApply.java @@ -32,7 +32,7 @@ import java.util.List; public class EliminateLimitUnderApply extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalLimit()).then(apply -> { + return logicalApply(any(), logicalLimit()).then(apply -> { List children = new ImmutableList.Builder() .add(apply.left()) .add(apply.right().child()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java index f7a03f5dfe..039e32ea5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java @@ -25,6 +25,8 @@ import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.nereids.util.TypeUtils; @@ -48,7 +50,8 @@ public class EliminateNotNull extends OneRewriteRuleFactory { public Rule build() { return logicalFilter() .when(filter -> filter.getConjuncts().stream().anyMatch(expr -> expr.isGeneratedIsNotNull)) - .then(filter -> { + .thenApply(ctx -> { + LogicalFilter filter = ctx.root; // Progress Example: `id > 0 and id is not null and name is not null(generated)` // predicatesNotContainIsNotNull: `id > 0` // predicatesNotContainIsNotNull infer nonNullable slots: `id` @@ -66,7 +69,8 @@ public class EliminateNotNull extends OneRewriteRuleFactory { predicatesNotContainIsNotNull.add(expr); } }); - Set inferNonNotSlots = ExpressionUtils.inferNotNullSlots(predicatesNotContainIsNotNull); + Set inferNonNotSlots = ExpressionUtils.inferNotNullSlots( + predicatesNotContainIsNotNull, ctx.cascadesContext); Set keepIsNotNull = slotsFromIsNotNull.stream() .filter(ExpressionTrait::nullable) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java index 33fe48fe61..0b6980e276 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java @@ -22,8 +22,8 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.util.TypeUtils; import org.apache.doris.nereids.util.Utils; @@ -45,7 +45,7 @@ public class EliminateOuterJoin extends OneRewriteRuleFactory { return logicalFilter( logicalJoin().when(join -> join.getJoinType().isOuterJoin()) ).then(filter -> { - LogicalJoin join = filter.child(); + LogicalJoin join = filter.child(); Builder conjunctsBuilder = ImmutableSet.builder(); Set notNullSlots = new HashSet<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateSortUnderApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateSortUnderApply.java index 62f092e7e6..64e61a0344 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateSortUnderApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateSortUnderApply.java @@ -34,7 +34,7 @@ public class EliminateSortUnderApply implements RewriteRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.ELIMINATE_SORT_UNDER_APPLY.build( - logicalApply(group(), logicalSort()).then(apply -> { + logicalApply(any(), logicalSort()).then(apply -> { List children = new ImmutableList.Builder() .add(apply.left()) .add(apply.right().child()) @@ -42,8 +42,8 @@ public class EliminateSortUnderApply implements RewriteRuleFactory { return apply.withChildren(children); }) ), - RuleType.ELIMINATE_SORT_UNDER_APPLY.build( - logicalApply(group(), logicalProject(logicalSort())).then(apply -> { + RuleType.ELIMINATE_SORT_UNDER_APPLY_PROJECT.build( + logicalApply(any(), logicalProject(logicalSort())).then(apply -> { List children = new ImmutableList.Builder() .add(apply.left()) .add(apply.right().withChildren(apply.right().child().child())) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java index 0754e6b6ee..f62ae4e96e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProject.java @@ -17,63 +17,92 @@ package org.apache.doris.nereids.rules.rewrite.logical; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.annotation.DependsRules; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.OutputSavePoint; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; -import com.google.common.collect.ImmutableList; - +import java.util.ArrayList; import java.util.List; /** * remove the project that output same with its child to avoid we get two consecutive projects in best plan. * for more information, please see this PR */ -public class EliminateUnnecessaryProject implements RewriteRuleFactory { +@DependsRules(ColumnPruning.class) +public class EliminateUnnecessaryProject implements CustomRewriter { + @Override - public List buildRules() { - return ImmutableList.of( - RuleType.MARK_NECESSARY_PROJECT.build( - logicalSetOperation(logicalProject(), group()) - .thenApply(ctx -> { - LogicalProject project = (LogicalProject) ctx.root.child(0); - return ctx.root.withChildren(project.withEliminate(false), ctx.root.child(1)); - }) - ), - RuleType.MARK_NECESSARY_PROJECT.build( - logicalSetOperation(group(), logicalProject()) - .thenApply(ctx -> { - LogicalProject project = (LogicalProject) ctx.root.child(1); - return ctx.root.withChildren(ctx.root.child(0), project.withEliminate(false)); - }) - ), - RuleType.ELIMINATE_UNNECESSARY_PROJECT.build( - logicalProject(any()) - .when(LogicalProject::canEliminate) - .when(project -> project.getOutputSet().equals(project.child().getOutputSet())) - .thenApply(ctx -> { - int rootGroupId = ctx.cascadesContext.getMemo().getRoot().getGroupId().asInt(); - LogicalProject project = ctx.root; - // if project is root, we need to ensure the output order is same. - if (project.getGroupExpression().get().getOwnerGroup().getGroupId().asInt() - == rootGroupId) { - if (project.getOutput().equals(project.child().getOutput())) { - return project.child(); - } else { - return null; - } - } else { - return project.child(); - } - }) - ), - RuleType.ELIMINATE_UNNECESSARY_PROJECT.build( - logicalProject(logicalEmptyRelation()) - .then(project -> new LogicalEmptyRelation(project.getProjects())) - ) - ); + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + return rewrite(plan, false); + } + + private Plan rewrite(Plan plan, boolean outputSavePoint) { + if (plan instanceof LogicalSetOperation) { + return rewriteLogicalSetOperation((LogicalSetOperation) plan, outputSavePoint); + } else if (plan instanceof LogicalProject) { + return rewriteProject((LogicalProject) plan, outputSavePoint); + } else if (plan instanceof OutputSavePoint) { + return rewriteChildren(plan, true); + } else { + return rewriteChildren(plan, outputSavePoint); + } + } + + private Plan rewriteProject(LogicalProject project, boolean outputSavePoint) { + if (project.child() instanceof LogicalEmptyRelation) { + // eliminate unnecessary project + return new LogicalEmptyRelation(project.getProjects()); + } else if (project.canEliminate() && outputSavePoint + && project.getOutputSet().equals(project.child().getOutputSet())) { + // eliminate unnecessary project + return rewrite(project.child(), outputSavePoint); + } else if (project.canEliminate() && project.getOutput().equals(project.child().getOutput())) { + // eliminate unnecessary project + return rewrite(project.child(), outputSavePoint); + } else { + return rewriteChildren(project, true); + } + } + + private Plan rewriteLogicalSetOperation(LogicalSetOperation set, boolean outputSavePoint) { + if (set.arity() == 2) { + Plan left = set.child(0); + Plan right = set.child(1); + boolean changed = false; + if (isCanEliminateProject(left)) { + changed = true; + left = ((LogicalProject) left).withEliminate(false); + } + if (isCanEliminateProject(right)) { + changed = true; + right = ((LogicalProject) right).withEliminate(false); + } + if (changed) { + set = (LogicalSetOperation) set.withChildren(left, right); + } + } + return rewriteChildren(set, outputSavePoint); + } + + private Plan rewriteChildren(Plan plan, boolean outputSavePoint) { + List newChildren = new ArrayList<>(); + boolean hasNewChildren = false; + for (Plan child : plan.children()) { + Plan newChild = rewrite(child, outputSavePoint); + if (newChild != child) { + hasNewChildren = true; + } + newChildren.add(newChild); + } + return hasNewChildren ? plan.withChildren(newChildren) : plan; + } + + private static boolean isCanEliminateProject(Plan plan) { + return plan instanceof LogicalProject && ((LogicalProject) plan).canEliminate(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java index f0adb815a4..6d01541d3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java @@ -21,8 +21,8 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanUtils; @@ -40,7 +40,7 @@ public class ExtractFilterFromCrossJoin extends OneRewriteRuleFactory { public Rule build() { return crossLogicalJoin() .then(join -> { - LogicalJoin newJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, + LogicalJoin newJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, join.getHint(), join.left(), join.right()); Set predicates = Stream.concat(join.getHashJoinConjuncts().stream(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunction.java index 3f88fb2533..5bb029e035 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunction.java @@ -115,7 +115,6 @@ public class ExtractSingleTableExpressionFromDisjunction extends OneRewriteRuleF redundants.add(ExpressionUtils.or(extractForAll)); } } - } if (redundants.isEmpty()) { return new LogicalFilter<>(filter.getConjuncts(), true, filter.child()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java index 6ee7546cf0..9700048e5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.annotation.DependsRules; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; @@ -43,6 +44,9 @@ import java.util.List; * CAUTION: * This rule must be applied after BindSlotReference */ +@DependsRules({ + PushFilterInsideJoin.class +}) public class FindHashConditionForJoin extends OneRewriteRuleFactory { @Override public Rule build() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java index 79e7305c1d..2577a31be2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/HideOneRowRelationUnderUnion.java @@ -50,7 +50,7 @@ public class HideOneRowRelationUnderUnion implements AnalysisRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build( - logicalUnion(logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode), group()) + logicalUnion(logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode), any()) .then(union -> { List newChildren = new ImmutableList.Builder() .add(((LogicalOneRowRelation) union.child(0)).withBuildUnionNode(false)) @@ -60,7 +60,7 @@ public class HideOneRowRelationUnderUnion implements AnalysisRuleFactory { }) ), RuleType.HIDE_ONE_ROW_RELATION_UNDER_UNION.build( - logicalUnion(group(), logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode)) + logicalUnion(any(), logicalOneRowRelation().when(LogicalOneRowRelation::buildUnionNode)) .then(union -> { List children = new ImmutableList.Builder() .add(union.child(0)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java index 02fac54ff3..f83f9369cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java @@ -21,6 +21,8 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.PlanUtils; @@ -43,9 +45,10 @@ public class InferFilterNotNull extends OneRewriteRuleFactory { public Rule build() { return logicalFilter() .when(filter -> filter.getConjuncts().stream().noneMatch(expr -> expr.isGeneratedIsNotNull)) - .then(filter -> { + .thenApply(ctx -> { + LogicalFilter filter = ctx.root; Set predicates = filter.getConjuncts(); - Set isNotNull = ExpressionUtils.inferNotNull(predicates); + Set isNotNull = ExpressionUtils.inferNotNull(predicates, ctx.cascadesContext); if (isNotNull.isEmpty() || predicates.containsAll(isNotNull)) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNull.java index c4ef028426..4887898637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNull.java @@ -44,7 +44,8 @@ public class InferJoinNotNull extends OneRewriteRuleFactory { // TODO: maybe consider ANTI? return logicalJoin().when(join -> join.getJoinType().isInnerJoin() || join.getJoinType().isSemiJoin()) .whenNot(LogicalJoin::isGenerateIsNotNull) - .then(join -> { + .thenApply(ctx -> { + LogicalJoin join = ctx.root; Set conjuncts = new HashSet<>(); conjuncts.addAll(join.getHashJoinConjuncts()); conjuncts.addAll(join.getOtherJoinConjuncts()); @@ -52,15 +53,19 @@ public class InferJoinNotNull extends OneRewriteRuleFactory { Plan left = join.left(); Plan right = join.right(); if (join.getJoinType().isInnerJoin()) { - Set leftNotNull = ExpressionUtils.inferNotNull(conjuncts, join.left().getOutputSet()); - Set rightNotNull = ExpressionUtils.inferNotNull(conjuncts, join.right().getOutputSet()); + Set leftNotNull = ExpressionUtils.inferNotNull( + conjuncts, join.left().getOutputSet(), ctx.cascadesContext); + Set rightNotNull = ExpressionUtils.inferNotNull( + conjuncts, join.right().getOutputSet(), ctx.cascadesContext); left = PlanUtils.filterOrSelf(leftNotNull, join.left()); right = PlanUtils.filterOrSelf(rightNotNull, join.right()); } else if (join.getJoinType() == JoinType.LEFT_SEMI_JOIN) { - Set leftNotNull = ExpressionUtils.inferNotNull(conjuncts, join.left().getOutputSet()); + Set leftNotNull = ExpressionUtils.inferNotNull( + conjuncts, join.left().getOutputSet(), ctx.cascadesContext); left = PlanUtils.filterOrSelf(leftNotNull, join.left()); } else { - Set rightNotNull = ExpressionUtils.inferNotNull(conjuncts, join.right().getOutputSet()); + Set rightNotNull = ExpressionUtils.inferNotNull( + conjuncts, join.right().getOutputSet(), ctx.cascadesContext); right = PlanUtils.filterOrSelf(rightNotNull, join.right()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicates.java index 12eb6f0e0a..38aa6bc6c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicates.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.util.ExpressionUtils; @@ -49,10 +50,15 @@ import java.util.stream.Collectors; * 2. put these predicates into `otherJoinConjuncts` , these predicates are processed in the next * round of predicate push-down */ -public class InferPredicates extends DefaultPlanRewriter { +public class InferPredicates extends DefaultPlanRewriter implements CustomRewriter { private final PredicatePropagation propagation = new PredicatePropagation(); private final PullUpPredicates pollUpPredicates = new PullUpPredicates(); + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + return plan.accept(this, jobContext); + } + @Override public Plan visitLogicalJoin(LogicalJoin join, JobContext context) { join = (LogicalJoin) super.visit(join, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeFilters.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeFilters.java index a992ca68c9..6b1cedd5f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeFilters.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeFilters.java @@ -30,16 +30,12 @@ import com.google.common.collect.ImmutableSet; * this rule aims to merge consecutive filters. * For example: * logical plan tree: - * project - * | * filter(a>0) * | * filter(b>0) * | * scan * transformed to: - * project - * | * filter(a>0 and b>0) * | * scan diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeGenerates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeGenerates.java index b596e0eba9..3a42bde534 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeGenerates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeGenerates.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.Function; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import com.google.common.collect.Lists; @@ -37,7 +37,7 @@ public class MergeGenerates extends OneRewriteRuleFactory { @Override public Rule build() { return logicalGenerate(logicalGenerate()).then(top -> { - LogicalGenerate bottom = top.child(); + LogicalGenerate bottom = top.child(); Set topGeneratorSlots = top.getInputSlots(); if (bottom.getGeneratorOutput().stream().anyMatch(topGeneratorSlots::contains)) { // top generators use bottom's generator's output, cannot merge. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java index 80dfcb7082..4afbbba883 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjects.java @@ -21,7 +21,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import java.util.List; @@ -47,7 +47,7 @@ public class MergeProjects extends OneRewriteRuleFactory { @Override public Rule build() { return logicalProject(logicalProject()).then(project -> { - LogicalProject childProject = project.child(); + LogicalProject childProject = project.child(); List projectExpressions = project.mergeProjections(childProject); return new LogicalProject<>(projectExpressions, childProject.child(0)); }).toRule(RuleType.MERGE_PROJECTS); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java index d3a7cf1526..56f924684c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeSetOperations.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.stream.Stream; /** * optimization. @@ -51,48 +52,44 @@ public class MergeSetOperations implements RewriteRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.MERGE_SET_OPERATION.build( - logicalSetOperation(logicalSetOperation(), group()).thenApply(ctx -> { - LogicalSetOperation parentSetOperation = ctx.root; - LogicalSetOperation childSetOperation = (LogicalSetOperation) parentSetOperation.child(0); + logicalSetOperation(any(), any()).when(MergeSetOperations::canMerge).then(parentSetOperation -> { + List newChildren = parentSetOperation.children() + .stream() + .flatMap(child -> { + if (canMerge(parentSetOperation, child)) { + return child.children().stream(); + } else { + return Stream.of(child); + } + }).collect(ImmutableList.toImmutableList()); - if (isSameClass(parentSetOperation, childSetOperation) - && isSameQualifierOrChildQualifierIsAll(parentSetOperation, childSetOperation)) { - List newChildren = new ImmutableList.Builder() - .addAll(childSetOperation.children()) - .add(parentSetOperation.child(1)) - .build(); - return parentSetOperation.withChildren(newChildren); - } - return parentSetOperation; - }) - ), - RuleType.MERGE_SET_OPERATION.build( - logicalSetOperation(group(), logicalSetOperation()).thenApply(ctx -> { - LogicalSetOperation parentSetOperation = ctx.root; - LogicalSetOperation childSetOperation = (LogicalSetOperation) parentSetOperation.child(1); - - if (isSameClass(parentSetOperation, childSetOperation) - && isSameQualifierOrChildQualifierIsAll(parentSetOperation, childSetOperation)) { - List newChildren = new ImmutableList.Builder() - .add(parentSetOperation.child(0)) - .addAll(childSetOperation.children()) - .build(); - return parentSetOperation.withNewChildren(newChildren); - } - return parentSetOperation; + return parentSetOperation.withChildren(newChildren); }) ) ); } - private boolean isSameQualifierOrChildQualifierIsAll(LogicalSetOperation parentSetOperation, + /** canMerge */ + public static boolean canMerge(LogicalSetOperation parent) { + Plan left = parent.child(0); + if (canMerge(parent, left)) { + return true; + } + Plan right = parent.child(1); + if (canMerge(parent, right)) { + return true; + } + return false; + } + + public static final boolean canMerge(LogicalSetOperation parent, Plan child) { + return child.getClass().equals(parent.getClass()) + && isSameQualifierOrChildQualifierIsAll(parent, (LogicalSetOperation) child); + } + + public static final boolean isSameQualifierOrChildQualifierIsAll(LogicalSetOperation parentSetOperation, LogicalSetOperation childSetOperation) { return parentSetOperation.getQualifier() == childSetOperation.getQualifier() || childSetOperation.getQualifier() == Qualifier.ALL; } - - private boolean isSameClass(LogicalSetOperation parentSetOperation, - LogicalSetOperation childSetOperation) { - return parentSetOperation.getClass().isAssignableFrom(childSetOperation.getClass()); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java index d8a31543fa..47df6ad4f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneAggChildColumns.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.ExpressionUtils; @@ -78,7 +78,7 @@ public class PruneAggChildColumns extends OneRewriteRuleFactory { * @return null, if there exists an aggregation function that its parameters contains non-constant expr. * else return a slot with min data type. */ - private boolean isAggregateWithConstant(LogicalAggregate agg) { + private boolean isAggregateWithConstant(LogicalAggregate agg) { for (NamedExpression output : agg.getOutputExpressions()) { if (output.anyMatch(SlotReference.class::isInstance)) { return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java index bcd925338e..b1f28def1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneFilterChildColumns.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -51,7 +50,7 @@ import java.util.stream.Stream; * | * scan(k1,k2,k3,v1) */ -public class PruneFilterChildColumns extends AbstractPushDownProjectRule> { +public class PruneFilterChildColumns extends AbstractPushDownProjectRule> { public PruneFilterChildColumns() { setRuleType(RuleType.COLUMN_PRUNE_FILTER_CHILD); @@ -59,7 +58,7 @@ public class PruneFilterChildColumns extends AbstractPushDownProjectRule filter, Set references) { + protected Plan pushDownProject(LogicalFilter filter, Set references) { Set filterInputSlots = filter.getInputSlots(); Set required = Stream.concat(references.stream(), filterInputSlots.stream()).collect(Collectors.toSet()); if (required.containsAll(filter.child().getOutput())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java index f05271d32d..daf4ad3a3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneJoinChildrenColumns.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -56,7 +55,7 @@ import java.util.stream.Stream; * scan scan */ public class PruneJoinChildrenColumns - extends AbstractPushDownProjectRule> { + extends AbstractPushDownProjectRule> { public PruneJoinChildrenColumns() { setRuleType(RuleType.COLUMN_PRUNE_JOIN_CHILD); @@ -64,7 +63,7 @@ public class PruneJoinChildrenColumns } @Override - protected Plan pushDownProject(LogicalJoin joinPlan, + protected Plan pushDownProject(LogicalJoin joinPlan, Set references) { Set exprIds = Stream.of(references, joinPlan.getInputSlots()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java index 372d03f8d0..90adbd067a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneSortChildColumns.java @@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; @@ -34,7 +33,7 @@ import java.util.stream.Stream; * prune join children output. * pattern: project(sort()) */ -public class PruneSortChildColumns extends AbstractPushDownProjectRule> { +public class PruneSortChildColumns extends AbstractPushDownProjectRule> { public PruneSortChildColumns() { setRuleType(RuleType.COLUMN_PRUNE_SORT_CHILD); @@ -42,7 +41,7 @@ public class PruneSortChildColumns extends AbstractPushDownProjectRule sortPlan, Set references) { + protected Plan pushDownProject(LogicalSort sortPlan, Set references) { Set sortSlots = sortPlan.getOutputSet(); Set required = Stream.concat(references.stream(), sortSlots.stream()).collect(Collectors.toSet()); if (required.containsAll(sortPlan.child().getOutput())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java similarity index 82% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java index 232d5d3190..403008a9d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnProjectUnderAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java @@ -21,12 +21,13 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.util.List; @@ -57,15 +58,15 @@ import java.util.List; * child * */ -public class ApplyPullFilterOnProjectUnderAgg extends OneRewriteRuleFactory { +public class PullUpCorrelatedFilterUnderApplyAggregateProject extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalAggregate(logicalProject(logicalFilter()))) + return logicalApply(any(), logicalAggregate(logicalProject(logicalFilter()))) .when(LogicalApply::isCorrelated).then(apply -> { - LogicalAggregate>> agg = apply.right(); + LogicalAggregate>> agg = apply.right(); - LogicalProject> project = agg.child(); - LogicalFilter filter = project.child(); + LogicalProject> project = agg.child(); + LogicalFilter filter = project.child(); List newProjects = Lists.newArrayList(); newProjects.addAll(project.getProjects()); filter.child().getOutput().forEach(slot -> { @@ -76,10 +77,9 @@ public class ApplyPullFilterOnProjectUnderAgg extends OneRewriteRuleFactory { LogicalProject newProject = new LogicalProject<>(newProjects, filter.child()); LogicalFilter newFilter = new LogicalFilter<>(filter.getConjuncts(), newProject); - LogicalAggregate newAgg = new LogicalAggregate<>(agg.getGroupByExpressions(), - agg.getOutputExpressions(), agg.isOrdinalIsResolved(), newFilter); + LogicalAggregate newAgg = agg.withChildren(ImmutableList.of(newFilter)); return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), apply.getCorrelationFilter(), apply.left(), newAgg); - }).toRule(RuleType.APPLY_PULL_FILTER_ON_PROJECT_UNDER_AGG); + }).toRule(RuleType.PULL_UP_CORRELATED_FILTER_UNDER_APPLY_AGGREGATE_PROJECT); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java similarity index 82% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderProject.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java index 0945b2706e..7f993f1798 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java @@ -20,8 +20,8 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -47,10 +47,10 @@ import java.util.List; * / \ * Input(output:b) child */ -public class PushApplyUnderProject extends OneRewriteRuleFactory { +public class PullUpProjectUnderApply extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalProject(any())) + return logicalApply(any(), logicalProject(any())) .when(LogicalApply::isCorrelated) .whenNot(apply -> apply.right().child() instanceof LogicalFilter && apply.isIn()) .whenNot(LogicalApply::alreadyExecutedEliminateFilter) @@ -58,12 +58,12 @@ public class PushApplyUnderProject extends OneRewriteRuleFactory { LogicalProject project = apply.right(); LogicalApply newCorrelate = new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), apply.getCorrelationFilter(), apply.left(), project.child()); - List newSlots = new ArrayList<>(); - newSlots.addAll(apply.left().getOutput()); + List newProjects = new ArrayList<>(); + newProjects.addAll(apply.left().getOutput()); if (apply.getSubqueryExpr() instanceof ScalarSubquery) { - newSlots.add(apply.right().getOutput().get(0)); + newProjects.add(project.getProjects().get(0)); } - return new LogicalProject(newSlots, newCorrelate); - }).toRule(RuleType.PUSH_APPLY_UNDER_PROJECT); + return new LogicalProject(newProjects, newCorrelate); + }).toRule(RuleType.PULL_UP_PROJECT_UNDER_APPLY); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java index 0ea7f62f86..b2d192c163 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java @@ -17,11 +17,12 @@ package org.apache.doris.nereids.rules.rewrite.logical; +import org.apache.doris.nereids.annotation.DependsRules; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import com.google.common.collect.Lists; @@ -31,6 +32,10 @@ import java.util.List; /** * Push the predicate in the LogicalFilter to the join children. */ +@DependsRules({ + InferPredicates.class, + EliminateOuterJoin.class +}) public class PushFilterInsideJoin extends OneRewriteRuleFactory { @Override public Rule build() { @@ -40,7 +45,7 @@ public class PushFilterInsideJoin extends OneRewriteRuleFactory { || filter.child().getJoinType().isInnerJoin()) .then(filter -> { List otherConditions = Lists.newArrayList(filter.getConjuncts()); - LogicalJoin join = filter.child(); + LogicalJoin join = filter.child(); otherConditions.addAll(join.getOtherJoinConjuncts()); return new LogicalJoin<>(join.getJoinType(), join.getHashJoinConjuncts(), otherConditions, join.getHint(), join.left(), join.right()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregation.java index 575f5c419e..a891eb07ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregation.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -66,7 +65,7 @@ public class PushdownFilterThroughAggregation extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter(logicalAggregate()).then(filter -> { - LogicalAggregate aggregate = filter.child(); + LogicalAggregate aggregate = filter.child(); Set canPushDownSlots = new HashSet<>(); if (aggregate.hasRepeat()) { // When there is a repeat, the push-down condition is consistent with the repeat diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java index e95f7bb229..64ff0cd514 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java @@ -24,8 +24,8 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.util.PlanUtils; @@ -94,7 +94,7 @@ public class PushdownFilterThroughJoin extends OneRewriteRuleFactory { public Rule build() { return logicalFilter(logicalJoin()).then(filter -> { - LogicalJoin join = filter.child(); + LogicalJoin join = filter.child(); Set predicates = filter.getConjuncts(); @@ -113,9 +113,9 @@ public class PushdownFilterThroughJoin extends OneRewriteRuleFactory { } } - Set leftPredicates = Sets.newHashSet(); - Set rightPredicates = Sets.newHashSet(); - Set remainingPredicates = Sets.newHashSet(); + Set leftPredicates = Sets.newLinkedHashSet(); + Set rightPredicates = Sets.newLinkedHashSet(); + Set remainingPredicates = Sets.newLinkedHashSet(); for (Expression p : filterPredicates) { Set slots = p.collect(SlotReference.class::isInstance); if (slots.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java index ac56572491..8712340f7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java @@ -19,12 +19,17 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.ImmutableList; + +import java.util.List; + /** * Push down filter through project. * input: @@ -32,18 +37,38 @@ import org.apache.doris.nereids.util.ExpressionUtils; * output: * project(c+d as a, e as b) -> filter(c+d>2, e=0). */ -public class PushdownFilterThroughProject extends OneRewriteRuleFactory { +public class PushdownFilterThroughProject implements RewriteRuleFactory { @Override - public Rule build() { - return logicalFilter(logicalProject()).then(filter -> { - LogicalProject project = filter.child(); - return new LogicalProject<>( - project.getProjects(), - new LogicalFilter<>( - ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()), - project.child() - ) - ); - }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT); + public List buildRules() { + return ImmutableList.of( + RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT.build( + logicalFilter(logicalProject()).then(filter -> { + LogicalProject project = filter.child(); + return new LogicalProject<>( + project.getProjects(), + new LogicalFilter<>( + ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()), + project.child() + ) + ); + }) + ), + // filter(project(limit)) will change to filter(limit(project)) by PushdownProjectThroughLimit, + // then we should change filter(limit(project)) to project(filter(limit)) + RuleType.PUSHDOWN_FILTER_THROUGH_PROJECT_UNDER_LIMIT.build( + logicalFilter(logicalLimit(logicalProject())).then(filter -> { + LogicalLimit> limit = filter.child(); + LogicalProject project = limit.child(); + + return new LogicalProject<>( + project.getProjects(), + new LogicalFilter<>( + ExpressionUtils.replace(filter.getConjuncts(), project.getAliasToProducer()), + limit.withChildren(project.child()) + ) + ); + }) + ) + ); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughRepeat.java index 0d5f2c3f64..e19b364042 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughRepeat.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; @@ -65,7 +64,7 @@ public class PushdownFilterThroughRepeat extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter(logicalRepeat()).then(filter -> { - LogicalRepeat repeat = filter.child(); + LogicalRepeat repeat = filter.child(); Set commonGroupingSetExpressions = repeat.getCommonGroupingSetExpressions(); if (commonGroupingSetExpressions.isEmpty()) { return filter; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java index 4faed0a5cf..b9f0f70d2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java @@ -20,7 +20,7 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -48,10 +48,10 @@ public class PushdownProjectThroughLimit extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalProject(logicalLimit(group())).thenApply(ctx -> { - LogicalProject> logicalProject = ctx.root; - LogicalLimit logicalLimit = logicalProject.child(); - return new LogicalLimit>(logicalLimit.getLimit(), + return logicalProject(logicalLimit(any())).thenApply(ctx -> { + LogicalProject> logicalProject = ctx.root; + LogicalLimit logicalLimit = logicalProject.child(); + return new LogicalLimit<>(logicalLimit.getLimit(), logicalLimit.getOffset(), new LogicalProject<>(logicalProject.getProjects(), logicalLimit.child())); }).toRule(RuleType.PUSHDOWN_PROJECT_THROUGH_LIMIT); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java index caea65b552..ede6586a3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java @@ -18,12 +18,12 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.annotation.DependsRules; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinHint.JoinHintType; import org.apache.doris.nereids.trees.plans.JoinType; @@ -68,6 +68,9 @@ import java.util.stream.Collectors; *
  • MultiJoin to {Join cluster}
    • * */ +@DependsRules({ + MergeFilters.class +}) public class ReorderJoin extends OneRewriteRuleFactory { @Override public Rule build() { @@ -94,8 +97,8 @@ public class ReorderJoin extends OneRewriteRuleFactory { */ public Plan joinToMultiJoin(Plan plan, Map planToHintType) { // subtree can't specify the end of Pattern. so end can be GroupPlan or Filter - if (plan instanceof GroupPlan - || (plan instanceof LogicalFilter && plan.child(0) instanceof GroupPlan)) { + if (nonJoinAndNonFilter(plan) + || (plan instanceof LogicalFilter && nonJoinAndNonFilter(plan.child(0)))) { return plan; } @@ -368,4 +371,8 @@ public class ReorderJoin extends OneRewriteRuleFactory { throw new RuntimeException("findInnerJoin: can't reach here"); } + + private boolean nonJoinAndNonFilter(Plan plan) { + return !(plan instanceof LogicalJoin) && !(plan instanceof LogicalFilter); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java similarity index 89% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java index 8998ba6afe..2b85431711 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -59,12 +59,12 @@ import java.util.Map; * Filter(Uncorrelated predicate) * */ -public class ApplyPullFilterOnAgg extends OneRewriteRuleFactory { +public class UnCorrelatedApplyAggregateFilter extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalAggregate(logicalFilter())).when(LogicalApply::isCorrelated).then(apply -> { - LogicalAggregate> agg = apply.right(); - LogicalFilter filter = agg.child(); + return logicalApply(any(), logicalAggregate(logicalFilter())).when(LogicalApply::isCorrelated).then(apply -> { + LogicalAggregate> agg = apply.right(); + LogicalFilter filter = agg.child(); Map> split = Utils.splitCorrelatedConjuncts( filter.getConjuncts(), apply.getCorrelationSlot()); List correlatedPredicate = split.get(true); @@ -89,6 +89,6 @@ public class ApplyPullFilterOnAgg extends OneRewriteRuleFactory { apply.getSubqueryExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), apply.left(), newAgg); - }).toRule(RuleType.APPLY_PULL_FILTER_ON_AGG); + }).toRule(RuleType.UN_CORRELATED_APPLY_AGGREGATE_FILTER); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java similarity index 89% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java index b8833a7a9d..1ffbcf2214 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -51,11 +50,11 @@ import java.util.Set; * / \ * Input(output:b) Filter(UnCorrelated predicate) */ -public class PushApplyUnderFilter extends OneRewriteRuleFactory { +public class UnCorrelatedApplyFilter extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalFilter()).when(LogicalApply::isCorrelated).then(apply -> { - LogicalFilter filter = apply.right(); + return logicalApply(any(), logicalFilter()).when(LogicalApply::isCorrelated).then(apply -> { + LogicalFilter filter = apply.right(); Set conjuncts = filter.getConjuncts(); Map> split = Utils.splitCorrelatedConjuncts( conjuncts, apply.getCorrelationSlot()); @@ -71,6 +70,6 @@ public class PushApplyUnderFilter extends OneRewriteRuleFactory { return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), apply.left(), child); - }).toRule(RuleType.PUSH_APPLY_UNDER_FILTER); + }).toRule(RuleType.UN_CORRELATED_APPLY_FILTER); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateFilterUnderApplyProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java similarity index 90% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateFilterUnderApplyProject.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java index e72302b9cb..23d9ec025a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateFilterUnderApplyProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -60,15 +59,15 @@ import java.util.Set; * | * child */ -public class EliminateFilterUnderApplyProject extends OneRewriteRuleFactory { +public class UnCorrelatedApplyProjectFilter extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalApply(group(), logicalProject(logicalFilter())) + return logicalApply(any(), logicalProject(logicalFilter())) .when(LogicalApply::isCorrelated) .when(LogicalApply::isIn) .then(apply -> { - LogicalProject> project = apply.right(); - LogicalFilter filter = project.child(); + LogicalProject> project = apply.right(); + LogicalFilter filter = project.child(); Set conjuncts = filter.getConjuncts(); Map> split = Utils.splitCorrelatedConjuncts( conjuncts, apply.getCorrelationSlot()); @@ -92,6 +91,6 @@ public class EliminateFilterUnderApplyProject extends OneRewriteRuleFactory { return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), apply.left(), newProject); - }).toRule(RuleType.ELIMINATE_FILTER_UNDER_APPLY_PROJECT); + }).toRule(RuleType.UN_CORRELATED_APPLY_PROJECT_FILTER); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 0320e3ee95..d8f10a362d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -179,4 +179,21 @@ public interface TreeNode> { return false; }); } + + /** + * equals by the full tree nodes + * @param that other tree node + * @return true if all the tree is equals + */ + default boolean deepEquals(TreeNode that) { + if (!equals(that)) { + return false; + } + for (int i = 0; i < arity(); i++) { + if (!child(i).deepEquals(that.child(i))) { + return false; + } + } + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java index d9cd7c695e..7f2628e03f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Exists.java @@ -77,15 +77,11 @@ public class Exists extends SubqueryExpr implements LeafExpression { @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { + if (!super.equals(o)) { return false; } Exists other = (Exists) o; - return Objects.equals(this.queryPlan, other.getQueryPlan()) - && Objects.equals(this.isNot, other.isNot()); + return this.isNot == other.isNot; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index d3080b283c..06697f82da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -199,7 +199,15 @@ public abstract class Expression extends AbstractTreeNode implements * This expression has unbound symbols or not. */ public boolean hasUnbound() { - return this.anyMatch(Unbound.class::isInstance); + if (this instanceof Unbound) { + return true; + } + for (Expression child : children) { + if (child.hasUnbound()) { + return true; + } + } + return false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java index 77423b165f..fe1dc5428f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InSubquery.java @@ -95,16 +95,12 @@ public class InSubquery extends SubqueryExpr { @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { + if (!super.equals(o)) { return false; } InSubquery inSubquery = (InSubquery) o; return Objects.equals(this.compareExpr, inSubquery.getCompareExpr()) - && checkEquals(this.queryPlan, inSubquery.queryPlan) - && Objects.equals(this.isNot, inSubquery.isNot); + && this.isNot == inSubquery.isNot; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java index 047c985de0..8e5e9080df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/IsNull.java @@ -60,7 +60,7 @@ public class IsNull extends Expression implements UnaryExpression { @Override public String toString() { - return toSql(); + return child().toString() + " IS NULL"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index e378b580ee..b0f4e829de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -148,7 +148,8 @@ public class SlotReference extends Slot { // The contains method needs to use hashCode, so similar to equals, it only compares exprId @Override public int hashCode() { - return Objects.hash(exprId); + // direct return exprId to speed up + return exprId.asInt(); } public Optional getColumn() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java index 44146728c9..d98a72f06a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SubqueryExpr.java @@ -77,6 +77,11 @@ public abstract class SubqueryExpr extends Expression { return queryPlan; } + @Override + public boolean hasUnbound() { + return super.hasUnbound() || !queryPlan.bound(); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -86,39 +91,8 @@ public abstract class SubqueryExpr extends Expression { return false; } SubqueryExpr other = (SubqueryExpr) o; - return checkEquals(queryPlan, other.queryPlan) - && Objects.equals(correlateSlots, other.correlateSlots); - } - - /** - * Compare whether all logical nodes under query are the same. - * @param i original query. - * @param o compared query. - * @return equal ? true : false; - */ - protected boolean checkEquals(Object i, Object o) { - if (!(i instanceof LogicalPlan) || !(o instanceof LogicalPlan)) { - return false; - } - LogicalPlan other = (LogicalPlan) o; - LogicalPlan input = (LogicalPlan) i; - if (other.children().size() != input.children().size()) { - return false; - } - boolean equal; - for (int j = 0; j < input.children().size(); j++) { - LogicalPlan childInput = (LogicalPlan) input.child(j); - LogicalPlan childOther = (LogicalPlan) other.child(j); - equal = Objects.equals(childInput, childOther); - if (!equal) { - return false; - } - if (childInput.children().size() != childOther.children().size()) { - return false; - } - checkEquals(childInput, childOther); - } - return true; + return Objects.equals(correlateSlots, other.correlateSlots) + && queryPlan.deepEquals(other.queryPlan); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index f3f829eade..198a28742a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.plans; import org.apache.doris.nereids.analyzer.Unbound; -import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.metrics.CounterType; @@ -32,6 +31,8 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.util.MutableState; +import org.apache.doris.nereids.util.MutableState.EmptyMutableState; import org.apache.doris.nereids.util.TreeStringUtils; import org.apache.doris.statistics.StatsDeriveResult; @@ -52,11 +53,19 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla EventChannel.getDefaultChannel() .addEnhancers(new AddCounterEventEnhancer()) .addConsumers(new LogConsumer(CounterEvent.class, EventChannel.LOG))); + protected final StatsDeriveResult statsDeriveResult; protected final PlanType type; protected final Optional groupExpression; protected final Supplier logicalPropertiesSupplier; + // this field is special, because other fields in tree node is immutable, but in some scenes, mutable + // state is necessary. e.g. the rewrite framework need distinguish whether the plan is created by + // rules, the framework can set this field to a state variable to quickly judge without new big plan. + // we should avoid using it as much as possible, because mutable state is easy to cause bugs and + // difficult to locate. + private MutableState mutableState = EmptyMutableState.INSTANCE; + public AbstractPlan(PlanType type, Plan... children) { this(type, Optional.empty(), Optional.empty(), null, children); } @@ -96,9 +105,7 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla @Override public boolean canBind() { - return !bound() - && !(this instanceof Unbound) - && childrenBound(); + return !bound() && childrenBound(); } /** @@ -161,9 +168,19 @@ public abstract class AbstractPlan extends AbstractTreeNode implements Pla @Override public LogicalProperties getLogicalProperties() { - if (this instanceof UnboundRelation) { + if (this instanceof Unbound) { return UnboundLogicalProperties.INSTANCE; } return logicalPropertiesSupplier.get(); } + + @Override + public Optional getMutableState(String key) { + return mutableState.get(key); + } + + @Override + public void setMutableState(String key, Object state) { + this.mutableState = this.mutableState.set(key, state); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java index 91a392732f..1708878fa4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java @@ -22,6 +22,8 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.MutableState; +import org.apache.doris.nereids.util.MutableState.MultiMutableState; import com.google.common.collect.ImmutableList; @@ -33,6 +35,7 @@ import java.util.Optional; * Used for unit test only. */ public class FakePlan implements Plan { + private MutableState mutableState = new MultiMutableState(); @Override public List children() { @@ -108,4 +111,14 @@ public class FakePlan implements Plan { public Plan withLogicalProperties(Optional logicalProperties) { return this; } + + @Override + public Optional getMutableState(String key) { + return (Optional) Optional.ofNullable(mutableState.get(key)); + } + + @Override + public void setMutableState(String key, Object mutableState) { + this.mutableState = this.mutableState.set(key, mutableState); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index e0702937a3..018af58ba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -129,4 +130,19 @@ public interface Plan extends TreeNode { Plan withGroupExpression(Optional groupExpression); Plan withLogicalProperties(Optional logicalProperties); + + Optional getMutableState(String key); + + /** getOrInitMutableState */ + default T getOrInitMutableState(String key, Supplier initState) { + Optional mutableState = getMutableState(key); + if (!mutableState.isPresent()) { + T state = initState.get(); + setMutableState(key, state); + return state; + } + return mutableState.get(); + } + + void setMutableState(String key, Object value); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 5afe8b4cf6..50ce5ff669 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -83,5 +83,9 @@ public enum PlanType { PHYSICAL_ASSERT_NUM_ROWS, PHYSICAL_UNION, PHYSICAL_EXCEPT, - PHYSICAL_INTERSECT + PHYSICAL_INTERSECT, + + COMMAND, + EXPLAIN_COMMAND, + CREATE_POLICY_COMMAND } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java index 9cec62708b..9bf9fb04b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/Command.java @@ -21,9 +21,13 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.statistics.StatsDeriveResult; + +import org.jetbrains.annotations.Nullable; import java.util.List; import java.util.Optional; @@ -31,75 +35,90 @@ import java.util.Optional; /** * All DDL and DML commands' super class. */ -public interface Command extends LogicalPlan { +public abstract class Command extends AbstractPlan implements LogicalPlan { + + public Command(PlanType type, Plan... children) { + super(type, children); + } + + public Command(PlanType type, Optional optLogicalProperties, Plan... children) { + super(type, optLogicalProperties, children); + } + + public Command(PlanType type, Optional groupExpression, + Optional optLogicalProperties, + @Nullable StatsDeriveResult statsDeriveResult, + Plan... children) { + super(type, groupExpression, optLogicalProperties, statsDeriveResult, children); + } @Override - default Optional getGroupExpression() { + public Optional getGroupExpression() { throw new RuntimeException("Command do not implement getGroupExpression"); } @Override - default List children() { + public List children() { throw new RuntimeException("Command do not implement children"); } @Override - default Plan child(int index) { + public Plan child(int index) { throw new RuntimeException("Command do not implement child"); } @Override - default int arity() { + public int arity() { throw new RuntimeException("Command do not implement arity"); } @Override - default Plan withChildren(List children) { + public Plan withChildren(List children) { throw new RuntimeException("Command do not implement withChildren"); } @Override - default PlanType getType() { + public PlanType getType() { throw new RuntimeException("Command do not implement getType"); } @Override - default List getExpressions() { + public List getExpressions() { throw new RuntimeException("Command do not implement getExpressions"); } @Override - default LogicalProperties getLogicalProperties() { + public LogicalProperties getLogicalProperties() { throw new RuntimeException("Command do not implement getLogicalProperties"); } @Override - default boolean canBind() { + public boolean canBind() { throw new RuntimeException("Command do not implement canResolve"); } @Override - default List getOutput() { + public List getOutput() { throw new RuntimeException("Command do not implement getOutput"); } @Override - default List getNonUserVisibleOutput() { + public List getNonUserVisibleOutput() { throw new RuntimeException("Command do not implement getNonUserVisibleOutput"); } @Override - default String treeString() { + public String treeString() { throw new RuntimeException("Command do not implement treeString"); } @Override - default Plan withGroupExpression(Optional groupExpression) { + public Plan withGroupExpression(Optional groupExpression) { throw new RuntimeException("Command do not implement withGroupExpression"); } @Override - default Plan withLogicalProperties(Optional logicalProperties) { + public Plan withLogicalProperties(Optional logicalProperties) { throw new RuntimeException("Command do not implement withLogicalProperties"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java index f6ddc8217e..a405a61cb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreatePolicyCommand.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.policy.PolicyTypeEnum; @@ -26,13 +27,14 @@ import java.util.Optional; /** * Create policy command. */ -public class CreatePolicyCommand implements Command { +public class CreatePolicyCommand extends Command { private PolicyTypeEnum type; private final Optional wherePredicate; public CreatePolicyCommand(PolicyTypeEnum type, Expression expr) { + super(PlanType.CREATE_POLICY_COMMAND); this.type = type; this.wherePredicate = Optional.of(expr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index adf9cd5ba4..c96a95cc23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -17,13 +17,14 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; /** * explain command. */ -public class ExplainCommand implements Command { +public class ExplainCommand extends Command { /** * explain level. @@ -51,6 +52,7 @@ public class ExplainCommand implements Command { private final LogicalPlan logicalPlan; public ExplainCommand(ExplainLevel level, LogicalPlan logicalPlan) { + super(PlanType.EXPLAIN_COMMAND); this.level = level; this.logicalPlan = logicalPlan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java index feef64e3a6..704a0dd222 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/AbstractLogicalPlan.java @@ -25,12 +25,15 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import java.util.Optional; +import java.util.function.Supplier; /** * Abstract class for all concrete logical plan. */ public abstract class AbstractLogicalPlan extends AbstractPlan implements LogicalPlan { + private Supplier hasUnboundExpressions = () -> super.hasUnboundExpression(); + public AbstractLogicalPlan(PlanType type, Plan... children) { super(type, children); } @@ -44,9 +47,15 @@ public abstract class AbstractLogicalPlan extends AbstractPlan implements Logica super(type, groupExpression, logicalProperties, null, children); } + @Override + public boolean hasUnboundExpression() { + return hasUnboundExpressions.get(); + } + @Override public LogicalProperties computeLogicalProperties() { - boolean hasUnboundChild = children.stream().map(Plan::getLogicalProperties) + boolean hasUnboundChild = children.stream() + .map(Plan::getLogicalProperties) .anyMatch(UnboundLogicalProperties.class::isInstance); if (hasUnboundChild || hasUnboundExpression()) { return UnboundLogicalProperties.INSTANCE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index b844c7eafa..6f3c0778de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -52,7 +52,7 @@ import java.util.Optional; */ public class LogicalAggregate extends LogicalUnary - implements Aggregate { + implements Aggregate, OutputSavePoint { private final boolean normalized; private final List groupByExpressions; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeaf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeaf.java index 56101773ee..f874ae8c4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeaf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLeaf.java @@ -29,7 +29,7 @@ import java.util.Optional; /** * Abstract class for all logical plan that have no child. */ -public abstract class LogicalLeaf extends AbstractLogicalPlan implements LeafPlan { +public abstract class LogicalLeaf extends AbstractLogicalPlan implements LeafPlan, OutputSavePoint { public LogicalLeaf(PlanType nodeType, Optional groupExpression, Optional logicalProperties) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index db4b9e2f0e..3d6a9f7363 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -297,6 +297,9 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, @Override public List computeNonUserVisibleOutput() { + // if (getTable().getIndexIdToMeta().size() == 1) { + // return ImmutableList.of(); + // } Set baseSchemaColNames = table.getBaseSchema().stream() .map(Column::getName) .collect(Collectors.toSet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 12037f81b2..101031220b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -40,7 +40,8 @@ import java.util.Optional; /** * Logical project plan. */ -public class LogicalProject extends LogicalUnary implements Project { +public class LogicalProject extends LogicalUnary + implements Project, OutputSavePoint { private final List projects; private final List excepts; @@ -116,6 +117,7 @@ public class LogicalProject extends LogicalUnary extends LogicalUnary implements Repeat { +public class LogicalRepeat extends LogicalUnary + implements Repeat, OutputSavePoint { // max num of distinct sets in grouping sets clause public static final int MAX_GROUPING_SETS_NUM = 64; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index 0be6e0efdf..b0534d04f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.plans.logical; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -51,7 +52,7 @@ import java.util.Optional; * * eg: select k1, k2 from t1 union select 1, 2 union select d1, d2 from t2; */ -public abstract class LogicalSetOperation extends AbstractLogicalPlan implements SetOperation { +public abstract class LogicalSetOperation extends AbstractLogicalPlan implements SetOperation, OutputSavePoint { // eg value: qualifier:DISTINCT protected final Qualifier qualifier; @@ -82,6 +83,19 @@ public abstract class LogicalSetOperation extends AbstractLogicalPlan implements this.outputs = ImmutableList.copyOf(outputs); } + @Override + public boolean hasUnboundExpression() { + return outputs.isEmpty() || super.hasUnboundExpression(); + } + + @Override + public LogicalProperties computeLogicalProperties() { + if (outputs.isEmpty()) { + return UnboundLogicalProperties.INSTANCE; + } + return super.computeLogicalProperties(); + } + @Override public List computeOutput() { return outputs.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputSavePoint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputSavePoint.java new file mode 100644 index 0000000000..fa566d14c1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/OutputSavePoint.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +/** OutputSavePoint is used to point out a plan whether exist output field */ +public interface OutputSavePoint { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CheckAnalysisJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CustomRewriter.java similarity index 60% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CheckAnalysisJob.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CustomRewriter.java index 45da20f408..fe719ee1f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CheckAnalysisJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CustomRewriter.java @@ -15,21 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.jobs.batch; +package org.apache.doris.nereids.trees.plans.visitor; -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.analysis.CheckAnalysis; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.trees.plans.Plan; -import com.google.common.collect.ImmutableList; +/** CustomRewriter */ +public interface CustomRewriter { -/** - * Execute check analysis rules. - */ -public class CheckAnalysisJob extends BatchRulesJob { - public CheckAnalysisJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - bottomUpBatch(ImmutableList.of(new CheckAnalysis())) - )); - } + // entrance method + Plan rewriteRoot(Plan plan, JobContext jobContext); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 6128aa39dc..f9cbd57a89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.util; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule; import org.apache.doris.nereids.trees.TreeNode; @@ -344,7 +345,7 @@ public class ExpressionUtils { /** * infer notNulls slot from predicate */ - public static Set inferNotNullSlots(Set predicates) { + public static Set inferNotNullSlots(Set predicates, CascadesContext cascadesContext) { Literal nullLiteral = Literal.of(null); Set notNullSlots = Sets.newHashSet(); for (Expression predicate : predicates) { @@ -353,7 +354,7 @@ public class ExpressionUtils { replaceMap.put(slot, nullLiteral); Expression evalExpr = FoldConstantRule.INSTANCE.rewrite( ExpressionUtils.replace(predicate, replaceMap), - new ExpressionRewriteContext(null)); + new ExpressionRewriteContext(cascadesContext)); if (nullLiteral.equals(evalExpr) || BooleanLiteral.FALSE.equals(evalExpr)) { notNullSlots.add(slot); } @@ -365,8 +366,8 @@ public class ExpressionUtils { /** * infer notNulls slot from predicate */ - public static Set inferNotNull(Set predicates) { - return inferNotNullSlots(predicates).stream() + public static Set inferNotNull(Set predicates, CascadesContext cascadesContext) { + return inferNotNullSlots(predicates, cascadesContext).stream() .map(slot -> { Not isNotNull = new Not(new IsNull(slot)); isNotNull.isGeneratedIsNotNull = true; @@ -377,8 +378,9 @@ public class ExpressionUtils { /** * infer notNulls slot from predicate but these slots must be in the given slots. */ - public static Set inferNotNull(Set predicates, Set slots) { - return inferNotNullSlots(predicates).stream() + public static Set inferNotNull(Set predicates, Set slots, + CascadesContext cascadesContext) { + return inferNotNullSlots(predicates, cascadesContext).stream() .filter(slots::contains) .map(slot -> { Not isNotNull = new Not(new IsNull(slot)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java new file mode 100644 index 0000000000..95ba886e3f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +/** MutableState */ +public interface MutableState { + Optional get(String key); + + MutableState set(String key, Object value); + + /** EmptyMutableState */ + class EmptyMutableState implements MutableState { + public static final EmptyMutableState INSTANCE = new EmptyMutableState(); + + private EmptyMutableState() {} + + @Override + public Optional get(String key) { + return Optional.empty(); + } + + @Override + public MutableState set(String key, Object value) { + return new SingleMutableState(key, value); + } + } + + /** SingleMutableState */ + class SingleMutableState implements MutableState { + public final String key; + public final Object value; + + public SingleMutableState(String key, Object value) { + this.key = key; + this.value = value; + } + + @Override + public Optional get(String key) { + if (this.key.equals(key)) { + return (Optional) Optional.ofNullable(value); + } + return Optional.empty(); + } + + @Override + public MutableState set(String key, Object value) { + if (this.key.equals(key)) { + return new SingleMutableState(key, value); + } + MultiMutableState multiMutableState = new MultiMutableState(); + multiMutableState.set(this.key, this.value); + multiMutableState.set(key, value); + return multiMutableState; + } + } + + /** MultiMutableState */ + class MultiMutableState implements MutableState { + private final Map states = new LinkedHashMap<>(); + + @Override + public Optional get(String key) { + return (Optional) Optional.ofNullable(states.get(key)); + } + + @Override + public MutableState set(String key, Object value) { + states.put(key, value); + return this; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index a5989d595f..e963d6131a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -244,6 +244,7 @@ public class Utils { for (int i = 0; i < list.size(); i++) { if (list.get(i) == item) { list.remove(i); + i--; return; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java index 76b2cf6dd0..1b0129d917 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/JoinHintTest.java @@ -23,7 +23,7 @@ import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.util.MatchingUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.HashJoinNode; import org.apache.doris.planner.HashJoinNode.DistributionMode; @@ -38,7 +38,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -class JoinHintTest extends TestWithFeService implements PatternMatchSupported { +class JoinHintTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/ssb/SSBJoinReorderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/ssb/SSBJoinReorderTest.java index 5f4fb55b23..583250afd1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/ssb/SSBJoinReorderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/datasets/ssb/SSBJoinReorderTest.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.datasets.ssb; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import com.google.common.collect.ImmutableList; @@ -25,7 +25,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class SSBJoinReorderTest extends SSBTestBase implements PatternMatchSupported { +public class SSBJoinReorderTest extends SSBTestBase implements MemoPatternMatchSupported { @Test public void q4_1() { test( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java index 7ddf8e9b1e..e268f5a091 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java @@ -40,8 +40,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.qe.ConnectContext; @@ -56,7 +56,7 @@ import java.util.ArrayList; import java.util.Map; import java.util.Objects; -class MemoTest implements PatternMatchSupported { +class MemoTest implements MemoPatternMatchSupported { private final ConnectContext connectContext = MemoTestUtils.createConnectContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/metrics/EventTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/metrics/EventTest.java index 94efff68d0..631022d175 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/metrics/EventTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/metrics/EventTest.java @@ -78,6 +78,11 @@ public class EventTest extends TestWithFeService { channel.start(); } + @Override + protected void runBeforeEach() throws Exception { + CounterEvent.clearCounter(); + } + @Override public void runAfterAll() { channel.stop(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java index cd35c5aef8..cb179ef531 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java @@ -18,13 +18,13 @@ package org.apache.doris.nereids.parser; import org.apache.doris.nereids.util.ExpressionParseChecker; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanParseChecker; /** * Base class to check SQL parsing result. */ -public abstract class ParserTestBase implements PatternMatchSupported { +public abstract class ParserTestBase implements MemoPatternMatchSupported { public PlanParseChecker parsePlan(String sql) { return new PlanParseChecker(sql); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java index efec2a2c67..4947570ace 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java @@ -319,7 +319,7 @@ public class GroupExpressionMatchingTest { } } - private org.apache.doris.nereids.pattern.GeneratedPatterns patterns() { + private org.apache.doris.nereids.pattern.GeneratedMemoPatterns patterns() { return () -> RulePromise.REWRITE; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java index d55083b718..5a568a8586 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java @@ -30,8 +30,8 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.util.FieldChecker; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -40,7 +40,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class AnalyzeSubQueryTest extends TestWithFeService implements PatternMatchSupported { +public class AnalyzeSubQueryTest extends TestWithFeService implements MemoPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); @@ -142,20 +142,20 @@ public class AnalyzeSubQueryTest extends TestWithFeService implements PatternMat ) ) ).when(FieldChecker.check("projects", ImmutableList.of( - new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT2")), - new SlotReference(new ExprId(1), "score", BigIntType.INSTANCE, true, ImmutableList.of("TT2")))) + new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT2")), + new SlotReference(new ExprId(3), "score", BigIntType.INSTANCE, true, ImmutableList.of("TT2")))) ) ) .when(FieldChecker.check("otherJoinConjuncts", ImmutableList.of(new EqualTo( - new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), - new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("T"))))) + new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), + new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("T"))))) ) ).when(FieldChecker.check("projects", ImmutableList.of( - new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), - new SlotReference(new ExprId(3), "score", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), - new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("T")), - new SlotReference(new ExprId(1), "score", BigIntType.INSTANCE, true, ImmutableList.of("T")))) + new SlotReference(new ExprId(0), "id", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), + new SlotReference(new ExprId(1), "score", BigIntType.INSTANCE, true, ImmutableList.of("TT1")), + new SlotReference(new ExprId(2), "id", BigIntType.INSTANCE, true, ImmutableList.of("T")), + new SlotReference(new ExprId(3), "score", BigIntType.INSTANCE, true, ImmutableList.of("T")))) ) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java index 0ab04d7456..b23b28b1c8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java @@ -26,15 +26,15 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.implementation.AggregateStrategies; -import org.apache.doris.nereids.rules.rewrite.logical.ApplyPullFilterOnAgg; -import org.apache.doris.nereids.rules.rewrite.logical.ApplyPullFilterOnProjectUnderAgg; -import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilterUnderApplyProject; import org.apache.doris.nereids.rules.rewrite.logical.ExistsApplyToJoin; import org.apache.doris.nereids.rules.rewrite.logical.InApplyToJoin; import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderFilter; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderProject; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpCorrelatedFilterUnderApplyAggregateProject; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpProjectUnderApply; import org.apache.doris.nereids.rules.rewrite.logical.ScalarApplyToJoin; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyAggregateFilter; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyFilter; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyProjectFilter; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -47,8 +47,8 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.util.FieldChecker; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -61,7 +61,7 @@ import org.junit.jupiter.api.Test; import java.util.List; import java.util.Optional; -public class AnalyzeWhereSubqueryTest extends TestWithFeService implements PatternMatchSupported { +public class AnalyzeWhereSubqueryTest extends TestWithFeService implements MemoPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); // scalar @@ -132,14 +132,18 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte }; for (String sql : testSql) { - NamedExpressionUtil.clear(); - StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); - PhysicalPlan plan = new NereidsPlanner(statementContext).plan( - parser.parseSingle(sql), - PhysicalProperties.ANY - ); - // Just to check whether translate will throw exception - new PhysicalPlanTranslator().translatePlan(plan, new PlanTranslatorContext()); + try { + NamedExpressionUtil.clear(); + StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); + PhysicalPlan plan = new NereidsPlanner(statementContext).plan( + parser.parseSingle(sql), + PhysicalProperties.ANY + ); + // Just to check whether translate will throw exception + new PhysicalPlanTranslator().translatePlan(plan, new PlanTranslatorContext()); + } catch (Throwable t) { + throw new IllegalStateException("Test sql failed: " + t.getMessage() + ", sql:\n" + sql, t); + } } } @@ -175,7 +179,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte // after aggFilter rule PlanChecker.from(connectContext) .analyze(sql2) - .applyBottomUp(new ApplyPullFilterOnAgg()) + .applyBottomUp(new UnCorrelatedApplyAggregateFilter()) .matches( logicalApply( any(), @@ -209,7 +213,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte // after Scalar CorrelatedJoin to join PlanChecker.from(connectContext) .analyze(sql2) - .applyBottomUp(new ApplyPullFilterOnAgg()) + .applyBottomUp(new UnCorrelatedApplyAggregateFilter()) .applyBottomUp(new ScalarApplyToJoin()) .matches( logicalJoin( @@ -247,7 +251,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testInSql4AfterEliminateFilterUnderApplyProjectRule() { PlanChecker.from(connectContext) .analyze(sql4) - .applyBottomUp(new EliminateFilterUnderApplyProject()) + .applyBottomUp(new UnCorrelatedApplyProjectFilter()) .matches( logicalApply( any(), @@ -271,7 +275,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testInSql4AfterInToJoin() { PlanChecker.from(connectContext) .analyze(sql4) - .applyBottomUp(new EliminateFilterUnderApplyProject()) + .applyBottomUp(new UnCorrelatedApplyProjectFilter()) .applyBottomUp(new InApplyToJoin()) .matches( logicalJoin().when(FieldChecker.check("joinType", JoinType.LEFT_SEMI_JOIN)) @@ -310,7 +314,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testExistSql6AfterPushProjectRule() { PlanChecker.from(connectContext) .analyze(sql6) - .applyBottomUp(new PushApplyUnderProject()) + .applyBottomUp(new PullUpProjectUnderApply()) .matches( logicalProject( logicalApply().when(FieldChecker.check("correlationFilter", Optional.empty())) @@ -329,8 +333,8 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testExistSql6AfterPushFilterRule() { PlanChecker.from(connectContext) .analyze(sql6) - .applyBottomUp(new PushApplyUnderProject()) - .applyBottomUp(new PushApplyUnderFilter()) + .applyBottomUp(new PullUpProjectUnderApply()) + .applyBottomUp(new UnCorrelatedApplyFilter()) .matches( logicalApply().when(FieldChecker.check("correlationFilter", Optional.of( new EqualTo(new SlotReference(new ExprId(1), "k2", BigIntType.INSTANCE, true, @@ -344,8 +348,8 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte public void testExistSql6AfterInToJoin() { PlanChecker.from(connectContext) .analyze(sql6) - .applyBottomUp(new PushApplyUnderProject()) - .applyBottomUp(new PushApplyUnderFilter()) + .applyBottomUp(new PullUpProjectUnderApply()) + .applyBottomUp(new UnCorrelatedApplyFilter()) .applyBottomUp(new ExistsApplyToJoin()) .matches( logicalJoin().when(FieldChecker.check("joinType", JoinType.LEFT_SEMI_JOIN)) @@ -407,7 +411,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte .analyze(sql10) .applyBottomUp(new LogicalSubQueryAliasToLogicalProject()) .applyTopDown(new MergeProjects()) - .applyBottomUp(new ApplyPullFilterOnProjectUnderAgg()) + .applyBottomUp(new PullUpCorrelatedFilterUnderApplyAggregateProject()) .matches( logicalApply( any(), @@ -439,8 +443,8 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte .analyze(sql10) .applyBottomUp(new LogicalSubQueryAliasToLogicalProject()) .applyTopDown(new MergeProjects()) - .applyBottomUp(new ApplyPullFilterOnProjectUnderAgg()) - .applyBottomUp(new ApplyPullFilterOnAgg()) + .applyBottomUp(new PullUpCorrelatedFilterUnderApplyAggregateProject()) + .applyBottomUp(new UnCorrelatedApplyAggregateFilter()) .matches( logicalApply( any(), @@ -465,8 +469,8 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements Patte .analyze(sql10) .applyBottomUp(new LogicalSubQueryAliasToLogicalProject()) .applyTopDown(new MergeProjects()) - .applyBottomUp(new ApplyPullFilterOnProjectUnderAgg()) - .applyBottomUp(new ApplyPullFilterOnAgg()) + .applyBottomUp(new PullUpCorrelatedFilterUnderApplyAggregateProject()) + .applyBottomUp(new UnCorrelatedApplyAggregateFilter()) .applyBottomUp(new ScalarApplyToJoin()) .matches( logicalJoin( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java index b5aebdc124..27d99fff42 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindFunctionTest.java @@ -21,13 +21,13 @@ import org.apache.doris.common.Config; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Test; -public class BindFunctionTest extends TestWithFeService implements PatternMatchSupported { +public class BindFunctionTest extends TestWithFeService implements MemoPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java index 553ee35e44..4c003cf283 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java @@ -23,14 +23,14 @@ import org.apache.doris.nereids.jobs.batch.CheckLegalityAfterRewrite; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewrite; import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.qe.ConnectContext; import org.junit.jupiter.api.Test; -public class CheckExpressionLegalityTest implements PatternMatchSupported { +public class CheckExpressionLegalityTest implements MemoPatternMatchSupported { @Test public void testAvg() { ConnectContext connectContext = MemoTestUtils.createConnectContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java index 3ecbd52a46..154a30955f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlotsTest.java @@ -39,7 +39,7 @@ import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.util.FieldChecker; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import com.google.common.collect.ImmutableList; @@ -49,7 +49,7 @@ import org.junit.jupiter.api.Test; import java.util.stream.Collectors; -public class FillUpMissingSlotsTest extends AnalyzeCheckTestBase implements PatternMatchSupported { +public class FillUpMissingSlotsTest extends AnalyzeCheckTestBase implements MemoPatternMatchSupported { @Override public void runBeforeAll() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java index 487fae2c58..a421aa0ac2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java @@ -31,8 +31,8 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.qe.ConnectContext; @@ -44,7 +44,7 @@ import java.util.List; import java.util.Map; // this ut will add more test case later -public class FunctionRegistryTest implements PatternMatchSupported { +public class FunctionRegistryTest implements MemoPatternMatchSupported { private ConnectContext connectContext = MemoTestUtils.createConnectContext(); @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java index f32f5959d9..37508d5387 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeatTest.java @@ -26,15 +26,15 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -public class NormalizeRepeatTest implements PatternMatchSupported { +public class NormalizeRepeatTest implements MemoPatternMatchSupported { @Test public void testKeepNullableAfterNormalizeRepeat() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java index 95b39c48a9..7142eecab5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/RegisterCTETest.java @@ -31,8 +31,8 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.implementation.AggregateStrategies; import org.apache.doris.nereids.rules.rewrite.logical.InApplyToJoin; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderFilter; -import org.apache.doris.nereids.rules.rewrite.logical.PushApplyUnderProject; +import org.apache.doris.nereids.rules.rewrite.logical.PullUpProjectUnderApply; +import org.apache.doris.nereids.rules.rewrite.logical.UnCorrelatedApplyFilter; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -46,8 +46,8 @@ import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.util.FieldChecker; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -60,7 +60,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class RegisterCTETest extends TestWithFeService implements PatternMatchSupported { +public class RegisterCTETest extends TestWithFeService implements MemoPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); @@ -205,8 +205,8 @@ public class RegisterCTETest extends TestWithFeService implements PatternMatchSu PlanChecker.from(connectContext) .analyze(sql3) - .applyBottomUp(new PushApplyUnderProject()) - .applyBottomUp(new PushApplyUnderFilter()) + .applyBottomUp(new PullUpProjectUnderApply()) + .applyBottomUp(new UnCorrelatedApplyFilter()) .applyBottomUp(new InApplyToJoin()) .matches( logicalProject( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutputTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutputTest.java index 67d043b88c..3238791145 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutputTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ReplaceExpressionByChildOutputTest.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -40,7 +40,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class ReplaceExpressionByChildOutputTest implements PatternMatchSupported { +public class ReplaceExpressionByChildOutputTest implements MemoPatternMatchSupported { @Test void testSortProject() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProjectTest.java index c272ce97cf..2111253168 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProjectTest.java @@ -32,8 +32,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -43,7 +43,7 @@ import org.junit.jupiter.api.Test; import java.util.List; import java.util.Objects; -class InnerJoinLAsscomProjectTest implements PatternMatchSupported { +class InnerJoinLAsscomProjectTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomTest.java index 02b340fbd2..5d89c29f8a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomTest.java @@ -32,8 +32,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -42,7 +42,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class InnerJoinLAsscomTest implements PatternMatchSupported { +public class InnerJoinLAsscomTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateTest.java index fede3fe0df..7b4b7187b8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociateTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class InnerJoinLeftAssociateTest implements PatternMatchSupported { +class InnerJoinLeftAssociateTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); private final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateTest.java index 82d284dade..c0c07513b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociateTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class InnerJoinRightAssociateTest implements PatternMatchSupported { +class InnerJoinRightAssociateTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); private final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java index 0e70e82083..20323d108e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -public class JoinCommuteTest implements PatternMatchSupported { +public class JoinCommuteTest implements MemoPatternMatchSupported { @Test public void testInnerJoinCommute() { LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeTest.java index 2909b960c2..8faa5d0a85 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinExchangeTest.java @@ -22,15 +22,15 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -class JoinExchangeTest implements PatternMatchSupported { +class JoinExchangeTest implements MemoPatternMatchSupported { @Test public void testSimple() { LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocTest.java index 49e7a13795..90eeefee02 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocTest.java @@ -18,12 +18,13 @@ package org.apache.doris.nereids.rules.exploration.join; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -31,10 +32,18 @@ import org.junit.jupiter.api.Test; import java.util.Objects; -class OuterJoinAssocTest implements PatternMatchSupported { - LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); - LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); - LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); +class OuterJoinAssocTest implements MemoPatternMatchSupported { + LogicalOlapScan scan1; + LogicalOlapScan scan2; + LogicalOlapScan scan3; + + public OuterJoinAssocTest() throws Exception { + // clear id so that slot id keep consistent every running + NamedExpressionUtil.clear(); + scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); + } @Test public void testInnerLeft() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java index 79d705f231..90b8266f25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProjectTest.java @@ -25,8 +25,8 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -36,7 +36,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -class OuterJoinLAsscomProjectTest implements PatternMatchSupported { +class OuterJoinLAsscomProjectTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomTest.java index 93143e22e1..7b75231c04 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomTest.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -37,7 +37,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class OuterJoinLAsscomTest implements PatternMatchSupported { +public class OuterJoinLAsscomTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java index fb4586d34e..ba82d58f2a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProjectTest.java @@ -22,15 +22,15 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -public class SemiJoinSemiJoinTransposeProjectTest implements PatternMatchSupported { +public class SemiJoinSemiJoinTransposeProjectTest implements MemoPatternMatchSupported { public static final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); public static final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); public static final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTestHelper.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTestHelper.java index 75824b2441..f75f474257 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTestHelper.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewriteTestHelper.java @@ -17,12 +17,15 @@ package org.apache.doris.nereids.rules.expression.rewrite; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.rules.expression.rewrite.rules.TypeCoercion; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -31,7 +34,9 @@ import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.nereids.util.MemoTestUtils; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.junit.jupiter.api.Assertions; @@ -43,21 +48,29 @@ public abstract class ExpressionRewriteTestHelper { protected static final NereidsParser PARSER = new NereidsParser(); protected ExpressionRuleExecutor executor; + protected ExpressionRewriteContext context; + + public ExpressionRewriteTestHelper() { + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( + new UnboundRelation(new RelationId(1), ImmutableList.of("tbl"))); + context = new ExpressionRewriteContext(cascadesContext); + } + protected final void assertRewrite(String expression, String expected) { Expression needRewriteExpression = PARSER.parseExpression(expression); Expression expectedExpression = PARSER.parseExpression(expected); - Expression rewrittenExpression = executor.rewrite(needRewriteExpression); + Expression rewrittenExpression = executor.rewrite(needRewriteExpression, context); Assertions.assertEquals(expectedExpression, rewrittenExpression); } protected void assertRewrite(String expression, Expression expectedExpression) { Expression needRewriteExpression = PARSER.parseExpression(expression); - Expression rewrittenExpression = executor.rewrite(needRewriteExpression); + Expression rewrittenExpression = executor.rewrite(needRewriteExpression, context); Assertions.assertEquals(expectedExpression, rewrittenExpression); } protected void assertRewrite(Expression expression, Expression expectedExpression) { - Expression rewrittenExpression = executor.rewrite(expression); + Expression rewrittenExpression = executor.rewrite(expression, context); Assertions.assertEquals(expectedExpression, rewrittenExpression); } @@ -66,7 +79,7 @@ public abstract class ExpressionRewriteTestHelper { Expression needRewriteExpression = PARSER.parseExpression(expression); needRewriteExpression = typeCoercion(replaceUnboundSlot(needRewriteExpression, mem)); Expression expectedExpression = PARSER.parseExpression(expected); - Expression rewrittenExpression = executor.rewrite(needRewriteExpression); + Expression rewrittenExpression = executor.rewrite(needRewriteExpression, context); Assertions.assertEquals(expectedExpression.toSql(), rewrittenExpression.toSql()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/FoldConstantTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/FoldConstantTest.java index d600d0a2da..7e8d132d06 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/FoldConstantTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/FoldConstantTest.java @@ -134,7 +134,7 @@ public class FoldConstantTest extends ExpressionRewriteTestHelper { // cast '1' as tinyint Cast c = new Cast(Literal.of("1"), TinyIntType.INSTANCE); - Expression rewritten = executor.rewrite(c); + Expression rewritten = executor.rewrite(c, context); Literal expected = Literal.of((byte) 1); Assertions.assertEquals(rewritten, expected); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/SimplifyRangeTest.java index 617ad971f7..5022828713 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/SimplifyRangeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rewrite/SimplifyRangeTest.java @@ -17,12 +17,15 @@ package org.apache.doris.nereids.rules.expression.rewrite; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyRange; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -30,6 +33,7 @@ import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.nereids.util.MemoTestUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -44,6 +48,13 @@ public class SimplifyRangeTest { private static final NereidsParser PARSER = new NereidsParser(); private ExpressionRuleExecutor executor; + private ExpressionRewriteContext context; + + public SimplifyRangeTest() { + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext( + new UnboundRelation(new RelationId(1), ImmutableList.of("tbl"))); + context = new ExpressionRewriteContext(cascadesContext); + } @Test public void testSimplify() { @@ -99,7 +110,7 @@ public class SimplifyRangeTest { Map mem = Maps.newHashMap(); Expression needRewriteExpression = replaceUnboundSlot(PARSER.parseExpression(expression), mem); Expression expectedExpression = replaceUnboundSlot(PARSER.parseExpression(expected), mem); - Expression rewrittenExpression = executor.rewrite(needRewriteExpression); + Expression rewrittenExpression = executor.rewrite(needRewriteExpression, context); Assertions.assertEquals(expectedExpression, rewrittenExpression); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java index 512e3f1e74..403d13b2cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java @@ -31,7 +31,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.HllUnionAgg; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.ScanNode; @@ -53,7 +53,7 @@ import java.util.stream.Collectors; * Tests ported from {@link org.apache.doris.planner.MaterializedViewFunctionTest} */ @Disabled("Disabled until nereids support advanced mv") -public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements PatternMatchSupported { +public class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoPatternMatchSupported { private static final String EMPS_TABLE_NAME = "emps"; private static final String EMPS_MV_NAME = "emps_mv"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java index 8f596b0d18..947a30e449 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java @@ -21,14 +21,14 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; import org.apache.doris.nereids.trees.plans.PreAggStatus; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements PatternMatchSupported { +class SelectRollupIndexTest extends BaseMaterializedIndexSelectTest implements MemoPatternMatchSupported { @Override protected void beforeCreatingConnectContext() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateStrategiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateStrategiesTest.java index 29b729fe1c..33c5c1f909 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateStrategiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateStrategiesTest.java @@ -37,8 +37,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -55,7 +55,7 @@ import java.util.List; import java.util.Optional; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class AggregateStrategiesTest implements PatternMatchSupported { +public class AggregateStrategiesTest implements MemoPatternMatchSupported { private Plan rStudent; @BeforeAll diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionTest.java index 9faa34487c..5bfa3635ef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/CheckAndStandardizeWindowFunctionTest.java @@ -40,8 +40,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -55,7 +55,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.List; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class CheckAndStandardizeWindowFunctionTest implements PatternMatchSupported { +public class CheckAndStandardizeWindowFunctionTest implements MemoPatternMatchSupported { private LogicalPlan rStudent; private NamedExpression gender; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java index bf2daa712e..7379df6bea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -35,7 +35,7 @@ import java.util.stream.Collectors; /** * column prune ut. */ -public class ColumnPruningTest extends TestWithFeService implements PatternMatchSupported { +public class ColumnPruningTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { createDatabase("test"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateDedupJoinConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateDedupJoinConditionTest.java index 24510cd97d..4011e79960 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateDedupJoinConditionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateDedupJoinConditionTest.java @@ -21,15 +21,15 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -class EliminateDedupJoinConditionTest implements PatternMatchSupported { +class EliminateDedupJoinConditionTest implements MemoPatternMatchSupported { @Test void testEliminate() { LogicalPlan plan = new LogicalPlanBuilder(PlanConstructor.scan1) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java index 4777251c56..623b29c159 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java @@ -20,13 +20,14 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.common.Pair; import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -34,9 +35,16 @@ import org.junit.jupiter.api.Test; import java.util.Objects; -class EliminateOuterJoinTest implements PatternMatchSupported { - private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); - private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); +class EliminateOuterJoinTest implements MemoPatternMatchSupported { + private final LogicalOlapScan scan1; + private final LogicalOlapScan scan2; + + public EliminateOuterJoinTest() throws Exception { + // clear id so that slot id keep consistent every running + NamedExpressionUtil.clear(); + scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + } @Test void testEliminateLeft() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java index 4c440e68f8..ecd6de1cd5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateUnnecessaryProjectTest.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.plans.Plan; @@ -33,12 +32,9 @@ import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.List; - /** * test ELIMINATE_UNNECESSARY_PROJECT rule. */ @@ -66,11 +62,10 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { .build(); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); - cascadesContext.topDownRewrite(rules); + cascadesContext.topDownRewrite(new EliminateUnnecessaryProject()); Plan actual = cascadesContext.getMemo().copyOut(); - Assertions.assertTrue(actual.child(0) instanceof LogicalOlapScan); + Assertions.assertTrue(actual.child(0) instanceof LogicalProject); } @Test @@ -80,8 +75,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { .build(); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); - cascadesContext.topDownRewrite(rules); + cascadesContext.topDownRewrite(new EliminateUnnecessaryProject()); Plan actual = cascadesContext.getMemo().copyOut(); Assertions.assertTrue(actual instanceof LogicalOlapScan); @@ -89,13 +83,12 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { @Test public void testNotEliminateTopProjectWhenOutputNotEquals() { - LogicalPlan unnecessaryProject = new LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0)) + LogicalPlan necessaryProject = new LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0)) .project(ImmutableList.of(1, 0)) .build(); - CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); - cascadesContext.topDownRewrite(rules); + CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(necessaryProject); + cascadesContext.topDownRewrite(new EliminateUnnecessaryProject()); Plan actual = cascadesContext.getMemo().copyOut(); Assertions.assertTrue(actual instanceof LogicalProject); @@ -109,8 +102,7 @@ public class EliminateUnnecessaryProjectTest extends TestWithFeService { .project(ImmutableList.of(1, 0)) .build(); CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(unnecessaryProject); - List rules = Lists.newArrayList(new EliminateUnnecessaryProject().buildRules()); - cascadesContext.topDownRewrite(rules); + cascadesContext.topDownRewrite(new EliminateUnnecessaryProject()); Plan actual = cascadesContext.getMemo().copyOut(); Assertions.assertTrue(actual instanceof LogicalEmptyRelation); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpressionTest.java index 39d62974a9..96c833a1de 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractAndNormalizeWindowExpressionTest.java @@ -34,8 +34,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -48,7 +48,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.List; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class ExtractAndNormalizeWindowExpressionTest implements PatternMatchSupported { +public class ExtractAndNormalizeWindowExpressionTest implements MemoPatternMatchSupported { private LogicalPlan rStudent; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoinTest.java index dc3aa0aa8f..48db2844f4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoinTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class ExtractFilterFromCrossJoinTest implements PatternMatchSupported { +class ExtractFilterFromCrossJoinTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunctionTest.java index eb1c619a9b..bfc7e1aaa2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractSingleTableExpressionFromDisjunctionTest.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -44,7 +44,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.Set; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class ExtractSingleTableExpressionFromDisjunctionTest implements PatternMatchSupported { +public class ExtractSingleTableExpressionFromDisjunctionTest implements MemoPatternMatchSupported { Plan student; Plan course; SlotReference courseCid; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java index 7c0d2623cc..3c689a2be1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java @@ -25,14 +25,14 @@ import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class InferFilterNotNullTest implements PatternMatchSupported { +class InferFilterNotNullTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNullTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNullTest.java index 81cd74874a..6ba1ed4bed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNullTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferJoinNotNullTest.java @@ -22,14 +22,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class InferJoinNotNullTest implements PatternMatchSupported { +class InferJoinNotNullTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); private final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); @@ -43,8 +43,8 @@ class InferJoinNotNullTest implements PatternMatchSupported { .applyTopDown(new InferJoinNotNull()) .matches( innerLogicalJoin( - logicalFilter().when(f -> f.getPredicate().toString().equals("( not id IS NULL)")), - logicalFilter().when(f -> f.getPredicate().toString().equals("( not id IS NULL)")) + logicalFilter().when(f -> f.getPredicate().toString().equals("( not id#0 IS NULL)")), + logicalFilter().when(f -> f.getPredicate().toString().equals("( not id#2 IS NULL)")) ) ); @@ -55,7 +55,7 @@ class InferJoinNotNullTest implements PatternMatchSupported { .applyTopDown(new InferJoinNotNull()) .matches( leftSemiLogicalJoin( - logicalFilter().when(f -> f.getPredicate().toString().equals("( not id IS NULL)")), + logicalFilter().when(f -> f.getPredicate().toString().equals("( not id#0 IS NULL)")), logicalOlapScan() ) ); @@ -68,7 +68,7 @@ class InferJoinNotNullTest implements PatternMatchSupported { .matches( rightSemiLogicalJoin( logicalOlapScan(), - logicalFilter().when(f -> f.getPredicate().toString().equals("( not id IS NULL)")) + logicalFilter().when(f -> f.getPredicate().toString().equals("( not id#2 IS NULL)")) ) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicatesTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicatesTest.java index a2addbbeb7..e7af340c32 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicatesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferPredicatesTest.java @@ -17,13 +17,13 @@ package org.apache.doris.nereids.rules.rewrite.logical; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Test; -public class InferPredicatesTest extends TestWithFeService implements PatternMatchSupported { +public class InferPredicatesTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LimitPushDownTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LimitPushDownTest.java index ef251c911f..89a2b4f0b5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LimitPushDownTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LimitPushDownTest.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.planner.OlapScanNode; @@ -48,7 +48,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -class LimitPushDownTest extends TestWithFeService implements PatternMatchSupported { +class LimitPushDownTest extends TestWithFeService implements MemoPatternMatchSupported { private Plan scanScore = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); private Plan scanStudent = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LogicalWindowToPhysicalWindowTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LogicalWindowToPhysicalWindowTest.java index 1abe5f56ca..0ba1f76ec3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LogicalWindowToPhysicalWindowTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/LogicalWindowToPhysicalWindowTest.java @@ -30,8 +30,8 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -45,7 +45,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.List; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class LogicalWindowToPhysicalWindowTest implements PatternMatchSupported { +public class LogicalWindowToPhysicalWindowTest implements MemoPatternMatchSupported { private Plan rStudent; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjectsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjectsTest.java index 024890753f..4c04874a06 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjectsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/MergeProjectsTest.java @@ -25,8 +25,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -39,7 +39,7 @@ import java.util.Objects; /** * MergeConsecutiveProjects ut */ -public class MergeProjectsTest implements PatternMatchSupported { +public class MergeProjectsTest implements MemoPatternMatchSupported { LogicalOlapScan score = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.score); @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeAggregateTest.java index 53848e4c50..254684eedb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/NormalizeAggregateTest.java @@ -33,8 +33,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.util.FieldChecker; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -47,7 +47,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.List; @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class NormalizeAggregateTest implements PatternMatchSupported { +public class NormalizeAggregateTest implements MemoPatternMatchSupported { private LogicalPlan rStudent; @BeforeAll diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PhysicalStorageLayerAggregateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PhysicalStorageLayerAggregateTest.java index 22e888ee7a..1556438652 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PhysicalStorageLayerAggregateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PhysicalStorageLayerAggregateTest.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.pattern.GeneratedPatterns; +import org.apache.doris.nereids.pattern.GeneratedMemoPatterns; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.rules.RuleType; @@ -42,7 +42,7 @@ import org.junit.jupiter.api.Test; import java.util.Collections; import java.util.Optional; -public class PhysicalStorageLayerAggregateTest implements GeneratedPatterns { +public class PhysicalStorageLayerAggregateTest implements GeneratedMemoPatterns { @Test public void testWithoutProject() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoinTest.java index 5111fea12c..1fd6cb9ac2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoinTest.java @@ -23,14 +23,14 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.junit.jupiter.api.Test; -class PushFilterInsideJoinTest implements PatternMatchSupported { +class PushFilterInsideJoinTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownExpressionsInHashConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownExpressionsInHashConditionTest.java index c002db8b9b..4b4278be9e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownExpressionsInHashConditionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownExpressionsInHashConditionTest.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -32,7 +32,7 @@ import org.junit.jupiter.api.Test; import java.util.List; import java.util.Set; -public class PushdownExpressionsInHashConditionTest extends TestWithFeService implements PatternMatchSupported { +public class PushdownExpressionsInHashConditionTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { createDatabase("test"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregationTest.java index bc6a08d994..474f1acd49 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughAggregationTest.java @@ -29,8 +29,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.RelationUtil; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -38,7 +38,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.Test; -public class PushdownFilterThroughAggregationTest implements PatternMatchSupported { +public class PushdownFilterThroughAggregationTest implements MemoPatternMatchSupported { /*- * origin plan: @@ -135,8 +135,8 @@ public class PushdownFilterThroughAggregationTest implements PatternMatchSupport logicalAggregate( logicalFilter( logicalOlapScan() - ).when(filter -> ImmutableList.copyOf(filter.getConjuncts()).get(0) instanceof LessThanEqual - && ImmutableList.copyOf(filter.getConjuncts()).get(1) instanceof GreaterThan) + ).when(filter -> ImmutableList.copyOf(filter.getConjuncts()).get(0) instanceof GreaterThan + && ImmutableList.copyOf(filter.getConjuncts()).get(1) instanceof LessThanEqual) ) ).when(filter -> ImmutableList.copyOf(filter.getConjuncts()).get(0) instanceof EqualTo) ) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java index ef357db025..620c1bf44b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java @@ -27,8 +27,8 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -44,7 +44,7 @@ import java.util.Set; * PushdownFilterThroughJoinTest UT. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) -public class PushdownFilterThroughJoinTest implements PatternMatchSupported { +public class PushdownFilterThroughJoinTest implements MemoPatternMatchSupported { private LogicalPlan rStudent; private LogicalPlan rScore; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java index f82f3c0787..77bf76af4c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java @@ -19,15 +19,15 @@ package org.apache.doris.nereids.rules.rewrite.logical; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -public class PushdownProjectThroughLimitTest implements PatternMatchSupported { +public class PushdownProjectThroughLimitTest implements MemoPatternMatchSupported { @Test public void testPushdownProjectThroughLimit() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java index 23f69e9b7c..bf37e5666f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoinTest.java @@ -23,8 +23,8 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -33,7 +33,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -class ReorderJoinTest implements PatternMatchSupported { +class ReorderJoinTest implements MemoPatternMatchSupported { private final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java index 7e81c648a2..48b5026d1e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java @@ -69,7 +69,7 @@ public class InferTest extends SqlTestBase { logicalOlapScan() ) ).when(f -> f.getPredicate().toString() - .equals("((id#0 = 4) OR ((id#0 > 4) AND score IS NULL))")) + .equals("((id#0 = 4) OR ((id#0 > 4) AND score#3 IS NULL))")) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java index f3b8899b5a..3151c2d7bd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java @@ -18,10 +18,10 @@ package org.apache.doris.nereids.sqltest; import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; -import org.apache.doris.nereids.util.PatternMatchSupported; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.utframe.TestWithFeService; -public abstract class SqlTestBase extends TestWithFeService implements PatternMatchSupported { +public abstract class SqlTestBase extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { createDatabase("test"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java index 23606f95f8..781c7840e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java @@ -22,8 +22,8 @@ import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; @@ -31,7 +31,7 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class SelectExceptTest implements PatternMatchSupported { +public class SelectExceptTest implements MemoPatternMatchSupported { @Test public void testExcept() { LogicalOlapScan olapScan = PlanConstructor.newLogicalOlapScan(0, "t1", 1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java index 4a72965010..71ab84f5a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.MemoTestUtils; -import org.apache.doris.nereids.util.PatternMatchSupported; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; @@ -36,7 +36,7 @@ import org.junit.jupiter.api.Test; import java.util.List; -public class ViewTest extends TestWithFeService implements PatternMatchSupported { +public class ViewTest extends TestWithFeService implements MemoPatternMatchSupported { @Override protected void runBeforeAll() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java index 8da90832fd..c09fa8623b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java @@ -91,7 +91,7 @@ public class PlanToStringTest { LogicalProject plan = new LogicalProject<>(ImmutableList.of( new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList())), child); - Assertions.assertTrue(plan.toString().matches("LogicalProject \\( projects=\\[a#\\d+], excepts=\\[], canEliminate=true \\)")); + Assertions.assertTrue(plan.toString().matches("LogicalProject \\( distinct=false, projects=\\[a#\\d+], excepts=\\[], canEliminate=true \\)")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MatchingUtils.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MatchingUtils.java index bc5bb1d78d..c698cdc0f4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MatchingUtils.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MatchingUtils.java @@ -36,20 +36,20 @@ public class MatchingUtils { Memo memo = new Memo(plan); if (plan instanceof PhysicalPlan) { assertMatches(memo, () -> new GroupExpressionMatching(patternDesc.pattern, - memo.getRoot().getPhysicalExpressions().get(0)).iterator().hasNext()); + memo.getRoot().getPhysicalExpressions().get(0)).iterator().hasNext(), + () -> plan.treeString()); } else if (plan instanceof LogicalPlan) { assertMatches(memo, () -> new GroupExpressionMatching(patternDesc.pattern, - memo.getRoot().getLogicalExpression()).iterator().hasNext()); + memo.getRoot().getLogicalExpression()).iterator().hasNext(), + () -> plan.treeString()); } else { throw new IllegalStateException("Input plan should be LogicalPlan or PhysicalPlan, but meet " + plan); } } - private static void assertMatches(Memo memo, Supplier asserter) { + private static void assertMatches(Memo memo, Supplier asserter, Supplier planString) { Assertions.assertTrue(asserter.get(), - () -> "pattern not match, plan :\n" - + memo.getRoot().getLogicalExpression().getPlan().treeString() - + "\n" + () -> "pattern not match, plan:\n" + planString.get() + "\n" ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PatternMatchSupported.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoPatternMatchSupported.java similarity index 87% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/util/PatternMatchSupported.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoPatternMatchSupported.java index 3c32a581c2..5916b6b1a0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PatternMatchSupported.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoPatternMatchSupported.java @@ -17,10 +17,10 @@ package org.apache.doris.nereids.util; -import org.apache.doris.nereids.pattern.GeneratedPatterns; +import org.apache.doris.nereids.pattern.GeneratedMemoPatterns; import org.apache.doris.nereids.rules.RulePromise; -public interface PatternMatchSupported extends GeneratedPatterns { +public interface MemoPatternMatchSupported extends GeneratedMemoPatterns { @Override default RulePromise defaultPromise() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java index f63b0842fc..f59b7a1342 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java @@ -81,8 +81,9 @@ public class MemoTestUtils { public static CascadesContext createCascadesContext(StatementContext statementContext, Plan initPlan) { PhysicalProperties requestProperties = NereidsPlanner.buildInitRequireProperties(initPlan); - CascadesContext cascadesContext = CascadesContext.newContext( + CascadesContext cascadesContext = CascadesContext.newRewriteContext( statementContext, initPlan, requestProperties); + cascadesContext.toMemo(); MemoValidator.validateInitState(cascadesContext.getMemo(), initPlan); return cascadesContext; } @@ -90,7 +91,7 @@ public class MemoTestUtils { public static LogicalPlan analyze(String sql) { CascadesContext cascadesContext = createCascadesContext(sql); cascadesContext.newAnalyzer().analyze(); - return (LogicalPlan) cascadesContext.getMemo().copyOut(); + return (LogicalPlan) cascadesContext.getRewritePlan(); } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 31fb819fa4..f49711aa89 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -26,8 +26,8 @@ import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.jobs.JobContext; -import org.apache.doris.nereids.jobs.batch.NereidsRewriteJobExecutor; -import org.apache.doris.nereids.jobs.batch.OptimizeRulesJob; +import org.apache.doris.nereids.jobs.batch.CascadesOptimizer; +import org.apache.doris.nereids.jobs.batch.NereidsRewriter; import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob; import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; import org.apache.doris.nereids.memo.CopyInResult; @@ -107,20 +107,17 @@ public class PlanChecker { return this; } - public PlanChecker analyze() { - MemoTestUtils.createCascadesContext(connectContext, parsedPlan); - return this; - } - public PlanChecker analyze(String sql) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql); this.cascadesContext.newAnalyzer().analyze(); + this.cascadesContext.toMemo(); return this; } public PlanChecker analyze(Plan plan) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); this.cascadesContext.newAnalyzer().analyze(); + this.cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); return this; } @@ -176,13 +173,14 @@ public class PlanChecker { } public PlanChecker rewrite() { - new NereidsRewriteJobExecutor(cascadesContext).execute(); + new NereidsRewriter(cascadesContext).execute(); + cascadesContext.toMemo(); return this; } public PlanChecker optimize() { double now = System.currentTimeMillis(); - new OptimizeRulesJob(cascadesContext).execute(); + new CascadesOptimizer(cascadesContext).execute(); System.out.println("cascades:" + (System.currentTimeMillis() - now)); return this; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/OptimizeRulesJob.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanPatternMatchSupported.java similarity index 66% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/OptimizeRulesJob.java rename to fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanPatternMatchSupported.java index 61c6887233..e664f219b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/OptimizeRulesJob.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanPatternMatchSupported.java @@ -15,20 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.jobs.batch; +package org.apache.doris.nereids.util; -import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; +import org.apache.doris.nereids.rules.RulePromise; -import com.google.common.collect.ImmutableList; - -/** - * cascade optimizer added. - */ -public class OptimizeRulesJob extends BatchRulesJob { - public OptimizeRulesJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - optimize() - )); +public interface PlanPatternMatchSupported extends GeneratedPlanPatterns { + @Override + default RulePromise defaultPromise() { + return RulePromise.PLAN_CHECK; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index 35f730726a..3068340c0b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -53,12 +53,10 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.planner.Planner; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; @@ -177,15 +175,14 @@ public abstract class TestWithFeService { protected CascadesContext createCascadesContext(String sql) { StatementContext statementCtx = createStatementCtx(sql); - LogicalPlan initPlan = new NereidsParser().parseSingle(sql); - PhysicalProperties requestProperties = NereidsPlanner.buildInitRequireProperties(initPlan); - return CascadesContext.newContext(statementCtx, initPlan, requestProperties); + return MemoTestUtils.createCascadesContext(statementCtx, sql); } public LogicalPlan analyze(String sql) { CascadesContext cascadesContext = createCascadesContext(sql); cascadesContext.newAnalyzer().analyze(); - return (LogicalPlan) cascadesContext.getMemo().copyOut(); + cascadesContext.toMemo(); + return (LogicalPlan) cascadesContext.getRewritePlan(); } protected ConnectContext createCtx(UserIdentity user, String host) throws IOException { diff --git a/regression-test/suites/nereids_benchmark_p2/explain_clickbench_benchmark.groovy b/regression-test/suites/nereids_benchmark_p2/explain_clickbench_benchmark.groovy index a8c491d855..8c52b52e9f 100644 --- a/regression-test/suites/nereids_benchmark_p2/explain_clickbench_benchmark.groovy +++ b/regression-test/suites/nereids_benchmark_p2/explain_clickbench_benchmark.groovy @@ -182,10 +182,11 @@ suite("explain_clickbench_benchmark") { sql "SET enable_nereids_planner=true" sql "SET enable_vectorized_engine=true" sql "SET enable_fallback_to_original_planner=false" + sql "SET enable_dphyp_optimizer=true" benchmark { warmUp true - executeTimes 3 + executeTimes 300 skipFailure false printResult true