From 5dfb59844f0b79ebff4e015f63ea0f47e496c5de Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Wed, 6 Jul 2022 18:36:31 +0800 Subject: [PATCH] [enhancement](Nereids)refactor PlannerContext and JobContext (#10485) Refactor Context in Cascades: use two context in cascades framework. JobContext is used in each job, contains such attributes: - reference to PlannerContext - current cost upper bound - current required physical properties PlannerContext is used to hold global info for query planner, contains such attributes: - reference to Memo - reference to connectContext - reference to ruleset could be used for plan - job pool to maintain unexecuted jobs - job scheduler to schedule unexecuted jobs - current job context for next job to be executed --- .../apache/doris/nereids/NereidsPlanner.java | 8 +- .../doris/nereids/OptimizerContext.java | 76 ----------------- .../apache/doris/nereids/PlannerContext.java | 84 +++++++++++-------- .../doris/nereids/jobs/BatchRulesJob.java | 16 ++-- .../org/apache/doris/nereids/jobs/Job.java | 9 +- .../apache/doris/nereids/jobs/JobContext.java | 48 +++++++++++ .../nereids/jobs/cascades/ApplyRuleJob.java | 10 +-- .../jobs/cascades/CostAndEnforcerJob.java | 4 +- .../nereids/jobs/cascades/DeriveStatsJob.java | 6 +- .../cascades/ExploreGroupExpressionJob.java | 6 +- .../jobs/cascades/ExploreGroupJob.java | 6 +- .../cascades/OptimizeGroupExpressionJob.java | 4 +- .../jobs/cascades/OptimizeGroupJob.java | 10 +-- .../jobs/rewrite/RewriteBottomUpJob.java | 13 +-- .../jobs/rewrite/RewriteTopDownJob.java | 12 +-- .../jobs/scheduler/SimpleJobScheduler.java | 2 +- .../apache/doris/nereids/AnalyzeSSBTest.java | 17 ++-- .../nereids/jobs/RewriteTopDownJobTest.java | 14 ++-- .../LogicalProjectToPhysicalProjectTest.java | 7 +- .../logical/PushDownPredicateTest.java | 38 +++++---- 20 files changed, 195 insertions(+), 195 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/OptimizerContext.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index f30c333502..bc7aed0bf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -31,6 +31,7 @@ import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.jobs.AnalyzeRulesJob; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.OptimizeRulesJob; import org.apache.doris.nereids.jobs.PredicatePushDownRulesJob; import org.apache.doris.nereids.memo.Group; @@ -129,8 +130,9 @@ public class NereidsPlanner extends Planner { Memo memo = new Memo(); memo.initialize(plan); - OptimizerContext optimizerContext = new OptimizerContext(memo); - plannerContext = new PlannerContext(optimizerContext, connectContext, outputProperties); + plannerContext = new PlannerContext(memo, connectContext); + JobContext jobContext = new JobContext(plannerContext, outputProperties, Double.MAX_VALUE); + plannerContext.setCurrentJobContext(jobContext); // Get plan directly. Just for SSB. return doPlan(); @@ -159,7 +161,7 @@ public class NereidsPlanner extends Planner { } public Group getRoot() { - return plannerContext.getOptimizerContext().getMemo().getRoot(); + return plannerContext.getMemo().getRoot(); } private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physicalProperties) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/OptimizerContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/OptimizerContext.java deleted file mode 100644 index 0b1605f230..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/OptimizerContext.java +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids; - -import org.apache.doris.nereids.jobs.Job; -import org.apache.doris.nereids.jobs.scheduler.JobPool; -import org.apache.doris.nereids.jobs.scheduler.JobScheduler; -import org.apache.doris.nereids.jobs.scheduler.JobStack; -import org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler; -import org.apache.doris.nereids.memo.Memo; -import org.apache.doris.nereids.rules.RuleSet; - -/** - * Context used in memo. - */ -public class OptimizerContext { - private final Memo memo; - private RuleSet ruleSet; - private JobPool jobPool; - private final JobScheduler jobScheduler; - - /** - * Constructor of OptimizerContext. - * - * @param memo {@link Memo} reference - */ - public OptimizerContext(Memo memo) { - this.memo = memo; - this.ruleSet = new RuleSet(); - this.jobPool = new JobStack(); - this.jobScheduler = new SimpleJobScheduler(); - } - - public JobPool getJobPool() { - return jobPool; - } - - public void setJobPool(JobPool jobPool) { - this.jobPool = jobPool; - } - - public RuleSet getRuleSet() { - return ruleSet; - } - - public void setRuleSet(RuleSet ruleSet) { - this.ruleSet = ruleSet; - } - - public Memo getMemo() { - return memo; - } - - public void pushJob(Job job) { - jobPool.push(job); - } - - public JobScheduler getJobScheduler() { - return jobScheduler; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerContext.java index 472d0a1a3b..150caa3df9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerContext.java @@ -17,63 +17,77 @@ package org.apache.doris.nereids; -import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.jobs.scheduler.JobPool; +import org.apache.doris.nereids.jobs.scheduler.JobScheduler; +import org.apache.doris.nereids.jobs.scheduler.JobStack; +import org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler; +import org.apache.doris.nereids.memo.Memo; +import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.qe.ConnectContext; -import com.google.common.collect.Sets; - -import java.util.Set; - /** - * Context used in all stage in Nereids. + * Context used in memo. */ public class PlannerContext { - private final OptimizerContext optimizerContext; + private final Memo memo; private final ConnectContext connectContext; - private final PhysicalProperties physicalProperties; - private double costUpperBound; - private Set neededSlots; + private RuleSet ruleSet; + private JobPool jobPool; + private final JobScheduler jobScheduler; + private JobContext currentJobContext; /** - * Constructor of OptimizationContext. + * Constructor of OptimizerContext. * - * @param optimizerContext context includes all data struct used in memo - * @param connectContext connect context of this query - * @param physicalProperties target physical properties + * @param memo {@link Memo} reference */ - public PlannerContext( - OptimizerContext optimizerContext, - ConnectContext connectContext, - PhysicalProperties physicalProperties) { - this.optimizerContext = optimizerContext; + public PlannerContext(Memo memo, ConnectContext connectContext) { + this.memo = memo; this.connectContext = connectContext; - this.physicalProperties = physicalProperties; - this.costUpperBound = Double.MAX_VALUE; - this.neededSlots = Sets.newHashSet(); + this.ruleSet = new RuleSet(); + this.jobPool = new JobStack(); + this.jobScheduler = new SimpleJobScheduler(); } - public double getCostUpperBound() { - return costUpperBound; + public void pushJob(Job job) { + jobPool.push(job); } - public void setCostUpperBound(double costUpperBound) { - this.costUpperBound = costUpperBound; - } - - public OptimizerContext getOptimizerContext() { - return optimizerContext; + public Memo getMemo() { + return memo; } public ConnectContext getConnectContext() { return connectContext; } - public PhysicalProperties getPhysicalProperties() { - return physicalProperties; + public RuleSet getRuleSet() { + return ruleSet; } - public Set getNeededAttributes() { - return neededSlots; + public void setRuleSet(RuleSet ruleSet) { + this.ruleSet = ruleSet; + } + + public JobPool getJobPool() { + return jobPool; + } + + public void setJobPool(JobPool jobPool) { + this.jobPool = jobPool; + } + + public JobScheduler getJobScheduler() { + return jobScheduler; + } + + public JobContext getCurrentJobContext() { + return currentJobContext; + } + + public void setCurrentJobContext(JobContext currentJobContext) { + this.currentJobContext = currentJobContext; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/BatchRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/BatchRulesJob.java index c2a9fed4e5..afa3bcf255 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/BatchRulesJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/BatchRulesJob.java @@ -50,9 +50,9 @@ public class BatchRulesJob { } Collections.reverse(rules); return new RewriteBottomUpJob( - plannerContext.getOptimizerContext().getMemo().getRoot(), + plannerContext.getMemo().getRoot(), rules, - plannerContext); + plannerContext.getCurrentJobContext()); } protected Job topDownBatch(List> ruleFactories) { @@ -62,21 +62,21 @@ public class BatchRulesJob { } Collections.reverse(rules); return new RewriteTopDownJob( - plannerContext.getOptimizerContext().getMemo().getRoot(), + plannerContext.getMemo().getRoot(), rules, - plannerContext); + plannerContext.getCurrentJobContext()); } protected Job optimize() { return new OptimizeGroupJob( - plannerContext.getOptimizerContext().getMemo().getRoot(), - plannerContext); + plannerContext.getMemo().getRoot(), + plannerContext.getCurrentJobContext()); } public void execute() { for (Job job : rulesJob) { - plannerContext.getOptimizerContext().pushJob(job); - plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext); + plannerContext.pushJob(job); + plannerContext.getJobScheduler().executeJobPool(plannerContext); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java index 3f7ef0200c..8363ef7af7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.jobs; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.Rule; @@ -33,19 +32,19 @@ import java.util.stream.Collectors; */ public abstract class Job> { protected JobType type; - protected PlannerContext context; + protected JobContext context; - public Job(JobType type, PlannerContext context) { + public Job(JobType type, JobContext context) { this.type = type; this.context = context; } public void pushTask(Job job) { - context.getOptimizerContext().pushJob(job); + context.getPlannerContext().pushJob(job); } public RuleSet getRuleSet() { - return context.getOptimizerContext().getRuleSet(); + return context.getPlannerContext().getRuleSet(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java new file mode 100644 index 0000000000..0aee081b9b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs; + +import org.apache.doris.nereids.PlannerContext; +import org.apache.doris.nereids.properties.PhysicalProperties; + +/** + * Context for one job in Nereids' cascades framework. + */ +public class JobContext { + private final PlannerContext plannerContext; + private final PhysicalProperties requiredProperties; + private final double costUpperBound; + + public JobContext(PlannerContext plannerContext, PhysicalProperties requiredProperties, double costUpperBound) { + this.plannerContext = plannerContext; + this.requiredProperties = requiredProperties; + this.costUpperBound = costUpperBound; + } + + public PlannerContext getPlannerContext() { + return plannerContext; + } + + public PhysicalProperties getRequiredProperties() { + return requiredProperties; + } + + public double getCostUpperBound() { + return costUpperBound; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java index fd16020ced..30d3e2b458 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java @@ -17,9 +17,9 @@ package org.apache.doris.nereids.jobs.cascades; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.pattern.GroupExpressionMatching; @@ -42,9 +42,9 @@ public class ApplyRuleJob extends Job { * * @param groupExpression apply rule on this {@link GroupExpression} * @param rule rule to be applied - * @param context context of optimization + * @param context context of current job */ - public ApplyRuleJob(GroupExpression groupExpression, Rule rule, PlannerContext context) { + public ApplyRuleJob(GroupExpression groupExpression, Rule rule, JobContext context) { super(JobType.APPLY_RULE, context); this.groupExpression = groupExpression; this.rule = rule; @@ -60,9 +60,9 @@ public class ApplyRuleJob extends Job { GroupExpressionMatching groupExpressionMatching = new GroupExpressionMatching(rule.getPattern(), groupExpression); for (Plan plan : groupExpressionMatching) { - List newPlans = rule.transform(plan, context); + List newPlans = rule.transform(plan, context.getPlannerContext()); for (Plan newPlan : newPlans) { - GroupExpression newGroupExpression = context.getOptimizerContext().getMemo() + GroupExpression newGroupExpression = context.getPlannerContext().getMemo() .copyIn(newPlan, groupExpression.getParent(), rule.isRewrite()); if (newPlan instanceof LogicalPlan) { pushTask(new DeriveStatsJob(newGroupExpression, context)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java index e7f71b3f3e..ee72328afc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java @@ -17,8 +17,8 @@ package org.apache.doris.nereids.jobs.cascades; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -30,7 +30,7 @@ import org.apache.doris.nereids.trees.plans.Plan; public class CostAndEnforcerJob extends Job { private final GroupExpression groupExpression; - public CostAndEnforcerJob(GroupExpression groupExpression, PlannerContext context) { + public CostAndEnforcerJob(GroupExpression groupExpression, JobContext context) { super(JobType.OPTIMIZE_CHILDREN, context); this.groupExpression = groupExpression; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java index 056c57f594..3a289d607f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java @@ -17,8 +17,8 @@ package org.apache.doris.nereids.jobs.cascades; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -35,9 +35,9 @@ public class DeriveStatsJob extends Job { * Constructor for DeriveStatsJob. * * @param groupExpression Derive stats on this {@link GroupExpression} - * @param context context of optimization + * @param context context of current job */ - public DeriveStatsJob(GroupExpression groupExpression, PlannerContext context) { + public DeriveStatsJob(GroupExpression groupExpression, JobContext context) { super(JobType.DERIVE_STATS, context); this.groupExpression = groupExpression; this.deriveChildren = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupExpressionJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupExpressionJob.java index b2f1da9a8e..8e2ca6c63a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupExpressionJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupExpressionJob.java @@ -17,8 +17,8 @@ package org.apache.doris.nereids.jobs.cascades; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -41,9 +41,9 @@ public class ExploreGroupExpressionJob extends Job { * Constructor for ExplorePlanJob. * * @param groupExpression {@link GroupExpression} to be explored - * @param context context of optimization + * @param context context of current job */ - public ExploreGroupExpressionJob(GroupExpression groupExpression, PlannerContext context) { + public ExploreGroupExpressionJob(GroupExpression groupExpression, JobContext context) { super(JobType.EXPLORE_PLAN, context); this.groupExpression = groupExpression; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupJob.java index daeff8046b..d5fcec6d19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupJob.java @@ -17,8 +17,8 @@ package org.apache.doris.nereids.jobs.cascades; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -34,9 +34,9 @@ public class ExploreGroupJob extends Job { * Constructor for ExploreGroupJob. * * @param group {@link Group} to be explored - * @param context context of optimization + * @param context context of current job */ - public ExploreGroupJob(Group group, PlannerContext context) { + public ExploreGroupJob(Group group, JobContext context) { super(JobType.EXPLORE_PLAN_SET, context); this.group = group; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java index 83f699e640..da815e2c54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java @@ -17,8 +17,8 @@ package org.apache.doris.nereids.jobs.cascades; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -36,7 +36,7 @@ import java.util.List; public class OptimizeGroupExpressionJob extends Job { private final GroupExpression groupExpression; - public OptimizeGroupExpressionJob(GroupExpression groupExpression, PlannerContext context) { + public OptimizeGroupExpressionJob(GroupExpression groupExpression, JobContext context) { super(JobType.OPTIMIZE_PLAN, context); this.groupExpression = groupExpression; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupJob.java index 56b97fb5dd..525de8a0de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupJob.java @@ -17,8 +17,8 @@ package org.apache.doris.nereids.jobs.cascades; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -30,7 +30,7 @@ import org.apache.doris.nereids.trees.plans.Plan; public class OptimizeGroupJob extends Job { private final Group group; - public OptimizeGroupJob(Group group, PlannerContext context) { + public OptimizeGroupJob(Group group, JobContext context) { super(JobType.OPTIMIZE_PLAN_SET, context); this.group = group; } @@ -38,16 +38,16 @@ public class OptimizeGroupJob extends Job { @Override public void execute() { if (group.getCostLowerBound() > context.getCostUpperBound() - || group.getLowestCostPlan(context.getPhysicalProperties()).isPresent()) { + || group.getLowestCostPlan(context.getRequiredProperties()).isPresent()) { return; } if (!group.isExplored()) { for (GroupExpression logicalGroupExpression : group.getLogicalExpressions()) { - context.getOptimizerContext().pushJob(new OptimizeGroupExpressionJob(logicalGroupExpression, context)); + context.getPlannerContext().pushJob(new OptimizeGroupExpressionJob(logicalGroupExpression, context)); } } for (GroupExpression physicalGroupExpression : group.getPhysicalExpressions()) { - context.getOptimizerContext().pushJob(new CostAndEnforcerJob(physicalGroupExpression, context)); + context.getPlannerContext().pushJob(new CostAndEnforcerJob(physicalGroupExpression, context)); } group.setExplored(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteBottomUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteBottomUpJob.java index 81a6719b13..4d9d525ab8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteBottomUpJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteBottomUpJob.java @@ -17,9 +17,9 @@ package org.apache.doris.nereids.jobs.rewrite; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -42,18 +42,19 @@ public class RewriteBottomUpJob extends Job { private final List> rules; private final boolean childrenOptimized; - public RewriteBottomUpJob(Group group, PlannerContext context, List> factories) { + + public RewriteBottomUpJob(Group group, JobContext context, List> factories) { this(group, factories.stream() .flatMap(factory -> factory.buildRules().stream()) .collect(Collectors.toList()), context, false); } - public RewriteBottomUpJob(Group group, List> rules, PlannerContext context) { + public RewriteBottomUpJob(Group group, List> rules, JobContext context) { this(group, rules, context, false); } private RewriteBottomUpJob(Group group, List> rules, - PlannerContext context, boolean childrenOptimized) { + JobContext context, boolean childrenOptimized) { super(JobType.BOTTOM_UP_REWRITE, context); this.group = Objects.requireNonNull(group, "group cannot be null"); this.rules = Objects.requireNonNull(rules, "rules cannot be null"); @@ -76,11 +77,11 @@ public class RewriteBottomUpJob extends Job { GroupExpressionMatching groupExpressionMatching = new GroupExpressionMatching(rule.getPattern(), logicalExpression); for (Plan before : groupExpressionMatching) { - List afters = rule.transform(before, context); + List afters = rule.transform(before, context.getPlannerContext()); Preconditions.checkArgument(afters.size() == 1); Plan after = afters.get(0); if (after != before) { - GroupExpression groupExpr = context.getOptimizerContext() + GroupExpression groupExpr = context.getPlannerContext() .getMemo() .copyIn(after, group, rule.isRewrite()); groupExpr.setApplied(rule); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java index 5a43b7a29c..b023a77dcd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java @@ -17,8 +17,8 @@ package org.apache.doris.nereids.jobs.rewrite; -import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.jobs.Job; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -40,7 +40,7 @@ public class RewriteTopDownJob extends Job { private final Group group; private final List> rules; - public RewriteTopDownJob(Group group, PlannerContext context, List> factories) { + public RewriteTopDownJob(Group group, JobContext context, List> factories) { this(group, factories.stream() .flatMap(factory -> factory.buildRules().stream()) .collect(Collectors.toList()), context); @@ -53,7 +53,7 @@ public class RewriteTopDownJob extends Job { * @param rules rewrite rules * @param context planner context */ - public RewriteTopDownJob(Group group, List> rules, PlannerContext context) { + public RewriteTopDownJob(Group group, List> rules, JobContext context) { super(JobType.TOP_DOWN_REWRITE, context); this.group = Objects.requireNonNull(group, "group cannot be null"); this.rules = Objects.requireNonNull(rules, "rules cannot be null"); @@ -68,12 +68,12 @@ public class RewriteTopDownJob extends Job { GroupExpressionMatching groupExpressionMatching = new GroupExpressionMatching(rule.getPattern(), logicalExpression); for (Plan before : groupExpressionMatching) { - List afters = rule.transform(before, context); + List afters = rule.transform(before, context.getPlannerContext()); Preconditions.checkArgument(afters.size() == 1); Plan after = afters.get(0); if (after != before) { - GroupExpression expression = context.getOptimizerContext().getMemo() - .copyIn(after, group, rule.isRewrite()); + GroupExpression expression = context.getPlannerContext() + .getMemo().copyIn(after, group, rule.isRewrite()); expression.setApplied(rule); pushTask(new RewriteTopDownJob(group, rules, context)); return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java index 58ef7482c8..395f153f2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java @@ -32,7 +32,7 @@ public class SimpleJobScheduler implements JobScheduler { @Override public void executeJobPool(PlannerContext plannerContext) throws AnalysisException { - JobPool pool = plannerContext.getOptimizerContext().getJobPool(); + JobPool pool = plannerContext.getJobPool(); while (!pool.isEmpty()) { Job job = pool.pop(); job.execute(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/AnalyzeSSBTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/AnalyzeSSBTest.java index fc742659ea..7780cb0e01 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/AnalyzeSSBTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/AnalyzeSSBTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids; import org.apache.doris.nereids.analyzer.Unbound; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.Memo; @@ -139,8 +140,10 @@ public class AnalyzeSSBTest extends TestWithFeService { private LogicalPlan analyze(LogicalPlan inputPlan, ConnectContext connectContext) { Memo memo = new Memo(); memo.initialize(inputPlan); - OptimizerContext optimizerContext = new OptimizerContext(memo); - PlannerContext plannerContext = new PlannerContext(optimizerContext, connectContext, new PhysicalProperties()); + + PlannerContext plannerContext = new PlannerContext(memo, connectContext); + JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE); + plannerContext.setCurrentJobContext(jobContext); executeRewriteBottomUpJob(plannerContext, new BindFunction()); executeRewriteBottomUpJob(plannerContext, new BindRelation()); @@ -150,11 +153,11 @@ public class AnalyzeSSBTest extends TestWithFeService { } private void executeRewriteBottomUpJob(PlannerContext plannerContext, RuleFactory ruleFactory) { - OptimizerContext optimizerContext = plannerContext.getOptimizerContext(); - Group rootGroup = optimizerContext.getMemo().getRoot(); - RewriteBottomUpJob job = new RewriteBottomUpJob(rootGroup, plannerContext, ImmutableList.of(ruleFactory)); - optimizerContext.pushJob(job); - optimizerContext.getJobScheduler().executeJobPool(plannerContext); + Group rootGroup = plannerContext.getMemo().getRoot(); + RewriteBottomUpJob job = new RewriteBottomUpJob(rootGroup, + plannerContext.getCurrentJobContext(), ImmutableList.of(ruleFactory)); + plannerContext.pushJob(job); + plannerContext.getJobScheduler().executeJobPool(plannerContext); } private boolean checkBound(LogicalPlan root) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java index 4c620c8764..d3416e463a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java @@ -20,7 +20,6 @@ package org.apache.doris.nereids.jobs; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; -import org.apache.doris.nereids.OptimizerContext; import org.apache.doris.nereids.PlannerContext; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob; @@ -40,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.Plans; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.StringType; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -72,12 +72,14 @@ public class RewriteTopDownJobTest implements Plans { Memo memo = new Memo(); memo.initialize(root); - OptimizerContext optimizerContext = new OptimizerContext(memo); - PlannerContext plannerContext = new PlannerContext(optimizerContext, null, new PhysicalProperties()); + PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext()); + JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE); + plannerContext.setCurrentJobContext(jobContext); List> fakeRules = Lists.newArrayList(new FakeRule().build()); - RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules, plannerContext); - plannerContext.getOptimizerContext().pushJob(rewriteTopDownJob); - plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext); + RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules, + plannerContext.getCurrentJobContext()); + plannerContext.pushJob(rewriteTopDownJob); + plannerContext.getJobScheduler().executeJobPool(plannerContext); Group rootGroup = memo.getRoot(); Assertions.assertEquals(1, rootGroup.getLogicalExpressions().size()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalProjectToPhysicalProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalProjectToPhysicalProjectTest.java index fbff548b8d..61cab85dc5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalProjectToPhysicalProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalProjectToPhysicalProjectTest.java @@ -17,8 +17,8 @@ package org.apache.doris.nereids.rules.implementation; -import org.apache.doris.nereids.OptimizerContext; import org.apache.doris.nereids.PlannerContext; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.Memo; import org.apache.doris.nereids.operators.OperatorType; @@ -45,8 +45,9 @@ public class LogicalProjectToPhysicalProjectTest implements Plans { Rule rule = new LogicalProjectToPhysicalProject().build(); - PlannerContext plannerContext = new PlannerContext(new OptimizerContext(new Memo()), new ConnectContext(), - new PhysicalProperties()); + PlannerContext plannerContext = new PlannerContext(new Memo(), new ConnectContext()); + JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE); + plannerContext.setCurrentJobContext(jobContext); List transform = rule.transform(plan, plannerContext); Assert.assertEquals(1, transform.size()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java index ee07b99194..015dcc8cc8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java @@ -21,8 +21,8 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; -import org.apache.doris.nereids.OptimizerContext; import org.apache.doris.nereids.PlannerContext; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.Memo; @@ -121,12 +121,14 @@ public class PushDownPredicateTest implements Plans { memo.initialize(root); System.out.println(memo.copyOut().treeString()); - OptimizerContext optimizerContext = new OptimizerContext(memo); - PlannerContext plannerContext = new PlannerContext(optimizerContext, null, new PhysicalProperties()); + PlannerContext plannerContext = new PlannerContext(memo, null); + JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE); + plannerContext.setCurrentJobContext(jobContext); + RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), - ImmutableList.of(new PushPredicateThroughJoin().build()), plannerContext); - plannerContext.getOptimizerContext().pushJob(rewriteTopDownJob); - plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext); + ImmutableList.of(new PushPredicateThroughJoin().build()), jobContext); + plannerContext.pushJob(rewriteTopDownJob); + plannerContext.getJobScheduler().executeJobPool(plannerContext); Group rootGroup = memo.getRoot(); System.out.println(memo.copyOut().treeString()); @@ -169,12 +171,14 @@ public class PushDownPredicateTest implements Plans { memo.initialize(root); System.out.println(memo.copyOut().treeString()); - OptimizerContext optimizerContext = new OptimizerContext(memo); - PlannerContext plannerContext = new PlannerContext(optimizerContext, null, new PhysicalProperties()); + PlannerContext plannerContext = new PlannerContext(memo, null); + JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE); + plannerContext.setCurrentJobContext(jobContext); + RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), - ImmutableList.of(new PushPredicateThroughJoin().build()), plannerContext); - plannerContext.getOptimizerContext().pushJob(rewriteTopDownJob); - plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext); + ImmutableList.of(new PushPredicateThroughJoin().build()), jobContext); + plannerContext.pushJob(rewriteTopDownJob); + plannerContext.getJobScheduler().executeJobPool(plannerContext); Group rootGroup = memo.getRoot(); System.out.println(memo.copyOut().treeString()); @@ -229,12 +233,14 @@ public class PushDownPredicateTest implements Plans { memo.initialize(root); System.out.println(memo.copyOut().treeString()); - OptimizerContext optimizerContext = new OptimizerContext(memo); - PlannerContext plannerContext = new PlannerContext(optimizerContext, null, new PhysicalProperties()); + PlannerContext plannerContext = new PlannerContext(memo, null); + JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE); + plannerContext.setCurrentJobContext(jobContext); + List> fakeRules = Lists.newArrayList(new PushPredicateThroughJoin().build()); - RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules, plannerContext); - plannerContext.getOptimizerContext().pushJob(rewriteTopDownJob); - plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext); + RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules, jobContext); + plannerContext.pushJob(rewriteTopDownJob); + plannerContext.getJobScheduler().executeJobPool(plannerContext); Group rootGroup = memo.getRoot(); System.out.println(memo.copyOut().treeString());