database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](Nereids)refactor PlannerContext and JobContext (#10485)

Browse Source 
Refactor Context in Cascades:
use two context in cascades framework.

JobContext is used in each job, contains such attributes:
- reference to PlannerContext
- current cost upper bound 
- current required physical properties

PlannerContext is used to hold global info for query planner, contains such attributes:
- reference to Memo
- reference to connectContext
- reference to ruleset could be used for plan
- job pool to maintain unexecuted jobs
- job scheduler to schedule unexecuted jobs
- current job context for next job to be executed
This commit is contained in:
morrySnow
2022-07-06 18:36:31 +08:00
committed by GitHub
parent 29d4809c80
commit 5dfb59844f
20 changed files with 195 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.nereids.jobs.AnalyzeRulesJob;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.OptimizeRulesJob;
import org.apache.doris.nereids.jobs.PredicatePushDownRulesJob;
import org.apache.doris.nereids.memo.Group;
@ -129,8 +130,9 @@ public class NereidsPlanner extends Planner {
Memo memo = new Memo();
memo.initialize(plan);
OptimizerContext optimizerContext = new OptimizerContext(memo);
plannerContext = new PlannerContext(optimizerContext, connectContext, outputProperties);
plannerContext = new PlannerContext(memo, connectContext);
JobContext jobContext = new JobContext(plannerContext, outputProperties, Double.MAX_VALUE);
plannerContext.setCurrentJobContext(jobContext);
// Get plan directly. Just for SSB.
return doPlan();
@ -159,7 +161,7 @@ public class NereidsPlanner extends Planner {
}
public Group getRoot() {
return plannerContext.getOptimizerContext().getMemo().getRoot();
return plannerContext.getMemo().getRoot();
}
private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physicalProperties)

76
fe/fe-core/src/main/java/org/apache/doris/nereids/OptimizerContext.java
View File

@ -1,76 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.scheduler.JobPool;
import org.apache.doris.nereids.jobs.scheduler.JobScheduler;
import org.apache.doris.nereids.jobs.scheduler.JobStack;
import org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler;
import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.rules.RuleSet;
/**
* Context used in memo.
*/
public class OptimizerContext {
private final Memo memo;
private RuleSet ruleSet;
private JobPool jobPool;
private final JobScheduler jobScheduler;
/**
* Constructor of OptimizerContext.
*
* @param memo {@link Memo} reference
*/
public OptimizerContext(Memo memo) {
this.memo = memo;
this.ruleSet = new RuleSet();
this.jobPool = new JobStack();
this.jobScheduler = new SimpleJobScheduler();
}
public JobPool getJobPool() {
return jobPool;
}
public void setJobPool(JobPool jobPool) {
this.jobPool = jobPool;
}
public RuleSet getRuleSet() {
return ruleSet;
}
public void setRuleSet(RuleSet ruleSet) {
this.ruleSet = ruleSet;
}
public Memo getMemo() {
return memo;
}
public void pushJob(Job job) {
jobPool.push(job);
}
public JobScheduler getJobScheduler() {
return jobScheduler;
}
}

84
fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerContext.java
View File

@ -17,63 +17,77 @@
package org.apache.doris.nereids;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.scheduler.JobPool;
import org.apache.doris.nereids.jobs.scheduler.JobScheduler;
import org.apache.doris.nereids.jobs.scheduler.JobStack;
import org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler;
import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Sets;
import java.util.Set;
/**
* Context used in all stage in Nereids.
* Context used in memo.
*/
public class PlannerContext {
private final OptimizerContext optimizerContext;
private final Memo memo;
private final ConnectContext connectContext;
private final PhysicalProperties physicalProperties;
private double costUpperBound;
private Set<Slot> neededSlots;
private RuleSet ruleSet;
private JobPool jobPool;
private final JobScheduler jobScheduler;
private JobContext currentJobContext;
/**
* Constructor of OptimizationContext.
* Constructor of OptimizerContext.
*
* @param optimizerContext context includes all data struct used in memo
* @param connectContext connect context of this query
* @param physicalProperties target physical properties
* @param memo {@link Memo} reference
*/
public PlannerContext(
OptimizerContext optimizerContext,
ConnectContext connectContext,
PhysicalProperties physicalProperties) {
this.optimizerContext = optimizerContext;
public PlannerContext(Memo memo, ConnectContext connectContext) {
this.memo = memo;
this.connectContext = connectContext;
this.physicalProperties = physicalProperties;
this.costUpperBound = Double.MAX_VALUE;
this.neededSlots = Sets.newHashSet();
this.ruleSet = new RuleSet();
this.jobPool = new JobStack();
this.jobScheduler = new SimpleJobScheduler();
}
public double getCostUpperBound() {
return costUpperBound;
public void pushJob(Job job) {
jobPool.push(job);
}
public void setCostUpperBound(double costUpperBound) {
this.costUpperBound = costUpperBound;
}
public OptimizerContext getOptimizerContext() {
return optimizerContext;
public Memo getMemo() {
return memo;
}
public ConnectContext getConnectContext() {
return connectContext;
}
public PhysicalProperties getPhysicalProperties() {
return physicalProperties;
public RuleSet getRuleSet() {
return ruleSet;
}
public Set<Slot> getNeededAttributes() {
return neededSlots;
public void setRuleSet(RuleSet ruleSet) {
this.ruleSet = ruleSet;
}
public JobPool getJobPool() {
return jobPool;
}
public void setJobPool(JobPool jobPool) {
this.jobPool = jobPool;
}
public JobScheduler getJobScheduler() {
return jobScheduler;
}
public JobContext getCurrentJobContext() {
return currentJobContext;
}
public void setCurrentJobContext(JobContext currentJobContext) {
this.currentJobContext = currentJobContext;
}
}

16
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/BatchRulesJob.java
View File

@ -50,9 +50,9 @@ public class BatchRulesJob {
}
Collections.reverse(rules);
return new RewriteBottomUpJob(
plannerContext.getOptimizerContext().getMemo().getRoot(),
plannerContext.getMemo().getRoot(),
rules,
plannerContext);
plannerContext.getCurrentJobContext());
}
protected Job<Plan> topDownBatch(List<RuleFactory<Plan>> ruleFactories) {
@ -62,21 +62,21 @@ public class BatchRulesJob {
}
Collections.reverse(rules);
return new RewriteTopDownJob(
plannerContext.getOptimizerContext().getMemo().getRoot(),
plannerContext.getMemo().getRoot(),
rules,
plannerContext);
plannerContext.getCurrentJobContext());
}
protected Job<Plan> optimize() {
return new OptimizeGroupJob(
plannerContext.getOptimizerContext().getMemo().getRoot(),
plannerContext);
plannerContext.getMemo().getRoot(),
plannerContext.getCurrentJobContext());
}
public void execute() {
for (Job job : rulesJob) {
plannerContext.getOptimizerContext().pushJob(job);
plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext);
plannerContext.pushJob(job);
plannerContext.getJobScheduler().executeJobPool(plannerContext);
}
}
}

9
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/Job.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.nereids.jobs;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.Rule;
@ -33,19 +32,19 @@ import java.util.stream.Collectors;
*/
public abstract class Job<NODE_TYPE extends TreeNode<NODE_TYPE>> {
protected JobType type;
protected PlannerContext context;
protected JobContext context;
public Job(JobType type, PlannerContext context) {
public Job(JobType type, JobContext context) {
this.type = type;
this.context = context;
}
public void pushTask(Job job) {
context.getOptimizerContext().pushJob(job);
context.getPlannerContext().pushJob(job);
}
public RuleSet getRuleSet() {
return context.getOptimizerContext().getRuleSet();
return context.getPlannerContext().getRuleSet();
}
/**

48
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/JobContext.java Normal file
View File

@ -0,0 +1,48 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.jobs;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.properties.PhysicalProperties;
/**
* Context for one job in Nereids' cascades framework.
*/
public class JobContext {
private final PlannerContext plannerContext;
private final PhysicalProperties requiredProperties;
private final double costUpperBound;
public JobContext(PlannerContext plannerContext, PhysicalProperties requiredProperties, double costUpperBound) {
this.plannerContext = plannerContext;
this.requiredProperties = requiredProperties;
this.costUpperBound = costUpperBound;
}
public PlannerContext getPlannerContext() {
return plannerContext;
}
public PhysicalProperties getRequiredProperties() {
return requiredProperties;
}
public double getCostUpperBound() {
return costUpperBound;
}
}

10
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ApplyRuleJob.java
View File

@ -17,9 +17,9 @@
package org.apache.doris.nereids.jobs.cascades;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.pattern.GroupExpressionMatching;
@ -42,9 +42,9 @@ public class ApplyRuleJob extends Job<Plan> {
*
* @param groupExpression apply rule on this {@link GroupExpression}
* @param rule rule to be applied
* @param context context of optimization
* @param context context of current job
*/
public ApplyRuleJob(GroupExpression groupExpression, Rule<Plan> rule, PlannerContext context) {
public ApplyRuleJob(GroupExpression groupExpression, Rule<Plan> rule, JobContext context) {
super(JobType.APPLY_RULE, context);
this.groupExpression = groupExpression;
this.rule = rule;
@ -60,9 +60,9 @@ public class ApplyRuleJob extends Job<Plan> {
GroupExpressionMatching groupExpressionMatching
= new GroupExpressionMatching(rule.getPattern(), groupExpression);
for (Plan plan : groupExpressionMatching) {
List<Plan> newPlans = rule.transform(plan, context);
List<Plan> newPlans = rule.transform(plan, context.getPlannerContext());
for (Plan newPlan : newPlans) {
GroupExpression newGroupExpression = context.getOptimizerContext().getMemo()
GroupExpression newGroupExpression = context.getPlannerContext().getMemo()
.copyIn(newPlan, groupExpression.getParent(), rule.isRewrite());
if (newPlan instanceof LogicalPlan) {
pushTask(new DeriveStatsJob(newGroupExpression, context));

4
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/CostAndEnforcerJob.java
View File

@ -17,8 +17,8 @@
package org.apache.doris.nereids.jobs.cascades;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
@ -30,7 +30,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
public class CostAndEnforcerJob extends Job<Plan> {
private final GroupExpression groupExpression;
public CostAndEnforcerJob(GroupExpression groupExpression, PlannerContext context) {
public CostAndEnforcerJob(GroupExpression groupExpression, JobContext context) {
super(JobType.OPTIMIZE_CHILDREN, context);
this.groupExpression = groupExpression;
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java
View File

@ -17,8 +17,8 @@
package org.apache.doris.nereids.jobs.cascades;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
@ -35,9 +35,9 @@ public class DeriveStatsJob extends Job<Plan> {
* Constructor for DeriveStatsJob.
*
* @param groupExpression Derive stats on this {@link GroupExpression}
* @param context context of optimization
* @param context context of current job
*/
public DeriveStatsJob(GroupExpression groupExpression, PlannerContext context) {
public DeriveStatsJob(GroupExpression groupExpression, JobContext context) {
super(JobType.DERIVE_STATS, context);
this.groupExpression = groupExpression;
this.deriveChildren = false;

6
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupExpressionJob.java
View File

@ -17,8 +17,8 @@
package org.apache.doris.nereids.jobs.cascades;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
@ -41,9 +41,9 @@ public class ExploreGroupExpressionJob extends Job<Plan> {
* Constructor for ExplorePlanJob.
*
* @param groupExpression {@link GroupExpression} to be explored
* @param context context of optimization
* @param context context of current job
*/
public ExploreGroupExpressionJob(GroupExpression groupExpression, PlannerContext context) {
public ExploreGroupExpressionJob(GroupExpression groupExpression, JobContext context) {
super(JobType.EXPLORE_PLAN, context);
this.groupExpression = groupExpression;
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/ExploreGroupJob.java
View File

@ -17,8 +17,8 @@
package org.apache.doris.nereids.jobs.cascades;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
@ -34,9 +34,9 @@ public class ExploreGroupJob extends Job<Plan> {
* Constructor for ExploreGroupJob.
*
* @param group {@link Group} to be explored
* @param context context of optimization
* @param context context of current job
*/
public ExploreGroupJob(Group group, PlannerContext context) {
public ExploreGroupJob(Group group, JobContext context) {
super(JobType.EXPLORE_PLAN_SET, context);
this.group = group;
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
View File

@ -17,8 +17,8 @@
package org.apache.doris.nereids.jobs.cascades;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
@ -36,7 +36,7 @@ import java.util.List;
public class OptimizeGroupExpressionJob extends Job<Plan> {
private final GroupExpression groupExpression;
public OptimizeGroupExpressionJob(GroupExpression groupExpression, PlannerContext context) {
public OptimizeGroupExpressionJob(GroupExpression groupExpression, JobContext context) {
super(JobType.OPTIMIZE_PLAN, context);
this.groupExpression = groupExpression;
}

10
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupJob.java
View File

@ -17,8 +17,8 @@
package org.apache.doris.nereids.jobs.cascades;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
@ -30,7 +30,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
public class OptimizeGroupJob extends Job<Plan> {
private final Group group;
public OptimizeGroupJob(Group group, PlannerContext context) {
public OptimizeGroupJob(Group group, JobContext context) {
super(JobType.OPTIMIZE_PLAN_SET, context);
this.group = group;
}
@ -38,16 +38,16 @@ public class OptimizeGroupJob extends Job<Plan> {
@Override
public void execute() {
if (group.getCostLowerBound() > context.getCostUpperBound()
|| group.getLowestCostPlan(context.getPhysicalProperties()).isPresent()) {
|| group.getLowestCostPlan(context.getRequiredProperties()).isPresent()) {
return;
}
if (!group.isExplored()) {
for (GroupExpression logicalGroupExpression : group.getLogicalExpressions()) {
context.getOptimizerContext().pushJob(new OptimizeGroupExpressionJob(logicalGroupExpression, context));
context.getPlannerContext().pushJob(new OptimizeGroupExpressionJob(logicalGroupExpression, context));
}
}
for (GroupExpression physicalGroupExpression : group.getPhysicalExpressions()) {
context.getOptimizerContext().pushJob(new CostAndEnforcerJob(physicalGroupExpression, context));
context.getPlannerContext().pushJob(new CostAndEnforcerJob(physicalGroupExpression, context));
}
group.setExplored(true);
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteBottomUpJob.java
View File

@ -17,9 +17,9 @@
package org.apache.doris.nereids.jobs.rewrite;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
@ -42,18 +42,19 @@ public class RewriteBottomUpJob extends Job<Plan> {
private final List<Rule<Plan>> rules;
private final boolean childrenOptimized;
public RewriteBottomUpJob(Group group, PlannerContext context, List<RuleFactory<Plan>> factories) {
public RewriteBottomUpJob(Group group, JobContext context, List<RuleFactory<Plan>> factories) {
this(group, factories.stream()
.flatMap(factory -> factory.buildRules().stream())
.collect(Collectors.toList()), context, false);
}
public RewriteBottomUpJob(Group group, List<Rule<Plan>> rules, PlannerContext context) {
public RewriteBottomUpJob(Group group, List<Rule<Plan>> rules, JobContext context) {
this(group, rules, context, false);
}
private RewriteBottomUpJob(Group group, List<Rule<Plan>> rules,
PlannerContext context, boolean childrenOptimized) {
JobContext context, boolean childrenOptimized) {
super(JobType.BOTTOM_UP_REWRITE, context);
this.group = Objects.requireNonNull(group, "group cannot be null");
this.rules = Objects.requireNonNull(rules, "rules cannot be null");
@ -76,11 +77,11 @@ public class RewriteBottomUpJob extends Job<Plan> {
GroupExpressionMatching groupExpressionMatching
= new GroupExpressionMatching(rule.getPattern(), logicalExpression);
for (Plan before : groupExpressionMatching) {
List<Plan> afters = rule.transform(before, context);
List<Plan> afters = rule.transform(before, context.getPlannerContext());
Preconditions.checkArgument(afters.size() == 1);
Plan after = afters.get(0);
if (after != before) {
GroupExpression groupExpr = context.getOptimizerContext()
GroupExpression groupExpr = context.getPlannerContext()
.getMemo()
.copyIn(after, group, rule.isRewrite());
groupExpr.setApplied(rule);

12
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java
View File

@ -17,8 +17,8 @@
package org.apache.doris.nereids.jobs.rewrite;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
@ -40,7 +40,7 @@ public class RewriteTopDownJob extends Job<Plan> {
private final Group group;
private final List<Rule<Plan>> rules;
public RewriteTopDownJob(Group group, PlannerContext context, List<RuleFactory<Plan>> factories) {
public RewriteTopDownJob(Group group, JobContext context, List<RuleFactory<Plan>> factories) {
this(group, factories.stream()
.flatMap(factory -> factory.buildRules().stream())
.collect(Collectors.toList()), context);
@ -53,7 +53,7 @@ public class RewriteTopDownJob extends Job<Plan> {
* @param rules rewrite rules
* @param context planner context
*/
public RewriteTopDownJob(Group group, List<Rule<Plan>> rules, PlannerContext context) {
public RewriteTopDownJob(Group group, List<Rule<Plan>> rules, JobContext context) {
super(JobType.TOP_DOWN_REWRITE, context);
this.group = Objects.requireNonNull(group, "group cannot be null");
this.rules = Objects.requireNonNull(rules, "rules cannot be null");
@ -68,12 +68,12 @@ public class RewriteTopDownJob extends Job<Plan> {
GroupExpressionMatching groupExpressionMatching
= new GroupExpressionMatching(rule.getPattern(), logicalExpression);
for (Plan before : groupExpressionMatching) {
List<Plan> afters = rule.transform(before, context);
List<Plan> afters = rule.transform(before, context.getPlannerContext());
Preconditions.checkArgument(afters.size() == 1);
Plan after = afters.get(0);
if (after != before) {
GroupExpression expression = context.getOptimizerContext().getMemo()
.copyIn(after, group, rule.isRewrite());
GroupExpression expression = context.getPlannerContext()
.getMemo().copyIn(after, group, rule.isRewrite());
expression.setApplied(rule);
pushTask(new RewriteTopDownJob(group, rules, context));
return;

2
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java
View File

@ -32,7 +32,7 @@ public class SimpleJobScheduler implements JobScheduler {
@Override
public void executeJobPool(PlannerContext plannerContext) throws AnalysisException {
JobPool pool = plannerContext.getOptimizerContext().getJobPool();
JobPool pool = plannerContext.getJobPool();
while (!pool.isEmpty()) {
Job job = pool.pop();
job.execute();

17
fe/fe-core/src/test/java/org/apache/doris/nereids/AnalyzeSSBTest.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.nereids;
import org.apache.doris.nereids.analyzer.Unbound;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.Memo;
@ -139,8 +140,10 @@ public class AnalyzeSSBTest extends TestWithFeService {
private LogicalPlan analyze(LogicalPlan inputPlan, ConnectContext connectContext) {
Memo memo = new Memo();
memo.initialize(inputPlan);
OptimizerContext optimizerContext = new OptimizerContext(memo);
PlannerContext plannerContext = new PlannerContext(optimizerContext, connectContext, new PhysicalProperties());
PlannerContext plannerContext = new PlannerContext(memo, connectContext);
JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
plannerContext.setCurrentJobContext(jobContext);
executeRewriteBottomUpJob(plannerContext, new BindFunction());
executeRewriteBottomUpJob(plannerContext, new BindRelation());
@ -150,11 +153,11 @@ public class AnalyzeSSBTest extends TestWithFeService {
}
private void executeRewriteBottomUpJob(PlannerContext plannerContext, RuleFactory<Plan> ruleFactory) {
OptimizerContext optimizerContext = plannerContext.getOptimizerContext();
Group rootGroup = optimizerContext.getMemo().getRoot();
RewriteBottomUpJob job = new RewriteBottomUpJob(rootGroup, plannerContext, ImmutableList.of(ruleFactory));
optimizerContext.pushJob(job);
optimizerContext.getJobScheduler().executeJobPool(plannerContext);
Group rootGroup = plannerContext.getMemo().getRoot();
RewriteBottomUpJob job = new RewriteBottomUpJob(rootGroup,
plannerContext.getCurrentJobContext(), ImmutableList.of(ruleFactory));
plannerContext.pushJob(job);
plannerContext.getJobScheduler().executeJobPool(plannerContext);
}
private boolean checkBound(LogicalPlan root) {

14
fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java
View File

@ -20,7 +20,6 @@ package org.apache.doris.nereids.jobs;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.OptimizerContext;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.analyzer.UnboundRelation;
import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob;
@ -40,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.Plans;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@ -72,12 +72,14 @@ public class RewriteTopDownJobTest implements Plans {
Memo memo = new Memo();
memo.initialize(root);
OptimizerContext optimizerContext = new OptimizerContext(memo);
PlannerContext plannerContext = new PlannerContext(optimizerContext, null, new PhysicalProperties());
PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
plannerContext.setCurrentJobContext(jobContext);
List<Rule<Plan>> fakeRules = Lists.newArrayList(new FakeRule().build());
RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules, plannerContext);
plannerContext.getOptimizerContext().pushJob(rewriteTopDownJob);
plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext);
RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules,
plannerContext.getCurrentJobContext());
plannerContext.pushJob(rewriteTopDownJob);
plannerContext.getJobScheduler().executeJobPool(plannerContext);
Group rootGroup = memo.getRoot();
Assertions.assertEquals(1, rootGroup.getLogicalExpressions().size());

7
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalProjectToPhysicalProjectTest.java
View File

@ -17,8 +17,8 @@
package org.apache.doris.nereids.rules.implementation;
import org.apache.doris.nereids.OptimizerContext;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.operators.OperatorType;
@ -45,8 +45,9 @@ public class LogicalProjectToPhysicalProjectTest implements Plans {
Rule<Plan> rule = new LogicalProjectToPhysicalProject().build();
PlannerContext plannerContext = new PlannerContext(new OptimizerContext(new Memo()), new ConnectContext(),
new PhysicalProperties());
PlannerContext plannerContext = new PlannerContext(new Memo(), new ConnectContext());
JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
plannerContext.setCurrentJobContext(jobContext);
List<Plan> transform = rule.transform(plan, plannerContext);
Assert.assertEquals(1, transform.size());

38
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java
View File

@ -21,8 +21,8 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.OptimizerContext;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.Memo;
@ -121,12 +121,14 @@ public class PushDownPredicateTest implements Plans {
memo.initialize(root);
System.out.println(memo.copyOut().treeString());
OptimizerContext optimizerContext = new OptimizerContext(memo);
PlannerContext plannerContext = new PlannerContext(optimizerContext, null, new PhysicalProperties());
PlannerContext plannerContext = new PlannerContext(memo, null);
JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
plannerContext.setCurrentJobContext(jobContext);
RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
ImmutableList.of(new PushPredicateThroughJoin().build()), plannerContext);
plannerContext.getOptimizerContext().pushJob(rewriteTopDownJob);
plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext);
ImmutableList.of(new PushPredicateThroughJoin().build()), jobContext);
plannerContext.pushJob(rewriteTopDownJob);
plannerContext.getJobScheduler().executeJobPool(plannerContext);
Group rootGroup = memo.getRoot();
System.out.println(memo.copyOut().treeString());
@ -169,12 +171,14 @@ public class PushDownPredicateTest implements Plans {
memo.initialize(root);
System.out.println(memo.copyOut().treeString());
OptimizerContext optimizerContext = new OptimizerContext(memo);
PlannerContext plannerContext = new PlannerContext(optimizerContext, null, new PhysicalProperties());
PlannerContext plannerContext = new PlannerContext(memo, null);
JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
plannerContext.setCurrentJobContext(jobContext);
RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
ImmutableList.of(new PushPredicateThroughJoin().build()), plannerContext);
plannerContext.getOptimizerContext().pushJob(rewriteTopDownJob);
plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext);
ImmutableList.of(new PushPredicateThroughJoin().build()), jobContext);
plannerContext.pushJob(rewriteTopDownJob);
plannerContext.getJobScheduler().executeJobPool(plannerContext);
Group rootGroup = memo.getRoot();
System.out.println(memo.copyOut().treeString());
@ -229,12 +233,14 @@ public class PushDownPredicateTest implements Plans {
memo.initialize(root);
System.out.println(memo.copyOut().treeString());
OptimizerContext optimizerContext = new OptimizerContext(memo);
PlannerContext plannerContext = new PlannerContext(optimizerContext, null, new PhysicalProperties());
PlannerContext plannerContext = new PlannerContext(memo, null);
JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
plannerContext.setCurrentJobContext(jobContext);
List<Rule<Plan>> fakeRules = Lists.newArrayList(new PushPredicateThroughJoin().build());
RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules, plannerContext);
plannerContext.getOptimizerContext().pushJob(rewriteTopDownJob);
plannerContext.getOptimizerContext().getJobScheduler().executeJobPool(plannerContext);
RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules, jobContext);
plannerContext.pushJob(rewriteTopDownJob);
plannerContext.getJobScheduler().executeJobPool(plannerContext);
Group rootGroup = memo.getRoot();
System.out.println(memo.copyOut().treeString());