diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index 2e32fc3ec0..aa8f6e1285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -109,6 +109,19 @@ public class CastExpr extends Expr { analysisDone(); } + /** + * Just use for nereids, put analyze() in finalizeImplForNereids + */ + public CastExpr(Type targetType, Expr e, Void v) { + super(); + Preconditions.checkArgument(targetType.isValid()); + Preconditions.checkNotNull(e); + type = targetType; + targetTypeDef = null; + isImplicit = true; + children.add(e); + } + /** * Copy c'tor used in clone(). */ @@ -531,6 +544,13 @@ public class CastExpr extends Expr { @Override public void finalizeImplForNereids() throws AnalysisException { + try { + analyze(); + } catch (AnalysisException ex) { + LOG.warn("Implicit casts fail", ex); + Preconditions.checkState(false, + "Implicit casts should never throw analysis exception."); + } FunctionName fnName = new FunctionName(getFnName(type)); Function searchDesc = new Function(fnName, Arrays.asList(collectChildReturnTypes()), Type.INVALID, false); if (type.isScalarType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index 1c3f59361d..84c2533972 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -232,7 +232,7 @@ public class ExpressionTranslator extends DefaultExpressionVisitor ruleInvokeTimes = Maps.newLinkedHashMap(); @@ -57,6 +58,14 @@ public class JobContext { this.costUpperBound = costUpperBound; } + public boolean isRewritten() { + return rewritten; + } + + public void setRewritten(boolean rewritten) { + this.rewritten = rewritten; + } + public void onInvokeRule(RuleType ruleType) { addRuleInvokeTimes(ruleType); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java index 3670f16b93..c77c0ec8e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java @@ -58,22 +58,35 @@ public abstract class BatchRulesJob { for (RuleFactory ruleFactory : ruleFactories) { rules.addAll(ruleFactory.buildRules()); } - return new RewriteTopDownJob( - cascadesContext.getMemo().getRoot(), - rules, + return new RewriteTopDownJob(cascadesContext.getMemo().getRoot(), rules, cascadesContext.getCurrentJobContext()); } + protected Job topDownBatch(List ruleFactories, boolean once) { + List rules = new ArrayList<>(); + for (RuleFactory ruleFactory : ruleFactories) { + rules.addAll(ruleFactory.buildRules()); + } + return new RewriteTopDownJob(cascadesContext.getMemo().getRoot(), rules, + cascadesContext.getCurrentJobContext(), once); + } + protected Job optimize() { return new OptimizeGroupJob( cascadesContext.getMemo().getRoot(), cascadesContext.getCurrentJobContext()); } + /** + * execute. + */ public void execute() { for (Job job : rulesJob) { - cascadesContext.pushJob(job); - cascadesContext.getJobScheduler().executeJobPool(cascadesContext); + do { + cascadesContext.getCurrentJobContext().setRewritten(false); + cascadesContext.pushJob(job); + cascadesContext.getJobScheduler().executeJobPool(cascadesContext); + } while (!job.isOnce() && cascadesContext.getCurrentJobContext().isRewritten()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/FindHashConditionForJoinJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/FindHashConditionForJoinJob.java deleted file mode 100644 index da2ccf1dc8..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/FindHashConditionForJoinJob.java +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.batch; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.rewrite.logical.FindHashConditionForJoin; - -import com.google.common.collect.ImmutableList; - -/** - * FindHashConditionForJoinJob - */ -public class FindHashConditionForJoinJob extends BatchRulesJob { - public FindHashConditionForJoinJob(CascadesContext cascadesContext) { - super(cascadesContext); - rulesJob.addAll(ImmutableList.of( - topDownBatch(ImmutableList.of(new FindHashConditionForJoin())) - )); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java index 38315f77a4..b77c599cc0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.Job; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization; +import org.apache.doris.nereids.rules.expression.rewrite.ExpressionOptimization; import org.apache.doris.nereids.rules.mv.SelectRollup; import org.apache.doris.nereids.rules.rewrite.AggregateDisassemble; import org.apache.doris.nereids.rules.rewrite.logical.ColumnPruning; @@ -57,10 +58,12 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob { .addAll(new AdjustApplyFromCorrelatToUnCorrelatJob(cascadesContext).rulesJob) .addAll(new ConvertApplyToJoinJob(cascadesContext).rulesJob) .add(topDownBatch(ImmutableList.of(new ExpressionNormalization()))) + .add(topDownBatch(ImmutableList.of(new ExpressionOptimization()))) .add(topDownBatch(ImmutableList.of(new NormalizeAggregate()))) + .add(topDownBatch(RuleSet.PUSH_DOWN_JOIN_CONDITION_RULES, false)) .add(topDownBatch(ImmutableList.of(new ReorderJoin()))) .add(topDownBatch(ImmutableList.of(new ColumnPruning()))) - .add(topDownBatch(RuleSet.PUSH_DOWN_JOIN_CONDITION_RULES)) + .add(topDownBatch(RuleSet.PUSH_DOWN_JOIN_CONDITION_RULES, false)) .add(topDownBatch(ImmutableList.of(new FindHashConditionForJoin()))) .add(topDownBatch(ImmutableList.of(new AggregateDisassemble()))) .add(topDownBatch(ImmutableList.of(new LimitPushDown()))) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java index 53359058b0..674a14f63e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteTopDownJob.java @@ -44,7 +44,11 @@ public class RewriteTopDownJob extends Job { public RewriteTopDownJob(Group group, JobContext context, List factories) { this(group, factories.stream() .flatMap(factory -> factory.buildRules().stream()) - .collect(Collectors.toList()), context); + .collect(Collectors.toList()), context, true); + } + + public RewriteTopDownJob(Group group, List rules, JobContext context) { + this(group, rules, context, true); } /** @@ -54,8 +58,8 @@ public class RewriteTopDownJob extends Job { * @param rules rewrite rules * @param context planner context */ - public RewriteTopDownJob(Group group, List rules, JobContext context) { - super(JobType.TOP_DOWN_REWRITE, context); + public RewriteTopDownJob(Group group, List rules, JobContext context, boolean once) { + super(JobType.TOP_DOWN_REWRITE, context, once); this.group = Objects.requireNonNull(group, "group cannot be null"); this.rules = Objects.requireNonNull(rules, "rules cannot be null"); } @@ -84,6 +88,7 @@ public class RewriteTopDownJob extends Job { if (result.generateNewExpression) { // new group-expr replaced the origin group-expr in `group`, // run this rule against this `group` again. + context.setRewritten(true); pushJob(new RewriteTopDownJob(group, rules, context)); return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MultiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MultiJoin.java index 1c343ba6fe..af20344205 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MultiJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MultiJoin.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -188,10 +189,18 @@ public class MultiJoin extends PlanVisitor { conjunctsForAllHashJoins.addAll(ExpressionUtils.extractConjunction(join.getOtherJoinCondition().get())); } - if (!(join.left() instanceof LogicalJoin)) { + Plan leftChild = join.left(); + if (join.left() instanceof LogicalFilter) { + leftChild = join.left().child(0); + } + if (leftChild instanceof GroupPlan) { joinInputs.add(join.left()); } - if (!(join.right() instanceof LogicalJoin)) { + Plan rightChild = join.right(); + if (join.right() instanceof LogicalFilter) { + rightChild = join.right().child(0); + } + if (rightChild instanceof GroupPlan) { joinInputs.add(join.right()); } return null;