database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](nereids) Normalize expressions before performing plan rewriting (#11299)

Browse Source 
Rules for normalizing expressions should be applied once before do some extra expression transforms.

Normalization rules include:
1. NormalizeBinaryPredicatesRule
2. BetweenToCompoundRule
3. SimplifyNotExprRule
This commit is contained in:
Adonis Ling
2022-08-01 17:15:04 +08:00
committed by GitHub
parent eb778da64a
commit 69bfbae856
13 changed files with 60 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.nereids.jobs.batch.DisassembleRulesJob;
import org.apache.doris.nereids.jobs.batch.JoinReorderRulesJob;
import org.apache.doris.nereids.jobs.batch.NormalizeExpressionRulesJob;
import org.apache.doris.nereids.jobs.batch.OptimizeRulesJob;
import org.apache.doris.nereids.jobs.batch.PredicatePushDownRulesJob;
import org.apache.doris.nereids.jobs.cascades.DeriveStatsJob;
@ -120,6 +121,7 @@ public class NereidsPlanner extends Planner {
* Logical plan rewrite based on a series of heuristic rules.
*/
private void rewrite() {
new NormalizeExpressionRulesJob(plannerContext).execute();
new JoinReorderRulesJob(plannerContext).execute();
new PredicatePushDownRulesJob(plannerContext).execute();
new DisassembleRulesJob(plannerContext).execute();

42
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NormalizeExpressionRulesJob.java Normal file
View File

@ -0,0 +1,42 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.jobs.batch;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization;
import com.google.common.collect.ImmutableList;
/**
* Apply rules to normalize expressions.
*/
public class NormalizeExpressionRulesJob extends BatchRulesJob {
/**
* Constructor.
* @param plannerContext context for applying rules.
*/
public NormalizeExpressionRulesJob(PlannerContext plannerContext) {
super(plannerContext);
rulesJob.addAll(ImmutableList.of(
topDownBatch(ImmutableList.of(
new ExpressionNormalization()
))
));
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
View File

@ -247,7 +247,7 @@ public class Memo {
private Plan replaceChildrenToGroupPlan(Plan plan, List<Group> childrenGroups) {
List<Plan> groupPlanChildren = childrenGroups.stream()
.map(group -> new GroupPlan(group))
.map(GroupPlan::new)
.collect(ImmutableList.toImmutableList());
LogicalProperties logicalProperties = plan.getLogicalProperties();
return plan.withChildren(groupPlanChildren)

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/NormalizeExpressionOfPlan.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionNormalization.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.expression.rewrite;
import org.apache.doris.nereids.rules.expression.rewrite.rules.BetweenToCompoundRule;
import org.apache.doris.nereids.rules.expression.rewrite.rules.NormalizeBinaryPredicatesRule;
import org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyNotExprRule;
import com.google.common.collect.ImmutableList;
@ -27,15 +28,16 @@ import java.util.List;
/**
* normalize expression of plan rule set.
*/
public class NormalizeExpressionOfPlan extends ExpressionOfPlanRewrite {
public class ExpressionNormalization extends ExpressionRewrite {
public static final List<ExpressionRewriteRule> NORMALIZE_REWRITE_RULES = ImmutableList.of(
NormalizeBinaryPredicatesRule.INSTANCE,
BetweenToCompoundRule.INSTANCE
BetweenToCompoundRule.INSTANCE,
SimplifyNotExprRule.INSTANCE
);
private static final ExpressionRuleExecutor EXECUTOR = new ExpressionRuleExecutor(NORMALIZE_REWRITE_RULES);
public NormalizeExpressionOfPlan() {
public ExpressionNormalization() {
super(EXECUTOR);
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/OptimizeExpressionOfPlan.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionOptimization.java
View File

@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.expression.rewrite;
import org.apache.doris.nereids.rules.expression.rewrite.rules.DistinctPredicatesRule;
import org.apache.doris.nereids.rules.expression.rewrite.rules.ExtractCommonFactorRule;
import org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyNotExprRule;
import com.google.common.collect.ImmutableList;
@ -28,14 +27,13 @@ import java.util.List;
/**
* optimize expression of plan rule set.
*/
public class OptimizeExpressionOfPlan extends ExpressionOfPlanRewrite {
public class ExpressionOptimization extends ExpressionRewrite {
public static final List<ExpressionRewriteRule> OPTIMIZE_REWRITE_RULES = ImmutableList.of(
SimplifyNotExprRule.INSTANCE,
ExtractCommonFactorRule.INSTANCE,
DistinctPredicatesRule.INSTANCE);
private static final ExpressionRuleExecutor EXECUTOR = new ExpressionRuleExecutor(OPTIMIZE_REWRITE_RULES);
public OptimizeExpressionOfPlan() {
public ExpressionOptimization() {
super(EXECUTOR);
}
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionOfPlanRewrite.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java
View File

@ -38,10 +38,10 @@ import java.util.stream.Collectors;
/**
* expression of plan rewrite rule.
*/
public class ExpressionOfPlanRewrite implements RewriteRuleFactory {
public class ExpressionRewrite implements RewriteRuleFactory {
private final ExpressionRuleExecutor rewriter;
public ExpressionOfPlanRewrite(ExpressionRuleExecutor rewriter) {
public ExpressionRewrite(ExpressionRuleExecutor rewriter) {
this.rewriter = Objects.requireNonNull(rewriter, "rewriter is null");
}

15
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRuleExecutor.java
View File

@ -17,12 +17,8 @@
package org.apache.doris.nereids.rules.expression.rewrite;
import org.apache.doris.nereids.rules.expression.rewrite.rules.BetweenToCompoundRule;
import org.apache.doris.nereids.rules.expression.rewrite.rules.NormalizeBinaryPredicatesRule;
import org.apache.doris.nereids.rules.expression.rewrite.rules.SimplifyNotExprRule;
import org.apache.doris.nereids.trees.expressions.Expression;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.util.List;
@ -33,20 +29,9 @@ import java.util.stream.Collectors;
*/
public class ExpressionRuleExecutor {
public static final List<ExpressionRewriteRule> REWRITE_RULES = ImmutableList.of(
new BetweenToCompoundRule(),
new SimplifyNotExprRule(),
new NormalizeBinaryPredicatesRule()
);
private final ExpressionRewriteContext ctx;
private final List<ExpressionRewriteRule> rules;
public ExpressionRuleExecutor() {
this.rules = REWRITE_RULES;
this.ctx = new ExpressionRewriteContext();
}
public ExpressionRuleExecutor(List<ExpressionRewriteRule> rules) {
this.rules = rules;
this.ctx = new ExpressionRewriteContext();

6
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushPredicateThroughJoin.java
View File

@ -19,7 +19,6 @@ package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRuleExecutor;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
@ -122,15 +121,14 @@ public class PushPredicateThroughJoin extends OneRewriteRuleFactory {
Expression left = ExpressionUtils.and(leftPredicates);
Expression right = ExpressionUtils.and(rightPredicates);
//todo expr should optimize again using expr rewrite
ExpressionRuleExecutor exprRewriter = new ExpressionRuleExecutor();
Plan leftPlan = joinPlan.left();
Plan rightPlan = joinPlan.right();
if (!left.equals(BooleanLiteral.TRUE)) {
leftPlan = new LogicalFilter(exprRewriter.rewrite(left), leftPlan);
leftPlan = new LogicalFilter(left, leftPlan);
}
if (!right.equals(BooleanLiteral.TRUE)) {
rightPlan = new LogicalFilter(exprRewriter.rewrite(right), rightPlan);
rightPlan = new LogicalFilter(right, rightPlan);
}
return new LogicalJoin<>(joinPlan.getJoinType(), Optional.of(joinConditions), leftPlan, rightPlan);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TernaryNode.java
View File

@ -18,7 +18,7 @@
package org.apache.doris.nereids.trees;
/**
* interface for all tree node that have two children.
* interface for all tree node that have three children.
*/
public interface TernaryNode<
NODE_TYPE extends TreeNode<NODE_TYPE>,

11
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryExpression.java
View File

@ -23,15 +23,4 @@ import org.apache.doris.nereids.trees.BinaryNode;
* Interface for all expression that have two children.
*/
public interface BinaryExpression extends BinaryNode<Expression, Expression, Expression> {
@Override
default Expression left() {
return child(0);
}
@Override
default Expression right() {
return child(1);
}
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TernaryExpression.java
View File

@ -23,17 +23,4 @@ import org.apache.doris.nereids.trees.TernaryNode;
* Interface for all expression that have three children.
*/
public interface TernaryExpression extends TernaryNode<Expression, Expression, Expression, Expression> {
default Expression first() {
return child(0);
}
default Expression second() {
return child(1);
}
default Expression third() {
return child(2);
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/UnaryExpression.java
View File

@ -23,9 +23,4 @@ import org.apache.doris.nereids.trees.UnaryNode;
* Abstract class for all expression that have one child.
*/
public interface UnaryExpression extends UnaryNode<Expression, Expression> {
@Override
default Expression child() {
return child(0);
}
}

4
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.rules.expression.rewrite.ExpressionNormalization;
import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.Between;
@ -242,6 +243,7 @@ public class PushDownPredicateTest {
}
private Memo rewrite(Plan plan) {
return PlanRewriter.topDownRewriteMemo(plan, new ConnectContext(), new PushPredicateThroughJoin());
Plan normalizedPlan = PlanRewriter.topDownRewrite(plan, new ConnectContext(), new ExpressionNormalization());
return PlanRewriter.topDownRewriteMemo(normalizedPlan, new ConnectContext(), new PushPredicateThroughJoin());
}
}