From 6d999f5b9587d091dc2259086ee2697e6e985836 Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:12:24 +0800 Subject: [PATCH] [enhancement](nereids)add eliminate filter on one row relation rule (#24980) 1.simplify PushdownFilterThroughSetOperation rule 2.add eliminate filter on one row relation rule --- .../apache/doris/nereids/rules/RuleType.java | 1 + .../rules/rewrite/EliminateFilter.java | 61 ++++++++++++++++--- .../PushdownFilterThroughSetOperation.java | 27 +------- .../nereids_p0/test_filter_pushdown_set.out | 24 ++++++++ .../test_filter_pushdown_set.groovy | 24 +++++++- 5 files changed, 98 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 7b00b524d7..51a20847ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -198,6 +198,7 @@ public enum RuleType { ELIMINATE_LIMIT_ON_ONE_ROW_RELATION(RuleTypeClass.REWRITE), ELIMINATE_LIMIT_ON_EMPTY_RELATION(RuleTypeClass.REWRITE), ELIMINATE_FILTER(RuleTypeClass.REWRITE), + ELIMINATE_FILTER_ON_ONE_RELATION(RuleTypeClass.REWRITE), ELIMINATE_NOT_NULL(RuleTypeClass.REWRITE), ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE), ELIMINATE_OUTER_JOIN(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java index 5f5d8b39a8..2f46c43ecf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java @@ -19,27 +19,36 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.util.ExpressionUtils; -import com.google.common.collect.Sets; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; -import java.util.Set; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** * Eliminate filter which is FALSE or TRUE. */ -public class EliminateFilter extends OneRewriteRuleFactory { +public class EliminateFilter implements RewriteRuleFactory { @Override - public Rule build() { - return logicalFilter() - .when(filter -> filter.getConjuncts().stream().anyMatch(BooleanLiteral.class::isInstance)) + public List buildRules() { + return ImmutableList.of(logicalFilter().when( + filter -> filter.getConjuncts().stream().anyMatch(BooleanLiteral.class::isInstance)) .thenApply(ctx -> { LogicalFilter filter = ctx.root; - Set newConjuncts = Sets.newHashSetWithExpectedSize(filter.getConjuncts().size()); + ImmutableSet.Builder newConjuncts = ImmutableSet.builder(); for (Expression expression : filter.getConjuncts()) { if (expression == BooleanLiteral.FALSE) { return new LogicalEmptyRelation(ctx.statementContext.getNextRelationId(), @@ -48,12 +57,44 @@ public class EliminateFilter extends OneRewriteRuleFactory { newConjuncts.add(expression); } } - if (newConjuncts.isEmpty()) { + + ImmutableSet conjuncts = newConjuncts.build(); + if (conjuncts.isEmpty()) { return filter.child(); } else { - return new LogicalFilter<>(newConjuncts, filter.child()); + return new LogicalFilter<>(conjuncts, filter.child()); } }) - .toRule(RuleType.ELIMINATE_FILTER); + .toRule(RuleType.ELIMINATE_FILTER), + logicalFilter(logicalOneRowRelation()).thenApply(ctx -> { + LogicalFilter filter = ctx.root; + Map replaceMap = + filter.child().getOutputs().stream().filter(e -> e instanceof Alias) + .collect(Collectors.toMap(NamedExpression::toSlot, e -> ((Alias) e).child())); + + ImmutableSet.Builder newConjuncts = ImmutableSet.builder(); + ExpressionRewriteContext context = + new ExpressionRewriteContext(ctx.cascadesContext); + for (Expression expression : filter.getConjuncts()) { + Expression newExpr = ExpressionUtils.replace(expression, replaceMap); + Expression foldExpression = + FoldConstantRule.INSTANCE.rewrite(newExpr, context); + + if (foldExpression == BooleanLiteral.FALSE) { + return new LogicalEmptyRelation( + ctx.statementContext.getNextRelationId(), filter.getOutput()); + } else if (foldExpression != BooleanLiteral.TRUE) { + newConjuncts.add(expression); + } + } + + ImmutableSet conjuncts = newConjuncts.build(); + if (conjuncts.isEmpty()) { + return filter.child(); + } else { + return new LogicalFilter<>(conjuncts, filter.child()); + } + }) + .toRule(RuleType.ELIMINATE_FILTER_ON_ONE_RELATION)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java index 19f5243fc9..6b78ab1f36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushdownFilterThroughSetOperation.java @@ -22,11 +22,8 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; -import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; -import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableSet; @@ -44,25 +41,10 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory { @Override public Rule build() { - return logicalFilter(logicalSetOperation()).when(f -> f.child().getQualifier() == Qualifier.ALL).then(f -> { + return logicalFilter(logicalSetOperation()).then(f -> { LogicalSetOperation setOperation = f.child(); - - if (setOperation instanceof LogicalUnion && ((LogicalUnion) setOperation).hasPushedFilter()) { - return f; - } - List newChildren = new ArrayList<>(); - boolean allOneRowRelation = true; - boolean hasOneRowRelation = false; for (Plan child : setOperation.children()) { - if (child instanceof OneRowRelation) { - // We shouldn't push down the 'filter' to 'oneRowRelation'. - hasOneRowRelation = true; - newChildren.add(child); - continue; - } else { - allOneRowRelation = false; - } Map replaceMap = new HashMap<>(); for (int i = 0; i < setOperation.getOutputs().size(); ++i) { NamedExpression output = setOperation.getOutputs().get(i); @@ -73,14 +55,7 @@ public class PushdownFilterThroughSetOperation extends OneRewriteRuleFactory { ExpressionUtils.replace(conjunct, replaceMap)).collect(ImmutableSet.toImmutableSet()); newChildren.add(new LogicalFilter<>(newFilterPredicates, child)); } - if (allOneRowRelation) { - return f; - } - if (hasOneRowRelation) { - // If there are some `OneRowRelation` exists, we need to keep the `filter`. - return f.withChildren(((LogicalUnion) setOperation).withHasPushedFilter().withChildren(newChildren)); - } return setOperation.withChildren(newChildren); }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_SET_OPERATION); } diff --git a/regression-test/data/nereids_p0/test_filter_pushdown_set.out b/regression-test/data/nereids_p0/test_filter_pushdown_set.out index 42ecbc5ad8..6f6a6062fe 100644 --- a/regression-test/data/nereids_p0/test_filter_pushdown_set.out +++ b/regression-test/data/nereids_p0/test_filter_pushdown_set.out @@ -1,10 +1,34 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select1 -- 1 2 +1 3 +1 3 -- !select2 -- 1 2 +1 3 -- !select3 -- 1 2 +-- !select4 -- +1 3 + +-- !select5 -- +1 2 +1 2 + +-- !select6 -- +1 2 + +-- !select7 -- +1 2 + +-- !select8 -- + +-- !select22 -- +1 2 + +-- !select23 -- +1 2 + diff --git a/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy b/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy index 03c1b25798..7d25aade29 100644 --- a/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy +++ b/regression-test/suites/nereids_p0/test_filter_pushdown_set.groovy @@ -18,8 +18,26 @@ suite("test_filter_pushdown_set") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" - qt_select1 'select * from (select 1 as a, 2 as b union all select 3, 3) t where a = 1;' + qt_select1 'select * from (select 1 as a, 2 as b union all select 1, 3 union all select 1, 3) t where a = 1 order by a, b;' + qt_select2 'select * from (select 1 as a, 2 as b union select 1, 3 union select 1, 3) t where a = 1 order by a, b;' + qt_select3 'select * from ((select 1 as a, 2 as b union all select 1, 3) intersect select 1, 2) t where a = 1 order by a, b;' + qt_select4 'select * from ((select 1 as a, 2 as b union all select 1, 3) except select 1, 2 ) t where a = 1 order by a, b;' + qt_select5 'select * from (select 1 as a, 2 as b union all select 1, 3 union all select 1, 2) t where b = 2 order by a, b;' + qt_select6 'select * from (select 1 as a, 2 as b union select 1, 3 union select 1, 2) t where b = 2 order by a, b;' + qt_select7 'select * from ((select 1 as a, 2 as b union all select 1, 3) intersect select 1, 2) t where b = 2 order by a, b;' + qt_select8 'select * from ((select 1 as a, 2 as b union all select 1, 3) except select 1, 2 ) t where b = 2 order by a, b;' + + explain { + sql("select * from ((select 1 as a, 2 as b union all select 1, 3) intersect select 1, 2) t where a = 1;") + notContains "VSELECT" + } + + explain { + sql("select * from ((select 1 as a, 2 as b union all select 1, 3) except select 1, 2 ) t where b = 2;") + notContains "1 | 3" + } + sql "SET enable_nereids_planner=false" - qt_select2 'select * from (select 1 as a, 2 as b union all select 3, 3) t where a = 1;' - qt_select3 'select * from (select 1 as a, 2 as b union select 3, 3) t where a = 1;' + qt_select22 'select * from (select 1 as a, 2 as b union all select 3, 3) t where a = 1 order by a, b;' + qt_select23 'select * from (select 1 as a, 2 as b union select 3, 3) t where a = 1 order by a, b;' } \ No newline at end of file