database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids): push down alias into union outputs. (#20543)

Browse Source
This commit is contained in:
jakevin
2023-06-09 11:53:44 +08:00
committed by GitHub
parent 019e2353d3
commit 44e20d9087
5 changed files with 105 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
View File

@ -72,6 +72,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters;
import org.apache.doris.nereids.rules.rewrite.logical.MergeGenerates;
import org.apache.doris.nereids.rules.rewrite.logical.MergeLimits;
import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownAliasIntoUnionAll;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownAliasThroughJoin;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownExpressionsInHashCondition;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughAggregation;
@ -126,7 +127,6 @@ public class RuleSet {
new PushdownFilterThroughSetOperation(),
new PushdownFilterThroughWindow(),
new PushdownProjectThroughLimit(),
new PushdownAliasThroughJoin(),
new EliminateOuterJoin(),
new MergeProjects(),
new MergeFilters(),
@ -135,7 +135,9 @@ public class RuleSet {
new PushdownFilterThroughCTE(),
new PushdownProjectThroughCTE(),
new PushdownFilterThroughCTEAnchor(),
new PushdownProjectThroughCTEAnchor());
new PushdownProjectThroughCTEAnchor(),
new PushdownAliasThroughJoin(),
new PushdownAliasIntoUnionAll());
public static final List<Rule> IMPLEMENTATION_RULES = planRuleFactories()
.add(new LogicalCTEProduceToPhysicalCTEProduce())

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -138,6 +138,7 @@ public enum RuleType {
PUSHDOWN_FILTER_THROUGH_WINDOW(RuleTypeClass.REWRITE),
PUSHDOWN_PROJECT_THROUGH_LIMIT(RuleTypeClass.REWRITE),
PUSHDOWN_ALIAS_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSHDOWN_ALIAS_INTO_UNION_ALL(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE),
PUSHDOWN_FILTER_THROUGH_SORT(RuleTypeClass.REWRITE),

86
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasIntoUnionAll.java Normal file
View File

@ -0,0 +1,86 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
import org.apache.doris.nereids.util.PlanUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Pushdown Alias (inside must be Slot) into UnionAll outputs.
* <pre>
* Project(c1, c2 as c2t)
* |
* UnionAll output(c1, c2, c3)
* ->
* Project(c1, c2t)
* |
* UnionAll output(c1, c2 as c2t, c3)
* </pre>
*/
public class PushdownAliasIntoUnionAll extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalProject(logicalUnion())
.when(project -> project.child().getQualifier() == Qualifier.ALL)
.when(project -> project.getProjects().stream().allMatch(expr ->
(expr instanceof Slot) || (expr instanceof Alias && ((Alias) expr).child() instanceof Slot)))
.when(project -> project.getProjects().stream().anyMatch(expr -> expr instanceof Alias))
.then(project -> {
LogicalUnion union = project.child();
// aliasMap { Slot -> Alias }
Map<Slot, Alias> aliasMap = project.getProjects().stream()
.filter(namedExpression -> namedExpression instanceof Alias)
.map(namedExpression -> (Alias) namedExpression)
.collect(Collectors.toMap(
alias -> (Slot) (alias.child()),
alias -> alias));
Preconditions.checkState(!aliasMap.isEmpty(), "aliasMap should not be empty");
List<NamedExpression> newOutput = union.getOutputs().stream()
.map(ne -> {
Slot outSlot = ne.toSlot();
Alias alias = aliasMap.get(outSlot);
if (alias == null) {
return outSlot;
}
if (ne instanceof Alias) {
return alias.withChildren(ImmutableList.of(((Alias) ne).child()));
} else {
return alias;
}
})
.collect(Collectors.toList());
List<NamedExpression> newProjects = project.getProjects().stream().map(NamedExpression::toSlot)
.collect(Collectors.toList());
return PlanUtils.projectOrSelf(newProjects, union.withNewOutputs(newOutput));
}).toRule(RuleType.PUSHDOWN_ALIAS_INTO_UNION_ALL);
}
}

35
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownAliasThroughJoin.java
View File

@ -27,12 +27,13 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ -41,22 +42,12 @@ import java.util.stream.Collectors;
* Pushdown Alias (inside must be Slot) through Join.
*/
public class PushdownAliasThroughJoin extends OneRewriteRuleFactory {
private boolean isAllSlotOrAliasSlot(LogicalProject<? extends Plan> project) {
return project.getProjects().stream().allMatch(expr -> {
if (expr instanceof Slot) {
return true;
}
if (expr instanceof Alias) {
return ((Alias) expr).child() instanceof Slot;
}
return false;
});
}
@Override
public Rule build() {
return logicalProject(logicalJoin())
.when(this::isAllSlotOrAliasSlot)
.when(project -> project.getProjects().stream().allMatch(expr ->
(expr instanceof Slot) || (expr instanceof Alias && ((Alias) expr).child() instanceof Slot)))
.when(project -> project.getProjects().stream().anyMatch(expr -> expr instanceof Alias))
.then(project -> {
LogicalJoin<? extends Plan, ? extends Plan> join = project.child();
// aliasMap { Slot -> List<Alias<Slot>> }
@ -71,9 +62,7 @@ public class PushdownAliasThroughJoin extends OneRewriteRuleFactory {
}
aliases.add(expr);
});
if (aliasMap.isEmpty()) {
return null;
}
Preconditions.checkState(!aliasMap.isEmpty(), "aliasMap should not be empty");
List<NamedExpression> newProjects = project.getProjects().stream().map(NamedExpression::toSlot)
.collect(Collectors.toList());
@ -122,15 +111,9 @@ public class PushdownAliasThroughJoin extends OneRewriteRuleFactory {
private List<NamedExpression> createNewOutput(List<Slot> oldOutput,
Map<Expression, List<NamedExpression>> aliasMap) {
List<NamedExpression> output = Lists.newArrayList();
oldOutput.stream().forEach(slot -> {
List<NamedExpression> alias = aliasMap.get(slot);
if (alias != null) {
output.addAll(alias);
} else {
output.add(slot);
}
});
List<NamedExpression> output = oldOutput.stream()
.flatMap(slot -> aliasMap.getOrDefault(slot, Collections.singletonList(slot)).stream())
.collect(Collectors.toList());
return output;
}
}

11
regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query2.out
View File

@ -7,12 +7,11 @@ CteAnchor[cteId= ( CTEId#4=] )
--------hashAgg[LOCAL]
----------PhysicalProject
------------hashJoin[INNER_JOIN](date_dim.d_date_sk = wscs.sold_date_sk)
--------------PhysicalProject
----------------PhysicalUnion
------------------PhysicalProject
--------------------PhysicalOlapScan[web_sales]
------------------PhysicalProject
--------------------PhysicalOlapScan[catalog_sales]
--------------PhysicalUnion
----------------PhysicalProject
------------------PhysicalOlapScan[web_sales]
----------------PhysicalProject
------------------PhysicalOlapScan[catalog_sales]
--------------PhysicalDistribute
----------------PhysicalProject
------------------PhysicalOlapScan[date_dim]