database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids): pull up Project under Limit/TopN (#25866)

Browse Source 
If project contains expression-eval, we need pull up through limit, it can improve performance.
This commit is contained in:
jakevin
2023-10-25 16:44:32 +08:00
committed by GitHub
parent 693982fd1a
commit 8a03a07339
26 changed files with 726 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View File

@ -85,6 +85,8 @@ import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition;
import org.apache.doris.nereids.rules.rewrite.PruneOlapScanTablet;
import org.apache.doris.nereids.rules.rewrite.PullUpCteAnchor;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderApply;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin;
@ -288,6 +290,10 @@ public class Rewriter extends AbstractBatchJobExecutor {
new PushdownLimitDistinctThroughJoin(),
new PushdownTopNThroughWindow(),
new CreatePartitionTopNFromWindow()
),
topDown(
new PullUpProjectUnderTopN(),
new PullUpProjectUnderLimit()
)
),
// TODO: these rules should be implementation rules, and generate alternative physical plans.
@ -341,7 +347,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
),
topic("eliminate empty relation",
bottomUp(new EliminateEmptyRelation())
bottomUp(new EliminateEmptyRelation())
)
);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -133,6 +133,8 @@ public enum RuleType {
ELIMINATE_SORT_UNDER_APPLY(RuleTypeClass.REWRITE),
ELIMINATE_SORT_UNDER_APPLY_PROJECT(RuleTypeClass.REWRITE),
PULL_UP_PROJECT_UNDER_APPLY(RuleTypeClass.REWRITE),
PULL_UP_PROJECT_UNDER_LIMIT(RuleTypeClass.REWRITE),
PULL_UP_PROJECT_UNDER_TOPN(RuleTypeClass.REWRITE),
AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION(RuleTypeClass.REWRITE),
UN_CORRELATED_APPLY_FILTER(RuleTypeClass.REWRITE),
UN_CORRELATED_APPLY_PROJECT_FILTER(RuleTypeClass.REWRITE),

55
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderLimit.java Normal file
View File

@ -0,0 +1,55 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.PlanUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Pull up Project under Limit.
*/
public class PullUpProjectUnderLimit extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalLimit(logicalProject().whenNot(p -> p.isAllSlots()))
.then(limit -> {
LogicalProject<Plan> project = limit.child();
Set<Slot> allUsedSlots = project.getProjects().stream().flatMap(ne -> ne.getInputSlots().stream())
.collect(Collectors.toSet());
Set<Slot> outputSet = project.child().getOutputSet();
if (outputSet.size() == allUsedSlots.size()) {
Preconditions.checkState(outputSet.equals(allUsedSlots));
return project.withChildren(limit.withChildren(project.child()));
} else {
Plan columnProject = PlanUtils.projectOrSelf(ImmutableList.copyOf(allUsedSlots),
project.child());
return project.withChildren(limit.withChildren(columnProject));
}
}).toRule(RuleType.PULL_UP_PROJECT_UNDER_LIMIT);
}
}

60
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderTopN.java Normal file
View File

@ -0,0 +1,60 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.PlanUtils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Pull up Project under TopN.
*/
public class PullUpProjectUnderTopN extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalTopN(logicalProject().whenNot(p -> p.isAllSlots()))
.then(topN -> {
LogicalProject<Plan> project = topN.child();
Set<Slot> outputSet = project.child().getOutputSet();
if (!topN.getOrderKeys().stream().map(OrderKey::getExpr).flatMap(e -> e.getInputSlots().stream())
.allMatch(outputSet::contains)) {
return null;
}
Set<Slot> allUsedSlots = project.getProjects().stream().flatMap(ne -> ne.getInputSlots().stream())
.collect(Collectors.toSet());
if (outputSet.size() == allUsedSlots.size()) {
Preconditions.checkState(outputSet.equals(allUsedSlots));
return project.withChildren(topN.withChildren(project.child()));
} else {
Plan columnProject = PlanUtils.projectOrSelf(ImmutableList.copyOf(allUsedSlots),
project.child());
return project.withChildren(topN.withChildren(columnProject));
}
}).toRule(RuleType.PULL_UP_PROJECT_UNDER_TOPN);
}
}