database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids) push down Project through Limit (#12490)

Browse Source 
This rule is rewrite project -> limit to limit -> project. The reason is we could get tree like project -> limit -> project -> other node. If we do not rewrite it. we could not merge the two project into one. And if we has more than one project on one node, the second one will overwrite the first one when translate. Then, be will core dump or return slot cannot find error.
This commit is contained in:
Kikyou1997
2022-09-13 13:26:12 +08:00
committed by GitHub
parent c3d7d4ce7a
commit d35a8a24a5
5 changed files with 115 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
View File

@ -31,8 +31,9 @@ import org.apache.doris.nereids.rules.rewrite.logical.MergeConsecutiveProjects;
import org.apache.doris.nereids.rules.rewrite.logical.NormalizeAggregate;
import org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanPartition;
import org.apache.doris.nereids.rules.rewrite.logical.PushPredicateThroughJoin;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughLimit;
import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin;
import org.apache.doris.nereids.rules.rewrite.logical.SwapFilterAndProject;
import com.google.common.collect.ImmutableList;
@ -54,6 +55,7 @@ public class RewriteJob extends BatchRulesJob {
* 1. Adjust the plan in correlated logicalApply
* so that there are no correlated columns in the subquery.
* 2. Convert logicalApply to a logicalJoin.
* TODO: group these rules to make sure the result plan is what we expected.
*/
.addAll(new AdjustApplyFromCorrelatToUnCorrelatJob(cascadesContext).rulesJob)
.addAll(new ConvertApplyToJoinJob(cascadesContext).rulesJob)
@ -65,7 +67,8 @@ public class RewriteJob extends BatchRulesJob {
.add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
.add(topDownBatch(ImmutableList.of(new ColumnPruning())))
.add(topDownBatch(ImmutableList.of(new AggregateDisassemble())))
.add(topDownBatch(ImmutableList.of(new SwapFilterAndProject())))
.add(topDownBatch(ImmutableList.of(new PushdownProjectThroughLimit())))
.add(topDownBatch(ImmutableList.of(new PushdownFilterThroughProject())))
.add(bottomUpBatch(ImmutableList.of(new MergeConsecutiveProjects())))
.add(topDownBatch(ImmutableList.of(new MergeConsecutiveFilters())))
.add(bottomUpBatch(ImmutableList.of(new MergeConsecutiveLimits())))

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -105,6 +105,7 @@ public enum RuleType {
OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
SWAP_FILTER_AND_PROJECT(RuleTypeClass.REWRITE),
LOGICAL_LIMIT_TO_LOGICAL_EMPTY_RELATION_RULE(RuleTypeClass.REWRITE),
SWAP_LIMIT_PROJECT(RuleTypeClass.REWRITE),
// exploration rules
TEST_EXPLORATION(RuleTypeClass.EXPLORATION),

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SwapFilterAndProject.java → fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java
View File

@ -32,7 +32,7 @@ import org.apache.doris.nereids.util.ExpressionUtils;
* output:
* project(c+d as a, e as b) -> filter(c+d>2, e=0).
*/
public class SwapFilterAndProject extends OneRewriteRuleFactory {
public class PushdownFilterThroughProject extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(logicalProject()).then(filter -> {

59
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimit.java Normal file
View File

@ -0,0 +1,59 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
/**
* Before:
* project
* │
* ▼
* limit
* │
* ▼
* plan node
*
* After:
*
* limit
* │
* ▼
* project
* │
* ▼
* plan node
*/
public class PushdownProjectThroughLimit extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalProject(logicalLimit(group())).thenApply(ctx -> {
LogicalProject<LogicalLimit<GroupPlan>> logicalProject = ctx.root;
LogicalLimit<GroupPlan> logicalLimit = logicalProject.child();
return new LogicalLimit<LogicalProject<GroupPlan>>(logicalLimit.getLimit(),
logicalLimit.getOffset(), new LogicalProject<>(logicalProject.getProjects(),
logicalLimit.child()));
}).toRule(RuleType.SWAP_LIMIT_PROJECT);
}
}

49
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownProjectThroughLimitTest.java Normal file
View File

@ -0,0 +1,49 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.types.IntegerType;
import com.google.common.collect.ImmutableList;
import mockit.Mocked;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class PushdownProjectThroughLimitTest {
@Mocked
private CascadesContext cascadesContext;
@Test
public void testPushdownProjectThroughLimit(@Mocked GroupPlan groupPlan) {
SlotReference slotRef = new SlotReference("col1", IntegerType.INSTANCE);
LogicalLimit logicalLimit = new LogicalLimit(1, 1, groupPlan);
LogicalProject logicalProject = new LogicalProject(ImmutableList.of(slotRef), logicalLimit);
PushdownProjectThroughLimit pushdownProjectThroughLimit = new PushdownProjectThroughLimit();
LogicalPlan rewrittenPlan =
(LogicalPlan) pushdownProjectThroughLimit.build().transform(logicalProject, cascadesContext).get(0);
Assertions.assertTrue(rewrittenPlan instanceof LogicalLimit);
Assertions.assertTrue(rewrittenPlan.child(0) instanceof LogicalProject);
}
}