database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](Nereids): Except just can merge with left deep shape (#30270)

Browse Source
This commit is contained in:
jakevin
2024-01-24 18:04:16 +08:00
committed by yiguolei
parent 1aa006c80f
commit 83ea486b15
6 changed files with 357 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View File

@ -82,6 +82,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeFilters;
import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion;
import org.apache.doris.nereids.rules.rewrite.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperations;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperationsExcept;
import org.apache.doris.nereids.rules.rewrite.MergeTopNs;
import org.apache.doris.nereids.rules.rewrite.NormalizeSort;
import org.apache.doris.nereids.rules.rewrite.OrExpansion;
@ -274,7 +275,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
topic("Set operation optimization",
// Do MergeSetOperation first because we hope to match pattern of Distinct SetOperator.
topDown(new PushProjectThroughUnion(), new MergeProjects()),
bottomUp(new MergeSetOperations()),
bottomUp(new MergeSetOperations(), new MergeSetOperationsExcept()),
bottomUp(new PushProjectIntoOneRowRelation()),
topDown(new MergeOneRowRelationIntoUnion()),
topDown(new PushProjectIntoUnion()),

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -245,6 +245,7 @@ public enum RuleType {
PUSH_PROJECT_INTO_ONE_ROW_RELATION(RuleTypeClass.REWRITE),
PUSH_PROJECT_INTO_UNION(RuleTypeClass.REWRITE),
MERGE_SET_OPERATION(RuleTypeClass.REWRITE),
MERGE_SET_OPERATION_EXCEPT(RuleTypeClass.REWRITE),
MERGE_TOP_N(RuleTypeClass.REWRITE),
BUILD_AGG_FOR_UNION(RuleTypeClass.REWRITE),
COUNT_DISTINCT_REWRITE(RuleTypeClass.REWRITE),

22
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperations.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import com.google.common.collect.ImmutableList;
@ -43,12 +44,22 @@ import java.util.List;
* / | \
* scan1 scan2 scan3
* </pre>
* Notice: this rule ignore Except.
* Relational Algebra: Union (R U S), Intersect Syntax: (R ∩ S), Except Syntax: (R - S)
* TODO: Except need other Rewrite.
* <ul> (R - S) U T = (R U T) - S </ul>
* <ul> (R - S) U (T - U) = (R U T) - (S U U) </ul>
* <ul> R - (S U T) = (R - S) - T </ul>
* <ul> R - (S - T) = (R - S) U T </ul>
* <ul> ...... and so on </ul>
*/
public class MergeSetOperations implements RewriteRuleFactory {
public class MergeSetOperations extends OneRewriteRuleFactory {
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalSetOperation(any(), any()).when(MergeSetOperations::canMerge).then(parentSetOperation -> {
public Rule build() {
return logicalSetOperation(any(), any())
.whenNot(LogicalExcept.class::isInstance)
.when(MergeSetOperations::canMerge)
.then(parentSetOperation -> {
ImmutableList.Builder<Plan> newChildren = ImmutableList.builder();
ImmutableList.Builder<List<SlotReference>> newChildrenOutputs = ImmutableList.builder();
for (int i = 0; i < parentSetOperation.arity(); i++) {
@ -64,8 +75,7 @@ public class MergeSetOperations implements RewriteRuleFactory {
}
return parentSetOperation.withChildrenAndTheirOutputs(
newChildren.build(), newChildrenOutputs.build());
}).toRule(RuleType.MERGE_SET_OPERATION)
);
}).toRule(RuleType.MERGE_SET_OPERATION);
}
/** canMerge */

64
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeSetOperationsExcept.java Normal file
View File

@ -0,0 +1,64 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
import com.google.common.collect.ImmutableList;
import java.util.List;
/**
* Merge left deep except
* <pre>
* Except
* / \
* Except scan3
* / \
* scan1 scan2
* -->
* Except
* / | \
* scan1 scan2 scan3
* </pre>
*/
public class MergeSetOperationsExcept extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalExcept(logicalExcept(), any())
.when(e -> e.getQualifier() == ((LogicalExcept) e.child(0)).getQualifier())
.then(parentExcept -> {
LogicalExcept childExcept = (LogicalExcept) parentExcept.child(0);
ImmutableList.Builder<Plan> newChildren = ImmutableList.builder();
ImmutableList.Builder<List<SlotReference>> newChildrenOutputs = ImmutableList.builder();
newChildren.addAll(childExcept.children());
newChildren.addAll(parentExcept.children().subList(1, parentExcept.children().size()));
newChildrenOutputs.addAll(childExcept.getRegularChildrenOutputs());
newChildrenOutputs.addAll(
parentExcept.getRegularChildrenOutputs().subList(1, parentExcept.children().size()));
return parentExcept.withChildrenAndTheirOutputs(newChildren.build(), newChildrenOutputs.build());
}).toRule(RuleType.MERGE_SET_OPERATION_EXCEPT);
}
}