database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[perf](Nereids) add back canEliminate temporary (#28017)

Browse Source
This commit is contained in:
morrySnow
2023-12-15 14:26:29 +08:00
committed by GitHub
parent 2018ab23f0
commit 97b033813a
90 changed files with 2979 additions and 3113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
View File

@ -120,7 +120,7 @@ public class Memo {
private Plan skipProject(Plan plan, Group targetGroup) {
// Some top project can't be eliminated
if (plan instanceof LogicalProject) {
if (plan instanceof LogicalProject && ((LogicalProject<?>) plan).canEliminate()) {
LogicalProject<?> logicalProject = (LogicalProject<?>) plan;
if (targetGroup != root) {
if (logicalProject.getOutputSet().equals(logicalProject.child().getOutputSet())) {

2
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java
View File

@ -129,7 +129,7 @@ public class RuntimeFilterContext {
private final Map<CTEId, Set<PhysicalHashJoin>> cteToJoinsMap = Maps.newHashMap();
// cte candidates which can be pushed into common runtime filter into from outside
private final Map<PhysicalCTEProducer, Map<EqualTo, PhysicalHashJoin>> cteRFPushDownMap = Maps.newHashMap();
private final Map<PhysicalCTEProducer, Map<EqualTo, PhysicalHashJoin>> cteRFPushDownMap = Maps.newLinkedHashMap();
private final Map<CTEId, PhysicalCTEProducer> cteProducerMap = Maps.newHashMap();

56
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java
View File

@ -23,6 +23,8 @@ import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.OutputSavePoint;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import java.util.ArrayList;
@ -37,34 +39,62 @@ public class EliminateUnnecessaryProject implements CustomRewriter {
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
return rewrite(plan);
return rewrite(plan, false);
}
private Plan rewrite(Plan plan) {
if (plan instanceof LogicalProject) {
return rewriteProject((LogicalProject<?>) plan);
private Plan rewrite(Plan plan, boolean outputSavePoint) {
if (plan instanceof LogicalSetOperation) {
return rewriteLogicalSetOperation((LogicalSetOperation) plan, outputSavePoint);
} else if (plan instanceof LogicalProject) {
return rewriteProject((LogicalProject) plan, outputSavePoint);
} else if (plan instanceof OutputSavePoint) {
return rewriteChildren(plan, true);
} else {
return rewriteChildren(plan);
return rewriteChildren(plan, outputSavePoint);
}
}
private Plan rewriteProject(LogicalProject<?> project) {
private Plan rewriteProject(LogicalProject<Plan> project, boolean outputSavePoint) {
if (project.child() instanceof LogicalEmptyRelation) {
// eliminate unnecessary project
return new LogicalEmptyRelation(StatementScopeIdGenerator.newRelationId(), project.getProjects());
} else if (project.getOutputSet().equals(project.child().getOutputSet())) {
} else if (project.canEliminate() && outputSavePoint
&& project.getOutputSet().equals(project.child().getOutputSet())) {
// eliminate unnecessary project
return rewrite(project.child());
return rewrite(project.child(), outputSavePoint);
} else if (project.canEliminate() && project.getOutput().equals(project.child().getOutput())) {
// eliminate unnecessary project
return rewrite(project.child(), outputSavePoint);
} else {
return rewriteChildren(project);
return rewriteChildren(project, true);
}
}
private Plan rewriteChildren(Plan plan) {
private Plan rewriteLogicalSetOperation(LogicalSetOperation set, boolean outputSavePoint) {
if (set.arity() == 2) {
Plan left = set.child(0);
Plan right = set.child(1);
boolean changed = false;
if (isCanEliminateProject(left)) {
changed = true;
left = ((LogicalProject) left).withEliminate(false);
}
if (isCanEliminateProject(right)) {
changed = true;
right = ((LogicalProject) right).withEliminate(false);
}
if (changed) {
set = (LogicalSetOperation) set.withChildren(left, right);
}
}
return rewriteChildren(set, outputSavePoint);
}
private Plan rewriteChildren(Plan plan, boolean outputSavePoint) {
List<Plan> newChildren = new ArrayList<>();
boolean hasNewChildren = false;
for (Plan child : plan.children()) {
Plan newChild = rewrite(child);
Plan newChild = rewrite(child, outputSavePoint);
if (newChild != child) {
hasNewChildren = true;
}
@ -72,4 +102,8 @@ public class EliminateUnnecessaryProject implements CustomRewriter {
}
return hasNewChildren ? plan.withChildren(newChildren) : plan;
}
private static boolean isCanEliminateProject(Plan plan) {
return plan instanceof LogicalProject && ((LogicalProject<?>) plan).canEliminate();
}
}

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java
View File

@ -51,7 +51,8 @@ public class MergeProjects extends OneRewriteRuleFactory {
public static Plan mergeProjects(LogicalProject<?> project) {
LogicalProject<? extends Plan> childProject = (LogicalProject<?>) project.child();
List<NamedExpression> projectExpressions = project.mergeProjections(childProject);
return project.withProjectsAndChild(projectExpressions, childProject.child(0));
LogicalProject<?> newProject = childProject.canEliminate() ? project : childProject;
return newProject.withProjectsAndChild(projectExpressions, childProject.child(0));
}
private boolean containsWindowExpression(List<NamedExpression> expressions) {

34
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
View File

@ -55,22 +55,29 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
private final List<NamedExpression> projects;
private final List<NamedExpression> excepts;
private final boolean isDistinct;
private final boolean canEliminate;
public LogicalProject(List<NamedExpression> projects, CHILD_TYPE child) {
this(projects, ImmutableList.of(), false, ImmutableList.of(child));
this(projects, ImmutableList.of(), false, true, ImmutableList.of(child));
}
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
boolean isDistinct, List<Plan> child) {
this(projects, excepts, isDistinct, Optional.empty(), Optional.empty(), child);
this(projects, excepts, isDistinct, true, Optional.empty(), Optional.empty(), child);
}
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
boolean isDistinct, Plan child) {
this(projects, excepts, isDistinct, Optional.empty(), Optional.empty(), ImmutableList.of(child));
this(projects, excepts, isDistinct, true, Optional.empty(), Optional.empty(), ImmutableList.of(child));
}
private LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
boolean isDistinct, boolean canEliminate, List<Plan> child) {
this(projects, excepts, isDistinct, canEliminate, Optional.empty(), Optional.empty(), child);
}
private LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, boolean isDistinct,
boolean canEliminate,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
List<Plan> child) {
super(PlanType.LOGICAL_PROJECT, groupExpression, logicalProperties, child);
@ -84,6 +91,7 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
: projects;
this.excepts = ImmutableList.copyOf(excepts);
this.isDistinct = isDistinct;
this.canEliminate = canEliminate;
}
/**
@ -151,18 +159,18 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
@Override
public int hashCode() {
return Objects.hash(projects);
return Objects.hash(projects, canEliminate);
}
@Override
public LogicalProject<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
return new LogicalProject<>(projects, excepts, isDistinct, ImmutableList.copyOf(children));
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, ImmutableList.copyOf(children));
}
@Override
public LogicalProject<Plan> withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalProject<>(projects, excepts, isDistinct,
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate,
groupExpression, Optional.of(getLogicalProperties()), children);
}
@ -170,22 +178,30 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
return new LogicalProject<>(projects, excepts, isDistinct,
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate,
groupExpression, logicalProperties, children);
}
public LogicalProject<Plan> withEliminate(boolean isEliminate) {
return new LogicalProject<>(projects, excepts, isDistinct, isEliminate, children);
}
public LogicalProject<Plan> withProjects(List<NamedExpression> projects) {
return new LogicalProject<>(projects, excepts, isDistinct, children);
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, children);
}
public LogicalProject<Plan> withProjectsAndChild(List<NamedExpression> projects, Plan child) {
return new LogicalProject<>(projects, excepts, isDistinct, ImmutableList.of(child));
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, ImmutableList.of(child));
}
public boolean isDistinct() {
return isDistinct;
}
public boolean canEliminate() {
return canEliminate;
}
@Override
public List<NamedExpression> getOutputs() {
return projects;

3
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.nereids.util.PlanConstructor;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
/**
@ -52,6 +53,7 @@ class EliminateUnnecessaryProjectTest extends TestWithFeService implements MemoP
+ ");");
}
@Disabled("enable this case when we remove canEliminate on LogicalProject again")
@Test
void testEliminateNonTopUnnecessaryProject() {
LogicalPlan unnecessaryProject = new LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0))
@ -75,6 +77,7 @@ class EliminateUnnecessaryProjectTest extends TestWithFeService implements MemoP
.matchesFromRoot(logicalOlapScan());
}
@Disabled("enable this case when we remove canEliminate on LogicalProject again")
@Test
void testEliminateTopProjectWhenOutputNotEquals() {
LogicalPlan necessaryProject = new LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0))