database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](Nereids) remove canEliminate flag on LogicalProject (#24362)

Browse Source 
since we have three infrastructure to ensure changing input column order
not lead to wrong result, we could remove this flag on LogicalProject to
eliminate project as mush as possible and let code clear.

1. output list in ResultSink node
2. regular children output in SetOperation node
3. producer to consumer slot id map in CteConsumer
This commit is contained in:
morrySnow
2023-09-18 12:22:33 +08:00
committed by GitHub
parent ae0b58fcde
commit c56d3237e8
31 changed files with 578 additions and 650 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
View File

@ -138,8 +138,8 @@ public class Memo {
private Plan skipProject(Plan plan, Group targetGroup) {
// Some top project can't be eliminated
if (plan instanceof LogicalProject && ((LogicalProject<?>) plan).canEliminate()) {
LogicalProject<Plan> logicalProject = (LogicalProject<Plan>) plan;
if (plan instanceof LogicalProject) {
LogicalProject<?> logicalProject = (LogicalProject<?>) plan;
if (targetGroup != root) {
if (logicalProject.getOutputSet().equals(logicalProject.child().getOutputSet())) {
return skipProject(logicalProject.child(), targetGroup);
@ -155,7 +155,7 @@ public class Memo {
private Plan skipProjectGetChild(Plan plan) {
if (plan instanceof LogicalProject) {
LogicalProject<Plan> logicalProject = (LogicalProject<Plan>) plan;
LogicalProject<?> logicalProject = (LogicalProject<?>) plan;
Plan child = logicalProject.child();
if (logicalProject.getOutputSet().equals(child.getOutputSet())) {
return skipProjectGetChild(child);
@ -915,7 +915,7 @@ public class Memo {
int prefix = 0;
for (GroupExpression groupExpression : extractGroupExpressionContainsProp(group, prop)) {
List<Pair<Long, Double>> possiblePlans = rankGroupExpression(groupExpression, prop);
if (possiblePlans.size() != 0 && rank - prefix <= possiblePlans.get(possiblePlans.size() - 1).first) {
if (!possiblePlans.isEmpty() && rank - prefix <= possiblePlans.get(possiblePlans.size() - 1).first) {
return unrankGroupExpression(groupExpression, prop, rank - prefix);
}
prefix += possiblePlans.size();
@ -944,10 +944,9 @@ public class Memo {
childrenPlan.add(unrankGroup(groupExpression.child(i), properties.get(i), childrenRanks.get(i)));
}
Plan plan = groupExpression.getPlan().withChildren(childrenPlan);
PhysicalPlan physicalPlan = ((PhysicalPlan) plan).withPhysicalPropertiesAndStats(
return ((PhysicalPlan) plan).withPhysicalPropertiesAndStats(
groupExpression.getOutputProperties(prop),
groupExpression.getOwnerGroup().getStatistics());
return physicalPlan;
}
/**
@ -957,7 +956,7 @@ public class Memo {
* 2: [2%1, 2%(1*2)]
*/
private List<Long> extractChildRanks(long rank, List<List<Pair<Long, Double>>> children) {
Preconditions.checkArgument(children.size() > 0);
Preconditions.checkArgument(!children.isEmpty(), "children should not empty in extractChildRanks");
int factor = children.get(0).size();
List<Long> indices = new ArrayList<>();
for (int i = 0; i < children.size() - 1; i++) {

56
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProject.java
View File

@ -23,8 +23,6 @@ import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation;
import org.apache.doris.nereids.trees.plans.logical.OutputSavePoint;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import java.util.ArrayList;
@ -39,62 +37,34 @@ public class EliminateUnnecessaryProject implements CustomRewriter {
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
return rewrite(plan, false);
return rewrite(plan);
}
private Plan rewrite(Plan plan, boolean outputSavePoint) {
if (plan instanceof LogicalSetOperation) {
return rewriteLogicalSetOperation((LogicalSetOperation) plan, outputSavePoint);
} else if (plan instanceof LogicalProject) {
return rewriteProject((LogicalProject) plan, outputSavePoint);
} else if (plan instanceof OutputSavePoint) {
return rewriteChildren(plan, true);
private Plan rewrite(Plan plan) {
if (plan instanceof LogicalProject) {
return rewriteProject((LogicalProject<?>) plan);
} else {
return rewriteChildren(plan, outputSavePoint);
return rewriteChildren(plan);
}
}
private Plan rewriteProject(LogicalProject<Plan> project, boolean outputSavePoint) {
private Plan rewriteProject(LogicalProject<?> project) {
if (project.child() instanceof LogicalEmptyRelation) {
// eliminate unnecessary project
return new LogicalEmptyRelation(StatementScopeIdGenerator.newRelationId(), project.getProjects());
} else if (project.canEliminate() && outputSavePoint
&& project.getOutputSet().equals(project.child().getOutputSet())) {
} else if (project.getOutputSet().equals(project.child().getOutputSet())) {
// eliminate unnecessary project
return rewrite(project.child(), outputSavePoint);
} else if (project.canEliminate() && project.getOutput().equals(project.child().getOutput())) {
// eliminate unnecessary project
return rewrite(project.child(), outputSavePoint);
return rewrite(project.child());
} else {
return rewriteChildren(project, true);
return rewriteChildren(project);
}
}
private Plan rewriteLogicalSetOperation(LogicalSetOperation set, boolean outputSavePoint) {
if (set.arity() == 2) {
Plan left = set.child(0);
Plan right = set.child(1);
boolean changed = false;
if (isCanEliminateProject(left)) {
changed = true;
left = ((LogicalProject) left).withEliminate(false);
}
if (isCanEliminateProject(right)) {
changed = true;
right = ((LogicalProject) right).withEliminate(false);
}
if (changed) {
set = (LogicalSetOperation) set.withChildren(left, right);
}
}
return rewriteChildren(set, outputSavePoint);
}
private Plan rewriteChildren(Plan plan, boolean outputSavePoint) {
private Plan rewriteChildren(Plan plan) {
List<Plan> newChildren = new ArrayList<>();
boolean hasNewChildren = false;
for (Plan child : plan.children()) {
Plan newChild = rewrite(child, outputSavePoint);
Plan newChild = rewrite(child);
if (newChild != child) {
hasNewChildren = true;
}
@ -102,8 +72,4 @@ public class EliminateUnnecessaryProject implements CustomRewriter {
}
return hasNewChildren ? plan.withChildren(newChildren) : plan;
}
private static boolean isCanEliminateProject(Plan plan) {
return plan instanceof LogicalProject && ((LogicalProject<?>) plan).canEliminate();
}
}

9
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeProjects.java
View File

@ -45,14 +45,13 @@ public class MergeProjects extends OneRewriteRuleFactory {
return logicalProject(logicalProject())
.whenNot(project -> containsWindowExpression(project.getProjects())
&& containsWindowExpression(project.child().getProjects()))
.then(project -> mergeProjects(project)).toRule(RuleType.MERGE_PROJECTS);
.then(MergeProjects::mergeProjects).toRule(RuleType.MERGE_PROJECTS);
}
public static Plan mergeProjects(LogicalProject project) {
LogicalProject<? extends Plan> childProject = (LogicalProject) project.child();
public static Plan mergeProjects(LogicalProject<?> project) {
LogicalProject<? extends Plan> childProject = (LogicalProject<?>) project.child();
List<NamedExpression> projectExpressions = project.mergeProjections(childProject);
LogicalProject newProject = childProject.canEliminate() ? project : childProject;
return newProject.withProjectsAndChild(projectExpressions, childProject.child(0));
return project.withProjectsAndChild(projectExpressions, childProject.child(0));
}
private boolean containsWindowExpression(List<NamedExpression> expressions) {

45
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
View File

@ -49,37 +49,30 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
private final List<NamedExpression> projects;
private final List<NamedExpression> excepts;
private final boolean isDistinct;
// For project nodes under union, erasure cannot be configured, so add this flag.
private final boolean canEliminate;
public LogicalProject(List<NamedExpression> projects, CHILD_TYPE child) {
this(projects, ImmutableList.of(), false, true, child);
this(projects, ImmutableList.of(), false, child);
}
/**
* only for test.
*/
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, CHILD_TYPE child) {
this(projects, excepts, false, true, child);
this(projects, excepts, false, child);
}
public LogicalProject(List<NamedExpression> projects, boolean isDistinct, CHILD_TYPE child) {
this(projects, ImmutableList.of(), isDistinct, true, child);
this(projects, ImmutableList.of(), isDistinct, child);
}
public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
boolean isDistinct, CHILD_TYPE child) {
this(projects, excepts, isDistinct, true, child);
}
private LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
boolean isDistinct, boolean canEliminate, CHILD_TYPE child) {
this(projects, excepts, isDistinct, canEliminate, Optional.empty(), Optional.empty(), child);
this(projects, excepts, isDistinct, Optional.empty(), Optional.empty(), child);
}
private LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, boolean isDistinct,
boolean canEliminate, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
CHILD_TYPE child) {
super(PlanType.LOGICAL_PROJECT, groupExpression, logicalProperties, child);
Preconditions.checkArgument(projects != null, "projects can not be null");
// only ColumnPrune rule may produce empty projects, this happens in rewrite phase
@ -91,7 +84,6 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
: projects;
this.excepts = ImmutableList.copyOf(excepts);
this.isDistinct = isDistinct;
this.canEliminate = canEliminate;
}
/**
@ -124,8 +116,7 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
return Utils.toSqlString("LogicalProject[" + id.asInt() + "]",
"distinct", isDistinct,
"projects", projects,
"excepts", excepts,
"canEliminate", canEliminate
"excepts", excepts
);
}
@ -150,7 +141,6 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
LogicalProject<?> that = (LogicalProject<?>) o;
boolean equal = projects.equals(that.projects)
&& excepts.equals(that.excepts)
&& canEliminate == that.canEliminate
&& isDistinct == that.isDistinct;
// TODO: should add exprId for UnBoundStar and BoundStar for equality comparison
if (!projects.isEmpty() && (projects.get(0) instanceof UnboundStar || projects.get(0) instanceof BoundStar)) {
@ -161,18 +151,18 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
@Override
public int hashCode() {
return Objects.hash(projects, canEliminate);
return Objects.hash(projects);
}
@Override
public LogicalProject<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, children.get(0));
return new LogicalProject<>(projects, excepts, isDistinct, children.get(0));
}
@Override
public LogicalProject<Plan> withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate,
return new LogicalProject<>(projects, excepts, isDistinct,
groupExpression, Optional.of(getLogicalProperties()), child());
}
@ -180,30 +170,22 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate,
return new LogicalProject<>(projects, excepts, isDistinct,
groupExpression, logicalProperties, children.get(0));
}
public LogicalProject<Plan> withEliminate(boolean isEliminate) {
return new LogicalProject<>(projects, excepts, isDistinct, isEliminate, child());
}
public LogicalProject<Plan> withProjects(List<NamedExpression> projects) {
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, child());
return new LogicalProject<>(projects, excepts, isDistinct, child());
}
public LogicalProject<Plan> withProjectsAndChild(List<NamedExpression> projects, Plan child) {
return new LogicalProject<>(projects, excepts, isDistinct, canEliminate, child);
return new LogicalProject<>(projects, excepts, isDistinct, child);
}
public boolean isDistinct() {
return isDistinct;
}
public boolean canEliminate() {
return canEliminate;
}
@Override
public List<NamedExpression> getOutputs() {
return projects;
@ -220,7 +202,6 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
JSONObject properties = new JSONObject();
properties.put("Projects", projects.toString());
properties.put("Excepts", excepts.toString());
properties.put("CanEliminate", canEliminate);
properties.put("IsDistinct", isDistinct);
logicalProject.put("Properties", properties);
return logicalProject;

6
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateUnnecessaryProjectTest.java
View File

@ -61,7 +61,7 @@ class EliminateUnnecessaryProjectTest extends TestWithFeService implements MemoP
PlanChecker.from(MemoTestUtils.createConnectContext(), unnecessaryProject)
.customRewrite(new EliminateUnnecessaryProject())
.matchesFromRoot(logicalFilter(logicalProject()));
.matchesFromRoot(logicalFilter(logicalOlapScan()));
}
@Test
@ -76,14 +76,14 @@ class EliminateUnnecessaryProjectTest extends TestWithFeService implements MemoP
}
@Test
void testNotEliminateTopProjectWhenOutputNotEquals() {
void testEliminateTopProjectWhenOutputNotEquals() {
LogicalPlan necessaryProject = new LogicalPlanBuilder(PlanConstructor.newLogicalOlapScan(0, "t1", 0))
.project(ImmutableList.of(1, 0))
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), necessaryProject)
.customRewrite(new EliminateUnnecessaryProject())
.matchesFromRoot(logicalProject());
.notMatchesFromRoot(logicalProject());
}
@Test

2
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
View File

@ -91,7 +91,7 @@ public class PlanToStringTest {
LogicalProject<Plan> plan = new LogicalProject<>(ImmutableList.of(
new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList())), child);
Assertions.assertTrue(plan.toString().matches("LogicalProject\\[\\d+\\] \\( distinct=false, projects=\\[a#\\d+], excepts=\\[], canEliminate=true \\)"));
Assertions.assertTrue(plan.toString().matches("LogicalProject\\[\\d+\\] \\( distinct=false, projects=\\[a#\\d+], excepts=\\[] \\)"));
}
@Test

7
fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
View File

@ -457,6 +457,13 @@ public class PlanChecker {
return this;
}
public PlanChecker notMatchesFromRoot(PatternDescriptor<? extends Plan> patternDesc) {
Memo memo = cascadesContext.getMemo();
assertMatches(memo, () -> !(new GroupExpressionMatching(patternDesc.pattern,
memo.getRoot().getLogicalExpression()).iterator().hasNext()));
return this;
}
public PlanChecker matches(PatternDescriptor<? extends Plan> patternDesc) {
Memo memo = cascadesContext.getMemo();
checkSlotFromChildren(memo);