database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[memo](nereids) record the chosen group expression in Group (#22661)

Browse Source 
1. remember the chosen plan in group
2. set groupId after RecomputeLogicalPropertiesProcessor
This commit is contained in:
minghong
2023-08-09 18:44:46 +08:00
committed by GitHub
parent 2019bb3870
commit e6a860fc9e
23 changed files with 70 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
View File

@ -70,6 +70,10 @@ public class Group {
private Statistics statistics;
private PhysicalProperties chosenProperties;
private int chosenGroupExpressionId = -1;
/**
* Constructor for Group.
*
@ -198,10 +202,17 @@ public class Group {
* @return {@link Optional} of cost and {@link GroupExpression} of physical plan pair.
*/
public Optional<Pair<Cost, GroupExpression>> getLowestCostPlan(PhysicalProperties physicalProperties) {
chosenProperties = physicalProperties;
if (physicalProperties == null || lowestCostPlans.isEmpty()) {
chosenGroupExpressionId = -1;
return Optional.empty();
}
return Optional.ofNullable(lowestCostPlans.get(physicalProperties));
Optional<Pair<Cost, GroupExpression>> costAndGroupExpression =
Optional.ofNullable(lowestCostPlans.get(physicalProperties));
if (costAndGroupExpression.isPresent()) {
chosenGroupExpressionId = costAndGroupExpression.get().second.getId().asInt();
}
return costAndGroupExpression;
}
public GroupExpression getBestPlan(PhysicalProperties properties) {
@ -431,6 +442,10 @@ public class Group {
for (GroupExpression enforcer : enforcers) {
str.append(" ").append(enforcer).append("\n");
}
if (chosenGroupExpressionId != -1) {
str.append(" chosen expression id: ").append(chosenGroupExpressionId).append("\n");
str.append(" chosen properties: ").append(chosenProperties).append("\n");
}
return str.toString();
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java
View File

@ -340,4 +340,8 @@ public class GroupExpression {
builder.append(" (plan=").append(plan.toString()).append(")");
return builder.toString();
}
public ObjectId getId() {
return id;
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RecomputeLogicalPropertiesProcessor.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.nereids.processor.post;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.util.MutableState;
/**
* merge consecutive projects
@ -28,6 +29,8 @@ public class RecomputeLogicalPropertiesProcessor extends PlanPostProcessor {
@Override
public Plan visit(Plan plan, CascadesContext ctx) {
PhysicalPlan physicalPlan = (PhysicalPlan) visitChildren(this, plan, ctx);
return physicalPlan.resetLogicalProperties();
physicalPlan.resetLogicalProperties();
physicalPlan.setMutableState(MutableState.KEY_GROUP, plan.getGroupIdAsString());
return physicalPlan;
}
}

17
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java
View File

@ -209,23 +209,6 @@ public abstract class AbstractPlan extends AbstractTreeNode<Plan> implements Pla
this.mutableState = this.mutableState.set(key, state);
}
/**
* used in treeString()
*
* @return "" if groupExpression is empty, o.w. string format of group id
*/
public String getGroupIdAsString() {
String groupId;
if (getGroupExpression().isPresent()) {
groupId = "@" + groupExpression.get().getOwnerGroup().getGroupId().asInt();
} else if (getMutableState("group").isPresent()) {
groupId = "@" + getMutableState("group").get();
} else {
groupId = "";
}
return groupId;
}
@Override
public boolean deepEquals(TreeNode o) {
AbstractPlan that = (AbstractPlan) o;

23
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
View File

@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.MutableState;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@ -40,7 +41,6 @@ import java.util.stream.Collectors;
* Abstract class for all plan node.
*/
public interface Plan extends TreeNode<Plan> {
PlanType getType();
// cache GroupExpression for fast exit from Memo.copyIn.
@ -165,4 +165,25 @@ public interface Plan extends TreeNode<Plan> {
default String shapeInfo() {
return this.getClass().getSimpleName();
}
/**
* used in treeString()
*
* @return "" if groupExpression is empty, o.w. string format of group id
*/
default String getGroupIdAsString() {
String groupId;
if (getGroupExpression().isPresent()) {
groupId = getGroupExpression().get().getOwnerGroup().getGroupId().asInt() + "";
} else if (getMutableState(MutableState.KEY_GROUP).isPresent()) {
groupId = getMutableState(MutableState.KEY_GROUP).get().toString();
} else {
groupId = "";
}
return groupId;
}
default String getGroupIdWithPrefix() {
return "@" + getGroupIdAsString();
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAssertNumRows.java
View File

@ -70,7 +70,7 @@ public class PhysicalAssertNumRows<CHILD_TYPE extends Plan> extends PhysicalUnar
@Override
public String toString() {
return Utils.toSqlString("PhysicalAssertNumRows" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalAssertNumRows" + getGroupIdWithPrefix(),
"assertNumRowsElement", assertNumRowsElement);
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java
View File

@ -72,7 +72,7 @@ public class PhysicalDistribute<CHILD_TYPE extends Plan> extends PhysicalUnary<C
@Override
public String toString() {
return Utils.toSqlString("PhysicalDistribute[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalDistribute[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"distributionSpec", distributionSpec,
"stats", statistics
);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java
View File

@ -74,7 +74,7 @@ public class PhysicalFilter<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD
@Override
public String toString() {
return Utils.toSqlString("PhysicalFilter[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalFilter[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"predicates", getPredicate(),
"stats", statistics
);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
View File

@ -196,7 +196,7 @@ public class PhysicalHashAggregate<CHILD_TYPE extends Plan> extends PhysicalUnar
@Override
public String toString() {
return Utils.toSqlString("PhysicalHashAggregate[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalHashAggregate[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"aggPhase", aggregateParam.aggPhase,
"aggMode", aggregateParam.aggMode,
"maybeUseStreaming", maybeUsingStream,

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
View File

@ -36,6 +36,7 @@ import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.MutableState;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.statistics.Statistics;
@ -161,7 +162,7 @@ public class PhysicalHashJoin<
args.add("runtimeFilters");
args.add(runtimeFilters.stream().map(rf -> rf.toString() + " ").collect(Collectors.toList()));
}
return Utils.toSqlString("PhysicalHashJoin[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalHashJoin[" + id.asInt() + "]" + getGroupIdWithPrefix(),
args.toArray());
}
@ -173,7 +174,7 @@ public class PhysicalHashJoin<
Optional.empty(), getLogicalProperties(), physicalProperties, statistics,
children.get(0), children.get(1));
if (groupExpression.isPresent()) {
newJoin.setMutableState("group", groupExpression.get().getOwnerGroup().getGroupId().asInt());
newJoin.setMutableState(MutableState.KEY_GROUP, groupExpression.get().getOwnerGroup().getGroupId().asInt());
}
return newJoin;
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLimit.java
View File

@ -156,7 +156,7 @@ public class PhysicalLimit<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD_
@Override
public String toString() {
return Utils.toSqlString("PhysicalLimit[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalLimit[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"limit", limit,
"offset", offset,
"phase", phase,

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.MutableState;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.statistics.Statistics;
@ -115,7 +116,7 @@ public class PhysicalNestedLoopJoin<
@Override
public String toString() {
// TODO: Maybe we could pull up this to the abstract class in the future.
return Utils.toSqlString("PhysicalNestedLoopJoin[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalNestedLoopJoin[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"type", joinType,
"otherJoinCondition", otherJoinConjuncts,
"isMarkJoin", markJoinSlotReference.isPresent(),
@ -131,7 +132,7 @@ public class PhysicalNestedLoopJoin<
hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, Optional.empty(),
getLogicalProperties(), physicalProperties, statistics, children.get(0), children.get(1));
if (groupExpression.isPresent()) {
newJoin.setMutableState("group", groupExpression.get().getOwnerGroup().getGroupId().asInt());
newJoin.setMutableState(MutableState.KEY_GROUP, groupExpression.get().getOwnerGroup().getGroupId().asInt());
}
return newJoin;
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
View File

@ -114,7 +114,7 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca
@Override
public String toString() {
return Utils.toSqlString("PhysicalOlapScan[" + relationId.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalOlapScan[" + relationId.asInt() + "]" + getGroupIdWithPrefix(),
"qualified", Utils.qualifiedName(qualifier, table.getName()),
"stats", statistics, "fr", getMutableState(AbstractPlan.FRAGMENT_ID)
);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java
View File

@ -108,7 +108,7 @@ public class PhysicalOneRowRelation extends PhysicalRelation implements OneRowRe
@Override
public String toString() {
return Utils.toSqlString("PhysicalOneRowRelation[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalOneRowRelation[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"expressions", projects
);
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPartitionTopN.java
View File

@ -178,7 +178,7 @@ public class PhysicalPartitionTopN<CHILD_TYPE extends Plan> extends PhysicalUnar
@Override
public String toString() {
return Utils.toSqlString("PhysicalPartitionTopN[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalPartitionTopN[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"function", function,
"partitionKeys", partitionKeys,
"orderKeys", orderKeys,

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java
View File

@ -77,7 +77,7 @@ public class PhysicalProject<CHILD_TYPE extends Plan> extends PhysicalUnary<CHIL
@Override
public String toString() {
return Utils.toSqlString("PhysicalProject[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalProject[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"projects", projects,
"stats", statistics
);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalQuickSort.java
View File

@ -101,7 +101,7 @@ public class PhysicalQuickSort<CHILD_TYPE extends Plan> extends AbstractPhysical
@Override
public String toString() {
return Utils.toSqlString("PhysicalQuickSort[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalQuickSort[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"orderKeys", orderKeys,
"phase", phase.toString()
);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRepeat.java
View File

@ -97,7 +97,7 @@ public class PhysicalRepeat<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD
@Override
public String toString() {
return Utils.toSqlString("PhysicalRepeat[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalRepeat[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"groupingSets", groupingSets,
"outputExpressions", outputExpressions,
"stats", statistics

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalStorageLayerAggregate.java
View File

@ -73,7 +73,7 @@ public class PhysicalStorageLayerAggregate extends PhysicalCatalogRelation {
@Override
public String toString() {
return Utils.toSqlString("PhysicalStorageLayerAggregate[" + relationId.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalStorageLayerAggregate[" + relationId.asInt() + "]" + getGroupIdWithPrefix(),
"pushDownAggOp", aggOp,
"relation", relation,
"stats", statistics

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTopN.java
View File

@ -149,7 +149,7 @@ public class PhysicalTopN<CHILD_TYPE extends Plan> extends AbstractPhysicalSort<
@Override
public String toString() {
return Utils.toSqlString("PhysicalTopN[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalTopN[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"limit", limit,
"offset", offset,
"orderKeys", orderKeys,

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalUnion.java
View File

@ -84,7 +84,7 @@ public class PhysicalUnion extends PhysicalSetOperation implements Union {
@Override
public String toString() {
return Utils.toSqlString("PhysicalUnion" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalUnion" + getGroupIdWithPrefix(),
"qualifier", qualifier,
"constantExprsList", constantExprsList,
"stats", statistics);

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalWindow.java
View File

@ -104,7 +104,7 @@ public class PhysicalWindow<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD
@Override
public String toString() {
return Utils.toSqlString("PhysicalWindow[" + id.asInt() + "]" + getGroupIdAsString(),
return Utils.toSqlString("PhysicalWindow[" + id.asInt() + "]" + getGroupIdWithPrefix(),
"windowFrameGroup", windowFrameGroup,
"requiredProperties", requireProperties
);

3
fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java
View File

@ -23,6 +23,9 @@ import java.util.Optional;
/** MutableState */
public interface MutableState {
String KEY_GROUP = "group";
String KEY_FRAGMENT = "fragment";
<T> Optional<T> get(String key);
MutableState set(String key, Object value);