From 3e7a9424e44495ecf357b06071ee02c69b5f1014 Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 4 Apr 2023 09:44:15 +0800 Subject: [PATCH] [feature](nereids) explain shape plan (#18296) `explain shape plan select ...` only print plan shape related information, including - node name - join type, join condition - filter condition - agg phase It is painful to maintain regression cases using explain since there are a lot of mutable information, like slot id. By this pr, we could use explain shape plan in regression cases. for exmaple: this is tpch q2 +-----------------------------------------------------------------------------------------------------------+ | Explain String | +-----------------------------------------------------------------------------------------------------------+ | PhysicalTopN | | --PhysicalDistribute | | ----PhysicalTopN | | ------PhysicalProject | | --------filter((cast(ps_supplycost as DECIMAL(27, 9)) = min(ps_supplycost) OVER(PARTITION BY p_partkey))) | | ----------PhysicalWindow | | ------------PhysicalQuickSort | | --------------PhysicalProject | | ----------------hashJoin[INNER_JOIN](supplier.s_suppkey = partsupp.ps_suppkey) | | ------------------PhysicalProject | | --------------------hashJoin[INNER_JOIN](part.p_partkey = partsupp.ps_partkey) | | ----------------------PhysicalProject | | ------------------------PhysicalOlapScan[partsupp] | | ----------------------PhysicalProject | | ------------------------filter((part.p_size = 15)(p_type like '%BRASS')) | | --------------------------PhysicalOlapScan[part] | | ------------------PhysicalDistribute | | --------------------hashJoin[INNER_JOIN](supplier.s_nationkey = nation.n_nationkey) | | ----------------------PhysicalOlapScan[supplier] | | ----------------------PhysicalDistribute | | ------------------------hashJoin[INNER_JOIN](nation.n_regionkey = region.r_regionkey) | | --------------------------PhysicalProject | | ----------------------------PhysicalOlapScan[nation] | | --------------------------PhysicalDistribute | | ----------------------------PhysicalProject | | ------------------------------filter((region.r_name = 'EUROPE')) | | --------------------------------PhysicalOlapScan[region] | +-----------------------------------------------------------------------------------------------------------+ --- .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 1 + .../apache/doris/nereids/NereidsPlanner.java | 7 +++++- .../trees/expressions/BinaryOperator.java | 5 ++++ .../nereids/trees/expressions/Expression.java | 4 +++ .../trees/expressions/SlotReference.java | 9 +++++++ .../doris/nereids/trees/plans/Plan.java | 25 +++++++++++++++++++ .../trees/plans/commands/ExplainCommand.java | 1 + .../plans/physical/AbstractPhysicalPlan.java | 1 - .../trees/plans/physical/PhysicalFilter.java | 9 +++++++ .../plans/physical/PhysicalHashAggregate.java | 7 ++++++ .../plans/physical/PhysicalHashJoin.java | 13 ++++++++++ .../physical/PhysicalNestedLoopJoin.java | 8 ++++++ .../plans/physical/PhysicalOlapScan.java | 8 ++++++ 14 files changed, 97 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 8b53d1fdde..b1d4fe0986 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -330,6 +330,7 @@ SESSION_USER: 'SESSION_USER'; SET: 'SET'; SETMINUS: 'MINUS'; SETS: 'SETS'; +SHAPE: 'SHAPE'; SHOW: 'SHOW'; SKEWED: 'SKEWED'; SOME: 'SOME'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 2749ac8d15..a3df7fa713 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -64,6 +64,7 @@ planType | ANALYZED | REWRITTEN | LOGICAL // same type | OPTIMIZED | PHYSICAL // same type + | SHAPE | ALL // default type ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 18a1ab7aed..1604bc46a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -219,7 +219,10 @@ public class NereidsPlanner extends Planner { LOG.info(tree); } - if (explainLevel == ExplainLevel.OPTIMIZED_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { + System.out.println(physicalPlan.shape(" ")); + if (explainLevel == ExplainLevel.OPTIMIZED_PLAN + || explainLevel == ExplainLevel.ALL_PLAN + || explainLevel == ExplainLevel.SHAPE_PLAN) { optimizedPlan = physicalPlan; } @@ -354,6 +357,8 @@ public class NereidsPlanner extends Planner { return rewrittenPlan.treeString(); case OPTIMIZED_PLAN: return "cost = " + cost + "\n" + optimizedPlan.treeString(); + case SHAPE_PLAN: + return optimizedPlan.shape(""); case ALL_PLAN: return "========== PARSED PLAN ==========\n" + parsedPlan.treeString() + "\n\n" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java index 7464a4cae7..68e05db3a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java @@ -55,6 +55,11 @@ public abstract class BinaryOperator extends Expression implements BinaryExpress return "(" + left().toString() + " " + symbol + " " + right().toString() + ")"; } + @Override + public String shapeInfo() { + return "(" + left().shapeInfo() + " " + symbol + " " + right().shapeInfo() + ")"; + } + @Override public int hashCode() { return Objects.hash(symbol, left(), right()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index 06697f82da..a217d44c29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -210,4 +210,8 @@ public abstract class Expression extends AbstractTreeNode implements return false; } + public String shapeInfo() { + return toSql(); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 25a9cb79c2..edc683c8fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -124,6 +124,15 @@ public class SlotReference extends Slot { return name + "#" + exprId; } + @Override + public String shapeInfo() { + if (qualifier.isEmpty()) { + return name; + } else { + return qualifier.get(qualifier.size() - 1) + "." + name; + } + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index 018af58ba6..da6eb24d43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -145,4 +145,29 @@ public interface Plan extends TreeNode { } void setMutableState(String key, Object value); + + /** + * a simple version of explain, used to verify plan shape + * @param prefix " " + * @return string format of plan shape + */ + default String shape(String prefix) { + StringBuilder builder = new StringBuilder(); + builder.append(prefix).append(shapeInfo()).append("\n"); + String childPrefix = prefix + "--"; + children().forEach( + child -> { + builder.append(child.shape(childPrefix)); + } + ); + return builder.toString(); + } + + /** + * used in shape() + * @return default value is its class name + */ + default String shapeInfo() { + return this.getClass().getSimpleName(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index ce6e550b61..0885946035 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -43,6 +43,7 @@ public class ExplainCommand extends Command implements NoForward { ANALYZED_PLAN(true), REWRITTEN_PLAN(true), OPTIMIZED_PLAN(true), + SHAPE_PLAN(true), ALL_PLAN(true) ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java index c3a283f46b..2cdab8c32b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java @@ -54,5 +54,4 @@ public abstract class AbstractPhysicalPlan extends AbstractPlan implements Physi public PhysicalProperties getPhysicalProperties() { return physicalProperties; } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java index 790dcb473e..866fdda4c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java @@ -123,4 +123,13 @@ public class PhysicalFilter extends PhysicalUnary(conjuncts, groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); } + + @Override + public String shapeInfo() { + StringBuilder builder = new StringBuilder(); + builder.append("filter("); + conjuncts.forEach(conjunct -> builder.append(conjunct.shapeInfo())); + builder.append(")"); + return builder.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java index e713f6dd87..79d13ccfc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java @@ -272,4 +272,11 @@ public class PhysicalHashAggregate extends PhysicalUnar aggregateParam, maybeUsingStream, Optional.empty(), getLogicalProperties(), requireProperties, physicalProperties, statistics, newChild); } + + @Override + public String shapeInfo() { + StringBuilder builder = new StringBuilder("hashAgg["); + builder.append(getAggPhase()).append("]"); + return builder.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java index 84c1c35e97..7351ec2e32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java @@ -152,4 +152,17 @@ public class PhysicalHashJoin< return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, groupExpression, getLogicalProperties(), physicalProperties, statistics, left(), right()); } + + @Override + public String shapeInfo() { + StringBuilder builder = new StringBuilder(); + builder.append("hashJoin[").append(joinType).append("]"); + hashJoinConjuncts.forEach(expr -> { + builder.append(expr.shapeInfo()); + }); + otherJoinConjuncts.forEach(expr -> { + builder.append(expr.shapeInfo()); + }); + return builder.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java index 29e92b5038..6b034a5138 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java @@ -167,4 +167,12 @@ public class PhysicalNestedLoopJoin< public boolean isBitMapRuntimeFilterConditionsEmpty() { return bitMapRuntimeFilterConditions.isEmpty(); } + + @Override + public String shapeInfo() { + StringBuilder builder = new StringBuilder("NestedLoopJoin"); + builder.append("[").append(joinType).append("]"); + otherJoinConjuncts.forEach(expr -> builder.append(expr.shapeInfo())); + return builder.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 2a238fdac5..2985ab8df4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -155,4 +155,12 @@ public class PhysicalOlapScan extends PhysicalRelation implements OlapScan { selectedPartitionIds, distributionSpec, preAggStatus, groupExpression, getLogicalProperties(), physicalProperties, statistics); } + + @Override + public String shapeInfo() { + StringBuilder builder = new StringBuilder(); + builder.append(this.getClass().getSimpleName()).append("[").append(olapTable.getName()).append("]"); + return builder.toString(); + } + }