From d7cb5cf3db2662d1a2e5adf52ab0b920cb4b1387 Mon Sep 17 00:00:00 2001 From: minghong Date: Sat, 11 Mar 2023 13:40:15 +0800 Subject: [PATCH] [feature](nereids) add session var: dump_nereids_memo (#17666) * dump_nereids_memo * print groupexpr id --- .../apache/doris/nereids/NereidsPlanner.java | 18 +++++++++++++++-- .../org/apache/doris/nereids/memo/Group.java | 8 ++++---- .../doris/nereids/memo/GroupExpression.java | 6 ++++-- .../org/apache/doris/nereids/memo/Memo.java | 20 ++++++++++++------- .../org/apache/doris/qe/SessionVariable.java | 13 ++++++++++++ 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 6f35193797..6a70fbbc79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -61,6 +61,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; /** @@ -180,10 +181,23 @@ public class NereidsPlanner extends Planner { optimize(); + //print memo before choose plan. + //if chooseNthPlan failed, we could get memo to debug + if (ConnectContext.get().getSessionVariable().isDumpNereidsMemo()) { + String memo = cascadesContext.getMemo().toString(); + LOG.info(memo); + } + int nth = ConnectContext.get().getSessionVariable().getNthOptimizedPlan(); PhysicalPlan physicalPlan = chooseNthPlan(getRoot(), requireProperties, nth); physicalPlan = postProcess(physicalPlan); + + if (ConnectContext.get().getSessionVariable().isDumpNereidsMemo()) { + String tree = physicalPlan.treeString(); + LOG.info(tree); + } + if (explainLevel == ExplainLevel.OPTIMIZED_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { optimizedPlan = physicalPlan; } @@ -293,8 +307,8 @@ public class NereidsPlanner extends Planner { if (!(plan instanceof PhysicalPlan)) { throw new AnalysisException("Result plan must be PhysicalPlan"); } - - // TODO: set (logical and physical)properties/statistics/... for physicalPlan. + // add groupExpression to plan so that we could print group id in plan.treeString() + plan = plan.withGroupExpression(Optional.of(groupExpression)); PhysicalPlan physicalPlan = ((PhysicalPlan) plan).withPhysicalPropertiesAndStats( groupExpression.getOutputProperties(physicalProperties), groupExpression.getOwnerGroup().getStatistics()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 36fe488e6c..8e1720fd14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -397,13 +397,13 @@ public class Group { @Override public String toString() { StringBuilder str = new StringBuilder("Group[" + groupId + "]\n"); - str.append("logical expressions:\n"); + str.append(" logical expressions:\n"); for (GroupExpression logicalExpression : logicalExpressions) { - str.append(" ").append(logicalExpression).append("\n"); + str.append(" ").append(logicalExpression).append("\n"); } - str.append("physical expressions:\n"); + str.append(" physical expressions:\n"); for (GroupExpression physicalExpression : physicalExpressions) { - str.append(" ").append(physicalExpression).append("\n"); + str.append(" ").append(physicalExpression).append("\n"); } return str.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java index efc35bc5d4..f9f7fef2d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/GroupExpression.java @@ -36,6 +36,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import java.text.DecimalFormat; import java.util.BitSet; import java.util.List; import java.util.Map; @@ -318,8 +319,9 @@ public class GroupExpression { builder.append("#").append(ownerGroup.getGroupId().asInt()); } - builder.append(" cost=").append((long) cost); - + DecimalFormat decimalFormat = new DecimalFormat(); + decimalFormat.setGroupingSize(3); + builder.append(" cost=").append(decimalFormat.format((long) cost)); builder.append(" estRows=").append(estOutputRowCount); builder.append(" (plan=").append(plan.toString()).append(") children=["); for (Group group : children) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index acb2cf1296..b1adb4cf05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -693,7 +693,7 @@ public class Memo { StringBuilder builder = new StringBuilder(); builder.append("root:").append(getRoot()).append("\n"); for (Group group : groups.values()) { - builder.append(group).append("\n"); + builder.append("\n\n").append(group); builder.append(" stats=").append(group.getStatistics()).append("\n"); StatsDeriveResult stats = group.getStatistics(); if (stats != null && !group.getLogicalExpressions().isEmpty() @@ -702,12 +702,18 @@ public class Memo { builder.append(" ").append(e.getKey()).append(":").append(e.getValue()).append("\n"); } } - for (GroupExpression groupExpression : group.getLogicalExpressions()) { - builder.append(" ").append(groupExpression.toString()).append("\n"); - } - for (GroupExpression groupExpression : group.getPhysicalExpressions()) { - builder.append(" ").append(groupExpression.toString()).append("\n"); - } + + builder.append(" lowest Plan(cost, properties, plan)"); + group.getAllProperties().forEach( + prop -> { + Optional> costAndGroupExpression = group.getLowestCostPlan(prop); + if (costAndGroupExpression.isPresent()) { + builder.append("\n " + costAndGroupExpression.get().first.getValue() + " " + prop) + .append("\n ").append(costAndGroupExpression.get().second); + } + } + ); + builder.append("\n"); } return builder.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 1133cf6807..ad471791ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -283,6 +283,8 @@ public class SessionVariable implements Serializable, Writable { public static final String SHOW_USER_DEFAULT_ROLE = "show_user_default_role"; + public static final String DUMP_NEREIDS_MEMO = "dump_nereids_memo"; + // fix replica to query. If num = 1, query the smallest replica, if 2 is the second smallest replica. public static final String USE_FIX_REPLICA = "use_fix_replica"; @@ -760,6 +762,9 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = USE_FIX_REPLICA) public int useFixReplica = -1; + @VariableMgr.VarAttr(name = DUMP_NEREIDS_MEMO) + public boolean dumpNereidsMemo = false; + // If set to true, all query will be executed without returning result @VariableMgr.VarAttr(name = DRY_RUN_QUERY, needForward = true) public boolean dryRunQuery = false; @@ -1847,4 +1852,12 @@ public class SessionVariable implements Serializable, Writable { } return ""; } + + public boolean isDumpNereidsMemo() { + return dumpNereidsMemo; + } + + public void setDumpNereidsMemo(boolean dumpNereidsMemo) { + this.dumpNereidsMemo = dumpNereidsMemo; + } }