From b7fd481dcb3aee082eefa6387558a6a328cb84dc Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Fri, 26 May 2023 18:03:34 +0800 Subject: [PATCH] [Fix](Nereids) Using switch to control minidump input serialize (#20049) Before change, when doing optimize use Nereids planner, input will serialize to memory first. And when bug happen, it would be dump to minidump file when catching the exception. We found that serialization process will cause the performance when statistic message too large or when optimization time be small enough. So the user minidump using should change to ONLY YOU OPEN MINIDUMP SWITCH(set enable_minidump=true;) can you use it. --- .../java/org/apache/doris/nereids/NereidsPlanner.java | 9 ++++++--- .../org/apache/doris/nereids/minidump/NereidsTracer.java | 2 +- .../nereids/trees/plans/commands/ExplainCommand.java | 1 + .../src/main/java/org/apache/doris/qe/StmtExecutor.java | 4 ++++ .../test/java/org/apache/doris/policy/PolicyTest.java | 4 ++-- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index a844b44cfd..cd3c521d27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -196,7 +196,8 @@ public class NereidsPlanner extends Planner { } // minidump of input must be serialized first, this process ensure minidump string not null - if (!statementContext.getConnectContext().getSessionVariable().isPlayNereidsDump()) { + if (!statementContext.getConnectContext().getSessionVariable().isPlayNereidsDump() + && statementContext.getConnectContext().getSessionVariable().isEnableMinidump()) { MinidumpUtils.init(); String queryId = DebugUtil.printId(statementContext.getConnectContext().queryId()); try { @@ -363,14 +364,16 @@ public class NereidsPlanner extends Planner { } private void serializeOutputToDumpFile(Plan resultPlan, ConnectContext connectContext) { - if (connectContext.getSessionVariable().isPlayNereidsDump()) { + if (connectContext.getSessionVariable().isPlayNereidsDump() + || !statementContext.getConnectContext().getSessionVariable().isEnableMinidump()) { return; } connectContext.getMinidump().put("ResultPlan", ((AbstractPlan) resultPlan).toJson()); } private void serializeStatUsed(ConnectContext connectContext) { - if (connectContext.getSessionVariable().isPlayNereidsDump()) { + if (connectContext.getSessionVariable().isPlayNereidsDump() + || !statementContext.getConnectContext().getSessionVariable().isEnableMinidump()) { return; } JSONObject jsonObj = connectContext.getMinidump(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/NereidsTracer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/NereidsTracer.java index 769a7a5334..80101dd750 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/NereidsTracer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/NereidsTracer.java @@ -64,7 +64,7 @@ public class NereidsTracer { } public static String getCurrentTime() { - return TimeUtils.getElapsedTimeMs(NereidsTracer.startTime) / 1000 + "us"; + return TimeUtils.getElapsedTimeMs(NereidsTracer.startTime) + "ms"; } /** log rewrite event when open switch */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index 0885946035..a312c1738c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -70,6 +70,7 @@ public class ExplainCommand extends Command implements NoForward { executor.setParsedStmt(logicalPlanAdapter); NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); + executor.setPlanner(planner); executor.handleExplainStmt(planner.getExplainString(new ExplainOptions(level))); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 7160ada0c3..79a69ac3d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -325,6 +325,10 @@ public class StmtExecutor { return planner; } + public void setPlanner(Planner planner) { + this.planner = planner; + } + public boolean isForwardToMaster() { if (Env.getCurrentEnv().isMaster()) { return false; diff --git a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java index 554e5dca70..a6a4dbbb47 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java @@ -123,10 +123,10 @@ public class PolicyTest extends TestWithFeService { createPolicy("CREATE ROW POLICY test_row_policy ON test.table1 AS PERMISSIVE TO test_policy USING (k1 = 1)"); String queryStr = "EXPLAIN select * from test.table1 a"; String explainString = getSQLPlanOrErrorMsg(queryStr); - Assertions.assertTrue(explainString.contains("`a`.`k1` = 1")); + Assertions.assertTrue(explainString.contains("k1[#0] = 1")); queryStr = "EXPLAIN select * from test.table1 b"; explainString = getSQLPlanOrErrorMsg(queryStr); - Assertions.assertTrue(explainString.contains("`b`.`k1` = 1")); + Assertions.assertTrue(explainString.contains("k1[#0] = 1")); dropPolicy("DROP ROW POLICY test_row_policy ON test.table1 FOR test_policy"); connectContext.getSessionVariable().setEnableNereidsPlanner(beforeConfig); }