database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](Nereids) Using switch to control minidump input serialize (#20049)

Browse Source 
Before change, when doing optimize use Nereids planner, input will serialize to memory first. And when bug happen, it would be dump to minidump file when catching the exception.
We found that serialization process will cause the performance when statistic message too large or when optimization time be small enough.
So the user minidump using should change to ONLY YOU OPEN MINIDUMP SWITCH(set enable_minidump=true;) can you use it.
This commit is contained in:
LiBinfeng
2023-05-26 18:03:34 +08:00
committed by GitHub
parent 9e70a9ef84
commit b7fd481dcb
5 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -196,7 +196,8 @@ public class NereidsPlanner extends Planner {
}
// minidump of input must be serialized first, this process ensure minidump string not null
if (!statementContext.getConnectContext().getSessionVariable().isPlayNereidsDump()) {
if (!statementContext.getConnectContext().getSessionVariable().isPlayNereidsDump()
&& statementContext.getConnectContext().getSessionVariable().isEnableMinidump()) {
MinidumpUtils.init();
String queryId = DebugUtil.printId(statementContext.getConnectContext().queryId());
try {
@ -363,14 +364,16 @@ public class NereidsPlanner extends Planner {
}
private void serializeOutputToDumpFile(Plan resultPlan, ConnectContext connectContext) {
if (connectContext.getSessionVariable().isPlayNereidsDump()) {
if (connectContext.getSessionVariable().isPlayNereidsDump()
|| !statementContext.getConnectContext().getSessionVariable().isEnableMinidump()) {
return;
}
connectContext.getMinidump().put("ResultPlan", ((AbstractPlan) resultPlan).toJson());
}
private void serializeStatUsed(ConnectContext connectContext) {
if (connectContext.getSessionVariable().isPlayNereidsDump()) {
if (connectContext.getSessionVariable().isPlayNereidsDump()
|| !statementContext.getConnectContext().getSessionVariable().isEnableMinidump()) {
return;
}
JSONObject jsonObj = connectContext.getMinidump();

2
fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/NereidsTracer.java
View File

@ -64,7 +64,7 @@ public class NereidsTracer {
}
public static String getCurrentTime() {
return TimeUtils.getElapsedTimeMs(NereidsTracer.startTime) / 1000 + "us";
return TimeUtils.getElapsedTimeMs(NereidsTracer.startTime) + "ms";
}
/** log rewrite event when open switch */

1
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
View File

@ -70,6 +70,7 @@ public class ExplainCommand extends Command implements NoForward {
executor.setParsedStmt(logicalPlanAdapter);
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
executor.setPlanner(planner);
executor.handleExplainStmt(planner.getExplainString(new ExplainOptions(level)));
}

4
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -325,6 +325,10 @@ public class StmtExecutor {
return planner;
}
public void setPlanner(Planner planner) {
this.planner = planner;
}
public boolean isForwardToMaster() {
if (Env.getCurrentEnv().isMaster()) {
return false;

4
fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java
View File

@ -123,10 +123,10 @@ public class PolicyTest extends TestWithFeService {
createPolicy("CREATE ROW POLICY test_row_policy ON test.table1 AS PERMISSIVE TO test_policy USING (k1 = 1)");
String queryStr = "EXPLAIN select * from test.table1 a";
String explainString = getSQLPlanOrErrorMsg(queryStr);
Assertions.assertTrue(explainString.contains("`a`.`k1` = 1"));
Assertions.assertTrue(explainString.contains("k1[#0] = 1"));
queryStr = "EXPLAIN select * from test.table1 b";
explainString = getSQLPlanOrErrorMsg(queryStr);
Assertions.assertTrue(explainString.contains("`b`.`k1` = 1"));
Assertions.assertTrue(explainString.contains("k1[#0] = 1"));
dropPolicy("DROP ROW POLICY test_row_policy ON test.table1 FOR test_policy");
connectContext.getSessionVariable().setEnableNereidsPlanner(beforeConfig);
}