database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](Nereids) show cost and execution time for each plan (#17123)

Browse Source 
1. Show cost in optimized plan
2. show plan time schedule time and so on in profile
This commit is contained in:
谢健
2023-02-28 18:59:57 +08:00
committed by GitHub
parent 34813bae13
commit e34e72dd51
6 changed files with 39 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
View File

@ -44,6 +44,13 @@ public class ShowQueryProfileStmt extends ShowStmt {
.addColumn(new Column("EndTime", ScalarType.createVarchar(128)))
.addColumn(new Column("TotalTime", ScalarType.createVarchar(128)))
.addColumn(new Column("QueryState", ScalarType.createVarchar(128)))
.addColumn(new Column("TraceId", ScalarType.createVarchar(128)))
.addColumn(new Column("AnalysisTime", ScalarType.createVarchar(128)))
.addColumn(new Column("PlanTime", ScalarType.createVarchar(128)))
.addColumn(new Column("ScheduleTime", ScalarType.createVarchar(128)))
.addColumn(new Column("FetchResultTime", ScalarType.createVarchar(128)))
.addColumn(new Column("WriteResultTime", ScalarType.createVarchar(128)))
.addColumn(new Column("WaitAndFetchResultTime", ScalarType.createVarchar(128)))
.build();
public static final ShowResultSetMetaData META_DATA_FRAGMENTS =

16
fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
View File

@ -79,6 +79,12 @@ public class ProfileManager {
public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE = "Parallel Fragment Exec Instance Num";
public static final String TRACE_ID = "Trace ID";
public static final String ANALYSIS_TIME = "Analysis Time";
public static final String FETCH_RESULT_TIME = "Fetch Result Time";
public static final String PLAN_TIME = "Plan Time";
public static final String SCHEDULE_TIME = "Schedule Time";
public static final String WRITE_RESULT_TIME = "Write Result Time";
public static final String WAIT_FETCH_RESULT_TIME = "Wait and Fetch Result Time";
public enum ProfileType {
QUERY,
@ -88,6 +94,9 @@ public class ProfileManager {
public static final List<String> PROFILE_HEADERS = Collections.unmodifiableList(
Arrays.asList(JOB_ID, QUERY_ID, USER, DEFAULT_DB, SQL_STATEMENT, QUERY_TYPE,
START_TIME, END_TIME, TOTAL_TIME, QUERY_STATE, TRACE_ID));
public static final List<String> EXECUTION_HEADERS = Collections.unmodifiableList(
Arrays.asList(ANALYSIS_TIME, PLAN_TIME, SCHEDULE_TIME, FETCH_RESULT_TIME,
WRITE_RESULT_TIME, WAIT_FETCH_RESULT_TIME));
private class ProfileElement {
public ProfileElement(RuntimeProfile profile) {
@ -146,6 +155,10 @@ public class ProfileManager {
for (String header : PROFILE_HEADERS) {
element.infoStrings.put(header, summaryProfile.getInfoString(header));
}
RuntimeProfile executionProfile = summaryProfile.getChildList().get(0).first;
for (String header : EXECUTION_HEADERS) {
element.infoStrings.put(header, executionProfile.getInfoString(header));
}
MultiProfileTreeBuilder builder = new MultiProfileTreeBuilder(profile);
try {
@ -215,6 +228,9 @@ public class ProfileManager {
for (String str : PROFILE_HEADERS) {
row.add(infoStrings.get(str));
}
for (String str : EXECUTION_HEADERS) {
row.add(infoStrings.get(str));
}
result.add(row);
}
} finally {

4
fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java
View File

@ -69,7 +69,9 @@ public class QueryProfileController extends BaseController {
private void addFinishedQueryInfo(Map<String, Object> result) {
List<List<String>> finishedQueries = ProfileManager.getInstance().getAllQueries();
List<String> columnHeaders = Lists.newLinkedList(ProfileManager.PROFILE_HEADERS);
List<String> columnHeaders = Lists.newLinkedList();
columnHeaders.addAll(ProfileManager.PROFILE_HEADERS);
columnHeaders.addAll(ProfileManager.EXECUTION_HEADERS);
int jobIdIndex = -1;
int queryIdIndex = -1;
int queryTypeIndex = -1;

14
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.analysis.DescriptorTable;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.common.NereidsException;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext.Lock;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
@ -76,6 +77,8 @@ public class NereidsPlanner extends Planner {
private Plan analyzedPlan;
private Plan rewrittenPlan;
private Plan optimizedPlan;
// The cost of optimized plan
private double cost = 0;
public NereidsPlanner(StatementContext statementContext) {
this.statementContext = statementContext;
@ -261,12 +264,16 @@ public class NereidsPlanner extends Planner {
private PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) {
if (nthPlan <= 1) {
cost = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow(
() -> new AnalysisException("lowestCostPlans with physicalProperties("
+ physicalProperties + ") doesn't exist in root group")).first;
return chooseBestPlan(rootGroup, physicalProperties);
}
Memo memo = cascadesContext.getMemo();
long id = memo.rank(nthPlan);
return memo.unrank(id);
Pair<Long, Double> idCost = memo.rank(nthPlan);
cost = idCost.second;
return memo.unrank(idCost.first);
}
private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physicalProperties)
@ -276,7 +283,6 @@ public class NereidsPlanner extends Planner {
() -> new AnalysisException("lowestCostPlans with physicalProperties("
+ physicalProperties + ") doesn't exist in root group")).second;
List<PhysicalProperties> inputPropertiesList = groupExpression.getInputPropertiesList(physicalProperties);
List<Plan> planChildren = Lists.newArrayList();
for (int i = 0; i < groupExpression.arity(); i++) {
planChildren.add(chooseBestPlan(groupExpression.child(i), inputPropertiesList.get(i)));
@ -310,7 +316,7 @@ public class NereidsPlanner extends Planner {
case REWRITTEN_PLAN:
return rewrittenPlan.treeString();
case OPTIMIZED_PLAN:
return optimizedPlan.treeString();
return "cost = " + cost + "\n" + optimizedPlan.treeString();
case ALL_PLAN:
return "========== PARSED PLAN ==========\n"
+ parsedPlan.treeString() + "\n\n"

4
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
View File

@ -719,7 +719,7 @@ public class Memo {
*
* In unrank() function, we will extract the actual physical function according the unique ID
*/
public long rank(long n) {
public Pair<Long, Double> rank(long n) {
double threshold = 0.000000001;
Preconditions.checkArgument(n > 0, "the n %d must be greater than 0 in nthPlan", n);
List<Pair<Long, Double>> plans = rankGroup(root, PhysicalProperties.GATHER);
@ -737,7 +737,7 @@ public class Memo {
pq.poll();
}
}
return pq.peek().first;
return pq.peek();
}
private List<Pair<Long, Double>> rankGroup(Group group, PhysicalProperties prop) {

2
fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java
View File

@ -55,7 +55,7 @@ public class RankTest extends TPCHTestBase {
.optimize()
.getCascadesContext()
.getMemo();
PhysicalPlan plan1 = memo.unrank(memo.rank(1));
PhysicalPlan plan1 = memo.unrank(memo.rank(1).first);
PhysicalPlan plan2 = PlanChecker.from(connectContext)
.analyze(field.get(null).toString())
.rewrite()