[feature](profilev2) Preliminary support for profilev2. (#24881)
You can set the level of counters on the backend using ADD_COUNTER_WITH_LEVEL/ADD_TIMER_WITH_LEVEL. The profile can then merge counters with level 1.
set profile_level = 1;
such as
sql
select count(*) from customer join item on c_customer_sk = i_item_sk
profile
Simple profile
PLAN FRAGMENT 0
OUTPUT EXPRS:
count(*)
PARTITION: UNPARTITIONED
VRESULT SINK
MYSQL_PROTOCAL
7:VAGGREGATE (merge finalize)
| output: count(partial_count(*))[#44]
| group by:
| cardinality=1
| TotalTime: avg 725.608us, max 725.608us, min 725.608us
| RowsReturned: 1
|
6:VEXCHANGE
offset: 0
TotalTime: avg 52.411us, max 52.411us, min 52.411us
RowsReturned: 8
PLAN FRAGMENT 1
PARTITION: HASH_PARTITIONED: c_customer_sk
STREAM DATA SINK
EXCHANGE ID: 06
UNPARTITIONED
TotalTime: avg 106.263us, max 118.38us, min 81.403us
BlocksSent: 8
5:VAGGREGATE (update serialize)
| output: partial_count(*)[#43]
| group by:
| cardinality=1
| TotalTime: avg 679.296us, max 739.395us, min 554.904us
| BuildTime: avg 33.198us, max 48.387us, min 28.880us
| ExecTime: avg 27.633us, max 40.278us, min 24.537us
| RowsReturned: 8
|
4:VHASH JOIN
| join op: INNER JOIN(PARTITIONED)[]
| equal join conjunct: c_customer_sk = i_item_sk
| runtime filters: RF000[bloom] <- i_item_sk(18000/16384/1048576)
| cardinality=17,740
| vec output tuple id: 3
| vIntermediate tuple ids: 2
| hash output slot ids: 22
| RowsReturned: 18.0K (18000)
| ProbeRows: 18.0K (18000)
| ProbeTime: avg 862.308us, max 1.576ms, min 666.28us
| BuildRows: 18.0K (18000)
| BuildTime: avg 3.8ms, max 3.860ms, min 2.317ms
|
|----1:VEXCHANGE
| offset: 0
| TotalTime: avg 48.822us, max 67.459us, min 30.380us
| RowsReturned: 18.0K (18000)
|
3:VEXCHANGE
offset: 0
TotalTime: avg 33.162us, max 39.480us, min 28.854us
RowsReturned: 18.0K (18000)
PLAN FRAGMENT 2
PARTITION: HASH_PARTITIONED: c_customer_id
STREAM DATA SINK
EXCHANGE ID: 03
HASH_PARTITIONED: c_customer_sk
TotalTime: avg 753.954us, max 1.210ms, min 499.470us
BlocksSent: 64
2:VOlapScanNode
TABLE: default_cluster:tpcds.customer(customer), PREAGGREGATION: ON
runtime filters: RF000[bloom] -> c_customer_sk
partitions=1/1, tablets=12/12, tabletList=1550745,1550747,1550749 ...
cardinality=100000, avgRowSize=0.0, numNodes=1
pushAggOp=NONE
TotalTime: avg 18.417us, max 41.319us, min 10.189us
RowsReturned: 18.0K (18000)
---------
Co-authored-by: yiguolei <676222867@qq.com>
This commit is contained in:
Mryange
@ -19,6 +19,7 @@ package org.apache.doris.common.profile;
|
||||
|
||||
import org.apache.doris.common.util.ProfileManager;
|
||||
import org.apache.doris.common.util.RuntimeProfile;
|
||||
import org.apache.doris.planner.Planner;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
@ -62,7 +63,7 @@ public class Profile {
|
||||
}
|
||||
|
||||
public synchronized void update(long startTime, Map<String, String> summaryInfo, boolean isFinished,
|
||||
boolean isSimpleProfile) {
|
||||
int profileLevel, Planner planner) {
|
||||
if (this.isFinished) {
|
||||
return;
|
||||
}
|
||||
@ -71,7 +72,8 @@ public class Profile {
|
||||
executionProfile.update(startTime, isFinished);
|
||||
}
|
||||
rootProfile.computeTimeInProfile();
|
||||
rootProfile.setProfileLevel(isSimpleProfile);
|
||||
rootProfile.setPlaner(planner);
|
||||
rootProfile.setProfileLevel(profileLevel);
|
||||
ProfileManager.getInstance().pushProfile(rootProfile);
|
||||
this.isFinished = isFinished;
|
||||
}
|
||||
|
||||
@ -24,6 +24,7 @@ public class Counter {
|
||||
private volatile long value;
|
||||
private volatile int type;
|
||||
private volatile boolean remove = false;
|
||||
private volatile long level;
|
||||
|
||||
public long getValue() {
|
||||
return value;
|
||||
@ -42,6 +43,10 @@ public class Counter {
|
||||
return TUnit.findByValue(type);
|
||||
}
|
||||
|
||||
public void setLevel(long level) {
|
||||
this.level = level;
|
||||
}
|
||||
|
||||
public void setType(TUnit type) {
|
||||
this.type = type.getValue();
|
||||
}
|
||||
@ -49,6 +54,13 @@ public class Counter {
|
||||
public Counter(TUnit type, long value) {
|
||||
this.value = value;
|
||||
this.type = type.getValue();
|
||||
this.level = 2;
|
||||
}
|
||||
|
||||
public Counter(TUnit type, long value, long level) {
|
||||
this.value = value;
|
||||
this.type = type.getValue();
|
||||
this.level = level;
|
||||
}
|
||||
|
||||
public void addValue(Counter other) {
|
||||
@ -77,4 +89,8 @@ public class Counter {
|
||||
public boolean isRemove() {
|
||||
return this.remove;
|
||||
}
|
||||
|
||||
public long getLevel() {
|
||||
return this.level;
|
||||
}
|
||||
}
|
||||
|
||||
@ -86,7 +86,7 @@ public class ProfileManager {
|
||||
if (profileContent == null) {
|
||||
// Simple profile will change the structure of the profile.
|
||||
try {
|
||||
profileContent = profile.getSimpleString();
|
||||
profileContent = profile.getProfileByLevel();
|
||||
} catch (Exception e) {
|
||||
LOG.warn("profile get error : " + e.toString());
|
||||
}
|
||||
|
||||
@ -0,0 +1,99 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.common.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
|
||||
/**
|
||||
* Used for collecting information obtained from the profile.
|
||||
*/
|
||||
public class ProfileStatistics {
|
||||
// Record statistical information based on nodeid.
|
||||
private HashMap<Integer, ArrayList<String>> statisticalInfo;
|
||||
|
||||
// Record statistical information based on fragment ID.
|
||||
// "Currently used to record sink nodes.
|
||||
private HashMap<Integer, ArrayList<String>> fragmentInfo;
|
||||
|
||||
private int fragmentId;
|
||||
|
||||
private boolean isDataSink;
|
||||
|
||||
public ProfileStatistics() {
|
||||
statisticalInfo = new HashMap<Integer, ArrayList<String>>();
|
||||
fragmentInfo = new HashMap<Integer, ArrayList<String>>();
|
||||
fragmentId = 0;
|
||||
isDataSink = false;
|
||||
}
|
||||
|
||||
private void addPlanNodeInfo(int id, String info) {
|
||||
if (!statisticalInfo.containsKey(id)) {
|
||||
statisticalInfo.put(id, new ArrayList<String>());
|
||||
}
|
||||
statisticalInfo.get(id).add(info);
|
||||
}
|
||||
|
||||
private void addDataSinkInfo(String info) {
|
||||
if (fragmentInfo.get(fragmentId) == null) {
|
||||
fragmentInfo.put(fragmentId, new ArrayList<String>());
|
||||
}
|
||||
fragmentInfo.get(fragmentId).add(info);
|
||||
}
|
||||
|
||||
public void addInfoFromProfile(RuntimeProfile profile, String info) {
|
||||
if (isDataSink) {
|
||||
addDataSinkInfo(info);
|
||||
} else {
|
||||
addPlanNodeInfo(profile.nodeId(), info);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean hasInfo(int id) {
|
||||
return statisticalInfo.containsKey(id);
|
||||
}
|
||||
|
||||
public void getInfoById(int id, String prefix, StringBuilder str) {
|
||||
if (!hasInfo(id)) {
|
||||
return;
|
||||
}
|
||||
ArrayList<String> infos = statisticalInfo.get(id);
|
||||
for (String info : infos) {
|
||||
str.append(prefix + info + "\n");
|
||||
}
|
||||
}
|
||||
|
||||
public void getDataSinkInfo(int fragmentIdx, String prefix, StringBuilder str) {
|
||||
if (!fragmentInfo.containsKey(fragmentIdx)) {
|
||||
return;
|
||||
}
|
||||
ArrayList<String> infos = fragmentInfo.get(fragmentIdx);
|
||||
for (String info : infos) {
|
||||
str.append(prefix + info + "\n");
|
||||
}
|
||||
}
|
||||
|
||||
public void setFragmentId(int id) {
|
||||
this.fragmentId = id;
|
||||
}
|
||||
|
||||
public void setIsDataSink(boolean dataSink) {
|
||||
this.isDataSink = dataSink;
|
||||
}
|
||||
|
||||
}
|
||||
@ -20,6 +20,7 @@ package org.apache.doris.common.util;
|
||||
import org.apache.doris.common.Pair;
|
||||
import org.apache.doris.common.Reference;
|
||||
import org.apache.doris.common.profile.SummaryProfile;
|
||||
import org.apache.doris.planner.Planner;
|
||||
import org.apache.doris.thrift.TCounter;
|
||||
import org.apache.doris.thrift.TRuntimeProfileNode;
|
||||
import org.apache.doris.thrift.TRuntimeProfileTree;
|
||||
@ -49,9 +50,9 @@ public class RuntimeProfile {
|
||||
private static final Logger LOG = LogManager.getLogger(RuntimeProfile.class);
|
||||
public static String ROOT_COUNTER = "";
|
||||
public static int FRAGMENT_DEPTH = 3;
|
||||
public static String MAX_TIME_PRE = "max: ";
|
||||
public static String MIN_TIME_PRE = "min: ";
|
||||
public static String AVG_TIME_PRE = "avg: ";
|
||||
public static String MAX_TIME_PRE = "max ";
|
||||
public static String MIN_TIME_PRE = "min ";
|
||||
public static String AVG_TIME_PRE = "avg ";
|
||||
private Counter counterTotalTime;
|
||||
private double localTimePercent;
|
||||
|
||||
@ -74,15 +75,18 @@ public class RuntimeProfile {
|
||||
|
||||
private Boolean isDone = false;
|
||||
private Boolean isCancel = false;
|
||||
private boolean enableSimplyProfile = false;
|
||||
private int profileLevel = 3;
|
||||
private Planner planner = null;
|
||||
private int nodeid = -1;
|
||||
|
||||
public RuntimeProfile(String name) {
|
||||
this();
|
||||
this.name = name;
|
||||
this.counterTotalTime = new Counter(TUnit.TIME_NS, 0, 1);
|
||||
}
|
||||
|
||||
public RuntimeProfile() {
|
||||
this.counterTotalTime = new Counter(TUnit.TIME_NS, 0);
|
||||
this.counterTotalTime = new Counter(TUnit.TIME_NS, 0, 1);
|
||||
this.localTimePercent = 0;
|
||||
this.counterMap.put("TotalTime", counterTotalTime);
|
||||
}
|
||||
@ -111,6 +115,10 @@ public class RuntimeProfile {
|
||||
return counterTotalTime;
|
||||
}
|
||||
|
||||
public int nodeId() {
|
||||
return this.nodeid;
|
||||
}
|
||||
|
||||
public Map<String, Counter> getCounterMap() {
|
||||
return counterMap;
|
||||
}
|
||||
@ -165,18 +173,23 @@ public class RuntimeProfile {
|
||||
// preorder traversal, idx should be modified in the traversal process
|
||||
private void update(List<TRuntimeProfileNode> nodes, Reference<Integer> idx) {
|
||||
TRuntimeProfileNode node = nodes.get(idx.getRef());
|
||||
// Make sure to update the latest LoadChannel profile according to the timestamp.
|
||||
// Make sure to update the latest LoadChannel profile according to the
|
||||
// timestamp.
|
||||
if (node.timestamp != -1 && node.timestamp < timestamp) {
|
||||
return;
|
||||
}
|
||||
if (node.isSetMetadata()) {
|
||||
this.nodeid = (int) node.getMetadata();
|
||||
}
|
||||
Preconditions.checkState(timestamp == -1 || node.timestamp != -1);
|
||||
// update this level's counters
|
||||
if (node.counters != null) {
|
||||
for (TCounter tcounter : node.counters) {
|
||||
Counter counter = counterMap.get(tcounter.name);
|
||||
if (counter == null) {
|
||||
counterMap.put(tcounter.name, new Counter(tcounter.type, tcounter.value));
|
||||
counterMap.put(tcounter.name, new Counter(tcounter.type, tcounter.value, tcounter.level));
|
||||
} else {
|
||||
counter.setLevel(tcounter.level);
|
||||
if (counter.getType() != tcounter.type) {
|
||||
LOG.error("Cannot update counters with the same name but different types"
|
||||
+ " type=" + tcounter.type);
|
||||
@ -188,8 +201,7 @@ public class RuntimeProfile {
|
||||
|
||||
if (node.child_counters_map != null) {
|
||||
// update childCounters
|
||||
for (Map.Entry<String, Set<String>> entry :
|
||||
node.child_counters_map.entrySet()) {
|
||||
for (Map.Entry<String, Set<String>> entry : node.child_counters_map.entrySet()) {
|
||||
String parentCounterName = entry.getKey();
|
||||
|
||||
counterLock.writeLock().lock();
|
||||
@ -343,43 +355,36 @@ public class RuntimeProfile {
|
||||
}
|
||||
}
|
||||
|
||||
public void simpleProfile(int depth) {
|
||||
public void simpleProfile(int depth, int childIdx, ProfileStatistics statistics) {
|
||||
if (depth == FRAGMENT_DEPTH) {
|
||||
mergeMutiInstance(childList);
|
||||
statistics.setFragmentId(childIdx);
|
||||
mergeMutiInstance(childList, statistics);
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < childList.size(); i++) {
|
||||
Pair<RuntimeProfile, Boolean> pair = childList.get(i);
|
||||
RuntimeProfile profile = pair.first;
|
||||
profile.simpleProfile(depth + 1);
|
||||
profile.simpleProfile(depth + 1, i, statistics);
|
||||
}
|
||||
}
|
||||
|
||||
private static void mergeMutiInstance(
|
||||
LinkedList<Pair<RuntimeProfile, Boolean>> childList) {
|
||||
/*
|
||||
* Fragment 1: Fragment 1:
|
||||
* Instance 0 Instance (total)
|
||||
* Instance 1
|
||||
* Instance 2
|
||||
*/
|
||||
int numInstance = childList.size();
|
||||
LinkedList<Pair<RuntimeProfile, Boolean>> childList, ProfileStatistics statistics) {
|
||||
Pair<RuntimeProfile, Boolean> pair = childList.get(0);
|
||||
RuntimeProfile mergedProfile = pair.first;
|
||||
LinkedList<RuntimeProfile> other = new LinkedList<RuntimeProfile>();
|
||||
for (int i = 1; i < childList.size(); i++) {
|
||||
other.add(childList.get(i).first);
|
||||
}
|
||||
mergeInstanceProfile(mergedProfile, other);
|
||||
childList.clear();
|
||||
mergedProfile.name = "Instance " + "(" + numInstance + ")";
|
||||
childList.add(Pair.of(mergedProfile, pair.second));
|
||||
mergeInstanceProfile(mergedProfile, other, statistics);
|
||||
}
|
||||
|
||||
private static LinkedList<RuntimeProfile> getChildListFromLists(int idx, LinkedList<RuntimeProfile> rhs) {
|
||||
LinkedList<RuntimeProfile> ret = new LinkedList<RuntimeProfile>();
|
||||
for (RuntimeProfile profile : rhs) {
|
||||
ret.add(profile.childList.get(idx).first);
|
||||
if (idx < profile.childList.size()) {
|
||||
ret.add(profile.childList.get(idx).first);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@ -392,40 +397,36 @@ public class RuntimeProfile {
|
||||
return ret;
|
||||
}
|
||||
|
||||
private static void mergeInstanceProfile(RuntimeProfile src, LinkedList<RuntimeProfile> rhs) {
|
||||
mergeProfileCounter(src, ROOT_COUNTER, rhs);
|
||||
mergeTotalTime(src, rhs);
|
||||
mergeProfileInfoStr(src, rhs);
|
||||
removePipelineContext(src);
|
||||
private static void mergeInstanceProfile(RuntimeProfile src, LinkedList<RuntimeProfile> rhs,
|
||||
ProfileStatistics statistics) {
|
||||
// data sink
|
||||
// plan node
|
||||
// other
|
||||
for (int i = 0; i < src.childList.size(); i++) {
|
||||
RuntimeProfile srcChild = src.childList.get(i).first;
|
||||
LinkedList<RuntimeProfile> rhsChild = getChildListFromLists(i, rhs);
|
||||
mergeInstanceProfile(srcChild, rhsChild);
|
||||
}
|
||||
}
|
||||
|
||||
private static void mergeTotalTime(RuntimeProfile src, LinkedList<RuntimeProfile> rhs) {
|
||||
Counter counter = src.counterMap.get("TotalTime");
|
||||
for (RuntimeProfile profile : rhs) {
|
||||
Counter othCounter = profile.counterMap.get("TotalTime");
|
||||
if (othCounter != null && counter != null) {
|
||||
counter.addValue(othCounter);
|
||||
if (i == 0) {
|
||||
statistics.setIsDataSink(true);
|
||||
} else {
|
||||
statistics.setIsDataSink(false);
|
||||
}
|
||||
mergePlanNodeProfile(srcChild, rhsChild, statistics);
|
||||
}
|
||||
}
|
||||
|
||||
private static void removePipelineContext(RuntimeProfile src) {
|
||||
LinkedList<Pair<RuntimeProfile, Boolean>> newChildList = new LinkedList<Pair<RuntimeProfile, Boolean>>();
|
||||
for (Pair<RuntimeProfile, Boolean> pair : src.childList) {
|
||||
RuntimeProfile profile = pair.first;
|
||||
if (!profile.name.equals("PipelineContext")) {
|
||||
newChildList.add(pair);
|
||||
}
|
||||
private static void mergePlanNodeProfile(RuntimeProfile src, LinkedList<RuntimeProfile> rhs,
|
||||
ProfileStatistics statistics) {
|
||||
mergeTotalTime(src, rhs, statistics);
|
||||
mergeProfileCounter(src, ROOT_COUNTER, rhs, statistics);
|
||||
for (int i = 0; i < src.childList.size(); i++) {
|
||||
RuntimeProfile srcChild = src.childList.get(i).first;
|
||||
LinkedList<RuntimeProfile> rhsChild = getChildListFromLists(i, rhs);
|
||||
mergePlanNodeProfile(srcChild, rhsChild, statistics);
|
||||
}
|
||||
src.childList = newChildList;
|
||||
}
|
||||
|
||||
private static void mergeProfileCounter(RuntimeProfile src, String counterName, LinkedList<RuntimeProfile> rhs) {
|
||||
private static void mergeProfileCounter(RuntimeProfile src, String counterName, LinkedList<RuntimeProfile> rhs,
|
||||
ProfileStatistics statistics) {
|
||||
Set<String> childCounterSet = src.childCounterMap.get(counterName);
|
||||
if (childCounterSet == null) {
|
||||
return;
|
||||
@ -433,49 +434,35 @@ public class RuntimeProfile {
|
||||
List<String> childCounterList = new LinkedList<>(childCounterSet);
|
||||
for (String childCounterName : childCounterList) {
|
||||
Counter counter = src.counterMap.get(childCounterName);
|
||||
LinkedList<Counter> rhsCounter = getCounterListFromLists(childCounterName, rhs);
|
||||
|
||||
mergeProfileCounter(src, childCounterName, rhs);
|
||||
mergeCounter(src, childCounterName, counter, rhsCounter);
|
||||
removeCounter(childCounterSet, childCounterName, counter);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private static void mergeProfileInfoStr(RuntimeProfile src, LinkedList<RuntimeProfile> rhs) {
|
||||
for (String key : src.infoStringsDisplayOrder) {
|
||||
Set<String> strList = new TreeSet<String>();
|
||||
strList.add(src.infoStrings.get(key));
|
||||
for (RuntimeProfile profile : rhs) {
|
||||
String value = profile.infoStrings.get(key);
|
||||
if (value != null) {
|
||||
strList.add(value);
|
||||
}
|
||||
}
|
||||
try {
|
||||
String joinedString = String.join(" | ", strList);
|
||||
src.infoStrings.put(key, joinedString);
|
||||
} catch (Exception e) {
|
||||
return;
|
||||
mergeProfileCounter(src, childCounterName, rhs, statistics);
|
||||
if (counter.getLevel() == 1) {
|
||||
LinkedList<Counter> rhsCounter = getCounterListFromLists(childCounterName, rhs);
|
||||
mergeCounter(src, childCounterName, counter, rhsCounter, statistics);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void removeCounter(Set<String> childCounterSet, String childCounterName, Counter counter) {
|
||||
if (counter.isRemove()) {
|
||||
childCounterSet.remove(childCounterName);
|
||||
private static void mergeTotalTime(RuntimeProfile src, LinkedList<RuntimeProfile> rhs,
|
||||
ProfileStatistics statistics) {
|
||||
String counterName = "TotalTime";
|
||||
Counter counter = src.counterMap.get(counterName);
|
||||
if (counter == null) {
|
||||
return;
|
||||
}
|
||||
LinkedList<Counter> rhsCounter = getCounterListFromLists(counterName, rhs);
|
||||
mergeCounter(src, counterName, counter, rhsCounter, statistics);
|
||||
}
|
||||
|
||||
private static void mergeCounter(RuntimeProfile src, String counterName, Counter counter,
|
||||
LinkedList<Counter> rhsCounter) {
|
||||
LinkedList<Counter> rhsCounter, ProfileStatistics statistics) {
|
||||
if (counter.getLevel() != 1) {
|
||||
return;
|
||||
}
|
||||
if (rhsCounter == null) {
|
||||
return;
|
||||
}
|
||||
if (rhsCounter.size() == 0) {
|
||||
return;
|
||||
}
|
||||
if (counter.isTimeType()) {
|
||||
Counter newCounter = new Counter(counter.getType(), counter.getValue());
|
||||
Counter maxCounter = new Counter(counter.getType(), counter.getValue());
|
||||
Counter minCounter = new Counter(counter.getType(), counter.getValue());
|
||||
for (Counter cnt : rhsCounter) {
|
||||
@ -490,40 +477,27 @@ public class RuntimeProfile {
|
||||
}
|
||||
for (Counter cnt : rhsCounter) {
|
||||
if (cnt != null) {
|
||||
counter.addValue(cnt);
|
||||
newCounter.addValue(cnt);
|
||||
}
|
||||
}
|
||||
long countNumber = rhsCounter.size() + 1;
|
||||
counter.divValue(countNumber);
|
||||
String maxCounterName = MAX_TIME_PRE + counterName;
|
||||
String minCounterName = MIN_TIME_PRE + counterName;
|
||||
src.counterMap.put(minCounterName, minCounter);
|
||||
src.counterMap.put(maxCounterName, maxCounter);
|
||||
TreeSet<String> childCounterSet = src.childCounterMap.get(counterName);
|
||||
if (childCounterSet == null) {
|
||||
src.childCounterMap.put(counterName, new TreeSet<String>());
|
||||
childCounterSet = src.childCounterMap.get(counterName);
|
||||
if (newCounter.getValue() > 0) {
|
||||
newCounter.divValue(countNumber);
|
||||
String infoString = counterName + ": "
|
||||
+ AVG_TIME_PRE + printCounter(newCounter.getValue(), newCounter.getType()) + ", "
|
||||
+ MAX_TIME_PRE + printCounter(maxCounter.getValue(), maxCounter.getType()) + ", "
|
||||
+ MIN_TIME_PRE + printCounter(minCounter.getValue(), minCounter.getType());
|
||||
statistics.addInfoFromProfile(src, infoString);
|
||||
}
|
||||
childCounterSet.add(minCounterName);
|
||||
childCounterSet.add(maxCounterName);
|
||||
if (counter.getValue() > 0) {
|
||||
src.infoStringsDisplayOrder.add(counterName);
|
||||
String infoString = "[ "
|
||||
+ AVG_TIME_PRE + printCounter(counter.getValue(), counter.getType()) + " , "
|
||||
+ MAX_TIME_PRE + printCounter(maxCounter.getValue(), maxCounter.getType()) + " , "
|
||||
+ MIN_TIME_PRE + printCounter(minCounter.getValue(), minCounter.getType()) + " ]";
|
||||
src.infoStrings.put(counterName, infoString);
|
||||
}
|
||||
counter.setCanRemove(); // value will remove in removeCounter
|
||||
} else {
|
||||
if (rhsCounter.size() == 0) {
|
||||
return;
|
||||
}
|
||||
Counter newCounter = new Counter(counter.getType(), counter.getValue());
|
||||
for (Counter cnt : rhsCounter) {
|
||||
if (cnt != null) {
|
||||
counter.addValue(cnt);
|
||||
newCounter.addValue(cnt);
|
||||
}
|
||||
}
|
||||
String infoString = counterName + ": " + printCounter(newCounter.getValue(), newCounter.getType());
|
||||
statistics.addInfoFromProfile(src, infoString);
|
||||
}
|
||||
}
|
||||
|
||||
@ -533,14 +507,19 @@ public class RuntimeProfile {
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String getSimpleString() {
|
||||
if (!this.enableSimplyProfile) {
|
||||
public String getProfileByLevel() {
|
||||
if (this.profileLevel == 3) {
|
||||
return toString();
|
||||
}
|
||||
if (this.planner == null) {
|
||||
return toString();
|
||||
}
|
||||
StringBuilder builder = new StringBuilder();
|
||||
simpleProfile(0);
|
||||
prettyPrint(builder, "");
|
||||
return builder.toString();
|
||||
ProfileStatistics statistics = new ProfileStatistics();
|
||||
simpleProfile(0, 0, statistics);
|
||||
String planerStr = this.planner.getExplainStringToProfile(statistics);
|
||||
return "Simple profile \n \n " + planerStr + "\n \n \n" + builder.toString();
|
||||
}
|
||||
|
||||
private void printChildCounters(String prefix, String counterName, StringBuilder builder) {
|
||||
@ -678,7 +657,8 @@ public class RuntimeProfile {
|
||||
}
|
||||
}
|
||||
|
||||
// Because the profile of summary and child fragment is not a real parent-child relationship
|
||||
// Because the profile of summary and child fragment is not a real parent-child
|
||||
// relationship
|
||||
// Each child profile needs to calculate the time proportion consumed by itself
|
||||
public void computeTimeInChildProfile() {
|
||||
childMap.values().forEach(RuntimeProfile::computeTimeInProfile);
|
||||
@ -688,8 +668,12 @@ public class RuntimeProfile {
|
||||
computeTimeInProfile(this.counterTotalTime.getValue());
|
||||
}
|
||||
|
||||
public void setProfileLevel(boolean isSimpleProfile) {
|
||||
this.enableSimplyProfile = isSimpleProfile;
|
||||
public void setProfileLevel(int profileLevel) {
|
||||
this.profileLevel = profileLevel;
|
||||
}
|
||||
|
||||
public void setPlaner(Planner planner) {
|
||||
this.planner = planner;
|
||||
}
|
||||
|
||||
private void computeTimeInProfile(long total) {
|
||||
@ -727,8 +711,10 @@ public class RuntimeProfile {
|
||||
this.childList.sort((profile1, profile2) -> Long.compare(profile2.first.getCounterTotalTime().getValue(),
|
||||
profile1.first.getCounterTotalTime().getValue()));
|
||||
} catch (IllegalArgumentException e) {
|
||||
// This exception may be thrown if the counter total time of the child is updated in the update method
|
||||
// during the sorting process. This sorting only affects the profile instance display order, so this
|
||||
// This exception may be thrown if the counter total time of the child is
|
||||
// updated in the update method
|
||||
// during the sorting process. This sorting only affects the profile instance
|
||||
// display order, so this
|
||||
// exception is temporarily ignored here.
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("sort child list error: ", e);
|
||||
@ -765,4 +751,3 @@ public class RuntimeProfile {
|
||||
return infoStrings;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -323,7 +323,7 @@ public class BrokerLoadJob extends BulkLoadJob {
|
||||
return;
|
||||
}
|
||||
jobProfile.update(createTimestamp, getSummaryInfo(true), true,
|
||||
Boolean.valueOf(sessionVariables.getOrDefault(SessionVariable.ENABLE_SIMPLY_PROFILE, "true")));
|
||||
Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.PROFILE_LEVEL, "3")), null);
|
||||
}
|
||||
|
||||
private Map<String, String> getSummaryInfo(boolean isFinished) {
|
||||
|
||||
@ -24,6 +24,7 @@ import org.apache.doris.catalog.MysqlTable;
|
||||
import org.apache.doris.catalog.OdbcTable;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.util.ProfileStatistics;
|
||||
import org.apache.doris.planner.external.odbc.OdbcTableSink;
|
||||
import org.apache.doris.thrift.TDataSink;
|
||||
import org.apache.doris.thrift.TExplainLevel;
|
||||
@ -39,7 +40,8 @@ public abstract class DataSink {
|
||||
protected PlanFragment fragment;
|
||||
|
||||
/**
|
||||
* Return an explain string for the DataSink. Each line of the explain will be prefixed
|
||||
* Return an explain string for the DataSink. Each line of the explain will be
|
||||
* prefixed
|
||||
* by "prefix"
|
||||
*
|
||||
* @param prefix each explain line will be started with the given prefix
|
||||
@ -47,6 +49,14 @@ public abstract class DataSink {
|
||||
*/
|
||||
public abstract String getExplainString(String prefix, TExplainLevel explainLevel);
|
||||
|
||||
public String getExplainStringToProfile(String prefix, TExplainLevel explainLevel, ProfileStatistics statistics,
|
||||
int fragmentIdx) {
|
||||
String dataSinkString = getExplainString(prefix, explainLevel);
|
||||
StringBuilder expBuilder = new StringBuilder();
|
||||
statistics.getDataSinkInfo(fragmentIdx, prefix + " ", expBuilder);
|
||||
return dataSinkString + "\n" + expBuilder.toString();
|
||||
}
|
||||
|
||||
protected abstract TDataSink toThrift();
|
||||
|
||||
public void setFragment(PlanFragment fragment) {
|
||||
|
||||
@ -27,6 +27,7 @@ import org.apache.doris.analysis.SlotRef;
|
||||
import org.apache.doris.analysis.StatementBase;
|
||||
import org.apache.doris.analysis.TupleDescriptor;
|
||||
import org.apache.doris.common.TreeNode;
|
||||
import org.apache.doris.common.util.ProfileStatistics;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.thrift.TExplainLevel;
|
||||
import org.apache.doris.thrift.TPartitionType;
|
||||
@ -334,6 +335,24 @@ public class PlanFragment extends TreeNode<PlanFragment> {
|
||||
return str.toString();
|
||||
}
|
||||
|
||||
public String getExplainStringToProfile(TExplainLevel explainLevel, ProfileStatistics statistics, int fragmentIdx) {
|
||||
StringBuilder str = new StringBuilder();
|
||||
Preconditions.checkState(dataPartition != null);
|
||||
if (CollectionUtils.isNotEmpty(outputExprs)) {
|
||||
str.append(" OUTPUT EXPRS:\n ");
|
||||
str.append(outputExprs.stream().map(Expr::toSql).collect(Collectors.joining("\n ")));
|
||||
}
|
||||
str.append("\n");
|
||||
str.append(" PARTITION: " + dataPartition.getExplainString(explainLevel) + "\n");
|
||||
if (sink != null) {
|
||||
str.append(sink.getExplainStringToProfile(" ", explainLevel, statistics, fragmentIdx) + "\n");
|
||||
}
|
||||
if (planRoot != null) {
|
||||
str.append(planRoot.getExplainStringToProfile(" ", " ", explainLevel, statistics));
|
||||
}
|
||||
return str.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if this fragment is partitioned.
|
||||
*/
|
||||
|
||||
@ -41,6 +41,7 @@ import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.NotImplementedException;
|
||||
import org.apache.doris.common.TreeNode;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.doris.common.util.ProfileStatistics;
|
||||
import org.apache.doris.statistics.PlanStats;
|
||||
import org.apache.doris.statistics.StatisticalType;
|
||||
import org.apache.doris.statistics.StatsDeriveResult;
|
||||
@ -574,6 +575,62 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
|
||||
return expBuilder.toString();
|
||||
}
|
||||
|
||||
protected final String getExplainStringToProfile(String rootPrefix, String prefix, TExplainLevel detailLevel,
|
||||
ProfileStatistics statistics) {
|
||||
StringBuilder expBuilder = new StringBuilder();
|
||||
String detailPrefix = prefix;
|
||||
boolean traverseChildren = children != null
|
||||
&& children.size() > 0
|
||||
&& !(this instanceof ExchangeNode);
|
||||
// if (children != null && children.size() > 0) {
|
||||
if (traverseChildren) {
|
||||
detailPrefix += "| ";
|
||||
} else {
|
||||
detailPrefix += " ";
|
||||
}
|
||||
|
||||
// Print the current node
|
||||
// The plan node header line will be prefixed by rootPrefix and the remaining
|
||||
// details
|
||||
// will be prefixed by detailPrefix.
|
||||
expBuilder.append(rootPrefix + id.asInt() + ":" + planNodeName + "\n");
|
||||
expBuilder.append(getNodeExplainString(detailPrefix, detailLevel));
|
||||
statistics.getInfoById(id.asInt(), detailPrefix, expBuilder);
|
||||
if (limit != -1) {
|
||||
expBuilder.append(detailPrefix + "limit: " + limit + "\n");
|
||||
}
|
||||
if (!CollectionUtils.isEmpty(projectList)) {
|
||||
expBuilder.append(detailPrefix).append("projections: ").append(getExplainString(projectList)).append("\n");
|
||||
expBuilder.append(detailPrefix).append("project output tuple id: ")
|
||||
.append(outputTupleDesc.getId().asInt()).append("\n");
|
||||
}
|
||||
// Output Tuple Ids only when explain plan level is set to verbose
|
||||
if (detailLevel.equals(TExplainLevel.VERBOSE)) {
|
||||
expBuilder.append(detailPrefix + "tuple ids: ");
|
||||
for (TupleId tupleId : tupleIds) {
|
||||
String nullIndicator = nullableTupleIds.contains(tupleId) ? "N" : "";
|
||||
expBuilder.append(tupleId.asInt() + nullIndicator + " ");
|
||||
}
|
||||
expBuilder.append("\n");
|
||||
}
|
||||
|
||||
// Print the children
|
||||
// if (children != null && children.size() > 0) {
|
||||
if (traverseChildren) {
|
||||
expBuilder.append(detailPrefix + "\n");
|
||||
String childHeadlinePrefix = prefix + "|----";
|
||||
String childDetailPrefix = prefix + "| ";
|
||||
for (int i = 1; i < children.size(); ++i) {
|
||||
expBuilder.append(
|
||||
children.get(i).getExplainStringToProfile(childHeadlinePrefix, childDetailPrefix,
|
||||
detailLevel, statistics));
|
||||
expBuilder.append(childDetailPrefix + "\n");
|
||||
}
|
||||
expBuilder.append(children.get(0).getExplainStringToProfile(prefix, prefix, detailLevel, statistics));
|
||||
}
|
||||
return expBuilder.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the node-specific details.
|
||||
* Subclass should override this function.
|
||||
|
||||
@ -29,6 +29,7 @@ import org.apache.doris.common.UserException;
|
||||
import org.apache.doris.common.profile.PlanTreeBuilder;
|
||||
import org.apache.doris.common.profile.PlanTreePrinter;
|
||||
import org.apache.doris.common.util.LiteralUtils;
|
||||
import org.apache.doris.common.util.ProfileStatistics;
|
||||
import org.apache.doris.qe.ResultSet;
|
||||
import org.apache.doris.thrift.TQueryOptions;
|
||||
|
||||
@ -89,6 +90,21 @@ public abstract class Planner {
|
||||
return str.toString();
|
||||
}
|
||||
|
||||
public String getExplainStringToProfile(ProfileStatistics statistics) {
|
||||
org.apache.doris.thrift.TExplainLevel explainLevel = org.apache.doris.thrift.TExplainLevel.NORMAL;
|
||||
StringBuilder str = new StringBuilder();
|
||||
for (int i = 0; i < fragments.size(); ++i) {
|
||||
PlanFragment fragment = fragments.get(i);
|
||||
if (i > 0) {
|
||||
// a blank line between plan fragments
|
||||
str.append("\n");
|
||||
}
|
||||
str.append("PLAN FRAGMENT " + i + "\n");
|
||||
str.append(fragment.getExplainStringToProfile(explainLevel, statistics, i));
|
||||
}
|
||||
return str.toString();
|
||||
}
|
||||
|
||||
protected void handleLiteralInFe(LiteralExpr literalExpr, List<String> data) {
|
||||
if (literalExpr instanceof NullLiteral) {
|
||||
data.add(null);
|
||||
|
||||
@ -112,7 +112,7 @@ public class SessionVariable implements Serializable, Writable {
|
||||
public static final String ENABLE_BUCKET_SHUFFLE_JOIN = "enable_bucket_shuffle_join";
|
||||
public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM = "parallel_fragment_exec_instance_num";
|
||||
public static final String PARALLEL_PIPELINE_TASK_NUM = "parallel_pipeline_task_num";
|
||||
public static final String ENABLE_SIMPLY_PROFILE = "enable_simply_profile";
|
||||
public static final String PROFILE_LEVEL = "profile_level";
|
||||
public static final String MAX_INSTANCE_NUM = "max_instance_num";
|
||||
public static final String ENABLE_INSERT_STRICT = "enable_insert_strict";
|
||||
public static final String ENABLE_SPILLING = "enable_spilling";
|
||||
@ -623,8 +623,8 @@ public class SessionVariable implements Serializable, Writable {
|
||||
@VariableMgr.VarAttr(name = PARALLEL_PIPELINE_TASK_NUM, fuzzy = true, needForward = true)
|
||||
public int parallelPipelineTaskNum = 0;
|
||||
|
||||
@VariableMgr.VarAttr(name = ENABLE_SIMPLY_PROFILE, fuzzy = true)
|
||||
public boolean enableSimplyProfile = true;
|
||||
@VariableMgr.VarAttr(name = PROFILE_LEVEL, fuzzy = true)
|
||||
public int profileLevel = 3;
|
||||
|
||||
@VariableMgr.VarAttr(name = MAX_INSTANCE_NUM)
|
||||
public int maxInstanceNum = 64;
|
||||
@ -2676,7 +2676,7 @@ public class SessionVariable implements Serializable, Writable {
|
||||
}
|
||||
}
|
||||
|
||||
public boolean getEnableSimplyProfile() {
|
||||
return this.enableSimplyProfile;
|
||||
public int getProfileLevel() {
|
||||
return this.profileLevel;
|
||||
}
|
||||
}
|
||||
|
||||
@ -885,8 +885,9 @@ public class StmtExecutor {
|
||||
if (!context.getSessionVariable().enableProfile()) {
|
||||
return;
|
||||
}
|
||||
|
||||
profile.update(context.startTime, getSummaryInfo(isFinished), isFinished,
|
||||
context.getSessionVariable().enableSimplyProfile);
|
||||
context.getSessionVariable().profileLevel, this.planner);
|
||||
}
|
||||
|
||||
// Analyze one statement to structure in memory.
|
||||
|
||||
Reference in New Issue
Block a user