[refactor](profile) remove unused code in profile (#26404)

This commit is contained in:
Mryange
2023-11-06 18:13:37 +08:00
committed by GitHub
parent 0057ec36a3
commit e42cb88c2d
6 changed files with 0 additions and 430 deletions

View File

@ -1,202 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.util;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
/**
* Used for collecting information obtained from the profile.
*/
public class ProfileStatistics {
class PipelineXStatistics {
ArrayList<String> sinkInfos;
ArrayList<String> infos;
int sinkInstance = 0;
int operatorInstance = 0;
PipelineXStatistics() {
sinkInfos = new ArrayList<String>();
infos = new ArrayList<String>();
}
void add(boolean isSink, String str) {
if (isSink) {
sinkInfos.add(str);
} else {
infos.add(str);
}
}
void updateInstance(boolean isSink, int instance) {
if (isSink) {
sinkInstance = Math.max(sinkInstance, instance);
} else {
operatorInstance = Math.max(operatorInstance, instance);
}
}
String to_String() {
return null;
}
void getInfo(ArrayList<String> infos, String prefix, StringBuilder str) {
Collections.sort(infos);
for (String info : infos) {
str.append(prefix + info + "\n");
}
}
void getOperator(String prefix, StringBuilder str) {
str.append(prefix + "Instance num : " + operatorInstance + "\n");
getInfo(infos, prefix, str);
}
void getSink(String prefix, StringBuilder str) {
str.append(prefix + "Instance num : " + sinkInstance + "\n");
getInfo(sinkInfos, prefix, str);
}
}
// Record statistical information based on nodeid.
private HashMap<Integer, ArrayList<String>> statisticalInfo;
private HashMap<Integer, Integer> statisticalInfoInstance;
// Record statistical information based on nodeid(use in pipelineX).
private HashMap<Integer, PipelineXStatistics> pipelineXStatisticalInfo;
// Record statistical information based on fragment ID.
// "Currently used to record sink nodes.
private HashMap<Integer, ArrayList<String>> fragmentInfo;
private HashMap<Integer, Integer> fragmentInfoInstance;
private int fragmentId;
private boolean isDataSink;
private boolean isPipelineX;
public ProfileStatistics(boolean isPipelineX) {
statisticalInfo = new HashMap<Integer, ArrayList<String>>();
statisticalInfoInstance = new HashMap<Integer, Integer>();
fragmentInfo = new HashMap<Integer, ArrayList<String>>();
pipelineXStatisticalInfo = new HashMap<Integer, PipelineXStatistics>();
fragmentInfoInstance = new HashMap<Integer, Integer>();
fragmentId = 0;
isDataSink = false;
this.isPipelineX = isPipelineX;
}
private void addPipelineXPlanNodeInfo(boolean isSink, int id, String info, int instance) {
if (!pipelineXStatisticalInfo.containsKey(id)) {
pipelineXStatisticalInfo.put(id, new PipelineXStatistics());
}
pipelineXStatisticalInfo.get(id).add(isSink, info);
pipelineXStatisticalInfo.get(id).updateInstance(isSink, instance);
}
private void addPlanNodeInfo(int id, String info, int instance) {
if (!statisticalInfo.containsKey(id)) {
statisticalInfo.put(id, new ArrayList<String>());
statisticalInfoInstance.put(id, new Integer(0));
}
statisticalInfo.get(id).add(info);
int ins = statisticalInfoInstance.get(id);
ins = Math.max(ins, instance);
statisticalInfoInstance.put(id, ins);
}
private void addDataSinkInfo(String info, int instance) {
if (fragmentInfo.get(fragmentId) == null) {
fragmentInfo.put(fragmentId, new ArrayList<String>());
fragmentInfoInstance.put(fragmentId, new Integer(0));
}
fragmentInfo.get(fragmentId).add(info);
int ins = fragmentInfoInstance.get(fragmentId);
ins = Math.max(ins, instance);
fragmentInfoInstance.put(fragmentId, ins);
}
public void addInfoFromProfile(RuntimeProfile profile, String name, String info, int instance) {
if (isPipelineX) {
addPipelineXPlanNodeInfo(profile.sinkOperator(), profile.nodeId(), name + ": " + info, instance);
} else {
if (isDataSink) {
addDataSinkInfo(name + ": " + info, instance);
} else {
addPlanNodeInfo(profile.nodeId(), name + ": " + info, instance);
}
}
}
public boolean hasInfo(int id) {
if (isPipelineX) {
return pipelineXStatisticalInfo.containsKey(id);
} else {
return statisticalInfo.containsKey(id);
}
}
public void getInfoById(int id, String prefix, StringBuilder str) {
if (!hasInfo(id)) {
return;
}
if (isPipelineX) {
getPipelineXInfoById(id, prefix, str);
} else {
ArrayList<String> infos = statisticalInfo.get(id);
str.append(prefix + "Instance num :" + statisticalInfoInstance.get(id) + "\n");
Collections.sort(infos);
for (String info : infos) {
str.append(prefix + info + "\n");
}
}
}
private void getPipelineXInfoById(int id, String prefix, StringBuilder str) {
PipelineXStatistics statistics = pipelineXStatisticalInfo.get(id);
str.append(prefix + "Operator: \n");
statistics.getOperator(prefix + " ", str);
str.append(prefix + "Sink: \n");
statistics.getSink(prefix + " ", str);
}
public void getDataSinkInfo(int fragmentIdx, String prefix, StringBuilder str) {
if (!fragmentInfo.containsKey(fragmentIdx)) {
return;
}
str.append(prefix + "Instance num :" + fragmentInfoInstance.get(fragmentIdx) + "\n");
ArrayList<String> infos = fragmentInfo.get(fragmentIdx);
Collections.sort(infos);
for (String info : infos) {
str.append(prefix + info + "\n");
}
}
public void setFragmentId(int id) {
this.fragmentId = id;
}
public void setIsDataSink(boolean dataSink) {
this.isDataSink = dataSink;
}
}

View File

@ -449,30 +449,6 @@ public class RuntimeProfile {
}
}
public void simpleProfile(int depth, int childIdx, ProfileStatistics statistics) {
if (depth == FRAGMENT_DEPTH) {
statistics.setFragmentId(childIdx);
mergeMutiInstance(childList, statistics);
return;
}
for (int i = 0; i < childList.size(); i++) {
Pair<RuntimeProfile, Boolean> pair = childList.get(i);
RuntimeProfile profile = pair.first;
profile.simpleProfile(depth + 1, i, statistics);
}
}
private static void mergeMutiInstance(
LinkedList<Pair<RuntimeProfile, Boolean>> childList, ProfileStatistics statistics) {
Pair<RuntimeProfile, Boolean> pair = childList.get(0);
RuntimeProfile mergedProfile = pair.first;
LinkedList<RuntimeProfile> other = new LinkedList<RuntimeProfile>();
for (int i = 1; i < childList.size(); i++) {
other.add(childList.get(i).first);
}
mergeInstanceProfile(mergedProfile, other, statistics);
}
private static LinkedList<RuntimeProfile> getChildListFromLists(int idx, LinkedList<RuntimeProfile> rhs) {
LinkedList<RuntimeProfile> ret = new LinkedList<RuntimeProfile>();
for (RuntimeProfile profile : rhs) {
@ -491,109 +467,6 @@ public class RuntimeProfile {
return ret;
}
private static void mergeInstanceProfile(RuntimeProfile src, LinkedList<RuntimeProfile> rhs,
ProfileStatistics statistics) {
// data sink
// plan node
// other
for (int i = 0; i < src.childList.size(); i++) {
RuntimeProfile srcChild = src.childList.get(i).first;
LinkedList<RuntimeProfile> rhsChild = getChildListFromLists(i, rhs);
if (i == 0) {
statistics.setIsDataSink(true);
} else {
statistics.setIsDataSink(false);
}
mergePlanNodeProfile(srcChild, rhsChild, statistics);
}
}
private static void mergePlanNodeProfile(RuntimeProfile src, LinkedList<RuntimeProfile> rhs,
ProfileStatistics statistics) {
mergeTotalTime(src, rhs, statistics);
mergeProfileCounter(src, ROOT_COUNTER, rhs, statistics);
for (int i = 0; i < src.childList.size(); i++) {
RuntimeProfile srcChild = src.childList.get(i).first;
LinkedList<RuntimeProfile> rhsChild = getChildListFromLists(i, rhs);
mergePlanNodeProfile(srcChild, rhsChild, statistics);
}
}
private static void mergeProfileCounter(RuntimeProfile src, String counterName, LinkedList<RuntimeProfile> rhs,
ProfileStatistics statistics) {
Set<String> childCounterSet = src.childCounterMap.get(counterName);
if (childCounterSet == null) {
return;
}
List<String> childCounterList = new LinkedList<>(childCounterSet);
for (String childCounterName : childCounterList) {
Counter counter = src.counterMap.get(childCounterName);
mergeProfileCounter(src, childCounterName, rhs, statistics);
if (counter.getLevel() == 1) {
LinkedList<Counter> rhsCounter = getCounterListFromLists(childCounterName, rhs);
mergeCounter(src, childCounterName, counter, rhsCounter, statistics);
}
}
}
private static void mergeTotalTime(RuntimeProfile src, LinkedList<RuntimeProfile> rhs,
ProfileStatistics statistics) {
String counterName = "TotalTime";
Counter counter = src.counterMap.get(counterName);
if (counter == null) {
return;
}
LinkedList<Counter> rhsCounter = getCounterListFromLists(counterName, rhs);
mergeCounter(src, counterName, counter, rhsCounter, statistics);
}
private static void mergeCounter(RuntimeProfile src, String counterName, Counter counter,
LinkedList<Counter> rhsCounter, ProfileStatistics statistics) {
if (counter.getLevel() != 1) {
return;
}
if (rhsCounter == null) {
return;
}
if (counter.isTimeType()) {
Counter newCounter = new Counter(counter.getType(), counter.getValue());
Counter maxCounter = new Counter(counter.getType(), counter.getValue());
Counter minCounter = new Counter(counter.getType(), counter.getValue());
for (Counter cnt : rhsCounter) {
if (cnt != null) {
if (cnt.getValue() > maxCounter.getValue()) {
maxCounter.setValue(cnt.getValue());
}
if (cnt.getValue() < minCounter.getValue()) {
minCounter.setValue(cnt.getValue());
}
}
}
for (Counter cnt : rhsCounter) {
if (cnt != null) {
newCounter.addValue(cnt);
}
}
long countNumber = rhsCounter.size() + 1;
if (newCounter.getValue() > 0) {
newCounter.divValue(countNumber);
String infoString = AVG_TIME_PRE + printCounter(newCounter.getValue(), newCounter.getType()) + ", "
+ MAX_TIME_PRE + printCounter(maxCounter.getValue(), maxCounter.getType()) + ", "
+ MIN_TIME_PRE + printCounter(minCounter.getValue(), minCounter.getType());
statistics.addInfoFromProfile(src, counterName, infoString, rhsCounter.size() + 1);
}
} else {
Counter newCounter = new Counter(counter.getType(), counter.getValue());
for (Counter cnt : rhsCounter) {
if (cnt != null) {
newCounter.addValue(cnt);
}
}
String infoString = printCounter(newCounter.getValue(), newCounter.getType());
statistics.addInfoFromProfile(src, counterName, infoString, rhsCounter.size() + 1);
}
}
public String toString() {
StringBuilder builder = new StringBuilder();
prettyPrint(builder, "", isPipelineX);

View File

@ -24,7 +24,6 @@ import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.util.ProfileStatistics;
import org.apache.doris.planner.external.odbc.OdbcTableSink;
import org.apache.doris.thrift.TDataSink;
import org.apache.doris.thrift.TExplainLevel;
@ -49,14 +48,6 @@ public abstract class DataSink {
*/
public abstract String getExplainString(String prefix, TExplainLevel explainLevel);
public String getExplainStringToProfile(String prefix, TExplainLevel explainLevel, ProfileStatistics statistics,
int fragmentIdx) {
String dataSinkString = getExplainString(prefix, explainLevel);
StringBuilder expBuilder = new StringBuilder();
statistics.getDataSinkInfo(fragmentIdx, prefix + " ", expBuilder);
return dataSinkString + "\n" + expBuilder.toString();
}
protected abstract TDataSink toThrift();
public void setFragment(PlanFragment fragment) {

View File

@ -27,7 +27,6 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.util.ProfileStatistics;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TExplainLevel;
import org.apache.doris.thrift.TPartitionType;
@ -338,24 +337,6 @@ public class PlanFragment extends TreeNode<PlanFragment> {
return str.toString();
}
public String getExplainStringToProfile(TExplainLevel explainLevel, ProfileStatistics statistics, int fragmentIdx) {
StringBuilder str = new StringBuilder();
Preconditions.checkState(dataPartition != null);
if (CollectionUtils.isNotEmpty(outputExprs)) {
str.append(" OUTPUT EXPRS:\n ");
str.append(outputExprs.stream().map(Expr::toSql).collect(Collectors.joining("\n ")));
}
str.append("\n");
str.append(" PARTITION: " + dataPartition.getExplainString(explainLevel) + "\n");
if (sink != null) {
str.append(sink.getExplainStringToProfile(" ", explainLevel, statistics, fragmentIdx) + "\n");
}
if (planRoot != null) {
str.append(planRoot.getExplainStringToProfile(" ", " ", explainLevel, statistics));
}
return str.toString();
}
public void getExplainStringMap(Map<Integer, String> planNodeMap) {
org.apache.doris.thrift.TExplainLevel explainLevel = org.apache.doris.thrift.TExplainLevel.NORMAL;
if (planRoot != null) {

View File

@ -41,7 +41,6 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.TreeNode;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.ProfileStatistics;
import org.apache.doris.statistics.PlanStats;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.statistics.StatsDeriveResult;
@ -576,62 +575,6 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
return expBuilder.toString();
}
protected final String getExplainStringToProfile(String rootPrefix, String prefix, TExplainLevel detailLevel,
ProfileStatistics statistics) {
StringBuilder expBuilder = new StringBuilder();
String detailPrefix = prefix;
boolean traverseChildren = children != null
&& children.size() > 0
&& !(this instanceof ExchangeNode);
// if (children != null && children.size() > 0) {
if (traverseChildren) {
detailPrefix += "| ";
} else {
detailPrefix += " ";
}
// Print the current node
// The plan node header line will be prefixed by rootPrefix and the remaining
// details
// will be prefixed by detailPrefix.
expBuilder.append(rootPrefix + id.asInt() + ":" + planNodeName + "\n");
expBuilder.append(getNodeExplainString(detailPrefix, detailLevel));
statistics.getInfoById(id.asInt(), detailPrefix, expBuilder);
if (limit != -1) {
expBuilder.append(detailPrefix + "limit: " + limit + "\n");
}
if (!CollectionUtils.isEmpty(projectList)) {
expBuilder.append(detailPrefix).append("projections: ").append(getExplainString(projectList)).append("\n");
expBuilder.append(detailPrefix).append("project output tuple id: ")
.append(outputTupleDesc.getId().asInt()).append("\n");
}
// Output Tuple Ids only when explain plan level is set to verbose
if (detailLevel.equals(TExplainLevel.VERBOSE)) {
expBuilder.append(detailPrefix + "tuple ids: ");
for (TupleId tupleId : tupleIds) {
String nullIndicator = nullableTupleIds.contains(tupleId) ? "N" : "";
expBuilder.append(tupleId.asInt() + nullIndicator + " ");
}
expBuilder.append("\n");
}
// Print the children
// if (children != null && children.size() > 0) {
if (traverseChildren) {
expBuilder.append(detailPrefix + "\n");
String childHeadlinePrefix = prefix + "|----";
String childDetailPrefix = prefix + "| ";
for (int i = 1; i < children.size(); ++i) {
expBuilder.append(
children.get(i).getExplainStringToProfile(childHeadlinePrefix, childDetailPrefix,
detailLevel, statistics));
expBuilder.append(childDetailPrefix + "\n");
}
expBuilder.append(children.get(0).getExplainStringToProfile(prefix, prefix, detailLevel, statistics));
}
return expBuilder.toString();
}
private String getplanNodeExplainString(String prefix, TExplainLevel detailLevel) {
StringBuilder expBuilder = new StringBuilder();
expBuilder.append(getNodeExplainString(prefix, detailLevel));

View File

@ -29,7 +29,6 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.profile.PlanTreeBuilder;
import org.apache.doris.common.profile.PlanTreePrinter;
import org.apache.doris.common.util.LiteralUtils;
import org.apache.doris.common.util.ProfileStatistics;
import org.apache.doris.qe.ResultSet;
import org.apache.doris.thrift.TQueryOptions;
@ -103,21 +102,6 @@ public abstract class Planner {
return str.toString();
}
public String getExplainStringToProfile(ProfileStatistics statistics) {
org.apache.doris.thrift.TExplainLevel explainLevel = org.apache.doris.thrift.TExplainLevel.NORMAL;
StringBuilder str = new StringBuilder();
for (int i = 0; i < fragments.size(); ++i) {
PlanFragment fragment = fragments.get(i);
if (i > 0) {
// a blank line between plan fragments
str.append("\n");
}
str.append("PLAN FRAGMENT " + i + "\n");
str.append(fragment.getExplainStringToProfile(explainLevel, statistics, i));
}
return str.toString();
}
public Map<Integer, String> getExplainStringMap() {
Map<Integer, String> planNodeMap = new HashMap<Integer, String>();
for (int i = 0; i < fragments.size(); ++i) {