diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileStatistics.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileStatistics.java deleted file mode 100644 index df4cb0cb01..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileStatistics.java +++ /dev/null @@ -1,202 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.util; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; - -/** - * Used for collecting information obtained from the profile. - */ -public class ProfileStatistics { - class PipelineXStatistics { - ArrayList sinkInfos; - ArrayList infos; - - int sinkInstance = 0; - int operatorInstance = 0; - - PipelineXStatistics() { - sinkInfos = new ArrayList(); - infos = new ArrayList(); - } - - void add(boolean isSink, String str) { - if (isSink) { - sinkInfos.add(str); - } else { - infos.add(str); - } - } - - void updateInstance(boolean isSink, int instance) { - if (isSink) { - sinkInstance = Math.max(sinkInstance, instance); - } else { - operatorInstance = Math.max(operatorInstance, instance); - } - } - - String to_String() { - return null; - } - - void getInfo(ArrayList infos, String prefix, StringBuilder str) { - Collections.sort(infos); - for (String info : infos) { - str.append(prefix + info + "\n"); - } - } - - void getOperator(String prefix, StringBuilder str) { - str.append(prefix + "Instance num : " + operatorInstance + "\n"); - getInfo(infos, prefix, str); - } - - void getSink(String prefix, StringBuilder str) { - str.append(prefix + "Instance num : " + sinkInstance + "\n"); - getInfo(sinkInfos, prefix, str); - } - } - - // Record statistical information based on nodeid. - private HashMap> statisticalInfo; - private HashMap statisticalInfoInstance; - - // Record statistical information based on nodeid(use in pipelineX). - private HashMap pipelineXStatisticalInfo; - - // Record statistical information based on fragment ID. - // "Currently used to record sink nodes. - private HashMap> fragmentInfo; - private HashMap fragmentInfoInstance; - private int fragmentId; - - private boolean isDataSink; - - private boolean isPipelineX; - - public ProfileStatistics(boolean isPipelineX) { - statisticalInfo = new HashMap>(); - statisticalInfoInstance = new HashMap(); - - fragmentInfo = new HashMap>(); - pipelineXStatisticalInfo = new HashMap(); - fragmentInfoInstance = new HashMap(); - fragmentId = 0; - isDataSink = false; - this.isPipelineX = isPipelineX; - } - - private void addPipelineXPlanNodeInfo(boolean isSink, int id, String info, int instance) { - if (!pipelineXStatisticalInfo.containsKey(id)) { - pipelineXStatisticalInfo.put(id, new PipelineXStatistics()); - } - pipelineXStatisticalInfo.get(id).add(isSink, info); - pipelineXStatisticalInfo.get(id).updateInstance(isSink, instance); - } - - private void addPlanNodeInfo(int id, String info, int instance) { - if (!statisticalInfo.containsKey(id)) { - statisticalInfo.put(id, new ArrayList()); - statisticalInfoInstance.put(id, new Integer(0)); - } - statisticalInfo.get(id).add(info); - int ins = statisticalInfoInstance.get(id); - ins = Math.max(ins, instance); - statisticalInfoInstance.put(id, ins); - } - - private void addDataSinkInfo(String info, int instance) { - if (fragmentInfo.get(fragmentId) == null) { - fragmentInfo.put(fragmentId, new ArrayList()); - fragmentInfoInstance.put(fragmentId, new Integer(0)); - } - fragmentInfo.get(fragmentId).add(info); - int ins = fragmentInfoInstance.get(fragmentId); - ins = Math.max(ins, instance); - fragmentInfoInstance.put(fragmentId, ins); - } - - public void addInfoFromProfile(RuntimeProfile profile, String name, String info, int instance) { - if (isPipelineX) { - addPipelineXPlanNodeInfo(profile.sinkOperator(), profile.nodeId(), name + ": " + info, instance); - } else { - if (isDataSink) { - addDataSinkInfo(name + ": " + info, instance); - } else { - addPlanNodeInfo(profile.nodeId(), name + ": " + info, instance); - } - } - } - - public boolean hasInfo(int id) { - if (isPipelineX) { - return pipelineXStatisticalInfo.containsKey(id); - } else { - return statisticalInfo.containsKey(id); - } - } - - public void getInfoById(int id, String prefix, StringBuilder str) { - if (!hasInfo(id)) { - return; - } - if (isPipelineX) { - getPipelineXInfoById(id, prefix, str); - } else { - ArrayList infos = statisticalInfo.get(id); - str.append(prefix + "Instance num :" + statisticalInfoInstance.get(id) + "\n"); - Collections.sort(infos); - for (String info : infos) { - str.append(prefix + info + "\n"); - } - } - } - - private void getPipelineXInfoById(int id, String prefix, StringBuilder str) { - PipelineXStatistics statistics = pipelineXStatisticalInfo.get(id); - str.append(prefix + "Operator: \n"); - statistics.getOperator(prefix + " ", str); - - str.append(prefix + "Sink: \n"); - statistics.getSink(prefix + " ", str); - } - - public void getDataSinkInfo(int fragmentIdx, String prefix, StringBuilder str) { - if (!fragmentInfo.containsKey(fragmentIdx)) { - return; - } - str.append(prefix + "Instance num :" + fragmentInfoInstance.get(fragmentIdx) + "\n"); - ArrayList infos = fragmentInfo.get(fragmentIdx); - Collections.sort(infos); - for (String info : infos) { - str.append(prefix + info + "\n"); - } - } - - public void setFragmentId(int id) { - this.fragmentId = id; - } - - public void setIsDataSink(boolean dataSink) { - this.isDataSink = dataSink; - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java index 2cfaf5e7a8..9f1b8234ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java @@ -449,30 +449,6 @@ public class RuntimeProfile { } } - public void simpleProfile(int depth, int childIdx, ProfileStatistics statistics) { - if (depth == FRAGMENT_DEPTH) { - statistics.setFragmentId(childIdx); - mergeMutiInstance(childList, statistics); - return; - } - for (int i = 0; i < childList.size(); i++) { - Pair pair = childList.get(i); - RuntimeProfile profile = pair.first; - profile.simpleProfile(depth + 1, i, statistics); - } - } - - private static void mergeMutiInstance( - LinkedList> childList, ProfileStatistics statistics) { - Pair pair = childList.get(0); - RuntimeProfile mergedProfile = pair.first; - LinkedList other = new LinkedList(); - for (int i = 1; i < childList.size(); i++) { - other.add(childList.get(i).first); - } - mergeInstanceProfile(mergedProfile, other, statistics); - } - private static LinkedList getChildListFromLists(int idx, LinkedList rhs) { LinkedList ret = new LinkedList(); for (RuntimeProfile profile : rhs) { @@ -491,109 +467,6 @@ public class RuntimeProfile { return ret; } - private static void mergeInstanceProfile(RuntimeProfile src, LinkedList rhs, - ProfileStatistics statistics) { - // data sink - // plan node - // other - for (int i = 0; i < src.childList.size(); i++) { - RuntimeProfile srcChild = src.childList.get(i).first; - LinkedList rhsChild = getChildListFromLists(i, rhs); - if (i == 0) { - statistics.setIsDataSink(true); - } else { - statistics.setIsDataSink(false); - } - mergePlanNodeProfile(srcChild, rhsChild, statistics); - } - } - - private static void mergePlanNodeProfile(RuntimeProfile src, LinkedList rhs, - ProfileStatistics statistics) { - mergeTotalTime(src, rhs, statistics); - mergeProfileCounter(src, ROOT_COUNTER, rhs, statistics); - for (int i = 0; i < src.childList.size(); i++) { - RuntimeProfile srcChild = src.childList.get(i).first; - LinkedList rhsChild = getChildListFromLists(i, rhs); - mergePlanNodeProfile(srcChild, rhsChild, statistics); - } - } - - private static void mergeProfileCounter(RuntimeProfile src, String counterName, LinkedList rhs, - ProfileStatistics statistics) { - Set childCounterSet = src.childCounterMap.get(counterName); - if (childCounterSet == null) { - return; - } - List childCounterList = new LinkedList<>(childCounterSet); - for (String childCounterName : childCounterList) { - Counter counter = src.counterMap.get(childCounterName); - mergeProfileCounter(src, childCounterName, rhs, statistics); - if (counter.getLevel() == 1) { - LinkedList rhsCounter = getCounterListFromLists(childCounterName, rhs); - mergeCounter(src, childCounterName, counter, rhsCounter, statistics); - } - } - } - - private static void mergeTotalTime(RuntimeProfile src, LinkedList rhs, - ProfileStatistics statistics) { - String counterName = "TotalTime"; - Counter counter = src.counterMap.get(counterName); - if (counter == null) { - return; - } - LinkedList rhsCounter = getCounterListFromLists(counterName, rhs); - mergeCounter(src, counterName, counter, rhsCounter, statistics); - } - - private static void mergeCounter(RuntimeProfile src, String counterName, Counter counter, - LinkedList rhsCounter, ProfileStatistics statistics) { - if (counter.getLevel() != 1) { - return; - } - if (rhsCounter == null) { - return; - } - if (counter.isTimeType()) { - Counter newCounter = new Counter(counter.getType(), counter.getValue()); - Counter maxCounter = new Counter(counter.getType(), counter.getValue()); - Counter minCounter = new Counter(counter.getType(), counter.getValue()); - for (Counter cnt : rhsCounter) { - if (cnt != null) { - if (cnt.getValue() > maxCounter.getValue()) { - maxCounter.setValue(cnt.getValue()); - } - if (cnt.getValue() < minCounter.getValue()) { - minCounter.setValue(cnt.getValue()); - } - } - } - for (Counter cnt : rhsCounter) { - if (cnt != null) { - newCounter.addValue(cnt); - } - } - long countNumber = rhsCounter.size() + 1; - if (newCounter.getValue() > 0) { - newCounter.divValue(countNumber); - String infoString = AVG_TIME_PRE + printCounter(newCounter.getValue(), newCounter.getType()) + ", " - + MAX_TIME_PRE + printCounter(maxCounter.getValue(), maxCounter.getType()) + ", " - + MIN_TIME_PRE + printCounter(minCounter.getValue(), minCounter.getType()); - statistics.addInfoFromProfile(src, counterName, infoString, rhsCounter.size() + 1); - } - } else { - Counter newCounter = new Counter(counter.getType(), counter.getValue()); - for (Counter cnt : rhsCounter) { - if (cnt != null) { - newCounter.addValue(cnt); - } - } - String infoString = printCounter(newCounter.getValue(), newCounter.getType()); - statistics.addInfoFromProfile(src, counterName, infoString, rhsCounter.size() + 1); - } - } - public String toString() { StringBuilder builder = new StringBuilder(); prettyPrint(builder, "", isPipelineX); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java index a859fb361f..7872560a1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java @@ -24,7 +24,6 @@ import org.apache.doris.catalog.MysqlTable; import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.util.ProfileStatistics; import org.apache.doris.planner.external.odbc.OdbcTableSink; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TExplainLevel; @@ -49,14 +48,6 @@ public abstract class DataSink { */ public abstract String getExplainString(String prefix, TExplainLevel explainLevel); - public String getExplainStringToProfile(String prefix, TExplainLevel explainLevel, ProfileStatistics statistics, - int fragmentIdx) { - String dataSinkString = getExplainString(prefix, explainLevel); - StringBuilder expBuilder = new StringBuilder(); - statistics.getDataSinkInfo(fragmentIdx, prefix + " ", expBuilder); - return dataSinkString + "\n" + expBuilder.toString(); - } - protected abstract TDataSink toThrift(); public void setFragment(PlanFragment fragment) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java index 3324f9de1b..2a4aa17553 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java @@ -27,7 +27,6 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.common.TreeNode; -import org.apache.doris.common.util.ProfileStatistics; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPartitionType; @@ -338,24 +337,6 @@ public class PlanFragment extends TreeNode { return str.toString(); } - public String getExplainStringToProfile(TExplainLevel explainLevel, ProfileStatistics statistics, int fragmentIdx) { - StringBuilder str = new StringBuilder(); - Preconditions.checkState(dataPartition != null); - if (CollectionUtils.isNotEmpty(outputExprs)) { - str.append(" OUTPUT EXPRS:\n "); - str.append(outputExprs.stream().map(Expr::toSql).collect(Collectors.joining("\n "))); - } - str.append("\n"); - str.append(" PARTITION: " + dataPartition.getExplainString(explainLevel) + "\n"); - if (sink != null) { - str.append(sink.getExplainStringToProfile(" ", explainLevel, statistics, fragmentIdx) + "\n"); - } - if (planRoot != null) { - str.append(planRoot.getExplainStringToProfile(" ", " ", explainLevel, statistics)); - } - return str.toString(); - } - public void getExplainStringMap(Map planNodeMap) { org.apache.doris.thrift.TExplainLevel explainLevel = org.apache.doris.thrift.TExplainLevel.NORMAL; if (planRoot != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index c323b3160f..a61e214f05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -41,7 +41,6 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.TreeNode; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.ProfileStatistics; import org.apache.doris.statistics.PlanStats; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.StatsDeriveResult; @@ -576,62 +575,6 @@ public abstract class PlanNode extends TreeNode implements PlanStats { return expBuilder.toString(); } - protected final String getExplainStringToProfile(String rootPrefix, String prefix, TExplainLevel detailLevel, - ProfileStatistics statistics) { - StringBuilder expBuilder = new StringBuilder(); - String detailPrefix = prefix; - boolean traverseChildren = children != null - && children.size() > 0 - && !(this instanceof ExchangeNode); - // if (children != null && children.size() > 0) { - if (traverseChildren) { - detailPrefix += "| "; - } else { - detailPrefix += " "; - } - - // Print the current node - // The plan node header line will be prefixed by rootPrefix and the remaining - // details - // will be prefixed by detailPrefix. - expBuilder.append(rootPrefix + id.asInt() + ":" + planNodeName + "\n"); - expBuilder.append(getNodeExplainString(detailPrefix, detailLevel)); - statistics.getInfoById(id.asInt(), detailPrefix, expBuilder); - if (limit != -1) { - expBuilder.append(detailPrefix + "limit: " + limit + "\n"); - } - if (!CollectionUtils.isEmpty(projectList)) { - expBuilder.append(detailPrefix).append("projections: ").append(getExplainString(projectList)).append("\n"); - expBuilder.append(detailPrefix).append("project output tuple id: ") - .append(outputTupleDesc.getId().asInt()).append("\n"); - } - // Output Tuple Ids only when explain plan level is set to verbose - if (detailLevel.equals(TExplainLevel.VERBOSE)) { - expBuilder.append(detailPrefix + "tuple ids: "); - for (TupleId tupleId : tupleIds) { - String nullIndicator = nullableTupleIds.contains(tupleId) ? "N" : ""; - expBuilder.append(tupleId.asInt() + nullIndicator + " "); - } - expBuilder.append("\n"); - } - - // Print the children - // if (children != null && children.size() > 0) { - if (traverseChildren) { - expBuilder.append(detailPrefix + "\n"); - String childHeadlinePrefix = prefix + "|----"; - String childDetailPrefix = prefix + "| "; - for (int i = 1; i < children.size(); ++i) { - expBuilder.append( - children.get(i).getExplainStringToProfile(childHeadlinePrefix, childDetailPrefix, - detailLevel, statistics)); - expBuilder.append(childDetailPrefix + "\n"); - } - expBuilder.append(children.get(0).getExplainStringToProfile(prefix, prefix, detailLevel, statistics)); - } - return expBuilder.toString(); - } - private String getplanNodeExplainString(String prefix, TExplainLevel detailLevel) { StringBuilder expBuilder = new StringBuilder(); expBuilder.append(getNodeExplainString(prefix, detailLevel)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java index 32fc0245c3..b113d83797 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java @@ -29,7 +29,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.profile.PlanTreeBuilder; import org.apache.doris.common.profile.PlanTreePrinter; import org.apache.doris.common.util.LiteralUtils; -import org.apache.doris.common.util.ProfileStatistics; import org.apache.doris.qe.ResultSet; import org.apache.doris.thrift.TQueryOptions; @@ -103,21 +102,6 @@ public abstract class Planner { return str.toString(); } - public String getExplainStringToProfile(ProfileStatistics statistics) { - org.apache.doris.thrift.TExplainLevel explainLevel = org.apache.doris.thrift.TExplainLevel.NORMAL; - StringBuilder str = new StringBuilder(); - for (int i = 0; i < fragments.size(); ++i) { - PlanFragment fragment = fragments.get(i); - if (i > 0) { - // a blank line between plan fragments - str.append("\n"); - } - str.append("PLAN FRAGMENT " + i + "\n"); - str.append(fragment.getExplainStringToProfile(explainLevel, statistics, i)); - } - return str.toString(); - } - public Map getExplainStringMap() { Map planNodeMap = new HashMap(); for (int i = 0; i < fragments.size(); ++i) {