From 0631ed61b016f91c6385fc0a500f3df9b89b0ef9 Mon Sep 17 00:00:00 2001 From: Mryange <59914473+Mryange@users.noreply.github.com> Date: Sat, 7 Oct 2023 11:16:53 +0800 Subject: [PATCH] [feature](profilev2) Preliminary support for profilev2. (#24881) You can set the level of counters on the backend using ADD_COUNTER_WITH_LEVEL/ADD_TIMER_WITH_LEVEL. The profile can then merge counters with level 1. set profile_level = 1; such as sql select count(*) from customer join item on c_customer_sk = i_item_sk profile Simple profile PLAN FRAGMENT 0 OUTPUT EXPRS: count(*) PARTITION: UNPARTITIONED VRESULT SINK MYSQL_PROTOCAL 7:VAGGREGATE (merge finalize) | output: count(partial_count(*))[#44] | group by: | cardinality=1 | TotalTime: avg 725.608us, max 725.608us, min 725.608us | RowsReturned: 1 | 6:VEXCHANGE offset: 0 TotalTime: avg 52.411us, max 52.411us, min 52.411us RowsReturned: 8 PLAN FRAGMENT 1 PARTITION: HASH_PARTITIONED: c_customer_sk STREAM DATA SINK EXCHANGE ID: 06 UNPARTITIONED TotalTime: avg 106.263us, max 118.38us, min 81.403us BlocksSent: 8 5:VAGGREGATE (update serialize) | output: partial_count(*)[#43] | group by: | cardinality=1 | TotalTime: avg 679.296us, max 739.395us, min 554.904us | BuildTime: avg 33.198us, max 48.387us, min 28.880us | ExecTime: avg 27.633us, max 40.278us, min 24.537us | RowsReturned: 8 | 4:VHASH JOIN | join op: INNER JOIN(PARTITIONED)[] | equal join conjunct: c_customer_sk = i_item_sk | runtime filters: RF000[bloom] <- i_item_sk(18000/16384/1048576) | cardinality=17,740 | vec output tuple id: 3 | vIntermediate tuple ids: 2 | hash output slot ids: 22 | RowsReturned: 18.0K (18000) | ProbeRows: 18.0K (18000) | ProbeTime: avg 862.308us, max 1.576ms, min 666.28us | BuildRows: 18.0K (18000) | BuildTime: avg 3.8ms, max 3.860ms, min 2.317ms | |----1:VEXCHANGE | offset: 0 | TotalTime: avg 48.822us, max 67.459us, min 30.380us | RowsReturned: 18.0K (18000) | 3:VEXCHANGE offset: 0 TotalTime: avg 33.162us, max 39.480us, min 28.854us RowsReturned: 18.0K (18000) PLAN FRAGMENT 2 PARTITION: HASH_PARTITIONED: c_customer_id STREAM DATA SINK EXCHANGE ID: 03 HASH_PARTITIONED: c_customer_sk TotalTime: avg 753.954us, max 1.210ms, min 499.470us BlocksSent: 64 2:VOlapScanNode TABLE: default_cluster:tpcds.customer(customer), PREAGGREGATION: ON runtime filters: RF000[bloom] -> c_customer_sk partitions=1/1, tablets=12/12, tabletList=1550745,1550747,1550749 ... cardinality=100000, avgRowSize=0.0, numNodes=1 pushAggOp=NONE TotalTime: avg 18.417us, max 41.319us, min 10.189us RowsReturned: 18.0K (18000) --------- Co-authored-by: yiguolei <676222867@qq.com> --- be/src/exec/exec_node.cpp | 3 +- .../pipeline/exec/exchange_sink_operator.cpp | 2 +- be/src/util/runtime_profile.cpp | 12 +- be/src/util/runtime_profile.h | 25 +- be/src/vec/exec/join/vjoin_node_base.cpp | 9 +- be/src/vec/exec/vaggregation_node.cpp | 4 +- be/src/vec/sink/vdata_stream_sender.cpp | 2 +- .../apache/doris/common/profile/Profile.java | 6 +- .../org/apache/doris/common/util/Counter.java | 16 ++ .../doris/common/util/ProfileManager.java | 2 +- .../doris/common/util/ProfileStatistics.java | 99 ++++++++ .../doris/common/util/RuntimeProfile.java | 215 ++++++++---------- .../doris/load/loadv2/BrokerLoadJob.java | 2 +- .../org/apache/doris/planner/DataSink.java | 12 +- .../apache/doris/planner/PlanFragment.java | 19 ++ .../org/apache/doris/planner/PlanNode.java | 57 +++++ .../org/apache/doris/planner/Planner.java | 16 ++ .../org/apache/doris/qe/SessionVariable.java | 10 +- .../org/apache/doris/qe/StmtExecutor.java | 3 +- gensrc/thrift/RuntimeProfile.thrift | 1 + 20 files changed, 375 insertions(+), 140 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileStatistics.java diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp index f3870c6346..352e111f1c 100644 --- a/be/src/exec/exec_node.cpp +++ b/be/src/exec/exec_node.cpp @@ -129,7 +129,8 @@ Status ExecNode::prepare(RuntimeState* state) { DCHECK(_runtime_profile.get() != nullptr); _span = state->get_tracer()->StartSpan(get_name()); OpentelemetryScope scope {_span}; - _rows_returned_counter = ADD_COUNTER(_runtime_profile, "RowsReturned", TUnit::UNIT); + _rows_returned_counter = + ADD_COUNTER_WITH_LEVEL(_runtime_profile, "RowsReturned", TUnit::UNIT, 1); _projection_timer = ADD_TIMER(_runtime_profile, "ProjectionTime"); _rows_returned_rate = runtime_profile()->add_derived_counter( ROW_THROUGHPUT_COUNTER, TUnit::UNIT_PER_SECOND, diff --git a/be/src/pipeline/exec/exchange_sink_operator.cpp b/be/src/pipeline/exec/exchange_sink_operator.cpp index 8532049793..46f5e99614 100644 --- a/be/src/pipeline/exec/exchange_sink_operator.cpp +++ b/be/src/pipeline/exec/exchange_sink_operator.cpp @@ -112,7 +112,7 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf _split_block_hash_compute_timer = ADD_TIMER(_profile, "SplitBlockHashComputeTime"); _split_block_distribute_by_channel_timer = ADD_TIMER(_profile, "SplitBlockDistributeByChannelTime"); - _blocks_sent_counter = ADD_COUNTER(_profile, "BlocksSent", TUnit::UNIT); + _blocks_sent_counter = ADD_COUNTER_WITH_LEVEL(_profile, "BlocksSent", TUnit::UNIT, 1); _overall_throughput = _profile->add_derived_counter( "OverallThroughput", TUnit::BYTES_PER_SECOND, std::bind(&RuntimeProfile::units_per_second, _bytes_sent_counter, diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp index f385256ab2..cbee0bee6a 100644 --- a/be/src/util/runtime_profile.cpp +++ b/be/src/util/runtime_profile.cpp @@ -52,7 +52,7 @@ RuntimeProfile::RuntimeProfile(const std::string& name, bool is_averaged_profile _metadata(-1), _timestamp(-1), _is_averaged_profile(is_averaged_profile), - _counter_total_time(TUnit::TIME_NS, 0), + _counter_total_time(TUnit::TIME_NS, 0, 1), _local_time_percent(0) { _counter_map["TotalTime"] = &_counter_total_time; } @@ -278,6 +278,9 @@ RuntimeProfile* RuntimeProfile::create_child(const std::string& name, bool inden std::lock_guard l(_children_lock); DCHECK(_child_map.find(name) == _child_map.end()); RuntimeProfile* child = _pool->add(new RuntimeProfile(name)); + if (this->is_set_metadata()) { + child->set_metadata(this->metadata()); + } if (_children.empty()) { add_child_unlock(child, indent, nullptr); } else { @@ -405,7 +408,8 @@ std::shared_ptr RuntimeProfile::AddSharedH } RuntimeProfile::Counter* RuntimeProfile::add_counter(const std::string& name, TUnit::type type, - const std::string& parent_counter_name) { + const std::string& parent_counter_name, + int64_t level) { std::lock_guard l(_counter_map_lock); // TODO(yingchun): Can we ensure that 'name' is not exist in '_counter_map'? Use CHECK instead? @@ -416,7 +420,7 @@ RuntimeProfile::Counter* RuntimeProfile::add_counter(const std::string& name, TU DCHECK(parent_counter_name == ROOT_COUNTER || _counter_map.find(parent_counter_name) != _counter_map.end()); - Counter* counter = _pool->add(new Counter(type, 0)); + Counter* counter = _pool->add(new Counter(type, 0, level)); _counter_map[name] = counter; std::set* child_counters = find_or_insert(&_child_counter_map, parent_counter_name, std::set()); @@ -631,7 +635,6 @@ void RuntimeProfile::to_thrift(std::vector* nodes) { node.metadata = _metadata; node.timestamp = _timestamp; node.indent = true; - CounterMap counter_map; { std::lock_guard l(_counter_map_lock); @@ -645,6 +648,7 @@ void RuntimeProfile::to_thrift(std::vector* nodes) { counter.name = iter->first; counter.value = iter->second->value(); counter.type = iter->second->type(); + counter.__set_level(iter->second->level()); node.counters.push_back(counter); } diff --git a/be/src/util/runtime_profile.h b/be/src/util/runtime_profile.h index 96d14d6540..23b4a1189b 100644 --- a/be/src/util/runtime_profile.h +++ b/be/src/util/runtime_profile.h @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -53,7 +54,11 @@ class TRuntimeProfileTree; #define ADD_LABEL_COUNTER(profile, name) (profile)->add_counter(name, TUnit::NONE) #define ADD_COUNTER(profile, name, type) (profile)->add_counter(name, type) +#define ADD_COUNTER_WITH_LEVEL(profile, name, type, level) \ + (profile)->add_counter_with_level(name, type, level) #define ADD_TIMER(profile, name) (profile)->add_counter(name, TUnit::TIME_NS) +#define ADD_TIMER_WITH_LEVEL(profile, name, level) \ + (profile)->add_counter_with_level(name, TUnit::TIME_NS, level) #define ADD_CHILD_COUNTER(profile, name, type, parent) (profile)->add_counter(name, type, parent) #define ADD_CHILD_TIMER(profile, name, parent) (profile)->add_counter(name, TUnit::TIME_NS, parent) #define SCOPED_TIMER(c) ScopedTimer MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c) @@ -86,7 +91,8 @@ class RuntimeProfile { public: class Counter { public: - Counter(TUnit::type type, int64_t value = 0) : _value(value), _type(type) {} + Counter(TUnit::type type, int64_t value = 0, int64_t level = 3) + : _value(value), _type(type), _level(level) {} virtual ~Counter() = default; virtual void update(int64_t delta) { _value.fetch_add(delta, std::memory_order_relaxed); } @@ -108,11 +114,14 @@ public: TUnit::type type() const { return _type; } + virtual int64_t level() { return _level; } + private: friend class RuntimeProfile; std::atomic _value; TUnit::type _type; + int64_t _level; }; /// A counter that keeps track of the highest value seen (reporting that @@ -276,11 +285,15 @@ public: // parent_counter_name. // If the counter already exists, the existing counter object is returned. Counter* add_counter(const std::string& name, TUnit::type type, - const std::string& parent_counter_name); + const std::string& parent_counter_name, int64_t level = 2); Counter* add_counter(const std::string& name, TUnit::type type) { return add_counter(name, type, ""); } + Counter* add_counter_with_level(const std::string& name, TUnit::type type, int64_t level) { + return add_counter(name, type, "", level); + } + // Add a derived counter with 'name'/'type'. The counter is owned by the // RuntimeProfile object. // If parent_counter_name is a non-empty string, the counter is added as a child of @@ -348,7 +361,12 @@ public: void set_name(const std::string& name) { _name = name; } int64_t metadata() const { return _metadata; } - void set_metadata(int64_t md) { _metadata = md; } + void set_metadata(int64_t md) { + _is_set_metadata = true; + _metadata = md; + } + + bool is_set_metadata() const { return _is_set_metadata; } time_t timestamp() const { return _timestamp; } void set_timestamp(time_t ss) { _timestamp = ss; } @@ -410,6 +428,7 @@ private: // user-supplied, uninterpreted metadata. int64_t _metadata; + bool _is_set_metadata = false; // The timestamp when the profile was modified, make sure the update is up to date. time_t _timestamp; diff --git a/be/src/vec/exec/join/vjoin_node_base.cpp b/be/src/vec/exec/join/vjoin_node_base.cpp index 1cb6c6e337..2a3a06f404 100644 --- a/be/src/vec/exec/join/vjoin_node_base.cpp +++ b/be/src/vec/exec/join/vjoin_node_base.cpp @@ -30,6 +30,7 @@ #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" +#include "util/runtime_profile.h" #include "util/telemetry/telemetry.h" #include "util/threadpool.h" #include "vec/columns/column.h" @@ -111,12 +112,16 @@ Status VJoinNodeBase::prepare(RuntimeState* state) { runtime_profile()->add_info_string("JoinType", to_string(_join_op)); _build_phase_profile = runtime_profile()->create_child("BuildPhase", true, true); + _build_get_next_timer = ADD_TIMER(_build_phase_profile, "BuildGetNextTime"); + _build_timer = ADD_TIMER_WITH_LEVEL(_build_phase_profile, "BuildTime", 1); + _build_rows_counter = ADD_COUNTER_WITH_LEVEL(_build_phase_profile, "BuildRows", TUnit::UNIT, 1); + _probe_phase_profile = runtime_profile()->create_child("ProbePhase", true, true); - _probe_timer = ADD_TIMER(_probe_phase_profile, "ProbeTime"); + _probe_timer = ADD_TIMER_WITH_LEVEL(_probe_phase_profile, "ProbeTime", 1); _join_filter_timer = ADD_CHILD_TIMER(_probe_phase_profile, "JoinFilterTimer", "ProbeTime"); _build_output_block_timer = ADD_CHILD_TIMER(_probe_phase_profile, "BuildOutputBlock", "ProbeTime"); - _probe_rows_counter = ADD_COUNTER(_probe_phase_profile, "ProbeRows", TUnit::UNIT); + _probe_rows_counter = ADD_COUNTER_WITH_LEVEL(_probe_phase_profile, "ProbeRows", TUnit::UNIT, 1); _push_down_timer = ADD_TIMER(runtime_profile(), "PublishRuntimeFilterTime"); _push_compute_timer = ADD_TIMER(runtime_profile(), "PushDownComputeTime"); diff --git a/be/src/vec/exec/vaggregation_node.cpp b/be/src/vec/exec/vaggregation_node.cpp index cee153af86..0202d67b25 100644 --- a/be/src/vec/exec/vaggregation_node.cpp +++ b/be/src/vec/exec/vaggregation_node.cpp @@ -291,10 +291,10 @@ Status AggregationNode::prepare_profile(RuntimeState* state) { _serialize_key_arena_memory_usage = runtime_profile()->AddHighWaterMarkCounter( "SerializeKeyArena", TUnit::BYTES, "MemoryUsage"); - _build_timer = ADD_TIMER(runtime_profile(), "BuildTime"); + _build_timer = ADD_TIMER_WITH_LEVEL(runtime_profile(), "BuildTime", 1); _build_table_convert_timer = ADD_TIMER(runtime_profile(), "BuildConvertToPartitionedTime"); _serialize_key_timer = ADD_TIMER(runtime_profile(), "SerializeKeyTime"); - _exec_timer = ADD_TIMER(runtime_profile(), "ExecTime"); + _exec_timer = ADD_TIMER_WITH_LEVEL(runtime_profile(), "ExecTime", 1); _merge_timer = ADD_TIMER(runtime_profile(), "MergeTime"); _expr_timer = ADD_TIMER(runtime_profile(), "ExprTime"); _get_results_timer = ADD_TIMER(runtime_profile(), "GetResultsTime"); diff --git a/be/src/vec/sink/vdata_stream_sender.cpp b/be/src/vec/sink/vdata_stream_sender.cpp index 4f4e4983e1..ad19dcd9cd 100644 --- a/be/src/vec/sink/vdata_stream_sender.cpp +++ b/be/src/vec/sink/vdata_stream_sender.cpp @@ -466,7 +466,7 @@ Status VDataStreamSender::prepare(RuntimeState* state) { _split_block_distribute_by_channel_timer = ADD_TIMER(profile(), "SplitBlockDistributeByChannelTime"); _merge_block_timer = ADD_TIMER(profile(), "MergeBlockTime"); - _blocks_sent_counter = ADD_COUNTER(profile(), "BlocksSent", TUnit::UNIT); + _blocks_sent_counter = ADD_COUNTER_WITH_LEVEL(profile(), "BlocksSent", TUnit::UNIT, 1); _overall_throughput = profile()->add_derived_counter( "OverallThroughput", TUnit::BYTES_PER_SECOND, std::bind(&RuntimeProfile::units_per_second, _bytes_sent_counter, diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java index bc7250ad2f..a2ec3a2cf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java @@ -19,6 +19,7 @@ package org.apache.doris.common.profile; import org.apache.doris.common.util.ProfileManager; import org.apache.doris.common.util.RuntimeProfile; +import org.apache.doris.planner.Planner; import com.google.common.collect.Lists; @@ -62,7 +63,7 @@ public class Profile { } public synchronized void update(long startTime, Map summaryInfo, boolean isFinished, - boolean isSimpleProfile) { + int profileLevel, Planner planner) { if (this.isFinished) { return; } @@ -71,7 +72,8 @@ public class Profile { executionProfile.update(startTime, isFinished); } rootProfile.computeTimeInProfile(); - rootProfile.setProfileLevel(isSimpleProfile); + rootProfile.setPlaner(planner); + rootProfile.setProfileLevel(profileLevel); ProfileManager.getInstance().pushProfile(rootProfile); this.isFinished = isFinished; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java index 77baf3ecd8..2beb845438 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Counter.java @@ -24,6 +24,7 @@ public class Counter { private volatile long value; private volatile int type; private volatile boolean remove = false; + private volatile long level; public long getValue() { return value; @@ -42,6 +43,10 @@ public class Counter { return TUnit.findByValue(type); } + public void setLevel(long level) { + this.level = level; + } + public void setType(TUnit type) { this.type = type.getValue(); } @@ -49,6 +54,13 @@ public class Counter { public Counter(TUnit type, long value) { this.value = value; this.type = type.getValue(); + this.level = 2; + } + + public Counter(TUnit type, long value, long level) { + this.value = value; + this.type = type.getValue(); + this.level = level; } public void addValue(Counter other) { @@ -77,4 +89,8 @@ public class Counter { public boolean isRemove() { return this.remove; } + + public long getLevel() { + return this.level; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java index dc175e7737..c4d593d897 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java @@ -86,7 +86,7 @@ public class ProfileManager { if (profileContent == null) { // Simple profile will change the structure of the profile. try { - profileContent = profile.getSimpleString(); + profileContent = profile.getProfileByLevel(); } catch (Exception e) { LOG.warn("profile get error : " + e.toString()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileStatistics.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileStatistics.java new file mode 100644 index 0000000000..758c5e69b3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileStatistics.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import java.util.ArrayList; +import java.util.HashMap; + +/** + * Used for collecting information obtained from the profile. + */ +public class ProfileStatistics { + // Record statistical information based on nodeid. + private HashMap> statisticalInfo; + + // Record statistical information based on fragment ID. + // "Currently used to record sink nodes. + private HashMap> fragmentInfo; + + private int fragmentId; + + private boolean isDataSink; + + public ProfileStatistics() { + statisticalInfo = new HashMap>(); + fragmentInfo = new HashMap>(); + fragmentId = 0; + isDataSink = false; + } + + private void addPlanNodeInfo(int id, String info) { + if (!statisticalInfo.containsKey(id)) { + statisticalInfo.put(id, new ArrayList()); + } + statisticalInfo.get(id).add(info); + } + + private void addDataSinkInfo(String info) { + if (fragmentInfo.get(fragmentId) == null) { + fragmentInfo.put(fragmentId, new ArrayList()); + } + fragmentInfo.get(fragmentId).add(info); + } + + public void addInfoFromProfile(RuntimeProfile profile, String info) { + if (isDataSink) { + addDataSinkInfo(info); + } else { + addPlanNodeInfo(profile.nodeId(), info); + } + } + + public boolean hasInfo(int id) { + return statisticalInfo.containsKey(id); + } + + public void getInfoById(int id, String prefix, StringBuilder str) { + if (!hasInfo(id)) { + return; + } + ArrayList infos = statisticalInfo.get(id); + for (String info : infos) { + str.append(prefix + info + "\n"); + } + } + + public void getDataSinkInfo(int fragmentIdx, String prefix, StringBuilder str) { + if (!fragmentInfo.containsKey(fragmentIdx)) { + return; + } + ArrayList infos = fragmentInfo.get(fragmentIdx); + for (String info : infos) { + str.append(prefix + info + "\n"); + } + } + + public void setFragmentId(int id) { + this.fragmentId = id; + } + + public void setIsDataSink(boolean dataSink) { + this.isDataSink = dataSink; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java index 9f1bdb96b5..9fc3597400 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java @@ -20,6 +20,7 @@ package org.apache.doris.common.util; import org.apache.doris.common.Pair; import org.apache.doris.common.Reference; import org.apache.doris.common.profile.SummaryProfile; +import org.apache.doris.planner.Planner; import org.apache.doris.thrift.TCounter; import org.apache.doris.thrift.TRuntimeProfileNode; import org.apache.doris.thrift.TRuntimeProfileTree; @@ -49,9 +50,9 @@ public class RuntimeProfile { private static final Logger LOG = LogManager.getLogger(RuntimeProfile.class); public static String ROOT_COUNTER = ""; public static int FRAGMENT_DEPTH = 3; - public static String MAX_TIME_PRE = "max: "; - public static String MIN_TIME_PRE = "min: "; - public static String AVG_TIME_PRE = "avg: "; + public static String MAX_TIME_PRE = "max "; + public static String MIN_TIME_PRE = "min "; + public static String AVG_TIME_PRE = "avg "; private Counter counterTotalTime; private double localTimePercent; @@ -74,15 +75,18 @@ public class RuntimeProfile { private Boolean isDone = false; private Boolean isCancel = false; - private boolean enableSimplyProfile = false; + private int profileLevel = 3; + private Planner planner = null; + private int nodeid = -1; public RuntimeProfile(String name) { this(); this.name = name; + this.counterTotalTime = new Counter(TUnit.TIME_NS, 0, 1); } public RuntimeProfile() { - this.counterTotalTime = new Counter(TUnit.TIME_NS, 0); + this.counterTotalTime = new Counter(TUnit.TIME_NS, 0, 1); this.localTimePercent = 0; this.counterMap.put("TotalTime", counterTotalTime); } @@ -111,6 +115,10 @@ public class RuntimeProfile { return counterTotalTime; } + public int nodeId() { + return this.nodeid; + } + public Map getCounterMap() { return counterMap; } @@ -165,18 +173,23 @@ public class RuntimeProfile { // preorder traversal, idx should be modified in the traversal process private void update(List nodes, Reference idx) { TRuntimeProfileNode node = nodes.get(idx.getRef()); - // Make sure to update the latest LoadChannel profile according to the timestamp. + // Make sure to update the latest LoadChannel profile according to the + // timestamp. if (node.timestamp != -1 && node.timestamp < timestamp) { return; } + if (node.isSetMetadata()) { + this.nodeid = (int) node.getMetadata(); + } Preconditions.checkState(timestamp == -1 || node.timestamp != -1); // update this level's counters if (node.counters != null) { for (TCounter tcounter : node.counters) { Counter counter = counterMap.get(tcounter.name); if (counter == null) { - counterMap.put(tcounter.name, new Counter(tcounter.type, tcounter.value)); + counterMap.put(tcounter.name, new Counter(tcounter.type, tcounter.value, tcounter.level)); } else { + counter.setLevel(tcounter.level); if (counter.getType() != tcounter.type) { LOG.error("Cannot update counters with the same name but different types" + " type=" + tcounter.type); @@ -188,8 +201,7 @@ public class RuntimeProfile { if (node.child_counters_map != null) { // update childCounters - for (Map.Entry> entry : - node.child_counters_map.entrySet()) { + for (Map.Entry> entry : node.child_counters_map.entrySet()) { String parentCounterName = entry.getKey(); counterLock.writeLock().lock(); @@ -343,43 +355,36 @@ public class RuntimeProfile { } } - public void simpleProfile(int depth) { + public void simpleProfile(int depth, int childIdx, ProfileStatistics statistics) { if (depth == FRAGMENT_DEPTH) { - mergeMutiInstance(childList); + statistics.setFragmentId(childIdx); + mergeMutiInstance(childList, statistics); return; } for (int i = 0; i < childList.size(); i++) { Pair pair = childList.get(i); RuntimeProfile profile = pair.first; - profile.simpleProfile(depth + 1); + profile.simpleProfile(depth + 1, i, statistics); } } private static void mergeMutiInstance( - LinkedList> childList) { - /* - * Fragment 1: Fragment 1: - * Instance 0 Instance (total) - * Instance 1 - * Instance 2 - */ - int numInstance = childList.size(); + LinkedList> childList, ProfileStatistics statistics) { Pair pair = childList.get(0); RuntimeProfile mergedProfile = pair.first; LinkedList other = new LinkedList(); for (int i = 1; i < childList.size(); i++) { other.add(childList.get(i).first); } - mergeInstanceProfile(mergedProfile, other); - childList.clear(); - mergedProfile.name = "Instance " + "(" + numInstance + ")"; - childList.add(Pair.of(mergedProfile, pair.second)); + mergeInstanceProfile(mergedProfile, other, statistics); } private static LinkedList getChildListFromLists(int idx, LinkedList rhs) { LinkedList ret = new LinkedList(); for (RuntimeProfile profile : rhs) { - ret.add(profile.childList.get(idx).first); + if (idx < profile.childList.size()) { + ret.add(profile.childList.get(idx).first); + } } return ret; } @@ -392,40 +397,36 @@ public class RuntimeProfile { return ret; } - private static void mergeInstanceProfile(RuntimeProfile src, LinkedList rhs) { - mergeProfileCounter(src, ROOT_COUNTER, rhs); - mergeTotalTime(src, rhs); - mergeProfileInfoStr(src, rhs); - removePipelineContext(src); + private static void mergeInstanceProfile(RuntimeProfile src, LinkedList rhs, + ProfileStatistics statistics) { + // data sink + // plan node + // other for (int i = 0; i < src.childList.size(); i++) { RuntimeProfile srcChild = src.childList.get(i).first; LinkedList rhsChild = getChildListFromLists(i, rhs); - mergeInstanceProfile(srcChild, rhsChild); - } - } - - private static void mergeTotalTime(RuntimeProfile src, LinkedList rhs) { - Counter counter = src.counterMap.get("TotalTime"); - for (RuntimeProfile profile : rhs) { - Counter othCounter = profile.counterMap.get("TotalTime"); - if (othCounter != null && counter != null) { - counter.addValue(othCounter); + if (i == 0) { + statistics.setIsDataSink(true); + } else { + statistics.setIsDataSink(false); } + mergePlanNodeProfile(srcChild, rhsChild, statistics); } } - private static void removePipelineContext(RuntimeProfile src) { - LinkedList> newChildList = new LinkedList>(); - for (Pair pair : src.childList) { - RuntimeProfile profile = pair.first; - if (!profile.name.equals("PipelineContext")) { - newChildList.add(pair); - } + private static void mergePlanNodeProfile(RuntimeProfile src, LinkedList rhs, + ProfileStatistics statistics) { + mergeTotalTime(src, rhs, statistics); + mergeProfileCounter(src, ROOT_COUNTER, rhs, statistics); + for (int i = 0; i < src.childList.size(); i++) { + RuntimeProfile srcChild = src.childList.get(i).first; + LinkedList rhsChild = getChildListFromLists(i, rhs); + mergePlanNodeProfile(srcChild, rhsChild, statistics); } - src.childList = newChildList; } - private static void mergeProfileCounter(RuntimeProfile src, String counterName, LinkedList rhs) { + private static void mergeProfileCounter(RuntimeProfile src, String counterName, LinkedList rhs, + ProfileStatistics statistics) { Set childCounterSet = src.childCounterMap.get(counterName); if (childCounterSet == null) { return; @@ -433,49 +434,35 @@ public class RuntimeProfile { List childCounterList = new LinkedList<>(childCounterSet); for (String childCounterName : childCounterList) { Counter counter = src.counterMap.get(childCounterName); - LinkedList rhsCounter = getCounterListFromLists(childCounterName, rhs); - - mergeProfileCounter(src, childCounterName, rhs); - mergeCounter(src, childCounterName, counter, rhsCounter); - removeCounter(childCounterSet, childCounterName, counter); - - } - } - - private static void mergeProfileInfoStr(RuntimeProfile src, LinkedList rhs) { - for (String key : src.infoStringsDisplayOrder) { - Set strList = new TreeSet(); - strList.add(src.infoStrings.get(key)); - for (RuntimeProfile profile : rhs) { - String value = profile.infoStrings.get(key); - if (value != null) { - strList.add(value); - } - } - try { - String joinedString = String.join(" | ", strList); - src.infoStrings.put(key, joinedString); - } catch (Exception e) { - return; + mergeProfileCounter(src, childCounterName, rhs, statistics); + if (counter.getLevel() == 1) { + LinkedList rhsCounter = getCounterListFromLists(childCounterName, rhs); + mergeCounter(src, childCounterName, counter, rhsCounter, statistics); } } } - private static void removeCounter(Set childCounterSet, String childCounterName, Counter counter) { - if (counter.isRemove()) { - childCounterSet.remove(childCounterName); + private static void mergeTotalTime(RuntimeProfile src, LinkedList rhs, + ProfileStatistics statistics) { + String counterName = "TotalTime"; + Counter counter = src.counterMap.get(counterName); + if (counter == null) { + return; } + LinkedList rhsCounter = getCounterListFromLists(counterName, rhs); + mergeCounter(src, counterName, counter, rhsCounter, statistics); } private static void mergeCounter(RuntimeProfile src, String counterName, Counter counter, - LinkedList rhsCounter) { + LinkedList rhsCounter, ProfileStatistics statistics) { + if (counter.getLevel() != 1) { + return; + } if (rhsCounter == null) { return; } - if (rhsCounter.size() == 0) { - return; - } if (counter.isTimeType()) { + Counter newCounter = new Counter(counter.getType(), counter.getValue()); Counter maxCounter = new Counter(counter.getType(), counter.getValue()); Counter minCounter = new Counter(counter.getType(), counter.getValue()); for (Counter cnt : rhsCounter) { @@ -490,40 +477,27 @@ public class RuntimeProfile { } for (Counter cnt : rhsCounter) { if (cnt != null) { - counter.addValue(cnt); + newCounter.addValue(cnt); } } long countNumber = rhsCounter.size() + 1; - counter.divValue(countNumber); - String maxCounterName = MAX_TIME_PRE + counterName; - String minCounterName = MIN_TIME_PRE + counterName; - src.counterMap.put(minCounterName, minCounter); - src.counterMap.put(maxCounterName, maxCounter); - TreeSet childCounterSet = src.childCounterMap.get(counterName); - if (childCounterSet == null) { - src.childCounterMap.put(counterName, new TreeSet()); - childCounterSet = src.childCounterMap.get(counterName); + if (newCounter.getValue() > 0) { + newCounter.divValue(countNumber); + String infoString = counterName + ": " + + AVG_TIME_PRE + printCounter(newCounter.getValue(), newCounter.getType()) + ", " + + MAX_TIME_PRE + printCounter(maxCounter.getValue(), maxCounter.getType()) + ", " + + MIN_TIME_PRE + printCounter(minCounter.getValue(), minCounter.getType()); + statistics.addInfoFromProfile(src, infoString); } - childCounterSet.add(minCounterName); - childCounterSet.add(maxCounterName); - if (counter.getValue() > 0) { - src.infoStringsDisplayOrder.add(counterName); - String infoString = "[ " - + AVG_TIME_PRE + printCounter(counter.getValue(), counter.getType()) + " , " - + MAX_TIME_PRE + printCounter(maxCounter.getValue(), maxCounter.getType()) + " , " - + MIN_TIME_PRE + printCounter(minCounter.getValue(), minCounter.getType()) + " ]"; - src.infoStrings.put(counterName, infoString); - } - counter.setCanRemove(); // value will remove in removeCounter } else { - if (rhsCounter.size() == 0) { - return; - } + Counter newCounter = new Counter(counter.getType(), counter.getValue()); for (Counter cnt : rhsCounter) { if (cnt != null) { - counter.addValue(cnt); + newCounter.addValue(cnt); } } + String infoString = counterName + ": " + printCounter(newCounter.getValue(), newCounter.getType()); + statistics.addInfoFromProfile(src, infoString); } } @@ -533,14 +507,19 @@ public class RuntimeProfile { return builder.toString(); } - public String getSimpleString() { - if (!this.enableSimplyProfile) { + public String getProfileByLevel() { + if (this.profileLevel == 3) { + return toString(); + } + if (this.planner == null) { return toString(); } StringBuilder builder = new StringBuilder(); - simpleProfile(0); prettyPrint(builder, ""); - return builder.toString(); + ProfileStatistics statistics = new ProfileStatistics(); + simpleProfile(0, 0, statistics); + String planerStr = this.planner.getExplainStringToProfile(statistics); + return "Simple profile \n \n " + planerStr + "\n \n \n" + builder.toString(); } private void printChildCounters(String prefix, String counterName, StringBuilder builder) { @@ -678,7 +657,8 @@ public class RuntimeProfile { } } - // Because the profile of summary and child fragment is not a real parent-child relationship + // Because the profile of summary and child fragment is not a real parent-child + // relationship // Each child profile needs to calculate the time proportion consumed by itself public void computeTimeInChildProfile() { childMap.values().forEach(RuntimeProfile::computeTimeInProfile); @@ -688,8 +668,12 @@ public class RuntimeProfile { computeTimeInProfile(this.counterTotalTime.getValue()); } - public void setProfileLevel(boolean isSimpleProfile) { - this.enableSimplyProfile = isSimpleProfile; + public void setProfileLevel(int profileLevel) { + this.profileLevel = profileLevel; + } + + public void setPlaner(Planner planner) { + this.planner = planner; } private void computeTimeInProfile(long total) { @@ -727,8 +711,10 @@ public class RuntimeProfile { this.childList.sort((profile1, profile2) -> Long.compare(profile2.first.getCounterTotalTime().getValue(), profile1.first.getCounterTotalTime().getValue())); } catch (IllegalArgumentException e) { - // This exception may be thrown if the counter total time of the child is updated in the update method - // during the sorting process. This sorting only affects the profile instance display order, so this + // This exception may be thrown if the counter total time of the child is + // updated in the update method + // during the sorting process. This sorting only affects the profile instance + // display order, so this // exception is temporarily ignored here. if (LOG.isDebugEnabled()) { LOG.debug("sort child list error: ", e); @@ -765,4 +751,3 @@ public class RuntimeProfile { return infoStrings; } } - diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index d049e0f508..408196283f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -323,7 +323,7 @@ public class BrokerLoadJob extends BulkLoadJob { return; } jobProfile.update(createTimestamp, getSummaryInfo(true), true, - Boolean.valueOf(sessionVariables.getOrDefault(SessionVariable.ENABLE_SIMPLY_PROFILE, "true"))); + Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.PROFILE_LEVEL, "3")), null); } private Map getSummaryInfo(boolean isFinished) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java index 839301e509..a859fb361f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.MysqlTable; import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.ProfileStatistics; import org.apache.doris.planner.external.odbc.OdbcTableSink; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TExplainLevel; @@ -39,7 +40,8 @@ public abstract class DataSink { protected PlanFragment fragment; /** - * Return an explain string for the DataSink. Each line of the explain will be prefixed + * Return an explain string for the DataSink. Each line of the explain will be + * prefixed * by "prefix" * * @param prefix each explain line will be started with the given prefix @@ -47,6 +49,14 @@ public abstract class DataSink { */ public abstract String getExplainString(String prefix, TExplainLevel explainLevel); + public String getExplainStringToProfile(String prefix, TExplainLevel explainLevel, ProfileStatistics statistics, + int fragmentIdx) { + String dataSinkString = getExplainString(prefix, explainLevel); + StringBuilder expBuilder = new StringBuilder(); + statistics.getDataSinkInfo(fragmentIdx, prefix + " ", expBuilder); + return dataSinkString + "\n" + expBuilder.toString(); + } + protected abstract TDataSink toThrift(); public void setFragment(PlanFragment fragment) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java index 16be7e17a0..6336799b46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java @@ -27,6 +27,7 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.common.TreeNode; +import org.apache.doris.common.util.ProfileStatistics; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPartitionType; @@ -334,6 +335,24 @@ public class PlanFragment extends TreeNode { return str.toString(); } + public String getExplainStringToProfile(TExplainLevel explainLevel, ProfileStatistics statistics, int fragmentIdx) { + StringBuilder str = new StringBuilder(); + Preconditions.checkState(dataPartition != null); + if (CollectionUtils.isNotEmpty(outputExprs)) { + str.append(" OUTPUT EXPRS:\n "); + str.append(outputExprs.stream().map(Expr::toSql).collect(Collectors.joining("\n "))); + } + str.append("\n"); + str.append(" PARTITION: " + dataPartition.getExplainString(explainLevel) + "\n"); + if (sink != null) { + str.append(sink.getExplainStringToProfile(" ", explainLevel, statistics, fragmentIdx) + "\n"); + } + if (planRoot != null) { + str.append(planRoot.getExplainStringToProfile(" ", " ", explainLevel, statistics)); + } + return str.toString(); + } + /** * Returns true if this fragment is partitioned. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index 8dcc8043b4..40cfe876b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -41,6 +41,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.TreeNode; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.ProfileStatistics; import org.apache.doris.statistics.PlanStats; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.StatsDeriveResult; @@ -574,6 +575,62 @@ public abstract class PlanNode extends TreeNode implements PlanStats { return expBuilder.toString(); } + protected final String getExplainStringToProfile(String rootPrefix, String prefix, TExplainLevel detailLevel, + ProfileStatistics statistics) { + StringBuilder expBuilder = new StringBuilder(); + String detailPrefix = prefix; + boolean traverseChildren = children != null + && children.size() > 0 + && !(this instanceof ExchangeNode); + // if (children != null && children.size() > 0) { + if (traverseChildren) { + detailPrefix += "| "; + } else { + detailPrefix += " "; + } + + // Print the current node + // The plan node header line will be prefixed by rootPrefix and the remaining + // details + // will be prefixed by detailPrefix. + expBuilder.append(rootPrefix + id.asInt() + ":" + planNodeName + "\n"); + expBuilder.append(getNodeExplainString(detailPrefix, detailLevel)); + statistics.getInfoById(id.asInt(), detailPrefix, expBuilder); + if (limit != -1) { + expBuilder.append(detailPrefix + "limit: " + limit + "\n"); + } + if (!CollectionUtils.isEmpty(projectList)) { + expBuilder.append(detailPrefix).append("projections: ").append(getExplainString(projectList)).append("\n"); + expBuilder.append(detailPrefix).append("project output tuple id: ") + .append(outputTupleDesc.getId().asInt()).append("\n"); + } + // Output Tuple Ids only when explain plan level is set to verbose + if (detailLevel.equals(TExplainLevel.VERBOSE)) { + expBuilder.append(detailPrefix + "tuple ids: "); + for (TupleId tupleId : tupleIds) { + String nullIndicator = nullableTupleIds.contains(tupleId) ? "N" : ""; + expBuilder.append(tupleId.asInt() + nullIndicator + " "); + } + expBuilder.append("\n"); + } + + // Print the children + // if (children != null && children.size() > 0) { + if (traverseChildren) { + expBuilder.append(detailPrefix + "\n"); + String childHeadlinePrefix = prefix + "|----"; + String childDetailPrefix = prefix + "| "; + for (int i = 1; i < children.size(); ++i) { + expBuilder.append( + children.get(i).getExplainStringToProfile(childHeadlinePrefix, childDetailPrefix, + detailLevel, statistics)); + expBuilder.append(childDetailPrefix + "\n"); + } + expBuilder.append(children.get(0).getExplainStringToProfile(prefix, prefix, detailLevel, statistics)); + } + return expBuilder.toString(); + } + /** * Return the node-specific details. * Subclass should override this function. diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java index 46c7bc91d5..8681455a39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java @@ -29,6 +29,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.profile.PlanTreeBuilder; import org.apache.doris.common.profile.PlanTreePrinter; import org.apache.doris.common.util.LiteralUtils; +import org.apache.doris.common.util.ProfileStatistics; import org.apache.doris.qe.ResultSet; import org.apache.doris.thrift.TQueryOptions; @@ -89,6 +90,21 @@ public abstract class Planner { return str.toString(); } + public String getExplainStringToProfile(ProfileStatistics statistics) { + org.apache.doris.thrift.TExplainLevel explainLevel = org.apache.doris.thrift.TExplainLevel.NORMAL; + StringBuilder str = new StringBuilder(); + for (int i = 0; i < fragments.size(); ++i) { + PlanFragment fragment = fragments.get(i); + if (i > 0) { + // a blank line between plan fragments + str.append("\n"); + } + str.append("PLAN FRAGMENT " + i + "\n"); + str.append(fragment.getExplainStringToProfile(explainLevel, statistics, i)); + } + return str.toString(); + } + protected void handleLiteralInFe(LiteralExpr literalExpr, List data) { if (literalExpr instanceof NullLiteral) { data.add(null); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index fbea68d0e1..a236918e18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -112,7 +112,7 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_BUCKET_SHUFFLE_JOIN = "enable_bucket_shuffle_join"; public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM = "parallel_fragment_exec_instance_num"; public static final String PARALLEL_PIPELINE_TASK_NUM = "parallel_pipeline_task_num"; - public static final String ENABLE_SIMPLY_PROFILE = "enable_simply_profile"; + public static final String PROFILE_LEVEL = "profile_level"; public static final String MAX_INSTANCE_NUM = "max_instance_num"; public static final String ENABLE_INSERT_STRICT = "enable_insert_strict"; public static final String ENABLE_SPILLING = "enable_spilling"; @@ -623,8 +623,8 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = PARALLEL_PIPELINE_TASK_NUM, fuzzy = true, needForward = true) public int parallelPipelineTaskNum = 0; - @VariableMgr.VarAttr(name = ENABLE_SIMPLY_PROFILE, fuzzy = true) - public boolean enableSimplyProfile = true; + @VariableMgr.VarAttr(name = PROFILE_LEVEL, fuzzy = true) + public int profileLevel = 3; @VariableMgr.VarAttr(name = MAX_INSTANCE_NUM) public int maxInstanceNum = 64; @@ -2676,7 +2676,7 @@ public class SessionVariable implements Serializable, Writable { } } - public boolean getEnableSimplyProfile() { - return this.enableSimplyProfile; + public int getProfileLevel() { + return this.profileLevel; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 26914ba481..dd5c0d7cd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -885,8 +885,9 @@ public class StmtExecutor { if (!context.getSessionVariable().enableProfile()) { return; } + profile.update(context.startTime, getSummaryInfo(isFinished), isFinished, - context.getSessionVariable().enableSimplyProfile); + context.getSessionVariable().profileLevel, this.planner); } // Analyze one statement to structure in memory. diff --git a/gensrc/thrift/RuntimeProfile.thrift b/gensrc/thrift/RuntimeProfile.thrift index 365050954d..637ff537ea 100644 --- a/gensrc/thrift/RuntimeProfile.thrift +++ b/gensrc/thrift/RuntimeProfile.thrift @@ -25,6 +25,7 @@ struct TCounter { 1: required string name 2: required Metrics.TUnit type 3: required i64 value + 4: optional i64 level } // A single runtime profile