database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](profile) set datalake profile level as 1 (#25686)

Browse Source 
Follow #25491, only the profile marked as 1 will be shown in simplified profile.
This commit is contained in:
Ashin Gau
2023-10-24 09:55:25 +08:00
committed by GitHub
parent 0e0f8090f7
commit d62e914205
5 changed files with 86 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
be/src/io/cache/block/block_file_cache_profile.h vendored
View File

@ -116,22 +116,23 @@ struct FileCacheProfileReporter {
FileCacheProfileReporter(RuntimeProfile* profile) {
static const char* cache_profile = "FileCache";
ADD_TIMER(profile, cache_profile);
num_local_io_total =
ADD_CHILD_COUNTER(profile, "NumLocalIOTotal", TUnit::UNIT, cache_profile);
num_remote_io_total =
ADD_CHILD_COUNTER(profile, "NumRemoteIOTotal", TUnit::UNIT, cache_profile);
local_io_timer = ADD_CHILD_TIMER(profile, "LocalIOUseTimer", cache_profile);
remote_io_timer = ADD_CHILD_TIMER(profile, "RemoteIOUseTimer", cache_profile);
write_cache_io_timer = ADD_CHILD_TIMER(profile, "WriteCacheIOUseTimer", cache_profile);
bytes_write_into_cache =
ADD_CHILD_COUNTER(profile, "BytesWriteIntoCache", TUnit::BYTES, cache_profile);
num_skip_cache_io_total =
ADD_CHILD_COUNTER(profile, "NumSkipCacheIOTotal", TUnit::UNIT, cache_profile);
bytes_scanned_from_cache =
ADD_CHILD_COUNTER(profile, "BytesScannedFromCache", TUnit::BYTES, cache_profile);
bytes_scanned_from_remote =
ADD_CHILD_COUNTER(profile, "BytesScannedFromRemote", TUnit::BYTES, cache_profile);
ADD_TIMER_WITH_LEVEL(profile, cache_profile, 1);
num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumLocalIOTotal", TUnit::UNIT,
cache_profile, 1);
num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumRemoteIOTotal", TUnit::UNIT,
cache_profile, 1);
local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LocalIOUseTimer", cache_profile, 1);
remote_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "RemoteIOUseTimer", cache_profile, 1);
write_cache_io_timer =
ADD_CHILD_TIMER_WITH_LEVEL(profile, "WriteCacheIOUseTimer", cache_profile, 1);
bytes_write_into_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "BytesWriteIntoCache",
TUnit::BYTES, cache_profile, 1);
num_skip_cache_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumSkipCacheIOTotal",
TUnit::UNIT, cache_profile, 1);
bytes_scanned_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "BytesScannedFromCache",
TUnit::BYTES, cache_profile, 1);
bytes_scanned_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "BytesScannedFromRemote",
TUnit::BYTES, cache_profile, 1);
}
void update(const FileCacheStatistics* statistics) {

19
be/src/io/fs/buffered_reader.h
View File

@ -143,14 +143,17 @@ public:
_remaining = TOTAL_BUFFER_SIZE;
if (_profile != nullptr) {
const char* random_profile = "MergedSmallIO";
ADD_TIMER(_profile, random_profile);
_copy_time = ADD_CHILD_TIMER(_profile, "CopyTime", random_profile);
_read_time = ADD_CHILD_TIMER(_profile, "ReadTime", random_profile);
_request_io = ADD_CHILD_COUNTER(_profile, "RequestIO", TUnit::UNIT, random_profile);
_merged_io = ADD_CHILD_COUNTER(_profile, "MergedIO", TUnit::UNIT, random_profile);
_request_bytes =
ADD_CHILD_COUNTER(_profile, "RequestBytes", TUnit::BYTES, random_profile);
_read_bytes = ADD_CHILD_COUNTER(_profile, "MergedBytes", TUnit::BYTES, random_profile);
ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1);
_copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", random_profile, 1);
_read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", random_profile, 1);
_request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT,
random_profile, 1);
_merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedIO", TUnit::UNIT,
random_profile, 1);
_request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES,
random_profile, 1);
_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedBytes", TUnit::BYTES,
random_profile, 1);
}
}

4
be/src/util/runtime_profile.h
View File

@ -60,7 +60,11 @@ class TRuntimeProfileTree;
#define ADD_TIMER_WITH_LEVEL(profile, name, level) \
(profile)->add_counter_with_level(name, TUnit::TIME_NS, level)
#define ADD_CHILD_COUNTER(profile, name, type, parent) (profile)->add_counter(name, type, parent)
#define ADD_CHILD_COUNTER_WITH_LEVEL(profile, name, type, parent, level) \
(profile)->add_counter(name, type, parent, level)
#define ADD_CHILD_TIMER(profile, name, parent) (profile)->add_counter(name, TUnit::TIME_NS, parent)
#define ADD_CHILD_TIMER_WITH_LEVEL(profile, name, parent, level) \
(profile)->add_counter(name, TUnit::TIME_NS, parent, level)
#define SCOPED_TIMER(c) ScopedTimer<MonotonicStopWatch> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)
#define SCOPED_TIMER_ATOMIC(c) \
ScopedTimer<MonotonicStopWatch, std::atomic_bool> MACRO_CONCAT(SCOPED_TIMER, __COUNTER__)(c)

27
be/src/vec/exec/format/orc/vorc_reader.cpp
View File

@ -198,20 +198,25 @@ int64_t OrcReader::size() const {
void OrcReader::_init_profile() {
if (_profile != nullptr) {
static const char* orc_profile = "OrcReader";
ADD_TIMER(_profile, orc_profile);
_orc_profile.read_time = ADD_TIMER(_profile, "FileReadTime");
_orc_profile.read_calls = ADD_COUNTER(_profile, "FileReadCalls", TUnit::UNIT);
_orc_profile.read_bytes = ADD_COUNTER(_profile, "FileReadBytes", TUnit::BYTES);
_orc_profile.column_read_time = ADD_CHILD_TIMER(_profile, "ColumnReadTime", orc_profile);
_orc_profile.get_batch_time = ADD_CHILD_TIMER(_profile, "GetBatchTime", orc_profile);
ADD_TIMER_WITH_LEVEL(_profile, orc_profile, 1);
_orc_profile.read_time = ADD_TIMER_WITH_LEVEL(_profile, "FileReadTime", 1);
_orc_profile.read_calls = ADD_COUNTER_WITH_LEVEL(_profile, "FileReadCalls", TUnit::UNIT, 1);
_orc_profile.read_bytes =
ADD_COUNTER_WITH_LEVEL(_profile, "FileReadBytes", TUnit::BYTES, 1);
_orc_profile.column_read_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ColumnReadTime", orc_profile, 1);
_orc_profile.get_batch_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "GetBatchTime", orc_profile, 1);
_orc_profile.create_reader_time =
ADD_CHILD_TIMER(_profile, "CreateReaderTime", orc_profile);
_orc_profile.init_column_time = ADD_CHILD_TIMER(_profile, "InitColumnTime", orc_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CreateReaderTime", orc_profile, 1);
_orc_profile.init_column_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "InitColumnTime", orc_profile, 1);
_orc_profile.set_fill_column_time =
ADD_CHILD_TIMER(_profile, "SetFillColumnTime", orc_profile);
_orc_profile.decode_value_time = ADD_CHILD_TIMER(_profile, "DecodeValueTime", orc_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "SetFillColumnTime", orc_profile, 1);
_orc_profile.decode_value_time =
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeValueTime", orc_profile, 1);
_orc_profile.decode_null_map_time =
ADD_CHILD_TIMER(_profile, "DecodeNullMapTime", orc_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeNullMapTime", orc_profile, 1);
}
}

74
be/src/vec/exec/format/parquet/vparquet_reader.cpp
View File

@ -106,58 +106,60 @@ ParquetReader::~ParquetReader() {
void ParquetReader::_init_profile() {
if (_profile != nullptr) {
static const char* parquet_profile = "ParquetReader";
ADD_TIMER(_profile, parquet_profile);
ADD_TIMER_WITH_LEVEL(_profile, parquet_profile, 1);
_parquet_profile.filtered_row_groups =
ADD_CHILD_COUNTER(_profile, "FilteredGroups", TUnit::UNIT, parquet_profile);
_parquet_profile.to_read_row_groups =
ADD_CHILD_COUNTER(_profile, "ReadGroups", TUnit::UNIT, parquet_profile);
_parquet_profile.filtered_group_rows =
ADD_CHILD_COUNTER(_profile, "FilteredRowsByGroup", TUnit::UNIT, parquet_profile);
_parquet_profile.filtered_page_rows =
ADD_CHILD_COUNTER(_profile, "FilteredRowsByPage", TUnit::UNIT, parquet_profile);
_parquet_profile.lazy_read_filtered_rows =
ADD_CHILD_COUNTER(_profile, "FilteredRowsByLazyRead", TUnit::UNIT, parquet_profile);
_parquet_profile.filtered_bytes =
ADD_CHILD_COUNTER(_profile, "FilteredBytes", TUnit::BYTES, parquet_profile);
_parquet_profile.raw_rows_read =
ADD_CHILD_COUNTER(_profile, "RawRowsRead", TUnit::UNIT, parquet_profile);
_parquet_profile.to_read_bytes =
ADD_CHILD_COUNTER(_profile, "ReadBytes", TUnit::BYTES, parquet_profile);
_parquet_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "FilteredGroups", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.to_read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "ReadGroups", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "FilteredRowsByGroup", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.filtered_page_rows = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "FilteredRowsByPage", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "FilteredRowsByLazyRead", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "FilteredBytes", TUnit::BYTES, parquet_profile, 1);
_parquet_profile.raw_rows_read = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.to_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "ReadBytes", TUnit::BYTES, parquet_profile, 1);
_parquet_profile.column_read_time =
ADD_CHILD_TIMER(_profile, "ColumnReadTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ColumnReadTime", parquet_profile, 1);
_parquet_profile.parse_meta_time =
ADD_CHILD_TIMER(_profile, "ParseMetaTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ParseMetaTime", parquet_profile, 1);
_parquet_profile.parse_footer_time =
ADD_CHILD_TIMER(_profile, "ParseFooterTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ParseFooterTime", parquet_profile, 1);
_parquet_profile.open_file_time =
ADD_CHILD_TIMER(_profile, "FileOpenTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FileOpenTime", parquet_profile, 1);
_parquet_profile.open_file_num =
ADD_CHILD_COUNTER(_profile, "FileNum", TUnit::UNIT, parquet_profile);
ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.page_index_filter_time =
ADD_CHILD_TIMER(_profile, "PageIndexFilterTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "PageIndexFilterTime", parquet_profile, 1);
_parquet_profile.row_group_filter_time =
ADD_CHILD_TIMER(_profile, "RowGroupFilterTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RowGroupFilterTime", parquet_profile, 1);
_parquet_profile.file_read_time = ADD_TIMER(_profile, "FileReadTime");
_parquet_profile.file_read_calls = ADD_COUNTER(_profile, "FileReadCalls", TUnit::UNIT);
_parquet_profile.file_read_time = ADD_TIMER_WITH_LEVEL(_profile, "FileReadTime", 1);
_parquet_profile.file_read_calls =
ADD_COUNTER_WITH_LEVEL(_profile, "FileReadCalls", TUnit::UNIT, 1);
_parquet_profile.file_meta_read_calls =
ADD_COUNTER(_profile, "FileMetaReadCalls", TUnit::UNIT);
_parquet_profile.file_read_bytes = ADD_COUNTER(_profile, "FileReadBytes", TUnit::BYTES);
ADD_COUNTER_WITH_LEVEL(_profile, "FileMetaReadCalls", TUnit::UNIT, 1);
_parquet_profile.file_read_bytes =
ADD_COUNTER_WITH_LEVEL(_profile, "FileReadBytes", TUnit::BYTES, 1);
_parquet_profile.decompress_time =
ADD_CHILD_TIMER(_profile, "DecompressTime", parquet_profile);
_parquet_profile.decompress_cnt =
ADD_CHILD_COUNTER(_profile, "DecompressCount", TUnit::UNIT, parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecompressTime", parquet_profile, 1);
_parquet_profile.decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(
_profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1);
_parquet_profile.decode_header_time =
ADD_CHILD_TIMER(_profile, "DecodeHeaderTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeHeaderTime", parquet_profile, 1);
_parquet_profile.decode_value_time =
ADD_CHILD_TIMER(_profile, "DecodeValueTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeValueTime", parquet_profile, 1);
_parquet_profile.decode_dict_time =
ADD_CHILD_TIMER(_profile, "DecodeDictTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeDictTime", parquet_profile, 1);
_parquet_profile.decode_level_time =
ADD_CHILD_TIMER(_profile, "DecodeLevelTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeLevelTime", parquet_profile, 1);
_parquet_profile.decode_null_map_time =
ADD_CHILD_TIMER(_profile, "DecodeNullMapTime", parquet_profile);
ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeNullMapTime", parquet_profile, 1);
}
}