database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](filecache) add profile for file cache (#16223)

Browse Source
This commit is contained in:
Ashin Gau
2023-01-30 10:46:31 +08:00
committed by GitHub
parent 28fcc093a8
commit 07d58e531a
2 changed files with 51 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
be/src/io/cache/block/block_file_cache_profile.h vendored
View File

@ -29,6 +29,7 @@
#include "olap/olap_common.h"
#include "util/doris_metrics.h"
#include "util/metrics.h"
#include "util/runtime_profile.h"
namespace doris {
namespace io {
@ -111,5 +112,49 @@ struct FileCacheProfile {
FileCacheStatistics report(int64_t table_id, int64_t partition_id);
};
struct FileCacheProfileReporter {
RuntimeProfile::Counter* num_io_total;
RuntimeProfile::Counter* num_io_hit_cache;
RuntimeProfile::Counter* num_io_bytes_read_total;
RuntimeProfile::Counter* num_io_bytes_read_from_file_cache;
RuntimeProfile::Counter* num_io_bytes_read_from_write_cache;
RuntimeProfile::Counter* num_io_written_in_file_cache;
RuntimeProfile::Counter* num_io_bytes_written_in_file_cache;
RuntimeProfile::Counter* num_io_bytes_skip_cache;
FileCacheProfileReporter(RuntimeProfile* profile) {
static const char* cache_profile = "FileCache";
ADD_TIMER(profile, cache_profile);
num_io_total = ADD_CHILD_COUNTER(profile, "IOTotalNum", TUnit::UNIT, cache_profile);
num_io_hit_cache = ADD_CHILD_COUNTER(profile, "IOHitCacheNum", TUnit::UNIT, cache_profile);
num_io_bytes_read_total =
ADD_CHILD_COUNTER(profile, "ReadTotalBytes", TUnit::BYTES, cache_profile);
num_io_bytes_read_from_file_cache =
ADD_CHILD_COUNTER(profile, "ReadFromFileCacheBytes", TUnit::BYTES, cache_profile);
num_io_bytes_read_from_write_cache =
ADD_CHILD_COUNTER(profile, "ReadFromWriteCacheBytes", TUnit::BYTES, cache_profile);
num_io_written_in_file_cache =
ADD_CHILD_COUNTER(profile, "WriteInFileCacheNum", TUnit::UNIT, cache_profile);
num_io_bytes_written_in_file_cache =
ADD_CHILD_COUNTER(profile, "WriteInFileCacheBytes", TUnit::BYTES, cache_profile);
num_io_bytes_skip_cache =
ADD_CHILD_COUNTER(profile, "SkipCacheBytes", TUnit::BYTES, cache_profile);
}
void update(FileCacheStatistics* statistics) {
COUNTER_UPDATE(num_io_total, statistics->num_io_total);
COUNTER_UPDATE(num_io_hit_cache, statistics->num_io_hit_cache);
COUNTER_UPDATE(num_io_bytes_read_total, statistics->num_io_bytes_read_total);
COUNTER_UPDATE(num_io_bytes_read_from_file_cache,
statistics->num_io_bytes_read_from_file_cache);
COUNTER_UPDATE(num_io_bytes_read_from_write_cache,
statistics->num_io_bytes_read_from_write_cache);
COUNTER_UPDATE(num_io_written_in_file_cache, statistics->num_io_written_in_file_cache);
COUNTER_UPDATE(num_io_bytes_written_in_file_cache,
statistics->num_io_bytes_written_in_file_cache);
COUNTER_UPDATE(num_io_bytes_skip_cache, statistics->num_io_bytes_skip_cache);
}
};
} // namespace io
} // namespace doris

6
be/src/vec/exec/scan/vfile_scanner.cpp
View File

@ -25,6 +25,7 @@
#include "common/logging.h"
#include "common/utils.h"
#include "exec/text_converter.hpp"
#include "io/cache/block/block_file_cache_profile.h"
#include "olap/iterators.h"
#include "runtime/descriptors.h"
#include "runtime/raw_value.h"
@ -752,6 +753,11 @@ Status VFileScanner::close(RuntimeState* state) {
_push_down_expr->close(state);
}
if (config::enable_file_cache) {
io::FileCacheProfileReporter cache_profile(_profile);
cache_profile.update(_file_cache_statistics.get());
}
RETURN_IF_ERROR(VScanner::close(state));
return Status::OK();
}