database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](file cache)Support set min file segment size while use block file cache (#19536)

Browse Source
This commit is contained in:
zxealous
2023-05-17 10:23:33 +08:00
committed by GitHub
parent 8fd1eb0d1e
commit 2d9cc8fe8f
3 changed files with 14 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
be/src/common/config.cpp
View File

@ -939,8 +939,16 @@ DEFINE_Bool(enable_file_cache, "false");
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
DEFINE_String(file_cache_path, "");
DEFINE_Int64(file_cache_max_file_segment_size, "4194304"); // 4MB
DEFINE_Validator(file_cache_max_file_segment_size,
[](const int64_t config) -> bool { return config >= 4096; }); // 4KB
// 4KB <= file_cache_max_file_segment_size <= 256MB
DEFINE_Validator(file_cache_max_file_segment_size, [](const int64_t config) -> bool {
return config >= 4096 && config <= 268435456;
});
DEFINE_Int64(file_cache_min_file_segment_size, "1048576"); // 1MB
// 4KB <= file_cache_min_file_segment_size <= 256MB
DEFINE_Validator(file_cache_min_file_segment_size, [](const int64_t config) -> bool {
return config >= 4096 && config <= 268435456 &&
config <= config::file_cache_max_file_segment_size;
});
DEFINE_Bool(clear_file_cache, "false");
DEFINE_Bool(enable_file_cache_query_limit, "false");

1
be/src/common/config.h
View File

@ -962,6 +962,7 @@ DECLARE_Bool(enable_file_cache);
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
DECLARE_String(file_cache_path);
DECLARE_Int64(file_cache_min_file_segment_size);
DECLARE_Int64(file_cache_max_file_segment_size);
DECLARE_Bool(clear_file_cache);
DECLARE_Bool(enable_file_cache_query_limit);

6
be/src/io/cache/block/cached_remote_file_reader.cpp vendored
View File

@ -76,9 +76,9 @@ Status CachedRemoteFileReader::close() {
std::pair<size_t, size_t> CachedRemoteFileReader::_align_size(size_t offset,
size_t read_size) const {
size_t min_size = 1024 * 1024; // 1MB;
size_t segment_size = std::min(std::max(read_size, min_size),
(size_t)config::file_cache_max_file_segment_size);
size_t segment_size =
std::min(std::max(read_size, (size_t)config::file_cache_min_file_segment_size),
(size_t)config::file_cache_max_file_segment_size);
segment_size = BitUtil::next_power_of_two(segment_size);
size_t left = offset;
size_t right = offset + read_size - 1;