database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](merge-on-write) increasing the max_write_buffer_number parameter to improve save meta performance (#21243)

Browse Source
This commit is contained in:
Xin Liao
2023-06-28 11:32:11 +08:00
committed by GitHub
parent 824c1fe165
commit 5d1fb33f2d
3 changed files with 20 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/common/config.cpp
View File

@ -1043,6 +1043,9 @@ DEFINE_Bool(enable_set_in_bitmap_value, "false");
DEFINE_Int64(max_hdfs_file_handle_cache_num, "20000");
DEFINE_Int64(max_external_file_meta_cache_num, "20000");
// max_write_buffer_number for rocksdb
DEFINE_Int32(rocksdb_max_write_buffer_number, "5");
#ifdef BE_TEST
// test s3
DEFINE_String(test_s3_resource, "resource");

3
be/src/common/config.h
View File

@ -1059,6 +1059,9 @@ DECLARE_Int64(max_hdfs_file_handle_cache_num);
// max number of meta info of external files, such as parquet footer
DECLARE_Int64(max_external_file_meta_cache_num);
// max_write_buffer_number for rocksdb
DECLARE_Int32(rocksdb_max_write_buffer_number);
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);

14
be/src/olap/olap_meta.cpp
View File

@ -19,6 +19,7 @@
#include <fmt/format.h>
#include <fmt/ranges.h>
#include <rocksdb/env.h>
#include <rocksdb/iterator.h>
#include <rocksdb/status.h>
#include <rocksdb/write_batch.h>
@ -62,12 +63,24 @@ OlapMeta::OlapMeta(const std::string& root_path) : _root_path(root_path) {}
OlapMeta::~OlapMeta() = default;
class RocksdbLogger : public rocksdb::Logger {
public:
void Logv(const char* format, va_list ap) override {
char buf[1024];
vsnprintf(buf, sizeof(buf), format, ap);
LOG(INFO) << "[Rocksdb] " << buf;
}
};
Status OlapMeta::init() {
// init db
DBOptions options;
options.IncreaseParallelism();
options.create_if_missing = true;
options.create_missing_column_families = true;
options.info_log = std::make_shared<RocksdbLogger>();
options.info_log_level = rocksdb::WARN_LEVEL;
std::string db_path = _root_path + META_POSTFIX;
std::vector<ColumnFamilyDescriptor> column_families;
// default column family is required
@ -76,6 +89,7 @@ Status OlapMeta::init() {
// meta column family add prefix extractor to improve performance and ensure correctness
ColumnFamilyOptions meta_column_family;
meta_column_family.max_write_buffer_number = config::rocksdb_max_write_buffer_number;
meta_column_family.prefix_extractor.reset(NewFixedPrefixTransform(PREFIX_LENGTH));
column_families.emplace_back(META_COLUMN_FAMILY, meta_column_family);