diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 1726a676b5..71fdf002bc 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1043,6 +1043,9 @@ DEFINE_Bool(enable_set_in_bitmap_value, "false"); DEFINE_Int64(max_hdfs_file_handle_cache_num, "20000"); DEFINE_Int64(max_external_file_meta_cache_num, "20000"); +// max_write_buffer_number for rocksdb +DEFINE_Int32(rocksdb_max_write_buffer_number, "5"); + #ifdef BE_TEST // test s3 DEFINE_String(test_s3_resource, "resource"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 81b5308132..06634747dc 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1059,6 +1059,9 @@ DECLARE_Int64(max_hdfs_file_handle_cache_num); // max number of meta info of external files, such as parquet footer DECLARE_Int64(max_external_file_meta_cache_num); +// max_write_buffer_number for rocksdb +DECLARE_Int32(rocksdb_max_write_buffer_number); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/olap/olap_meta.cpp b/be/src/olap/olap_meta.cpp index 889090d36c..4df89c04dc 100644 --- a/be/src/olap/olap_meta.cpp +++ b/be/src/olap/olap_meta.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -62,12 +63,24 @@ OlapMeta::OlapMeta(const std::string& root_path) : _root_path(root_path) {} OlapMeta::~OlapMeta() = default; +class RocksdbLogger : public rocksdb::Logger { +public: + void Logv(const char* format, va_list ap) override { + char buf[1024]; + vsnprintf(buf, sizeof(buf), format, ap); + LOG(INFO) << "[Rocksdb] " << buf; + } +}; + Status OlapMeta::init() { // init db DBOptions options; options.IncreaseParallelism(); options.create_if_missing = true; options.create_missing_column_families = true; + options.info_log = std::make_shared(); + options.info_log_level = rocksdb::WARN_LEVEL; + std::string db_path = _root_path + META_POSTFIX; std::vector column_families; // default column family is required @@ -76,6 +89,7 @@ Status OlapMeta::init() { // meta column family add prefix extractor to improve performance and ensure correctness ColumnFamilyOptions meta_column_family; + meta_column_family.max_write_buffer_number = config::rocksdb_max_write_buffer_number; meta_column_family.prefix_extractor.reset(NewFixedPrefixTransform(PREFIX_LENGTH)); column_families.emplace_back(META_COLUMN_FAMILY, meta_column_family);