database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](merge-on-write) add primary key data page size config (#20961)

Browse Source
This commit is contained in:
Xin Liao
2023-06-20 19:51:02 +08:00
committed by GitHub
parent 19dd35f908
commit 9eade148dd
6 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/common/config.cpp
View File

@ -306,6 +306,8 @@ DEFINE_Bool(disable_storage_row_cache, "true");
// Cache for mow primary key storage page size
DEFINE_String(pk_storage_page_cache_limit, "10%");
// data page size for primary key index
DEFINE_Int32(primary_key_data_page_size, "32768");
DEFINE_Bool(enable_low_cardinality_optimize, "true");
DEFINE_Bool(enable_low_cardinality_cache_code, "true");

2
be/src/common/config.h
View File

@ -347,6 +347,8 @@ DECLARE_Bool(disable_storage_row_cache);
// Cache for mow primary key storage page size, it's seperated from
// storage_page_cache_limit
DECLARE_String(pk_storage_page_cache_limit);
// data page size for primary key index
DECLARE_Int32(primary_key_data_page_size);
DECLARE_Bool(enable_low_cardinality_optimize);
DECLARE_Bool(enable_low_cardinality_cache_code);

1
be/src/olap/primary_key_index.cpp
View File

@ -38,6 +38,7 @@ Status PrimaryKeyIndexBuilder::init() {
segment_v2::IndexedColumnWriterOptions options;
options.write_ordinal_index = true;
options.write_value_index = true;
options.data_page_size = config::primary_key_data_page_size;
options.encoding = segment_v2::EncodingInfo::get_default_encoding(type_info, true);
// TODO(liaoxin) test to confirm whether it needs to be compressed
options.compression = segment_v2::NO_COMPRESSION; // currently not compressed

3
be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
View File

@ -37,6 +37,8 @@ using namespace ErrorCode;
namespace segment_v2 {
static bvar::Adder<uint64_t> g_index_reader_bytes("doris_pk", "index_reader_bytes");
static bvar::Adder<uint64_t> g_index_reader_compressed_bytes("doris_pk",
"index_reader_compressed_bytes");
static bvar::PerSecond<bvar::Adder<uint64_t>> g_index_reader_bytes_per_second(
"doris_pk", "index_reader_bytes_per_second", &g_index_reader_bytes, 60);
static bvar::Adder<uint64_t> g_index_reader_pages("doris_pk", "index_reader_pages");
@ -119,6 +121,7 @@ Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle,
opts.pre_decode = pre_decode;
auto st = PageIO::read_and_decompress_page(opts, handle, body, footer);
g_index_reader_compressed_bytes << pp.size;
g_index_reader_bytes << footer->uncompressed_size();
g_index_reader_pages << 1;
return st;

1
be/src/olap/rowset/segment_v2/indexed_column_writer.cpp
View File

@ -63,6 +63,7 @@ Status IndexedColumnWriter::init() {
PageBuilder* data_page_builder = nullptr;
PageBuilderOptions builder_option;
builder_option.need_check_bitmap = false;
builder_option.data_page_size = _options.data_page_size;
RETURN_IF_ERROR(encoding_info->create_page_builder(builder_option, &data_page_builder));
_data_page_builder.reset(data_page_builder);

1
be/src/olap/rowset/segment_v2/indexed_column_writer.h
View File

@ -47,6 +47,7 @@ class PageBuilder;
struct IndexedColumnWriterOptions {
size_t index_page_size = 64 * 1024;
size_t data_page_size = 1024 * 1024;
bool write_ordinal_index = false;
bool write_value_index = false;
EncodingTypePB encoding = DEFAULT_ENCODING;