diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp index b8d3fa98c6..f86c16f134 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp @@ -37,7 +37,9 @@ BinaryDictPageBuilder::BinaryDictPageBuilder(const PageBuilderOptions& options) // TODO: the data page builder type can be created by Factory according to user config _data_page_builder.reset(new BitshufflePageBuilder(options)); PageBuilderOptions dict_builder_options; - dict_builder_options.data_page_size = _options.dict_page_size; + dict_builder_options.data_page_size = + std::min(_options.data_page_size, _options.dict_page_size); + dict_builder_options.is_dict_page = true; _dict_builder.reset(new BinaryPlainPageBuilder(dict_builder_options)); reset(); } @@ -118,6 +120,10 @@ Status BinaryDictPageBuilder::add(const uint8_t* vals, size_t* count) { } OwnedSlice BinaryDictPageBuilder::finish() { + if (VLOG_DEBUG_IS_ON && _encoding_type == DICT_ENCODING) { + VLOG_DEBUG << "dict page size:" << _dict_builder->size(); + } + DCHECK(!_finished); _finished = true; diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h b/be/src/olap/rowset/segment_v2/binary_plain_page.h index 96cfc392a5..9faaeab3eb 100644 --- a/be/src/olap/rowset/segment_v2/binary_plain_page.h +++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h @@ -53,8 +53,14 @@ public: } bool is_page_full() override { - // data_page_size is 0, do not limit the page size - return _options.data_page_size != 0 && _size_estimate > _options.data_page_size; + bool ret = false; + if (_options.is_dict_page) { + // dict_page_size is 0, do not limit the page size + ret = _options.dict_page_size != 0 && _size_estimate > _options.dict_page_size; + } else { + ret = _options.data_page_size != 0 && _size_estimate > _options.data_page_size; + } + return ret; } Status add(const uint8_t* vals, size_t* count) override { @@ -104,7 +110,9 @@ public: void reset() override { _offsets.clear(); _buffer.clear(); - _buffer.reserve(_options.data_page_size == 0 ? 1024 : _options.data_page_size); + _buffer.reserve(_options.data_page_size == 0 + ? 1024 + : std::min(_options.data_page_size, _options.dict_page_size)); _size_estimate = sizeof(uint32_t); _finished = false; _last_value_size = 0; diff --git a/be/src/olap/rowset/segment_v2/options.h b/be/src/olap/rowset/segment_v2/options.h index 9405eb19cf..19041f4c51 100644 --- a/be/src/olap/rowset/segment_v2/options.h +++ b/be/src/olap/rowset/segment_v2/options.h @@ -30,6 +30,8 @@ struct PageBuilderOptions { size_t dict_page_size = DEFAULT_PAGE_SIZE; bool need_check_bitmap = true; + + bool is_dict_page = false; // page used for saving dictionary }; struct PageDecoderOptions {