database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improve](load) add switch for vertical segment writer (#26996)

Browse Source
This commit is contained in:
Kaijie Chen
2023-11-15 08:19:12 +08:00
committed by GitHub
parent 82a88366f1
commit 89215306d3
3 changed files with 17 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/common/config.cpp
View File

@ -351,6 +351,9 @@ DEFINE_Int32(vertical_compaction_max_row_source_memory_mb, "200");
// In vertical compaction, max dest segment file size
DEFINE_mInt64(vertical_compaction_max_segment_size, "268435456");
// If enabled, segments will be flushed column by column
DEFINE_mBool(enable_vertical_segment_writer, "true");
// In ordered data compaction, min segment size for input rowset
DEFINE_mInt32(ordered_data_compaction_min_segment_size, "10485760");

3
be/src/common/config.h
View File

@ -404,6 +404,9 @@ DECLARE_Int32(vertical_compaction_max_row_source_memory_mb);
// In vertical compaction, max dest segment file size
DECLARE_mInt64(vertical_compaction_max_segment_size);
// If enabled, segments will be flushed column by column
DECLARE_mBool(enable_vertical_segment_writer);
// In ordered data compaction, min segment size for input rowset
DECLARE_mInt32(ordered_data_compaction_min_segment_size);

15
be/src/olap/rowset/segment_creator.cpp
View File

@ -50,11 +50,18 @@ Status SegmentFlusher::flush_single_block(const vectorized::Block* block, int32_
if (block->rows() == 0) {
return Status::OK();
}
std::unique_ptr<segment_v2::VerticalSegmentWriter> writer;
bool no_compression = block->bytes() <= config::segment_compression_threshold_kb * 1024;
RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema));
RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows()));
RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
if (config::enable_vertical_segment_writer) {
std::unique_ptr<segment_v2::VerticalSegmentWriter> writer;
RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema));
RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows()));
RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
} else {
std::unique_ptr<segment_v2::SegmentWriter> writer;
RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema));
RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows()));
RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
}
return Status::OK();
}