From 89215306d325f386e1ba3cfc91e59c0ec79a342f Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 15 Nov 2023 08:19:12 +0800 Subject: [PATCH] [improve](load) add switch for vertical segment writer (#26996) --- be/src/common/config.cpp | 3 +++ be/src/common/config.h | 3 +++ be/src/olap/rowset/segment_creator.cpp | 15 +++++++++++---- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index a9b0848984..4d58b1fc94 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -351,6 +351,9 @@ DEFINE_Int32(vertical_compaction_max_row_source_memory_mb, "200"); // In vertical compaction, max dest segment file size DEFINE_mInt64(vertical_compaction_max_segment_size, "268435456"); +// If enabled, segments will be flushed column by column +DEFINE_mBool(enable_vertical_segment_writer, "true"); + // In ordered data compaction, min segment size for input rowset DEFINE_mInt32(ordered_data_compaction_min_segment_size, "10485760"); diff --git a/be/src/common/config.h b/be/src/common/config.h index b6e911438e..6317fc7636 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -404,6 +404,9 @@ DECLARE_Int32(vertical_compaction_max_row_source_memory_mb); // In vertical compaction, max dest segment file size DECLARE_mInt64(vertical_compaction_max_segment_size); +// If enabled, segments will be flushed column by column +DECLARE_mBool(enable_vertical_segment_writer); + // In ordered data compaction, min segment size for input rowset DECLARE_mInt32(ordered_data_compaction_min_segment_size); diff --git a/be/src/olap/rowset/segment_creator.cpp b/be/src/olap/rowset/segment_creator.cpp index f40d899bdc..5eabc2cc23 100644 --- a/be/src/olap/rowset/segment_creator.cpp +++ b/be/src/olap/rowset/segment_creator.cpp @@ -50,11 +50,18 @@ Status SegmentFlusher::flush_single_block(const vectorized::Block* block, int32_ if (block->rows() == 0) { return Status::OK(); } - std::unique_ptr writer; bool no_compression = block->bytes() <= config::segment_compression_threshold_kb * 1024; - RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema)); - RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows())); - RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size)); + if (config::enable_vertical_segment_writer) { + std::unique_ptr writer; + RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema)); + RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows())); + RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size)); + } else { + std::unique_ptr writer; + RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema)); + RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows())); + RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size)); + } return Status::OK(); }