From e8360f5eee11164ea3386ea8088c69f48d5b9362 Mon Sep 17 00:00:00 2001 From: ZHAO Chun Date: Wed, 16 Jan 2019 18:57:04 +0800 Subject: [PATCH] Add counters to OlapScanNode (#538) There is unnegligible cost to covnert VectorRowBatch to RowBatch, When we seek block, we only read one row from engine to minimize this convert cost. This patch can optimize some query's time from 5s to 2s --- be/src/exec/olap_scan_node.cpp | 3 +++ be/src/exec/olap_scan_node.h | 3 +++ be/src/exec/olap_scanner.cpp | 3 +++ be/src/olap/column_data.cpp | 12 ++++++++---- be/src/olap/column_data.h | 4 ++-- be/src/olap/olap_common.h | 2 ++ be/src/olap/segment_reader.cpp | 1 + 7 files changed, 22 insertions(+), 6 deletions(-) diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp index b0a456d99b..b9b4afd7e2 100644 --- a/be/src/exec/olap_scan_node.cpp +++ b/be/src/exec/olap_scan_node.cpp @@ -109,6 +109,7 @@ void OlapScanNode::_init_counter(RuntimeState* state) { #endif ADD_TIMER(_runtime_profile, "ShowHintsTime"); + _reader_init_timer = ADD_TIMER(_runtime_profile, "ReaderInitTime"); _read_compressed_counter = ADD_COUNTER(_runtime_profile, "CompressedBytesRead", TUnit::BYTES); _read_uncompressed_counter = @@ -120,6 +121,8 @@ void OlapScanNode::_init_counter(RuntimeState* state) { ADD_TIMER(_runtime_profile, "BlockFetchTime"); _raw_rows_counter = ADD_COUNTER(_runtime_profile, "RawRowsRead", TUnit::UNIT); + _block_convert_timer = ADD_TIMER(_runtime_profile, "BlockConvertTime"); + _block_seek_timer = ADD_TIMER(_runtime_profile, "BlockSeekTime"); _rows_vec_cond_counter = ADD_COUNTER(_runtime_profile, "RowsVectorPredFiltered", TUnit::UNIT); diff --git a/be/src/exec/olap_scan_node.h b/be/src/exec/olap_scan_node.h index 0db834d0a0..935896315d 100644 --- a/be/src/exec/olap_scan_node.h +++ b/be/src/exec/olap_scan_node.h @@ -245,6 +245,7 @@ private: RuntimeProfile::Counter* _scan_timer; RuntimeProfile::Counter* _tablet_counter; RuntimeProfile::Counter* _rows_pushed_cond_filtered_counter = nullptr; + RuntimeProfile::Counter* _reader_init_timer = nullptr; TResourceInfo* _resource_info; @@ -265,6 +266,8 @@ private: RuntimeProfile::Counter* _stats_filtered_counter = nullptr; RuntimeProfile::Counter* _del_filtered_counter = nullptr; + RuntimeProfile::Counter* _block_seek_timer = nullptr; + RuntimeProfile::Counter* _block_convert_timer = nullptr; RuntimeProfile::Counter* _block_load_timer = nullptr; RuntimeProfile::Counter* _block_load_counter = nullptr; RuntimeProfile::Counter* _block_fetch_timer = nullptr; diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index e3576b0dac..2f1ea579ba 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -121,6 +121,7 @@ Status OlapScanner::_prepare( Status OlapScanner::open() { RETURN_IF_ERROR(_ctor_status); + SCOPED_TIMER(_parent->_reader_init_timer); if (_conjunct_ctxs.size() > _direct_conjunct_size) { _use_pushdown_conjuncts = true; @@ -430,6 +431,8 @@ void OlapScanner::update_counter() { COUNTER_UPDATE(_parent->_block_load_timer, _reader->stats().block_load_ns); COUNTER_UPDATE(_parent->_block_load_counter, _reader->stats().blocks_load); COUNTER_UPDATE(_parent->_block_fetch_timer, _reader->stats().block_fetch_ns); + COUNTER_UPDATE(_parent->_block_seek_timer, _reader->stats().block_seek_ns); + COUNTER_UPDATE(_parent->_block_convert_timer, _reader->stats().block_convert_ns); COUNTER_UPDATE(_parent->_raw_rows_counter, _reader->stats().raw_rows_read); // COUNTER_UPDATE(_parent->_filtered_rows_counter, _reader->stats().num_rows_filtered); diff --git a/be/src/olap/column_data.cpp b/be/src/olap/column_data.cpp index 5d6f146c1b..2d59b3b935 100644 --- a/be/src/olap/column_data.cpp +++ b/be/src/olap/column_data.cpp @@ -344,7 +344,7 @@ const RowCursor* ColumnData::seek_and_get_current_row(const RowBlockPosition& po << ", segment:" << position.segment << ", block:" << position.data_offset; return nullptr; } - res = _get_block(true); + res = _get_block(true, 1); if (res != OLAP_SUCCESS) { LOG(WARNING) << "Fail to get block in seek_and_get_current_row, res=" << res << ", segment:" << position.segment << ", block:" << position.data_offset; @@ -634,7 +634,7 @@ OLAPStatus ColumnData::_schema_change_init() { } OLAPStatus ColumnData::_get_block_from_reader( - VectorizedRowBatch** got_batch, bool without_filter) { + VectorizedRowBatch** got_batch, bool without_filter, int rows_read) { VectorizedRowBatch* vec_batch = nullptr; if (_is_normal_read) { vec_batch = _read_vector_batch.get(); @@ -652,6 +652,9 @@ OLAPStatus ColumnData::_get_block_from_reader( << ", _segment_eof:" << _segment_eof; #endif vec_batch->clear(); + if (rows_read > 0) { + vec_batch->set_limit(rows_read); + } // If we are going to read last block, we need to set batch limit to the end of key // if without_filter is true and _end_key_is_set is true, this must seek to start row's // block, we must load the entire block. @@ -689,10 +692,10 @@ OLAPStatus ColumnData::_get_block_from_reader( return OLAP_SUCCESS; } -OLAPStatus ColumnData::_get_block(bool without_filter) { +OLAPStatus ColumnData::_get_block(bool without_filter, int rows_read) { do { VectorizedRowBatch* vec_batch = nullptr; - auto res = _get_block_from_reader(&vec_batch, without_filter); + auto res = _get_block_from_reader(&vec_batch, without_filter, rows_read); if (res != OLAP_SUCCESS) { return res; } @@ -709,6 +712,7 @@ OLAPStatus ColumnData::_get_block(bool without_filter) { if (vec_batch->size() == 0) { continue; } + SCOPED_RAW_TIMER(&_stats->block_convert_ns); // when reach here, we have already read a block successfully _read_block->clear(); vec_batch->dump_to_row_block(_read_block.get()); diff --git a/be/src/olap/column_data.h b/be/src/olap/column_data.h index 78220bf0c9..af9b96a129 100644 --- a/be/src/olap/column_data.h +++ b/be/src/olap/column_data.h @@ -149,10 +149,10 @@ private: // get block from reader, just read vector batch from _current_segment. // The read batch return by got_batch. OLAPStatus _get_block_from_reader( - VectorizedRowBatch** got_batch, bool without_filter); + VectorizedRowBatch** got_batch, bool without_filter, int rows_read); // get block from segment reader. If this function returns OLAP_SUCCESS - OLAPStatus _get_block(bool without_filter); + OLAPStatus _get_block(bool without_filter, int rows_read = 0); const RowCursor* _current_row() { _read_block->get_row(_read_block->pos(), &_cursor); diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 598fb75dd0..ddeddcee6f 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -222,6 +222,8 @@ struct OlapReaderStatistics { int64_t block_load_ns = 0; int64_t blocks_load = 0; int64_t block_fetch_ns = 0; + int64_t block_seek_ns = 0; + int64_t block_convert_ns = 0; int64_t raw_rows_read = 0; diff --git a/be/src/olap/segment_reader.cpp b/be/src/olap/segment_reader.cpp index 74febd7fff..9876d48b34 100644 --- a/be/src/olap/segment_reader.cpp +++ b/be/src/olap/segment_reader.cpp @@ -846,6 +846,7 @@ OLAPStatus SegmentReader::_seek_to_block_directly( // no need to execute seek return OLAP_SUCCESS; } + SCOPED_RAW_TIMER(&_stats->block_seek_ns); for (auto cid : cids) { // If column is added through schema change, column index may not exist because of // linked schema change. So we need to ignore this column's seek