diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp index dfc9a293ee..48f0ab0e25 100644 --- a/be/src/exec/olap_scan_node.cpp +++ b/be/src/exec/olap_scan_node.cpp @@ -159,6 +159,13 @@ void OlapScanNode::_init_counter(RuntimeState* state) { // time of node to wait for batch/block queue _olap_wait_batch_queue_timer = ADD_TIMER(_runtime_profile, "BatchQueueWaitTime"); + + // for the purpose of debugging or profiling + for (int i = 0; i < sizeof(_general_debug_timer)/sizeof(*_general_debug_timer); ++i) { + char name[64]; + snprintf(name, sizeof(name), "GeneralDebugTimer%d", i); + _general_debug_timer[i] = ADD_TIMER(_segment_profile, name); + } } Status OlapScanNode::prepare(RuntimeState* state) { diff --git a/be/src/exec/olap_scan_node.h b/be/src/exec/olap_scan_node.h index f9905e256d..d57a92da57 100644 --- a/be/src/exec/olap_scan_node.h +++ b/be/src/exec/olap_scan_node.h @@ -325,6 +325,9 @@ protected: RuntimeProfile::Counter* _olap_wait_batch_queue_timer = nullptr; + // for debugging or profiling, record any info as you want + RuntimeProfile::Counter* _general_debug_timer[GENERAL_DEBUG_COUNT] = {}; + vectorized::VExpr* _dfs_peel_conjunct(vectorized::VExpr* expr, int& leaf_index); void _peel_pushed_conjuncts(); // remove pushed expr from conjunct tree }; diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index 2e05c5d12c..a1efc1d602 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -541,6 +541,11 @@ void OlapScanner::update_counter() { COUNTER_UPDATE(_parent->_index_load_timer, stats.index_load_ns); + size_t timer_count = sizeof(stats.general_debug_ns) / sizeof(*stats.general_debug_ns); + for (size_t i = 0; i < timer_count; ++i) { + COUNTER_UPDATE(_parent->_general_debug_timer[i], stats.general_debug_ns[i]); + } + COUNTER_UPDATE(_parent->_total_pages_num_counter, stats.total_pages_num); COUNTER_UPDATE(_parent->_cached_pages_num_counter, stats.cached_pages_num); diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 64802e5708..623528d343 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -238,6 +238,8 @@ class Field; class WrapperField; using KeyRange = std::pair; +static const int GENERAL_DEBUG_COUNT = 4; + // ReaderStatistics used to collect statistics when scan data from storage struct OlapReaderStatistics { int64_t io_ns = 0; @@ -285,6 +287,17 @@ struct OlapReaderStatistics { int64_t filtered_segment_number = 0; // total number of segment int64_t total_segment_number = 0; + // general_debug_ns is designed for the purpose of DEBUG, to record any infomations of debugging or profiling. + // different from specific meaningful timer such as index_load_ns, general_debug_ns can be used flexibly. + // general_debug_ns has associated with OlapScanNode's _general_debug_timer already. + // so general_debug_ns' values will update to _general_debug_timer automaticly, + // the timer result can be checked through QueryProfile web page easily. + // when search general_debug_ns, you can find that general_debug_ns has not been used, + // this is because such codes added for debug purpose should not commit, it's just for debuging. + // so, please do not delete general_debug_ns defined here + // usage example: + // SCOPED_RAW_TIMER(&_stats->general_debug_ns[1]); + int64_t general_debug_ns[GENERAL_DEBUG_COUNT] = {}; }; typedef uint32_t ColumnId; diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h b/be/src/olap/rowset/segment_v2/binary_plain_page.h index 3c55d2b0fb..3444cf1bf4 100644 --- a/be/src/olap/rowset/segment_v2/binary_plain_page.h +++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h @@ -154,8 +154,6 @@ private: class BinaryPlainPageDecoder : public PageDecoder { public: - BinaryPlainPageDecoder(Slice data) : BinaryPlainPageDecoder(data, PageDecoderOptions()) {} - BinaryPlainPageDecoder(Slice data, const PageDecoderOptions& options) : _data(data), _options(options), diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 9b14ff4201..4c651f2ca3 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -652,6 +652,7 @@ Status FileColumnIterator::_read_data_page(const OrdinalPageIndexIterator& iter) RETURN_IF_ERROR(_reader->read_page(_opts, iter.page(), &handle, &page_body, &footer)); // parse data page RETURN_IF_ERROR(ParsedPage::create(std::move(handle), page_body, footer.data_page_footer(), + _opts.stats, _reader->encoding_info(), iter.page(), iter.page_index(), &_page)); @@ -672,7 +673,7 @@ Status FileColumnIterator::_read_data_page(const OrdinalPageIndexIterator& iter) &_dict_page_handle, &dict_data, &dict_footer)); // ignore dict_footer.dict_page_footer().encoding() due to only // PLAIN_ENCODING is supported for dict page right now - _dict_decoder.reset(new BinaryPlainPageDecoder(dict_data)); + _dict_decoder.reset(new BinaryPlainPageDecoder(dict_data, _opts.stats)); RETURN_IF_ERROR(_dict_decoder->init()); auto* pd_decoder = (BinaryPlainPageDecoder*)_dict_decoder.get(); diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp index e5a3bed7dc..27c3d84ded 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp @@ -102,7 +102,7 @@ Status IndexedColumnIterator::_read_data_page(const PagePointer& pp) { // parse data page // note that page_index is not used in IndexedColumnIterator, so we pass 0 return ParsedPage::create(std::move(handle), body, footer.data_page_footer(), - _reader->encoding_info(), pp, 0, &_data_page); + nullptr, _reader->encoding_info(), pp, 0, &_data_page); } Status IndexedColumnIterator::seek_to_ordinal(ordinal_t idx) { diff --git a/be/src/olap/rowset/segment_v2/options.h b/be/src/olap/rowset/segment_v2/options.h index 0cb2dafa39..12db113008 100644 --- a/be/src/olap/rowset/segment_v2/options.h +++ b/be/src/olap/rowset/segment_v2/options.h @@ -32,7 +32,10 @@ struct PageBuilderOptions { size_t dict_page_size = DEFAULT_PAGE_SIZE; }; -struct PageDecoderOptions {}; +struct PageDecoderOptions { + struct OlapReaderStatistics* stats; + PageDecoderOptions(struct OlapReaderStatistics* stats) : stats(stats) {} +}; } // namespace segment_v2 } // namespace doris diff --git a/be/src/olap/rowset/segment_v2/parsed_page.h b/be/src/olap/rowset/segment_v2/parsed_page.h index cf49b897f3..d7ba5ea819 100644 --- a/be/src/olap/rowset/segment_v2/parsed_page.h +++ b/be/src/olap/rowset/segment_v2/parsed_page.h @@ -36,6 +36,7 @@ namespace segment_v2 { // this object struct ParsedPage { static Status create(PageHandle handle, const Slice& body, const DataPageFooterPB& footer, + struct OlapReaderStatistics* stats, const EncodingInfo* encoding, const PagePointer& page_pointer, uint32_t page_index, std::unique_ptr* result) { std::unique_ptr page(new ParsedPage); @@ -51,7 +52,7 @@ struct ParsedPage { } Slice data_slice(body.data, body.size - null_size); - PageDecoderOptions opts; + PageDecoderOptions opts(stats); RETURN_IF_ERROR(encoding->create_page_decoder(data_slice, opts, &page->data_decoder)); RETURN_IF_ERROR(page->data_decoder->init()); @@ -102,4 +103,4 @@ private: }; } // namespace segment_v2 -} // namespace doris \ No newline at end of file +} // namespace doris