database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Chore](profile) add some profile on ReaderInit #45556 (#47427)

Browse Source 
pick from #45556
This commit is contained in:
Pxl
2025-01-26 10:51:08 +08:00
committed by GitHub
parent 0e13ee3db6
commit 631d1f43cf
12 changed files with 220 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
be/src/olap/olap_common.h
View File

@ -390,6 +390,30 @@ struct OlapReaderStatistics {
int64_t collect_iterator_merge_next_timer = 0;
int64_t collect_iterator_normal_next_timer = 0;
int64_t delete_bitmap_get_agg_ns = 0;
int64_t tablet_reader_init_timer_ns = 0;
int64_t tablet_reader_capture_rs_readers_timer_ns = 0;
int64_t tablet_reader_init_return_columns_timer_ns = 0;
int64_t tablet_reader_init_keys_param_timer_ns = 0;
int64_t tablet_reader_init_orderby_keys_param_timer_ns = 0;
int64_t tablet_reader_init_conditions_param_timer_ns = 0;
int64_t tablet_reader_init_delete_condition_param_timer_ns = 0;
int64_t block_reader_vcollect_iter_init_timer_ns = 0;
int64_t block_reader_rs_readers_init_timer_ns = 0;
int64_t block_reader_build_heap_init_timer_ns = 0;
int64_t rowset_reader_get_segment_iterators_timer_ns = 0;
int64_t rowset_reader_create_iterators_timer_ns = 0;
int64_t rowset_reader_init_iterators_timer_ns = 0;
int64_t rowset_reader_load_segments_timer_ns = 0;
int64_t segment_iterator_init_timer_ns = 0;
int64_t segment_iterator_init_return_column_iterators_timer_ns = 0;
int64_t segment_iterator_init_bitmap_index_iterators_timer_ns = 0;
int64_t segment_iterator_init_inverted_index_iterators_timer_ns = 0;
int64_t segment_create_column_readers_timer_ns = 0;
int64_t segment_load_index_timer_ns = 0;
};
using ColumnId = uint32_t;

19
be/src/olap/rowset/beta_rowset_reader.cpp
View File

@ -78,7 +78,6 @@ bool BetaRowsetReader::update_profile(RuntimeProfile* profile) {
Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context,
std::vector<RowwiseIteratorUPtr>* out_iters,
bool use_cache) {
RETURN_IF_ERROR(_rowset->load());
_read_context = read_context;
// The segment iterator is created with its own statistics,
// and the member variable '_stats' is initialized by '_stats(&owned_stats)'.
@ -92,6 +91,9 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
if (_read_context->stats != nullptr) {
_stats = _read_context->stats;
}
SCOPED_RAW_TIMER(&_stats->rowset_reader_get_segment_iterators_timer_ns);
RETURN_IF_ERROR(_rowset->load());
// convert RowsetReaderContext to StorageReadOptions
_read_options.block_row_max = read_context->batch_size;
@ -207,8 +209,11 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
// load segments
bool should_use_cache = use_cache || _read_context->reader_type == ReaderType::READER_QUERY;
SegmentCacheHandle segment_cache_handle;
RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(_rowset, &segment_cache_handle,
should_use_cache));
{
SCOPED_RAW_TIMER(&_stats->rowset_reader_load_segments_timer_ns);
RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(_rowset, &segment_cache_handle,
should_use_cache));
}
// create iterator for each segment
auto& segments = segment_cache_handle.get_segments();
@ -234,6 +239,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
const bool use_lazy_init_iterators =
!is_merge_iterator && _read_context->reader_type == ReaderType::READER_QUERY;
for (int i = seg_start; i < seg_end; i++) {
SCOPED_RAW_TIMER(&_stats->rowset_reader_create_iterators_timer_ns);
auto& seg_ptr = segments[i];
std::unique_ptr<RowwiseIterator> iter;
@ -298,6 +304,8 @@ Status BetaRowsetReader::_init_iterator() {
std::vector<RowwiseIteratorUPtr> iterators;
RETURN_IF_ERROR(get_segment_iterators(_read_context, &iterators));
SCOPED_RAW_TIMER(&_stats->rowset_reader_init_iterators_timer_ns);
if (_read_context->merged_rows == nullptr) {
_read_context->merged_rows = &_merged_rows;
}
@ -333,8 +341,8 @@ Status BetaRowsetReader::_init_iterator() {
}
Status BetaRowsetReader::next_block(vectorized::Block* block) {
SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
RETURN_IF_ERROR(_init_iterator_once());
SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
if (_empty) {
return Status::Error<END_OF_FILE>("BetaRowsetReader is empty");
}
@ -362,9 +370,8 @@ Status BetaRowsetReader::next_block(vectorized::Block* block) {
}
Status BetaRowsetReader::next_block_view(vectorized::BlockView* block_view) {
SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
RETURN_IF_ERROR(_init_iterator_once());
SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
RuntimeState* runtime_state = nullptr;
if (_read_context != nullptr) {
runtime_state = _read_context->runtime_state;

24
be/src/olap/rowset/segment_v2/segment.cpp
View File

@ -147,7 +147,7 @@ Status Segment::_open_inverted_index() {
Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_options,
std::unique_ptr<RowwiseIterator>* iter) {
RETURN_IF_ERROR(_create_column_readers_once());
RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
read_options.stats->total_segment_number++;
// trying to prune the current segment by segment-level zone map
@ -208,7 +208,11 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o
}
}
RETURN_IF_ERROR(load_index());
{
SCOPED_RAW_TIMER(&read_options.stats->segment_load_index_timer_ns);
RETURN_IF_ERROR(load_index());
}
if (read_options.delete_condition_predicates->num_of_column_predicate() == 0 &&
read_options.push_down_agg_type_opt != TPushAggOp::NONE &&
read_options.push_down_agg_type_opt != TPushAggOp::COUNT_ON_INDEX) {
@ -418,7 +422,8 @@ vectorized::DataTypePtr Segment::get_data_type_of(const ColumnIdentifier& identi
return nullptr;
}
Status Segment::_create_column_readers_once() {
Status Segment::_create_column_readers_once(OlapReaderStatistics* stats) {
SCOPED_RAW_TIMER(&stats->segment_create_column_readers_timer_ns);
return _create_column_readers_once_call.call([&] {
DCHECK(_footer_pb);
Defer defer([&]() { _footer_pb.reset(); });
@ -642,7 +647,7 @@ Status Segment::new_column_iterator_with_path(const TabletColumn& tablet_column,
Status Segment::new_column_iterator(const TabletColumn& tablet_column,
std::unique_ptr<ColumnIterator>* iter,
const StorageReadOptions* opt) {
RETURN_IF_ERROR(_create_column_readers_once());
RETURN_IF_ERROR(_create_column_readers_once(opt->stats));
// init column iterator by path info
if (tablet_column.has_path_info() || tablet_column.is_variant_type()) {
@ -670,8 +675,9 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
return Status::OK();
}
Status Segment::new_column_iterator(int32_t unique_id, std::unique_ptr<ColumnIterator>* iter) {
RETURN_IF_ERROR(_create_column_readers_once());
Status Segment::new_column_iterator(int32_t unique_id, const StorageReadOptions* opt,
std::unique_ptr<ColumnIterator>* iter) {
RETURN_IF_ERROR(_create_column_readers_once(opt->stats));
ColumnIterator* it;
RETURN_IF_ERROR(_column_readers.at(unique_id)->new_iterator(&it));
iter->reset(it);
@ -699,8 +705,9 @@ ColumnReader* Segment::_get_column_reader(const TabletColumn& col) {
}
Status Segment::new_bitmap_index_iterator(const TabletColumn& tablet_column,
const StorageReadOptions& read_options,
std::unique_ptr<BitmapIndexIterator>* iter) {
RETURN_IF_ERROR(_create_column_readers_once());
RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
ColumnReader* reader = _get_column_reader(tablet_column);
if (reader != nullptr && reader->has_bitmap_index()) {
BitmapIndexIterator* it;
@ -715,7 +722,7 @@ Status Segment::new_inverted_index_iterator(const TabletColumn& tablet_column,
const TabletIndex* index_meta,
const StorageReadOptions& read_options,
std::unique_ptr<InvertedIndexIterator>* iter) {
RETURN_IF_ERROR(_create_column_readers_once());
RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
ColumnReader* reader = _get_column_reader(tablet_column);
if (reader != nullptr && index_meta) {
if (_inverted_index_file_reader == nullptr) {
@ -872,6 +879,7 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescripto
OlapReaderStatistics& stats,
std::unique_ptr<ColumnIterator>& iterator_hint) {
StorageReadOptions storage_read_opt;
storage_read_opt.stats = &stats;
storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
segment_v2::ColumnIteratorOptions opt {
.use_page_cache = !config::disable_storage_page_cache,

6
be/src/olap/rowset/segment_v2/segment.h
View File

@ -104,9 +104,11 @@ public:
std::unique_ptr<ColumnIterator>* iter,
const StorageReadOptions* opt);
Status new_column_iterator(int32_t unique_id, std::unique_ptr<ColumnIterator>* iter);
Status new_column_iterator(int32_t unique_id, const StorageReadOptions* opt,
std::unique_ptr<ColumnIterator>* iter);
Status new_bitmap_index_iterator(const TabletColumn& tablet_column,
const StorageReadOptions& read_options,
std::unique_ptr<BitmapIndexIterator>* iter);
Status new_inverted_index_iterator(const TabletColumn& tablet_column,
@ -220,7 +222,7 @@ private:
Status _open_inverted_index();
Status _create_column_readers_once();
Status _create_column_readers_once(OlapReaderStatistics* stats);
private:
friend class SegmentIterator;

10
be/src/olap/rowset/segment_v2/segment_iterator.cpp
View File

@ -280,9 +280,10 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
if (_inited) {
return Status::OK();
}
_opts = opts;
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_timer_ns);
_inited = true;
_file_reader = _segment->_file_reader;
_opts = opts;
_col_predicates.clear();
for (const auto& predicate : opts.column_predicates) {
@ -1008,6 +1009,7 @@ bool SegmentIterator::_check_all_conditions_passed_inverted_index_for_column(Col
}
Status SegmentIterator::_init_return_column_iterators() {
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_return_column_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}
@ -1050,19 +1052,21 @@ Status SegmentIterator::_init_return_column_iterators() {
}
Status SegmentIterator::_init_bitmap_index_iterators() {
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_bitmap_index_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}
for (auto cid : _schema->column_ids()) {
if (_bitmap_index_iterators[cid] == nullptr) {
RETURN_IF_ERROR(_segment->new_bitmap_index_iterator(_opts.tablet_schema->column(cid),
&_bitmap_index_iterators[cid]));
RETURN_IF_ERROR(_segment->new_bitmap_index_iterator(
_opts.tablet_schema->column(cid), _opts, &_bitmap_index_iterators[cid]));
}
}
return Status::OK();
}
Status SegmentIterator::_init_inverted_index_iterators() {
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_inverted_index_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}

4
be/src/olap/tablet.cpp
View File

@ -2725,7 +2725,9 @@ Status Tablet::_get_segment_column_iterator(
rowset->rowset_id().to_string(), segid));
}
segment_v2::SegmentSharedPtr segment = *it;
RETURN_IF_ERROR(segment->new_column_iterator(target_column, column_iterator, nullptr));
StorageReadOptions opts;
opts.stats = stats;
RETURN_IF_ERROR(segment->new_column_iterator(target_column, column_iterator, &opts));
segment_v2::ColumnIteratorOptions opt {
.use_page_cache = !config::disable_storage_page_cache,
.file_reader = segment->file_reader().get(),

7
be/src/olap/tablet_reader.cpp
View File

@ -120,6 +120,7 @@ TabletReader::~TabletReader() {
}
Status TabletReader::init(const ReaderParams& read_params) {
SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns);
_predicate_arena.reset(new vectorized::Arena());
Status res = _init_params(read_params);
@ -159,6 +160,7 @@ bool TabletReader::_optimize_for_single_rowset(
}
Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) {
SCOPED_RAW_TIMER(&_stats.tablet_reader_capture_rs_readers_timer_ns);
if (read_params.rs_splits.empty()) {
return Status::InternalError("fail to acquire data sources. tablet={}",
_tablet->tablet_id());
@ -330,6 +332,7 @@ Status TabletReader::_init_params(const ReaderParams& read_params) {
}
Status TabletReader::_init_return_columns(const ReaderParams& read_params) {
SCOPED_RAW_TIMER(&_stats.tablet_reader_init_return_columns_timer_ns);
if (read_params.reader_type == ReaderType::READER_QUERY) {
_return_columns = read_params.return_columns;
_tablet_columns_convert_to_null_set = read_params.tablet_columns_convert_to_null_set;
@ -386,6 +389,7 @@ Status TabletReader::_init_return_columns(const ReaderParams& read_params) {
}
Status TabletReader::_init_keys_param(const ReaderParams& read_params) {
SCOPED_RAW_TIMER(&_stats.tablet_reader_init_keys_param_timer_ns);
if (read_params.start_key.empty()) {
return Status::OK();
}
@ -460,6 +464,7 @@ Status TabletReader::_init_keys_param(const ReaderParams& read_params) {
}
Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) {
SCOPED_RAW_TIMER(&_stats.tablet_reader_init_orderby_keys_param_timer_ns);
// UNIQUE_KEYS will compare all keys as before
if (_tablet_schema->keys_type() == DUP_KEYS || (_tablet_schema->keys_type() == UNIQUE_KEYS &&
_tablet->enable_unique_key_merge_on_write())) {
@ -486,6 +491,7 @@ Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) {
}
Status TabletReader::_init_conditions_param(const ReaderParams& read_params) {
SCOPED_RAW_TIMER(&_stats.tablet_reader_init_conditions_param_timer_ns);
for (auto& condition : read_params.conditions) {
TCondition tmp_cond = condition;
RETURN_IF_ERROR(_tablet_schema->have_column(tmp_cond.column_name));
@ -612,6 +618,7 @@ ColumnPredicate* TabletReader::_parse_to_predicate(const FunctionFilter& functio
}
Status TabletReader::_init_delete_condition(const ReaderParams& read_params) {
SCOPED_RAW_TIMER(&_stats.tablet_reader_init_delete_condition_param_timer_ns);
// If it's cumu and not allow do delete when cumu
if (read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION ||
(read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION &&

41
be/src/pipeline/exec/olap_scan_operator.cpp
View File

@ -149,6 +149,47 @@ Status OlapScanLocalState::_init_profile() {
_tablet_counter = ADD_COUNTER(_runtime_profile, "TabletNum", TUnit::UNIT);
_key_range_counter = ADD_COUNTER(_runtime_profile, "KeyRangesNum", TUnit::UNIT);
_runtime_filter_info = ADD_LABEL_COUNTER_WITH_LEVEL(_runtime_profile, "RuntimeFilterInfo", 1);
_tablet_reader_init_timer = ADD_TIMER(_scanner_profile, "TabletReaderInitTimer");
_tablet_reader_capture_rs_readers_timer =
ADD_TIMER(_scanner_profile, "TabletReaderCaptureRsReadersTimer");
_tablet_reader_init_return_columns_timer =
ADD_TIMER(_scanner_profile, "TabletReaderInitReturnColumnsTimer");
_tablet_reader_init_keys_param_timer =
ADD_TIMER(_scanner_profile, "TabletReaderInitKeysParamTimer");
_tablet_reader_init_orderby_keys_param_timer =
ADD_TIMER(_scanner_profile, "TabletReaderInitOrderbyKeysParamTimer");
_tablet_reader_init_conditions_param_timer =
ADD_TIMER(_scanner_profile, "TabletReaderInitConditionsParamTimer");
_tablet_reader_init_delete_condition_param_timer =
ADD_TIMER(_scanner_profile, "TabletReaderInitDeleteConditionParamTimer");
_block_reader_vcollect_iter_init_timer =
ADD_TIMER(_scanner_profile, "BlockReaderVcollectIterInitTimer");
_block_reader_rs_readers_init_timer =
ADD_TIMER(_scanner_profile, "BlockReaderRsReadersInitTimer");
_block_reader_build_heap_init_timer =
ADD_TIMER(_scanner_profile, "BlockReaderBuildHeapInitTimer");
_rowset_reader_get_segment_iterators_timer =
ADD_TIMER(_scanner_profile, "RowsetReaderGetSegmentIteratorsTimer");
_rowset_reader_create_iterators_timer =
ADD_TIMER(_scanner_profile, "RowsetReaderCreateIteratorsTimer");
_rowset_reader_init_iterators_timer =
ADD_TIMER(_scanner_profile, "RowsetReaderInitIteratorsTimer");
_rowset_reader_load_segments_timer =
ADD_TIMER(_scanner_profile, "RowsetReaderLoadSegmentsTimer");
_segment_iterator_init_timer = ADD_TIMER(_scanner_profile, "SegmentIteratorInitTimer");
_segment_iterator_init_return_column_iterators_timer =
ADD_TIMER(_scanner_profile, "SegmentIteratorInitReturnColumnIteratorsTimer");
_segment_iterator_init_bitmap_index_iterators_timer =
ADD_TIMER(_scanner_profile, "SegmentIteratorInitBitmapIndexIteratorsTimer");
_segment_iterator_init_inverted_index_iterators_timer =
ADD_TIMER(_scanner_profile, "SegmentIteratorInitInvertedIndexIteratorsTimer");
_segment_create_column_readers_timer =
ADD_TIMER(_scanner_profile, "SegmentCreateColumnReadersTimer");
_segment_load_index_timer = ADD_TIMER(_scanner_profile, "SegmentLoadIndexTimer");
return Status::OK();
}

27
be/src/pipeline/exec/olap_scan_operator.h
View File

@ -191,6 +191,33 @@ private:
RuntimeProfile::Counter* _runtime_filter_info = nullptr;
// timer about tablet reader
RuntimeProfile::Counter* _tablet_reader_init_timer = nullptr;
RuntimeProfile::Counter* _tablet_reader_capture_rs_readers_timer = nullptr;
RuntimeProfile::Counter* _tablet_reader_init_return_columns_timer = nullptr;
RuntimeProfile::Counter* _tablet_reader_init_keys_param_timer = nullptr;
RuntimeProfile::Counter* _tablet_reader_init_orderby_keys_param_timer = nullptr;
RuntimeProfile::Counter* _tablet_reader_init_conditions_param_timer = nullptr;
RuntimeProfile::Counter* _tablet_reader_init_delete_condition_param_timer = nullptr;
// timer about block reader
RuntimeProfile::Counter* _block_reader_vcollect_iter_init_timer = nullptr;
RuntimeProfile::Counter* _block_reader_rs_readers_init_timer = nullptr;
RuntimeProfile::Counter* _block_reader_build_heap_init_timer = nullptr;
RuntimeProfile::Counter* _rowset_reader_get_segment_iterators_timer = nullptr;
RuntimeProfile::Counter* _rowset_reader_create_iterators_timer = nullptr;
RuntimeProfile::Counter* _rowset_reader_init_iterators_timer = nullptr;
RuntimeProfile::Counter* _rowset_reader_load_segments_timer = nullptr;
RuntimeProfile::Counter* _segment_iterator_init_timer = nullptr;
RuntimeProfile::Counter* _segment_iterator_init_return_column_iterators_timer = nullptr;
RuntimeProfile::Counter* _segment_iterator_init_bitmap_index_iterators_timer = nullptr;
RuntimeProfile::Counter* _segment_iterator_init_inverted_index_iterators_timer = nullptr;
RuntimeProfile::Counter* _segment_create_column_readers_timer = nullptr;
RuntimeProfile::Counter* _segment_load_index_timer = nullptr;
std::mutex _profile_mtx;
};

43
be/src/vec/exec/scan/new_olap_scanner.cpp
View File

@ -656,10 +656,53 @@ void NewOlapScanner::_collect_profile_before_close() {
} else {
pipeline::OlapScanLocalState* local_state = (pipeline::OlapScanLocalState*)_local_state;
INCR_COUNTER(local_state);
COUNTER_UPDATE(local_state->_tablet_reader_init_timer, stats.tablet_reader_init_timer_ns);
COUNTER_UPDATE(local_state->_tablet_reader_capture_rs_readers_timer,
stats.tablet_reader_capture_rs_readers_timer_ns);
COUNTER_UPDATE(local_state->_tablet_reader_init_return_columns_timer,
stats.tablet_reader_init_return_columns_timer_ns);
COUNTER_UPDATE(local_state->_tablet_reader_init_keys_param_timer,
stats.tablet_reader_init_keys_param_timer_ns);
COUNTER_UPDATE(local_state->_tablet_reader_init_orderby_keys_param_timer,
stats.tablet_reader_init_orderby_keys_param_timer_ns);
COUNTER_UPDATE(local_state->_tablet_reader_init_conditions_param_timer,
stats.tablet_reader_init_conditions_param_timer_ns);
COUNTER_UPDATE(local_state->_tablet_reader_init_delete_condition_param_timer,
stats.tablet_reader_init_delete_condition_param_timer_ns);
COUNTER_UPDATE(local_state->_block_reader_vcollect_iter_init_timer,
stats.block_reader_vcollect_iter_init_timer_ns);
COUNTER_UPDATE(local_state->_block_reader_rs_readers_init_timer,
stats.block_reader_rs_readers_init_timer_ns);
COUNTER_UPDATE(local_state->_block_reader_build_heap_init_timer,
stats.block_reader_build_heap_init_timer_ns);
COUNTER_UPDATE(local_state->_rowset_reader_get_segment_iterators_timer,
stats.rowset_reader_get_segment_iterators_timer_ns);
COUNTER_UPDATE(local_state->_rowset_reader_create_iterators_timer,
stats.rowset_reader_create_iterators_timer_ns);
COUNTER_UPDATE(local_state->_rowset_reader_init_iterators_timer,
stats.rowset_reader_init_iterators_timer_ns);
COUNTER_UPDATE(local_state->_rowset_reader_load_segments_timer,
stats.rowset_reader_load_segments_timer_ns);
COUNTER_UPDATE(local_state->_segment_iterator_init_timer,
stats.segment_iterator_init_timer_ns);
COUNTER_UPDATE(local_state->_segment_iterator_init_return_column_iterators_timer,
stats.segment_iterator_init_return_column_iterators_timer_ns);
COUNTER_UPDATE(local_state->_segment_iterator_init_bitmap_index_iterators_timer,
stats.segment_iterator_init_bitmap_index_iterators_timer_ns);
COUNTER_UPDATE(local_state->_segment_iterator_init_inverted_index_iterators_timer,
stats.segment_iterator_init_inverted_index_iterators_timer_ns);
COUNTER_UPDATE(local_state->_segment_create_column_readers_timer,
stats.segment_create_column_readers_timer_ns);
COUNTER_UPDATE(local_state->_segment_load_index_timer, stats.segment_load_index_timer_ns);
}
#undef INCR_COUNTER
#endif
// Update metrics
DorisMetrics::instance()->query_scan_bytes->increment(_scan_bytes);
DorisMetrics::instance()->query_scan_rows->increment(_scan_rows);

57
be/src/vec/olap/block_reader.cpp
View File

@ -107,40 +107,49 @@ Status BlockReader::_init_collect_iter(const ReaderParams& read_params) {
return res;
}
// check if rowsets are noneoverlapping
_is_rowsets_overlapping = _rowsets_mono_asc_disjoint(read_params);
_vcollect_iter.init(this, _is_rowsets_overlapping, read_params.read_orderby_key,
read_params.read_orderby_key_reverse);
{
SCOPED_RAW_TIMER(&_stats.block_reader_vcollect_iter_init_timer_ns);
_is_rowsets_overlapping = _rowsets_mono_asc_disjoint(read_params);
_vcollect_iter.init(this, _is_rowsets_overlapping, read_params.read_orderby_key,
read_params.read_orderby_key_reverse);
}
std::vector<RowsetReaderSharedPtr> valid_rs_readers;
RuntimeState* runtime_state = read_params.runtime_state;
for (int i = 0; i < read_params.rs_splits.size(); ++i) {
if (runtime_state != nullptr && runtime_state->is_cancelled()) {
return Status::Cancelled(runtime_state->cancel_reason());
}
{
SCOPED_RAW_TIMER(&_stats.block_reader_rs_readers_init_timer_ns);
for (int i = 0; i < read_params.rs_splits.size(); ++i) {
if (runtime_state != nullptr && runtime_state->is_cancelled()) {
return Status::Cancelled(runtime_state->cancel_reason());
}
auto& rs_split = read_params.rs_splits[i];
auto& rs_split = read_params.rs_splits[i];
// _vcollect_iter.topn_next() will init rs_reader by itself
if (!_vcollect_iter.use_topn_next()) {
RETURN_IF_ERROR(rs_split.rs_reader->init(&_reader_context, rs_split));
}
// _vcollect_iter.topn_next() will init rs_reader by itself
if (!_vcollect_iter.use_topn_next()) {
RETURN_IF_ERROR(rs_split.rs_reader->init(&_reader_context, rs_split));
}
Status res = _vcollect_iter.add_child(rs_split);
if (!res.ok() && !res.is<END_OF_FILE>()) {
LOG(WARNING) << "failed to add child to iterator, err=" << res;
return res;
}
if (res.ok()) {
valid_rs_readers.push_back(rs_split.rs_reader);
Status res = _vcollect_iter.add_child(rs_split);
if (!res.ok() && !res.is<END_OF_FILE>()) {
LOG(WARNING) << "failed to add child to iterator, err=" << res;
return res;
}
if (res.ok()) {
valid_rs_readers.push_back(rs_split.rs_reader);
}
}
}
RETURN_IF_ERROR(_vcollect_iter.build_heap(valid_rs_readers));
// _vcollect_iter.topn_next() can not use current_row
if (!_vcollect_iter.use_topn_next()) {
auto status = _vcollect_iter.current_row(&_next_row);
_eof = status.is<END_OF_FILE>();
{
SCOPED_RAW_TIMER(&_stats.block_reader_build_heap_init_timer_ns);
RETURN_IF_ERROR(_vcollect_iter.build_heap(valid_rs_readers));
// _vcollect_iter.topn_next() can not use current_row
if (!_vcollect_iter.use_topn_next()) {
auto status = _vcollect_iter.current_row(&_next_row);
_eof = status.is<END_OF_FILE>();
}
}
return Status::OK();

5
be/src/vec/olap/vgeneric_iterators.cpp
View File

@ -50,9 +50,8 @@ Status VStatisticsIterator::init(const StorageReadOptions& opts) {
auto cid = _schema.column_id(i);
auto unique_id = _schema.column(cid)->unique_id();
if (_column_iterators_map.count(unique_id) < 1) {
RETURN_IF_ERROR(_segment->new_column_iterator(opts.tablet_schema->column(cid),
&_column_iterators_map[unique_id],
nullptr));
RETURN_IF_ERROR(_segment->new_column_iterator(
opts.tablet_schema->column(cid), &_column_iterators_map[unique_id], &opts));
}
_column_iterators.push_back(_column_iterators_map[unique_id].get());
}