From 6dd60c6ebbf272cd24b3e0d2d63a2fc9a2df0db2 Mon Sep 17 00:00:00 2001 From: plat1ko Date: Thu, 26 Oct 2023 10:00:28 +0800 Subject: [PATCH] [Enhance](BE) Add -Wshadow-field compile option to avoid unexpected shadowing behavior (#25698) * Fix `Tablet::_meta_lock` shadows member inherited from `BaseTablet` * Add -Wshadow-field compile option to avoid unexpected shadowing behavior --- be/CMakeLists.txt | 1 + be/src/io/fs/s3_file_writer.h | 1 - .../rowset/segment_v2/inverted_index_cache.h | 7 ++++++ .../segment_v2/inverted_index_reader.cpp | 7 ++++++ .../segment_v2/inverted_index_writer.cpp | 7 ++++++ .../rowset/segment_v2/ngram_bloom_filter.h | 8 +++++++ be/src/pipeline/exec/es_scan_operator.h | 8 +++++++ .../exec/multi_cast_data_stream_source.h | 8 +++++++ be/src/pipeline/exec/olap_scan_operator.h | 3 --- .../exec/partition_sort_source_operator.h | 4 +--- .../streaming_aggregation_sink_operator.h | 4 ---- be/src/pipeline/pipeline_x/operator.h | 2 +- .../pipeline_x/pipeline_x_fragment_context.h | 7 ++++++ .../aggregate_function_collect.h | 6 ++--- .../aggregate_function_hll_union_agg.h | 5 ++-- .../aggregate_function_java_udaf.h | 8 +++---- .../aggregate_function_reader_first_last.h | 6 ++--- .../aggregate_function_rpc.h | 8 +++---- .../aggregate_function_state_merge.h | 10 ++++---- .../aggregate_function_state_union.h | 10 ++++---- .../aggregate_function_window.h | 6 ++--- be/src/vec/aggregate_functions/helpers.h | 14 +++++------ .../vec/common/hash_table/hash_map_context.h | 9 ++++++- be/src/vec/exec/format/csv/csv_reader.cpp | 24 +++++++++---------- be/src/vec/exec/format/csv/csv_reader.h | 6 ++--- be/src/vec/exec/scan/new_es_scan_node.h | 8 +++++++ be/src/vec/exec/scan/new_es_scanner.cpp | 10 ++++---- be/src/vec/exec/scan/new_es_scanner.h | 1 - be/src/vec/exec/scan/new_jdbc_scanner.cpp | 4 ++-- be/src/vec/exec/scan/new_jdbc_scanner.h | 2 -- be/src/vec/exec/scan/new_odbc_scanner.cpp | 5 ++-- be/src/vec/exec/scan/new_odbc_scanner.h | 2 -- be/src/vec/exec/scan/new_olap_scan_node.h | 3 --- be/src/vec/exec/scan/pip_scanner_context.h | 8 +++---- be/src/vec/exec/scan/vscan_node.h | 2 -- be/src/vec/sink/vmemory_scratch_sink.h | 2 -- 36 files changed, 138 insertions(+), 88 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index cf3c0be1dc..941a5bb972 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -271,6 +271,7 @@ if (COMPILER_CLANG) add_compile_options(-fcolor-diagnostics -Wpedantic + -Wshadow-field -Wunused -Wunused-command-line-argument -Wunused-exception-parameter diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index b8d53cf648..119f07199b 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -64,7 +64,6 @@ private: std::string _bucket; std::string _key; - bool _closed = false; bool _aborted = false; std::shared_ptr _client; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.h b/be/src/olap/rowset/segment_v2/inverted_index_cache.h index 19a9eb7734..79439ac462 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_cache.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.h @@ -17,7 +17,14 @@ #pragma once +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif #include // IWYU pragma: keep +#ifdef __clang__ +#pragma clang diagnostic pop +#endif #include #include #include diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 244e00d71d..2776fa4a8d 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -47,7 +47,14 @@ #include #include +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif #include "CLucene/analysis/standard95/StandardAnalyzer.h" +#ifdef __clang__ +#pragma clang diagnostic pop +#endif #include "common/config.h" #include "common/logging.h" #include "common/status.h" diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index b6682e3ae2..4be1c72b53 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -30,7 +30,14 @@ #include #include +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif #include "CLucene/analysis/standard95/StandardAnalyzer.h" +#ifdef __clang__ +#pragma clang diagnostic pop +#endif #include "common/config.h" #include "olap/field.h" #include "olap/inverted_index_parser.h" diff --git a/be/src/olap/rowset/segment_v2/ngram_bloom_filter.h b/be/src/olap/rowset/segment_v2/ngram_bloom_filter.h index b69e467831..527f8a6e99 100644 --- a/be/src/olap/rowset/segment_v2/ngram_bloom_filter.h +++ b/be/src/olap/rowset/segment_v2/ngram_bloom_filter.h @@ -48,7 +48,15 @@ public: bool is_ngram_bf() const override { return true; } private: +// FIXME: non-static data member '_size' of 'NGramBloomFilter' shadows member inherited from type 'BloomFilter' +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif size_t _size; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif size_t words; std::vector filter; }; diff --git a/be/src/pipeline/exec/es_scan_operator.h b/be/src/pipeline/exec/es_scan_operator.h index 60a5f04510..7c8a5ceae5 100644 --- a/be/src/pipeline/exec/es_scan_operator.h +++ b/be/src/pipeline/exec/es_scan_operator.h @@ -55,7 +55,15 @@ private: std::vector> _scan_ranges; std::unique_ptr _es_profile; + // FIXME: non-static data member '_rows_read_counter' of 'EsScanLocalState' shadows member inherited from type 'ScanLocalStateBase' +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif RuntimeProfile::Counter* _rows_read_counter; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif RuntimeProfile::Counter* _read_timer; RuntimeProfile::Counter* _materialize_timer; }; diff --git a/be/src/pipeline/exec/multi_cast_data_stream_source.h b/be/src/pipeline/exec/multi_cast_data_stream_source.h index 22a50fa1fc..ee724b1ff1 100644 --- a/be/src/pipeline/exec/multi_cast_data_stream_source.h +++ b/be/src/pipeline/exec/multi_cast_data_stream_source.h @@ -182,7 +182,15 @@ private: const int _consumer_id; const TDataStreamSink _t_data_stream_sink; vectorized::VExprContextSPtrs _output_expr_contexts; + // FIXME: non-static data member '_row_descriptor' of 'MultiCastDataStreamerSourceOperatorX' shadows member inherited from type 'OperatorXBase' +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif const RowDescriptor& _row_descriptor; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif const RowDescriptor& _row_desc() { return _row_descriptor; } }; diff --git a/be/src/pipeline/exec/olap_scan_operator.h b/be/src/pipeline/exec/olap_scan_operator.h index f15d085b51..5f7c86c9c3 100644 --- a/be/src/pipeline/exec/olap_scan_operator.h +++ b/be/src/pipeline/exec/olap_scan_operator.h @@ -169,9 +169,6 @@ private: RuntimeProfile::Counter* _output_index_result_column_timer = nullptr; - // number of created olap scanners - RuntimeProfile::Counter* _num_scanners = nullptr; - // number of segment filtered by column stat when creating seg iterator RuntimeProfile::Counter* _filtered_segment_counter = nullptr; // total number of segment related to this scan node diff --git a/be/src/pipeline/exec/partition_sort_source_operator.h b/be/src/pipeline/exec/partition_sort_source_operator.h index 0bb1149603..40b802c35e 100644 --- a/be/src/pipeline/exec/partition_sort_source_operator.h +++ b/be/src/pipeline/exec/partition_sort_source_operator.h @@ -58,8 +58,7 @@ public: PartitionSortSourceLocalState(RuntimeState* state, OperatorXBase* parent) : PipelineXLocalState(state, parent), _get_sorted_timer(nullptr), - _get_next_timer(nullptr), - _num_rows_returned(0) {} + _get_next_timer(nullptr) {} Status init(RuntimeState* state, LocalStateInfo& info) override; @@ -67,7 +66,6 @@ private: friend class PartitionSortSourceOperatorX; RuntimeProfile::Counter* _get_sorted_timer; RuntimeProfile::Counter* _get_next_timer; - int64_t _num_rows_returned; int _sort_idx = 0; }; diff --git a/be/src/pipeline/exec/streaming_aggregation_sink_operator.h b/be/src/pipeline/exec/streaming_aggregation_sink_operator.h index 6f49ac86e2..031515ba94 100644 --- a/be/src/pipeline/exec/streaming_aggregation_sink_operator.h +++ b/be/src/pipeline/exec/streaming_aggregation_sink_operator.h @@ -94,10 +94,6 @@ private: doris::vectorized::Block* out_block); bool _should_expand_preagg_hash_tables(); - vectorized::Block _preagg_block = vectorized::Block(); - - vectorized::PODArray _places; - RuntimeProfile::Counter* _queue_byte_size_counter; RuntimeProfile::Counter* _queue_size_counter; RuntimeProfile::Counter* _streaming_agg_timer; diff --git a/be/src/pipeline/pipeline_x/operator.h b/be/src/pipeline/pipeline_x/operator.h index 2e51600d6f..a6ef8a81ac 100644 --- a/be/src/pipeline/pipeline_x/operator.h +++ b/be/src/pipeline/pipeline_x/operator.h @@ -103,7 +103,7 @@ protected: friend class OperatorXBase; ObjectPool* _pool; - int64_t _num_rows_returned; + int64_t _num_rows_returned {0}; std::unique_ptr _runtime_profile; diff --git a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h index 8c7777f95b..45eb7f48cc 100644 --- a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h +++ b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h @@ -157,7 +157,14 @@ private: // TODO: remove the _sink and _multi_cast_stream_sink_senders to set both // of it in pipeline task not the fragment_context +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif DataSinkOperatorXPtr _sink; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif std::atomic_bool _canceled = false; diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.h b/be/src/vec/aggregate_functions/aggregate_function_collect.h index 63ff0a680b..3a4b0bad20 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_collect.h +++ b/be/src/vec/aggregate_functions/aggregate_function_collect.h @@ -432,12 +432,12 @@ class AggregateFunctionCollect public: using BaseHelper = IAggregateFunctionHelper>; - AggregateFunctionCollect(const DataTypes& argument_types, + AggregateFunctionCollect(const DataTypes& argument_types_, UInt64 max_size_ = std::numeric_limits::max()) : IAggregateFunctionDataHelper>( - {argument_types}), - return_type(argument_types[0]) {} + {argument_types_}), + return_type(argument_types_[0]) {} std::string get_name() const override { if constexpr (ShowNull::value) { diff --git a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h index d4a94a190a..bb2ab75d6c 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h +++ b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h @@ -109,8 +109,9 @@ template class AggregateFunctionHLLUnion : public IAggregateFunctionDataHelper> { public: - AggregateFunctionHLLUnion(const DataTypes& argument_types) - : IAggregateFunctionDataHelper>(argument_types) {} + AggregateFunctionHLLUnion(const DataTypes& argument_types_) + : IAggregateFunctionDataHelper>(argument_types_) { + } String get_name() const override { return Data::name(); } diff --git a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h index 37888b35ac..d79154a004 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h +++ b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h @@ -247,18 +247,18 @@ class AggregateJavaUdaf final : public IAggregateFunctionDataHelper { public: ENABLE_FACTORY_CREATOR(AggregateJavaUdaf); - AggregateJavaUdaf(const TFunction& fn, const DataTypes& argument_types, + AggregateJavaUdaf(const TFunction& fn, const DataTypes& argument_types_, const DataTypePtr& return_type) - : IAggregateFunctionDataHelper(argument_types), + : IAggregateFunctionDataHelper(argument_types_), _fn(fn), _return_type(return_type), _first_created(true), _exec_place(nullptr) {} ~AggregateJavaUdaf() override = default; - static AggregateFunctionPtr create(const TFunction& fn, const DataTypes& argument_types, + static AggregateFunctionPtr create(const TFunction& fn, const DataTypes& argument_types_, const DataTypePtr& return_type) { - return std::make_shared(fn, argument_types, return_type); + return std::make_shared(fn, argument_types_, return_type); } //Note: The condition is added because maybe the BE can't find java-udaf impl jar //So need to check as soon as possible, before call Data function diff --git a/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h b/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h index e3c8105684..b10d69b2d4 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h +++ b/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h @@ -195,9 +195,9 @@ template class ReaderFunctionData final : public IAggregateFunctionDataHelper> { public: - ReaderFunctionData(const DataTypes& argument_types) - : IAggregateFunctionDataHelper>(argument_types), - _argument_type(argument_types[0]) {} + ReaderFunctionData(const DataTypes& argument_types_) + : IAggregateFunctionDataHelper>(argument_types_), + _argument_type(argument_types_[0]) {} String get_name() const override { return Data::name(); } diff --git a/be/src/vec/aggregate_functions/aggregate_function_rpc.h b/be/src/vec/aggregate_functions/aggregate_function_rpc.h index 4c0fe441f7..21e5aa290d 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_rpc.h +++ b/be/src/vec/aggregate_functions/aggregate_function_rpc.h @@ -336,14 +336,14 @@ public: class AggregateRpcUdaf final : public IAggregateFunctionDataHelper { public: - AggregateRpcUdaf(const TFunction& fn, const DataTypes& argument_types, + AggregateRpcUdaf(const TFunction& fn, const DataTypes& argument_types_, const DataTypePtr& return_type) - : IAggregateFunctionDataHelper(argument_types), _fn(fn), _return_type(return_type) {} + : IAggregateFunctionDataHelper(argument_types_), _fn(fn), _return_type(return_type) {} ~AggregateRpcUdaf() = default; - static AggregateFunctionPtr create(const TFunction& fn, const DataTypes& argument_types, + static AggregateFunctionPtr create(const TFunction& fn, const DataTypes& argument_types_, const DataTypePtr& return_type) { - return std::make_shared(fn, argument_types, return_type); + return std::make_shared(fn, argument_types_, return_type); } void create(AggregateDataPtr __restrict place) const override { diff --git a/be/src/vec/aggregate_functions/aggregate_function_state_merge.h b/be/src/vec/aggregate_functions/aggregate_function_state_merge.h index 136eca0837..d16c823b02 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_state_merge.h +++ b/be/src/vec/aggregate_functions/aggregate_function_state_merge.h @@ -26,18 +26,18 @@ class AggregateStateMerge : public AggregateStateUnion { public: using AggregateStateUnion::create; - AggregateStateMerge(AggregateFunctionPtr function, const DataTypes& argument_types, + AggregateStateMerge(AggregateFunctionPtr function, const DataTypes& argument_types_, const DataTypePtr& return_type) - : AggregateStateUnion(function, argument_types, return_type) {} + : AggregateStateUnion(function, argument_types_, return_type) {} static AggregateFunctionPtr create(AggregateFunctionPtr function, - const DataTypes& argument_types, + const DataTypes& argument_types_, const DataTypePtr& return_type) { - CHECK(argument_types.size() == 1); + CHECK(argument_types_.size() == 1); if (function == nullptr) { return nullptr; } - return std::make_shared(function, argument_types, return_type); + return std::make_shared(function, argument_types_, return_type); } void set_version(const int version_) override { diff --git a/be/src/vec/aggregate_functions/aggregate_function_state_union.h b/be/src/vec/aggregate_functions/aggregate_function_state_union.h index b2ff81b168..4134b7f79d 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_state_union.h +++ b/be/src/vec/aggregate_functions/aggregate_function_state_union.h @@ -25,21 +25,21 @@ const static std::string AGG_UNION_SUFFIX = "_union"; class AggregateStateUnion : public IAggregateFunctionHelper { public: - AggregateStateUnion(AggregateFunctionPtr function, const DataTypes& argument_types, + AggregateStateUnion(AggregateFunctionPtr function, const DataTypes& argument_types_, const DataTypePtr& return_type) - : IAggregateFunctionHelper(argument_types), + : IAggregateFunctionHelper(argument_types_), _function(function), _return_type(return_type) {} ~AggregateStateUnion() override = default; static AggregateFunctionPtr create(AggregateFunctionPtr function, - const DataTypes& argument_types, + const DataTypes& argument_types_, const DataTypePtr& return_type) { - CHECK(argument_types.size() == 1); + CHECK(argument_types_.size() == 1); if (function == nullptr) { return nullptr; } - return std::make_shared(function, argument_types, return_type); + return std::make_shared(function, argument_types_, return_type); } void set_version(const int version_) override { diff --git a/be/src/vec/aggregate_functions/aggregate_function_window.h b/be/src/vec/aggregate_functions/aggregate_function_window.h index ad8eeece8d..6fadced2d3 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_window.h +++ b/be/src/vec/aggregate_functions/aggregate_function_window.h @@ -389,9 +389,9 @@ template class WindowFunctionData final : public IAggregateFunctionDataHelper> { public: - WindowFunctionData(const DataTypes& argument_types) - : IAggregateFunctionDataHelper>(argument_types), - _argument_type(argument_types[0]) {} + WindowFunctionData(const DataTypes& argument_types_) + : IAggregateFunctionDataHelper>(argument_types_), + _argument_type(argument_types_[0]) {} String get_name() const override { return Data::name(); } diff --git a/be/src/vec/aggregate_functions/helpers.h b/be/src/vec/aggregate_functions/helpers.h index 58ddd455bc..63b814e3e6 100644 --- a/be/src/vec/aggregate_functions/helpers.h +++ b/be/src/vec/aggregate_functions/helpers.h @@ -113,17 +113,17 @@ struct creator_without_type { } template - static AggregateFunctionPtr create(const DataTypes& argument_types, + static AggregateFunctionPtr create(const DataTypes& argument_types_, const bool result_is_nullable, TArgs&&... args) { IAggregateFunction* result(new AggregateFunctionTemplate(std::forward(args)..., - remove_nullable(argument_types))); - if (have_nullable(argument_types)) { + remove_nullable(argument_types_))); + if (have_nullable(argument_types_)) { std::visit( [&](auto multi_arguments, auto result_is_nullable) { result = new NullableT(result, argument_types); + AggregateFunctionTemplate>(result, argument_types_); }, - make_bool_variant(argument_types.size() > 1), + make_bool_variant(argument_types_.size() > 1), make_bool_variant(result_is_nullable)); } @@ -134,11 +134,11 @@ struct creator_without_type { /// AggregateFunctionTemplate will handle the nullable arguments, no need to use /// AggregateFunctionNullVariadicInline/AggregateFunctionNullUnaryInline template - static AggregateFunctionPtr create_ignore_nullable(const DataTypes& argument_types, + static AggregateFunctionPtr create_ignore_nullable(const DataTypes& argument_types_, const bool /*result_is_nullable*/, TArgs&&... args) { IAggregateFunction* result( - new AggregateFunctionTemplate(std::forward(args)..., argument_types)); + new AggregateFunctionTemplate(std::forward(args)..., argument_types_)); CHECK_AGG_FUNCTION_SERIALIZED_TYPE(AggregateFunctionTemplate); return AggregateFunctionPtr(result); } diff --git a/be/src/vec/common/hash_table/hash_map_context.h b/be/src/vec/common/hash_table/hash_map_context.h index f40a351f9d..0464380b18 100644 --- a/be/src/vec/common/hash_table/hash_map_context.h +++ b/be/src/vec/common/hash_table/hash_map_context.h @@ -134,6 +134,11 @@ struct MethodBase { size_t num_rows) = 0; }; +// FIXME: parameter 'keys' shadows member inherited from type `MethodBase` +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif template struct MethodSerialized : public MethodBase { using Base = MethodBase; @@ -464,7 +469,6 @@ struct MethodSingleNullableColumn : public SingleColumnMethod { using Base = SingleColumnMethod; using State = ColumnsHashing::HashMethodSingleLowNullableColumn; - void insert_keys_into_columns(std::vector& keys, MutableColumns& key_columns, const size_t num_rows) override { auto* col = key_columns[0].get(); @@ -476,6 +480,9 @@ struct MethodSingleNullableColumn : public SingleColumnMethod { } } }; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif template using SerializedHashTableContext = MethodSerialized>; diff --git a/be/src/vec/exec/format/csv/csv_reader.cpp b/be/src/vec/exec/format/csv/csv_reader.cpp index 2d7c116a34..f5cb42082d 100644 --- a/be/src/vec/exec/format/csv/csv_reader.cpp +++ b/be/src/vec/exec/format/csv/csv_reader.cpp @@ -75,12 +75,12 @@ void EncloseCsvTextFieldSplitter::do_split(const Slice& line, std::vector const auto& column_sep_positions = _text_line_reader_ctx->column_sep_positions(); size_t value_start_offset = 0; for (auto idx : column_sep_positions) { - process_value_func(data, value_start_offset, idx - value_start_offset, trimming_char, + process_value_func(data, value_start_offset, idx - value_start_offset, _trimming_char, splitted_values); - value_start_offset = idx + value_sep_len; + value_start_offset = idx + _value_sep_len; } // process the last column - process_value_func(data, value_start_offset, line.size - value_start_offset, trimming_char, + process_value_func(data, value_start_offset, line.size - value_start_offset, _trimming_char, splitted_values); } @@ -91,11 +91,11 @@ void PlainCsvTextFieldSplitter::_split_field_single_char(const Slice& line, size_t value_start = 0; for (size_t i = 0; i < size; ++i) { if (data[i] == _value_sep[0]) { - process_value_func(data, value_start, i - value_start, trimming_char, splitted_values); - value_start = i + value_sep_len; + process_value_func(data, value_start, i - value_start, _trimming_char, splitted_values); + value_start = i + _value_sep_len; } } - process_value_func(data, value_start, size - value_start, trimming_char, splitted_values); + process_value_func(data, value_start, size - value_start, _trimming_char, splitted_values); } void PlainCsvTextFieldSplitter::_split_field_multi_char(const Slice& line, @@ -114,9 +114,9 @@ void PlainCsvTextFieldSplitter::_split_field_multi_char(const Slice& line, // curpos curpos //kmp - vector next(value_sep_len); + vector next(_value_sep_len); next[0] = -1; - for (int i = 1, j = -1; i < value_sep_len; i++) { + for (int i = 1, j = -1; i < _value_sep_len; i++) { while (j > -1 && _value_sep[i] != _value_sep[j + 1]) { j = next[j]; } @@ -135,8 +135,8 @@ void PlainCsvTextFieldSplitter::_split_field_multi_char(const Slice& line, if (line[i] == _value_sep[j + 1]) { j++; } - if (j == value_sep_len - 1) { - curpos = i - value_sep_len + 1; + if (j == _value_sep_len - 1) { + curpos = i - _value_sep_len + 1; /* * column_separator : "xx" @@ -154,7 +154,7 @@ void PlainCsvTextFieldSplitter::_split_field_multi_char(const Slice& line, */ if (curpos >= start) { - process_value_func(line.data, start, curpos - start, trimming_char, + process_value_func(line.data, start, curpos - start, _trimming_char, splitted_values); start = i + 1; } @@ -162,7 +162,7 @@ void PlainCsvTextFieldSplitter::_split_field_multi_char(const Slice& line, j = next[j]; } } - process_value_func(line.data, start, line.size - start, trimming_char, splitted_values); + process_value_func(line.data, start, line.size - start, _trimming_char, splitted_values); } void PlainCsvTextFieldSplitter::do_split(const Slice& line, std::vector* splitted_values) { diff --git a/be/src/vec/exec/format/csv/csv_reader.h b/be/src/vec/exec/format/csv/csv_reader.h index f22922c48c..5eb3572e72 100644 --- a/be/src/vec/exec/format/csv/csv_reader.h +++ b/be/src/vec/exec/format/csv/csv_reader.h @@ -90,7 +90,7 @@ class BaseCsvTextFieldSplitter : public BaseLineFieldSplitter; @@ -111,8 +111,8 @@ public: } protected: - const char trimming_char; - const size_t value_sep_len; + const char _trimming_char; + const size_t _value_sep_len; ProcessValueFunc process_value_func; private: diff --git a/be/src/vec/exec/scan/new_es_scan_node.h b/be/src/vec/exec/scan/new_es_scan_node.h index 13b337290f..2cb6193d64 100644 --- a/be/src/vec/exec/scan/new_es_scan_node.h +++ b/be/src/vec/exec/scan/new_es_scan_node.h @@ -75,7 +75,15 @@ private: // Profile std::unique_ptr _es_profile; + // FIXME: non-static data member '_rows_read_counter' of 'NewEsScanNode' shadows member inherited from type 'VScanNode' +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif RuntimeProfile::Counter* _rows_read_counter; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif RuntimeProfile::Counter* _read_timer; RuntimeProfile::Counter* _materialize_timer; }; diff --git a/be/src/vec/exec/scan/new_es_scanner.cpp b/be/src/vec/exec/scan/new_es_scanner.cpp index e66a1c01a8..afc2412f2b 100644 --- a/be/src/vec/exec/scan/new_es_scanner.cpp +++ b/be/src/vec/exec/scan/new_es_scanner.cpp @@ -47,7 +47,6 @@ NewEsScanner::NewEsScanner(RuntimeState* state, NewEsScanNode* parent, int64_t l const std::map& docvalue_context, bool doc_value_mode, RuntimeProfile* profile) : VScanner(state, static_cast(parent), limit, profile), - _is_init(false), _es_eof(false), _properties(properties), _line_eof(false), @@ -57,7 +56,9 @@ NewEsScanner::NewEsScanner(RuntimeState* state, NewEsScanNode* parent, int64_t l _es_reader(nullptr), _es_scroll_parser(nullptr), _docvalue_context(docvalue_context), - _doc_value_mode(doc_value_mode) {} + _doc_value_mode(doc_value_mode) { + _is_init = false; +} NewEsScanner::NewEsScanner(RuntimeState* state, pipeline::ScanLocalStateBase* local_state, int64_t limit, TupleId tuple_id, @@ -65,7 +66,6 @@ NewEsScanner::NewEsScanner(RuntimeState* state, pipeline::ScanLocalStateBase* lo const std::map& docvalue_context, bool doc_value_mode, RuntimeProfile* profile) : VScanner(state, local_state, limit, profile), - _is_init(false), _es_eof(false), _properties(properties), _line_eof(false), @@ -75,7 +75,9 @@ NewEsScanner::NewEsScanner(RuntimeState* state, pipeline::ScanLocalStateBase* lo _es_reader(nullptr), _es_scroll_parser(nullptr), _docvalue_context(docvalue_context), - _doc_value_mode(doc_value_mode) {} + _doc_value_mode(doc_value_mode) { + _is_init = false; +} Status NewEsScanner::prepare(RuntimeState* state, const VExprContextSPtrs& conjuncts) { VLOG_CRITICAL << NEW_SCANNER_TYPE << "::prepare"; diff --git a/be/src/vec/exec/scan/new_es_scanner.h b/be/src/vec/exec/scan/new_es_scanner.h index 10ee1c438d..247fb3ff33 100644 --- a/be/src/vec/exec/scan/new_es_scanner.h +++ b/be/src/vec/exec/scan/new_es_scanner.h @@ -74,7 +74,6 @@ private: Status _get_next(std::vector& columns); private: - bool _is_init; bool _es_eof; const std::map& _properties; diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.cpp b/be/src/vec/exec/scan/new_jdbc_scanner.cpp index fb6c356074..25b6301c13 100644 --- a/be/src/vec/exec/scan/new_jdbc_scanner.cpp +++ b/be/src/vec/exec/scan/new_jdbc_scanner.cpp @@ -40,12 +40,12 @@ NewJdbcScanner::NewJdbcScanner(RuntimeState* state, NewJdbcScanNode* parent, int const TupleId& tuple_id, const std::string& query_string, TOdbcTableType::type table_type, RuntimeProfile* profile) : VScanner(state, static_cast(parent), limit, profile), - _is_init(false), _jdbc_eos(false), _tuple_id(tuple_id), _query_string(query_string), _tuple_desc(nullptr), _table_type(table_type) { + _is_init = false; _load_jar_timer = ADD_TIMER(get_parent()->_scanner_profile, "LoadJarTime"); _init_connector_timer = ADD_TIMER(get_parent()->_scanner_profile, "InitConnectorTime"); _check_type_timer = ADD_TIMER(get_parent()->_scanner_profile, "CheckTypeTime"); @@ -63,12 +63,12 @@ NewJdbcScanner::NewJdbcScanner(RuntimeState* state, const TupleId& tuple_id, const std::string& query_string, TOdbcTableType::type table_type, RuntimeProfile* profile) : VScanner(state, local_state, limit, profile), - _is_init(false), _jdbc_eos(false), _tuple_id(tuple_id), _query_string(query_string), _tuple_desc(nullptr), _table_type(table_type) { + _is_init = false; _load_jar_timer = ADD_TIMER(local_state->_scanner_profile, "LoadJarTime"); _init_connector_timer = ADD_TIMER(local_state->_scanner_profile, "InitConnectorTime"); _check_type_timer = ADD_TIMER(local_state->_scanner_profile, "CheckTypeTime"); diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.h b/be/src/vec/exec/scan/new_jdbc_scanner.h index 4d00fa21c6..cdb9525eb6 100644 --- a/be/src/vec/exec/scan/new_jdbc_scanner.h +++ b/be/src/vec/exec/scan/new_jdbc_scanner.h @@ -72,8 +72,6 @@ protected: private: void _update_profile(); - bool _is_init; - bool _jdbc_eos; // Tuple id resolved in prepare() to set _tuple_desc; diff --git a/be/src/vec/exec/scan/new_odbc_scanner.cpp b/be/src/vec/exec/scan/new_odbc_scanner.cpp index 55c52edfcb..c3460e95c7 100644 --- a/be/src/vec/exec/scan/new_odbc_scanner.cpp +++ b/be/src/vec/exec/scan/new_odbc_scanner.cpp @@ -52,13 +52,14 @@ namespace doris::vectorized { NewOdbcScanner::NewOdbcScanner(RuntimeState* state, NewOdbcScanNode* parent, int64_t limit, const TOdbcScanNode& odbc_scan_node, RuntimeProfile* profile) : VScanner(state, static_cast(parent), limit, profile), - _is_init(false), _odbc_eof(false), _table_name(odbc_scan_node.table_name), _connect_string(odbc_scan_node.connect_string), _query_string(odbc_scan_node.query_string), _tuple_id(odbc_scan_node.tuple_id), - _tuple_desc(nullptr) {} + _tuple_desc(nullptr) { + _is_init = false; +} Status NewOdbcScanner::prepare(RuntimeState* state, const VExprContextSPtrs& conjuncts) { VLOG_CRITICAL << NEW_SCANNER_TYPE << "::prepare"; diff --git a/be/src/vec/exec/scan/new_odbc_scanner.h b/be/src/vec/exec/scan/new_odbc_scanner.h index 9305f8361d..e2388c86a0 100644 --- a/be/src/vec/exec/scan/new_odbc_scanner.h +++ b/be/src/vec/exec/scan/new_odbc_scanner.h @@ -61,8 +61,6 @@ protected: Status _get_block_impl(RuntimeState* state, Block* block, bool* eos) override; private: - bool _is_init; - // Indicates whether there are more rows to process. Set in _odbc_connector.next(). bool _odbc_eof; diff --git a/be/src/vec/exec/scan/new_olap_scan_node.h b/be/src/vec/exec/scan/new_olap_scan_node.h index cbbf307287..a26c300ffc 100644 --- a/be/src/vec/exec/scan/new_olap_scan_node.h +++ b/be/src/vec/exec/scan/new_olap_scan_node.h @@ -193,9 +193,6 @@ private: RuntimeProfile::Counter* _output_index_result_column_timer = nullptr; - // number of created olap scanners - RuntimeProfile::Counter* _num_scanners = nullptr; - // number of segment filtered by column stat when creating seg iterator RuntimeProfile::Counter* _filtered_segment_counter = nullptr; // total number of segment related to this scan node diff --git a/be/src/vec/exec/scan/pip_scanner_context.h b/be/src/vec/exec/scan/pip_scanner_context.h index 66eaed7f28..8e3e00f5f4 100644 --- a/be/src/vec/exec/scan/pip_scanner_context.h +++ b/be/src/vec/exec/scan/pip_scanner_context.h @@ -31,20 +31,20 @@ class PipScannerContext : public vectorized::ScannerContext { public: PipScannerContext(RuntimeState* state, vectorized::VScanNode* parent, const TupleDescriptor* output_tuple_desc, - const std::list& scanners, int64_t limit, + const std::list& scanners, int64_t limit_, int64_t max_bytes_in_blocks_queue, const std::vector& col_distribute_ids, const int num_parallel_instances) - : vectorized::ScannerContext(state, parent, output_tuple_desc, scanners, limit, + : vectorized::ScannerContext(state, parent, output_tuple_desc, scanners, limit_, max_bytes_in_blocks_queue, num_parallel_instances), _col_distribute_ids(col_distribute_ids), _need_colocate_distribute(!_col_distribute_ids.empty()) {} PipScannerContext(RuntimeState* state, ScanLocalStateBase* local_state, const TupleDescriptor* output_tuple_desc, - const std::list& scanners, int64_t limit, + const std::list& scanners, int64_t limit_, int64_t max_bytes_in_blocks_queue, const std::vector& col_distribute_ids, const int num_parallel_instances) - : vectorized::ScannerContext(state, nullptr, output_tuple_desc, scanners, limit, + : vectorized::ScannerContext(state, nullptr, output_tuple_desc, scanners, limit_, max_bytes_in_blocks_queue, num_parallel_instances, local_state), _col_distribute_ids(col_distribute_ids), diff --git a/be/src/vec/exec/scan/vscan_node.h b/be/src/vec/exec/scan/vscan_node.h index 458cb318b7..be72c7ca1f 100644 --- a/be/src/vec/exec/scan/vscan_node.h +++ b/be/src/vec/exec/scan/vscan_node.h @@ -306,8 +306,6 @@ protected: VExprContextSPtrs _stale_expr_ctxs; VExprContextSPtrs _common_expr_ctxs_push_down; - RuntimeState* _state; - // If sort info is set, push limit to each scanner; int64_t _limit_per_scanner = -1; diff --git a/be/src/vec/sink/vmemory_scratch_sink.h b/be/src/vec/sink/vmemory_scratch_sink.h index 848952b1d7..e91d130547 100644 --- a/be/src/vec/sink/vmemory_scratch_sink.h +++ b/be/src/vec/sink/vmemory_scratch_sink.h @@ -69,8 +69,6 @@ private: BlockQueueSharedPtr _queue; - RuntimeProfile* _profile; // Allocated from _pool - // Owned by the RuntimeState. const std::vector& _t_output_expr; VExprContextSPtrs _output_vexpr_ctxs;