From 50864aca7d44ae006c4be005fe2b940c8b32f2ef Mon Sep 17 00:00:00 2001 From: Zhengguo Yang Date: Sat, 19 Feb 2022 11:29:02 +0800 Subject: [PATCH] [refactor] fix warings when compile with clang (#8069) --- be/CMakeLists.txt | 96 +--- be/src/agent/cgroups_mgr.cpp | 5 +- be/src/agent/cgroups_mgr.h | 1 - be/src/agent/task_worker_pool.cpp | 21 +- be/src/common/config.h | 5 +- be/src/common/daemon.h | 2 +- be/src/common/status.h | 3 + be/src/env/env.h | 12 +- be/src/env/env_posix.cpp | 36 +- be/src/env/env_posix.h | 15 +- be/src/env/env_remote.cpp | 105 ++-- be/src/env/env_remote.h | 15 +- be/src/exec/CMakeLists.txt | 1 - be/src/exec/aggregation_node.cpp | 464 ------------------ be/src/exec/aggregation_node.h | 132 ----- be/src/exec/base_scanner.cpp | 74 +-- be/src/exec/broker_scan_node.h | 2 - be/src/exec/broker_scanner.h | 4 +- be/src/exec/csv_scan_node.cpp | 7 +- be/src/exec/csv_scan_node.h | 2 +- be/src/exec/es/es_predicate.cpp | 6 +- be/src/exec/es/es_predicate.h | 3 +- be/src/exec/es/es_scroll_parser.cpp | 286 ++++++----- be/src/exec/es/es_scroll_parser.h | 4 +- be/src/exec/es_http_scan_node.h | 6 +- be/src/exec/es_scan_node.h | 2 +- be/src/exec/exchange_node.h | 12 +- be/src/exec/exec_node.cpp | 26 +- be/src/exec/odbc_connector.cpp | 1 - be/src/exec/odbc_connector.h | 1 - be/src/exec/parquet_reader.cpp | 10 +- be/src/exec/parquet_scanner.h | 7 +- be/src/exec/parquet_writer.cpp | 10 +- be/src/exec/partitioned_aggregation_node.cc | 2 - be/src/exec/partitioned_aggregation_node.h | 3 - .../schema_scanner/schema_dummy_scanner.cpp | 2 +- .../schema_scanner/schema_dummy_scanner.h | 2 +- .../schema_scanner/schema_variables_scanner.h | 1 - be/src/exec/tablet_sink.cpp | 19 +- be/src/exec/tablet_sink.h | 11 +- be/src/exprs/agg_fn_evaluator.cpp | 11 +- be/src/exprs/aggregate_functions.cpp | 51 +- be/src/exprs/arithmetic_expr.h | 110 ++--- be/src/exprs/binary_predicate.h | 40 +- be/src/exprs/bloomfilter_predicate.h | 3 +- be/src/exprs/case_expr.h | 28 +- be/src/exprs/cast_expr.h | 32 +- be/src/exprs/compound_predicate.h | 12 +- be/src/exprs/conditional_functions.h | 96 ++-- be/src/exprs/expr.h | 2 +- be/src/exprs/hash_functions.h | 4 +- be/src/exprs/hybrid_set.h | 12 +- be/src/exprs/in_predicate.h | 8 +- be/src/exprs/info_func.h | 6 +- be/src/exprs/literal.h | 26 +- be/src/exprs/minmax_predicate.h | 4 +- be/src/exprs/new_agg_fn_evaluator.cc | 2 - be/src/exprs/null_literal.h | 22 +- be/src/exprs/scalar_fn_call.h | 35 +- be/src/exprs/slot_ref.h | 37 +- be/src/exprs/tuple_is_null_predicate.h | 12 +- be/src/geo/wkt_yacc.y | 2 +- be/src/gutil/hash/hash.cc | 14 +- be/src/gutil/hash/hash.h | 132 +---- be/src/gutil/strings/join.h | 6 - be/src/http/action/checksum_action.cpp | 6 +- be/src/http/action/checksum_action.h | 8 +- be/src/http/action/health_action.cpp | 2 +- be/src/http/action/health_action.h | 7 +- be/src/http/action/pprof_actions.cpp | 9 +- be/src/http/action/snapshot_action.cpp | 5 +- be/src/http/action/snapshot_action.h | 5 +- .../http/action/tablet_migration_action.cpp | 9 +- be/src/olap/bloom_filter_reader.h | 1 - be/src/olap/byte_buffer.cpp | 2 +- be/src/olap/collect_iterator.h | 37 +- be/src/olap/comparison_predicate.cpp | 248 +++++----- be/src/olap/cumulative_compaction_policy.h | 19 +- be/src/olap/data_dir.cpp | 5 +- be/src/olap/decimal12.h | 10 +- be/src/olap/fs/block_manager.h | 16 +- be/src/olap/fs/file_block_manager.h | 8 +- be/src/olap/fs/remote_block_manager.h | 8 +- be/src/olap/hll.h | 67 ++- be/src/olap/lru_cache.h | 16 +- be/src/olap/memory/typed_column_writer.h | 3 +- be/src/olap/memtable.cpp | 16 +- be/src/olap/memtable.h | 29 +- be/src/olap/olap_cond.h | 2 +- be/src/olap/push_handler.h | 4 +- be/src/olap/rowset/column_reader.cpp | 3 +- be/src/olap/rowset/column_reader.h | 19 +- be/src/olap/rowset/column_writer.h | 54 +- be/src/olap/rowset/rowset_factory.h | 5 +- be/src/olap/rowset/rowset_writer.h | 2 +- .../olap/rowset/segment_v2/bitshuffle_page.h | 19 +- .../segment_v2/bloom_filter_index_writer.h | 2 +- be/src/olap/rowset/segment_v2/encoding_info.h | 4 +- be/src/olap/rowset/segment_v2/plain_page.h | 4 +- be/src/olap/rowset/segment_v2/segment.cpp | 33 +- .../rowset/segment_v2/segment_iterator.cpp | 101 ++-- .../olap/rowset/segment_v2/segment_iterator.h | 23 +- be/src/olap/schema.cpp | 157 +++--- be/src/olap/schema_change.h | 25 +- be/src/olap/short_key_index.h | 4 +- be/src/olap/task/engine_alter_tablet_task.cpp | 11 +- be/src/olap/task/engine_alter_tablet_task.h | 8 +- be/src/olap/types.h | 8 +- be/src/olap/uint24.h | 2 + be/src/runtime/buffered_block_mgr2.h | 4 +- be/src/runtime/bufferpool/buffer_pool.h | 4 +- be/src/runtime/collection_value.cpp | 4 +- be/src/runtime/descriptors.cpp | 7 +- be/src/runtime/descriptors.h | 8 +- be/src/runtime/disk_io_mgr.cc | 2 - be/src/runtime/export_sink.h | 2 +- be/src/runtime/file_result_writer.h | 2 +- be/src/runtime/fold_constant_executor.h | 17 +- be/src/runtime/load_path_mgr.h | 1 - be/src/runtime/memory/chunk_allocator.h | 2 +- be/src/runtime/memory_scratch_sink.h | 1 - be/src/runtime/result_buffer_mgr.h | 2 +- be/src/runtime/result_file_sink.h | 14 +- be/src/runtime/result_sink.h | 15 +- be/src/runtime/result_writer.h | 2 +- be/src/runtime/row_batch.cpp | 58 +-- be/src/runtime/small_file_mgr.cpp | 3 +- be/src/runtime/snapshot_loader.h | 2 +- be/src/runtime/stream_load/stream_load_pipe.h | 4 +- be/src/service/backend_service.h | 35 +- be/src/service/http_service.cpp | 6 +- be/src/service/internal_service.h | 6 +- be/src/udf/udf.h | 8 +- be/src/udf/udf_internal.h | 2 +- be/src/util/arrow/row_batch.cpp | 24 +- be/src/util/arrow/row_block.cpp | 2 +- be/src/util/cidr.cpp | 8 +- be/src/util/crc32c.cpp | 2 +- be/src/util/date_func.h | 2 +- be/src/util/metrics.h | 24 +- be/src/util/mysql_load_error_hub.cpp | 4 +- be/src/util/mysql_row_buffer.cpp | 14 +- be/src/util/system_metrics.h | 14 +- be/src/util/topn_counter.h | 2 +- be/src/util/tuple_row_zorder_compare.h | 31 +- be/src/vec/columns/column_complex.h | 20 +- be/src/vec/columns/column_vector.h | 18 +- be/src/vec/columns/predicate_column.h | 152 +++--- be/src/vec/common/cow.h | 87 ++-- be/src/vec/core/block.cpp | 38 +- be/src/vec/data_types/data_type_bitmap.h | 8 +- be/src/vec/data_types/data_type_date.h | 2 +- be/src/vec/data_types/data_type_date_time.h | 2 +- be/src/vec/data_types/data_type_decimal.h | 20 +- be/src/vec/data_types/data_type_hll.h | 7 +- be/src/vec/data_types/data_type_nullable.h | 2 +- be/src/vec/data_types/data_type_number_base.h | 4 +- be/src/vec/data_types/data_type_string.h | 4 +- be/src/vec/exec/join/vhash_join_node.h | 18 +- be/src/vec/exec/vaggregation_node.h | 160 +++--- be/src/vec/exec/vanalytic_eval_node.h | 3 +- be/src/vec/exec/vassert_num_rows_node.h | 6 +- be/src/vec/exec/vcross_join_node.h | 18 +- be/src/vec/exec/vempty_set_node.h | 20 +- be/src/vec/exec/ves_http_scan_node.h | 5 +- be/src/vec/exec/vexcept_node.h | 3 +- be/src/vec/exec/vintersect_node.h | 3 +- be/src/vec/exec/vmysql_scan_node.h | 1 + be/src/vec/exec/vodbc_scan_node.h | 1 + be/src/vec/exec/vrepeat_node.h | 1 + be/src/vec/exec/vschema_scan_node.h | 4 +- be/src/vec/exec/vset_operation_node.h | 4 +- be/src/vec/exec/vsort_node.h | 27 +- be/src/vec/exprs/vcase_expr.h | 10 +- be/src/vec/exprs/vcast_expr.h | 10 +- be/src/vec/exprs/vectorized_fn_call.h | 10 +- be/src/vec/exprs/vin_predicate.h | 12 +- be/src/vec/exprs/vslot_ref.h | 8 +- be/src/vec/functions/function_cast.h | 4 +- be/src/vec/functions/function_rpc.cpp | 4 +- be/src/vec/olap/vcollect_iterator.h | 2 - be/src/vec/runtime/vdata_stream_recvr.h | 4 +- be/src/vec/sink/result_sink.h | 3 +- be/src/vec/sink/vtablet_sink.h | 10 +- be/test/env/env_posix_test.cpp | 17 +- be/test/exec/hash_table_test.cpp | 3 - be/test/gutil/strings/numbers_test.cpp | 9 +- be/test/http/http_client_test.cpp | 5 +- be/test/http/stream_load_test.cpp | 2 +- be/test/olap/column_reader_test.cpp | 70 +-- be/test/olap/delta_writer_test.cpp | 13 +- .../segment_v2/bitshuffle_page_test.cpp | 4 +- .../rowset/segment_v2/plain_page_test.cpp | 2 +- be/test/olap/schema_change_test.cpp | 2 +- be/test/olap/storage_types_test.cpp | 2 +- .../olap/timestamped_version_tracker_test.cpp | 3 +- be/test/runtime/result_queue_mgr_test.cpp | 16 +- be/test/tools/benchmark_tool.cpp | 13 +- be/test/util/broker_storage_backend_test.cpp | 1 - env.sh | 3 + 200 files changed, 1750 insertions(+), 2617 deletions(-) delete mode 100644 be/src/exec/aggregation_node.cpp delete mode 100644 be/src/exec/aggregation_node.h diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 377ddd6e30..95b448c2e8 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -327,16 +327,12 @@ check_function_exists(sched_getcpu HAVE_SCHED_GETCPU) # -Wall: Enable all warnings. # -Wno-sign-compare: suppress warnings for comparison between signed and unsigned # integers -# -Wno-unknown-pragmas: suppress warnings for unknown (compiler specific) pragmas -# -Wno-deprecated: gutil contains deprecated headers -# -Wno-vla: we use C99-style variable-length arrays # -pthread: enable multithreaded malloc # -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG: enable nanosecond precision for boost # -fno-omit-frame-pointers: Keep frame pointer for functions in register -set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -Wno-unknown-pragmas -pthread -Werror") +set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -pthread -Werror") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fstrict-aliasing -fno-omit-frame-pointer") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++17 -D__STDC_FORMAT_MACROS") -set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED") # Enable the cpu and heap profile of brpc @@ -364,96 +360,6 @@ if (COMPILER_CLANG) if(MAKE_TEST STREQUAL "OFF") add_compile_options(-Qunused-arguments) endif() - - # TODO: Fix all these warnings - add_compile_options( - -Wno-address-of-packed-member - -Wno-ambiguous-reversed-operator - -Wno-atomic-implicit-seq-cst - -Wno-cast-align - -Wno-cast-qual - -Wno-comma - -Wno-conditional-uninitialized - -Wno-constant-conversion - -Wno-covered-switch-default - -Wno-delete-abstract-non-virtual-dtor - -Wno-delete-non-abstract-non-virtual-dtor - -Wno-deprecated - -Wno-deprecated-copy-with-dtor - -Wno-deprecated-copy-with-user-provided-copy - -Wno-deprecated-copy-with-user-provided-dtor - -Wno-deprecated-declarations - -Wno-deprecated-this-capture - -Wno-documentation - -Wno-embedded-directive - -Wno-enum-compare-switch - -Wno-extra-semi - -Wno-extra-semi-stmt - -Wno-format-nonliteral - -Wno-frame-larger-than - -Wno-gnu-label-as-value - -Wno-gnu-redeclared-enum - -Wno-gnu-statement-expression - -Wno-gnu-zero-variadic-macro-arguments - -Wno-header-hygiene - -Wno-ignored-qualifiers - -Wno-implicit-const-int-float-conversion - -Wno-implicit-exception-spec-mismatch - -Wno-implicit-fallthrough - -Wno-inconsistent-missing-destructor-override - -Wno-inconsistent-missing-override - -Wno-instantiation-after-specialization - -Wno-macro-redefined - -Wno-mismatched-new-delete - -Wno-mismatched-tags - -Wno-missing-field-initializers - -Wno-missing-noreturn - -Wno-newline-eof - -Wno-non-virtual-dtor - -Wno-null-conversion - -Wno-null-dereference - -Wno-old-style-cast - -Wno-overloaded-virtual - -Wno-pedantic - -Wno-pessimizing-move - -Wno-pragma-once-outside-header - -Wno-reserved-identifier - -Wno-reserved-macro-identifier - -Wno-return-type - -Wno-shadow - -Wno-shadow-field - -Wno-shadow-field-in-constructor - -Wno-shadow-uncaptured-local - -Wno-shift-count-overflow - -Wno-sign-compare - -Wno-string-plus-int - -Wno-suggest-destructor-override - -Wno-suggest-override - -Wno-tautological-constant-out-of-range-compare - -Wno-tautological-unsigned-zero-compare - -Wno-undef - -Wno-uninitialized - -Wno-unknown-pragmas - -Wno-unknown-warning-option - -Wno-unreachable-code - -Wno-unreachable-code-break - -Wno-unreachable-code-return - -Wno-unused-but-set-variable - -Wno-unused-const-variable - -Wno-unused-exception-parameter - -Wno-unused-function - -Wno-unused-lambda-capture - -Wno-unused-local-typedef - -Wno-unused-macros - -Wno-unused-member-function - -Wno-unused-parameter - -Wno-unused-private-field - -Wno-unused-result - -Wno-unused-value - -Wno-unused-variable - -Wno-variadic-macros - -Wno-zero-as-null-pointer-constant - ) endif () # https://github.com/boostorg/uuid/issues/92 diff --git a/be/src/agent/cgroups_mgr.cpp b/be/src/agent/cgroups_mgr.cpp index ffcbe210b9..a5df832eef 100644 --- a/be/src/agent/cgroups_mgr.cpp +++ b/be/src/agent/cgroups_mgr.cpp @@ -54,10 +54,7 @@ std::map CgroupsMgr::_s_resource_cgroups = { {TResourceType::type::TRESOURCE_IO_SHARE, "blkio.weight"}}; CgroupsMgr::CgroupsMgr(ExecEnv* exec_env, const string& root_cgroups_path) - : _exec_env(exec_env), - _root_cgroups_path(root_cgroups_path), - _is_cgroups_init_success(false), - _cur_version(-1) { + : _root_cgroups_path(root_cgroups_path), _is_cgroups_init_success(false), _cur_version(-1) { if (s_global_cg_mgr == nullptr) { s_global_cg_mgr = this; } diff --git a/be/src/agent/cgroups_mgr.h b/be/src/agent/cgroups_mgr.h index de46a47eb2..a0886f8278 100644 --- a/be/src/agent/cgroups_mgr.h +++ b/be/src/agent/cgroups_mgr.h @@ -155,7 +155,6 @@ public: const static std::string _s_system_group; private: - ExecEnv* _exec_env; std::string _root_cgroups_path; int32_t _drop_retry_times = 10; bool _is_cgroups_init_success; diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 457f8da65b..2eeed4505f 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -203,7 +203,8 @@ void TaskWorkerPool::start() { break; case TaskWorkerType::SUBMIT_TABLE_COMPACTION: _worker_count = 1; - cb = std::bind(&TaskWorkerPool::_submit_table_compaction_worker_thread_callback, this); + cb = std::bind(&TaskWorkerPool::_submit_table_compaction_worker_thread_callback, + this); break; default: // pass @@ -298,8 +299,9 @@ void TaskWorkerPool::_finish_task(const TFinishTaskRequest& finish_task_request) break; } else { DorisMetrics::instance()->finish_task_requests_failed->increment(1); - LOG(WARNING) << "finish task failed. type=" << to_string(finish_task_request.task_type) << ", signature=" - << finish_task_request.signature << ", status_code=" << result.status.status_code; + LOG(WARNING) << "finish task failed. type=" << to_string(finish_task_request.task_type) + << ", signature=" << finish_task_request.signature + << ", status_code=" << result.status.status_code; try_time += 1; } sleep(config::sleep_one_second); @@ -534,8 +536,7 @@ void TaskWorkerPool::_alter_tablet(const TAgentTaskRequest& agent_task_req, int6 if (status == DORIS_SUCCESS) { new_tablet_id = agent_task_req.alter_tablet_req_v2.new_tablet_id; new_schema_hash = agent_task_req.alter_tablet_req_v2.new_schema_hash; - EngineAlterTabletTask engine_task(agent_task_req.alter_tablet_req_v2, signature, task_type, - &error_msgs, process_name); + EngineAlterTabletTask engine_task(agent_task_req.alter_tablet_req_v2); OLAPStatus sc_status = _env->storage_engine()->execute_task(&engine_task); if (sc_status != OLAP_SUCCESS) { if (sc_status == OLAP_ERR_DATA_QUALITY_ERR) { @@ -1072,9 +1073,9 @@ void TaskWorkerPool::_check_consistency_worker_thread_callback() { TStatus task_status; uint32_t checksum = 0; - EngineChecksumTask engine_task( - check_consistency_req.tablet_id, check_consistency_req.schema_hash, - check_consistency_req.version, &checksum); + EngineChecksumTask engine_task(check_consistency_req.tablet_id, + check_consistency_req.schema_hash, + check_consistency_req.version, &checksum); OLAPStatus res = _env->storage_engine()->execute_task(&engine_task); if (res != OLAP_SUCCESS) { LOG(WARNING) << "check consistency failed. status: " << res @@ -1690,8 +1691,8 @@ void TaskWorkerPool::_submit_table_compaction_worker_thread_callback() { continue; } - Status status = StorageEngine::instance()->submit_compaction_task( - tablet_ptr, compaction_type); + Status status = + StorageEngine::instance()->submit_compaction_task(tablet_ptr, compaction_type); if (!status.ok()) { LOG(WARNING) << "failed to submit table compaction task. " << status.to_string(); } diff --git a/be/src/common/config.h b/be/src/common/config.h index 074d7a5f39..b3c60f5618 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -208,7 +208,7 @@ CONF_mInt32(tablet_rowset_stale_sweep_time_sec, "1800"); CONF_Int32(max_garbage_sweep_interval, "3600"); CONF_Int32(min_garbage_sweep_interval, "180"); CONF_mInt32(snapshot_expire_time_sec, "172800"); -// It is only a recommended value. When the disk space is insufficient, +// It is only a recommended value. When the disk space is insufficient, // the file storage period under trash dose not have to comply with this parameter. CONF_mInt32(trash_file_expire_time_sec, "259200"); // check row nums for BE/CE and schema change. true is open, false is closed. @@ -409,9 +409,6 @@ CONF_Bool(enable_quadratic_probing, "false"); // for pprof CONF_String(pprof_profile_dir, "${DORIS_HOME}/log"); -// for partition -CONF_Bool(enable_partitioned_aggregation, "true"); - // to forward compatibility, will be removed later CONF_mBool(enable_token_check, "true"); diff --git a/be/src/common/daemon.h b/be/src/common/daemon.h index 4544c1a9a1..cd03625e4b 100644 --- a/be/src/common/daemon.h +++ b/be/src/common/daemon.h @@ -26,7 +26,7 @@ namespace doris { -class StorePath; +struct StorePath; class Thread; class Daemon { diff --git a/be/src/common/status.h b/be/src/common/status.h index 23bf764192..aba8fc9e85 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -306,5 +306,8 @@ private: } \ } while (false); } // namespace doris +#ifdef WARN_UNUSED_RESULT +#undef WARN_UNUSED_RESULT +#endif #define WARN_UNUSED_RESULT __attribute__((warn_unused_result)) diff --git a/be/src/env/env.h b/be/src/env/env.h index d47e5a75ca..957ad36b05 100644 --- a/be/src/env/env.h +++ b/be/src/env/env.h @@ -11,8 +11,8 @@ #include #include "common/status.h" -#include "util/slice.h" #include "gen_cpp/Types_types.h" +#include "util/slice.h" namespace doris { @@ -176,7 +176,8 @@ public: virtual Status link_file(const std::string& /*old_path*/, const std::string& /*new_path*/) = 0; // get space info for local and remote system - virtual Status get_space_info(const std::string& path, int64_t* capacity, int64_t* available) = 0; + virtual Status get_space_info(const std::string& path, int64_t* capacity, + int64_t* available) = 0; virtual bool is_remote_env() = 0; @@ -198,9 +199,7 @@ private: }; struct FilePathDesc { - FilePathDesc(const std::string& path) { - filepath = path; - } + FilePathDesc(const std::string& path) { filepath = path; } FilePathDesc() {} TStorageMedium::type storage_medium = TStorageMedium::HDD; std::string filepath; @@ -268,6 +267,7 @@ public: } return path_desc; } + private: TStorageMedium::type _storage_medium = TStorageMedium::HDD; std::stringstream _filepath_stream; @@ -424,7 +424,7 @@ public: RandomRWFile() {} virtual ~RandomRWFile() {} - virtual Status read_at(uint64_t offset, const Slice* result) const = 0; + virtual Status read_at(uint64_t offset, const Slice& result) const = 0; virtual Status readv_at(uint64_t offset, const Slice* res, size_t res_cnt) const = 0; diff --git a/be/src/env/env_posix.cpp b/be/src/env/env_posix.cpp index 046650495f..9478329548 100644 --- a/be/src/env/env_posix.cpp +++ b/be/src/env/env_posix.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include "env/env_posix.h" + #include #include #include @@ -23,13 +25,12 @@ #include #include -#include #include #include +#include #include "common/logging.h" #include "env/env.h" -#include "env/env_posix.h" #include "gutil/gscoped_ptr.h" #include "gutil/macros.h" #include "gutil/port.h" @@ -288,10 +289,9 @@ public: Status read_all(std::string* content) const override { std::fstream fs(_filename.c_str(), std::fstream::in); if (!fs.is_open()) { - RETURN_NOT_OK_STATUS_WITH_WARN( - Status::IOError( - strings::Substitute("failed to open cluster id file $0", _filename)), - "open file failed"); + RETURN_NOT_OK_STATUS_WITH_WARN(Status::IOError(strings::Substitute( + "failed to open cluster id file $0", _filename)), + "open file failed"); } std::string data; fs >> data; @@ -303,7 +303,8 @@ public: Status::Corruption(strings::Substitute( "read_all from file $0 is corrupt. [eofbit=$1 failbit=$2 badbit=$3]", _filename, fs.rdstate() & std::fstream::eofbit, - fs.rdstate() & std::fstream::failbit, fs.rdstate() & std::fstream::badbit)), + fs.rdstate() & std::fstream::failbit, + fs.rdstate() & std::fstream::badbit)), "read_all is error"); } return Status::OK(); @@ -449,8 +450,8 @@ public: ~PosixRandomRWFile() { WARN_IF_ERROR(close(), "Failed to close " + _filename); } - Status read_at(uint64_t offset, const Slice* result) const override { - return readv_at(offset, result, 1); + Status read_at(uint64_t offset, const Slice& result) const override { + return readv_at(offset, &result, 1); } Status readv_at(uint64_t offset, const Slice* result, size_t res_cnt) const override { @@ -531,8 +532,7 @@ Status PosixEnv::init_conf() { return Status::OK(); } -Status PosixEnv::new_sequential_file(const string& fname, - std::unique_ptr* result) { +Status PosixEnv::new_sequential_file(const string& fname, std::unique_ptr* result) { FILE* f; POINTER_RETRY_ON_EINTR(f, fopen(fname.c_str(), "r")); if (f == nullptr) { @@ -544,12 +544,13 @@ Status PosixEnv::new_sequential_file(const string& fname, // get a RandomAccessFile pointer without file cache Status PosixEnv::new_random_access_file(const std::string& fname, - std::unique_ptr* result) { + std::unique_ptr* result) { return new_random_access_file(RandomAccessFileOptions(), fname, result); } -Status PosixEnv::new_random_access_file(const RandomAccessFileOptions& opts, const std::string& fname, - std::unique_ptr* result) { +Status PosixEnv::new_random_access_file(const RandomAccessFileOptions& opts, + const std::string& fname, + std::unique_ptr* result) { int fd; RETRY_ON_EINTR(fd, open(fname.c_str(), O_RDONLY)); if (fd < 0) { @@ -564,7 +565,7 @@ Status PosixEnv::new_writable_file(const string& fname, std::unique_ptr* result) { + std::unique_ptr* result) { int fd; RETURN_IF_ERROR(do_open(fname, opts.mode, &fd)); @@ -581,7 +582,7 @@ Status PosixEnv::new_random_rw_file(const string& fname, std::unique_ptr* result) { + std::unique_ptr* result) { int fd; RETURN_IF_ERROR(do_open(fname, opts.mode, &fd)); result->reset(new PosixRandomRWFile(fname, fd, opts.sync_on_close)); @@ -609,8 +610,7 @@ Status PosixEnv::get_children(const std::string& dir, std::vector* return Status::OK(); } -Status PosixEnv::iterate_dir(const std::string& dir, - const std::function& cb) { +Status PosixEnv::iterate_dir(const std::string& dir, const std::function& cb) { DIR* d = opendir(dir.c_str()); if (d == nullptr) { return io_error(dir, errno); diff --git a/be/src/env/env_posix.h b/be/src/env/env_posix.h index d9caf6b2b7..84a35cc71b 100644 --- a/be/src/env/env_posix.h +++ b/be/src/env/env_posix.h @@ -33,7 +33,7 @@ class PosixEnv : public Env { public: ~PosixEnv() override {} - Status init_conf(); + Status init_conf() override; Status new_sequential_file(const std::string& fname, std::unique_ptr* result) override; @@ -45,12 +45,14 @@ public: Status new_random_access_file(const RandomAccessFileOptions& opts, const std::string& fname, std::unique_ptr* result) override; - Status new_writable_file(const std::string& fname, std::unique_ptr* result) override; + Status new_writable_file(const std::string& fname, + std::unique_ptr* result) override; Status new_writable_file(const WritableFileOptions& opts, const std::string& fname, std::unique_ptr* result) override; - Status new_random_rw_file(const std::string& fname, std::unique_ptr* result) override; + Status new_random_rw_file(const std::string& fname, + std::unique_ptr* result) override; Status new_random_rw_file(const RandomRWFileOptions& opts, const std::string& fname, std::unique_ptr* result) override; @@ -59,8 +61,7 @@ public: Status get_children(const std::string& dir, std::vector* result) override; - Status iterate_dir(const std::string& dir, - const std::function& cb) override; + Status iterate_dir(const std::string& dir, const std::function& cb) override; Status delete_file(const std::string& fname) override; @@ -93,9 +94,7 @@ public: Status get_space_info(const std::string& path, int64_t* capacity, int64_t* available) override; - bool is_remote_env() override { - return false; - } + bool is_remote_env() override { return false; } }; } // namespace doris \ No newline at end of file diff --git a/be/src/env/env_remote.cpp b/be/src/env/env_remote.cpp index a75e84ddf3..c992d69bbb 100644 --- a/be/src/env/env_remote.cpp +++ b/be/src/env/env_remote.cpp @@ -15,16 +15,16 @@ // specific language governing permissions and limitations // under the License. +#include "env/env_remote.h" + #include "common/config.h" #include "common/logging.h" #include "common/status.h" #include "env/env.h" -#include "env/env_remote.h" +#include "gutil/strings/substitute.h" #include "util/s3_storage_backend.h" #include "util/s3_util.h" -#include "gutil/strings/substitute.h" - namespace doris { using std::string; @@ -34,8 +34,7 @@ class RemoteRandomAccessFile : public RandomAccessFile { public: RemoteRandomAccessFile(std::string filename, std::shared_ptr storage_backend) : _filename(std::move(filename)), _storage_backend(storage_backend) {} - ~RemoteRandomAccessFile() { - } + ~RemoteRandomAccessFile() {} Status read_at(uint64_t offset, const Slice* result) const override { return readv_at(offset, result, 1); @@ -47,9 +46,7 @@ public: Status read_all(std::string* content) const override { return _storage_backend->direct_download(_filename, content); } - Status size(uint64_t* size) const override { - return Status::NotSupported("No support", 1, ""); - } + Status size(uint64_t* size) const override { return Status::NotSupported("No support", 1, ""); } const std::string& file_name() const override { return _filename; } @@ -60,7 +57,8 @@ private: class RemoteWritableFile : public WritableFile { public: - RemoteWritableFile(std::string filename, std::shared_ptr storage_backend, uint64_t filesize) + RemoteWritableFile(std::string filename, std::shared_ptr storage_backend, + uint64_t filesize) : _filename(std::move(filename)), _storage_backend(storage_backend), _filesize(filesize) {} @@ -79,8 +77,9 @@ public: bytes_written += data[i].size; } Status status = _storage_backend->direct_upload(_filename, content); - RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute( - "direct_upload failed: $0, err=$1", _filename, status.to_string())); + RETURN_NOT_OK_STATUS_WITH_WARN( + status, strings::Substitute("direct_upload failed: $0, err=$1", _filename, + status.to_string())); _filesize += bytes_written; return Status::OK(); } @@ -89,17 +88,11 @@ public: return Status::NotSupported("No support", 1, ""); } - Status close() override { - return Status::OK(); - } + Status close() override { return Status::OK(); } - Status flush(FlushMode mode) override { - return Status::OK(); - } + Status flush(FlushMode mode) override { return Status::OK(); } - Status sync() override { - return Status::OK(); - } + Status sync() override { return Status::OK(); } uint64_t size() const override { return _filesize; } const string& filename() const override { return _filename; } @@ -115,7 +108,6 @@ public: RemoteRandomRWFile(const FilePathDesc& path_desc) : _path_desc(path_desc) {} ~RemoteRandomRWFile() { WARN_IF_ERROR(close(), "Failed to close " + _path_desc.filepath); } - virtual Status read_at(uint64_t offset, const Slice& result) const { return Status::NotSupported("No support", 1, ""); } @@ -136,39 +128,36 @@ public: return Status::NotSupported("No support", 1, ""); } - Status sync() { - return Status::NotSupported("No support", 1, ""); - } + Status sync() { return Status::NotSupported("No support", 1, ""); } - Status close() { - return Status::NotSupported("No support", 1, ""); - } + Status close() { return Status::NotSupported("No support", 1, ""); } - Status size(uint64_t* size) const { - return Status::NotSupported("No support", 1, ""); - } + Status size(uint64_t* size) const { return Status::NotSupported("No support", 1, ""); } const string& filename() const { return _path_desc.filepath; } private: const FilePathDesc _path_desc; - const bool _sync_on_close = false; - bool _closed = false; }; Status RemoteEnv::init_conf() { std::map storage_prop; - if (doris::config::default_remote_storage_s3_ak.empty() || doris::config::default_remote_storage_s3_sk.empty() - || doris::config::default_remote_storage_s3_endpoint.empty() || doris::config::default_remote_storage_s3_region.empty()) { + if (doris::config::default_remote_storage_s3_ak.empty() || + doris::config::default_remote_storage_s3_sk.empty() || + doris::config::default_remote_storage_s3_endpoint.empty() || + doris::config::default_remote_storage_s3_region.empty()) { return Status::OK(); } storage_prop[S3_AK] = doris::config::default_remote_storage_s3_ak; storage_prop[S3_SK] = doris::config::default_remote_storage_s3_sk; storage_prop[S3_ENDPOINT] = doris::config::default_remote_storage_s3_endpoint; storage_prop[S3_REGION] = doris::config::default_remote_storage_s3_region; - storage_prop[S3_MAX_CONN_SIZE] = std::to_string(doris::config::default_remote_storage_s3_max_conn); - storage_prop[S3_REQUEST_TIMEOUT_MS] = std::to_string(doris::config::default_remote_storage_s3_request_timeout_ms); - storage_prop[S3_CONN_TIMEOUT_MS] = std::to_string(doris::config::default_remote_storage_s3_conn_timeout_ms); + storage_prop[S3_MAX_CONN_SIZE] = + std::to_string(doris::config::default_remote_storage_s3_max_conn); + storage_prop[S3_REQUEST_TIMEOUT_MS] = + std::to_string(doris::config::default_remote_storage_s3_request_timeout_ms); + storage_prop[S3_CONN_TIMEOUT_MS] = + std::to_string(doris::config::default_remote_storage_s3_conn_timeout_ms); if (ClientFactory::is_s3_conf_valid(storage_prop)) { _storage_backend.reset(new S3StorageBackend(storage_prop)); @@ -187,13 +176,15 @@ Status RemoteEnv::new_random_access_file(const std::string& fname, return new_random_access_file(RandomAccessFileOptions(), fname, result); } -Status RemoteEnv::new_random_access_file(const RandomAccessFileOptions& opts, const std::string& fname, +Status RemoteEnv::new_random_access_file(const RandomAccessFileOptions& opts, + const std::string& fname, std::unique_ptr* result) { result->reset(new RemoteRandomAccessFile(fname, _storage_backend)); return Status::OK(); } -Status RemoteEnv::new_writable_file(const std::string& fname, std::unique_ptr* result) { +Status RemoteEnv::new_writable_file(const std::string& fname, + std::unique_ptr* result) { return new_writable_file(WritableFileOptions(), fname, result); } @@ -207,7 +198,8 @@ Status RemoteEnv::new_writable_file(const WritableFileOptions& opts, const std:: return Status::OK(); } -Status RemoteEnv::new_random_rw_file(const std::string& fname, std::unique_ptr* result) { +Status RemoteEnv::new_random_rw_file(const std::string& fname, + std::unique_ptr* result) { return new_random_rw_file(RandomRWFileOptions(), fname, result); } @@ -224,8 +216,8 @@ Status RemoteEnv::path_exists(const std::string& fname, bool is_dir) { } else { status = storage_backend->exist(fname); } - RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute( - "path_exists failed: $0, err=$1", fname, status.to_string())); + RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute("path_exists failed: $0, err=$1", + fname, status.to_string())); return Status::OK(); } @@ -233,16 +225,15 @@ Status RemoteEnv::get_children(const std::string& dir, std::vector* return Status::IOError(strings::Substitute("Unable to get_children $0", dir), 0, ""); } -Status RemoteEnv::iterate_dir(const std::string& dir, - const std::function& cb) { +Status RemoteEnv::iterate_dir(const std::string& dir, const std::function& cb) { return Status::IOError(strings::Substitute("Unable to iterate_dir $0", dir), 0, ""); } Status RemoteEnv::delete_file(const std::string& fname) { std::shared_ptr storage_backend = get_storage_backend(); Status status = storage_backend->rm(fname); - RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute( - "delete_file failed: $0, err=$1", fname, status.to_string())); + RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute("delete_file failed: $0, err=$1", + fname, status.to_string())); return Status::OK(); } @@ -269,8 +260,8 @@ Status RemoteEnv::create_dirs(const string& dirname) { Status RemoteEnv::delete_dir(const std::string& dirname) { std::shared_ptr storage_backend = get_storage_backend(); Status status = storage_backend->rmdir(dirname); - RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute( - "delete_dir failed: $0, err=$1", dirname, status.to_string())); + RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute("delete_dir failed: $0, err=$1", + dirname, status.to_string())); return Status::OK(); } @@ -313,7 +304,8 @@ Status RemoteEnv::get_file_size(const std::string& fname, uint64_t* size) { } Status RemoteEnv::get_file_modified_time(const std::string& fname, uint64_t* file_mtime) { - return Status::IOError(strings::Substitute("Unable to get_file_modified_time $0", fname), 0, ""); + return Status::IOError(strings::Substitute("Unable to get_file_modified_time $0", fname), 0, + ""); } Status RemoteEnv::copy_path(const std::string& src, const std::string& target) { @@ -323,24 +315,27 @@ Status RemoteEnv::copy_path(const std::string& src, const std::string& target) { Status RemoteEnv::rename_file(const std::string& src, const std::string& target) { std::shared_ptr storage_backend = get_storage_backend(); Status status = storage_backend->rename(src, target); - RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute( - "rename_file failed: from $0 to $1, err=$2", src, target, status.to_string())); + RETURN_NOT_OK_STATUS_WITH_WARN( + status, strings::Substitute("rename_file failed: from $0 to $1, err=$2", src, target, + status.to_string())); return Status::OK(); } Status RemoteEnv::rename_dir(const std::string& src, const std::string& target) { std::shared_ptr storage_backend = get_storage_backend(); Status status = storage_backend->rename_dir(src, target); - RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute( - "rename_dir failed: from $0 to $1, err=$2", src, target, status.to_string())); + RETURN_NOT_OK_STATUS_WITH_WARN( + status, strings::Substitute("rename_dir failed: from $0 to $1, err=$2", src, target, + status.to_string())); return Status::OK(); } Status RemoteEnv::link_file(const std::string& old_path, const std::string& new_path) { std::shared_ptr storage_backend = get_storage_backend(); Status status = storage_backend->copy(old_path, new_path); - RETURN_NOT_OK_STATUS_WITH_WARN(status, strings::Substitute( - "link_file failed: from $0 to $1, err=$2", old_path, new_path, status.to_string())); + RETURN_NOT_OK_STATUS_WITH_WARN( + status, strings::Substitute("link_file failed: from $0 to $1, err=$2", old_path, + new_path, status.to_string())); return Status::OK(); } diff --git a/be/src/env/env_remote.h b/be/src/env/env_remote.h index dc7fd530ef..d606bdc36e 100644 --- a/be/src/env/env_remote.h +++ b/be/src/env/env_remote.h @@ -34,7 +34,7 @@ class RemoteEnv : public Env { public: ~RemoteEnv() override {} - Status init_conf(); + Status init_conf() override; Status new_sequential_file(const std::string& fname, std::unique_ptr* result) override; @@ -46,12 +46,14 @@ public: Status new_random_access_file(const RandomAccessFileOptions& opts, const std::string& fname, std::unique_ptr* result) override; - Status new_writable_file(const std::string& fname, std::unique_ptr* result) override; + Status new_writable_file(const std::string& fname, + std::unique_ptr* result) override; Status new_writable_file(const WritableFileOptions& opts, const std::string& fname, std::unique_ptr* result) override; - Status new_random_rw_file(const std::string& fname, std::unique_ptr* result) override; + Status new_random_rw_file(const std::string& fname, + std::unique_ptr* result) override; Status new_random_rw_file(const RandomRWFileOptions& opts, const std::string& fname, std::unique_ptr* result) override; @@ -60,8 +62,7 @@ public: Status get_children(const std::string& dir, std::vector* result) override; - Status iterate_dir(const std::string& dir, - const std::function& cb) override; + Status iterate_dir(const std::string& dir, const std::function& cb) override; Status delete_file(const std::string& fname) override; @@ -94,9 +95,7 @@ public: Status get_space_info(const std::string& path, int64_t* capacity, int64_t* available) override; - bool is_remote_env() override { - return true; - } + bool is_remote_env() override { return true; } std::shared_ptr get_storage_backend(); diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 36e2af6693..709cb4de84 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -22,7 +22,6 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/exec") set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/exec") set(EXEC_FILES - aggregation_node.cpp analytic_eval_node.cpp blocking_join_node.cpp broker_scan_node.cpp diff --git a/be/src/exec/aggregation_node.cpp b/be/src/exec/aggregation_node.cpp deleted file mode 100644 index 82c8f6c267..0000000000 --- a/be/src/exec/aggregation_node.cpp +++ /dev/null @@ -1,464 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "exec/aggregation_node.h" - -#include -#include - -#include - -#include "exec/hash_table.hpp" -#include "exprs/agg_fn_evaluator.h" -#include "exprs/expr.h" -#include "exprs/slot_ref.h" -#include "gen_cpp/Exprs_types.h" -#include "gen_cpp/PlanNodes_types.h" -#include "runtime/descriptors.h" -#include "runtime/mem_pool.h" -#include "runtime/raw_value.h" -#include "runtime/row_batch.h" -#include "runtime/runtime_state.h" -#include "runtime/string_value.hpp" -#include "runtime/tuple.h" -#include "runtime/tuple_row.h" -#include "util/runtime_profile.h" - -namespace doris { - -// TODO: pass in maximum size; enforce by setting limit in mempool -// TODO: have a Status ExecNode::init(const TPlanNode&) member function -// that does initialization outside of c'tor, so we can indicate errors -AggregationNode::AggregationNode(ObjectPool* pool, const TPlanNode& tnode, - const DescriptorTbl& descs) - : ExecNode(pool, tnode, descs), - _intermediate_tuple_id(tnode.agg_node.intermediate_tuple_id), - _intermediate_tuple_desc(nullptr), - _output_tuple_id(tnode.agg_node.output_tuple_id), - _output_tuple_desc(nullptr), - _singleton_output_tuple(nullptr), - //_tuple_pool(new MemPool()), - // - _process_row_batch_fn(nullptr), - _needs_finalize(tnode.agg_node.need_finalize), - _build_timer(nullptr), - _get_results_timer(nullptr), - _hash_table_buckets_counter(nullptr) {} - -AggregationNode::~AggregationNode() {} - -Status AggregationNode::init(const TPlanNode& tnode, RuntimeState* state) { - RETURN_IF_ERROR(ExecNode::init(tnode, state)); - // ignore return status for now , so we need to introduce ExecNode::init() - RETURN_IF_ERROR( - Expr::create_expr_trees(_pool, tnode.agg_node.grouping_exprs, &_probe_expr_ctxs)); - - for (int i = 0; i < tnode.agg_node.aggregate_functions.size(); ++i) { - AggFnEvaluator* evaluator = nullptr; - AggFnEvaluator::create(_pool, tnode.agg_node.aggregate_functions[i], &evaluator); - _aggregate_evaluators.push_back(evaluator); - } - return Status::OK(); -} - -Status AggregationNode::prepare(RuntimeState* state) { - RETURN_IF_ERROR(ExecNode::prepare(state)); - _build_timer = ADD_TIMER(runtime_profile(), "BuildTime"); - _get_results_timer = ADD_TIMER(runtime_profile(), "GetResultsTime"); - _hash_table_buckets_counter = ADD_COUNTER(runtime_profile(), "BuildBuckets", TUnit::UNIT); - _hash_table_load_factor_counter = - ADD_COUNTER(runtime_profile(), "LoadFactor", TUnit::DOUBLE_VALUE); - - SCOPED_TIMER(_runtime_profile->total_time_counter()); - - _intermediate_tuple_desc = state->desc_tbl().get_tuple_descriptor(_intermediate_tuple_id); - _output_tuple_desc = state->desc_tbl().get_tuple_descriptor(_output_tuple_id); - DCHECK_EQ(_intermediate_tuple_desc->slots().size(), _output_tuple_desc->slots().size()); - RETURN_IF_ERROR( - Expr::prepare(_probe_expr_ctxs, state, child(0)->row_desc(), expr_mem_tracker())); - - // Construct build exprs from _agg_tuple_desc - for (int i = 0; i < _probe_expr_ctxs.size(); ++i) { - SlotDescriptor* desc = _intermediate_tuple_desc->slots()[i]; - Expr* expr = new SlotRef(desc); - state->obj_pool()->add(expr); - _build_expr_ctxs.push_back(new ExprContext(expr)); - state->obj_pool()->add(_build_expr_ctxs.back()); - } - - // Construct a new row desc for preparing the build exprs because neither the child's - // nor this node's output row desc may contain the intermediate tuple, e.g., - // in a single-node plan with an intermediate tuple different from the output tuple. - RowDescriptor build_row_desc(_intermediate_tuple_desc, false); - RETURN_IF_ERROR(Expr::prepare(_build_expr_ctxs, state, build_row_desc, expr_mem_tracker())); - - _tuple_pool.reset(new MemPool(mem_tracker().get())); - - _agg_fn_ctxs.resize(_aggregate_evaluators.size()); - int j = _probe_expr_ctxs.size(); - for (int i = 0; i < _aggregate_evaluators.size(); ++i, ++j) { - // skip non-materialized slots; we don't have evaluators instantiated for those - // while (!_agg_tuple_desc->slots()[j]->is_materialized()) { - // DCHECK_LT(j, _agg_tuple_desc->slots().size() - 1) - // << "#eval= " << _aggregate_evaluators.size() - // << " #probe=" << _probe_expr_ctxs.size(); - // ++j; - // } - SlotDescriptor* intermediate_slot_desc = _intermediate_tuple_desc->slots()[j]; - SlotDescriptor* output_slot_desc = _output_tuple_desc->slots()[j]; - RETURN_IF_ERROR(_aggregate_evaluators[i]->prepare( - state, child(0)->row_desc(), _tuple_pool.get(), intermediate_slot_desc, - output_slot_desc, mem_tracker(), &_agg_fn_ctxs[i])); - state->obj_pool()->add(_agg_fn_ctxs[i]); - } - - // TODO: how many buckets? - _hash_tbl.reset(new HashTable(_build_expr_ctxs, _probe_expr_ctxs, 1, true, - std::vector(_build_expr_ctxs.size(), false), id(), - mem_tracker(), 1024)); - - if (_probe_expr_ctxs.empty()) { - // create single output tuple now; we need to output something - // even if our input is empty - _singleton_output_tuple = construct_intermediate_tuple(); - } - - return Status::OK(); -} - -Status AggregationNode::open(RuntimeState* state) { - RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::OPEN)); - SCOPED_TIMER(_runtime_profile->total_time_counter()); - RETURN_IF_ERROR(ExecNode::open(state)); - RETURN_IF_ERROR(Expr::open(_probe_expr_ctxs, state)); - RETURN_IF_ERROR(Expr::open(_build_expr_ctxs, state)); - - for (int i = 0; i < _aggregate_evaluators.size(); ++i) { - RETURN_IF_ERROR(_aggregate_evaluators[i]->open(state, _agg_fn_ctxs[i])); - } - - RETURN_IF_ERROR(_children[0]->open(state)); - - RowBatch batch(_children[0]->row_desc(), state->batch_size(), mem_tracker().get()); - int64_t num_input_rows = 0; - int64_t num_agg_rows = 0; - - bool early_return = false; - bool limit_with_no_agg = (limit() != -1 && (_aggregate_evaluators.size() == 0)); - DCHECK_EQ(_aggregate_evaluators.size(), _agg_fn_ctxs.size()); - - while (true) { - bool eos = false; - RETURN_IF_CANCELLED(state); - RETURN_IF_ERROR(state->check_query_state("Aggregation, before getting next from child 0.")); - RETURN_IF_ERROR(_children[0]->get_next(state, &batch, &eos)); - // SCOPED_TIMER(_build_timer); - if (VLOG_ROW_IS_ON) { - for (int i = 0; i < batch.num_rows(); ++i) { - TupleRow* row = batch.get_row(i); - VLOG_ROW << "id=" << id() - << " input row: " << row->to_string(_children[0]->row_desc()); - } - } - - int64_t agg_rows_before = _hash_tbl->size(); - - if (_process_row_batch_fn != nullptr) { - _process_row_batch_fn(this, &batch); - } else if (_singleton_output_tuple != nullptr) { - SCOPED_TIMER(_build_timer); - process_row_batch_no_grouping(&batch, _tuple_pool.get()); - } else { - process_row_batch_with_grouping(&batch, _tuple_pool.get()); - if (limit_with_no_agg) { - if (_hash_tbl->size() >= limit()) { - early_return = true; - } - } - } - - // RETURN_IF_LIMIT_EXCEEDED(state); - RETURN_IF_ERROR(state->check_query_state("Aggregation, after hashing the child 0.")); - - COUNTER_SET(_hash_table_buckets_counter, _hash_tbl->num_buckets()); - COUNTER_SET(_hash_table_load_factor_counter, _hash_tbl->load_factor()); - num_agg_rows += (_hash_tbl->size() - agg_rows_before); - num_input_rows += batch.num_rows(); - - batch.reset(); - - RETURN_IF_ERROR(state->check_query_state("Aggregation, after setting the counter.")); - if (eos) { - break; - } - if (early_return) { - break; - } - } - - if (_singleton_output_tuple != nullptr) { - _hash_tbl->insert(reinterpret_cast(&_singleton_output_tuple)); - ++num_agg_rows; - } - - VLOG_ROW << "id=" << id() << " aggregated " << num_input_rows << " input rows into " - << num_agg_rows << " output rows"; - _output_iterator = _hash_tbl->begin(); - return Status::OK(); -} - -Status AggregationNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) { - // 1. `!need_finalize` means this aggregation node not the level two aggregation node - // 2. `_singleton_output_tuple != nullptr` means is not group by - // 3. `child(0)->rows_returned() == 0` mean not data from child - // in level two aggregation node should return nullptr result - // level one aggregation node set `eos = true` return directly - if (UNLIKELY(!_needs_finalize && _singleton_output_tuple != nullptr && - child(0)->rows_returned() == 0)) { - *eos = true; - return Status::OK(); - } - SCOPED_TIMER(_runtime_profile->total_time_counter()); - RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::GETNEXT)); - RETURN_IF_CANCELLED(state); - RETURN_IF_ERROR(state->check_query_state("Aggregation, before evaluating conjuncts.")); - SCOPED_TIMER(_get_results_timer); - - if (reached_limit()) { - *eos = true; - return Status::OK(); - } - - ExprContext** ctxs = &_conjunct_ctxs[0]; - int num_ctxs = _conjunct_ctxs.size(); - - int count = 0; - const int N = state->batch_size(); - while (!_output_iterator.at_end() && !row_batch->at_capacity()) { - // This loop can go on for a long time if the conjuncts are very selective. Do query - // maintenance every N iterations. - if (count++ % N == 0) { - RETURN_IF_CANCELLED(state); - RETURN_IF_ERROR(state->check_query_state("Aggregation, while evaluating conjuncts.")); - } - int row_idx = row_batch->add_row(); - TupleRow* row = row_batch->get_row(row_idx); - Tuple* intermediate_tuple = _output_iterator.get_row()->get_tuple(0); - Tuple* output_tuple = finalize_tuple(intermediate_tuple, row_batch->tuple_data_pool()); - row->set_tuple(0, output_tuple); - - if (ExecNode::eval_conjuncts(ctxs, num_ctxs, row)) { - VLOG_ROW << "output row: " << row->to_string(row_desc()); - row_batch->commit_last_row(); - ++_num_rows_returned; - - if (reached_limit()) { - break; - } - } - - _output_iterator.next(); - } - - *eos = _output_iterator.at_end() || reached_limit(); - if (*eos) { - if (_hash_tbl.get() != nullptr && _hash_table_buckets_counter != nullptr) { - COUNTER_SET(_hash_table_buckets_counter, _hash_tbl->num_buckets()); - } - } - COUNTER_SET(_rows_returned_counter, _num_rows_returned); - return Status::OK(); -} - -Status AggregationNode::close(RuntimeState* state) { - if (is_closed()) { - return Status::OK(); - } - - // Iterate through the remaining rows in the hash table and call Serialize/Finalize on - // them in order to free any memory allocated by UDAs. Finalize() requires a dst tuple - // but we don't actually need the result, so allocate a single dummy tuple to avoid - // accumulating memory. - Tuple* dummy_dst = nullptr; - if (_needs_finalize && _output_tuple_desc != nullptr) { - dummy_dst = Tuple::create(_output_tuple_desc->byte_size(), _tuple_pool.get()); - } - while (!_output_iterator.at_end()) { - Tuple* tuple = _output_iterator.get_row()->get_tuple(0); - if (_needs_finalize) { - AggFnEvaluator::finalize(_aggregate_evaluators, _agg_fn_ctxs, tuple, dummy_dst); - } else { - AggFnEvaluator::serialize(_aggregate_evaluators, _agg_fn_ctxs, tuple); - } - _output_iterator.next(); - } - - for (int i = 0; i < _aggregate_evaluators.size(); ++i) { - _aggregate_evaluators[i]->close(state); - if (!_agg_fn_ctxs.empty() && _agg_fn_ctxs[i] && _agg_fn_ctxs[i]->impl()) { - _agg_fn_ctxs[i]->impl()->close(); - } - } - - if (_tuple_pool.get() != nullptr) { - _tuple_pool->free_all(); - } - if (_hash_tbl.get() != nullptr) { - _hash_tbl->close(); - } - - Expr::close(_probe_expr_ctxs, state); - Expr::close(_build_expr_ctxs, state); - - return ExecNode::close(state); -} - -Tuple* AggregationNode::construct_intermediate_tuple() { - Tuple* agg_tuple = Tuple::create(_intermediate_tuple_desc->byte_size(), _tuple_pool.get()); - std::vector::const_iterator slot_desc = - _intermediate_tuple_desc->slots().begin(); - - // copy grouping values - for (int i = 0; i < _probe_expr_ctxs.size(); ++i, ++slot_desc) { - if (_hash_tbl->last_expr_value_null(i)) { - agg_tuple->set_null((*slot_desc)->null_indicator_offset()); - } else { - void* src = _hash_tbl->last_expr_value(i); - void* dst = agg_tuple->get_slot((*slot_desc)->tuple_offset()); - RawValue::write(src, dst, (*slot_desc)->type(), _tuple_pool.get()); - } - } - - // Initialize aggregate output. - for (int i = 0; i < _aggregate_evaluators.size(); ++i, ++slot_desc) { - while (!(*slot_desc)->is_materialized()) { - ++slot_desc; - } - - AggFnEvaluator* evaluator = _aggregate_evaluators[i]; - evaluator->init(_agg_fn_ctxs[i], agg_tuple); - - // Codegen specific path. - // To minimize branching on the UpdateAggTuple path, initialize the result value - // so that UpdateAggTuple doesn't have to check if the aggregation - // dst slot is null. - // - sum/count: 0 - // - min: max_value - // - max: min_value - // TODO: remove when we don't use the irbuilder for codegen here. - // This optimization no longer applies with AnyVal - if (!(*slot_desc)->type().is_string_type() && !(*slot_desc)->type().is_date_type()) { - ExprValue default_value; - void* default_value_ptr = nullptr; - - switch (evaluator->agg_op()) { - case TAggregationOp::MIN: - default_value_ptr = default_value.set_to_max((*slot_desc)->type()); - RawValue::write(default_value_ptr, agg_tuple, *slot_desc, nullptr); - break; - - case TAggregationOp::MAX: - default_value_ptr = default_value.set_to_min((*slot_desc)->type()); - RawValue::write(default_value_ptr, agg_tuple, *slot_desc, nullptr); - break; - - default: - break; - } - } - } - - return agg_tuple; -} - -void AggregationNode::update_tuple(Tuple* tuple, TupleRow* row) { - DCHECK(tuple != nullptr); - - AggFnEvaluator::add(_aggregate_evaluators, _agg_fn_ctxs, row, tuple); -} - -Tuple* AggregationNode::finalize_tuple(Tuple* tuple, MemPool* pool) { - DCHECK(tuple != nullptr); - - Tuple* dst = tuple; - if (_needs_finalize && _intermediate_tuple_id != _output_tuple_id) { - dst = Tuple::create(_output_tuple_desc->byte_size(), pool); - } - if (_needs_finalize) { - AggFnEvaluator::finalize( - _aggregate_evaluators, _agg_fn_ctxs, tuple, dst, - _singleton_output_tuple != nullptr && child(0)->rows_returned() == 0); - } else { - AggFnEvaluator::serialize(_aggregate_evaluators, _agg_fn_ctxs, tuple); - } - // Copy grouping values from tuple to dst. - // TODO: Codegen this. - if (dst != tuple) { - int num_grouping_slots = _probe_expr_ctxs.size(); - for (int i = 0; i < num_grouping_slots; ++i) { - SlotDescriptor* src_slot_desc = _intermediate_tuple_desc->slots()[i]; - SlotDescriptor* dst_slot_desc = _output_tuple_desc->slots()[i]; - bool src_slot_null = tuple->is_null(src_slot_desc->null_indicator_offset()); - void* src_slot = nullptr; - if (!src_slot_null) src_slot = tuple->get_slot(src_slot_desc->tuple_offset()); - RawValue::write(src_slot, dst, dst_slot_desc, nullptr); - } - } - return dst; -} - -void AggregationNode::debug_string(int indentation_level, std::stringstream* out) const { - *out << std::string(indentation_level * 2, ' '); - *out << "AggregationNode(intermediate_tuple_id=" << _intermediate_tuple_id - << " output_tuple_id=" << _output_tuple_id << " needs_finalize=" - << _needs_finalize - // << " probe_exprs=" << Expr::debug_string(_probe_exprs) - << " agg_exprs=" << AggFnEvaluator::debug_string(_aggregate_evaluators); - ExecNode::debug_string(indentation_level, out); - *out << ")"; -} - -void AggregationNode::push_down_predicate(RuntimeState* state, std::list* expr_ctxs) { - // groupby can pushdown, agg can't pushdown - // Now we doesn't pushdown for easy. - return; -} - -void AggregationNode::process_row_batch_no_grouping(RowBatch* batch, MemPool* pool) { - for (int i = 0; i < batch->num_rows(); ++i) { - update_tuple(_singleton_output_tuple, batch->get_row(i)); - } -} - -void AggregationNode::process_row_batch_with_grouping(RowBatch* batch, MemPool* pool) { - for (int i = 0; i < batch->num_rows(); ++i) { - TupleRow* row = batch->get_row(i); - Tuple* agg_tuple = nullptr; - HashTable::Iterator it = _hash_tbl->find(row); - - if (it.at_end()) { - agg_tuple = construct_intermediate_tuple(); - _hash_tbl->insert(reinterpret_cast(&agg_tuple)); - } else { - agg_tuple = it.get_row()->get_tuple(0); - } - - update_tuple(agg_tuple, row); - } -} - -} // namespace doris diff --git a/be/src/exec/aggregation_node.h b/be/src/exec/aggregation_node.h deleted file mode 100644 index a45236ecb9..0000000000 --- a/be/src/exec/aggregation_node.h +++ /dev/null @@ -1,132 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_QUERY_EXEC_AGGREGATION_NODE_H -#define DORIS_BE_SRC_QUERY_EXEC_AGGREGATION_NODE_H - -#include - -#include "exec/exec_node.h" -#include "exec/hash_table.h" -#include "runtime/descriptors.h" -#include "runtime/free_list.hpp" -#include "runtime/mem_pool.h" -#include "runtime/string_value.h" - -namespace doris { - -class AggFnEvaluator; -class RowBatch; -class RuntimeState; -struct StringValue; -class Tuple; -class TupleDescriptor; -class SlotDescriptor; - -// Node for in-memory hash aggregation. -// The node creates a hash set of aggregation output tuples, which -// contain slots for all grouping and aggregation exprs (the grouping -// slots precede the aggregation expr slots in the output tuple descriptor). -// -// For string aggregation, we need to append additional data to the tuple object -// to reduce the number of string allocations (since we cannot know the length of -// the output string beforehand). For each string slot in the output tuple, a int32 -// will be appended to the end of the normal tuple data that stores the size of buffer -// for that string slot. This also results in the correct alignment because StringValue -// slots are 8-byte aligned and form the tail end of the tuple. -class AggregationNode : public ExecNode { -public: - AggregationNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); - virtual ~AggregationNode(); - - virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr); - virtual Status prepare(RuntimeState* state); - virtual Status open(RuntimeState* state); - virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos); - virtual Status close(RuntimeState* state); - - virtual void debug_string(int indentation_level, std::stringstream* out) const; - virtual void push_down_predicate(RuntimeState* state, std::list* expr_ctxs); - -private: - std::unique_ptr _hash_tbl; - HashTable::Iterator _output_iterator; - - std::vector _aggregate_evaluators; - - /// FunctionContext for each agg fn and backing pool. - std::vector _agg_fn_ctxs; - std::unique_ptr _agg_fn_pool; - - // Exprs used to evaluate input rows - std::vector _probe_expr_ctxs; - // Exprs used to insert constructed aggregation tuple into the hash table. - // All the exprs are simply SlotRefs for the agg tuple. - std::vector _build_expr_ctxs; - - /// Tuple into which Update()/Merge()/Serialize() results are stored. - TupleId _intermediate_tuple_id; - TupleDescriptor* _intermediate_tuple_desc; - - /// Tuple into which Finalize() results are stored. Possibly the same as - /// the intermediate tuple. - TupleId _output_tuple_id; - TupleDescriptor* _output_tuple_desc; - - Tuple* _singleton_output_tuple; // result of aggregation w/o GROUP BY - std::unique_ptr _tuple_pool; - - typedef void (*ProcessRowBatchFn)(AggregationNode*, RowBatch*); - // Jitted ProcessRowBatch function pointer. Null if codegen is disabled. - ProcessRowBatchFn _process_row_batch_fn; - - // Certain aggregates require a finalize step, which is the final step of the - // aggregate after consuming all input rows. The finalize step converts the aggregate - // value into its final form. This is true if this node contains aggregate that requires - // a finalize step. - bool _needs_finalize; - - // Time spent processing the child rows - RuntimeProfile::Counter* _build_timer; - // Time spent returning the aggregated rows - RuntimeProfile::Counter* _get_results_timer; - // Num buckets in hash table - RuntimeProfile::Counter* _hash_table_buckets_counter; - // Load factor in hash table - RuntimeProfile::Counter* _hash_table_load_factor_counter; - - // Constructs a new aggregation output tuple (allocated from _tuple_pool), - // initialized to grouping values computed over '_current_row'. - // Aggregation expr slots are set to their initial values. - Tuple* construct_intermediate_tuple(); - - // Updates the aggregation output tuple 'tuple' with aggregation values - // computed over 'row'. - void update_tuple(Tuple* tuple, TupleRow* row); - - // Called when all rows have been aggregated for the aggregation tuple to compute final - // aggregate values - Tuple* finalize_tuple(Tuple* tuple, MemPool* pool); - - // Do the aggregation for all tuple rows in the batch - void process_row_batch_no_grouping(RowBatch* batch, MemPool* pool); - void process_row_batch_with_grouping(RowBatch* batch, MemPool* pool); -}; - -} // namespace doris - -#endif diff --git a/be/src/exec/base_scanner.cpp b/be/src/exec/base_scanner.cpp index f9fb5e3898..f7c0a1bb03 100644 --- a/be/src/exec/base_scanner.cpp +++ b/be/src/exec/base_scanner.cpp @@ -33,8 +33,7 @@ namespace doris { BaseScanner::BaseScanner(RuntimeState* state, RuntimeProfile* profile, const TBrokerScanRangeParams& params, - const std::vector& pre_filter_texprs, - ScannerCounter* counter) + const std::vector& pre_filter_texprs, ScannerCounter* counter) : _state(state), _params(params), _counter(counter), @@ -43,9 +42,9 @@ BaseScanner::BaseScanner(RuntimeState* state, RuntimeProfile* profile, #if BE_TEST _mem_tracker(new MemTracker()), #else - _mem_tracker( - MemTracker::CreateTracker(-1, "BaseScanner:" + std::to_string(state->load_job_id()), - state->instance_mem_tracker())), + _mem_tracker(MemTracker::CreateTracker( + -1, "BaseScanner:" + std::to_string(state->load_job_id()), + state->instance_mem_tracker())), #endif _mem_pool(_mem_tracker.get()), _dest_tuple_desc(nullptr), @@ -56,7 +55,7 @@ BaseScanner::BaseScanner(RuntimeState* state, RuntimeProfile* profile, _rows_read_counter(nullptr), _read_timer(nullptr), _materialize_timer(nullptr), - _success(nullptr), + _success(false), _scanner_eof(false) { } @@ -107,7 +106,8 @@ Status BaseScanner::init_expr_ctxes() { // preceding filter expr should be initialized by using `_row_desc`, which is the source row descriptor if (!_pre_filter_texprs.empty()) { - RETURN_IF_ERROR(Expr::create_expr_trees(_state->obj_pool(), _pre_filter_texprs, &_pre_filter_ctxs)); + RETURN_IF_ERROR( + Expr::create_expr_trees(_state->obj_pool(), _pre_filter_texprs, &_pre_filter_ctxs)); RETURN_IF_ERROR(Expr::prepare(_pre_filter_ctxs, _state, *_row_desc, _mem_tracker)); RETURN_IF_ERROR(Expr::open(_pre_filter_ctxs, _state)); } @@ -157,7 +157,7 @@ Status BaseScanner::init_expr_ctxes() { Status BaseScanner::fill_dest_tuple(Tuple* dest_tuple, MemPool* mem_pool) { // filter src tuple by preceding filter first - if (!ExecNode::eval_conjuncts(&_pre_filter_ctxs[0], _pre_filter_ctxs.size(), _src_tuple_row)) { + if (!ExecNode::eval_conjuncts(&_pre_filter_ctxs[0], _pre_filter_ctxs.size(), _src_tuple_row)) { _counter->num_rows_unselected++; _success = false; return Status::OK(); @@ -177,8 +177,11 @@ Status BaseScanner::fill_dest_tuple(Tuple* dest_tuple, MemPool* mem_pool) { // Only when the expr return value is null, we will check the error message. std::string expr_error = ctx->get_error_msg(); if (!expr_error.empty()) { - RETURN_IF_ERROR(_state->append_error_msg_to_file([&]() -> std::string { return _src_tuple_row->to_string(*(_row_desc.get())); }, - [&]() -> std::string { return expr_error; }, &_scanner_eof)); + RETURN_IF_ERROR(_state->append_error_msg_to_file( + [&]() -> std::string { + return _src_tuple_row->to_string(*(_row_desc.get())); + }, + [&]() -> std::string { return expr_error; }, &_scanner_eof)); _counter->num_rows_filtered++; // The ctx is reused, so must clear the error state and message. ctx->clear_error_msg(); @@ -189,32 +192,44 @@ Status BaseScanner::fill_dest_tuple(Tuple* dest_tuple, MemPool* mem_pool) { if (_strict_mode && (_src_slot_descs_order_by_dest[dest_index] != nullptr) && !_src_tuple->is_null( _src_slot_descs_order_by_dest[dest_index]->null_indicator_offset())) { - - RETURN_IF_ERROR(_state->append_error_msg_to_file([&]() -> std::string { return _src_tuple_row->to_string(*(_row_desc.get())); }, + RETURN_IF_ERROR(_state->append_error_msg_to_file( [&]() -> std::string { - // Type of the slot is must be Varchar in _src_tuple. - StringValue* raw_value = _src_tuple->get_string_slot( - _src_slot_descs_order_by_dest[dest_index]->tuple_offset()); - std::string raw_string; - if (raw_value != nullptr) { //is not null then get raw value - raw_string = raw_value->to_string(); - } - fmt::memory_buffer error_msg; - fmt::format_to(error_msg, "column({}) value is incorrect while strict mode is {}, src value is {}", - slot_desc->col_name(), _strict_mode, raw_string); - return error_msg.data(); - }, &_scanner_eof)); + return _src_tuple_row->to_string(*(_row_desc.get())); + }, + [&]() -> std::string { + // Type of the slot is must be Varchar in _src_tuple. + StringValue* raw_value = _src_tuple->get_string_slot( + _src_slot_descs_order_by_dest[dest_index]->tuple_offset()); + std::string raw_string; + if (raw_value != nullptr) { //is not null then get raw value + raw_string = raw_value->to_string(); + } + fmt::memory_buffer error_msg; + fmt::format_to(error_msg, + "column({}) value is incorrect while strict mode is {}, " + "src value is {}", + slot_desc->col_name(), _strict_mode, raw_string); + return error_msg.data(); + }, + &_scanner_eof)); _counter->num_rows_filtered++; _success = false; return Status::OK(); } if (!slot_desc->is_nullable()) { - RETURN_IF_ERROR(_state->append_error_msg_to_file([&]() -> std::string { return _src_tuple_row->to_string(*(_row_desc.get())); }, + RETURN_IF_ERROR(_state->append_error_msg_to_file( [&]() -> std::string { - fmt::memory_buffer error_msg; - fmt::format_to(error_msg, "column({}) values is null while columns is not nullable", slot_desc->col_name()); - return error_msg.data(); - }, &_scanner_eof)); + return _src_tuple_row->to_string(*(_row_desc.get())); + }, + [&]() -> std::string { + fmt::memory_buffer error_msg; + fmt::format_to( + error_msg, + "column({}) values is null while columns is not nullable", + slot_desc->col_name()); + return error_msg.data(); + }, + &_scanner_eof)); _counter->num_rows_filtered++; _success = false; return Status::OK(); @@ -259,5 +274,4 @@ void BaseScanner::close() { } } - } // namespace doris diff --git a/be/src/exec/broker_scan_node.h b/be/src/exec/broker_scan_node.h index 39581404bc..c4494731d8 100644 --- a/be/src/exec/broker_scan_node.h +++ b/be/src/exec/broker_scan_node.h @@ -102,8 +102,6 @@ private: std::deque> _batch_queue; int _num_running_scanners; - // Indicate if all scanners have been finished scan worker - bool _all_scanners_finished; std::atomic _scan_finished; diff --git a/be/src/exec/broker_scanner.h b/be/src/exec/broker_scanner.h index ca66cd0c44..d6deb7d64f 100644 --- a/be/src/exec/broker_scanner.h +++ b/be/src/exec/broker_scanner.h @@ -36,7 +36,7 @@ namespace doris { class Tuple; class SlotDescriptor; -class Slice; +struct Slice; class TextConverter; class FileReader; class LineReader; @@ -63,7 +63,7 @@ public: Status open() override; // Get next tuple - Status get_next(Tuple* tuple, MemPool* tuple_pool, bool* eof, bool *fill_tuple) override; + Status get_next(Tuple* tuple, MemPool* tuple_pool, bool* eof, bool* fill_tuple) override; // Close this scanner void close() override; diff --git a/be/src/exec/csv_scan_node.cpp b/be/src/exec/csv_scan_node.cpp index e005b81d90..847554065a 100644 --- a/be/src/exec/csv_scan_node.cpp +++ b/be/src/exec/csv_scan_node.cpp @@ -112,8 +112,8 @@ CsvScanNode::~CsvScanNode() { // do nothing } -Status CsvScanNode::init(const TPlanNode& tnode) { - return ExecNode::init(tnode); +Status CsvScanNode::init(const TPlanNode& tnode, RuntimeState* state) { + return ExecNode::init(tnode, state); } Status CsvScanNode::prepare(RuntimeState* state) { @@ -556,7 +556,8 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state) fields[i].length(), slot, state, &error_msg); if (flag == false) { - LOG(INFO) << error_msg.str();; + LOG(INFO) << error_msg.str(); + ; return false; } } diff --git a/be/src/exec/csv_scan_node.h b/be/src/exec/csv_scan_node.h index 36df97f9ef..07ba7b6fd9 100644 --- a/be/src/exec/csv_scan_node.h +++ b/be/src/exec/csv_scan_node.h @@ -41,7 +41,7 @@ public: CsvScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); ~CsvScanNode(); - virtual Status init(const TPlanNode& tnode); + virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr); // initialize _csv_scanner, and create _text_converter. virtual Status prepare(RuntimeState* state); diff --git a/be/src/exec/es/es_predicate.cpp b/be/src/exec/es/es_predicate.cpp index 63add75e50..3b8be666aa 100644 --- a/be/src/exec/es/es_predicate.cpp +++ b/be/src/exec/es/es_predicate.cpp @@ -167,11 +167,7 @@ std::string ExtLiteral::get_largeint_string() { } EsPredicate::EsPredicate(ExprContext* context, const TupleDescriptor* tuple_desc, ObjectPool* pool) - : _context(context), - _disjuncts_num(0), - _tuple_desc(tuple_desc), - _es_query_status(Status::OK()), - _pool(pool) {} + : _context(context), _tuple_desc(tuple_desc), _es_query_status(Status::OK()), _pool(pool) {} EsPredicate::~EsPredicate() { for (int i = 0; i < _disjuncts.size(); i++) { diff --git a/be/src/exec/es/es_predicate.h b/be/src/exec/es/es_predicate.h index 0ffbe54388..825bc5ef57 100644 --- a/be/src/exec/es/es_predicate.h +++ b/be/src/exec/es/es_predicate.h @@ -33,7 +33,7 @@ namespace doris { class Status; class ExprContext; -class ExtBinaryPredicate; +struct ExtBinaryPredicate; class EsPredicate; class ExtLiteral { @@ -157,7 +157,6 @@ private: const SlotDescriptor* get_slot_desc(const SlotRef* slotRef); ExprContext* _context; - int _disjuncts_num; const TupleDescriptor* _tuple_desc; std::vector _disjuncts; Status _es_query_status; diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index 86cd16a934..6e149e09d8 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -198,8 +198,7 @@ static Status get_float_value(const rapidjson::Value& col, PrimitiveType type, v template static Status insert_float_value(const rapidjson::Value& col, PrimitiveType type, - vectorized::IColumn* col_ptr, bool pure_doc_value, - bool nullable) { + vectorized::IColumn* col_ptr, bool pure_doc_value, bool nullable) { static_assert(sizeof(T) == 4 || sizeof(T) == 8); if (col.IsNumber() && nullable) { T value = (T)(sizeof(T) == 4 ? col.GetFloat() : col.GetDouble()); @@ -229,8 +228,7 @@ static Status insert_float_value(const rapidjson::Value& col, PrimitiveType type template static Status insert_int_value(const rapidjson::Value& col, PrimitiveType type, - vectorized::IColumn* col_ptr, bool pure_doc_value, - bool nullable) { + vectorized::IColumn* col_ptr, bool pure_doc_value, bool nullable) { if (col.IsNumber()) { T value = (T)(sizeof(T) < 8 ? col.GetInt() : col.GetInt64()); col_ptr->insert_data(const_cast(reinterpret_cast(&value)), 0); @@ -258,8 +256,7 @@ static Status insert_int_value(const rapidjson::Value& col, PrimitiveType type, return Status::OK(); } -ScrollParser::ScrollParser(bool doc_value_mode) - : _scroll_id(""), _size(0), _line_index(0), _doc_value_mode(doc_value_mode) {} +ScrollParser::ScrollParser(bool doc_value_mode) : _scroll_id(""), _size(0), _line_index(0) {} ScrollParser::~ScrollParser() {} @@ -503,7 +500,7 @@ Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc, Tuple* tuple, strings::Substitute(ERROR_INVALID_COL_DATA, "BOOLEAN")); } - const rapidjson::Value& str_col = is_nested_str? col[0]: col; + const rapidjson::Value& str_col = is_nested_str ? col[0] : col; const std::string& val = str_col.GetString(); size_t val_size = str_col.GetStringLength(); StringParser::ParseResult result; @@ -643,12 +640,110 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, return Status::RuntimeError(details); } switch (type) { - case TYPE_CHAR: - case TYPE_VARCHAR: - case TYPE_STRING: { - // sometimes elasticsearch user post some not-string value to Elasticsearch Index. - // because of reading value from _source, we can not process all json type and then just transfer the value to original string representation - // this may be a tricky, but we can workaround this issue + case TYPE_CHAR: + case TYPE_VARCHAR: + case TYPE_STRING: { + // sometimes elasticsearch user post some not-string value to Elasticsearch Index. + // because of reading value from _source, we can not process all json type and then just transfer the value to original string representation + // this may be a tricky, but we can workaround this issue + std::string val; + if (pure_doc_value) { + if (!col[0].IsString()) { + val = json_value_to_string(col[0]); + } else { + val = col[0].GetString(); + } + } else { + RETURN_ERROR_IF_COL_IS_ARRAY(col, type); + if (!col.IsString()) { + val = json_value_to_string(col); + } else { + val = col.GetString(); + } + } + size_t val_size = val.length(); + col_ptr->insert_data(const_cast(val.data()), val_size); + break; + } + + case TYPE_TINYINT: { + insert_int_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); + break; + } + + case TYPE_SMALLINT: { + insert_int_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); + break; + } + + case TYPE_INT: { + insert_int_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); + break; + } + + case TYPE_BIGINT: { + insert_int_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); + break; + } + + case TYPE_LARGEINT: { + insert_int_value<__int128>(col, type, col_ptr, pure_doc_value, + slot_desc->is_nullable()); + break; + } + + case TYPE_DOUBLE: { + insert_float_value(col, type, col_ptr, pure_doc_value, + slot_desc->is_nullable()); + break; + } + + case TYPE_FLOAT: { + insert_float_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); + break; + } + + case TYPE_BOOLEAN: { + if (col.IsBool()) { + int8_t val = col.GetBool(); + col_ptr->insert_data(const_cast(reinterpret_cast(&val)), 0); + break; + } + + if (col.IsNumber()) { + int8_t val = col.GetInt(); + col_ptr->insert_data(const_cast(reinterpret_cast(&val)), 0); + break; + } + + bool is_nested_str = false; + if (pure_doc_value && col.IsArray() && col[0].IsBool()) { + int8_t val = col[0].GetBool(); + col_ptr->insert_data(const_cast(reinterpret_cast(&val)), 0); + break; + } else if (pure_doc_value && col.IsArray() && col[0].IsString()) { + is_nested_str = true; + } else if (pure_doc_value && col.IsArray()) { + return Status::InternalError( + strings::Substitute(ERROR_INVALID_COL_DATA, "BOOLEAN")); + } + + const rapidjson::Value& str_col = is_nested_str ? col[0] : col; + + const std::string& val = str_col.GetString(); + size_t val_size = str_col.GetStringLength(); + StringParser::ParseResult result; + bool b = StringParser::string_to_bool(val.c_str(), val_size, &result); + RETURN_ERROR_IF_PARSING_FAILED(result, str_col, type); + col_ptr->insert_data(const_cast(reinterpret_cast(&b)), 0); + break; + } + case TYPE_DECIMALV2: { + DecimalV2Value data; + + if (col.IsDouble()) { + data.assign_from_double(col.GetDouble()); + } else { std::string val; if (pure_doc_value) { if (!col[0].IsString()) { @@ -664,142 +759,44 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, val = col.GetString(); } } - size_t val_size = val.length(); - col_ptr->insert_data( - const_cast(val.data()), val_size); - break; + data.parse_from_str(val.data(), val.length()); } + col_ptr->insert_data(const_cast(reinterpret_cast(&data)), 0); + break; + } - case TYPE_TINYINT: { - insert_int_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); - break; - } - - case TYPE_SMALLINT: { - insert_int_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); - break; - } - - case TYPE_INT: { - insert_int_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); - break; - } - - case TYPE_BIGINT: { - insert_int_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); - break; - } - - case TYPE_LARGEINT: { - insert_int_value<__int128>(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); - break; - } - - case TYPE_DOUBLE: { - insert_float_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); - break; - } - - case TYPE_FLOAT: { - insert_float_value(col, type, col_ptr, pure_doc_value, slot_desc->is_nullable()); - break; - } - - case TYPE_BOOLEAN: { - if (col.IsBool()) { - int8_t val = col.GetBool(); - col_ptr->insert_data(const_cast(reinterpret_cast(&val)), 0); + case TYPE_DATE: + case TYPE_DATETIME: { + // this would happend just only when `enable_docvalue_scan = false`, and field has timestamp format date from _source + if (col.IsNumber()) { + // ES process date/datetime field would use millisecond timestamp for index or docvalue + // processing date type field, if a number is encountered, Doris On ES will force it to be processed according to ms + // Doris On ES needs to be consistent with ES, so just divided by 1000 because the unit for from_unixtime is seconds + RETURN_IF_ERROR(fill_date_col_with_timestamp(col_ptr, col, type)); + } else if (col.IsArray() && pure_doc_value) { + // this would happened just only when `enable_docvalue_scan = true` + // ES add default format for all field after ES 6.4, if we not provided format for `date` field ES would impose + // a standard date-format for date field as `2020-06-16T00:00:00.000Z` + // At present, we just process this string format date. After some PR were merged into Doris, we would impose `epoch_mills` for + // date field's docvalue + if (col[0].IsString()) { + RETURN_IF_ERROR(fill_date_col_with_strval(col_ptr, col[0], type)); break; } - - if (col.IsNumber()) { - int8_t val = col.GetInt(); - col_ptr->insert_data(const_cast(reinterpret_cast(&val)), 0); - break; - } - - bool is_nested_str = false; - if (pure_doc_value && col.IsArray() && col[0].IsBool()) { - int8_t val = col[0].GetBool(); - col_ptr->insert_data(const_cast(reinterpret_cast(&val)), 0); - break; - } else if (pure_doc_value && col.IsArray() && col[0].IsString()) { - is_nested_str = true; - } else if (pure_doc_value && col.IsArray()) { - return Status::InternalError( - strings::Substitute(ERROR_INVALID_COL_DATA, "BOOLEAN")); - } - - const rapidjson::Value& str_col = is_nested_str? col[0]: col; - - const std::string& val = str_col.GetString(); - size_t val_size = str_col.GetStringLength(); - StringParser::ParseResult result; - bool b = StringParser::string_to_bool(val.c_str(), val_size, &result); - RETURN_ERROR_IF_PARSING_FAILED(result, str_col, type); - col_ptr->insert_data(const_cast(reinterpret_cast(&b)), 0); - break; - } - case TYPE_DECIMALV2: { - DecimalV2Value data; - - if (col.IsDouble()) { - data.assign_from_double(col.GetDouble()); - } else { - std::string val; - if (pure_doc_value) { - if (!col[0].IsString()) { - val = json_value_to_string(col[0]); - } else { - val = col[0].GetString(); - } - } else { - RETURN_ERROR_IF_COL_IS_ARRAY(col, type); - if (!col.IsString()) { - val = json_value_to_string(col); - } else { - val = col.GetString(); - } - } - data.parse_from_str(val.data(), val.length()); - } - col_ptr->insert_data( - const_cast(reinterpret_cast(&data)), 0); - break; - } - - case TYPE_DATE: - case TYPE_DATETIME: { - // this would happend just only when `enable_docvalue_scan = false`, and field has timestamp format date from _source - if (col.IsNumber()) { - // ES process date/datetime field would use millisecond timestamp for index or docvalue - // processing date type field, if a number is encountered, Doris On ES will force it to be processed according to ms - // Doris On ES needs to be consistent with ES, so just divided by 1000 because the unit for from_unixtime is seconds - RETURN_IF_ERROR(fill_date_col_with_timestamp(col_ptr, col, type)); - } else if (col.IsArray() && pure_doc_value) { - // this would happened just only when `enable_docvalue_scan = true` - // ES add default format for all field after ES 6.4, if we not provided format for `date` field ES would impose - // a standard date-format for date field as `2020-06-16T00:00:00.000Z` - // At present, we just process this string format date. After some PR were merged into Doris, we would impose `epoch_mills` for - // date field's docvalue - if (col[0].IsString()) { - RETURN_IF_ERROR(fill_date_col_with_strval(col_ptr, col[0], type)); - break; - } - // ES would return millisecond timestamp for date field, divided by 1000 because the unit for from_unixtime is seconds - RETURN_IF_ERROR(fill_date_col_with_timestamp(col_ptr, col, type)); - } else { - // this would happened just only when `enable_docvalue_scan = false`, and field has string format date from _source - RETURN_ERROR_IF_COL_IS_ARRAY(col, type); - RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type); - RETURN_IF_ERROR(fill_date_col_with_strval(col_ptr, col, type)); - } - break; - } - default: { - DCHECK(false); - break; + // ES would return millisecond timestamp for date field, divided by 1000 because the unit for from_unixtime is seconds + RETURN_IF_ERROR(fill_date_col_with_timestamp(col_ptr, col, type)); + } else { + // this would happened just only when `enable_docvalue_scan = false`, and field has string format date from _source + RETURN_ERROR_IF_COL_IS_ARRAY(col, type); + RETURN_ERROR_IF_COL_IS_NOT_STRING(col, type); + RETURN_IF_ERROR(fill_date_col_with_strval(col_ptr, col, type)); } + break; + } + default: { + DCHECK(false); + break; + } } } @@ -875,5 +872,4 @@ Status ScrollParser::fill_date_col_with_timestamp(vectorized::IColumn* col_ptr, return Status::OK(); } - } // namespace doris diff --git a/be/src/exec/es/es_scroll_parser.h b/be/src/exec/es/es_scroll_parser.h index 30eabaff2c..561150ac04 100644 --- a/be/src/exec/es/es_scroll_parser.h +++ b/be/src/exec/es/es_scroll_parser.h @@ -54,7 +54,7 @@ private: Status fill_date_slot_with_timestamp(void* slot, const rapidjson::Value& col, PrimitiveType type); Status fill_date_col_with_timestamp(vectorized::IColumn* col_ptr, const rapidjson::Value& col, - PrimitiveType type); + PrimitiveType type); private: std::string _scroll_id; @@ -68,6 +68,6 @@ private: // including remove some variables in the current implementation, e.g. pure_doc_value. // All above will be done in the DOE refactoring projects. // Current bug fixes minimize the scope of changes to avoid introducing other new bugs. - bool _doc_value_mode; + // bool _doc_value_mode; }; } // namespace doris diff --git a/be/src/exec/es_http_scan_node.h b/be/src/exec/es_http_scan_node.h index b90bf0e466..2564294f39 100644 --- a/be/src/exec/es_http_scan_node.h +++ b/be/src/exec/es_http_scan_node.h @@ -29,9 +29,9 @@ #include "common/status.h" #include "exec/es_http_scanner.h" -#include "vec/exec/ves_http_scanner.h" #include "exec/scan_node.h" #include "gen_cpp/PaloInternalService_types.h" +#include "vec/exec/ves_http_scanner.h" namespace doris { @@ -100,13 +100,11 @@ private: const std::vector& conjunct_ctxs, EsScanCounter* counter); virtual Status scanner_scan(std::unique_ptr scanner) { - return Status::NotSupported("vectorized scan in EsHttpScanNode is not supported!"); + return Status::NotSupported("vectorized scan in EsHttpScanNode is not supported!"); }; Status build_conjuncts_list(); - bool _all_scanners_finished; - std::vector _scanner_threads; std::vector> _scanners_status; std::map _properties; diff --git a/be/src/exec/es_scan_node.h b/be/src/exec/es_scan_node.h index 518d18e061..173cddaf68 100644 --- a/be/src/exec/es_scan_node.h +++ b/be/src/exec/es_scan_node.h @@ -47,7 +47,7 @@ public: protected: // Write debug string of this into out. - virtual void debug_string(int indentation_level, std::stringstream* out) const; + virtual void debug_string(int indentation_level, std::stringstream* out) const override; private: Status open_es(TNetworkAddress& address, TExtOpenResult& result, TExtOpenParams& params); diff --git a/be/src/exec/exchange_node.h b/be/src/exec/exchange_node.h index 6077dd4651..eb07e86bd6 100644 --- a/be/src/exec/exchange_node.h +++ b/be/src/exec/exchange_node.h @@ -43,20 +43,20 @@ public: ExchangeNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); virtual ~ExchangeNode() {} - virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr); - virtual Status prepare(RuntimeState* state); + virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr) override; + virtual Status prepare(RuntimeState* state) override; // Blocks until the first batch is available for consumption via GetNext(). - virtual Status open(RuntimeState* state); - virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos); + virtual Status open(RuntimeState* state) override; + virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) override; Status collect_query_statistics(QueryStatistics* statistics) override; - virtual Status close(RuntimeState* state); + virtual Status close(RuntimeState* state) override; // the number of senders needs to be set after the c'tor, because it's not // recorded in TPlanNode, and before calling prepare() void set_num_senders(int num_senders) { _num_senders = num_senders; } protected: - virtual void debug_string(int indentation_level, std::stringstream* out) const; + virtual void debug_string(int indentation_level, std::stringstream* out) const override; private: // Implements GetNext() for the case where _is_merging is true. Delegates the GetNext() diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp index 4501d42343..06ded1db1c 100644 --- a/be/src/exec/exec_node.cpp +++ b/be/src/exec/exec_node.cpp @@ -24,7 +24,6 @@ #include "common/object_pool.h" #include "common/status.h" -#include "exec/aggregation_node.h" #include "exec/analytic_eval_node.h" #include "exec/assert_num_rows_node.h" #include "exec/broker_scan_node.h" @@ -60,27 +59,26 @@ #include "runtime/runtime_state.h" #include "util/debug_util.h" #include "util/runtime_profile.h" - #include "vec/core/block.h" #include "vec/exec/join/vhash_join_node.h" #include "vec/exec/vaggregation_node.h" -#include "vec/exec/ves_http_scan_node.h" +#include "vec/exec/vanalytic_eval_node.h" +#include "vec/exec/vassert_num_rows_node.h" #include "vec/exec/vcross_join_node.h" +#include "vec/exec/vempty_set_node.h" +#include "vec/exec/ves_http_scan_node.h" +#include "vec/exec/vexcept_node.h" #include "vec/exec/vexchange_node.h" +#include "vec/exec/vintersect_node.h" #include "vec/exec/vmysql_scan_node.h" #include "vec/exec/vodbc_scan_node.h" #include "vec/exec/volap_scan_node.h" +#include "vec/exec/vrepeat_node.h" +#include "vec/exec/vschema_scan_node.h" +#include "vec/exec/vselect_node.h" #include "vec/exec/vsort_node.h" #include "vec/exec/vunion_node.h" -#include "vec/exec/vintersect_node.h" -#include "vec/exec/vexcept_node.h" -#include "vec/exec/vanalytic_eval_node.h" -#include "vec/exec/vassert_num_rows_node.h" -#include "vec/exec/vselect_node.h" #include "vec/exprs/vexpr.h" -#include "vec/exec/vempty_set_node.h" -#include "vec/exec/vschema_scan_node.h" -#include "vec/exec/vrepeat_node.h" namespace doris { const std::string ExecNode::ROW_THROUGHPUT_COUNTER = "RowsReturnedRate"; @@ -455,11 +453,7 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN if (state->enable_vectorized_exec()) { *node = pool->add(new vectorized::AggregationNode(pool, tnode, descs)); } else { - if (config::enable_partitioned_aggregation) { - *node = pool->add(new PartitionedAggregationNode(pool, tnode, descs)); - } else { - *node = pool->add(new AggregationNode(pool, tnode, descs)); - } + *node = pool->add(new PartitionedAggregationNode(pool, tnode, descs)); } return Status::OK(); diff --git a/be/src/exec/odbc_connector.cpp b/be/src/exec/odbc_connector.cpp index bc2d787534..e459f58754 100644 --- a/be/src/exec/odbc_connector.cpp +++ b/be/src/exec/odbc_connector.cpp @@ -59,7 +59,6 @@ ODBCConnector::ODBCConnector(const ODBCConnectorParam& param) _output_expr_ctxs(param.output_expr_ctxs), _is_open(false), _field_num(0), - _row_count(0), _env(nullptr), _dbc(nullptr), _stmt(nullptr) {} diff --git a/be/src/exec/odbc_connector.h b/be/src/exec/odbc_connector.h index c5d90162f1..690289c47b 100644 --- a/be/src/exec/odbc_connector.h +++ b/be/src/exec/odbc_connector.h @@ -112,7 +112,6 @@ private: bool _is_in_transaction; SQLSMALLINT _field_num; - uint64_t _row_count; SQLHENV _env; SQLHDBC _dbc; diff --git a/be/src/exec/parquet_reader.cpp b/be/src/exec/parquet_reader.cpp index 689fe0aaf5..ec2f31c2da 100644 --- a/be/src/exec/parquet_reader.cpp +++ b/be/src/exec/parquet_reader.cpp @@ -121,7 +121,10 @@ Status ParquetReaderWrap::init_parquet_reader(const std::vector } void ParquetReaderWrap::close() { - _parquet->Close(); + arrow::Status st = _parquet->Close(); + if (!st.ok()) { + LOG(WARNING) << "close parquet file error"; + } } Status ParquetReaderWrap::size(int64_t* size) { @@ -537,7 +540,10 @@ Status ParquetReaderWrap::read(Tuple* tuple, const std::vector& ParquetFile::ParquetFile(FileReader* file) : _file(file) {} ParquetFile::~ParquetFile() { - Close(); + arrow::Status st = Close(); + if (!st.ok()) { + LOG(WARNING) << "close parquet file error"; + } } arrow::Status ParquetFile::Close() { diff --git a/be/src/exec/parquet_scanner.h b/be/src/exec/parquet_scanner.h index de53fb196d..1e54c05d93 100644 --- a/be/src/exec/parquet_scanner.h +++ b/be/src/exec/parquet_scanner.h @@ -35,7 +35,7 @@ namespace doris { class Tuple; class SlotDescriptor; -class Slice; +struct Slice; class ParquetReaderWrap; class RuntimeState; class ExprContext; @@ -52,8 +52,7 @@ public: const TBrokerScanRangeParams& params, const std::vector& ranges, const std::vector& broker_addresses, - const std::vector& pre_filter_texprs, - ScannerCounter* counter); + const std::vector& pre_filter_texprs, ScannerCounter* counter); ~ParquetScanner(); @@ -61,7 +60,7 @@ public: virtual Status open(); // Get next tuple - virtual Status get_next(Tuple* tuple, MemPool* tuple_pool, bool* eof, bool *fill_tuple); + virtual Status get_next(Tuple* tuple, MemPool* tuple_pool, bool* eof, bool* fill_tuple); // Close this scanner virtual void close(); diff --git a/be/src/exec/parquet_writer.cpp b/be/src/exec/parquet_writer.cpp index 13cafacd87..f45060b0b3 100644 --- a/be/src/exec/parquet_writer.cpp +++ b/be/src/exec/parquet_writer.cpp @@ -42,7 +42,10 @@ ParquetOutputStream::ParquetOutputStream(FileWriter* file_writer) } ParquetOutputStream::~ParquetOutputStream() { - Close(); + arrow::Status st = Close(); + if (!st.ok()) { + LOG(WARNING) << "close parquet file error"; + } } arrow::Status ParquetOutputStream::Write(const void* data, int64_t nbytes) { @@ -463,7 +466,10 @@ void ParquetWriterWrapper::close() { _rg_writer = nullptr; } _writer->Close(); - _outstream->Close(); + arrow::Status st = _outstream->Close(); + if (!st.ok()) { + LOG(WARNING) << "close parquet file error"; + } } catch (const std::exception& e) { _rg_writer = nullptr; LOG(WARNING) << "Parquet writer close error: " << e.what(); diff --git a/be/src/exec/partitioned_aggregation_node.cc b/be/src/exec/partitioned_aggregation_node.cc index 98651998b6..a30c2f5a47 100644 --- a/be/src/exec/partitioned_aggregation_node.cc +++ b/be/src/exec/partitioned_aggregation_node.cc @@ -125,7 +125,6 @@ PartitionedAggregationNode::PartitionedAggregationNode(ObjectPool* pool, const T num_passthrough_rows_(nullptr), preagg_estimated_reduction_(nullptr), preagg_streaming_ht_min_reduction_(nullptr), - // estimated_input_cardinality_(tnode.agg_node.estimated_input_cardinality), singleton_output_tuple_(nullptr), singleton_output_tuple_returned_(true), partition_eos_(false), @@ -623,7 +622,6 @@ bool PartitionedAggregationNode::ShouldExpandPreaggHashTables() const { const int64_t input_rows = _children[0]->rows_returned(); const int64_t aggregated_input_rows = input_rows - _num_rows_returned; // TODO chenhao - // const int64_t expected_input_rows = estimated_input_cardinality_ - num_rows_returned_; double current_reduction = static_cast(aggregated_input_rows) / ht_rows; // TODO: workaround for IMPALA-2490: subplan node rows_returned counter may be diff --git a/be/src/exec/partitioned_aggregation_node.h b/be/src/exec/partitioned_aggregation_node.h index d627162418..e77a1366fe 100644 --- a/be/src/exec/partitioned_aggregation_node.h +++ b/be/src/exec/partitioned_aggregation_node.h @@ -323,9 +323,6 @@ private: /// Expose the minimum reduction factor to continue growing the hash tables. RuntimeProfile::Counter* preagg_streaming_ht_min_reduction_; - /// The estimated number of input rows from the planner. - int64_t estimated_input_cardinality_; - ///////////////////////////////////////// /// BEGIN: Members that must be Reset() diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp index bbc409e375..1319a5d21d 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp @@ -30,7 +30,7 @@ SchemaDummyScanner::SchemaDummyScanner() SchemaDummyScanner::~SchemaDummyScanner() {} -Status SchemaDummyScanner::start() { +Status SchemaDummyScanner::start(RuntimeState* state) { return Status::OK(); } diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.h b/be/src/exec/schema_scanner/schema_dummy_scanner.h index b5ba514051..63fafe18b5 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.h +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.h @@ -27,7 +27,7 @@ class SchemaDummyScanner : public SchemaScanner { public: SchemaDummyScanner(); virtual ~SchemaDummyScanner(); - virtual Status start(); + virtual Status start(RuntimeState* state = nullptr); virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); private: diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.h b/be/src/exec/schema_scanner/schema_variables_scanner.h index 2abe8385c4..e957a4eb3e 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.h +++ b/be/src/exec/schema_scanner/schema_variables_scanner.h @@ -42,7 +42,6 @@ private: Status fill_one_row(Tuple* tuple, MemPool* pool); - int _index; static SchemaScanner::ColumnDesc _s_vars_columns[]; TShowVariableResult _var_result; diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp index 6aad7df770..998936ba0b 100644 --- a/be/src/exec/tablet_sink.cpp +++ b/be/src/exec/tablet_sink.cpp @@ -43,16 +43,14 @@ namespace doris { namespace stream_load { -NodeChannel::NodeChannel(OlapTableSink* parent, IndexChannel* index_channel, int64_t node_id, - int32_t schema_hash) - : _parent(parent), _index_channel(index_channel), _node_id(node_id), _schema_hash(schema_hash) { - +NodeChannel::NodeChannel(OlapTableSink* parent, IndexChannel* index_channel, int64_t node_id) + : _parent(parent), _index_channel(index_channel), _node_id(node_id) { if (_parent->_transfer_data_by_brpc_attachment) { _tuple_data_buffer_ptr = &_tuple_data_buffer; } } -NodeChannel::~NodeChannel() { +NodeChannel::~NodeChannel() noexcept { if (_open_closure != nullptr) { if (_open_closure->unref()) { delete _open_closure; @@ -459,7 +457,8 @@ void NodeChannel::try_send_batch() { if (row_batch->num_rows() > 0) { SCOPED_ATOMIC_TIMER(&_serialize_batch_ns); size_t uncompressed_bytes = 0, compressed_bytes = 0; - Status st = row_batch->serialize(request.mutable_row_batch(), &uncompressed_bytes, &compressed_bytes, _tuple_data_buffer_ptr); + Status st = row_batch->serialize(request.mutable_row_batch(), &uncompressed_bytes, + &compressed_bytes, _tuple_data_buffer_ptr); if (!st.ok()) { cancel(fmt::format("{}, err: {}", channel_info(), st.get_error_msg())); return; @@ -499,8 +498,8 @@ void NodeChannel::try_send_batch() { if (_parent->_transfer_data_by_brpc_attachment && request.has_row_batch()) { request_row_batch_transfer_attachment>( - &request, _tuple_data_buffer, _add_batch_closure); + ReusableClosure>( + &request, _tuple_data_buffer, _add_batch_closure); } _add_batch_closure->set_in_flight(); _stub->tablet_writer_add_batch(&_add_batch_closure->cntl, &request, &_add_batch_closure->result, @@ -550,7 +549,7 @@ Status IndexChannel::init(RuntimeState* state, const std::vector_pool. // Because the deconstruction of NodeChannel may take a long time to wait rpc finish. // but the ObjectPool will hold a spin lock to delete objects. - channel = std::make_shared(_parent, this, node_id, _schema_hash); + channel = std::make_shared(_parent, this, node_id); _node_channels.emplace(node_id, channel); } else { channel = it->second; @@ -797,7 +796,7 @@ Status OlapTableSink::prepare(RuntimeState* state) { tablets.emplace_back(std::move(tablet_with_partition)); } } - auto channel = _pool->add(new IndexChannel(this, index->index_id, index->schema_hash)); + auto channel = _pool->add(new IndexChannel(this, index->index_id)); RETURN_IF_ERROR(channel->init(state, tablets)); _channels.emplace_back(channel); } diff --git a/be/src/exec/tablet_sink.h b/be/src/exec/tablet_sink.h index 916b01208e..8bd1e9660b 100644 --- a/be/src/exec/tablet_sink.h +++ b/be/src/exec/tablet_sink.h @@ -151,7 +151,7 @@ private: class IndexChannel; class NodeChannel { public: - NodeChannel(OlapTableSink* parent, IndexChannel* index_channel, int64_t node_id, int32_t schema_hash); + NodeChannel(OlapTableSink* parent, IndexChannel* index_channel, int64_t node_id); ~NodeChannel() noexcept; // called before open, used to add tablet located in this backend @@ -221,7 +221,6 @@ private: OlapTableSink* _parent = nullptr; IndexChannel* _index_channel = nullptr; int64_t _node_id = -1; - int32_t _schema_hash = 0; std::string _load_info; std::string _name; @@ -286,8 +285,7 @@ private: class IndexChannel { public: - IndexChannel(OlapTableSink* parent, int64_t index_id, int32_t schema_hash) - : _parent(parent), _index_id(index_id), _schema_hash(schema_hash) {} + IndexChannel(OlapTableSink* parent, int64_t index_id) : _parent(parent), _index_id(index_id) {} ~IndexChannel(); Status init(RuntimeState* state, const std::vector& tablets); @@ -315,7 +313,6 @@ private: OlapTableSink* _parent; int64_t _index_id; - int32_t _schema_hash; // from backend channel to tablet_id // ATTN: must be placed before `_node_channels` and `_channels_by_tablet`. @@ -372,8 +369,8 @@ private: // return number of invalid/filtered rows. // invalid row number is set in Bitmap // set stop_processing is we want to stop the whole process now. - Status _validate_data(RuntimeState* state, RowBatch* batch, Bitmap* filter_bitmap, int* filtered_rows, - bool* stop_processing); + Status _validate_data(RuntimeState* state, RowBatch* batch, Bitmap* filter_bitmap, + int* filtered_rows, bool* stop_processing); // the consumer func of sending pending batches in every NodeChannel. // use polling & NodeChannel::try_send_and_fetch_status() to achieve nonblocking sending. diff --git a/be/src/exprs/agg_fn_evaluator.cpp b/be/src/exprs/agg_fn_evaluator.cpp index d83920c43b..2b0679cf4c 100644 --- a/be/src/exprs/agg_fn_evaluator.cpp +++ b/be/src/exprs/agg_fn_evaluator.cpp @@ -14,16 +14,21 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Warray-bounds" +#elif defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Warray-bounds" #pragma GCC diagnostic ignored "-Wstringop-overflow=" +#endif #include "exprs/agg_fn_evaluator.h" #include #include "common/logging.h" -#include "exec/aggregation_node.h" #include "exprs/aggregate_functions.h" #include "exprs/anyval_util.h" #include "runtime/datetime_value.h" @@ -951,4 +956,8 @@ std::string AggFnEvaluator::debug_string() const { } // namespace doris +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop +#endif diff --git a/be/src/exprs/aggregate_functions.cpp b/be/src/exprs/aggregate_functions.cpp index 93166cf463..f8efe675b1 100644 --- a/be/src/exprs/aggregate_functions.cpp +++ b/be/src/exprs/aggregate_functions.cpp @@ -2201,19 +2201,6 @@ void AggregateFunctions::first_val_update(FunctionContext* ctx, const T& src, T* *dst = src; } -template <> -void AggregateFunctions::first_val_update(FunctionContext* ctx, const IntVal& src, IntVal* dst) { - // The first call to FirstValUpdate sets the value of dst. - if (ctx->impl()->num_updates() > 1) { - return; - } - // num_updates is incremented before calling Update(), so it should never be 0. - // Remove() should never be called for FIRST_VALUE. - DCHECK_GT(ctx->impl()->num_updates(), 0); - DCHECK_EQ(ctx->impl()->num_removes(), 0); - *dst = src; -} - template <> void AggregateFunctions::first_val_update(FunctionContext* ctx, const StringVal& src, StringVal* dst) { @@ -2272,40 +2259,21 @@ void AggregateFunctions::offset_fn_init(FunctionContext* ctx, StringVal* dst) { } } -/* -template <> -void AggregateFunctions::offset_fn_init(FunctionContext* ctx, IntVal* dst) { - DCHECK_EQ(ctx->get_num_args(), 3); - DCHECK(ctx->is_arg_constant(1)); - DCHECK(ctx->is_arg_constant(2)); - - // DCHECK_EQ(*ctx->GetArgType(0), *ctx->GetArgType(2)); - *dst = *static_cast(ctx->get_constant_arg(2)); -} -*/ template void AggregateFunctions::offset_fn_update(FunctionContext* ctx, const T& src, const BigIntVal&, const T& default_value, T* dst) { *dst = src; } -template <> -void AggregateFunctions::offset_fn_update(FunctionContext* ctx, const IntVal& src, const BigIntVal&, - const IntVal& default_value, IntVal* dst) { - *dst = src; -} - // Stamp out the templates for the types we need. template void AggregateFunctions::init_zero_null(FunctionContext*, BigIntVal* dst); template void AggregateFunctions::init_zero_null(FunctionContext*, LargeIntVal* dst); template void AggregateFunctions::init_zero_null(FunctionContext*, DoubleVal* dst); -template void AggregateFunctions::init_zero_null(FunctionContext*, DecimalV2Val* dst); // Stamp out the templates for the types we need. template void AggregateFunctions::init_zero(FunctionContext*, BigIntVal* dst); template void AggregateFunctions::init_zero(FunctionContext*, LargeIntVal* dst); template void AggregateFunctions::init_zero(FunctionContext*, DoubleVal* dst); -template void AggregateFunctions::init_zero(FunctionContext*, DecimalV2Val* dst); template void AggregateFunctions::init_zero_not_null(FunctionContext*, BigIntVal* dst); @@ -2329,9 +2297,6 @@ template void AggregateFunctions::sum_remove(FunctionContex template void AggregateFunctions::sum_remove(FunctionContext*, const DoubleVal& src, DoubleVal* dst); -template void AggregateFunctions::sum_remove(FunctionContext*, - const DecimalV2Val& src, - DecimalV2Val* dst); template void AggregateFunctions::sum_remove(FunctionContext*, const LargeIntVal& src, LargeIntVal* dst); @@ -2379,8 +2344,9 @@ template void AggregateFunctions::sum(FunctionContext*, const BigIntVal* dst); template void AggregateFunctions::sum(FunctionContext*, const BigIntVal& src, BigIntVal* dst); -template void AggregateFunctions::sum(FunctionContext*, const LargeIntVal& src, - LargeIntVal* dst); +template void AggregateFunctions::sum(FunctionContext*, + const LargeIntVal& src, + LargeIntVal* dst); template void AggregateFunctions::sum(FunctionContext*, const FloatVal& src, DoubleVal* dst); template void AggregateFunctions::sum(FunctionContext*, const DoubleVal& src, @@ -2417,8 +2383,6 @@ template void AggregateFunctions::min(FunctionContext*, const FloatVal FloatVal* dst); template void AggregateFunctions::min(FunctionContext*, const DoubleVal& src, DoubleVal* dst); -template void AggregateFunctions::min(FunctionContext*, const StringVal& src, - StringVal* dst); template void AggregateFunctions::avg_remove(doris_udf::FunctionContext*, doris_udf::BooleanVal const&, @@ -2467,8 +2431,6 @@ template void AggregateFunctions::max(FunctionContext*, const FloatVal FloatVal* dst); template void AggregateFunctions::max(FunctionContext*, const DoubleVal& src, DoubleVal* dst); -template void AggregateFunctions::max(FunctionContext*, const StringVal& src, - StringVal* dst); template void AggregateFunctions::pc_update(FunctionContext*, const BooleanVal&, StringVal*); template void AggregateFunctions::pc_update(FunctionContext*, const TinyIntVal&, StringVal*); @@ -2619,8 +2581,7 @@ template void AggregateFunctions::first_val_update(FunctionContext*, c FloatVal* dst); template void AggregateFunctions::first_val_update(FunctionContext*, const DoubleVal& src, DoubleVal* dst); -template void AggregateFunctions::first_val_update(FunctionContext*, - const StringVal& src, StringVal* dst); + template void AggregateFunctions::first_val_update(FunctionContext*, const DateTimeVal& src, DateTimeVal* dst); @@ -2685,8 +2646,6 @@ template void AggregateFunctions::last_val_update(FunctionContext*, co FloatVal* dst); template void AggregateFunctions::last_val_update(FunctionContext*, const DoubleVal& src, DoubleVal* dst); -template void AggregateFunctions::last_val_update(FunctionContext*, const StringVal& src, - StringVal* dst); template void AggregateFunctions::last_val_update(FunctionContext*, const DateTimeVal& src, DateTimeVal* dst); @@ -2711,8 +2670,6 @@ template void AggregateFunctions::last_val_remove(FunctionContext*, co FloatVal* dst); template void AggregateFunctions::last_val_remove(FunctionContext*, const DoubleVal& src, DoubleVal* dst); -template void AggregateFunctions::last_val_remove(FunctionContext*, const StringVal& src, - StringVal* dst); template void AggregateFunctions::last_val_remove(FunctionContext*, const DateTimeVal& src, DateTimeVal* dst); diff --git a/be/src/exprs/arithmetic_expr.h b/be/src/exprs/arithmetic_expr.h index 2c6c790f2e..5fd56a9cd7 100644 --- a/be/src/exprs/arithmetic_expr.h +++ b/be/src/exprs/arithmetic_expr.h @@ -49,13 +49,13 @@ public: AddExpr(const TExprNode& node) : ArithmeticExpr(node) {} virtual ~AddExpr() {} virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new AddExpr(*this)); } - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); - virtual FloatVal get_float_val(ExprContext* context, TupleRow*); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow*); + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow*) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow*) override; }; class SubExpr : public ArithmeticExpr { @@ -63,13 +63,13 @@ public: SubExpr(const TExprNode& node) : ArithmeticExpr(node) {} virtual ~SubExpr() {} virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new SubExpr(*this)); } - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); - virtual FloatVal get_float_val(ExprContext* context, TupleRow*); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow*); + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow*) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow*) override; }; class MulExpr : public ArithmeticExpr { @@ -77,13 +77,13 @@ public: MulExpr(const TExprNode& node) : ArithmeticExpr(node) {} virtual ~MulExpr() {} virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new MulExpr(*this)); } - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); - virtual FloatVal get_float_val(ExprContext* context, TupleRow*); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow*); + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow*) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow*) override; }; class DivExpr : public ArithmeticExpr { @@ -91,13 +91,13 @@ public: DivExpr(const TExprNode& node) : ArithmeticExpr(node) {} virtual ~DivExpr() {} virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new DivExpr(*this)); } - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); - virtual FloatVal get_float_val(ExprContext* context, TupleRow*); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow*); + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow*) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow*) override; }; class ModExpr : public ArithmeticExpr { @@ -105,13 +105,13 @@ public: ModExpr(const TExprNode& node) : ArithmeticExpr(node) {} virtual ~ModExpr() {} virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new ModExpr(*this)); } - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); - virtual FloatVal get_float_val(ExprContext* context, TupleRow*); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow*); + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow*) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow*) override; }; class BitAndExpr : public ArithmeticExpr { @@ -121,11 +121,11 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new BitAndExpr(*this)); } - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; }; class BitOrExpr : public ArithmeticExpr { @@ -133,11 +133,11 @@ public: BitOrExpr(const TExprNode& node) : ArithmeticExpr(node) {} virtual ~BitOrExpr() {} virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new BitOrExpr(*this)); } - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; }; class BitXorExpr : public ArithmeticExpr { @@ -147,11 +147,11 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new BitXorExpr(*this)); } - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; }; class BitNotExpr : public ArithmeticExpr { @@ -161,11 +161,11 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new BitNotExpr(*this)); } - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; }; } // namespace doris diff --git a/be/src/exprs/binary_predicate.h b/be/src/exprs/binary_predicate.h index bbf68a30d6..eedee17b2b 100644 --- a/be/src/exprs/binary_predicate.h +++ b/be/src/exprs/binary_predicate.h @@ -40,16 +40,16 @@ protected: virtual std::string debug_string() const; }; -#define BIN_PRED_CLASS_DEFINE(CLASS) \ - class CLASS : public BinaryPredicate { \ - public: \ - CLASS(const TExprNode& node) : BinaryPredicate(node) {} \ - virtual ~CLASS() {} \ - virtual Expr* clone(ObjectPool* pool) const override { \ - return pool->add(new CLASS(*this)); \ - } \ - \ - virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row); \ +#define BIN_PRED_CLASS_DEFINE(CLASS) \ + class CLASS : public BinaryPredicate { \ + public: \ + CLASS(const TExprNode& node) : BinaryPredicate(node) {} \ + virtual ~CLASS() {} \ + virtual Expr* clone(ObjectPool* pool) const override { \ + return pool->add(new CLASS(*this)); \ + } \ + \ + virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row) override; \ }; #define BIN_PRED_CLASSES_DEFINE(TYPE) \ @@ -72,16 +72,16 @@ BIN_PRED_CLASSES_DEFINE(StringVal) BIN_PRED_CLASSES_DEFINE(DateTimeVal) BIN_PRED_CLASSES_DEFINE(DecimalV2Val) -#define BIN_PRED_FOR_NULL_CLASS_DEFINE(CLASS) \ - class CLASS : public BinaryPredicate { \ - public: \ - CLASS(const TExprNode& node) : BinaryPredicate(node) {} \ - virtual ~CLASS() {} \ - virtual Expr* clone(ObjectPool* pool) const override { \ - return pool->add(new CLASS(*this)); \ - } \ - \ - virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row); \ +#define BIN_PRED_FOR_NULL_CLASS_DEFINE(CLASS) \ + class CLASS : public BinaryPredicate { \ + public: \ + CLASS(const TExprNode& node) : BinaryPredicate(node) {} \ + virtual ~CLASS() {} \ + virtual Expr* clone(ObjectPool* pool) const override { \ + return pool->add(new CLASS(*this)); \ + } \ + \ + virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row) override; \ }; #define BIN_PRED_FOR_NULL_CLASSES_DEFINE(TYPE) BIN_PRED_FOR_NULL_CLASS_DEFINE(EqForNull##TYPE##Pred) diff --git a/be/src/exprs/bloomfilter_predicate.h b/be/src/exprs/bloomfilter_predicate.h index a6b7f83636..2af9e32211 100644 --- a/be/src/exprs/bloomfilter_predicate.h +++ b/be/src/exprs/bloomfilter_predicate.h @@ -302,7 +302,7 @@ public: ~BloomFilterFunc() = default; - void insert(const void* data) { + void insert(const void* data) override { DCHECK(this->_bloom_filter != nullptr); dummy.insert(*this->_bloom_filter, data); } @@ -329,6 +329,7 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new BloomFilterPredicate(*this)); } + using Predicate::prepare; Status prepare(RuntimeState* state, IBloomFilterFuncBase* bloomfilterfunc); std::shared_ptr get_bloom_filter_func() { return _filter; } diff --git a/be/src/exprs/case_expr.h b/be/src/exprs/case_expr.h index dd10ceafce..8bcbc48269 100644 --- a/be/src/exprs/case_expr.h +++ b/be/src/exprs/case_expr.h @@ -31,16 +31,16 @@ class CaseExpr : public Expr { public: virtual ~CaseExpr(); virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new CaseExpr(*this)); } - virtual BooleanVal get_boolean_val(ExprContext* ctx, TupleRow* row); - virtual TinyIntVal get_tiny_int_val(ExprContext* ctx, TupleRow* row); - virtual SmallIntVal get_small_int_val(ExprContext* ctx, TupleRow* row); - virtual IntVal get_int_val(ExprContext* ctx, TupleRow* row); - virtual BigIntVal get_big_int_val(ExprContext* ctx, TupleRow* row); - virtual FloatVal get_float_val(ExprContext* ctx, TupleRow* row); - virtual DoubleVal get_double_val(ExprContext* ctx, TupleRow* row); - virtual StringVal get_string_val(ExprContext* ctx, TupleRow* row); - virtual DateTimeVal get_datetime_val(ExprContext* ctx, TupleRow* row); - virtual DecimalV2Val get_decimalv2_val(ExprContext* ctx, TupleRow* row); + virtual BooleanVal get_boolean_val(ExprContext* ctx, TupleRow* row) override; + virtual TinyIntVal get_tiny_int_val(ExprContext* ctx, TupleRow* row) override; + virtual SmallIntVal get_small_int_val(ExprContext* ctx, TupleRow* row) override; + virtual IntVal get_int_val(ExprContext* ctx, TupleRow* row) override; + virtual BigIntVal get_big_int_val(ExprContext* ctx, TupleRow* row) override; + virtual FloatVal get_float_val(ExprContext* ctx, TupleRow* row) override; + virtual DoubleVal get_double_val(ExprContext* ctx, TupleRow* row) override; + virtual StringVal get_string_val(ExprContext* ctx, TupleRow* row) override; + virtual DateTimeVal get_datetime_val(ExprContext* ctx, TupleRow* row) override; + virtual DecimalV2Val get_decimalv2_val(ExprContext* ctx, TupleRow* row) override; protected: friend class Expr; @@ -51,13 +51,13 @@ protected: CaseExpr(const TExprNode& node); virtual Status prepare(RuntimeState* state, const RowDescriptor& row_desc, - ExprContext* context); + ExprContext* context) override; virtual Status open(RuntimeState* state, ExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual void close(RuntimeState* state, ExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; - virtual std::string debug_string() const; + virtual std::string debug_string() const override; bool has_case_expr() { return _has_case_expr; } diff --git a/be/src/exprs/cast_expr.h b/be/src/exprs/cast_expr.h index 6c9df4a61e..28b3602627 100644 --- a/be/src/exprs/cast_expr.h +++ b/be/src/exprs/cast_expr.h @@ -30,22 +30,22 @@ public: static Expr* from_thrift(const TExprNode& node); }; -#define CAST_EXPR_DEFINE(CLASS) \ - class CLASS : public CastExpr { \ - public: \ - CLASS(const TExprNode& node) : CastExpr(node) {} \ - virtual ~CLASS() {} \ - virtual Expr* clone(ObjectPool* pool) const override { \ - return pool->add(new CLASS(*this)); \ - } \ - virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow*); \ - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); \ - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); \ - virtual IntVal get_int_val(ExprContext* context, TupleRow*); \ - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); \ - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); \ - virtual FloatVal get_float_val(ExprContext* context, TupleRow*); \ - virtual DoubleVal get_double_val(ExprContext* context, TupleRow*); \ +#define CAST_EXPR_DEFINE(CLASS) \ + class CLASS : public CastExpr { \ + public: \ + CLASS(const TExprNode& node) : CastExpr(node) {} \ + virtual ~CLASS() {} \ + virtual Expr* clone(ObjectPool* pool) const override { \ + return pool->add(new CLASS(*this)); \ + } \ + virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow*) override; \ + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; \ + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; \ + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; \ + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; \ + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; \ + virtual FloatVal get_float_val(ExprContext* context, TupleRow*) override; \ + virtual DoubleVal get_double_val(ExprContext* context, TupleRow*) override; \ }; CAST_EXPR_DEFINE(CastBooleanExpr); diff --git a/be/src/exprs/compound_predicate.h b/be/src/exprs/compound_predicate.h index 25d0858770..be8fa52878 100644 --- a/be/src/exprs/compound_predicate.h +++ b/be/src/exprs/compound_predicate.h @@ -51,13 +51,13 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new AndPredicate(*this)); } - virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*); + virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*) override; protected: friend class Expr; AndPredicate(const TExprNode& node) : CompoundPredicate(node) {} - virtual std::string debug_string() const { + virtual std::string debug_string() const override { std::stringstream out; out << "AndPredicate(" << Expr::debug_string() << ")"; return out.str(); @@ -73,13 +73,13 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new OrPredicate(*this)); } - virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*); + virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*) override; protected: friend class Expr; OrPredicate(const TExprNode& node) : CompoundPredicate(node) {} - virtual std::string debug_string() const { + virtual std::string debug_string() const override { std::stringstream out; out << "OrPredicate(" << Expr::debug_string() << ")"; return out.str(); @@ -95,13 +95,13 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new NotPredicate(*this)); } - virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*); + virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*) override; protected: friend class Expr; NotPredicate(const TExprNode& node) : CompoundPredicate(node) {} - virtual std::string debug_string() const { + virtual std::string debug_string() const override { std::stringstream out; out << "NotPredicate(" << Expr::debug_string() << ")"; return out.str(); diff --git a/be/src/exprs/conditional_functions.h b/be/src/exprs/conditional_functions.h index 42312a3ae0..cbf60d941b 100644 --- a/be/src/exprs/conditional_functions.h +++ b/be/src/exprs/conditional_functions.h @@ -41,19 +41,19 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new IfNullExpr(*this)); } - virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row); - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow* row); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow* row); - virtual IntVal get_int_val(ExprContext* context, TupleRow* row); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow* row); - virtual FloatVal get_float_val(ExprContext* context, TupleRow* row); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow* row); - virtual StringVal get_string_val(ExprContext* context, TupleRow* row); - virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow* row); - virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow* row); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow* row); + virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row) override; + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow* row) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow* row) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow* row) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow* row) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow* row) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow* row) override; + virtual StringVal get_string_val(ExprContext* context, TupleRow* row) override; + virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow* row) override; + virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow* row) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow* row) override; - virtual std::string debug_string() const { return Expr::debug_string("IfNullExpr"); } + virtual std::string debug_string() const override { return Expr::debug_string("IfNullExpr"); } protected: friend class Expr; @@ -66,19 +66,19 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new NullIfExpr(*this)); } - virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row); - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow* row); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow* row); - virtual IntVal get_int_val(ExprContext* context, TupleRow* row); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow* row); - virtual FloatVal get_float_val(ExprContext* context, TupleRow* row); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow* row); - virtual StringVal get_string_val(ExprContext* context, TupleRow* row); - virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow* row); - virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow* row); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow* row); + virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row) override; + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow* row) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow* row) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow* row) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow* row) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow* row) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow* row) override; + virtual StringVal get_string_val(ExprContext* context, TupleRow* row) override; + virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow* row) override; + virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow* row) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow* row) override; - virtual std::string debug_string() const { return Expr::debug_string("NullIfExpr"); } + virtual std::string debug_string() const override { return Expr::debug_string("NullIfExpr"); } protected: friend class Expr; @@ -89,19 +89,19 @@ class IfExpr : public Expr { public: virtual ~IfExpr(); virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new IfExpr(*this)); } - virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row); - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow* row); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow* row); - virtual IntVal get_int_val(ExprContext* context, TupleRow* row); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow* row); - virtual FloatVal get_float_val(ExprContext* context, TupleRow* row); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow* row); - virtual StringVal get_string_val(ExprContext* context, TupleRow* row); - virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow* row); - virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow* row); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow* row); + virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row) override; + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow* row) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow* row) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow* row) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow* row) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow* row) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow* row) override; + virtual StringVal get_string_val(ExprContext* context, TupleRow* row) override; + virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow* row) override; + virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow* row) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow* row) override; - virtual std::string debug_string() const { return Expr::debug_string("IfExpr"); } + virtual std::string debug_string() const override { return Expr::debug_string("IfExpr"); } protected: friend class Expr; @@ -115,19 +115,19 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new CoalesceExpr(*this)); } - virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row); - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow* row); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow* row); - virtual IntVal get_int_val(ExprContext* context, TupleRow* row); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow* row); - virtual FloatVal get_float_val(ExprContext* context, TupleRow* row); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow* row); - virtual StringVal get_string_val(ExprContext* context, TupleRow* row); - virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow* row); - virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow* row); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow* row); + virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row) override; + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow* row) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow* row) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow* row) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow* row) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow* row) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow* row) override; + virtual StringVal get_string_val(ExprContext* context, TupleRow* row) override; + virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow* row) override; + virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow* row) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow* row) override; - virtual std::string debug_string() const { return Expr::debug_string("CoalesceExpr"); } + virtual std::string debug_string() const override { return Expr::debug_string("CoalesceExpr"); } protected: friend class Expr; diff --git a/be/src/exprs/expr.h b/be/src/exprs/expr.h index 795dca6bc0..fdb47553e0 100644 --- a/be/src/exprs/expr.h +++ b/be/src/exprs/expr.h @@ -55,7 +55,7 @@ class TupleIsNullPredicate; class VectorizedRowBatch; class Literal; class MemTracker; -class UserFunctionCacheEntry; +struct UserFunctionCacheEntry; // This is the superclass of all expr evaluation nodes. class Expr { diff --git a/be/src/exprs/hash_functions.h b/be/src/exprs/hash_functions.h index 4b2318e9dc..9fcfb9a7aa 100644 --- a/be/src/exprs/hash_functions.h +++ b/be/src/exprs/hash_functions.h @@ -19,8 +19,8 @@ namespace doris_udf { class FunctionContext; -class IntVal; -class StringVal; +struct IntVal; +struct StringVal; } // namespace doris_udf namespace doris { diff --git a/be/src/exprs/hybrid_set.h b/be/src/exprs/hybrid_set.h index fef7146fdd..e47e53e65e 100644 --- a/be/src/exprs/hybrid_set.h +++ b/be/src/exprs/hybrid_set.h @@ -99,9 +99,9 @@ public: typename phmap::flat_hash_set<_iT>::iterator end) : _begin(begin), _end(end) {} ~Iterator() override = default; - virtual bool has_next() const { return !(_begin == _end); } - virtual const void* get_value() { return _begin.operator->(); } - virtual void next() { ++_begin; } + virtual bool has_next() const override { return !(_begin == _end); } + virtual const void* get_value() override { return _begin.operator->(); } + virtual void next() override { ++_begin; } private: typename phmap::flat_hash_set<_iT>::iterator _begin; @@ -162,13 +162,13 @@ public: phmap::flat_hash_set::iterator end) : _begin(begin), _end(end) {} ~Iterator() override = default; - virtual bool has_next() const { return !(_begin == _end); } - virtual const void* get_value() { + virtual bool has_next() const override { return !(_begin == _end); } + virtual const void* get_value() override { _value.ptr = const_cast(_begin->data()); _value.len = _begin->length(); return &_value; } - virtual void next() { ++_begin; } + virtual void next() override { ++_begin; } private: typename phmap::flat_hash_set::iterator _begin; diff --git a/be/src/exprs/in_predicate.h b/be/src/exprs/in_predicate.h index d7d21e2662..19b5a5f8e0 100644 --- a/be/src/exprs/in_predicate.h +++ b/be/src/exprs/in_predicate.h @@ -40,11 +40,11 @@ public: Status prepare(RuntimeState* state, HybridSetBase* hset); Status open(RuntimeState* state, ExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual Status prepare(RuntimeState* state, const RowDescriptor& row_desc, - ExprContext* context); + ExprContext* context) override; - virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row); + virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow* row) override; // this function add one item in hashset, not add to children. // if add to children, when List is long, copy is a expensive op. @@ -62,7 +62,7 @@ protected: InPredicate(const TExprNode& node); // virtual Status prepare(RuntimeState* state, const RowDescriptor& desc); - virtual std::string debug_string() const; + virtual std::string debug_string() const override; private: const bool _is_not_in; diff --git a/be/src/exprs/info_func.h b/be/src/exprs/info_func.h index c8082c3a1b..dd9dbf663e 100644 --- a/be/src/exprs/info_func.h +++ b/be/src/exprs/info_func.h @@ -38,10 +38,10 @@ protected: InfoFunc(const TExprNode& node); - virtual StringVal get_string_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); + virtual StringVal get_string_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; - virtual std::string debug_string() const; + virtual std::string debug_string() const override; private: static void* compute_fn(Expr* e, TupleRow* row); diff --git a/be/src/exprs/literal.h b/be/src/exprs/literal.h index 49b402f792..a00f0e9408 100644 --- a/be/src/exprs/literal.h +++ b/be/src/exprs/literal.h @@ -32,21 +32,21 @@ public: virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new Literal(*this)); } - virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow*); - virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual IntVal get_int_val(ExprContext* context, TupleRow*); - virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); - virtual FloatVal get_float_val(ExprContext* context, TupleRow*); - virtual DoubleVal get_double_val(ExprContext* context, TupleRow*); - virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow*); - virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow*); - virtual StringVal get_string_val(ExprContext* context, TupleRow* row); - virtual CollectionVal get_array_val(ExprContext* context, TupleRow*); + virtual BooleanVal get_boolean_val(ExprContext* context, TupleRow*) override; + virtual TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; + virtual FloatVal get_float_val(ExprContext* context, TupleRow*) override; + virtual DoubleVal get_double_val(ExprContext* context, TupleRow*) override; + virtual DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow*) override; + virtual DateTimeVal get_datetime_val(ExprContext* context, TupleRow*) override; + virtual StringVal get_string_val(ExprContext* context, TupleRow* row) override; + virtual CollectionVal get_array_val(ExprContext* context, TupleRow*) override; // init val before use virtual Status prepare(RuntimeState* state, const RowDescriptor& row_desc, - ExprContext* context); + ExprContext* context) override; protected: friend class Expr; diff --git a/be/src/exprs/minmax_predicate.h b/be/src/exprs/minmax_predicate.h index eda8b78614..61f7f1f768 100644 --- a/be/src/exprs/minmax_predicate.h +++ b/be/src/exprs/minmax_predicate.h @@ -34,6 +34,8 @@ public: virtual Status assign(void* min_data, void* max_data) = 0; // merge from other minmax_func virtual Status merge(MinMaxFuncBase* minmax_func, ObjectPool* pool) = 0; + virtual ~MinMaxFuncBase() = default; + ; }; template @@ -113,4 +115,4 @@ private: bool _empty = true; }; -} \ No newline at end of file +} // namespace doris \ No newline at end of file diff --git a/be/src/exprs/new_agg_fn_evaluator.cc b/be/src/exprs/new_agg_fn_evaluator.cc index 7a2209ba7f..851291708b 100644 --- a/be/src/exprs/new_agg_fn_evaluator.cc +++ b/be/src/exprs/new_agg_fn_evaluator.cc @@ -88,8 +88,6 @@ typedef StringVal (*SerializeFn)(FunctionContext*, const StringVal&); typedef AnyVal (*GetValueFn)(FunctionContext*, const AnyVal&); typedef AnyVal (*FinalizeFn)(FunctionContext*, const AnyVal&); -const int DEFAULT_MULTI_DISTINCT_COUNT_STRING_BUFFER_SIZE = 1024; - NewAggFnEvaluator::NewAggFnEvaluator(const AggFn& agg_fn, MemPool* mem_pool, const std::shared_ptr& tracker, bool is_clone) : _total_mem_consumption(0), diff --git a/be/src/exprs/null_literal.h b/be/src/exprs/null_literal.h index 38d7dcd07a..6fd6a8e929 100644 --- a/be/src/exprs/null_literal.h +++ b/be/src/exprs/null_literal.h @@ -31,17 +31,17 @@ public: return pool->add(new NullLiteral(*this)); } // NullLiteral(PrimitiveType type); - virtual doris_udf::BooleanVal get_boolean_val(ExprContext*, TupleRow*); - virtual doris_udf::TinyIntVal get_tiny_int_val(ExprContext*, TupleRow*); - virtual doris_udf::SmallIntVal get_small_int_val(ExprContext*, TupleRow*); - virtual doris_udf::IntVal get_int_val(ExprContext*, TupleRow*); - virtual doris_udf::BigIntVal get_big_int_val(ExprContext*, TupleRow*); - virtual doris_udf::FloatVal get_float_val(ExprContext*, TupleRow*); - virtual doris_udf::DoubleVal get_double_val(ExprContext*, TupleRow*); - virtual doris_udf::StringVal get_string_val(ExprContext*, TupleRow*); - virtual doris_udf::DateTimeVal get_datetime_val(ExprContext*, TupleRow*); - virtual doris_udf::DecimalV2Val get_decimalv2_val(ExprContext*, TupleRow*); - virtual CollectionVal get_array_val(ExprContext* context, TupleRow*); + virtual doris_udf::BooleanVal get_boolean_val(ExprContext*, TupleRow*) override; + virtual doris_udf::TinyIntVal get_tiny_int_val(ExprContext*, TupleRow*) override; + virtual doris_udf::SmallIntVal get_small_int_val(ExprContext*, TupleRow*) override; + virtual doris_udf::IntVal get_int_val(ExprContext*, TupleRow*) override; + virtual doris_udf::BigIntVal get_big_int_val(ExprContext*, TupleRow*) override; + virtual doris_udf::FloatVal get_float_val(ExprContext*, TupleRow*) override; + virtual doris_udf::DoubleVal get_double_val(ExprContext*, TupleRow*) override; + virtual doris_udf::StringVal get_string_val(ExprContext*, TupleRow*) override; + virtual doris_udf::DateTimeVal get_datetime_val(ExprContext*, TupleRow*) override; + virtual doris_udf::DecimalV2Val get_decimalv2_val(ExprContext*, TupleRow*) override; + virtual CollectionVal get_array_val(ExprContext* context, TupleRow*) override; protected: friend class Expr; diff --git a/be/src/exprs/scalar_fn_call.h b/be/src/exprs/scalar_fn_call.h index cf8f1f7c85..093a970c20 100644 --- a/be/src/exprs/scalar_fn_call.h +++ b/be/src/exprs/scalar_fn_call.h @@ -48,7 +48,7 @@ class TExprNode; /// - Allow more functions to be nullptr in UDA test harness class ScalarFnCall : public Expr { public: - virtual std::string debug_string() const; + virtual std::string debug_string() const override; virtual ~ScalarFnCall(); virtual Expr* clone(ObjectPool* pool) const override { return pool->add(new ScalarFnCall(*this)); @@ -63,26 +63,27 @@ protected: friend class Expr; ScalarFnCall(const TExprNode& node); - virtual Status prepare(RuntimeState* state, const RowDescriptor& desc, ExprContext* context); + virtual Status prepare(RuntimeState* state, const RowDescriptor& desc, + ExprContext* context) override; virtual Status open(RuntimeState* state, ExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual void close(RuntimeState* state, ExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; - virtual bool is_constant() const; + virtual bool is_constant() const override; - virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*); - virtual doris_udf::TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::IntVal get_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::FloatVal get_float_val(ExprContext* context, TupleRow*); - virtual doris_udf::DoubleVal get_double_val(ExprContext* context, TupleRow*); - virtual doris_udf::StringVal get_string_val(ExprContext* context, TupleRow*); - virtual doris_udf::DateTimeVal get_datetime_val(ExprContext* context, TupleRow*); - virtual doris_udf::DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow*); - virtual CollectionVal get_array_val(ExprContext* context, TupleRow*); + virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::FloatVal get_float_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::DoubleVal get_double_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::StringVal get_string_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::DateTimeVal get_datetime_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow*) override; + virtual CollectionVal get_array_val(ExprContext* context, TupleRow*) override; private: /// If this function has var args, children()[_vararg_start_idx] is the first vararg diff --git a/be/src/exprs/slot_ref.h b/be/src/exprs/slot_ref.h index 110140e470..3aa52b93de 100644 --- a/be/src/exprs/slot_ref.h +++ b/be/src/exprs/slot_ref.h @@ -42,33 +42,34 @@ public: Status prepare(const SlotDescriptor* slot_desc, const RowDescriptor& row_desc); - virtual Status prepare(RuntimeState* state, const RowDescriptor& row_desc, ExprContext* ctx); + virtual Status prepare(RuntimeState* state, const RowDescriptor& row_desc, + ExprContext* ctx) override; static void* get_value(Expr* expr, TupleRow* row); void* get_slot(TupleRow* row); Tuple* get_tuple(TupleRow* row); bool is_null_bit_set(TupleRow* row); static bool vector_compute_fn(Expr* expr, VectorizedRowBatch* batch); static bool is_nullable(Expr* expr); - virtual std::string debug_string() const; - virtual bool is_constant() const { return false; } - virtual bool is_vectorized() const { return true; } - virtual bool is_bound(std::vector* tuple_ids) const; - virtual int get_slot_ids(std::vector* slot_ids) const; + virtual std::string debug_string() const override; + virtual bool is_constant() const override { return false; } + virtual bool is_vectorized() const override { return true; } + virtual bool is_bound(std::vector* tuple_ids) const override; + virtual int get_slot_ids(std::vector* slot_ids) const override; SlotId slot_id() const { return _slot_id; } inline NullIndicatorOffset null_indicator_offset() const { return _null_indicator_offset; } - virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*); - virtual doris_udf::TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::SmallIntVal get_small_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::IntVal get_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::BigIntVal get_big_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::LargeIntVal get_large_int_val(ExprContext* context, TupleRow*); - virtual doris_udf::FloatVal get_float_val(ExprContext* context, TupleRow*); - virtual doris_udf::DoubleVal get_double_val(ExprContext* context, TupleRow*); - virtual doris_udf::StringVal get_string_val(ExprContext* context, TupleRow*); - virtual doris_udf::DateTimeVal get_datetime_val(ExprContext* context, TupleRow*); - virtual doris_udf::DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow*); - virtual doris_udf::CollectionVal get_array_val(ExprContext* context, TupleRow*); + virtual doris_udf::BooleanVal get_boolean_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::TinyIntVal get_tiny_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::SmallIntVal get_small_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::IntVal get_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::BigIntVal get_big_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::LargeIntVal get_large_int_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::FloatVal get_float_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::DoubleVal get_double_val(ExprContext* context, TupleRow* row) override; + virtual doris_udf::StringVal get_string_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::DateTimeVal get_datetime_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::DecimalV2Val get_decimalv2_val(ExprContext* context, TupleRow*) override; + virtual doris_udf::CollectionVal get_array_val(ExprContext* context, TupleRow*) override; private: int _tuple_idx; // within row diff --git a/be/src/exprs/tuple_is_null_predicate.h b/be/src/exprs/tuple_is_null_predicate.h index 58069d5a20..283b787ea7 100644 --- a/be/src/exprs/tuple_is_null_predicate.h +++ b/be/src/exprs/tuple_is_null_predicate.h @@ -31,18 +31,18 @@ public: return pool->add(new TupleIsNullPredicate(*this)); } - bool is_constant() const override { - return false; - } + bool is_constant() const override { return false; } + protected: friend class Expr; TupleIsNullPredicate(const TExprNode& node); - virtual Status prepare(RuntimeState* state, const RowDescriptor& row_desc, ExprContext* ctx); + virtual Status prepare(RuntimeState* state, const RowDescriptor& row_desc, + ExprContext* ctx) override; - virtual BooleanVal get_boolean_val(ExprContext* ctx, TupleRow* row); - virtual std::string debug_string() const; + virtual BooleanVal get_boolean_val(ExprContext* ctx, TupleRow* row) override; + virtual std::string debug_string() const override; private: std::vector _tuple_ids; diff --git a/be/src/geo/wkt_yacc.y b/be/src/geo/wkt_yacc.y index b43f75d4b5..7757cbef36 100644 --- a/be/src/geo/wkt_yacc.y +++ b/be/src/geo/wkt_yacc.y @@ -22,7 +22,7 @@ #include "geo/wkt_parse_type.h" #include "geo/geo_types.h" -class WktParseContext; +struct WktParseContext; void wkt_error(WktParseContext* ctx, const char* msg) { } /* forward declare this class for wkt_parse declaration in yacc.y.cpp */ diff --git a/be/src/gutil/hash/hash.cc b/be/src/gutil/hash/hash.cc index 9b1e440ffb..d895c484df 100644 --- a/be/src/gutil/hash/hash.cc +++ b/be/src/gutil/hash/hash.cc @@ -14,6 +14,7 @@ #include "gutil/hash/jenkins.h" #include "gutil/hash/jenkins_lookup2.h" #include "gutil/integral_types.h" +#include "gutil/logging-inl.h" // For components that ship code externally (notably the Google Search // Appliance) we want to change the fingerprint function so that @@ -186,16 +187,3 @@ uint64 FingerprintInterleavedImplementation(const char* s, uint32 len) { mix(d, e, f); return CombineFingerprintHalves(c, f); } - -// Extern template definitions. - -#if defined(__GNUC__) -#include -namespace __gnu_cxx { - -template class hash_set; -template class hash_map; - -} // namespace __gnu_cxx - -#endif diff --git a/be/src/gutil/hash/hash.h b/be/src/gutil/hash/hash.h index 8d76001c65..7595791434 100644 --- a/be/src/gutil/hash/hash.h +++ b/be/src/gutil/hash/hash.h @@ -20,8 +20,8 @@ // RECOMMENDED HASH FOR STRINGS: GoodFastHash // // It is a functor, so you can use it like this: -// hash_map> -// hash_set> +// hash_map > +// hash_set > // // RECOMMENDED HASH FOR NUMBERS: hash<> // @@ -78,13 +78,6 @@ #include #include -#include -using __gnu_cxx::hash; -using __gnu_cxx::hash_map; // hacky way to make sure we import standard hash<> fns -#include -using __gnu_cxx::hash; -using __gnu_cxx::hash_set; -#include #include #include @@ -99,28 +92,6 @@ using __gnu_cxx::hash_set; #include "gutil/integral_types.h" #include "gutil/macros.h" #include "gutil/port.h" -namespace __gnu_cxx { - -// STLport and MSVC 10.0 above already define these. -#if !defined(_STLP_LONG_LONG) && !(defined(_MSC_VER) && _MSC_VER >= 1600) - -#if defined(_MSC_VER) -// MSVC's stl implementation with _MSC_VER less than 1600 doesn't have -// this hash struct. STLport already defines this. -template -struct hash { - size_t operator()(const T& t) const; -}; -#endif // defined(_MSC_VER) - -#endif // !defined(_STLP_LONG_LONG) && !(defined(_MSC_VER) && _MSC_VER >= 1600) - -template <> -struct hash { - size_t operator()(bool x) const { return static_cast(x); } -}; - -} // namespace __gnu_cxx // ---------------------------------------------------------------------- // Fingerprint() @@ -211,8 +182,7 @@ inline uint64 FingerprintCat(uint64 fp1, uint64 fp2) { return Hash64NumWithSeed(fp1, fp2); } -#include -namespace __gnu_cxx { +namespace std { // This intended to be a "good" hash function. It may change from time to time. template <> @@ -231,88 +201,23 @@ struct hash { return c; } } - // Less than operator for MSVC use. - bool operator()(const uint128& a, const uint128& b) const { return a < b; } - static const size_t bucket_size = 4; // These are required by MSVC static const size_t min_buckets = 8; // 4 and 8 are defaults. }; -// Avoid collision with definition in port_hash.h (via port.h). -#ifndef HAVE_DEFINED_HASH_FOR_POINTERS -#define HAVE_DEFINED_HASH_FOR_POINTERS -// Hash pointers as if they were int's, but bring more entropy to -// the lower bits. -template -struct hash { - size_t operator()(T* x) const { - size_t k = reinterpret_cast(x); - return k + (k >> 6); - } -}; -#endif // HAVE_DEFINED_HASH_FOR_POINTERS - -#if defined(__GNUC__) -// Use our nice hash function for strings -template -struct hash> { - size_t operator()(const std::basic_string<_CharT, _Traits, _Alloc>& k) const { - return HashTo32(k.data(), static_cast(k.length())); - } -}; - -// they don't define a hash for const string at all -template <> -struct hash { - size_t operator()(const std::string& k) const { - return HashTo32(k.data(), static_cast(k.length())); - } -}; -#endif // defined(__GNUC__) - -// MSVC's STL requires an ever-so slightly different decl -#if defined(STL_MSVC) -template <> -struct hash { - size_t operator()(char const* const k) const { return HashTo32(k, strlen(k)); } - // Less than operator: - bool operator()(char const* const a, char const* const b) const { return strcmp(a, b) < 0; } - static const size_t bucket_size = 4; // These are required by MSVC - static const size_t min_buckets = 8; // 4 and 8 are defaults. -}; - -// MSVC 10.0 and above have already defined this. -#if !defined(_MSC_VER) || _MSC_VER < 1600 -template <> -struct hash { - size_t operator()(const std::string& k) const { return HashTo32(k.data(), k.length()); } - // Less than operator: - bool operator()(const std::string& a, const std::string& b) const { return a < b; } - static const size_t bucket_size = 4; // These are required by MSVC - static const size_t min_buckets = 8; // 4 and 8 are defaults. -}; -#endif // !defined(_MSC_VER) || _MSC_VER < 1600 - -#endif // defined(STL_MSVC) - // Hasher for STL pairs. Requires hashers for both members to be defined template -struct hash> { +struct hash > { size_t operator()(const pair& p) const { - size_t h1 = hash()(p.first); - size_t h2 = hash()(p.second); + size_t h1 = std::hash()(p.first); + size_t h2 = std::hash()(p.second); // The decision below is at compile time return (sizeof(h1) <= sizeof(uint32)) ? Hash32NumWithSeed(h1, h2) : Hash64NumWithSeed(h1, h2); } - // Less than operator for MSVC. - bool operator()(const pair& a, const pair& b) const { - return a < b; - } - static const size_t bucket_size = 4; // These are required by MSVC static const size_t min_buckets = 8; // 4 and 8 are defaults. }; -} // namespace __gnu_cxx +} // namespace std // If you want an excellent string hash function, and you don't mind if it // might change when you sync and recompile, please use GoodFastHash<>. @@ -350,7 +255,7 @@ struct GoodFastHash { // This intended to be a "good" hash function. It may change from time to time. template -struct GoodFastHash> { +struct GoodFastHash > { size_t operator()(const std::basic_string<_CharT, _Traits, _Alloc>& k) const { return HashStringThoroughly(k.data(), k.length() * sizeof(k[0])); } @@ -365,7 +270,7 @@ struct GoodFastHash> { // This intended to be a "good" hash function. It may change from time to time. template -struct GoodFastHash> { +struct GoodFastHash > { size_t operator()(const std::basic_string<_CharT, _Traits, _Alloc>& k) const { return HashStringThoroughly(k.data(), k.length() * sizeof(k[0])); } @@ -378,23 +283,4 @@ struct GoodFastHash> { static const size_t min_buckets = 8; // 4 and 8 are defaults. }; -// Extern template declarations. -// -// gcc only for now. msvc and others: this technique is likely to work with -// your compiler too. changelists welcome. -// -// This technique is limited to template specializations whose hash key -// functions are declared in this file. - -#if defined(__GNUC__) -#include -namespace __gnu_cxx { - -extern template class hash_set; -extern template class hash_map; - -} // namespace __gnu_cxx - -#endif // defined(__GNUC__) - #endif // UTIL_HASH_HASH_H_ diff --git a/be/src/gutil/strings/join.h b/be/src/gutil/strings/join.h index f8a22f583b..115e4aace5 100644 --- a/be/src/gutil/strings/join.h +++ b/be/src/gutil/strings/join.h @@ -10,12 +10,6 @@ #include #include -#include -using __gnu_cxx::hash; -using __gnu_cxx::hash_map; // Not used in this file. -#include -using __gnu_cxx::hash; -using __gnu_cxx::hash_set; // Not used in this file. #include using std::back_insert_iterator; using std::iterator_traits; diff --git a/be/src/http/action/checksum_action.cpp b/be/src/http/action/checksum_action.cpp index d1757d84e2..2ba5bd0956 100644 --- a/be/src/http/action/checksum_action.cpp +++ b/be/src/http/action/checksum_action.cpp @@ -41,7 +41,7 @@ const std::string TABLET_ID = "tablet_id"; const std::string TABLET_VERSION = "version"; const std::string SCHEMA_HASH = "schema_hash"; -ChecksumAction::ChecksumAction(ExecEnv* exec_env) : _exec_env(exec_env) {} +ChecksumAction::ChecksumAction() {} void ChecksumAction::handle(HttpRequest* req) { LOG(INFO) << "accept one request " << req->debug_string(); @@ -105,8 +105,8 @@ void ChecksumAction::handle(HttpRequest* req) { LOG(INFO) << "deal with checksum request finished! tablet id: " << tablet_id; } -int64_t ChecksumAction::do_checksum(int64_t tablet_id, int64_t version, - int32_t schema_hash, HttpRequest* req) { +int64_t ChecksumAction::do_checksum(int64_t tablet_id, int64_t version, int32_t schema_hash, + HttpRequest* req) { OLAPStatus res = OLAP_SUCCESS; uint32_t checksum; EngineChecksumTask engine_task(tablet_id, schema_hash, version, &checksum); diff --git a/be/src/http/action/checksum_action.h b/be/src/http/action/checksum_action.h index 9b85429222..386b74d58f 100644 --- a/be/src/http/action/checksum_action.h +++ b/be/src/http/action/checksum_action.h @@ -28,18 +28,14 @@ class ExecEnv; class ChecksumAction : public HttpHandler { public: - explicit ChecksumAction(ExecEnv* exec_env); + explicit ChecksumAction(); virtual ~ChecksumAction() {} void handle(HttpRequest* req) override; private: - int64_t do_checksum(int64_t tablet_id, int64_t version, - int32_t schema_hash, HttpRequest* req); - - ExecEnv* _exec_env; - + int64_t do_checksum(int64_t tablet_id, int64_t version, int32_t schema_hash, HttpRequest* req); }; // end class ChecksumAction } // end namespace doris diff --git a/be/src/http/action/health_action.cpp b/be/src/http/action/health_action.cpp index 2614740393..163c782b5f 100644 --- a/be/src/http/action/health_action.cpp +++ b/be/src/http/action/health_action.cpp @@ -30,7 +30,7 @@ namespace doris { const static std::string HEADER_JSON = "application/json"; -HealthAction::HealthAction(ExecEnv* exec_env) : _exec_env(exec_env) {} +HealthAction::HealthAction() {} void HealthAction::handle(HttpRequest* req) { std::stringstream ss; diff --git a/be/src/http/action/health_action.h b/be/src/http/action/health_action.h index fd2bd596e5..658e950627 100644 --- a/be/src/http/action/health_action.h +++ b/be/src/http/action/health_action.h @@ -27,14 +27,11 @@ class ExecEnv; // Get BE health state from http API. class HealthAction : public HttpHandler { public: - HealthAction(ExecEnv* exec_env); + HealthAction(); - virtual ~HealthAction(){}; + virtual ~HealthAction() {}; void handle(HttpRequest* req) override; - -private: - ExecEnv* _exec_env; }; } // end namespace doris diff --git a/be/src/http/action/pprof_actions.cpp b/be/src/http/action/pprof_actions.cpp index d576f88e66..d3b3863645 100644 --- a/be/src/http/action/pprof_actions.cpp +++ b/be/src/http/action/pprof_actions.cpp @@ -59,6 +59,7 @@ public: }; void HeapAction::handle(HttpRequest* req) { + std::lock_guard lock(kPprofActionMutex); #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER) (void)kPprofDefaultSampleSecs; // Avoid unused variable warning. @@ -66,8 +67,6 @@ void HeapAction::handle(HttpRequest* req) { HttpChannel::send_reply(req, str); #else - std::lock_guard lock(kPprofActionMutex); - int seconds = kPprofDefaultSampleSecs; const std::string& seconds_str = req->param(SECOND_KEY); if (!seconds_str.empty()) { @@ -292,8 +291,10 @@ Status PprofActions::setup(ExecEnv* exec_env, EvHttpServer* http_server, ObjectP http_server->register_handler(HttpMethod::GET, "/pprof/heap", pool.add(new HeapAction())); http_server->register_handler(HttpMethod::GET, "/pprof/growth", pool.add(new GrowthAction())); http_server->register_handler(HttpMethod::GET, "/pprof/profile", pool.add(new ProfileAction())); - http_server->register_handler(HttpMethod::GET, "/pprof/pmuprofile", pool.add(new PmuProfileAction())); - http_server->register_handler(HttpMethod::GET, "/pprof/contention", pool.add(new ContentionAction())); + http_server->register_handler(HttpMethod::GET, "/pprof/pmuprofile", + pool.add(new PmuProfileAction())); + http_server->register_handler(HttpMethod::GET, "/pprof/contention", + pool.add(new ContentionAction())); http_server->register_handler(HttpMethod::GET, "/pprof/cmdline", pool.add(new CmdlineAction())); auto action = pool.add(new SymbolAction(exec_env->bfd_parser())); http_server->register_handler(HttpMethod::GET, "/pprof/symbol", action); diff --git a/be/src/http/action/snapshot_action.cpp b/be/src/http/action/snapshot_action.cpp index ccdaea3cd4..86336b45d9 100644 --- a/be/src/http/action/snapshot_action.cpp +++ b/be/src/http/action/snapshot_action.cpp @@ -39,7 +39,7 @@ namespace doris { const std::string TABLET_ID = "tablet_id"; const std::string SCHEMA_HASH = "schema_hash"; -SnapshotAction::SnapshotAction(ExecEnv* exec_env) : _exec_env(exec_env) {} +SnapshotAction::SnapshotAction() {} void SnapshotAction::handle(HttpRequest* req) { LOG(INFO) << "accept one request " << req->debug_string(); @@ -101,7 +101,8 @@ int64_t SnapshotAction::make_snapshot(int64_t tablet_id, int32_t schema_hash, OLAPStatus res = OLAPStatus::OLAP_SUCCESS; bool allow_incremental_clone; // not used - res = SnapshotManager::instance()->make_snapshot(request, snapshot_path, &allow_incremental_clone); + res = SnapshotManager::instance()->make_snapshot(request, snapshot_path, + &allow_incremental_clone); if (res != OLAPStatus::OLAP_SUCCESS) { LOG(WARNING) << "make snapshot failed. status: " << res << ", signature: " << tablet_id; return -1L; diff --git a/be/src/http/action/snapshot_action.h b/be/src/http/action/snapshot_action.h index 665a5db4a3..673d761182 100644 --- a/be/src/http/action/snapshot_action.h +++ b/be/src/http/action/snapshot_action.h @@ -31,7 +31,7 @@ class ExecEnv; // be_host:be_http_port/api/snapshot?tablet_id=123&schema_hash=456 class SnapshotAction : public HttpHandler { public: - explicit SnapshotAction(ExecEnv* exec_env); + explicit SnapshotAction(); virtual ~SnapshotAction() {} @@ -39,9 +39,6 @@ public: private: int64_t make_snapshot(int64_t tablet_id, int schema_hash, std::string* snapshot_path); - - ExecEnv* _exec_env; - }; // end class SnapshotAction } // end namespace doris diff --git a/be/src/http/action/tablet_migration_action.cpp b/be/src/http/action/tablet_migration_action.cpp index ed7ea21976..6e94463cbe 100644 --- a/be/src/http/action/tablet_migration_action.cpp +++ b/be/src/http/action/tablet_migration_action.cpp @@ -63,7 +63,8 @@ void TabletMigrationAction::handle(HttpRequest* req) { do { { std::unique_lock lock(_migration_status_mutex); - std::map::iterator it_task = _migration_tasks.find(current_task); + std::map::iterator it_task = + _migration_tasks.find(current_task); if (it_task != _migration_tasks.end()) { status = Status::AlreadyExist(strings::Substitute( "There is a migration task for this tablet already exists. " @@ -73,8 +74,7 @@ void TabletMigrationAction::handle(HttpRequest* req) { } _migration_tasks[current_task] = "submitted"; } - auto st = _migration_thread_pool->submit_func([&, tablet_id, schema_hash, - dest_disk, current_task]() { + auto st = _migration_thread_pool->submit_func([&, dest_disk, current_task]() { { std::unique_lock lock(_migration_status_mutex); _migration_tasks[current_task] = "running"; @@ -123,7 +123,8 @@ void TabletMigrationAction::handle(HttpRequest* req) { std::string status_result; do { std::unique_lock lock(_migration_status_mutex); - std::map::iterator it_task = _migration_tasks.find(current_task); + std::map::iterator it_task = + _migration_tasks.find(current_task); if (it_task != _migration_tasks.end()) { status_result = "{\"status\": \"Success\", \"msg\": \"migration task is " + it_task->second + "\", \"dest_disk\": \"" + diff --git a/be/src/olap/bloom_filter_reader.h b/be/src/olap/bloom_filter_reader.h index e383fd9d33..9507d42fc6 100644 --- a/be/src/olap/bloom_filter_reader.h +++ b/be/src/olap/bloom_filter_reader.h @@ -55,7 +55,6 @@ private: size_t _step_size; // Bloom filter param - uint32_t _bit_num; uint32_t _hash_function_num; // BloomFilterIndexReader will not release bloom filter index buffer in destructor diff --git a/be/src/olap/byte_buffer.cpp b/be/src/olap/byte_buffer.cpp index 0cc877ee51..6b2747c12e 100644 --- a/be/src/olap/byte_buffer.cpp +++ b/be/src/olap/byte_buffer.cpp @@ -62,7 +62,7 @@ StorageByteBuffer* StorageByteBuffer::create(uint64_t capacity) { } SAFE_DELETE(buf); - SAFE_DELETE(memory); + SAFE_DELETE_ARRAY(memory); return nullptr; } diff --git a/be/src/olap/collect_iterator.h b/be/src/olap/collect_iterator.h index e0dd44d2e3..fb6747790e 100644 --- a/be/src/olap/collect_iterator.h +++ b/be/src/olap/collect_iterator.h @@ -53,7 +53,7 @@ private: // It currently contains two implementations, one is Level0Iterator, // which only reads data from the rowset reader, and the other is Level1Iterator, // which can read merged data from multiple LevelIterators through MergeHeap. - // By using Level1Iterator, some rowset readers can be merged in advance and + // By using Level1Iterator, some rowset readers can be merged in advance and // then merged with other rowset readers. class LevelIterator { public: @@ -68,13 +68,9 @@ private: virtual OLAPStatus next(const RowCursor** row, bool* delete_flag) = 0; virtual ~LevelIterator() = 0; - bool need_skip() const { - return _skip_row; - } + bool need_skip() const { return _skip_row; } - void set_need_skip(bool skip) const { - _skip_row = skip; - } + void set_need_skip(bool skip) const { _skip_row = skip; } // Only use in unique reader. Heap will set _skip_row = true. // when build heap find the row in LevelIterator have same key but lower version or sequence @@ -86,27 +82,28 @@ private: // if row cursors equal, compare data version. class LevelIteratorComparator { public: - LevelIteratorComparator(const bool reverse = false, int sequence_id_idx = -1) : - _reverse(reverse), _sequence_id_idx(sequence_id_idx) {} + LevelIteratorComparator(const bool reverse = false, int sequence_id_idx = -1) + : _reverse(reverse), _sequence_id_idx(sequence_id_idx) {} virtual bool operator()(const LevelIterator* a, const LevelIterator* b); + virtual ~LevelIteratorComparator() {} private: bool _reverse; int _sequence_id_idx; }; - class LevelZorderIteratorComparator: public LevelIteratorComparator { + class LevelZorderIteratorComparator : public LevelIteratorComparator { public: - LevelZorderIteratorComparator(const bool reverse = false, int sequence_id_idx = -1, const size_t sort_col_num = 0) : - _reverse(reverse), _sequence_id_idx(sequence_id_idx), _sort_col_num(sort_col_num) { + LevelZorderIteratorComparator(const bool reverse = false, int sequence_id_idx = -1, + const size_t sort_col_num = 0) + : _reverse(reverse) { _comparator = TupleRowZOrderComparator(sort_col_num); } virtual bool operator()(const LevelIterator* a, const LevelIterator* b); + virtual ~LevelZorderIteratorComparator() = default; private: bool _reverse = false; - int _sequence_id_idx; - size_t _sort_col_num = 0; TupleRowZOrderComparator _comparator; }; @@ -119,8 +116,7 @@ private: std::shared_ptr _cmp; }; - typedef std::priority_queue, - BaseComparator> + typedef std::priority_queue, BaseComparator> MergeHeap; // Iterate from rowset reader. This Iterator usually like a leaf node class Level0Iterator : public LevelIterator { @@ -146,19 +142,19 @@ private: OLAPStatus _refresh_current_row_v2(); RowsetReaderSharedPtr _rs_reader; - const RowCursor* _current_row = nullptr; // It points to the returned row + const RowCursor* _current_row = nullptr; // It points to the returned row bool _is_delete = false; TabletReader* _reader = nullptr; - RowCursor _row_cursor; // It points to rows inside `_row_block`, maybe not returned + RowCursor _row_cursor; // It points to rows inside `_row_block`, maybe not returned RowBlock* _row_block = nullptr; }; // Iterate from LevelIterators (maybe Level0Iterators or Level1Iterator or mixed) class Level1Iterator : public LevelIterator { public: - Level1Iterator(const std::list& children, bool merge, bool reverse, - int sequence_id_idx, uint64_t* merge_count, SortType sort_type, int sort_col_num); + int sequence_id_idx, uint64_t* merge_count, SortType sort_type, + int sort_col_num); OLAPStatus init() override; @@ -195,7 +191,6 @@ private: // The child LevelIterator should be either in _heap or in _children std::unique_ptr _heap; // used when `_merge == false` - int _child_idx = 0; int _sequence_id_idx = -1; uint64_t* _merged_rows = nullptr; diff --git a/be/src/olap/comparison_predicate.cpp b/be/src/olap/comparison_predicate.cpp index a154a04499..ff5db3f94f 100644 --- a/be/src/olap/comparison_predicate.cpp +++ b/be/src/olap/comparison_predicate.cpp @@ -145,68 +145,81 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(LessEqualPredicate, <=) COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterPredicate, >) COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=) -#define COMPARISON_PRED_COLUMN_EVALUATE(CLASS, OP) \ - template \ - void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const { \ - uint16_t new_size = 0; \ - if (column.is_nullable()) { \ - auto* nullable_column = vectorized::check_and_get_column(column);\ - auto& null_bitmap = reinterpret_cast&>(*(nullable_column->get_null_map_column_ptr())).get_data(); \ - auto* nest_column_vector = vectorized::check_and_get_column>(nullable_column->get_nested_column());\ - auto& data_array = nest_column_vector->get_data(); \ - for (uint16_t i = 0; i < *size; i++) { \ - uint16_t idx = sel[i]; \ - sel[new_size] = idx; \ - const type& cell_value = reinterpret_cast(data_array[idx]); \ - bool ret = !null_bitmap[idx] && (cell_value OP _value); \ - new_size += _opposite ? !ret : ret; \ - } \ - *size = new_size; \ - } else {\ - auto& pred_column_ref = reinterpret_cast&>(column);\ - auto& data_array = pred_column_ref.get_data(); \ - for (uint16_t i = 0; i < *size; i++) { \ - uint16_t idx = sel[i]; \ - sel[new_size] = idx; \ - const type& cell_value = reinterpret_cast(data_array[idx]); \ - auto ret = cell_value OP _value; \ - new_size += _opposite ? !ret : ret; \ - } \ - *size = new_size; \ - }\ +#define COMPARISON_PRED_COLUMN_EVALUATE(CLASS, OP) \ + template \ + void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const { \ + uint16_t new_size = 0; \ + if (column.is_nullable()) { \ + auto* nullable_column = \ + vectorized::check_and_get_column(column); \ + auto& null_bitmap = reinterpret_cast&>( \ + *(nullable_column->get_null_map_column_ptr())) \ + .get_data(); \ + auto* nest_column_vector = \ + vectorized::check_and_get_column>( \ + nullable_column->get_nested_column()); \ + auto& data_array = nest_column_vector->get_data(); \ + for (uint16_t i = 0; i < *size; i++) { \ + uint16_t idx = sel[i]; \ + sel[new_size] = idx; \ + const type& cell_value = reinterpret_cast(data_array[idx]); \ + bool ret = !null_bitmap[idx] && (cell_value OP _value); \ + new_size += _opposite ? !ret : ret; \ + } \ + *size = new_size; \ + } else { \ + auto& pred_column_ref = \ + reinterpret_cast&>(column); \ + auto& data_array = pred_column_ref.get_data(); \ + for (uint16_t i = 0; i < *size; i++) { \ + uint16_t idx = sel[i]; \ + sel[new_size] = idx; \ + const type& cell_value = reinterpret_cast(data_array[idx]); \ + auto ret = cell_value OP _value; \ + new_size += _opposite ? !ret : ret; \ + } \ + *size = new_size; \ + } \ } - + COMPARISON_PRED_COLUMN_EVALUATE(EqualPredicate, ==) COMPARISON_PRED_COLUMN_EVALUATE(NotEqualPredicate, !=) COMPARISON_PRED_COLUMN_EVALUATE(LessPredicate, <) COMPARISON_PRED_COLUMN_EVALUATE(LessEqualPredicate, <=) COMPARISON_PRED_COLUMN_EVALUATE(GreaterPredicate, >) COMPARISON_PRED_COLUMN_EVALUATE(GreaterEqualPredicate, >=) - -#define COMPARISON_PRED_COLUMN_EVALUATE_VEC(CLASS, OP) \ - template \ - void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const { \ - if (column.is_nullable()) { \ - auto* nullable_column = vectorized::check_and_get_column(column); \ - auto& data_array = reinterpret_cast&>(nullable_column->get_nested_column()).get_data(); \ - auto& null_bitmap = reinterpret_cast&>(*(nullable_column->get_null_map_column_ptr())).get_data(); \ - for (uint16_t i = 0; i < size; i++) { \ - flags[i] = (data_array[i] OP _value) && (!null_bitmap[i]); \ - } \ - } else { \ - auto& predicate_column = reinterpret_cast&>(column); \ - auto& data_array = predicate_column.get_data(); \ - for (uint16_t i = 0; i < size; i++) { \ - flags[i] = data_array[i] OP _value; \ - } \ - } \ - if (_opposite) { \ - for (uint16_t i = 0; i < size; i++) { \ - flags[i] = !flags[i]; \ - } \ - } \ + +#define COMPARISON_PRED_COLUMN_EVALUATE_VEC(CLASS, OP) \ + template \ + void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ + const { \ + if (column.is_nullable()) { \ + auto* nullable_column = \ + vectorized::check_and_get_column(column); \ + auto& data_array = reinterpret_cast&>( \ + nullable_column->get_nested_column()) \ + .get_data(); \ + auto& null_bitmap = reinterpret_cast&>( \ + *(nullable_column->get_null_map_column_ptr())) \ + .get_data(); \ + for (uint16_t i = 0; i < size; i++) { \ + flags[i] = (data_array[i] OP _value) && (!null_bitmap[i]); \ + } \ + } else { \ + auto& predicate_column = \ + reinterpret_cast&>(column); \ + auto& data_array = predicate_column.get_data(); \ + for (uint16_t i = 0; i < size; i++) { \ + flags[i] = data_array[i] OP _value; \ + } \ + } \ + if (_opposite) { \ + for (uint16_t i = 0; i < size; i++) { \ + flags[i] = !flags[i]; \ + } \ + } \ } - + COMPARISON_PRED_COLUMN_EVALUATE_VEC(EqualPredicate, ==) COMPARISON_PRED_COLUMN_EVALUATE_VEC(NotEqualPredicate, !=) COMPARISON_PRED_COLUMN_EVALUATE_VEC(LessPredicate, <) @@ -247,12 +260,11 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(GreaterPredicate, >) COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(GreaterEqualPredicate, >=) // todo(wb) support it -#define COMPARISON_PRED_COLUMN_EVALUATE_OR(CLASS, OP) \ - template \ - void CLASS::evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const { \ - \ - } - +#define COMPARISON_PRED_COLUMN_EVALUATE_OR(CLASS, OP) \ + template \ + void CLASS::evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size, \ + bool* flags) const {} + COMPARISON_PRED_COLUMN_EVALUATE_OR(EqualPredicate, ==) COMPARISON_PRED_COLUMN_EVALUATE_OR(NotEqualPredicate, !=) COMPARISON_PRED_COLUMN_EVALUATE_OR(LessPredicate, <) @@ -293,13 +305,11 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(GreaterPredicate, >) COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(GreaterEqualPredicate, >=) //todo(wb) support it -#define COMPARISON_PRED_COLUMN_EVALUATE_AND(CLASS, OP) \ - template \ - void CLASS::evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const { \ - \ - \ - } - +#define COMPARISON_PRED_COLUMN_EVALUATE_AND(CLASS, OP) \ + template \ + void CLASS::evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size, \ + bool* flags) const {} + COMPARISON_PRED_COLUMN_EVALUATE_AND(EqualPredicate, ==) COMPARISON_PRED_COLUMN_EVALUATE_AND(NotEqualPredicate, !=) COMPARISON_PRED_COLUMN_EVALUATE_AND(LessPredicate, <) @@ -431,8 +441,6 @@ COMPARISON_PRED_BITMAP_EVALUATE(GreaterEqualPredicate, >=) template CLASS::CLASS(uint32_t column_id, const double& value, bool opposite); \ template CLASS::CLASS(uint32_t column_id, const decimal12_t& value, \ bool opposite); \ - template CLASS::CLASS(uint32_t column_id, const StringValue& value, \ - bool opposite); \ template CLASS::CLASS(uint32_t column_id, const uint24_t& value, bool opposite); \ template CLASS::CLASS(uint32_t column_id, const uint64_t& value, bool opposite); \ template CLASS::CLASS(uint32_t column_id, const bool& value, bool opposite); @@ -541,61 +549,65 @@ COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(LessEqualPredicate) COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(GreaterPredicate) COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(GreaterEqualPredicate) -#define COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(CLASS) \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) \ - const; \ - template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t* size) const; - +#define COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(CLASS) \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; \ + template void CLASS::evaluate(vectorized::IColumn& column, uint16_t* sel, \ + uint16_t* size) const; + COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(EqualPredicate) COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(NotEqualPredicate) COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(LessPredicate) COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(LessEqualPredicate) COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(GreaterPredicate) COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(GreaterEqualPredicate) - -#define COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(CLASS) \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) \ - const; \ - template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const; - + +#define COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(CLASS) \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; \ + template void CLASS::evaluate_vec(vectorized::IColumn& column, uint16_t size, \ + bool* flags) const; + COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(EqualPredicate) COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(NotEqualPredicate) COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(LessPredicate) diff --git a/be/src/olap/cumulative_compaction_policy.h b/be/src/olap/cumulative_compaction_policy.h index 48c023325e..2ff414566e 100644 --- a/be/src/olap/cumulative_compaction_policy.h +++ b/be/src/olap/cumulative_compaction_policy.h @@ -64,9 +64,9 @@ public: /// param all_rowsets, all rowsets in tablet. /// param current_cumulative_point, current cumulative point value. /// return score, the result score after calculate. - virtual void calc_cumulative_compaction_score(TabletState state, - const std::vector& all_rowsets, int64_t current_cumulative_point, - uint32_t* score) = 0; + virtual void calc_cumulative_compaction_score( + TabletState state, const std::vector& all_rowsets, + int64_t current_cumulative_point, uint32_t* score) = 0; /// This function implements the policy which represents how to pick the candidate rowsets for compaction. /// This base class gives a unified implementation. Its derived classes also can override this function each other. @@ -156,11 +156,12 @@ public: /// Num based cumulative compaction policy implements calc cumulative compaction score function. /// Its main policy is calculating the accumulative compaction score after current cumulative_point in tablet. - void calc_cumulative_compaction_score(TabletState state, const std::vector& all_rowsets, + void calc_cumulative_compaction_score(TabletState state, + const std::vector& all_rowsets, int64_t current_cumulative_point, uint32_t* score) override; - std::string name() { return CUMULATIVE_NUM_BASED_POLICY; } + std::string name() override { return CUMULATIVE_NUM_BASED_POLICY; } }; /// SizeBased cumulative compaction policy implemention. SizeBased policy which derives CumulativeCompactionPolicy is a optimized @@ -206,15 +207,17 @@ public: /// Its main policy is judging the output rowset size whether satisfied the promotion size. /// If it satisfied, this policy will update the cumulative point. void update_cumulative_point(Tablet* tablet, const std::vector& input_rowsets, - RowsetSharedPtr _output_rowset, Version& last_delete_version); + RowsetSharedPtr _output_rowset, + Version& last_delete_version) override; /// Num based cumulative compaction policy implements calc cumulative compaction score function. /// Its main policy is calculating the accumulative compaction score after current cumulative_point in tablet. - void calc_cumulative_compaction_score(TabletState state, const std::vector& all_rowsets, + void calc_cumulative_compaction_score(TabletState state, + const std::vector& all_rowsets, int64_t current_cumulative_point, uint32_t* score) override; - std::string name() { return CUMULATIVE_SIZE_BASED_POLICY; } + std::string name() override { return CUMULATIVE_SIZE_BASED_POLICY; } private: /// calculate promotion size using current base rowset meta size and promotion configs diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index e3e643751d..3820f2c186 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -58,7 +58,6 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(disks_state, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(disks_compaction_score, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(disks_compaction_num, MetricUnit::NOUNIT); -static const char* const kMtabPath = "/etc/mtab"; static const char* const kTestFilePath = "/.testfile"; DataDir::DataDir(const std::string& path, int64_t capacity_bytes, @@ -353,8 +352,8 @@ std::string DataDir::get_root_path_from_schema_hash_path_in_trash( } OLAPStatus DataDir::_check_incompatible_old_format_tablet() { - auto check_incompatible_old_func = [this](int64_t tablet_id, int32_t schema_hash, - const std::string& value) -> bool { + auto check_incompatible_old_func = [](int64_t tablet_id, int32_t schema_hash, + const std::string& value) -> bool { // if strict check incompatible old format, then log fatal if (config::storage_strict_check_incompatible_old_format) { LOG(FATAL) diff --git a/be/src/olap/decimal12.h b/be/src/olap/decimal12.h index bcab67bae3..b73a44bb8c 100644 --- a/be/src/olap/decimal12.h +++ b/be/src/olap/decimal12.h @@ -109,12 +109,16 @@ struct decimal12_t { integer = 999999999999999999; fraction = 999999999; } else { + int32_t f = 0; + int64_t i = 0; if (sepr == value_string) { - sscanf(value_string, ".%9d", &fraction); - integer = 0; + int32_t f = 0; + sscanf(value_string, ".%9d", &f); } else { - sscanf(value_string, "%18ld.%9d", &integer, &fraction); + sscanf(value_string, "%18ld.%9d", &i, &f); } + integer = i; + fraction = f; int32_t frac_len = (nullptr != sepr) ? MAX_FRAC_DIGITS_NUM - strlen(sepr + 1) : MAX_FRAC_DIGITS_NUM; diff --git a/be/src/olap/fs/block_manager.h b/be/src/olap/fs/block_manager.h index 55be413ceb..58b15e5b8e 100644 --- a/be/src/olap/fs/block_manager.h +++ b/be/src/olap/fs/block_manager.h @@ -31,7 +31,7 @@ namespace doris { class BlockId; class Env; class MemTracker; -class Slice; +struct Slice; namespace fs { @@ -171,12 +171,8 @@ public: // used to specify directories based on block type (e.g. to prefer bloom block // placement into SSD-backed directories). struct CreateBlockOptions { - CreateBlockOptions(const FilePathDesc& new_path_desc) { - path_desc = new_path_desc; - } - CreateBlockOptions(const std::string& path) { - path_desc.filepath = path; - } + CreateBlockOptions(const FilePathDesc& new_path_desc) { path_desc = new_path_desc; } + CreateBlockOptions(const std::string& path) { path_desc.filepath = path; } // const std::string tablet_id; FilePathDesc path_desc; }; @@ -236,7 +232,8 @@ public: // may fail. // // Does not modify 'block' on error. - virtual Status open_block(const FilePathDesc& path_desc, std::unique_ptr* block) = 0; + virtual Status open_block(const FilePathDesc& path_desc, + std::unique_ptr* block) = 0; // Retrieves the IDs of all blocks under management by this block manager. // These include ReadableBlocks as well as WritableBlocks. @@ -249,7 +246,8 @@ public: virtual Status delete_block(const FilePathDesc& path_desc, bool is_dir = false) = 0; - virtual Status link_file(const FilePathDesc& src_path_desc, const FilePathDesc& dest_path_desc) = 0; + virtual Status link_file(const FilePathDesc& src_path_desc, + const FilePathDesc& dest_path_desc) = 0; static const std::string block_manager_preflush_control; }; diff --git a/be/src/olap/fs/file_block_manager.h b/be/src/olap/fs/file_block_manager.h index 118d619880..cf8e49d9a3 100644 --- a/be/src/olap/fs/file_block_manager.h +++ b/be/src/olap/fs/file_block_manager.h @@ -70,7 +70,8 @@ public: Status create_block(const CreateBlockOptions& opts, std::unique_ptr* block) override; - Status open_block(const FilePathDesc& path_desc, std::unique_ptr* block) override; + Status open_block(const FilePathDesc& path_desc, + std::unique_ptr* block) override; Status get_all_block_ids(std::vector* block_ids) override { // TODO(lingbin): to be implemented after we assign each block an id @@ -84,9 +85,10 @@ public: // the actual deletion will take place after the last open reader or // writer is closed. // is_dir: whether this path is a dir or file. if it is true, delete all files in this path - Status delete_block(const FilePathDesc& path_desc, bool is_dir = false); + Status delete_block(const FilePathDesc& path_desc, bool is_dir = false) override; - Status link_file(const FilePathDesc& src_path_desc, const FilePathDesc& dest_path_desc) override; + Status link_file(const FilePathDesc& src_path_desc, + const FilePathDesc& dest_path_desc) override; private: friend class internal::FileReadableBlock; diff --git a/be/src/olap/fs/remote_block_manager.h b/be/src/olap/fs/remote_block_manager.h index 56db5d96c0..c41f5f0589 100644 --- a/be/src/olap/fs/remote_block_manager.h +++ b/be/src/olap/fs/remote_block_manager.h @@ -47,16 +47,18 @@ public: Status create_block(const CreateBlockOptions& opts, std::unique_ptr* block) override; - Status open_block(const FilePathDesc& path_desc, std::unique_ptr* block) override; + Status open_block(const FilePathDesc& path_desc, + std::unique_ptr* block) override; Status get_all_block_ids(std::vector* block_ids) override { // TODO(lingbin): to be implemented after we assign each block an id return Status::OK(); }; - Status delete_block(const FilePathDesc& path_desc, bool is_dir = false); + Status delete_block(const FilePathDesc& path_desc, bool is_dir = false) override; - Status link_file(const FilePathDesc& src_path_desc, const FilePathDesc& dest_path_desc) override; + Status link_file(const FilePathDesc& src_path_desc, + const FilePathDesc& dest_path_desc) override; private: Env* _local_env; diff --git a/be/src/olap/hll.h b/be/src/olap/hll.h index 481ddcf03c..1ae56e2254 100644 --- a/be/src/olap/hll.h +++ b/be/src/olap/hll.h @@ -18,9 +18,10 @@ #ifndef DORIS_BE_SRC_OLAP_HLL_H #define DORIS_BE_SRC_OLAP_HLL_H -#include #include #include + +#include #include #include @@ -28,7 +29,7 @@ namespace doris { -class Slice; +struct Slice; const static int HLL_COLUMN_PRECISION = 14; const static int HLL_ZERO_COUNT_BITS = (64 - HLL_COLUMN_PRECISION); @@ -65,7 +66,7 @@ const static int HLL_EMPTY_SIZE = 1; // (1 + 4 + 3 * 4096) = 12293. // // HLL_DATA_FULL: most space-consuming, store all registers -// +// // A HLL value will change in the sequence empty -> explicit -> sparse -> full, and not // allow reverse. // @@ -80,9 +81,8 @@ enum HllDataType { class HyperLogLog { public: - HyperLogLog() = default; - explicit HyperLogLog(uint64_t hash_value): _type(HLL_DATA_EXPLICIT) { + explicit HyperLogLog(uint64_t hash_value) : _type(HLL_DATA_EXPLICIT) { _explicit_data = new uint64_t[HLL_EXPLICIT_INT64_NUM_DOUBLE]; _explicit_data[0] = hash_value; _explicit_data_num = 1; @@ -212,15 +212,13 @@ public: explicit HyperLogLog(const Slice& src); - ~HyperLogLog() { - clear(); - } + ~HyperLogLog() { clear(); } void clear() { _type = HLL_DATA_EMPTY; - delete [] _registers; + delete[] _registers; _registers = nullptr; - delete [] _explicit_data; + delete[] _explicit_data; _explicit_data = nullptr; _explicit_data_num = 0; } @@ -267,7 +265,7 @@ public: // Check if input slice is a valid serialized binary of HyperLogLog. // This function only check the encoded type in slice, whose complex - // function is O(1). + // function is O(1). static bool is_valid(const Slice& slice); // only for debug @@ -277,16 +275,15 @@ public: return {}; case HLL_DATA_EXPLICIT: case HLL_DATA_SPARSE: - case HLL_DATA_FULL: - { - std::string str {"hash set size: "}; - str.append(std::to_string((size_t)_explicit_data_num)); - str.append("\ncardinality:\t"); - str.append(std::to_string(estimate_cardinality())); - str.append("\ntype:\t"); - str.append(std::to_string(_type)); - return str; - } + case HLL_DATA_FULL: { + std::string str {"hash set size: "}; + str.append(std::to_string((size_t)_explicit_data_num)); + str.append("\ncardinality:\t"); + str.append(std::to_string(estimate_cardinality())); + str.append("\ntype:\t"); + str.append(std::to_string(_type)); + return str; + } default: return {}; } @@ -342,7 +339,7 @@ private: size_t n = (_explicit_data_num - i) * sizeof(*_explicit_data); if (n) { - memmove(_explicit_data+i+1, _explicit_data+i, n); + memmove(_explicit_data + i + 1, _explicit_data + i, n); } //insert data @@ -355,12 +352,13 @@ private: // todo(kks): remove this when dpp_sink class was removed class HllSetResolver { public: - HllSetResolver() : _buf_ref(nullptr), - _buf_len(0), - _set_type(HLL_DATA_EMPTY), - _full_value_position(nullptr), - _explicit_value(nullptr), - _explicit_num(0) {} + HllSetResolver() + : _buf_ref(nullptr), + _buf_len(0), + _set_type(HLL_DATA_EMPTY), + _full_value_position(nullptr), + _explicit_value(nullptr), + _explicit_num(0) {} ~HllSetResolver() {} @@ -371,7 +369,7 @@ public: typedef uint8_t SparseValueType; // only save pointer - void init(char* buf, int len){ + void init(char* buf, int len) { this->_buf_ref = buf; this->_buf_len = len; } @@ -398,10 +396,11 @@ public: // parse set , call after copy() or init() void parse(); -private : - char* _buf_ref; // set - int _buf_len; // set len - HllDataType _set_type; //set type + +private: + char* _buf_ref; // set + int _buf_len; // set len + HllDataType _set_type; //set type char* _full_value_position; uint64_t* _explicit_value; ExplicitLengthValueType _explicit_num; @@ -418,6 +417,6 @@ public: const int set_len, int& len); }; -} // namespace doris +} // namespace doris #endif // DORIS_BE_SRC_OLAP_HLL_H diff --git a/be/src/olap/lru_cache.h b/be/src/olap/lru_cache.h index 2ea6bda38e..0c4bf69795 100644 --- a/be/src/olap/lru_cache.h +++ b/be/src/olap/lru_cache.h @@ -368,15 +368,15 @@ public: virtual ~ShardedLRUCache(); virtual Handle* insert(const CacheKey& key, void* value, size_t charge, void (*deleter)(const CacheKey& key, void* value), - CachePriority priority = CachePriority::NORMAL); - virtual Handle* lookup(const CacheKey& key); - virtual void release(Handle* handle); - virtual void erase(const CacheKey& key); - virtual void* value(Handle* handle); + CachePriority priority = CachePriority::NORMAL) override; + virtual Handle* lookup(const CacheKey& key) override; + virtual void release(Handle* handle) override; + virtual void erase(const CacheKey& key) override; + virtual void* value(Handle* handle) override; Slice value_slice(Handle* handle) override; - virtual uint64_t new_id(); - virtual int64_t prune(); - virtual int64_t prune_if(CacheValuePredicate pred); + virtual uint64_t new_id() override; + virtual int64_t prune() override; + virtual int64_t prune_if(CacheValuePredicate pred) override; private: void update_cache_metrics() const; diff --git a/be/src/olap/memory/typed_column_writer.h b/be/src/olap/memory/typed_column_writer.h index f180be1b00..19b2326fd0 100644 --- a/be/src/olap/memory/typed_column_writer.h +++ b/be/src/olap/memory/typed_column_writer.h @@ -40,11 +40,12 @@ private: template class UpdateType { public: - bool& isnull() { return *static_cast(0); /*unused*/ } + bool& isnull() { return _fake_isnull; /*unused*/ } T& value() { return _value; } private: T _value; + bool _fake_isnull; }; // ColumnWriter typed implementations diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 2ec1ccbbc1..835842abb1 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -36,7 +36,6 @@ MemTable::MemTable(int64_t tablet_id, Schema* schema, const TabletSchema* tablet : _tablet_id(tablet_id), _schema(schema), _tablet_schema(tablet_schema), - _tuple_desc(tuple_desc), _slot_descs(slot_descs), _keys_type(keys_type), _mem_tracker(MemTracker::CreateTracker(-1, "MemTable", parent_tracker)), @@ -45,12 +44,13 @@ MemTable::MemTable(int64_t tablet_id, Schema* schema, const TabletSchema* tablet _schema_size(_schema->schema_size()), _rowset_writer(rowset_writer) { if (tablet_schema->sort_type() == SortType::ZORDER) { - _row_comparator = std::make_shared(_schema, tablet_schema->sort_col_num()); + _row_comparator = + std::make_shared(_schema, tablet_schema->sort_col_num()); } else { _row_comparator = std::make_shared(_schema); } - _skip_list = new Table(_row_comparator.get(), _table_mem_pool.get(), _keys_type == KeysType::DUP_KEYS); - + _skip_list = new Table(_row_comparator.get(), _table_mem_pool.get(), + _keys_type == KeysType::DUP_KEYS); } MemTable::~MemTable() { @@ -158,10 +158,8 @@ OLAPStatus MemTable::close() { return flush(); } -MemTable::Iterator::Iterator(MemTable* memtable): - _mem_table(memtable), - _it(memtable->_skip_list) { -} +MemTable::Iterator::Iterator(MemTable* memtable) + : _mem_table(memtable), _it(memtable->_skip_list) {} void MemTable::Iterator::seek_to_first() { _it.SeekToFirst(); @@ -176,7 +174,7 @@ void MemTable::Iterator::next() { } ContiguousRow MemTable::Iterator::get_current_row() { - char* row = (char*) _it.key(); + char* row = (char*)_it.key(); ContiguousRow dst_row(_mem_table->_schema, row); agg_finalize_row(&dst_row, _mem_table->_table_mem_pool.get()); return dst_row; diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index fdb574cb1e..4282ba5c06 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -28,7 +28,7 @@ namespace doris { -class ContiguousRow; +struct ContiguousRow; class RowsetWriter; class Schema; class SlotDescriptor; @@ -47,14 +47,14 @@ public: int64_t tablet_id() const { return _tablet_id; } size_t memory_usage() const { return _mem_tracker->consumption(); } void insert(const Tuple* tuple); - /// Flush + /// Flush OLAPStatus flush(); OLAPStatus close(); int64_t flush_size() const { return _flush_size; } private: - class RowCursorComparator: public RowComparator { + class RowCursorComparator : public RowComparator { public: RowCursorComparator(const Schema* schema); virtual int operator()(const char* left, const char* right) const; @@ -71,18 +71,18 @@ public: /// The iterator of memtable, so that the data in this memtable /// can be visited outside. class Iterator { - public: - Iterator(MemTable* mem_table); - ~Iterator() {} + public: + Iterator(MemTable* mem_table); + ~Iterator() {} - void seek_to_first(); - bool valid(); - void next(); - ContiguousRow get_current_row(); + void seek_to_first(); + bool valid(); + void next(); + ContiguousRow get_current_row(); - private: - MemTable* _mem_table; - Table::Iterator _it; + private: + MemTable* _mem_table; + Table::Iterator _it; }; private: @@ -92,7 +92,6 @@ private: int64_t _tablet_id; Schema* _schema; const TabletSchema* _tablet_schema; - TupleDescriptor* _tuple_desc; // the slot in _slot_descs are in order of tablet's schema const std::vector* _slot_descs; KeysType _keys_type; @@ -109,7 +108,7 @@ private: // The object buffer pool for convert tuple to row ObjectPool _agg_buffer_pool; // Only the rows will be inserted into SkipList can acquire the owner ship from - // `_agg_buffer_pool` + // `_agg_buffer_pool` ObjectPool _agg_object_pool; size_t _schema_size; diff --git a/be/src/olap/olap_cond.h b/be/src/olap/olap_cond.h index 84e56be142..6146859d00 100644 --- a/be/src/olap/olap_cond.h +++ b/be/src/olap/olap_cond.h @@ -35,7 +35,7 @@ namespace doris { class WrapperField; -class RowCursorCell; +struct RowCursorCell; enum CondOp { OP_NULL = -1, // invalid op diff --git a/be/src/olap/push_handler.h b/be/src/olap/push_handler.h index c1499b2858..60ed4f5815 100644 --- a/be/src/olap/push_handler.h +++ b/be/src/olap/push_handler.h @@ -36,7 +36,7 @@ namespace doris { class BinaryFile; class BinaryReader; -class ColumnMapping; +struct ColumnMapping; class RowCursor; struct TabletVars { @@ -197,7 +197,7 @@ public: return OLAP_SUCCESS; } bool eof() { return _eof; } - bool is_fill_tuple() { return _fill_tuple; } + bool is_fill_tuple() { return _fill_tuple; } MemPool* mem_pool() { return _mem_pool.get(); } private: diff --git a/be/src/olap/rowset/column_reader.cpp b/be/src/olap/rowset/column_reader.cpp index e870a3c947..d75a7cb652 100644 --- a/be/src/olap/rowset/column_reader.cpp +++ b/be/src/olap/rowset/column_reader.cpp @@ -72,8 +72,7 @@ OLAPStatus IntegerColumnReader::next(int64_t* value) { StringColumnDirectReader::StringColumnDirectReader(uint32_t column_unique_id, uint32_t dictionary_size) - : _eof(false), - _column_unique_id(column_unique_id), + : _column_unique_id(column_unique_id), _values(nullptr), _data_stream(nullptr), _length_reader(nullptr) {} diff --git a/be/src/olap/rowset/column_reader.h b/be/src/olap/rowset/column_reader.h index b585a2453c..d3db8dcc89 100644 --- a/be/src/olap/rowset/column_reader.h +++ b/be/src/olap/rowset/column_reader.h @@ -114,7 +114,6 @@ public: size_t get_buffer_size() { return sizeof(RunLengthByteReader); } private: - bool _eof; uint32_t _column_unique_id; Slice* _values; ReadOnlyFileStream* _data_stream; @@ -154,7 +153,7 @@ private: //StorageByteBuffer* _dictionary_data_buffer; // Save dict data std::vector _dictionary; // Used to read the actual data (represented by an integer) - RunLengthIntegerReader* _data_reader; + RunLengthIntegerReader* _data_reader; }; // ColumnReader is used to read a column and is the base class of other XXXColumnReader @@ -395,9 +394,10 @@ public: _stats = stats; return OLAP_SUCCESS; } - virtual OLAPStatus seek(PositionProvider* positions) { return OLAP_SUCCESS; } - virtual OLAPStatus skip(uint64_t row_count) { return OLAP_SUCCESS; } - virtual OLAPStatus next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool) { + virtual OLAPStatus seek(PositionProvider* positions) override { return OLAP_SUCCESS; } + virtual OLAPStatus skip(uint64_t row_count) override { return OLAP_SUCCESS; } + virtual OLAPStatus next_vector(ColumnVector* column_vector, uint32_t size, + MemPool* mem_pool) override { column_vector->set_no_nulls(false); column_vector->set_is_null(_is_null); _stats->bytes_read += size; @@ -792,11 +792,12 @@ public: virtual ~DecimalColumnReader(); OLAPStatus init(std::map* streams, int size, MemPool* mem_pool, OlapReaderStatistics* stats) override; - virtual OLAPStatus seek(PositionProvider* positions); - virtual OLAPStatus skip(uint64_t row_count); - virtual OLAPStatus next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool); + virtual OLAPStatus seek(PositionProvider* positions) override; + virtual OLAPStatus skip(uint64_t row_count) override; + virtual OLAPStatus next_vector(ColumnVector* column_vector, uint32_t size, + MemPool* mem_pool) override; - virtual size_t get_buffer_size() { return sizeof(RunLengthByteReader) * 2; } + virtual size_t get_buffer_size() override { return sizeof(RunLengthByteReader) * 2; } private: bool _eof; diff --git a/be/src/olap/rowset/column_writer.h b/be/src/olap/rowset/column_writer.h index e40ad38f31..42bc54caa2 100644 --- a/be/src/olap/rowset/column_writer.h +++ b/be/src/olap/rowset/column_writer.h @@ -128,7 +128,7 @@ public: ByteColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp); virtual ~ByteColumnWriter(); - virtual OLAPStatus init(); + virtual OLAPStatus init() override; OLAPStatus write_batch(RowBlock* block, RowCursor* cursor) override { for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { @@ -154,9 +154,9 @@ public: return OLAP_SUCCESS; } - virtual OLAPStatus finalize(ColumnDataHeaderMessage* header); - virtual void record_position(); - virtual OLAPStatus flush() { return OLAP_SUCCESS; } + virtual OLAPStatus finalize(ColumnDataHeaderMessage* header) override; + virtual void record_position() override; + virtual OLAPStatus flush() override { return OLAP_SUCCESS; } private: RunLengthByteWriter* _writer; @@ -199,7 +199,7 @@ public: virtual ~IntegerColumnWriterWrapper() {} - virtual OLAPStatus init() { + virtual OLAPStatus init() override { OLAPStatus res = OLAP_SUCCESS; res = ColumnWriter::init(); @@ -241,7 +241,7 @@ public: return OLAP_SUCCESS; } - virtual OLAPStatus flush() { + virtual OLAPStatus flush() override { OLAPStatus res = ColumnWriter::flush(); if (OLAP_SUCCESS != res) { @@ -259,7 +259,7 @@ public: return res; } - virtual OLAPStatus finalize(ColumnDataHeaderMessage* header) { + virtual OLAPStatus finalize(ColumnDataHeaderMessage* header) override { OLAPStatus res = ColumnWriter::finalize(header); if (OLAP_UNLIKELY(OLAP_SUCCESS != res)) { @@ -270,7 +270,7 @@ public: return _writer.finalize(header); } - virtual void record_position() { + virtual void record_position() override { ColumnWriter::record_position(); _writer.record_position(index_entry()); } @@ -291,7 +291,7 @@ public: virtual ~DoubleColumnWriterBase() {} - virtual OLAPStatus init() { + virtual OLAPStatus init() override { OLAPStatus res = OLAP_SUCCESS; res = ColumnWriter::init(); @@ -311,7 +311,7 @@ public: return OLAP_SUCCESS; } - virtual OLAPStatus flush() { return OLAP_SUCCESS; } + virtual OLAPStatus flush() override { return OLAP_SUCCESS; } OLAPStatus write_batch(RowBlock* block, RowCursor* cursor) override { for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { @@ -336,7 +336,7 @@ public: return OLAP_SUCCESS; } - virtual OLAPStatus finalize(ColumnDataHeaderMessage* header) { + virtual OLAPStatus finalize(ColumnDataHeaderMessage* header) override { OLAPStatus res = OLAP_SUCCESS; res = ColumnWriter::finalize(header); @@ -354,7 +354,7 @@ public: return OLAP_SUCCESS; } - virtual void record_position() { + virtual void record_position() override { ColumnWriter::record_position(); _stream->get_position(index_entry()); } @@ -375,7 +375,7 @@ public: VarStringColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp); virtual ~VarStringColumnWriter(); - virtual OLAPStatus init(); + virtual OLAPStatus init() override; OLAPStatus write_batch(RowBlock* block, RowCursor* cursor) override { for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { @@ -399,11 +399,11 @@ public: return OLAP_SUCCESS; } - virtual uint64_t estimate_buffered_memory(); - virtual OLAPStatus finalize(ColumnDataHeaderMessage* header); - virtual void save_encoding(ColumnEncodingMessage* encoding); - virtual void record_position(); - virtual OLAPStatus flush() { return OLAP_SUCCESS; } + virtual uint64_t estimate_buffered_memory() override; + virtual OLAPStatus finalize(ColumnDataHeaderMessage* header) override; + virtual void save_encoding(ColumnEncodingMessage* encoding) override; + virtual void record_position() override; + virtual OLAPStatus flush() override { return OLAP_SUCCESS; } protected: //Write a piece of data directly without using cursor @@ -476,7 +476,7 @@ public: return OLAP_SUCCESS; } - virtual OLAPStatus flush() { return OLAP_SUCCESS; } + virtual OLAPStatus flush() override { return OLAP_SUCCESS; } private: uint32_t _length; @@ -495,7 +495,7 @@ public: DecimalColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp); virtual ~DecimalColumnWriter(); - virtual OLAPStatus init(); + virtual OLAPStatus init() override; OLAPStatus write_batch(RowBlock* block, RowCursor* cursor) override { for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { @@ -525,9 +525,9 @@ public: return OLAP_SUCCESS; } - virtual OLAPStatus finalize(ColumnDataHeaderMessage* header); - virtual void record_position(); - virtual OLAPStatus flush() { return OLAP_SUCCESS; } + virtual OLAPStatus finalize(ColumnDataHeaderMessage* header) override; + virtual void record_position() override; + virtual OLAPStatus flush() override { return OLAP_SUCCESS; } private: RunLengthIntegerWriter* _int_writer; @@ -541,7 +541,7 @@ public: LargeIntColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp); virtual ~LargeIntColumnWriter(); - virtual OLAPStatus init(); + virtual OLAPStatus init() override; OLAPStatus write_batch(RowBlock* block, RowCursor* cursor) override { for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { @@ -570,9 +570,9 @@ public: return OLAP_SUCCESS; } - virtual OLAPStatus finalize(ColumnDataHeaderMessage* header); - virtual void record_position(); - virtual OLAPStatus flush() { return OLAP_SUCCESS; } + virtual OLAPStatus finalize(ColumnDataHeaderMessage* header) override; + virtual void record_position() override; + virtual OLAPStatus flush() override { return OLAP_SUCCESS; } private: RunLengthIntegerWriter* _high_writer; diff --git a/be/src/olap/rowset/rowset_factory.h b/be/src/olap/rowset/rowset_factory.h index 2d1382e6c2..bad3e7efca 100644 --- a/be/src/olap/rowset/rowset_factory.h +++ b/be/src/olap/rowset/rowset_factory.h @@ -25,13 +25,14 @@ namespace doris { class RowsetWriter; -class RowsetWriterContext; +struct RowsetWriterContext; class RowsetFactory { public: // return OLAP_SUCCESS and set inited rowset in `*rowset`. // return others if failed to create or init rowset. - static OLAPStatus create_rowset(const TabletSchema* schema, const FilePathDesc& rowset_path_desc, + static OLAPStatus create_rowset(const TabletSchema* schema, + const FilePathDesc& rowset_path_desc, RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset); // create and init rowset writer. diff --git a/be/src/olap/rowset/rowset_writer.h b/be/src/olap/rowset/rowset_writer.h index 19063e0dd4..a36f0f8ceb 100644 --- a/be/src/olap/rowset/rowset_writer.h +++ b/be/src/olap/rowset/rowset_writer.h @@ -27,7 +27,7 @@ namespace doris { -class ContiguousRow; +struct ContiguousRow; class MemTable; class RowCursor; diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.h b/be/src/olap/rowset/segment_v2/bitshuffle_page.h index df6a59ba0b..fe7ba566fa 100644 --- a/be/src/olap/rowset/segment_v2/bitshuffle_page.h +++ b/be/src/olap/rowset/segment_v2/bitshuffle_page.h @@ -37,8 +37,8 @@ #include "util/coding.h" #include "util/faststring.h" #include "util/slice.h" -#include "vec/runtime/vdatetime_value.h" #include "vec/columns/column_nullable.h" +#include "vec/runtime/vdatetime_value.h" namespace doris { namespace segment_v2 { @@ -124,7 +124,7 @@ public: _remain_element_capacity = block_size / SIZE_OF_TYPE; } - size_t count() const { return _count; } + size_t count() const override { return _count; } uint64_t size() const override { return _buffer.size(); } @@ -217,9 +217,7 @@ public: _size_of_element(0), _cur_index(0) {} - ~BitShufflePageDecoder() { - ChunkAllocator::instance()->free(_chunk); - } + ~BitShufflePageDecoder() { ChunkAllocator::instance()->free(_chunk); } Status init() override { CHECK(!_parsed); @@ -350,20 +348,20 @@ public: return Status::OK(); } - Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) override { + Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst) override { DCHECK(_parsed); if (PREDICT_FALSE(*n == 0 || _cur_index >= _num_elements)) { *n = 0; return Status::OK(); } - + size_t max_fetch = std::min(*n, static_cast(_num_elements - _cur_index)); - + dst->insert_many_fix_len_data((char*)&_chunk.data[_cur_index * SIZE_OF_TYPE], max_fetch); *n = max_fetch; _cur_index += max_fetch; - + return Status::OK(); }; @@ -383,7 +381,8 @@ private: Status _decode() { if (_num_elements > 0) { int64_t bytes; - if (!ChunkAllocator::instance()->allocate_align(_num_element_after_padding * _size_of_element, &_chunk)) { + if (!ChunkAllocator::instance()->allocate_align( + _num_element_after_padding * _size_of_element, &_chunk)) { return Status::RuntimeError("Decoded Memory Alloc failed"); } char* in = const_cast(&_data[BITSHUFFLE_PAGE_HEADER_SIZE]); diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.h b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.h index c9666265bc..14e2a126d1 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.h +++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.h @@ -34,7 +34,7 @@ class WritableBlock; namespace segment_v2 { -class BloomFilterOptions; +struct BloomFilterOptions; class BloomFilterIndexWriter { public: diff --git a/be/src/olap/rowset/segment_v2/encoding_info.h b/be/src/olap/rowset/segment_v2/encoding_info.h index 821e25be90..e5daa6ed2d 100644 --- a/be/src/olap/rowset/segment_v2/encoding_info.h +++ b/be/src/olap/rowset/segment_v2/encoding_info.h @@ -31,8 +31,8 @@ namespace segment_v2 { class PageBuilder; class PageDecoder; -class PageBuilderOptions; -class PageDecoderOptions; +struct PageBuilderOptions; +struct PageDecoderOptions; class EncodingInfo { public: diff --git a/be/src/olap/rowset/segment_v2/plain_page.h b/be/src/olap/rowset/segment_v2/plain_page.h index 09e4280474..67881dc165 100644 --- a/be/src/olap/rowset/segment_v2/plain_page.h +++ b/be/src/olap/rowset/segment_v2/plain_page.h @@ -71,7 +71,7 @@ public: _buffer.resize(PLAIN_PAGE_HEADER_SIZE); } - size_t count() const { return _count; } + size_t count() const override { return _count; } uint64_t size() const override { return _buffer.size(); } @@ -186,7 +186,7 @@ public: Status next_batch(size_t* n, ColumnBlockView* dst) override { return next_batch(n, dst); } - Status next_batch(size_t* n, vectorized::MutableColumnPtr &dst) override { + Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst) override { return Status::NotSupported("plain page not implement vec op now"); }; diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 470efc0a9e..2235f0dd7d 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -37,8 +37,8 @@ namespace segment_v2 { using strings::Substitute; -Status Segment::open(const FilePathDesc& path_desc, uint32_t segment_id, const TabletSchema* tablet_schema, - std::shared_ptr* output) { +Status Segment::open(const FilePathDesc& path_desc, uint32_t segment_id, + const TabletSchema* tablet_schema, std::shared_ptr* output) { std::shared_ptr segment(new Segment(path_desc, segment_id, tablet_schema)); if (!Env::get_env(path_desc.storage_medium)->is_remote_env()) { RETURN_IF_ERROR(segment->_open()); @@ -47,11 +47,12 @@ Status Segment::open(const FilePathDesc& path_desc, uint32_t segment_id, const T return Status::OK(); } -Segment::Segment(const FilePathDesc& path_desc, uint32_t segment_id, const TabletSchema* tablet_schema) - : _path_desc(path_desc), _segment_id(segment_id), - _tablet_schema(tablet_schema) { +Segment::Segment(const FilePathDesc& path_desc, uint32_t segment_id, + const TabletSchema* tablet_schema) + : _path_desc(path_desc), _segment_id(segment_id), _tablet_schema(tablet_schema) { #ifndef BE_TEST - _mem_tracker = MemTracker::CreateTracker(-1, "Segment", StorageEngine::instance()->tablet_mem_tracker(), false); + _mem_tracker = MemTracker::CreateTracker( + -1, "Segment", StorageEngine::instance()->tablet_mem_tracker(), false); #else _mem_tracker = MemTracker::CreateTracker(-1, "Segment", nullptr, false); #endif @@ -74,7 +75,6 @@ Status Segment::new_iterator(const Schema& schema, const StorageReadOptions& rea if (!_is_open) { RETURN_IF_ERROR(_open()); } - DCHECK_NOTNULL(read_options.stats); read_options.stats->total_segment_number++; // trying to prune the current segment by segment-level zone map if (read_options.conditions != nullptr) { @@ -109,8 +109,8 @@ Status Segment::_parse_footer() { RETURN_IF_ERROR(rblock->size(&file_size)); if (file_size < 12) { - return Status::Corruption( - strings::Substitute("Bad segment file $0: file size $1 < 12", _path_desc.filepath, file_size)); + return Status::Corruption(strings::Substitute("Bad segment file $0: file size $1 < 12", + _path_desc.filepath, file_size)); } uint8_t fixed_buf[12]; @@ -118,15 +118,16 @@ Status Segment::_parse_footer() { // validate magic number if (memcmp(fixed_buf + 8, k_segment_magic, k_segment_magic_length) != 0) { - return Status::Corruption( - strings::Substitute("Bad segment file $0: magic number not match", _path_desc.filepath)); + return Status::Corruption(strings::Substitute("Bad segment file $0: magic number not match", + _path_desc.filepath)); } // read footer PB uint32_t footer_length = decode_fixed32_le(fixed_buf); if (file_size < 12 + footer_length) { return Status::Corruption(strings::Substitute("Bad segment file $0: file size $1 < $2", - _path_desc.filepath, file_size, 12 + footer_length)); + _path_desc.filepath, file_size, + 12 + footer_length)); } _mem_tracker->Consume(footer_length); @@ -146,7 +147,7 @@ Status Segment::_parse_footer() { // deserialize footer PB if (!_footer.ParseFromString(footer_buf)) { return Status::Corruption(strings::Substitute( - "Bad segment file $0: failed to parse SegmentFooterPB", _path_desc.filepath)); + "Bad segment file $0: failed to parse SegmentFooterPB", _path_desc.filepath)); } return Status::OK(); } @@ -202,7 +203,8 @@ Status Segment::_create_column_readers() { return Status::OK(); } -Status Segment::new_column_iterator(uint32_t cid, std::shared_ptr parent, ColumnIterator** iter) { +Status Segment::new_column_iterator(uint32_t cid, std::shared_ptr parent, + ColumnIterator** iter) { if (_column_readers[cid] == nullptr) { const TabletColumn& tablet_column = _tablet_schema->column(cid); if (!tablet_column.has_default_value() && !tablet_column.is_nullable()) { @@ -214,7 +216,8 @@ Status Segment::new_column_iterator(uint32_t cid, std::shared_ptr pa tablet_column.has_default_value(), tablet_column.default_value(), tablet_column.is_nullable(), type_info, tablet_column.length())); ColumnIteratorOptions iter_opts; - iter_opts.mem_tracker = MemTracker::CreateTracker(-1, "DefaultColumnIterator", parent, false); + iter_opts.mem_tracker = + MemTracker::CreateTracker(-1, "DefaultColumnIterator", parent, false); RETURN_IF_ERROR(default_value_iter->init(iter_opts)); *iter = default_value_iter.release(); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 66751882e6..2754aefe6d 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -24,13 +24,13 @@ #include "gutil/strings/substitute.h" #include "olap/column_predicate.h" #include "olap/fs/fs_util.h" +#include "olap/in_list_predicate.h" #include "olap/row.h" #include "olap/row_block2.h" #include "olap/row_cursor.h" #include "olap/rowset/segment_v2/column_reader.h" #include "olap/rowset/segment_v2/segment.h" #include "olap/short_key_index.h" -#include "olap/in_list_predicate.h" #include "util/doris_metrics.h" #include "util/simd/bits.h" @@ -67,7 +67,7 @@ public: range_size++; if (UNLIKELY(_buf_pos == _buf_size)) { // read next batch _read_next_batch(); - if (_eof) { + if (_eof) { break; } } @@ -592,7 +592,8 @@ Status SegmentIterator::next_batch(RowBlockV2* block) { void SegmentIterator::_vec_init_lazy_materialization() { _is_pred_column.resize(_schema.columns().size(), false); - std::set pred_column_ids; // including short_cir_pred_col_id_set and vec_pred_col_id_set + std::set + pred_column_ids; // including short_cir_pred_col_id_set and vec_pred_col_id_set _is_all_column_basic_type = true; bool is_predicate_column_exists = false; bool is_non_predicate_column_exists = false; @@ -610,8 +611,9 @@ void SegmentIterator::_vec_init_lazy_materialization() { pred_column_ids.insert(cid); // for date type which can not be executed in a vectorized way, using short circuit execution - if (type == OLAP_FIELD_TYPE_VARCHAR || type == OLAP_FIELD_TYPE_CHAR || type == OLAP_FIELD_TYPE_DECIMAL - || type == OLAP_FIELD_TYPE_DATE || predicate->is_in_predicate()) { + if (type == OLAP_FIELD_TYPE_VARCHAR || type == OLAP_FIELD_TYPE_CHAR || + type == OLAP_FIELD_TYPE_DECIMAL || type == OLAP_FIELD_TYPE_DATE || + predicate->is_in_predicate()) { short_cir_pred_col_id_set.insert(cid); _short_cir_eval_predicate.push_back(predicate); _is_all_column_basic_type = false; @@ -620,7 +622,8 @@ void SegmentIterator::_vec_init_lazy_materialization() { if (_pre_eval_block_predicate == nullptr) { _pre_eval_block_predicate.reset(new AndBlockColumnPredicate()); } - _pre_eval_block_predicate->add_column_predicate(new SingleColumnBlockPredicate(predicate)); + _pre_eval_block_predicate->add_column_predicate( + new SingleColumnBlockPredicate(predicate)); } } @@ -640,7 +643,8 @@ void SegmentIterator::_vec_init_lazy_materialization() { } _vec_pred_column_ids.assign(vec_pred_col_id_set.cbegin(), vec_pred_col_id_set.cend()); - _short_cir_pred_column_ids.assign(short_cir_pred_col_id_set.cbegin(), short_cir_pred_col_id_set.cend()); + _short_cir_pred_column_ids.assign(short_cir_pred_col_id_set.cbegin(), + short_cir_pred_col_id_set.cend()); } else { _is_all_column_basic_type = false; is_non_predicate_column_exists = true; @@ -657,7 +661,8 @@ void SegmentIterator::_vec_init_lazy_materialization() { // todo(wb) need further research more lazy materialization rule, such as get more info from `statistics` for better decision if (_is_all_column_basic_type) { std::set pred_set(_vec_pred_column_ids.begin(), _vec_pred_column_ids.end()); - std::set non_pred_set(_non_predicate_columns.begin(), _non_predicate_columns.end()); + std::set non_pred_set(_non_predicate_columns.begin(), + _non_predicate_columns.end()); // when _is_all_column_basic_type = true, _first_read_column_ids should keep the same order with _schema.column_ids which stands for return column order for (int i = 0; i < _schema.num_column_ids(); i++) { @@ -666,7 +671,8 @@ void SegmentIterator::_vec_init_lazy_materialization() { _first_read_column_ids.push_back(cid); } else if (non_pred_set.find(cid) != non_pred_set.end()) { _first_read_column_ids.push_back(cid); - _is_pred_column[cid] = true; // in this case, non-predicate column should also be filtered by sel idx, so we regard it as pred columns + // in this case, non-predicate column should also be filtered by sel idx, so we regard it as pred columns + _is_pred_column[cid] = true; } } @@ -687,10 +693,10 @@ void SegmentIterator::_vec_init_lazy_materialization() { auto cid = _schema.column_id(i); _schema_block_id_map[cid] = i; } - } -Status SegmentIterator::_read_columns(const std::vector& column_ids, vectorized::MutableColumns& column_block, size_t nrows) { +Status SegmentIterator::_read_columns(const std::vector& column_ids, + vectorized::MutableColumns& column_block, size_t nrows) { for (auto cid : column_ids) { auto& column = column_block[cid]; size_t rows_read = nrows; @@ -700,8 +706,9 @@ Status SegmentIterator::_read_columns(const std::vector& column_ids, v return Status::OK(); } -void SegmentIterator::_init_current_block(vectorized::Block* block, std::vector& current_columns) { - bool is_block_mem_reuse= block->mem_reuse(); +void SegmentIterator::_init_current_block( + vectorized::Block* block, std::vector& current_columns) { + bool is_block_mem_reuse = block->mem_reuse(); if (is_block_mem_reuse) { block->clear_column_data(_schema.num_column_ids()); } else { // pre fill output block here @@ -710,7 +717,9 @@ void SegmentIterator::_init_current_block(vectorized::Block* block, std::vector< auto column_desc = _schema.column(cid); auto data_type = Schema::get_data_type_ptr(column_desc->type()); if (column_desc->is_nullable()) { - block->insert({nullptr, std::make_shared(std::move(data_type)), column_desc->name()}); + block->insert({nullptr, + std::make_shared(std::move(data_type)), + column_desc->name()}); } else { block->insert({nullptr, std::move(data_type), column_desc->name()}); } @@ -719,7 +728,7 @@ void SegmentIterator::_init_current_block(vectorized::Block* block, std::vector< for (size_t i = 0; i < _schema.num_column_ids(); i++) { auto cid = _schema.column_id(i); - if (_is_pred_column[cid]) { //todo(wb) maybe we can relase it after output block + if (_is_pred_column[cid]) { //todo(wb) maybe we can relase it after output block current_columns[cid]->clear(); } else { // non-predicate column auto column_desc = _schema.column(cid); @@ -729,7 +738,7 @@ void SegmentIterator::_init_current_block(vectorized::Block* block, std::vector< auto data_type = Schema::get_data_type_ptr(column_desc->type()); if (column_desc->is_nullable()) { current_columns[cid] = doris::vectorized::ColumnNullable::create( - std::move(data_type->create_column()), doris::vectorized::ColumnUInt8::create()); + data_type->create_column(), doris::vectorized::ColumnUInt8::create()); } else { current_columns[cid] = data_type->create_column(); } @@ -746,27 +755,31 @@ void SegmentIterator::_init_current_block(vectorized::Block* block, std::vector< void SegmentIterator::_output_non_pred_columns(vectorized::Block* block, bool is_block_mem_reuse) { for (auto cid : _non_predicate_columns) { - block->replace_by_position(_schema_block_id_map[cid], std::move(_current_return_columns[cid])); + block->replace_by_position(_schema_block_id_map[cid], + std::move(_current_return_columns[cid])); } - } +} -Status SegmentIterator::_output_column_by_sel_idx(vectorized::Block* block, const std::vector& columnIds, - uint16_t* sel_rowid_idx, uint16_t select_size, bool is_block_mem_reuse) { +Status SegmentIterator::_output_column_by_sel_idx(vectorized::Block* block, + const std::vector& columnIds, + uint16_t* sel_rowid_idx, uint16_t select_size, + bool is_block_mem_reuse) { for (auto cid : columnIds) { int block_cid = _schema_block_id_map[cid]; - RETURN_IF_ERROR(block->copy_column_data_to_block(is_block_mem_reuse, - _current_return_columns[cid].get(), sel_rowid_idx, select_size, block_cid, _opts.block_row_max)); + RETURN_IF_ERROR(block->copy_column_data_to_block( + is_block_mem_reuse, _current_return_columns[cid].get(), sel_rowid_idx, select_size, + block_cid, _opts.block_row_max)); } return Status::OK(); - } +} - -Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint32_t& nrows_read, bool set_block_rowid) { +Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint32_t& nrows_read, + bool set_block_rowid) { do { uint32_t range_from; uint32_t range_to; bool has_next_range = - _range_iter->next_range(nrows_read_limit - nrows_read, &range_from, &range_to); + _range_iter->next_range(nrows_read_limit - nrows_read, &range_from, &range_to); if (!has_next_range) { break; } @@ -775,7 +788,8 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint32 RETURN_IF_ERROR(_seek_columns(_first_read_column_ids, _cur_rowid)); } size_t rows_to_read = range_to - range_from; - RETURN_IF_ERROR(_read_columns(_first_read_column_ids, _current_return_columns, rows_to_read)); + RETURN_IF_ERROR( + _read_columns(_first_read_column_ids, _current_return_columns, rows_to_read)); _cur_rowid += rows_to_read; if (set_block_rowid) { for (uint32_t rid = range_from; rid < range_to; rid++) { @@ -788,7 +802,8 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint32 return Status::OK(); } -void SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, uint16_t& selected_size) { +void SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, + uint16_t& selected_size) { if (_vec_pred_column_ids.empty()) { for (uint32_t i = 0; i < selected_size; ++i) { sel_rowid_idx[i] = i; @@ -827,7 +842,8 @@ void SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, selected_size = new_size; } -void SegmentIterator::_evaluate_short_circuit_predicate(uint16_t* vec_sel_rowid_idx, uint16_t* selected_size_ptr) { +void SegmentIterator::_evaluate_short_circuit_predicate(uint16_t* vec_sel_rowid_idx, + uint16_t* selected_size_ptr) { if (_short_cir_pred_column_ids.empty()) { return; } @@ -839,15 +855,19 @@ void SegmentIterator::_evaluate_short_circuit_predicate(uint16_t* vec_sel_rowid_ } // evaluate delete condition - _opts.delete_condition_predicates->evaluate(_current_return_columns, vec_sel_rowid_idx, selected_size_ptr); + _opts.delete_condition_predicates->evaluate(_current_return_columns, vec_sel_rowid_idx, + selected_size_ptr); } -void SegmentIterator::_read_columns_by_rowids(std::vector& read_column_ids, std::vector& rowid_vector, - uint16_t* sel_rowid_idx, size_t select_size, vectorized::MutableColumns* mutable_columns) { +void SegmentIterator::_read_columns_by_rowids(std::vector& read_column_ids, + std::vector& rowid_vector, + uint16_t* sel_rowid_idx, size_t select_size, + vectorized::MutableColumns* mutable_columns) { size_t start_idx = 0; while (start_idx < select_size) { size_t end_idx = start_idx + 1; - while (end_idx < select_size && (rowid_vector[sel_rowid_idx[end_idx - 1]] == rowid_vector[sel_rowid_idx[end_idx]] - 1)) { + while (end_idx < select_size && (rowid_vector[sel_rowid_idx[end_idx - 1]] == + rowid_vector[sel_rowid_idx[end_idx]] - 1)) { end_idx++; } size_t range = end_idx - start_idx; @@ -871,7 +891,8 @@ Status SegmentIterator::next_batch(vectorized::Block* block) { auto cid = _schema.column_id(i); if (_is_pred_column[cid]) { auto column_desc = _schema.column(cid); - _current_return_columns[cid] = Schema::get_predicate_column_nullable_ptr(column_desc->type(), column_desc->is_nullable()); + _current_return_columns[cid] = Schema::get_predicate_column_nullable_ptr( + column_desc->type(), column_desc->is_nullable()); _current_return_columns[cid]->reserve(_opts.block_row_max); } } @@ -920,7 +941,8 @@ Status SegmentIterator::next_batch(vectorized::Block* block) { // When predicate column and no-predicate column are both basic type, lazy materialization is eliminate // So output block directly after vectorization evaluation if (_is_all_column_basic_type) { - return _output_column_by_sel_idx(block, _first_read_column_ids, sel_rowid_idx, selected_size, is_mem_reuse); + return _output_column_by_sel_idx(block, _first_read_column_ids, sel_rowid_idx, + selected_size, is_mem_reuse); } // step 2: evaluate short ciruit predicate @@ -931,7 +953,8 @@ Status SegmentIterator::next_batch(vectorized::Block* block) { // step3: read non_predicate column if (!_non_predicate_columns.empty()) { - _read_columns_by_rowids(_non_predicate_columns, _block_rowids, sel_rowid_idx, selected_size, &_current_return_columns); + _read_columns_by_rowids(_non_predicate_columns, _block_rowids, sel_rowid_idx, + selected_size, &_current_return_columns); } // step4: output columns @@ -939,9 +962,11 @@ Status SegmentIterator::next_batch(vectorized::Block* block) { _output_non_pred_columns(block, is_mem_reuse); // 4.2 output short circuit predicate column - RETURN_IF_ERROR(_output_column_by_sel_idx(block, _short_cir_pred_column_ids, sel_rowid_idx, selected_size, is_mem_reuse)); + RETURN_IF_ERROR(_output_column_by_sel_idx(block, _short_cir_pred_column_ids, sel_rowid_idx, + selected_size, is_mem_reuse)); // 4.3 output vectorizatioin predicate column - return _output_column_by_sel_idx(block, _vec_pred_column_ids, sel_rowid_idx, selected_size, is_mem_reuse); + return _output_column_by_sel_idx(block, _vec_pred_column_ids, sel_rowid_idx, selected_size, + is_mem_reuse); } return Status::OK(); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index 2b91204c17..ba82be3a90 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -57,7 +57,7 @@ public: const Schema& schema() const override { return _schema; } bool is_lazy_materialization_read() const override { return _lazy_materialization_read; } - uint64_t data_id() const { return _segment->id(); } + uint64_t data_id() const override { return _segment->id(); } private: Status _init(bool is_vec = false); @@ -89,9 +89,12 @@ private: size_t row_offset, size_t nrows); // for vectorization implementation - Status _read_columns(const std::vector& column_ids, vectorized::MutableColumns& column_block, size_t nrows); - Status _read_columns_by_index(uint32_t nrows_read_limit, uint32_t& nrows_read, bool set_block_rowid); - void _init_current_block(vectorized::Block* block, std::vector& non_pred_vector); + Status _read_columns(const std::vector& column_ids, + vectorized::MutableColumns& column_block, size_t nrows); + Status _read_columns_by_index(uint32_t nrows_read_limit, uint32_t& nrows_read, + bool set_block_rowid); + void _init_current_block(vectorized::Block* block, + std::vector& non_pred_vector); void _evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, uint16_t& selected_size); void _evaluate_short_circuit_predicate(uint16_t* sel_rowid_idx, uint16_t* selected_size); void _output_non_pred_columns(vectorized::Block* block, bool is_block_mem_reuse); @@ -127,11 +130,13 @@ private: // could be a local variable of next_batch(), kept here to reuse vector memory std::vector _block_rowids; - // fields for vectorization execution + // fields for vectorization execution bool _is_all_column_basic_type; - std::vector _vec_pred_column_ids; // keep columnId of columns for vectorized predicate evaluation - std::vector _short_cir_pred_column_ids; // keep columnId of columns for short circuit predicate evaluation - vector _is_pred_column; // columns hold by segmentIter + std::vector + _vec_pred_column_ids; // keep columnId of columns for vectorized predicate evaluation + std::vector + _short_cir_pred_column_ids; // keep columnId of columns for short circuit predicate evaluation + vector _is_pred_column; // columns hold by segmentIter vectorized::MutableColumns _current_return_columns; std::unique_ptr _pre_eval_block_predicate; std::vector _short_cir_eval_predicate; @@ -149,8 +154,6 @@ private: // make a copy of `_opts.column_predicates` in order to make local changes std::vector _col_predicates; - int16_t** _select_vec; - // row schema of the key to seek // only used in `_get_row_ranges_by_keys` std::unique_ptr _seek_schema; diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp index 1ff9d285e3..593ba5babd 100644 --- a/be/src/olap/schema.cpp +++ b/be/src/olap/schema.cpp @@ -18,11 +18,11 @@ #include "olap/schema.h" #include "olap/row_block2.h" -#include "vec/columns/column_nullable.h" +#include "olap/uint24.h" #include "vec/columns/column_complex.h" +#include "vec/columns/column_nullable.h" #include "vec/columns/predicate_column.h" #include "vec/core/types.h" -#include "olap/uint24.h" namespace doris { @@ -110,107 +110,108 @@ Schema::~Schema() { vectorized::DataTypePtr Schema::get_data_type_ptr(FieldType type) { switch (type) { - case OLAP_FIELD_TYPE_BOOL: - return std::make_shared(); + case OLAP_FIELD_TYPE_BOOL: + return std::make_shared(); - case OLAP_FIELD_TYPE_TINYINT: - return std::make_shared(); + case OLAP_FIELD_TYPE_TINYINT: + return std::make_shared(); - case OLAP_FIELD_TYPE_SMALLINT: - return std::make_shared(); + case OLAP_FIELD_TYPE_SMALLINT: + return std::make_shared(); - case OLAP_FIELD_TYPE_INT: - return std::make_shared(); + case OLAP_FIELD_TYPE_INT: + return std::make_shared(); - case OLAP_FIELD_TYPE_FLOAT: - return std::make_shared(); + case OLAP_FIELD_TYPE_FLOAT: + return std::make_shared(); - case OLAP_FIELD_TYPE_BIGINT: - return std::make_shared(); + case OLAP_FIELD_TYPE_BIGINT: + return std::make_shared(); - case OLAP_FIELD_TYPE_LARGEINT: - return std::make_shared(); + case OLAP_FIELD_TYPE_LARGEINT: + return std::make_shared(); - case OLAP_FIELD_TYPE_DATE: - return std::make_shared(); + case OLAP_FIELD_TYPE_DATE: + return std::make_shared(); - case OLAP_FIELD_TYPE_DATETIME: - return std::make_shared(); + case OLAP_FIELD_TYPE_DATETIME: + return std::make_shared(); - case OLAP_FIELD_TYPE_DOUBLE: - return std::make_shared(); + case OLAP_FIELD_TYPE_DOUBLE: + return std::make_shared(); - case OLAP_FIELD_TYPE_CHAR: - case OLAP_FIELD_TYPE_VARCHAR: - case OLAP_FIELD_TYPE_STRING: - return std::make_shared(); - case OLAP_FIELD_TYPE_HLL: - return std::make_shared(); - case TYPE_OBJECT: - return std::make_shared(); + case OLAP_FIELD_TYPE_CHAR: + case OLAP_FIELD_TYPE_VARCHAR: + case OLAP_FIELD_TYPE_STRING: + return std::make_shared(); + case OLAP_FIELD_TYPE_HLL: + return std::make_shared(); + case OLAP_FIELD_TYPE_OBJECT: + return std::make_shared(); - case OLAP_FIELD_TYPE_DECIMAL: - return std::make_shared>(27, 9); + case OLAP_FIELD_TYPE_DECIMAL: + return std::make_shared>(27, 9); - default: - DCHECK(false); + default: + DCHECK(false); + return nullptr; } - // For llvm complain - return nullptr; } -vectorized::IColumn::MutablePtr Schema::get_predicate_column_nullable_ptr(FieldType type, bool is_null) { +vectorized::IColumn::MutablePtr Schema::get_predicate_column_nullable_ptr(FieldType type, + bool is_null) { vectorized::IColumn::MutablePtr ptr = Schema::get_predicate_column_ptr(type); if (is_null) { - return doris::vectorized::ColumnNullable::create(std::move(ptr), doris::vectorized::ColumnUInt8::create()); + return doris::vectorized::ColumnNullable::create(std::move(ptr), + doris::vectorized::ColumnUInt8::create()); } return ptr; } vectorized::IColumn::MutablePtr Schema::get_predicate_column_ptr(FieldType type) { switch (type) { - case OLAP_FIELD_TYPE_BOOL: - return doris::vectorized::PredicateColumnType::create();; - case OLAP_FIELD_TYPE_TINYINT: - return doris::vectorized::PredicateColumnType::create(); - - case OLAP_FIELD_TYPE_SMALLINT: - return doris::vectorized::PredicateColumnType::create(); - - case OLAP_FIELD_TYPE_INT: - return doris::vectorized::PredicateColumnType::create(); - - case OLAP_FIELD_TYPE_FLOAT: - return doris::vectorized::PredicateColumnType::create(); - - case OLAP_FIELD_TYPE_DOUBLE: - return doris::vectorized::PredicateColumnType::create(); - - case OLAP_FIELD_TYPE_BIGINT: - return doris::vectorized::PredicateColumnType::create(); - - case OLAP_FIELD_TYPE_LARGEINT: - return doris::vectorized::PredicateColumnType::create(); - - case OLAP_FIELD_TYPE_DATE: - return doris::vectorized::PredicateColumnType::create(); + case OLAP_FIELD_TYPE_BOOL: + return doris::vectorized::PredicateColumnType::create(); - case OLAP_FIELD_TYPE_DATETIME: - return doris::vectorized::PredicateColumnType::create(); - - case OLAP_FIELD_TYPE_CHAR: - case OLAP_FIELD_TYPE_VARCHAR: - case OLAP_FIELD_TYPE_STRING: - return doris::vectorized::PredicateColumnType::create(); - - case OLAP_FIELD_TYPE_DECIMAL: - return doris::vectorized::PredicateColumnType::create(); - - default: - DCHECK(false); + case OLAP_FIELD_TYPE_TINYINT: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_SMALLINT: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_INT: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_FLOAT: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_DOUBLE: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_BIGINT: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_LARGEINT: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_DATE: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_DATETIME: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_CHAR: + case OLAP_FIELD_TYPE_VARCHAR: + case OLAP_FIELD_TYPE_STRING: + return doris::vectorized::PredicateColumnType::create(); + + case OLAP_FIELD_TYPE_DECIMAL: + return doris::vectorized::PredicateColumnType::create(); + + default: + DCHECK(false); + return nullptr; } - // For llvm complain - return nullptr; } } // namespace doris diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h index 53aa34934a..fc97a7d354 100644 --- a/be/src/olap/schema_change.h +++ b/be/src/olap/schema_change.h @@ -76,7 +76,8 @@ private: class RowBlockAllocator { public: - RowBlockAllocator(const TabletSchema& tablet_schema, std::shared_ptr parent, size_t memory_limitation); + RowBlockAllocator(const TabletSchema& tablet_schema, std::shared_ptr parent, + size_t memory_limitation); virtual ~RowBlockAllocator(); OLAPStatus allocate(RowBlock** row_block, size_t num_rows, bool null_supported); @@ -85,7 +86,6 @@ public: private: const TabletSchema& _tablet_schema; - size_t _memory_allocated; std::shared_ptr _mem_tracker; size_t _row_len; size_t _memory_limitation; @@ -93,7 +93,8 @@ private: class SchemaChange { public: - SchemaChange(std::shared_ptr tracker) : _mem_tracker(std::move(tracker)), _filtered_rows(0), _merged_rows(0) {} + SchemaChange(std::shared_ptr tracker) + : _mem_tracker(std::move(tracker)), _filtered_rows(0), _merged_rows(0) {} virtual ~SchemaChange() = default; virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, @@ -111,8 +112,10 @@ public: void reset_filtered_rows() { _filtered_rows = 0; } void reset_merged_rows() { _merged_rows = 0; } + protected: std::shared_ptr _mem_tracker; + private: uint64_t _filtered_rows; uint64_t _merged_rows; @@ -120,7 +123,8 @@ private: class LinkedSchemaChange : public SchemaChange { public: - explicit LinkedSchemaChange(const RowBlockChanger& row_block_changer, std::shared_ptr mem_tracker) + explicit LinkedSchemaChange(const RowBlockChanger& row_block_changer, + std::shared_ptr mem_tracker) : SchemaChange(mem_tracker), _row_block_changer(row_block_changer) {} ~LinkedSchemaChange() {} @@ -137,7 +141,8 @@ class SchemaChangeDirectly : public SchemaChange { public: // @params tablet the instance of tablet which has new schema. // @params row_block_changer changer to modify the data of RowBlock - explicit SchemaChangeDirectly(const RowBlockChanger& row_block_changer, std::shared_ptr mem_tracker); + explicit SchemaChangeDirectly(const RowBlockChanger& row_block_changer, + std::shared_ptr mem_tracker); virtual ~SchemaChangeDirectly(); virtual OLAPStatus process(RowsetReaderSharedPtr rowset_reader, RowsetWriter* new_rowset_writer, @@ -156,7 +161,8 @@ private: // @breif schema change with sorting class SchemaChangeWithSorting : public SchemaChange { public: - explicit SchemaChangeWithSorting(const RowBlockChanger& row_block_changer, std::shared_ptr mem_tracker, + explicit SchemaChangeWithSorting(const RowBlockChanger& row_block_changer, + std::shared_ptr mem_tracker, size_t memory_limitation); virtual ~SchemaChangeWithSorting(); @@ -166,9 +172,9 @@ public: private: bool _internal_sorting(const std::vector& row_block_arr, - const Version& temp_delta_versions, - TabletSharedPtr new_tablet, RowsetTypePB new_rowset_type, - SegmentsOverlapPB segments_overlap, RowsetSharedPtr* rowset); + const Version& temp_delta_versions, TabletSharedPtr new_tablet, + RowsetTypePB new_rowset_type, SegmentsOverlapPB segments_overlap, + RowsetSharedPtr* rowset); bool _external_sorting(std::vector& src_rowsets, RowsetWriter* rowset_writer, TabletSharedPtr new_tablet); @@ -237,6 +243,7 @@ private: static OLAPStatus _init_column_mapping(ColumnMapping* column_mapping, const TabletColumn& column_schema, const std::string& value); + private: SchemaChangeHandler(); virtual ~SchemaChangeHandler(); diff --git a/be/src/olap/short_key_index.h b/be/src/olap/short_key_index.h index ce3b2f91a3..439814f118 100644 --- a/be/src/olap/short_key_index.h +++ b/be/src/olap/short_key_index.h @@ -259,9 +259,7 @@ public: private: template ShortKeyIndexIterator seek(const Slice& key) const { - auto comparator = [this](const Slice& lhs, const Slice& rhs) { - return lhs.compare(rhs) < 0; - }; + auto comparator = [](const Slice& lhs, const Slice& rhs) { return lhs.compare(rhs) < 0; }; if (lower_bound) { return std::lower_bound(begin(), end(), key, comparator); } else { diff --git a/be/src/olap/task/engine_alter_tablet_task.cpp b/be/src/olap/task/engine_alter_tablet_task.cpp index 51f029648a..5c26e93589 100644 --- a/be/src/olap/task/engine_alter_tablet_task.cpp +++ b/be/src/olap/task/engine_alter_tablet_task.cpp @@ -23,15 +23,8 @@ namespace doris { using std::to_string; -EngineAlterTabletTask::EngineAlterTabletTask(const TAlterTabletReqV2& request, int64_t signature, - const TTaskType::type task_type, - std::vector* error_msgs, - const string& process_name) - : _alter_tablet_req(request), - _signature(signature), - _task_type(task_type), - _error_msgs(error_msgs), - _process_name(process_name) {} +EngineAlterTabletTask::EngineAlterTabletTask(const TAlterTabletReqV2& request) + : _alter_tablet_req(request) {} OLAPStatus EngineAlterTabletTask::execute() { DorisMetrics::instance()->create_rollup_requests_total->increment(1); diff --git a/be/src/olap/task/engine_alter_tablet_task.h b/be/src/olap/task/engine_alter_tablet_task.h index 73dd5514ac..cf57b45d80 100644 --- a/be/src/olap/task/engine_alter_tablet_task.h +++ b/be/src/olap/task/engine_alter_tablet_task.h @@ -31,17 +31,11 @@ public: virtual OLAPStatus execute(); public: - EngineAlterTabletTask(const TAlterTabletReqV2& alter_tablet_request, int64_t signature, - const TTaskType::type task_type, vector* error_msgs, - const string& process_name); + EngineAlterTabletTask(const TAlterTabletReqV2& alter_tablet_request); ~EngineAlterTabletTask() {} private: const TAlterTabletReqV2& _alter_tablet_req; - int64_t _signature; - const TTaskType::type _task_type; - vector* _error_msgs; - const string& _process_name; }; // EngineTask diff --git a/be/src/olap/types.h b/be/src/olap/types.h index ec59a92d4b..0a16eef39c 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -46,6 +46,8 @@ class TabletColumn; class TypeInfo { public: + virtual ~TypeInfo() = default; + ; virtual bool equal(const void* left, const void* right) const = 0; virtual int cmp(const void* left, const void* right) const = 0; @@ -161,7 +163,7 @@ class ArrayTypeInfo : public TypeInfo { public: explicit ArrayTypeInfo(TypeInfo* item_type_info) : _item_type_info(item_type_info), _item_size(item_type_info->size()) {} - + ~ArrayTypeInfo() = default; inline bool equal(const void* left, const void* right) const override { auto l_value = reinterpret_cast(left); auto r_value = reinterpret_cast(right); @@ -248,7 +250,7 @@ public: dest_value->shallow_copy(src_value); } - inline void deep_copy(void* dest, const void* src, MemPool* mem_pool) const { + inline void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override { auto dest_value = reinterpret_cast(dest); auto src_value = reinterpret_cast(src); @@ -308,7 +310,7 @@ public: return OLAPStatus::OLAP_ERR_FUNC_NOT_IMPLEMENTED; } - OLAPStatus from_string(void* buf, const std::string& scan_key) const { + OLAPStatus from_string(void* buf, const std::string& scan_key) const override { return OLAPStatus::OLAP_ERR_FUNC_NOT_IMPLEMENTED; } diff --git a/be/src/olap/uint24.h b/be/src/olap/uint24.h index 965646975f..1605d893c9 100644 --- a/be/src/olap/uint24.h +++ b/be/src/olap/uint24.h @@ -22,6 +22,8 @@ #include #include +#include "olap/olap_common.h" + namespace doris { // 24bit int type, used to store date type in storage diff --git a/be/src/runtime/buffered_block_mgr2.h b/be/src/runtime/buffered_block_mgr2.h index 493398e8d9..cf923dcb39 100644 --- a/be/src/runtime/buffered_block_mgr2.h +++ b/be/src/runtime/buffered_block_mgr2.h @@ -384,7 +384,7 @@ public: } private: - friend struct Client; + friend class Client; // Descriptor for a single memory buffer in the pool. struct BufferDescriptor : public InternalQueue::Node { @@ -578,8 +578,6 @@ private: // Counters and timers to track behavior. std::unique_ptr _profile; - // These have a fixed value for the lifetime of the manager and show memory usage. - RuntimeProfile::Counter* _mem_tracker_counter; RuntimeProfile::Counter* _block_size_counter; // Total number of blocks created. diff --git a/be/src/runtime/bufferpool/buffer_pool.h b/be/src/runtime/bufferpool/buffer_pool.h index 3dae928782..46f95e41e5 100644 --- a/be/src/runtime/bufferpool/buffer_pool.h +++ b/be/src/runtime/bufferpool/buffer_pool.h @@ -148,7 +148,7 @@ class MemTracker; /// same Client, PageHandle or BufferHandle. class BufferPool : public CacheLineAligned { public: - class BufferAllocator; + struct BufferAllocator; class BufferHandle; class ClientHandle; class PageHandle; @@ -295,7 +295,7 @@ private: class Client; class FreeBufferArena; class PageList; - struct Page; + class Page; /// Allocator for allocating and freeing all buffer memory and managing lists of free /// buffers and clean pages. diff --git a/be/src/runtime/collection_value.cpp b/be/src/runtime/collection_value.cpp index 0729646e13..9568f9bbf0 100644 --- a/be/src/runtime/collection_value.cpp +++ b/be/src/runtime/collection_value.cpp @@ -46,7 +46,7 @@ Status type_check(PrimitiveType type) { case TYPE_NULL: break; default: - return Status::InvalidArgument("Type not implemented: " + type); + return Status::InvalidArgument(fmt::format("Type not implemented: {}", type)); } return Status::OK(); @@ -91,7 +91,7 @@ Status CollectionValue::init_collection(ObjectPool* pool, uint32_t size, Primiti } val->_length = size; - val->_null_signs = pool->add_array(new bool[size]{0}); + val->_null_signs = pool->add_array(new bool[size] {0}); val->_data = pool->add_array(new uint8_t[size * sizeof_type(child_type)]); return Status::OK(); diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp index 2e7041b493..162545ac15 100644 --- a/be/src/runtime/descriptors.cpp +++ b/be/src/runtime/descriptors.cpp @@ -24,9 +24,8 @@ #include "common/object_pool.h" #include "gen_cpp/Descriptors_types.h" #include "gen_cpp/descriptors.pb.h" - -#include "vec/core/columns_with_type_and_name.h" #include "vec/columns/column_nullable.h" +#include "vec/core/columns_with_type_and_name.h" #include "vec/data_types/data_type_nullable.h" namespace doris { @@ -87,7 +86,8 @@ void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const { vectorized::MutableColumnPtr SlotDescriptor::get_empty_mutable_column() const { auto data_column = type().get_data_type_ptr()->create_column(); if (is_nullable()) { - return doris::vectorized::ColumnNullable::create(std::move(data_column), doris::vectorized::ColumnUInt8::create()); + return doris::vectorized::ColumnNullable::create(std::move(data_column), + doris::vectorized::ColumnUInt8::create()); } return data_column; } @@ -109,7 +109,6 @@ std::string SlotDescriptor::debug_string() const { TableDescriptor::TableDescriptor(const TTableDescriptor& tdesc) : _name(tdesc.tableName), _database(tdesc.dbName), - _id(tdesc.id), _num_cols(tdesc.numCols), _num_clustering_cols(tdesc.numClusteringCols) {} diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index 708a154225..0bdc22b5ea 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -33,7 +33,7 @@ #include "runtime/types.h" namespace doris::vectorized { -class ColumnWithTypeAndName; +struct ColumnWithTypeAndName; } namespace doris { @@ -106,6 +106,7 @@ public: doris::vectorized::MutableColumnPtr get_empty_mutable_column() const; doris::vectorized::DataTypePtr get_data_type_ptr() const; + private: friend class DescriptorTbl; friend class TupleDescriptor; @@ -159,7 +160,6 @@ public: private: std::string _name; std::string _database; - TableId _id; int _num_cols; int _num_clustering_cols; }; @@ -380,9 +380,7 @@ public: // to GetAvgRowSize() int get_row_size() const; - int num_materialized_slots() const { - return _num_materialized_slots; - } + int num_materialized_slots() const { return _num_materialized_slots; } int num_null_slots() const { return _num_null_slots; } diff --git a/be/src/runtime/disk_io_mgr.cc b/be/src/runtime/disk_io_mgr.cc index 9b9b350fee..2060320ad2 100644 --- a/be/src/runtime/disk_io_mgr.cc +++ b/be/src/runtime/disk_io_mgr.cc @@ -728,7 +728,6 @@ char* DiskIoMgr::get_free_buffer(int64_t* buffer_size) { void DiskIoMgr::gc_io_buffers() { unique_lock lock(_free_buffers_lock); int buffers_freed = 0; - int bytes_freed = 0; for (int idx = 0; idx < _free_buffers.size(); ++idx) { for (list::iterator iter = _free_buffers[idx].begin(); iter != _free_buffers[idx].end(); ++iter) { @@ -738,7 +737,6 @@ void DiskIoMgr::gc_io_buffers() { delete[] * iter; ++buffers_freed; - bytes_freed += buffer_size; } _free_buffers[idx].clear(); } diff --git a/be/src/runtime/export_sink.h b/be/src/runtime/export_sink.h index c6bb7fe7cd..8dda722bc2 100644 --- a/be/src/runtime/export_sink.h +++ b/be/src/runtime/export_sink.h @@ -54,7 +54,7 @@ public: // hosts. Further send() calls are illegal after calling close(). virtual Status close(RuntimeState* state, Status exec_status) override; - virtual RuntimeProfile* profile() { return _profile; } + virtual RuntimeProfile* profile() override { return _profile; } private: Status open_file_writer(); diff --git a/be/src/runtime/file_result_writer.h b/be/src/runtime/file_result_writer.h index ea0c36312d..95e329cf6f 100644 --- a/be/src/runtime/file_result_writer.h +++ b/be/src/runtime/file_result_writer.h @@ -94,7 +94,7 @@ public: virtual Status close() override; // file result writer always return statistic result in one row - virtual int64_t get_written_rows() const { return 1; } + virtual int64_t get_written_rows() const override { return 1; } private: Status _write_csv_file(const RowBatch& batch); diff --git a/be/src/runtime/fold_constant_executor.h b/be/src/runtime/fold_constant_executor.h index 84c52f781d..1351687c03 100644 --- a/be/src/runtime/fold_constant_executor.h +++ b/be/src/runtime/fold_constant_executor.h @@ -15,15 +15,15 @@ // specific language governing permissions and limitations // under the License. -#include "runtime/tuple_row.h" -#include "util/runtime_profile.h" -#include "exprs/expr_context.h" -#include "exprs/expr.h" #include "common/object_pool.h" #include "common/status.h" -#include "runtime/exec_env.h" -#include "gen_cpp/internal_service.pb.h" +#include "exprs/expr.h" +#include "exprs/expr_context.h" #include "gen_cpp/PaloInternalService_types.h" +#include "gen_cpp/internal_service.pb.h" +#include "runtime/exec_env.h" +#include "runtime/tuple_row.h" +#include "util/runtime_profile.h" namespace doris { @@ -39,6 +39,7 @@ public: // fold constant vexpr Status fold_constant_vexpr(const TFoldConstantParams& params, PConstantExprResult* response); + private: // init runtime_state and mem_tracker Status _init(const TQueryGlobals& query_globals); @@ -53,9 +54,7 @@ private: std::shared_ptr _mem_tracker; RuntimeProfile* _runtime_profile = nullptr; std::unique_ptr _mem_pool; - ExecEnv* _exec_env; ObjectPool _pool; static TUniqueId _dummy_id; }; -} - +} // namespace doris diff --git a/be/src/runtime/load_path_mgr.h b/be/src/runtime/load_path_mgr.h index bf891571a5..8166b5d66a 100644 --- a/be/src/runtime/load_path_mgr.h +++ b/be/src/runtime/load_path_mgr.h @@ -64,7 +64,6 @@ private: std::vector _path_vec; int _idx; int _reserved_hours; - pthread_t _cleaner_id; std::string _error_log_dir; uint32_t _next_shard; uint32_t _error_path_next_shard; diff --git a/be/src/runtime/memory/chunk_allocator.h b/be/src/runtime/memory/chunk_allocator.h index d7eb22fefe..cfdcb0bb1f 100644 --- a/be/src/runtime/memory/chunk_allocator.h +++ b/be/src/runtime/memory/chunk_allocator.h @@ -25,7 +25,7 @@ namespace doris { -class Chunk; +struct Chunk; class ChunkArena; class MetricEntity; diff --git a/be/src/runtime/memory_scratch_sink.h b/be/src/runtime/memory_scratch_sink.h index 658aa0eb64..19c71f7c00 100644 --- a/be/src/runtime/memory_scratch_sink.h +++ b/be/src/runtime/memory_scratch_sink.h @@ -68,7 +68,6 @@ public: private: Status prepare_exprs(RuntimeState* state); - ObjectPool* _obj_pool; // Owned by the RuntimeState. const RowDescriptor& _row_desc; std::shared_ptr _arrow_schema; diff --git a/be/src/runtime/result_buffer_mgr.h b/be/src/runtime/result_buffer_mgr.h index 1af035ac6e..d6f8e60693 100644 --- a/be/src/runtime/result_buffer_mgr.h +++ b/be/src/runtime/result_buffer_mgr.h @@ -34,7 +34,7 @@ namespace doris { class TFetchDataResult; class BufferControlBlock; -class GetResultBatchCtx; +struct GetResultBatchCtx; class PUniqueId; // manage all result buffer control block in one backend diff --git a/be/src/runtime/result_file_sink.h b/be/src/runtime/result_file_sink.h index cef47cc10a..b68678df4c 100644 --- a/be/src/runtime/result_file_sink.h +++ b/be/src/runtime/result_file_sink.h @@ -35,7 +35,7 @@ class BufferControlBlock; class ExprContext; class ResultWriter; class MemTracker; -class ResultFileOptions; +struct ResultFileOptions; class ResultFileSink : public DataStreamSender { public: @@ -46,16 +46,16 @@ public: const std::vector& destinations, ObjectPool* pool, int sender_id, DescriptorTbl& descs); virtual ~ResultFileSink(); - virtual Status init(const TDataSink& thrift_sink); - virtual Status prepare(RuntimeState* state); - virtual Status open(RuntimeState* state); + virtual Status init(const TDataSink& thrift_sink) override; + virtual Status prepare(RuntimeState* state) override; + virtual Status open(RuntimeState* state) override; // send data in 'batch' to this backend stream mgr // Blocks until all rows in batch are placed in the buffer - virtual Status send(RuntimeState* state, RowBatch* batch); + virtual Status send(RuntimeState* state, RowBatch* batch) override; // Flush all buffered data and close all existing channels to destination // hosts. Further send() calls are illegal after calling close(). - virtual Status close(RuntimeState* state, Status exec_status); - virtual RuntimeProfile* profile() { return _profile; } + virtual Status close(RuntimeState* state, Status exec_status) override; + virtual RuntimeProfile* profile() override { return _profile; } void set_query_statistics(std::shared_ptr statistics) override; diff --git a/be/src/runtime/result_sink.h b/be/src/runtime/result_sink.h index 08fd6338c3..b94f4e0830 100644 --- a/be/src/runtime/result_sink.h +++ b/be/src/runtime/result_sink.h @@ -34,7 +34,7 @@ class BufferControlBlock; class ExprContext; class ResultWriter; class MemTracker; -class ResultFileOptions; +struct ResultFileOptions; namespace vectorized { class VExprContext; @@ -45,15 +45,15 @@ public: ResultSink(const RowDescriptor& row_desc, const std::vector& select_exprs, const TResultSink& sink, int buffer_size); virtual ~ResultSink(); - virtual Status prepare(RuntimeState* state); - virtual Status open(RuntimeState* state); + virtual Status prepare(RuntimeState* state) override; + virtual Status open(RuntimeState* state) override; // send data in 'batch' to this backend stream mgr // Blocks until all rows in batch are placed in the buffer - virtual Status send(RuntimeState* state, RowBatch* batch); + virtual Status send(RuntimeState* state, RowBatch* batch) override; // Flush all buffered data and close all existing channels to destination // hosts. Further send() calls are illegal after calling close(). - virtual Status close(RuntimeState* state, Status exec_status); - virtual RuntimeProfile* profile() { return _profile; } + virtual Status close(RuntimeState* state, Status exec_status) override; + virtual RuntimeProfile* profile() override { return _profile; } void set_query_statistics(std::shared_ptr statistics) override; @@ -63,7 +63,6 @@ private: // set file options when sink type is FILE std::unique_ptr _file_opts; - ObjectPool* _obj_pool; // Owned by the RuntimeState. const RowDescriptor& _row_desc; @@ -75,8 +74,6 @@ private: std::shared_ptr _writer; RuntimeProfile* _profile; // Allocated from _pool int _buf_size; // Allocated from _pool - - bool _is_vec; }; } // namespace doris diff --git a/be/src/runtime/result_writer.h b/be/src/runtime/result_writer.h index 5e2bf3349e..c52f489a40 100644 --- a/be/src/runtime/result_writer.h +++ b/be/src/runtime/result_writer.h @@ -25,7 +25,7 @@ namespace doris { class Status; class RowBatch; class RuntimeState; -class TypeDescriptor; +struct TypeDescriptor; namespace vectorized { class Block; diff --git a/be/src/runtime/row_batch.cpp b/be/src/runtime/row_batch.cpp index eb223d661d..b58615791e 100644 --- a/be/src/runtime/row_batch.cpp +++ b/be/src/runtime/row_batch.cpp @@ -29,7 +29,6 @@ #include "runtime/runtime_state.h" #include "runtime/string_value.h" #include "runtime/tuple_row.h" - #include "vec/columns/column_vector.h" #include "vec/core/block.h" @@ -58,13 +57,9 @@ RowBatch::RowBatch(const RowDescriptor& row_desc, int capacity, MemTracker* mem_ _tuple_ptrs_size = _capacity * _num_tuples_per_row * sizeof(Tuple*); DCHECK_GT(_tuple_ptrs_size, 0); // TODO: switch to Init() pattern so we can check memory limit and return Status. - if (config::enable_partitioned_aggregation) { - _mem_tracker->Consume(_tuple_ptrs_size); - _tuple_ptrs = (Tuple**)(malloc(_tuple_ptrs_size)); - DCHECK(_tuple_ptrs != nullptr); - } else { - _tuple_ptrs = (Tuple**)(_tuple_data_pool.allocate(_tuple_ptrs_size)); - } + _mem_tracker->Consume(_tuple_ptrs_size); + _tuple_ptrs = (Tuple**)(malloc(_tuple_ptrs_size)); + DCHECK(_tuple_ptrs != nullptr); } // TODO: we want our input_batch's tuple_data to come from our (not yet implemented) @@ -90,13 +85,9 @@ RowBatch::RowBatch(const RowDescriptor& row_desc, const PRowBatch& input_batch, _tuple_ptrs_size = _num_rows * _num_tuples_per_row * sizeof(Tuple*); DCHECK_GT(_tuple_ptrs_size, 0); // TODO: switch to Init() pattern so we can check memory limit and return Status. - if (config::enable_partitioned_aggregation) { - _mem_tracker->Consume(_tuple_ptrs_size); - _tuple_ptrs = (Tuple**)(malloc(_tuple_ptrs_size)); - DCHECK(_tuple_ptrs != nullptr); - } else { - _tuple_ptrs = (Tuple**)_tuple_data_pool.allocate(_tuple_ptrs_size); - } + _mem_tracker->Consume(_tuple_ptrs_size); + _tuple_ptrs = (Tuple**)(malloc(_tuple_ptrs_size)); + DCHECK(_tuple_ptrs != nullptr); char* tuple_data = nullptr; if (input_batch.is_compressed()) { @@ -234,12 +225,10 @@ void RowBatch::clear() { for (int i = 0; i < _blocks.size(); ++i) { _blocks[i]->del(); } - if (config::enable_partitioned_aggregation) { - DCHECK(_tuple_ptrs != nullptr); - free(_tuple_ptrs); - _mem_tracker->Release(_tuple_ptrs_size); - _tuple_ptrs = nullptr; - } + DCHECK(_tuple_ptrs != nullptr); + free(_tuple_ptrs); + _mem_tracker->Release(_tuple_ptrs_size); + _tuple_ptrs = nullptr; _cleared = true; } @@ -247,8 +236,8 @@ RowBatch::~RowBatch() { clear(); } -Status RowBatch::serialize(PRowBatch* output_batch, size_t* uncompressed_size, size_t* compressed_size, - std::string* allocated_buf) { +Status RowBatch::serialize(PRowBatch* output_batch, size_t* uncompressed_size, + size_t* compressed_size, std::string* allocated_buf) { // num_rows output_batch->set_num_rows(_num_rows); // row_tuples @@ -333,8 +322,11 @@ Status RowBatch::serialize(PRowBatch* output_batch, size_t* uncompressed_size, s *compressed_size = pb_size; if (pb_size > std::numeric_limits::max()) { // the protobuf has a hard limit of 2GB for serialized data. - return Status::InternalError(fmt::format("The rowbatch is large than 2GB({}), can not send by Protobuf. " - "please set BE config 'transfer_data_by_brpc_attachment' to true and restart BE.", pb_size)); + return Status::InternalError( + fmt::format("The rowbatch is large than 2GB({}), can not send by Protobuf. " + "please set BE config 'transfer_data_by_brpc_attachment' to true " + "and restart BE.", + pb_size)); } } else { *uncompressed_size = pb_size + size; @@ -343,12 +335,11 @@ Status RowBatch::serialize(PRowBatch* output_batch, size_t* uncompressed_size, s return Status::OK(); } -// when row from files can't fill into tuple with schema limitation, increase the _num_uncommitted_rows in row batch, +// when row from files can't fill into tuple with schema limitation, increase the _num_uncommitted_rows in row batch, void RowBatch::increase_uncommitted_rows() { _num_uncommitted_rows++; } - void RowBatch::add_io_buffer(DiskIoMgr::BufferDescriptor* buffer) { DCHECK(buffer != nullptr); _io_buffers.push_back(buffer); @@ -411,9 +402,6 @@ void RowBatch::reset() { } _blocks.clear(); _auxiliary_mem_usage = 0; - if (!config::enable_partitioned_aggregation) { - _tuple_ptrs = (Tuple**)(_tuple_data_pool.allocate(_tuple_ptrs_size)); - } _need_to_return = false; _flush = FlushMode::NO_FLUSH_RESOURCES; _needs_deep_copy = false; @@ -557,14 +545,8 @@ void RowBatch::acquire_state(RowBatch* src) { _num_rows = src->_num_rows; _capacity = src->_capacity; _need_to_return = src->_need_to_return; - if (!config::enable_partitioned_aggregation) { - // Tuple pointers are allocated from tuple_data_pool_ so are transferred. - _tuple_ptrs = src->_tuple_ptrs; - src->_tuple_ptrs = nullptr; - } else { - // tuple_ptrs_ were allocated with malloc so can be swapped between batches. - std::swap(_tuple_ptrs, src->_tuple_ptrs); - } + // tuple_ptrs_ were allocated with malloc so can be swapped between batches. + std::swap(_tuple_ptrs, src->_tuple_ptrs); src->transfer_resource_ownership(this); } diff --git a/be/src/runtime/small_file_mgr.cpp b/be/src/runtime/small_file_mgr.cpp index e4fa8f388d..fc0aa2056a 100644 --- a/be/src/runtime/small_file_mgr.cpp +++ b/be/src/runtime/small_file_mgr.cpp @@ -84,7 +84,8 @@ Status SmallFileMgr::_load_single_file(const std::string& path, const std::strin std::string md5 = parts[1]; if (_file_cache.find(file_id) != _file_cache.end()) { - return Status::InternalError("File with same id is already been loaded: " + file_id); + return Status::InternalError( + fmt::format("File with same id is already been loaded: {}", file_id)); } std::string file_md5; diff --git a/be/src/runtime/snapshot_loader.h b/be/src/runtime/snapshot_loader.h index 0d2386b6e9..9b1ebd2f50 100644 --- a/be/src/runtime/snapshot_loader.h +++ b/be/src/runtime/snapshot_loader.h @@ -33,7 +33,7 @@ namespace doris { class ExecEnv; class StorageBackend; -class FileStat; +struct FileStat; /* * Upload: diff --git a/be/src/runtime/stream_load/stream_load_pipe.h b/be/src/runtime/stream_load/stream_load_pipe.h index cab7908301..7f9cb5110e 100644 --- a/be/src/runtime/stream_load/stream_load_pipe.h +++ b/be/src/runtime/stream_load/stream_load_pipe.h @@ -22,10 +22,10 @@ #include #include "exec/file_reader.h" +#include "gen_cpp/internal_service.pb.h" #include "runtime/message_body_sink.h" #include "util/bit_util.h" #include "util/byte_buffer.h" -#include "gen_cpp/internal_service.pb.h" namespace doris { @@ -148,7 +148,7 @@ public: return Status::OK(); } - Status readat(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) { + Status readat(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) override { return Status::InternalError("Not implemented"); } diff --git a/be/src/service/backend_service.h b/be/src/service/backend_service.h index 36e908e65f..3c9b3bd13c 100644 --- a/be/src/service/backend_service.h +++ b/be/src/service/backend_service.h @@ -79,48 +79,53 @@ public: // Agent service virtual void submit_tasks(TAgentResult& return_value, - const std::vector& tasks) { + const std::vector& tasks) override { _agent_server->submit_tasks(return_value, tasks); } virtual void make_snapshot(TAgentResult& return_value, - const TSnapshotRequest& snapshot_request) { + const TSnapshotRequest& snapshot_request) override { _agent_server->make_snapshot(return_value, snapshot_request); } - virtual void release_snapshot(TAgentResult& return_value, const std::string& snapshot_path) { + virtual void release_snapshot(TAgentResult& return_value, + const std::string& snapshot_path) override { _agent_server->release_snapshot(return_value, snapshot_path); } - virtual void publish_cluster_state(TAgentResult& result, const TAgentPublishRequest& request) { + virtual void publish_cluster_state(TAgentResult& result, + const TAgentPublishRequest& request) override { _agent_server->publish_cluster_state(result, request); } - virtual void submit_etl_task(TAgentResult& result, const TMiniLoadEtlTaskRequest& request) { + virtual void submit_etl_task(TAgentResult& result, + const TMiniLoadEtlTaskRequest& request) override { VLOG_RPC << "submit_etl_task. request is " << apache::thrift::ThriftDebugString(request).c_str(); _agent_server->submit_etl_task(result, request); } virtual void get_etl_status(TMiniLoadEtlStatusResult& result, - const TMiniLoadEtlStatusRequest& request) { + const TMiniLoadEtlStatusRequest& request) override { _agent_server->get_etl_status(result, request); } - virtual void delete_etl_files(TAgentResult& result, const TDeleteEtlFilesRequest& request) { + virtual void delete_etl_files(TAgentResult& result, + const TDeleteEtlFilesRequest& request) override { _agent_server->delete_etl_files(result, request); } // DorisServer service virtual void exec_plan_fragment(TExecPlanFragmentResult& return_val, - const TExecPlanFragmentParams& params); + const TExecPlanFragmentParams& params) override; virtual void cancel_plan_fragment(TCancelPlanFragmentResult& return_val, - const TCancelPlanFragmentParams& params); + const TCancelPlanFragmentParams& params) override; - virtual void transmit_data(TTransmitDataResult& return_val, const TTransmitDataParams& params); + virtual void transmit_data(TTransmitDataResult& return_val, + const TTransmitDataParams& params) override; - virtual void fetch_data(TFetchDataResult& return_val, const TFetchDataParams& params); + virtual void fetch_data(TFetchDataResult& return_val, const TFetchDataParams& params) override; void submit_export_task(TStatus& t_status, const TExportTaskRequest& request) override; @@ -138,17 +143,17 @@ public: const std::vector& tasks) override; // used for external service, open means start the scan procedure - virtual void open_scanner(TScanOpenResult& result_, const TScanOpenParams& params); + virtual void open_scanner(TScanOpenResult& result_, const TScanOpenParams& params) override; // used for external service, external use getNext to fetch data batch after batch until eos = true - virtual void get_next(TScanBatchResult& result_, const TScanNextBatchParams& params); + virtual void get_next(TScanBatchResult& result_, const TScanNextBatchParams& params) override; // used for external service, close some context and release resource related with this context - virtual void close_scanner(TScanCloseResult& result_, const TScanCloseParams& params); + virtual void close_scanner(TScanCloseResult& result_, const TScanCloseParams& params) override; virtual void get_stream_load_record(TStreamLoadRecordResult& result, const int64_t last_stream_record_time) override; - + virtual void clean_trash() override; private: diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index be7519f2e2..c50d14ad72 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -91,7 +91,7 @@ Status HttpService::start() { error_log_download_action); // Register BE health action - HealthAction* health_action = _pool.add(new HealthAction(_env)); + HealthAction* health_action = _pool.add(new HealthAction()); _ev_http_server->register_handler(HttpMethod::GET, "/api/health", health_action); // Register Tablets Info action @@ -124,7 +124,7 @@ Status HttpService::start() { #ifndef BE_TEST // Register BE checksum action - ChecksumAction* checksum_action = _pool.add(new ChecksumAction(_env)); + ChecksumAction* checksum_action = _pool.add(new ChecksumAction()); _ev_http_server->register_handler(HttpMethod::GET, "/api/checksum", checksum_action); // Register BE reload tablet action @@ -136,7 +136,7 @@ Status HttpService::start() { restore_tablet_action); // Register BE snapshot action - SnapshotAction* snapshot_action = _pool.add(new SnapshotAction(_env)); + SnapshotAction* snapshot_action = _pool.add(new SnapshotAction()); _ev_http_server->register_handler(HttpMethod::GET, "/api/snapshot", snapshot_action); #endif diff --git a/be/src/service/internal_service.h b/be/src/service/internal_service.h index 702bec1608..c4073bf86e 100644 --- a/be/src/service/internal_service.h +++ b/be/src/service/internal_service.h @@ -97,11 +97,11 @@ public: ::google::protobuf::Closure* done) override; void send_data(google::protobuf::RpcController* controller, const PSendDataRequest* request, - PSendDataResult* response, google::protobuf::Closure* done); + PSendDataResult* response, google::protobuf::Closure* done) override; void commit(google::protobuf::RpcController* controller, const PCommitRequest* request, - PCommitResult* response, google::protobuf::Closure* done); + PCommitResult* response, google::protobuf::Closure* done) override; void rollback(google::protobuf::RpcController* controller, const PRollbackRequest* request, - PRollbackResult* response, google::protobuf::Closure* done); + PRollbackResult* response, google::protobuf::Closure* done) override; void fold_constant_expr(google::protobuf::RpcController* controller, const PConstantExprRequest* request, PConstantExprResult* response, google::protobuf::Closure* done) override; diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h index 141219afec..c262c8c061 100644 --- a/be/src/udf/udf.h +++ b/be/src/udf/udf.h @@ -28,11 +28,11 @@ // object serves as the interface object between the UDF/UDA and the doris process. namespace doris { class FunctionContextImpl; -class ColumnPtrWrapper; +struct ColumnPtrWrapper; struct StringValue; -struct BitmapValue; -struct DecimalV2Value; -struct DateTimeValue; +class BitmapValue; +class DecimalV2Value; +class DateTimeValue; struct CollectionValue; } // namespace doris diff --git a/be/src/udf/udf_internal.h b/be/src/udf/udf_internal.h index 36cf8ad474..f4c99fdccf 100644 --- a/be/src/udf/udf_internal.h +++ b/be/src/udf/udf_internal.h @@ -32,7 +32,7 @@ namespace doris { class FreePool; class MemPool; class RuntimeState; -class ColumnPtrWrapper; +struct ColumnPtrWrapper; class PFunctionContext; // This class actually implements the interface of FunctionContext. This is split to diff --git a/be/src/util/arrow/row_batch.cpp b/be/src/util/arrow/row_batch.cpp index 3b0e0ca1db..b7432e04e3 100644 --- a/be/src/util/arrow/row_batch.cpp +++ b/be/src/util/arrow/row_batch.cpp @@ -205,7 +205,7 @@ public: arrow::Status Visit(const arrow::StringType& type) override { arrow::StringBuilder builder(_pool); size_t num_rows = _batch.num_rows(); - builder.Reserve(num_rows); + ARROW_RETURN_NOT_OK(builder.Reserve(num_rows)); for (size_t i = 0; i < num_rows; ++i) { bool is_null = _cur_slot_ref->is_null_bit_set(_batch.get_row(i)); if (is_null) { @@ -258,7 +258,7 @@ public: std::make_shared(27, 9); arrow::Decimal128Builder builder(s_decimal_ptr, _pool); size_t num_rows = _batch.num_rows(); - builder.Reserve(num_rows); + ARROW_RETURN_NOT_OK(builder.Reserve(num_rows)); for (size_t i = 0; i < num_rows; ++i) { bool is_null = _cur_slot_ref->is_null_bit_set(_batch.get_row(i)); if (is_null) { @@ -275,10 +275,10 @@ public: return builder.Finish(&_arrays[_cur_field_idx]); } // process boolean - arrow::Status Visit(const arrow::BooleanType& type) { + arrow::Status Visit(const arrow::BooleanType& type) override { arrow::BooleanBuilder builder(_pool); size_t num_rows = _batch.num_rows(); - builder.Reserve(num_rows); + ARROW_RETURN_NOT_OK(builder.Reserve(num_rows)); for (size_t i = 0; i < num_rows; ++i) { bool is_null = _cur_slot_ref->is_null_bit_set(_batch.get_row(i)); if (is_null) { @@ -300,7 +300,7 @@ private: arrow::NumericBuilder builder(_pool); size_t num_rows = _batch.num_rows(); - builder.Reserve(num_rows); + ARROW_RETURN_NOT_OK(builder.Reserve(num_rows)); for (size_t i = 0; i < num_rows; ++i) { bool is_null = _cur_slot_ref->is_null_bit_set(_batch.get_row(i)); if (is_null) { @@ -490,7 +490,12 @@ Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::strin msg << "write record batch failure, reason: " << a_st.ToString(); return Status::InternalError(msg.str()); } - record_batch_writer->Close(); + a_st = record_batch_writer->Close(); + if (!a_st.ok()) { + std::stringstream msg; + msg << "Close failed, reason: " << a_st.ToString(); + return Status::InternalError(msg.str()); + } auto finish_res = sink->Finish(); if (!finish_res.ok()) { std::stringstream msg; @@ -499,7 +504,12 @@ Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::strin } *result = finish_res.ValueOrDie()->ToString(); // close the sink - sink->Close(); + a_st = sink->Close(); + if (!a_st.ok()) { + std::stringstream msg; + msg << "Close failed, reason: " << a_st.ToString(); + return Status::InternalError(msg.str()); + } return Status::OK(); } diff --git a/be/src/util/arrow/row_block.cpp b/be/src/util/arrow/row_block.cpp index ebefb825e7..569ca67389 100644 --- a/be/src/util/arrow/row_block.cpp +++ b/be/src/util/arrow/row_block.cpp @@ -171,7 +171,7 @@ private: _visit(const T& type) { arrow::NumericBuilder builder(_pool); size_t num_rows = _block.num_rows(); - builder.Reserve(num_rows); + ARROW_RETURN_NOT_OK(builder.Reserve(num_rows)); auto column_block = _block.column_block(_cur_field_idx); for (size_t i = 0; i < num_rows; ++i) { diff --git a/be/src/util/cidr.cpp b/be/src/util/cidr.cpp index e222b78924..bf8e888b9f 100644 --- a/be/src/util/cidr.cpp +++ b/be/src/util/cidr.cpp @@ -55,15 +55,13 @@ bool CIDR::reset(const std::string& cidr_str) { char* endptr = nullptr; int32_t mask_length = strtol(cidr_items[1].c_str(), &endptr, 10); - if ((errno == ERANGE && (mask_length == LONG_MAX || mask_length == LONG_MIN)) || - (errno != 0 && mask_length == 0)) { + if (errno != 0 && mask_length == 0) { char errmsg[64]; // Ignore unused return value - if (strerror_r(errno, errmsg, 64)) - ; + auto ret = strerror_r(errno, errmsg, 64); LOG(WARNING) << "wrong CIDR mask format. network=" << cidr_str << ", mask_length=" << mask_length << ", errno=" << errno - << ", errmsg=" << errmsg; + << ", errmsg=" << errmsg << ", strerror_r returns=" << ret; return false; } if (mask_length <= 0 || mask_length > 32) { diff --git a/be/src/util/crc32c.cpp b/be/src/util/crc32c.cpp index 0a13b11d0b..56e9fb83c1 100644 --- a/be/src/util/crc32c.cpp +++ b/be/src/util/crc32c.cpp @@ -191,7 +191,7 @@ static inline uint64_t LE_LOAD64(const uint8_t* p) { } #endif -static inline void Slow_CRC32(uint64_t* l, uint8_t const** p) { +[[maybe_unused]] static inline void Slow_CRC32(uint64_t* l, uint8_t const** p) { uint32_t c = static_cast(*l ^ LE_LOAD32(*p)); *p += 4; *l = table3_[c & 0xff] ^ table2_[(c >> 8) & 0xff] ^ table1_[(c >> 16) & 0xff] ^ diff --git a/be/src/util/date_func.h b/be/src/util/date_func.h index 53c02af817..f3cb926a62 100644 --- a/be/src/util/date_func.h +++ b/be/src/util/date_func.h @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#pragma +#pragma once #include #include diff --git a/be/src/util/metrics.h b/be/src/util/metrics.h index fca08cfd87..c5b005f0e6 100644 --- a/be/src/util/metrics.h +++ b/be/src/util/metrics.h @@ -32,8 +32,8 @@ #include "common/config.h" #include "util/core_local.h" -#include "util/spinlock.h" #include "util/histogram.h" +#include "util/spinlock.h" namespace doris { @@ -70,8 +70,7 @@ public: Metric() {} virtual ~Metric() {} virtual std::string to_string() const = 0; - virtual std::string to_prometheus(const std::string& display_name, - const Labels& entity_labels, + virtual std::string to_prometheus(const std::string& display_name, const Labels& entity_labels, const Labels& metric_labels) const; virtual rj::Value to_json_value(rj::Document::AllocatorType& allocator) const = 0; @@ -94,7 +93,9 @@ public: void set_value(const T& value) { _value.store(value); } - rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { return rj::Value(value()); } + rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { + return rj::Value(value()); + } protected: std::atomic _value; @@ -123,7 +124,9 @@ public: _value = value; } - rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { return rj::Value(value()); } + rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { + return rj::Value(value()); + } protected: // We use spinlock instead of std::atomic is because atomic don't support @@ -159,7 +162,9 @@ public: void increment(const T& delta) { __sync_fetch_and_add(_value.access(), delta); } - rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { return rj::Value(value()); } + rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override { + return rj::Value(value()); + } protected: CoreLocalValue _value; @@ -188,8 +193,7 @@ public: double average() const; double standard_deviation() const; std::string to_string() const override; - std::string to_prometheus(const std::string& display_name, - const Labels& entity_labels, + std::string to_prometheus(const std::string& display_name, const Labels& entity_labels, const Labels& metric_labels) const override; rj::Value to_json_value(rj::Document::AllocatorType& allocator) const override; @@ -363,8 +367,8 @@ public: private: friend class MetricRegistry; - friend class MetricEntityHash; - friend class MetricEntityEqualTo; + friend struct MetricEntityHash; + friend struct MetricEntityEqualTo; MetricEntityType _type; std::string _name; diff --git a/be/src/util/mysql_load_error_hub.cpp b/be/src/util/mysql_load_error_hub.cpp index 920ceead2b..daf484ef9c 100644 --- a/be/src/util/mysql_load_error_hub.cpp +++ b/be/src/util/mysql_load_error_hub.cpp @@ -70,7 +70,7 @@ Status MysqlLoadErrorHub::write_mysql() { return st; } - Defer close_mysql_conn{[=]() { mysql_close(my_conn); }}; + Defer close_mysql_conn {[=]() { mysql_close(my_conn); }}; Status status; std::stringstream sql_stream; @@ -96,12 +96,10 @@ Status MysqlLoadErrorHub::write_mysql() { Status MysqlLoadErrorHub::gen_sql(MYSQL* my_conn, const LoadErrorHub::ErrorMsg& error_msg, std::stringstream* sql_stream) { char* sql_start = &_escape_buff[0]; - char* sql_end = sql_start; size_t msg_size = error_msg.msg.size(); if (msg_size > EXPORTER_MAX_LINE_SIZE) { msg_size = EXPORTER_MAX_LINE_SIZE; } - sql_end += mysql_real_escape_string(my_conn, sql_start, error_msg.msg.c_str(), msg_size); (*sql_stream) << "insert into " << _info.table << " (job_id, error_msg) values(" << error_msg.job_id << ", '" << sql_start << "'); "; diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp index 9c9846201b..33844d0d54 100644 --- a/be/src/util/mysql_row_buffer.cpp +++ b/be/src/util/mysql_row_buffer.cpp @@ -29,12 +29,16 @@ namespace doris { +static uint8_t NEXT_TWO_BYTE = 252; +static uint8_t NEXT_THREE_BYTE = 253; +static uint8_t NEXT_EIGHT_BYTE = 254; + // the first byte: // <= 250: length // = 251: nullptr // = 252: the next two byte is length // = 253: the next three byte is length -// = 254: the next eighth byte is length +// = 254: the next eight byte is length static char* pack_vlen(char* packet, uint64_t length) { if (length < 251ULL) { int1store(packet, length); @@ -43,18 +47,18 @@ static char* pack_vlen(char* packet, uint64_t length) { /* 251 is reserved for nullptr */ if (length < 65536ULL) { - *packet++ = 252; + *packet++ = NEXT_TWO_BYTE; int2store(packet, length); return packet + 2; } if (length < 16777216ULL) { - *packet++ = 253; + *packet++ = NEXT_THREE_BYTE; int3store(packet, length); return packet + 3; } - *packet++ = 254; + *packet++ = NEXT_EIGHT_BYTE; int8store(packet, length); return packet + 8; } @@ -73,7 +77,7 @@ MysqlRowBuffer::~MysqlRowBuffer() { void MysqlRowBuffer::open_dynamic_mode() { if (!_dynamic_mode) { - *_pos++ = 254; + *_pos++ = NEXT_EIGHT_BYTE; // write length when dynamic mode close _len_pos = _pos; _pos = _pos + 8; diff --git a/be/src/util/system_metrics.h b/be/src/util/system_metrics.h index e9c2458376..87dc2a368f 100644 --- a/be/src/util/system_metrics.h +++ b/be/src/util/system_metrics.h @@ -23,13 +23,13 @@ namespace doris { -class CpuMetrics; -class MemoryMetrics; -class DiskMetrics; -class NetworkMetrics; -class FileDescriptorMetrics; -class SnmpMetrics; -class LoadAverageMetrics; +struct CpuMetrics; +struct MemoryMetrics; +struct DiskMetrics; +struct NetworkMetrics; +struct FileDescriptorMetrics; +struct SnmpMetrics; +struct LoadAverageMetrics; class SystemMetrics { public: diff --git a/be/src/util/topn_counter.h b/be/src/util/topn_counter.h index be6bd9c54a..3be5325d97 100644 --- a/be/src/util/topn_counter.h +++ b/be/src/util/topn_counter.h @@ -31,7 +31,7 @@ namespace doris { static const uint32_t DEFAULT_SPACE_EXPAND_RATE = 50; -class Slice; +struct Slice; class Counter { public: diff --git a/be/src/util/tuple_row_zorder_compare.h b/be/src/util/tuple_row_zorder_compare.h index e44008c4f6..043493bfd2 100644 --- a/be/src/util/tuple_row_zorder_compare.h +++ b/be/src/util/tuple_row_zorder_compare.h @@ -21,24 +21,24 @@ #include "exec/sort_exec_exprs.h" #include "exprs/expr.h" #include "exprs/expr_context.h" +#include "olap/row.h" +#include "olap/row_cursor.h" +#include "olap/schema.h" #include "runtime/descriptors.h" #include "runtime/raw_value.h" #include "runtime/tuple.h" #include "runtime/tuple_row.h" -#include "olap/schema.h" -#include "olap/row.h" -#include "olap/row_cursor.h" namespace doris { class RowComparator { public: - RowComparator()=default; + RowComparator() = default; RowComparator(Schema* schema); + virtual ~RowComparator() = default; virtual int operator()(const char* left, const char* right) const; }; -class TupleRowZOrderComparator: public RowComparator { - +class TupleRowZOrderComparator : public RowComparator { private: typedef __uint128_t uint128_t; int _max_col_size = 0; @@ -49,22 +49,23 @@ public: TupleRowZOrderComparator(); TupleRowZOrderComparator(int sort_col_num); TupleRowZOrderComparator(Schema* schema, int sort_col_num); + virtual ~TupleRowZOrderComparator() {} int compare(const char* lhs, const char* rhs) const; void max_col_size(const RowCursor& rc); int compare_row(const RowCursor& lhs, const RowCursor& rhs); - template + template int compare_based_on_size(LhsRowType& lhs, LhsRowType& rhs) const; - template - U get_shared_representation(const void *val, FieldType type) const; - template + template + U get_shared_representation(const void* val, FieldType type) const; + template U inline get_shared_int_representation(const T val, U mask) const; - template - U inline get_shared_float_representation(const void *val, U mask) const; - template - U inline get_shared_string_representation(const char *char_ptr, int length) const; + template + U inline get_shared_float_representation(const void* val, U mask) const; + template + U inline get_shared_string_representation(const char* char_ptr, int length) const; virtual int operator()(const char* lhs, const char* rhs) const; int get_type_byte_size(FieldType type) const; }; -} +} // namespace doris #endif //DORIS_TUPLE_ROW_ZORDER_COMPARE_H \ No newline at end of file diff --git a/be/src/vec/columns/column_complex.h b/be/src/vec/columns/column_complex.h index 45c0579822..0cdf3a34f4 100644 --- a/be/src/vec/columns/column_complex.h +++ b/be/src/vec/columns/column_complex.h @@ -142,7 +142,7 @@ public: LOG(FATAL) << "get field not implemented"; } - void insert_range_from(const IColumn& src, size_t start, size_t length) { + void insert_range_from(const IColumn& src, size_t start, size_t length) override { auto& col = static_cast(src); auto& src_data = col.get_data(); auto st = src_data.begin() + start; @@ -150,7 +150,8 @@ public: data.insert(data.end(), st, ed); } - void insert_indices_from(const IColumn& src, const int* indices_begin, const int* indices_end) override { + void insert_indices_from(const IColumn& src, const int* indices_begin, + const int* indices_end) override { const Self& src_vec = assert_cast(src); data.reserve(size() + (indices_end - indices_begin)); for (auto x = indices_begin; x != indices_end; ++x) { @@ -158,27 +159,27 @@ public: } } - void pop_back(size_t n) { data.erase(data.end() - n, data.end()); } + void pop_back(size_t n) override { data.erase(data.end() - n, data.end()); } // it's impossable to use ComplexType as key , so we don't have to implemnt them [[noreturn]] StringRef serialize_value_into_arena(size_t n, Arena& arena, - char const*& begin) const { + char const*& begin) const override { LOG(FATAL) << "serialize_value_into_arena not implemented"; } - [[noreturn]] const char* deserialize_and_insert_from_arena(const char* pos) { + [[noreturn]] const char* deserialize_and_insert_from_arena(const char* pos) override { LOG(FATAL) << "deserialize_and_insert_from_arena not implemented"; } - void update_hash_with_value(size_t n, SipHash& hash) const { + void update_hash_with_value(size_t n, SipHash& hash) const override { // TODO add hash function } [[noreturn]] int compare_at(size_t n, size_t m, const IColumn& rhs, - int nan_direction_hint) const { + int nan_direction_hint) const override { LOG(FATAL) << "compare_at not implemented"; } - void get_extremes(Field& min, Field& max) const { + void get_extremes(Field& min, Field& max) const override { LOG(FATAL) << "get_extremes not implemented"; } @@ -319,7 +320,8 @@ ColumnPtr ColumnComplexType::replicate(const IColumn::Offsets& offsets) const } template -void ColumnComplexType::replicate(const uint32_t* counts, size_t target_size, IColumn& column) const { +void ColumnComplexType::replicate(const uint32_t* counts, size_t target_size, + IColumn& column) const { size_t size = data.size(); if (0 == size) return; diff --git a/be/src/vec/columns/column_vector.h b/be/src/vec/columns/column_vector.h index ce1d479419..c6b2311597 100644 --- a/be/src/vec/columns/column_vector.h +++ b/be/src/vec/columns/column_vector.h @@ -22,12 +22,12 @@ #include +#include "olap/uint24.h" #include "vec/columns/column.h" #include "vec/columns/column_impl.h" #include "vec/columns/column_vector_helper.h" #include "vec/common/unaligned.h" #include "vec/core/field.h" -#include "olap/uint24.h" namespace doris::vectorized { @@ -120,8 +120,9 @@ private: /// Sugar constructor. ColumnVector(std::initializer_list il) : data {il} {} - void insert_res_column(const uint16_t* sel, size_t sel_size, vectorized::ColumnVector* res_ptr) { - auto& res_data = res_ptr->data; + void insert_res_column(const uint16_t* sel, size_t sel_size, + vectorized::ColumnVector* res_ptr) { + auto& res_data = res_ptr->data; DCHECK(res_data.empty()); res_data.reserve(sel_size); T* t = (T*)res_data.get_end_ptr(); @@ -139,7 +140,7 @@ private: } res_val_ptr += num; data.set_end_ptr(res_val_ptr); - } + } public: bool is_numeric() const override { return IsNumber; } @@ -165,7 +166,7 @@ public: res_ptr += num * sizeof(T); data.set_end_ptr(res_ptr); } - + void insert_date_column(const char* data_ptr, size_t num) { size_t value_size = sizeof(uint24_t); for (int i = 0; i < num; i++) { @@ -181,7 +182,7 @@ public: data.push_back_without_reserve(date); } } - + void insert_datetime_column(const char* data_ptr, size_t num) { size_t value_size = sizeof(uint64_t); for (int i = 0; i < num; i++) { @@ -191,7 +192,7 @@ public: data.push_back_without_reserve(date); } } - + /* use by date, datetime, basic type */ @@ -270,7 +271,8 @@ public: void insert_range_from(const IColumn& src, size_t start, size_t length) override; - void insert_indices_from(const IColumn& src, const int* indices_begin, const int* indices_end) override; + void insert_indices_from(const IColumn& src, const int* indices_begin, + const int* indices_end) override; void fill(const value_type& element, size_t num) { auto old_size = data.size(); diff --git a/be/src/vec/columns/predicate_column.h b/be/src/vec/columns/predicate_column.h index 8bc12427af..890d9a2bc1 100644 --- a/be/src/vec/columns/predicate_column.h +++ b/be/src/vec/columns/predicate_column.h @@ -17,14 +17,13 @@ #pragma once -#include "vec/columns/column.h" -#include "vec/columns/column_impl.h" - -#include "runtime/string_value.h" #include "olap/decimal12.h" #include "olap/uint24.h" -#include "vec/columns/column_string.h" +#include "runtime/string_value.h" +#include "vec/columns/column.h" #include "vec/columns/column_decimal.h" +#include "vec/columns/column_impl.h" +#include "vec/columns/column_string.h" #include "vec/columns/column_vector.h" #include "vec/core/types.h" @@ -32,7 +31,7 @@ namespace doris::vectorized { /** * used to keep predicate column in storage layer - * + * * T = predicate column type */ template @@ -56,7 +55,8 @@ private: return value; } - void insert_date_to_res_column(const uint16_t* sel, size_t sel_size, vectorized::ColumnVector* res_ptr) { + void insert_date_to_res_column(const uint16_t* sel, size_t sel_size, + vectorized::ColumnVector* res_ptr) { for (size_t i = 0; i < sel_size; i++) { VecDateTimeValue date; date.from_olap_date(get_date_at(sel[i])); @@ -64,7 +64,8 @@ private: } } - void insert_datetime_to_res_column(const uint16_t* sel, size_t sel_size, vectorized::ColumnVector* res_ptr) { + void insert_datetime_to_res_column(const uint16_t* sel, size_t sel_size, + vectorized::ColumnVector* res_ptr) { for (size_t i = 0; i < sel_size; i++) { uint64_t value = data[sel[i]]; vectorized::VecDateTimeValue date(value); @@ -72,7 +73,8 @@ private: } } - void insert_string_to_res_column(const uint16_t* sel, size_t sel_size, vectorized::ColumnString* res_ptr) { + void insert_string_to_res_column(const uint16_t* sel, size_t sel_size, + vectorized::ColumnString* res_ptr) { for (size_t i = 0; i < sel_size; i++) { uint16_t n = sel[i]; auto& sv = reinterpret_cast(data[n]); @@ -80,7 +82,8 @@ private: } } - void insert_decimal_to_res_column(const uint16_t* sel, size_t sel_size, vectorized::ColumnDecimal* res_ptr) { + void insert_decimal_to_res_column(const uint16_t* sel, size_t sel_size, + vectorized::ColumnDecimal* res_ptr) { for (size_t i = 0; i < sel_size; i++) { uint16_t n = sel[i]; auto& dv = reinterpret_cast(data[n]); @@ -90,7 +93,8 @@ private: } template - void insert_default_value_res_column(const uint16_t* sel, size_t sel_size, vectorized::ColumnVector* res_ptr) { + void insert_default_value_res_column(const uint16_t* sel, size_t sel_size, + vectorized::ColumnVector* res_ptr) { static_assert(std::is_same_v); auto& res_data = res_ptr->get_data(); DCHECK(res_data.empty()); @@ -102,7 +106,8 @@ private: res_data.set_end_ptr(y + sel_size); } - void insert_byte_to_res_column(const uint16_t* sel, size_t sel_size, vectorized::IColumn* res_ptr) { + void insert_byte_to_res_column(const uint16_t* sel, size_t sel_size, + vectorized::IColumn* res_ptr) { for (size_t i = 0; i < sel_size; i++) { uint16_t n = sel[i]; char* ch_val = reinterpret_cast(&data[n]); @@ -120,7 +125,7 @@ private: res_val_ptr += num; data.set_end_ptr(res_val_ptr); } - + void insert_many_in_copy_way(const char* data_ptr, size_t num) { char* res_ptr = (char*)data.get_end_ptr(); memcpy(res_ptr, data_ptr, num * sizeof(T)); @@ -139,20 +144,21 @@ public: size_t size() const override { return data.size(); } - [[noreturn]] StringRef get_data_at(size_t n) const override { - LOG(FATAL) << "get_data_at not supported in PredicateColumnType"; + [[noreturn]] StringRef get_data_at(size_t n) const override { + LOG(FATAL) << "get_data_at not supported in PredicateColumnType"; } void insert_from(const IColumn& src, size_t n) override { - LOG(FATAL) << "insert_from not supported in PredicateColumnType"; + LOG(FATAL) << "insert_from not supported in PredicateColumnType"; } void insert_range_from(const IColumn& src, size_t start, size_t length) override { - LOG(FATAL) << "insert_range_from not supported in PredicateColumnType"; + LOG(FATAL) << "insert_range_from not supported in PredicateColumnType"; } - void insert_indices_from(const IColumn& src, const int* indices_begin, const int* indices_end) override { - LOG(FATAL) << "insert_indices_from not supported in PredicateColumnType"; + void insert_indices_from(const IColumn& src, const int* indices_begin, + const int* indices_end) override { + LOG(FATAL) << "insert_indices_from not supported in PredicateColumnType"; } void pop_back(size_t n) override { @@ -160,7 +166,7 @@ public: } void update_hash_with_value(size_t n, SipHash& hash) const override { - LOG(FATAL) << "update_hash_with_value not supported in PredicateColumnType"; + LOG(FATAL) << "update_hash_with_value not supported in PredicateColumnType"; } void insert_string_value(char* data_ptr, size_t length) { @@ -181,7 +187,7 @@ public: memcpy(&val, data_ptr, sizeof(val)); data.push_back_without_reserve(val); } - + void insert_default_type(char* data_ptr, size_t length) { T* val = (T*)data_ptr; data.push_back_without_reserve(*val); @@ -191,13 +197,13 @@ public: char* ch = const_cast(data_ptr); if constexpr (std::is_same_v) { insert_string_value(ch, length); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { insert_decimal_value(ch, length); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { insert_in_copy_way(ch, length); - } else { + } else { insert_default_type(ch, length); - } + } } void insert_many_fix_len_data(const char* data_ptr, size_t num) override { @@ -211,9 +217,10 @@ public: insert_many_default_type(data_ptr, num); } } - - void insert_many_dict_data(const int32_t* data_array, size_t start_index, const uint32_t* start_offset_array, - const uint32_t* len_array, char* dict_data, size_t num) override { + + void insert_many_dict_data(const int32_t* data_array, size_t start_index, + const uint32_t* start_offset_array, const uint32_t* len_array, + char* dict_data, size_t num) override { if constexpr (std::is_same_v) { for (int i = 0; i < num; i++, start_index++) { int32_t codeword = data_array[start_index]; @@ -223,8 +230,9 @@ public: } } } - - void insert_many_binary_data(char* data_array, uint32_t* len_array, uint32_t* start_offset_array, size_t num) override { + + void insert_many_binary_data(char* data_array, uint32_t* len_array, + uint32_t* start_offset_array, size_t num) override { if constexpr (std::is_same_v) { for (size_t i = 0; i < num; i++) { uint32_t len = len_array[i]; @@ -234,34 +242,28 @@ public: } } - void insert_default() override { - data.push_back(T()); - } + void insert_default() override { data.push_back(T()); } void clear() override { data.clear(); } - size_t byte_size() const override { - return data.size() * sizeof(T); - } + size_t byte_size() const override { return data.size() * sizeof(T); } size_t allocated_bytes() const override { return byte_size(); } void protect() override {} void get_permutation(bool reverse, size_t limit, int nan_direction_hint, - IColumn::Permutation& res) const override { + IColumn::Permutation& res) const override { LOG(FATAL) << "get_permutation not supported in PredicateColumnType"; } - void reserve(size_t n) override { - data.reserve(n); - } + void reserve(size_t n) override { data.reserve(n); } - [[noreturn]] const char* get_family_name() const override { + [[noreturn]] const char* get_family_name() const override { LOG(FATAL) << "get_family_name not supported in PredicateColumnType"; } - [[noreturn]] MutableColumnPtr clone_resized(size_t size) const override { + [[noreturn]] MutableColumnPtr clone_resized(size_t size) const override { LOG(FATAL) << "clone_resized not supported in PredicateColumnType"; } @@ -299,20 +301,20 @@ public: // it's impossable to use ComplexType as key , so we don't have to implemnt them [[noreturn]] StringRef serialize_value_into_arena(size_t n, Arena& arena, - char const*& begin) const { + char const*& begin) const override { LOG(FATAL) << "serialize_value_into_arena not supported in PredicateColumnType"; } - [[noreturn]] const char* deserialize_and_insert_from_arena(const char* pos) { + [[noreturn]] const char* deserialize_and_insert_from_arena(const char* pos) override { LOG(FATAL) << "deserialize_and_insert_from_arena not supported in PredicateColumnType"; } [[noreturn]] int compare_at(size_t n, size_t m, const IColumn& rhs, - int nan_direction_hint) const { + int nan_direction_hint) const override { LOG(FATAL) << "compare_at not supported in PredicateColumnType"; } - void get_extremes(Field& min, Field& max) const { + void get_extremes(Field& min, Field& max) const override { LOG(FATAL) << "get_extremes not supported in PredicateColumnType"; } @@ -326,15 +328,16 @@ public: } [[noreturn]] bool structure_equals(const IColumn& rhs) const override { - LOG(FATAL) << "structure_equals not supported in PredicateColumnType"; + LOG(FATAL) << "structure_equals not supported in PredicateColumnType"; } - [[noreturn]] ColumnPtr filter(const IColumn::Filter& filt, ssize_t result_size_hint) const override { - LOG(FATAL) << "filter not supported in PredicateColumnType"; + [[noreturn]] ColumnPtr filter(const IColumn::Filter& filt, + ssize_t result_size_hint) const override { + LOG(FATAL) << "filter not supported in PredicateColumnType"; }; - [[noreturn]] ColumnPtr permute(const IColumn::Permutation& perm, size_t limit) const override { - LOG(FATAL) << "permute not supported in PredicateColumnType"; + [[noreturn]] ColumnPtr permute(const IColumn::Permutation& perm, size_t limit) const override { + LOG(FATAL) << "permute not supported in PredicateColumnType"; }; Container& get_data() { return data; } @@ -352,27 +355,49 @@ public: Status filter_by_selector(const uint16_t* sel, size_t sel_size, IColumn* col_ptr) override { if constexpr (std::is_same_v) { - insert_string_to_res_column(sel, sel_size, reinterpret_cast(col_ptr)); + insert_string_to_res_column(sel, sel_size, + reinterpret_cast(col_ptr)); } else if constexpr (std::is_same_v) { - insert_decimal_to_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_decimal_to_res_column( + sel, sel_size, + reinterpret_cast*>(col_ptr)); } else if constexpr (std::is_same_v) { - insert_default_value_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>(col_ptr)); } else if constexpr (std::is_same_v) { - insert_default_value_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>(col_ptr)); } else if constexpr (std::is_same_v) { - insert_default_value_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>(col_ptr)); } else if constexpr (std::is_same_v) { - insert_default_value_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>(col_ptr)); } else if constexpr (std::is_same_v) { - insert_default_value_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>( + col_ptr)); } else if constexpr (std::is_same_v) { - insert_default_value_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>( + col_ptr)); } else if constexpr (std::is_same_v) { - insert_datetime_to_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_datetime_to_res_column( + sel, sel_size, reinterpret_cast*>(col_ptr)); } else if constexpr (std::is_same_v) { - insert_date_to_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_date_to_res_column(sel, sel_size, + reinterpret_cast*>(col_ptr)); } else if constexpr (std::is_same_v) { - insert_default_value_res_column(sel, sel_size, reinterpret_cast*>(col_ptr)); + insert_default_value_res_column( + sel, sel_size, + reinterpret_cast*>( + col_ptr)); } else if (std::is_same_v) { insert_byte_to_res_column(sel, sel_size, col_ptr); } else { @@ -381,7 +406,6 @@ public: return Status::OK(); } - void replace_column_data(const IColumn&, size_t row, size_t self_row = 0) override { LOG(FATAL) << "should not call replace_column_data in predicate column"; } @@ -395,4 +419,4 @@ private: }; using ColumnStringValue = PredicateColumnType; -} // namespace +} // namespace doris::vectorized diff --git a/be/src/vec/common/cow.h b/be/src/vec/common/cow.h index a235c64d9a..c9e9da9ec4 100644 --- a/be/src/vec/common/cow.h +++ b/be/src/vec/common/cow.h @@ -24,7 +24,6 @@ #include #include - /** Copy-on-write shared ptr. * Allows to work with shared immutable objects and sometimes unshare and mutate you own unique copy. * @@ -101,13 +100,9 @@ protected: COW(COW const&) : ref_counter(0) {} - COW& operator=(COW const&) { - return *this; - } + COW& operator=(COW const&) { return *this; } - void add_ref() { - ++ref_counter; - } + void add_ref() { ++ref_counter; } void release_ref() { if (--ref_counter == 0) { @@ -124,7 +119,7 @@ protected: public: intrusive_ptr() : t(nullptr) {} - intrusive_ptr(T* t, bool add_ref=true) : t(t) { + intrusive_ptr(T* t, bool add_ref = true) : t(t) { if (t && add_ref) ((std::remove_const_t*)t)->add_ref(); } @@ -146,27 +141,33 @@ protected: intrusive_ptr(rhs).swap(*this); return *this; } - +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wuninitialized" +#elif defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" - intrusive_ptr(intrusive_ptr&& rhs) : t(rhs.t) { - rhs.t = nullptr; - } +#endif + intrusive_ptr(intrusive_ptr&& rhs) : t(rhs.t) { rhs.t = nullptr; } +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop - +#endif intrusive_ptr& operator=(intrusive_ptr&& rhs) { intrusive_ptr(static_cast(rhs)).swap(*this); return *this; } - template friend class intrusive_ptr; + template + friend class intrusive_ptr; - template + template intrusive_ptr(intrusive_ptr&& rhs) : t(rhs.t) { rhs.t = nullptr; } - template + template intrusive_ptr& operator=(intrusive_ptr&& rhs) { intrusive_ptr(static_cast&&>(rhs)).swap(*this); return *this; @@ -182,21 +183,13 @@ protected: return *this; } - void reset() { - intrusive_ptr().swap(*this); - } + void reset() { intrusive_ptr().swap(*this); } - void reset(T* rhs) { - intrusive_ptr(rhs).swap(*this); - } + void reset(T* rhs) { intrusive_ptr(rhs).swap(*this); } - void reset(T* rhs, bool add_ref) { - intrusive_ptr(rhs, add_ref).swap(*this); - } + void reset(T* rhs, bool add_ref) { intrusive_ptr(rhs, add_ref).swap(*this); } - T* get() const { - return t; - } + T* get() const { return t; } T* detach() { T* ret = t; @@ -210,25 +203,15 @@ protected: rhs.t = tmp; } - T& operator*() const& { - return *t; - } + T& operator*() const& { return *t; } - T&& operator*() const&& { - return const_cast&&>(*t); - } + T&& operator*() const&& { return const_cast&&>(*t); } - T* operator->() const { - return t; - } + T* operator->() const { return t; } - operator bool() const { - return t != nullptr; - } + operator bool() const { return t != nullptr; } - operator T*() const { - return t; - } + operator T*() const { return t; } private: T* t; @@ -240,10 +223,13 @@ protected: private: using Base = intrusive_ptr; - template friend class COW; - template friend class COWHelper; + template + friend class COW; + template + friend class COWHelper; explicit mutable_ptr(T* ptr) : Base(ptr) {} + public: /// Copy: not possible. mutable_ptr(const mutable_ptr&) = delete; @@ -264,9 +250,7 @@ protected: public: using MutablePtr = mutable_ptr; - unsigned int use_count() const { - return ref_counter.load(); - } + unsigned int use_count() const { return ref_counter.load(); } protected: template @@ -274,10 +258,13 @@ protected: private: using Base = intrusive_ptr; - template friend class COW; - template friend class COWHelper; + template + friend class COW; + template + friend class COWHelper; explicit immutable_ptr(const T* ptr) : Base(ptr) {} + public: /// Copy from immutable ptr: ok. immutable_ptr(const immutable_ptr&) = default; diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 11fa55325f..f888ad1bb1 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -99,16 +99,16 @@ inline DataTypePtr create_data_type(const PColumnMeta& pcolumn_meta) { return std::make_shared(); } case PGenericType::DECIMAL32: { - return std::make_shared>(pcolumn_meta.decimal_param().precision(), - pcolumn_meta.decimal_param().scale()); + return std::make_shared>( + pcolumn_meta.decimal_param().precision(), pcolumn_meta.decimal_param().scale()); } case PGenericType::DECIMAL64: { - return std::make_shared>(pcolumn_meta.decimal_param().precision(), - pcolumn_meta.decimal_param().scale()); + return std::make_shared>( + pcolumn_meta.decimal_param().precision(), pcolumn_meta.decimal_param().scale()); } case PGenericType::DECIMAL128: { - return std::make_shared>(pcolumn_meta.decimal_param().precision(), - pcolumn_meta.decimal_param().scale()); + return std::make_shared>( + pcolumn_meta.decimal_param().precision(), pcolumn_meta.decimal_param().scale()); } case PGenericType::BITMAP: { return std::make_shared(); @@ -136,14 +136,16 @@ Block::Block(const PBlock& pblock) { const char* buf = nullptr; std::string compression_scratch; if (pblock.compressed()) { - // Decompress + // Decompress const char* compressed_data = pblock.column_values().c_str(); size_t compressed_size = pblock.column_values().size(); size_t uncompressed_size = 0; - bool success = snappy::GetUncompressedLength(compressed_data, compressed_size, &uncompressed_size); + bool success = + snappy::GetUncompressedLength(compressed_data, compressed_size, &uncompressed_size); DCHECK(success) << "snappy::GetUncompressedLength failed"; compression_scratch.resize(uncompressed_size); - success = snappy::RawUncompress(compressed_data, compressed_size, compression_scratch.data()); + success = + snappy::RawUncompress(compressed_data, compressed_size, compression_scratch.data()); DCHECK(success) << "snappy::RawUncompress failed"; buf = compression_scratch.data(); } else { @@ -154,14 +156,14 @@ Block::Block(const PBlock& pblock) { DataTypePtr type = create_data_type(pcol_meta); MutableColumnPtr data_column; if (pcol_meta.is_nullable()) { - data_column = ColumnNullable::create(std::move(type->create_column()), ColumnUInt8::create()); + data_column = ColumnNullable::create(type->create_column(), ColumnUInt8::create()); type = make_nullable(type); } else { data_column = type->create_column(); } buf = type->deserialize(buf, data_column.get()); data.emplace_back(data_column->get_ptr(), type, pcol_meta.name()); - } + } initialize_index_by_name(); } @@ -704,8 +706,8 @@ Status Block::filter_block(Block* block, int filter_column_id, int column_to_kee return Status::OK(); } -Status Block::serialize(PBlock* pblock, size_t* uncompressed_bytes, - size_t* compressed_bytes, std::string* allocated_buf) const { +Status Block::serialize(PBlock* pblock, size_t* uncompressed_bytes, size_t* compressed_bytes, + std::string* allocated_buf) const { // calc uncompressed size for allocation size_t content_uncompressed_size = 0; for (const auto& c : *this) { @@ -722,7 +724,8 @@ Status Block::serialize(PBlock* pblock, size_t* uncompressed_bytes, for (const auto& c : *this) { buf = c.type->serialize(*(c.column), buf); } - CHECK(content_uncompressed_size == (buf - start_buf)) << content_uncompressed_size << " vs. " << (buf - start_buf); + CHECK(content_uncompressed_size == (buf - start_buf)) + << content_uncompressed_size << " vs. " << (buf - start_buf); *uncompressed_bytes = content_uncompressed_size; // compress @@ -735,7 +738,8 @@ Status Block::serialize(PBlock* pblock, size_t* uncompressed_bytes, size_t compressed_size = 0; char* compressed_output = compression_scratch.data(); - snappy::RawCompress(allocated_buf->data(), content_uncompressed_size, compressed_output, &compressed_size); + snappy::RawCompress(allocated_buf->data(), content_uncompressed_size, compressed_output, + &compressed_size); if (LIKELY(compressed_size < content_uncompressed_size)) { compression_scratch.resize(compressed_size); @@ -746,13 +750,13 @@ Status Block::serialize(PBlock* pblock, size_t* uncompressed_bytes, *compressed_bytes = content_uncompressed_size; } - VLOG_ROW << "uncompressed size: " << content_uncompressed_size << ", compressed size: " << compressed_size; + VLOG_ROW << "uncompressed size: " << content_uncompressed_size + << ", compressed size: " << compressed_size; } return Status::OK(); } - void Block::serialize(RowBatch* output_batch, const RowDescriptor& row_desc) { auto num_rows = rows(); auto mem_pool = output_batch->tuple_data_pool(); diff --git a/be/src/vec/data_types/data_type_bitmap.h b/be/src/vec/data_types/data_type_bitmap.h index e0c7937830..ba8e96be75 100644 --- a/be/src/vec/data_types/data_type_bitmap.h +++ b/be/src/vec/data_types/data_type_bitmap.h @@ -67,10 +67,12 @@ public: bool can_be_inside_low_cardinality() const override { return false; } - std::string to_string(const IColumn& column, size_t row_num) const { return "BitMap()"; } - void to_string(const IColumn &column, size_t row_num, BufferWritable &ostr) const override; + std::string to_string(const IColumn& column, size_t row_num) const override { + return "BitMap()"; + } + void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; - [[noreturn]] virtual Field get_default() const { + [[noreturn]] virtual Field get_default() const override { LOG(FATAL) << "Method get_default() is not implemented for data type " << get_name(); __builtin_unreachable(); } diff --git a/be/src/vec/data_types/data_type_date.h b/be/src/vec/data_types/data_type_date.h index 685363c396..afc67eded6 100644 --- a/be/src/vec/data_types/data_type_date.h +++ b/be/src/vec/data_types/data_type_date.h @@ -34,7 +34,7 @@ public: bool can_be_inside_nullable() const override { return true; } bool equals(const IDataType& rhs) const override; - std::string to_string(const IColumn& column, size_t row_num) const; + std::string to_string(const IColumn& column, size_t row_num) const override; void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; static void cast_to_date(Int64& x); diff --git a/be/src/vec/data_types/data_type_date_time.h b/be/src/vec/data_types/data_type_date_time.h index a0b21f18ef..6f2ef2c3f9 100644 --- a/be/src/vec/data_types/data_type_date_time.h +++ b/be/src/vec/data_types/data_type_date_time.h @@ -60,7 +60,7 @@ public: bool equals(const IDataType& rhs) const override; - std::string to_string(const IColumn& column, size_t row_num) const; + std::string to_string(const IColumn& column, size_t row_num) const override; void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h index f57274f10f..8792a95213 100644 --- a/be/src/vec/data_types/data_type_decimal.h +++ b/be/src/vec/data_types/data_type_decimal.h @@ -143,8 +143,8 @@ public: bool is_summable() const override { return true; } bool can_be_used_in_boolean_context() const override { return true; } bool can_be_inside_nullable() const override { return true; } - std::string to_string(const IColumn& column, size_t row_num) const; - void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const; + std::string to_string(const IColumn& column, size_t row_num) const override; + void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; /// Decimal specific @@ -330,19 +330,9 @@ convert_to_decimal(const typename FromDataType::FieldType& value, UInt32 scale) } auto out = value * ToDataType::get_scale_multiplier(scale); - if constexpr (std::is_same_v) { - static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64; - static constexpr __int128 max_int128 = - (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll; - if (out <= static_cast(min_int128) || - out >= static_cast(max_int128)) { - LOG(FATAL) << "Decimal convert overflow. Float is out of Decimal range"; - } - } else { - if (out <= std::numeric_limits::min() || - out >= std::numeric_limits::max()) { - LOG(FATAL) << "Decimal convert overflow. Float is out of Decimal range"; - } + if (out <= static_cast(std::numeric_limits::min()) || + out >= static_cast(std::numeric_limits::max())) { + LOG(FATAL) << "Decimal convert overflow. Float is out of Decimal range"; } return out; } else { diff --git a/be/src/vec/data_types/data_type_hll.h b/be/src/vec/data_types/data_type_hll.h index 45a039b187..f63890e754 100644 --- a/be/src/vec/data_types/data_type_hll.h +++ b/be/src/vec/data_types/data_type_hll.h @@ -65,12 +65,13 @@ public: bool can_be_inside_low_cardinality() const override { return false; } - std::string to_string(const IColumn& column, size_t row_num) const { return "HLL()"; } + std::string to_string(const IColumn& column, size_t row_num) const override { return "HLL()"; } void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; - [[noreturn]] virtual Field get_default() const { + virtual Field get_default() const override { LOG(FATAL) << "Method get_default() is not implemented for data type " << get_name(); - __builtin_unreachable(); + // unreachable + return String(); } static void serialize_as_stream(const HyperLogLog& value, BufferWritable& buf); diff --git a/be/src/vec/data_types/data_type_nullable.h b/be/src/vec/data_types/data_type_nullable.h index a2655527b0..537d0bf449 100644 --- a/be/src/vec/data_types/data_type_nullable.h +++ b/be/src/vec/data_types/data_type_nullable.h @@ -80,7 +80,7 @@ public: bool can_be_inside_low_cardinality() const override { return nested_data_type->can_be_inside_low_cardinality(); } - std::string to_string(const IColumn& column, size_t row_num) const; + std::string to_string(const IColumn& column, size_t row_num) const override; const DataTypePtr& get_nested_type() const { return nested_data_type; } diff --git a/be/src/vec/data_types/data_type_number_base.h b/be/src/vec/data_types/data_type_number_base.h index e945607ba0..1ffdfb58d3 100644 --- a/be/src/vec/data_types/data_type_number_base.h +++ b/be/src/vec/data_types/data_type_number_base.h @@ -64,8 +64,8 @@ public: bool is_categorial() const override { return is_value_represented_by_integer(); } bool can_be_inside_low_cardinality() const override { return true; } - void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const; - std::string to_string(const IColumn& column, size_t row_num) const; + void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; + std::string to_string(const IColumn& column, size_t row_num) const override; }; } // namespace doris::vectorized diff --git a/be/src/vec/data_types/data_type_string.h b/be/src/vec/data_types/data_type_string.h index 654773723a..8cf10fd9d4 100644 --- a/be/src/vec/data_types/data_type_string.h +++ b/be/src/vec/data_types/data_type_string.h @@ -56,8 +56,8 @@ public: bool is_categorial() const override { return true; } bool can_be_inside_nullable() const override { return true; } bool can_be_inside_low_cardinality() const override { return true; } - std::string to_string(const IColumn& column, size_t row_num) const; - void to_string(const IColumn &column, size_t row_num, BufferWritable &ostr) const override; + std::string to_string(const IColumn& column, size_t row_num) const override; + void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; }; } // namespace doris::vectorized diff --git a/be/src/vec/exec/join/vhash_join_node.h b/be/src/vec/exec/join/vhash_join_node.h index 00a1eaf75c..ba6f394b3f 100644 --- a/be/src/vec/exec/join/vhash_join_node.h +++ b/be/src/vec/exec/join/vhash_join_node.h @@ -147,12 +147,12 @@ public: HashJoinNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); ~HashJoinNode() override; - virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr); - virtual Status prepare(RuntimeState* state); - virtual Status open(RuntimeState* state); - virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos); - virtual Status get_next(RuntimeState* state, Block* block, bool* eos); - virtual Status close(RuntimeState* state); + virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr) override; + virtual Status prepare(RuntimeState* state) override; + virtual Status open(RuntimeState* state) override; + virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) override; + virtual Status get_next(RuntimeState* state, Block* block, bool* eos) override; + virtual Status close(RuntimeState* state) override; HashTableVariants& get_hash_table_variants() { return _hash_table_variants; } void init_join_op(); @@ -240,13 +240,13 @@ private: void _hash_table_init(); template - friend class ProcessHashTableBuild; + friend struct ProcessHashTableBuild; template - friend class ProcessHashTableProbe; + friend struct ProcessHashTableProbe; template - friend class ProcessRuntimeFilterBuild; + friend struct ProcessRuntimeFilterBuild; std::vector _runtime_filter_descs; std::unordered_map> _inserted_rows; diff --git a/be/src/vec/exec/vaggregation_node.h b/be/src/vec/exec/vaggregation_node.h index 4df933f917..f020b90a6e 100644 --- a/be/src/vec/exec/vaggregation_node.h +++ b/be/src/vec/exec/vaggregation_node.h @@ -59,7 +59,7 @@ struct AggregationMethodSerialized { using State = ColumnsHashing::HashMethodSerialized; static void insert_key_into_columns(const StringRef& key, MutableColumns& key_columns, - const Sizes&) { + const Sizes&) { auto pos = key.data; for (auto& column : key_columns) pos = column->deserialize_and_insert_from_arena(pos); } @@ -77,10 +77,8 @@ using AggregatedDataWithStringKey = HashMapWithSavedHash -struct AggregationMethodOneNumber -{ +template +struct AggregationMethodOneNumber { using Data = TData; using Key = typename Data::key_type; using Mapped = typename Data::mapped_type; @@ -93,16 +91,17 @@ struct AggregationMethodOneNumber AggregationMethodOneNumber() = default; template - AggregationMethodOneNumber(const Other & other) : data(other.data) {} + AggregationMethodOneNumber(const Other& other) : data(other.data) {} /// To use one `Method` in different threads, use different `State`. - using State = ColumnsHashing::HashMethodOneNumber; + using State = ColumnsHashing::HashMethodOneNumber; // Insert the key from the hash table into columns. - static void insert_key_into_columns(const Key & key, MutableColumns & key_columns, const Sizes & /*key_sizes*/) { - const auto * key_holder = reinterpret_cast(&key); - auto * column = static_cast(key_columns[0].get()); + static void insert_key_into_columns(const Key& key, MutableColumns& key_columns, + const Sizes& /*key_sizes*/) { + const auto* key_holder = reinterpret_cast(&key); + auto* column = static_cast(key_columns[0].get()); column->insert_raw_data(key_holder); } @@ -118,7 +117,7 @@ template struct AggregationDataWithNullKey : public Base { using Base::Base; - bool & has_null_key_data() { return has_null_key; } + bool& has_null_key_data() { return has_null_key; } AggregateDataPtr& get_null_key_data() { return null_key_data; } bool has_null_key_data() const { return has_null_key; } const AggregateDataPtr get_null_key_data() const { return null_key_data; } @@ -155,30 +154,32 @@ struct AggregationMethodKeysFixed { AggregationMethodKeysFixed() {} template - AggregationMethodKeysFixed(const Other & other) : data(other.data) {} + AggregationMethodKeysFixed(const Other& other) : data(other.data) {} - using State = ColumnsHashing::HashMethodKeysFixed; + using State = ColumnsHashing::HashMethodKeysFixed; - static void insert_key_into_columns(const Key & key, MutableColumns & key_columns, const Sizes & key_sizes) { + static void insert_key_into_columns(const Key& key, MutableColumns& key_columns, + const Sizes& key_sizes) { size_t keys_size = key_columns.size(); - static constexpr auto bitmap_size = has_nullable_keys ? std::tuple_size>::value : 0; + static constexpr auto bitmap_size = + has_nullable_keys ? std::tuple_size>::value : 0; /// In any hash key value, column values to be read start just after the bitmap, if it exists. size_t pos = bitmap_size; for (size_t i = 0; i < keys_size; ++i) { - IColumn * observed_column; - ColumnUInt8 * null_map; + IColumn* observed_column; + ColumnUInt8* null_map; bool column_nullable = false; - if constexpr (has_nullable_keys) - column_nullable = is_column_nullable(*key_columns[i]); + if constexpr (has_nullable_keys) column_nullable = is_column_nullable(*key_columns[i]); /// If we have a nullable column, get its nested column and its null map. if (column_nullable) { - ColumnNullable & nullable_col = assert_cast(*key_columns[i]); + ColumnNullable& nullable_col = assert_cast(*key_columns[i]); observed_column = &nullable_col.get_nested_column(); - null_map = assert_cast(&nullable_col.get_null_map_column()); + null_map = assert_cast(&nullable_col.get_null_map_column()); } else { observed_column = key_columns[i].get(); null_map = nullptr; @@ -190,7 +191,7 @@ struct AggregationMethodKeysFixed { /// corresponding key is nullable. Update the null map accordingly. size_t bucket = i / 8; size_t offset = i % 8; - UInt8 val = (reinterpret_cast(&key)[bucket] >> offset) & 1; + UInt8 val = (reinterpret_cast(&key)[bucket] >> offset) & 1; null_map->insert_value(val); is_null = val == 1; } @@ -199,7 +200,7 @@ struct AggregationMethodKeysFixed { observed_column->insert_default(); else { size_t size = key_sizes[i]; - observed_column->insert_data(reinterpret_cast(&key) + pos, size); + observed_column->insert_data(reinterpret_cast(&key) + pos, size); pos += size; } } @@ -228,23 +229,24 @@ struct AggregationMethodSingleNullableColumn : public SingleColumnMethod { AggregationMethodSingleNullableColumn() = default; template - explicit AggregationMethodSingleNullableColumn(const Other & other) : Base(other) {} + explicit AggregationMethodSingleNullableColumn(const Other& other) : Base(other) {} using State = ColumnsHashing::HashMethodSingleLowNullableColumn; - static void insert_key_into_columns(const Key & key, - MutableColumns & key_columns, const Sizes & /*key_sizes*/) { + static void insert_key_into_columns(const Key& key, MutableColumns& key_columns, + const Sizes& /*key_sizes*/) { auto col = key_columns[0].get(); if constexpr (std::is_same_v) { col->insert_data(key.data, key.size); } else { - col->insert_data(reinterpret_cast(&key), sizeof(key)); + col->insert_data(reinterpret_cast(&key), sizeof(key)); } } }; -using AggregatedDataWithUInt8Key = FixedImplicitZeroHashMapWithCalculatedSize; +using AggregatedDataWithUInt8Key = + FixedImplicitZeroHashMapWithCalculatedSize; using AggregatedDataWithUInt16Key = FixedImplicitZeroHashMap; using AggregatedDataWithUInt32Key = HashMap>; using AggregatedDataWithUInt64Key = HashMap>; @@ -255,25 +257,32 @@ using AggregatedDataWithNullableUInt8Key = AggregationDataWithNullKey; using AggregatedDataWithNullableUInt32Key = AggregationDataWithNullKey; using AggregatedDataWithNullableUInt64Key = AggregationDataWithNullKey; -using AggregatedDataWithNullableUInt128Key = AggregationDataWithNullKey; +using AggregatedDataWithNullableUInt128Key = + AggregationDataWithNullKey; -using AggregatedMethodVariants = std::variant, - AggregationMethodOneNumber, - AggregationMethodOneNumber, - AggregationMethodOneNumber, - AggregationMethodOneNumber, - AggregationMethodOneNumber, - AggregationMethodSingleNullableColumn>, - AggregationMethodSingleNullableColumn>, - AggregationMethodSingleNullableColumn>, - AggregationMethodSingleNullableColumn>, - AggregationMethodSingleNullableColumn>, - AggregationMethodKeysFixed, - AggregationMethodKeysFixed, - AggregationMethodKeysFixed, - AggregationMethodKeysFixed, - AggregationMethodKeysFixed, - AggregationMethodKeysFixed>; +using AggregatedMethodVariants = std::variant< + AggregationMethodSerialized, + AggregationMethodOneNumber, + AggregationMethodOneNumber, + AggregationMethodOneNumber, + AggregationMethodOneNumber, + AggregationMethodOneNumber, + AggregationMethodSingleNullableColumn< + AggregationMethodOneNumber>, + AggregationMethodSingleNullableColumn< + AggregationMethodOneNumber>, + AggregationMethodSingleNullableColumn< + AggregationMethodOneNumber>, + AggregationMethodSingleNullableColumn< + AggregationMethodOneNumber>, + AggregationMethodSingleNullableColumn< + AggregationMethodOneNumber>, + AggregationMethodKeysFixed, + AggregationMethodKeysFixed, + AggregationMethodKeysFixed, + AggregationMethodKeysFixed, + AggregationMethodKeysFixed, + AggregationMethodKeysFixed>; struct AggregatedDataVariants { AggregatedDataVariants() = default; @@ -305,62 +314,82 @@ struct AggregatedDataVariants { case Type::without_key: break; case Type::serialized: - _aggregated_method_variant.emplace>(); + _aggregated_method_variant + .emplace>(); break; case Type::int8_key: if (is_nullable) { - _aggregated_method_variant.emplace>>(); + _aggregated_method_variant + .emplace>>(); } else { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant.emplace< + AggregationMethodOneNumber>(); } break; case Type::int16_key: if (is_nullable) { - _aggregated_method_variant.emplace>>(); + _aggregated_method_variant + .emplace>>(); } else { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant.emplace< + AggregationMethodOneNumber>(); } break; case Type::int32_key: if (is_nullable) { - _aggregated_method_variant.emplace>>(); + _aggregated_method_variant.emplace>>(); } else { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant + .emplace>(); } break; case Type::int64_key: if (is_nullable) { - _aggregated_method_variant.emplace>>(); + _aggregated_method_variant.emplace>>(); } else { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant + .emplace>(); } break; case Type::int128_key: if (is_nullable) { - _aggregated_method_variant.emplace>>(); + _aggregated_method_variant + .emplace>>(); } else { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant.emplace< + AggregationMethodOneNumber>(); } break; case Type::int64_keys: if (is_nullable) { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant + .emplace>(); } else { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant + .emplace>(); } break; case Type::int128_keys: if (is_nullable) { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant + .emplace>(); } else { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant + .emplace>(); } break; case Type::int256_keys: if (is_nullable) { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant + .emplace>(); } else { - _aggregated_method_variant.emplace>(); + _aggregated_method_variant + .emplace>(); } break; default: @@ -427,9 +456,6 @@ private: bool _should_expand_hash_table = true; std::vector _streaming_pre_places; - /// Expose the minimum reduction factor to continue growing the hash tables. - RuntimeProfile::Counter* preagg_streaming_ht_min_reduction_; - private: /// Return true if we should keep expanding hash tables in the preagg. If false, /// the preagg should pass through any rows it can't fit in its tables. diff --git a/be/src/vec/exec/vanalytic_eval_node.h b/be/src/vec/exec/vanalytic_eval_node.h index 3289dd145e..18845eb85d 100644 --- a/be/src/vec/exec/vanalytic_eval_node.h +++ b/be/src/vec/exec/vanalytic_eval_node.h @@ -49,6 +49,7 @@ public: virtual Status close(RuntimeState* state); protected: + using ExecNode::debug_string; virtual std::string debug_string(); private: @@ -72,7 +73,7 @@ private: Status _output_current_block(Block* block); BlockRowPos _get_partition_by_end(); BlockRowPos _compare_row_to_find_end(int idx, BlockRowPos start, BlockRowPos end); - + Status _fetch_next_block_data(RuntimeState* state); Status _consumed_block_and_init_partition(RuntimeState* state, bool* next_partition, bool* eos); bool whether_need_next_partition(BlockRowPos found_partition_end); diff --git a/be/src/vec/exec/vassert_num_rows_node.h b/be/src/vec/exec/vassert_num_rows_node.h index 95e11540a6..cce963f25f 100644 --- a/be/src/vec/exec/vassert_num_rows_node.h +++ b/be/src/vec/exec/vassert_num_rows_node.h @@ -30,8 +30,8 @@ public: return Status::NotSupported("Not Implemented VAnalyticEvalNode::get_next."); } - virtual Status open(RuntimeState* state); - virtual Status get_next(RuntimeState* state, Block* block, bool* eos); + virtual Status open(RuntimeState* state) override; + virtual Status get_next(RuntimeState* state, Block* block, bool* eos) override; private: int64_t _desired_num_rows; @@ -39,4 +39,4 @@ private: TAssertion::type _assertion; }; -} // namespace doris +} // namespace doris::vectorized diff --git a/be/src/vec/exec/vcross_join_node.h b/be/src/vec/exec/vcross_join_node.h index aeeeb3a2db..ba517861bf 100644 --- a/be/src/vec/exec/vcross_join_node.h +++ b/be/src/vec/exec/vcross_join_node.h @@ -27,7 +27,6 @@ #include "gen_cpp/PlanNodes_types.h" #include "runtime/descriptors.h" #include "runtime/mem_pool.h" - #include "vec/core/block.h" #include "vec/exec/vblocking_join_node.h" @@ -40,18 +39,19 @@ namespace doris::vectorized { // the left child as necessary in get_next(). class VCrossJoinNode final : public VBlockingJoinNode { public: - VCrossJoinNode(ObjectPool *pool, const TPlanNode &tnode, const DescriptorTbl &descs); + VCrossJoinNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); - Status prepare(RuntimeState *state) override; + Status prepare(RuntimeState* state) override; + using VBlockingJoinNode::get_next; Status get_next(RuntimeState* state, Block* block, bool* eos) override; - Status close(RuntimeState *state) override; + Status close(RuntimeState* state) override; protected: void init_get_next(int first_left_row) override; - Status construct_build_side(RuntimeState *state) override; + Status construct_build_side(RuntimeState* state) override; private: // List of build blocks, constructed in prepare() @@ -68,18 +68,18 @@ private: // if block can mem reuse, just clear data in block // else build a new block and alloc mem of column from left and right child block MutableColumns get_mutable_columns(Block* block); - + // Processes a block from the left child. // dst_columns: left_child_row and now_process_build_block to construct a bundle column of new block // now_process_build_block: right child block now to process - void process_left_child_block(MutableColumns& dst_columns, const Block& now_process_build_block); + void process_left_child_block(MutableColumns& dst_columns, + const Block& now_process_build_block); // Returns a debug string for _build_rows. This is used for debugging during the // build list construction and before doing the join. std::string build_list_debug_string(); }; -} +} // namespace doris::vectorized #endif - diff --git a/be/src/vec/exec/vempty_set_node.h b/be/src/vec/exec/vempty_set_node.h index 501ba17d2a..900f0c6016 100644 --- a/be/src/vec/exec/vempty_set_node.h +++ b/be/src/vec/exec/vempty_set_node.h @@ -21,15 +21,15 @@ namespace doris { namespace vectorized { - /// Node that returns an empty result set, i.e., just sets eos_ in GetNext(). - /// Corresponds to EmptySetNode.java in the FE. - class VEmptySetNode : public ExecNode { - public: - VEmptySetNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); - virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) { - return Status::NotSupported("Not Implemented get RowBatch in vecorized execution."); - } - virtual Status get_next(RuntimeState* state, Block* block, bool* eos) override; - }; +/// Node that returns an empty result set, i.e., just sets eos_ in GetNext(). +/// Corresponds to EmptySetNode.java in the FE. +class VEmptySetNode : public ExecNode { +public: + VEmptySetNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); + virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) override { + return Status::NotSupported("Not Implemented get RowBatch in vecorized execution."); + } + virtual Status get_next(RuntimeState* state, Block* block, bool* eos) override; +}; } // namespace vectorized } // namespace doris diff --git a/be/src/vec/exec/ves_http_scan_node.h b/be/src/vec/exec/ves_http_scan_node.h index d1835a1dba..7825390177 100644 --- a/be/src/vec/exec/ves_http_scan_node.h +++ b/be/src/vec/exec/ves_http_scan_node.h @@ -35,12 +35,13 @@ public: VEsHttpScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); ~VEsHttpScanNode(); - virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos); + using EsHttpScanNode::get_next; + virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos) override; virtual Status close(RuntimeState* state) override; private: - virtual Status scanner_scan(std::unique_ptr scanner); + virtual Status scanner_scan(std::unique_ptr scanner) override; std::deque> _block_queue; std::mutex _block_queue_lock; diff --git a/be/src/vec/exec/vexcept_node.h b/be/src/vec/exec/vexcept_node.h index e986c28694..7bbed571e7 100644 --- a/be/src/vec/exec/vexcept_node.h +++ b/be/src/vec/exec/vexcept_node.h @@ -28,12 +28,13 @@ public: virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr); virtual Status prepare(RuntimeState* state); virtual Status open(RuntimeState* state); + using VSetOperationNode::get_next; virtual Status get_next(RuntimeState* state, vectorized::Block* output_block, bool* eos); virtual Status close(RuntimeState* state); private: template - friend class HashTableProbe; + friend struct HashTableProbe; }; } // namespace vectorized } // namespace doris diff --git a/be/src/vec/exec/vintersect_node.h b/be/src/vec/exec/vintersect_node.h index 4e9b21e327..1316fcd761 100644 --- a/be/src/vec/exec/vintersect_node.h +++ b/be/src/vec/exec/vintersect_node.h @@ -38,12 +38,13 @@ public: virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr); virtual Status prepare(RuntimeState* state); virtual Status open(RuntimeState* state); + using VSetOperationNode::get_next; virtual Status get_next(RuntimeState* state, vectorized::Block* output_block, bool* eos); virtual Status close(RuntimeState* state); private: template - friend class HashTableProbe; + friend struct HashTableProbe; }; } // namespace vectorized } // namespace doris \ No newline at end of file diff --git a/be/src/vec/exec/vmysql_scan_node.h b/be/src/vec/exec/vmysql_scan_node.h index c297caec79..c4d1d2c97c 100644 --- a/be/src/vec/exec/vmysql_scan_node.h +++ b/be/src/vec/exec/vmysql_scan_node.h @@ -37,6 +37,7 @@ public: VMysqlScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); ~VMysqlScanNode(); + using MysqlScanNode::get_next; // Fill the next block by calling next() on the _mysql_scanner, // converting text data in MySQL cells to binary data. virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos); diff --git a/be/src/vec/exec/vodbc_scan_node.h b/be/src/vec/exec/vodbc_scan_node.h index 7c644cdbb8..d1cd428c58 100644 --- a/be/src/vec/exec/vodbc_scan_node.h +++ b/be/src/vec/exec/vodbc_scan_node.h @@ -27,6 +27,7 @@ public: VOdbcScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); ~VOdbcScanNode(); + using OdbcScanNode::get_next; Status get_next(RuntimeState* state, Block* block, bool* eos); }; } // namespace vectorized diff --git a/be/src/vec/exec/vrepeat_node.h b/be/src/vec/exec/vrepeat_node.h index cccf7ad74f..7f3f91f8c6 100644 --- a/be/src/vec/exec/vrepeat_node.h +++ b/be/src/vec/exec/vrepeat_node.h @@ -35,6 +35,7 @@ public: virtual Status prepare(RuntimeState* state) override; virtual Status open(RuntimeState* state) override; + using RepeatNode::get_next; virtual Status get_next(RuntimeState* state, Block* block, bool* eos) override; virtual Status close(RuntimeState* state) override; diff --git a/be/src/vec/exec/vschema_scan_node.h b/be/src/vec/exec/vschema_scan_node.h index 51b523256c..96b214122c 100644 --- a/be/src/vec/exec/vschema_scan_node.h +++ b/be/src/vec/exec/vschema_scan_node.h @@ -36,8 +36,8 @@ public: VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); ~VSchemaScanNode(); Status prepare(RuntimeState* state) override; - - virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos); + using SchemaScanNode::get_next; + virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos) override; private: Status write_slot_to_vectorized_column(void* slot, SlotDescriptor* slot_desc, diff --git a/be/src/vec/exec/vset_operation_node.h b/be/src/vec/exec/vset_operation_node.h index 93a7f8b61a..f31532ff8f 100644 --- a/be/src/vec/exec/vset_operation_node.h +++ b/be/src/vec/exec/vset_operation_node.h @@ -90,9 +90,9 @@ protected: RuntimeProfile::Counter* _probe_timer; // time to probe template - friend class HashTableBuild; + friend struct HashTableBuild; template - friend class HashTableProbe; + friend struct HashTableProbe; }; template diff --git a/be/src/vec/exec/vsort_node.h b/be/src/vec/exec/vsort_node.h index 66876aa149..f67326afa6 100644 --- a/be/src/vec/exec/vsort_node.h +++ b/be/src/vec/exec/vsort_node.h @@ -17,10 +17,9 @@ #pragma once -#include "exec/exec_node.h" - #include +#include "exec/exec_node.h" #include "vec/core/block.h" #include "vec/core/sort_cursor.h" #include "vec/exec/vsort_exec_exprs.h" @@ -33,30 +32,30 @@ namespace doris::vectorized { // support spill to disk in the future class VSortNode : public doris::ExecNode { public: - VSortNode(ObjectPool *pool, const TPlanNode &tnode, const DescriptorTbl &descs); + VSortNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); ~VSortNode() override = default; - virtual Status init(const TPlanNode &tnode, RuntimeState *state = nullptr); + virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr) override; - virtual Status prepare(RuntimeState *state); + virtual Status prepare(RuntimeState* state) override; - virtual Status open(RuntimeState *state); + virtual Status open(RuntimeState* state) override; - virtual Status get_next(RuntimeState *state, RowBatch *row_batch, bool *eos); + virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) override; - virtual Status get_next(RuntimeState* state, Block* block, bool* eos); + virtual Status get_next(RuntimeState* state, Block* block, bool* eos) override; - virtual Status reset(RuntimeState *state); + virtual Status reset(RuntimeState* state) override; - virtual Status close(RuntimeState *state); + virtual Status close(RuntimeState* state) override; protected: - virtual void debug_string(int indentation_level, std::stringstream *out) const; + virtual void debug_string(int indentation_level, std::stringstream* out) const override; private: // Fetch input rows and feed them to the sorter until the input is exhausted. - Status sort_input(RuntimeState *state); + Status sort_input(RuntimeState* state); Status pretreat_block(Block& block); @@ -87,6 +86,4 @@ private: std::priority_queue _block_priority_queue; }; -} // end namespace doris - - +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/vcase_expr.h b/be/src/vec/exprs/vcase_expr.h index 6291f9e074..af6d74a2b0 100644 --- a/be/src/vec/exprs/vcase_expr.h +++ b/be/src/vec/exprs/vcase_expr.h @@ -26,12 +26,14 @@ class VCaseExpr final : public VExpr { public: VCaseExpr(const TExprNode& node); ~VCaseExpr() = default; - virtual Status execute(VExprContext* context, vectorized::Block* block, int* result_column_id); - virtual Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context); + virtual Status execute(VExprContext* context, vectorized::Block* block, + int* result_column_id) override; + virtual Status prepare(RuntimeState* state, const RowDescriptor& desc, + VExprContext* context) override; virtual Status open(RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual void close(RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual VExpr* clone(ObjectPool* pool) const override { return pool->add(new VCaseExpr(*this)); } diff --git a/be/src/vec/exprs/vcast_expr.h b/be/src/vec/exprs/vcast_expr.h index b33828c60c..7f62a4c1de 100644 --- a/be/src/vec/exprs/vcast_expr.h +++ b/be/src/vec/exprs/vcast_expr.h @@ -20,18 +20,18 @@ #include "vec/functions/function.h" namespace doris::vectorized { -class VCastExpr final: public VExpr { +class VCastExpr final : public VExpr { public: VCastExpr(const TExprNode& node) : VExpr(node) {} ~VCastExpr() = default; virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block, - int* result_column_id); + int* result_column_id) override; virtual doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc, - VExprContext* context); + VExprContext* context) override; virtual doris::Status open(doris::RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual void close(doris::RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual VExpr* clone(doris::ObjectPool* pool) const override { return pool->add(new VCastExpr(*this)); } diff --git a/be/src/vec/exprs/vectorized_fn_call.h b/be/src/vec/exprs/vectorized_fn_call.h index 9776917ba7..cdd3ef211e 100644 --- a/be/src/vec/exprs/vectorized_fn_call.h +++ b/be/src/vec/exprs/vectorized_fn_call.h @@ -25,18 +25,18 @@ class VectorizedFnCall : public VExpr { public: VectorizedFnCall(const doris::TExprNode& node); virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block, - int* result_column_id); + int* result_column_id) override; virtual doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc, - VExprContext* context); + VExprContext* context) override; virtual doris::Status open(doris::RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual void close(doris::RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual VExpr* clone(doris::ObjectPool* pool) const override { return pool->add(new VectorizedFnCall(*this)); } virtual const std::string& expr_name() const override; - virtual std::string debug_string() const; + virtual std::string debug_string() const override; static std::string debug_string(const std::vector& exprs); private: diff --git a/be/src/vec/exprs/vin_predicate.h b/be/src/vec/exprs/vin_predicate.h index 96e9a0ae3d..2a63827f7e 100644 --- a/be/src/vec/exprs/vin_predicate.h +++ b/be/src/vec/exprs/vin_predicate.h @@ -18,23 +18,22 @@ #pragma once #include "exprs/hybrid_set.h" - #include "vec/exprs/vexpr.h" #include "vec/functions/function.h" namespace doris::vectorized { -class VInPredicate final: public VExpr { +class VInPredicate final : public VExpr { public: VInPredicate(const TExprNode& node); ~VInPredicate() = default; virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block, - int* result_column_id); + int* result_column_id) override; virtual doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc, - VExprContext* context); + VExprContext* context) override; virtual doris::Status open(doris::RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual void close(doris::RuntimeState* state, VExprContext* context, - FunctionContext::FunctionStateScope scope); + FunctionContext::FunctionStateScope scope) override; virtual VExpr* clone(doris::ObjectPool* pool) const override { return pool->add(new VInPredicate(*this)); } @@ -46,6 +45,7 @@ private: const bool _is_not_in; bool _is_prepare; + private: static const constexpr char* function_name = "in"; }; diff --git a/be/src/vec/exprs/vslot_ref.h b/be/src/vec/exprs/vslot_ref.h index 61d3eb99e3..975b90a223 100644 --- a/be/src/vec/exprs/vslot_ref.h +++ b/be/src/vec/exprs/vslot_ref.h @@ -28,16 +28,16 @@ public: VSlotRef(const doris::TExprNode& node); VSlotRef(const SlotDescriptor* desc); virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block, - int* result_column_id); + int* result_column_id) override; virtual doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc, - VExprContext* context); + VExprContext* context) override; virtual VExpr* clone(doris::ObjectPool* pool) const override { return pool->add(new VSlotRef(*this)); } virtual const std::string& expr_name() const override; - virtual std::string debug_string() const; - virtual bool is_constant() const { return false; } + virtual std::string debug_string() const override; + virtual bool is_constant() const override { return false; } private: FunctionPtr _function; diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index 769b5ef79d..053d44c0f4 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -374,9 +374,9 @@ struct ToNumberMonotonicity { Float64 right_float = right.get(); if (left_float >= std::numeric_limits::min() && - left_float <= std::numeric_limits::max() && + left_float <= static_cast(std::numeric_limits::max()) && right_float >= std::numeric_limits::min() && - right_float <= std::numeric_limits::max()) + right_float <= static_cast(std::numeric_limits::max())) return {true}; return {}; diff --git a/be/src/vec/functions/function_rpc.cpp b/be/src/vec/functions/function_rpc.cpp index 43d5a694e1..1003e0d3f8 100644 --- a/be/src/vec/functions/function_rpc.cpp +++ b/be/src/vec/functions/function_rpc.cpp @@ -495,8 +495,8 @@ void convert_to_block(Block& block, const PValues& result, size_t pos) { null_map_data[i] = false; } } - block.replace_by_position( - pos, std::move(ColumnNullable::create(std::move(data_col), std::move(null_col)))); + block.replace_by_position(pos, + ColumnNullable::create(std::move(data_col), std::move(null_col))); } else { auto column = data_type->create_column(); convert_to_column(column, result); diff --git a/be/src/vec/olap/vcollect_iterator.h b/be/src/vec/olap/vcollect_iterator.h index 999a12e70e..366d12658c 100644 --- a/be/src/vec/olap/vcollect_iterator.h +++ b/be/src/vec/olap/vcollect_iterator.h @@ -181,8 +181,6 @@ private: bool _skip_same; // used when `_merge == true` std::unique_ptr _heap; - // used when `_merge == false` - int _child_idx = 0; }; std::unique_ptr _inner_iter; diff --git a/be/src/vec/runtime/vdata_stream_recvr.h b/be/src/vec/runtime/vdata_stream_recvr.h index 1292b978b5..9b41e76d61 100644 --- a/be/src/vec/runtime/vdata_stream_recvr.h +++ b/be/src/vec/runtime/vdata_stream_recvr.h @@ -89,7 +89,7 @@ public: private: class SenderQueue; - friend class ReceiveQueueSortCursorImpl; + friend struct ReceiveQueueSortCursorImpl; bool exceeds_limit(int batch_size) { return _num_buffered_bytes + batch_size > _total_buffer_limit; @@ -140,7 +140,7 @@ public: private: std::condition_variable _cv; -}; +}; class VDataStreamRecvr::SenderQueue { public: diff --git a/be/src/vec/sink/result_sink.h b/be/src/vec/sink/result_sink.h index eba9d4dd78..f2dcd2a174 100644 --- a/be/src/vec/sink/result_sink.h +++ b/be/src/vec/sink/result_sink.h @@ -28,7 +28,7 @@ class BufferControlBlock; class ExprContext; class ResultWriter; class MemTracker; -class ResultFileOptions; +struct ResultFileOptions; namespace vectorized { class VExprContext; @@ -58,7 +58,6 @@ private: // set file options when sink type is FILE std::unique_ptr _file_opts; - ObjectPool* _obj_pool; // Owned by the RuntimeState. const RowDescriptor& _row_desc; diff --git a/be/src/vec/sink/vtablet_sink.h b/be/src/vec/sink/vtablet_sink.h index 8785b5ba18..5514ff1909 100644 --- a/be/src/vec/sink/vtablet_sink.h +++ b/be/src/vec/sink/vtablet_sink.h @@ -37,8 +37,8 @@ class OlapTableSink; class VOlapTableSink : public OlapTableSink { public: // Construct from thrift struct which is generated by FE. - VOlapTableSink(ObjectPool* pool, const RowDescriptor& row_desc, const std::vector& texprs, - Status* status); + VOlapTableSink(ObjectPool* pool, const RowDescriptor& row_desc, + const std::vector& texprs, Status* status); Status init(const TDataSink& sink) override; // TODO: unify the code of prepare/open/close with result sink @@ -47,7 +47,7 @@ public: Status open(RuntimeState* state) override; Status close(RuntimeState* state, Status close_status) override; - + using OlapTableSink::send; Status send(RuntimeState* state, vectorized::Block* block) override; private: @@ -55,8 +55,8 @@ private: // return number of invalid/filtered rows. // invalid row number is set in Bitmap // set stop_processing is we want to stop the whole process now. - Status _validate_data(RuntimeState* state, vectorized::Block* block, Bitmap* filter_bitmap, int* filtered_rows, - bool* stop_processing); + Status _validate_data(RuntimeState* state, vectorized::Block* block, Bitmap* filter_bitmap, + int* filtered_rows, bool* stop_processing); VOlapTablePartitionParam* _vpartition = nullptr; std::vector _output_vexpr_ctxs; diff --git a/be/test/env/env_posix_test.cpp b/be/test/env/env_posix_test.cpp index e2124f3ec7..0503b01502 100644 --- a/be/test/env/env_posix_test.cpp +++ b/be/test/env/env_posix_test.cpp @@ -41,7 +41,8 @@ TEST_F(EnvPosixTest, file_path_desc) { path_desc.storage_medium = TStorageMedium::S3; path_desc.remote_path = "/remote"; FilePathDescStream path_desc_stream; - path_desc_stream << path_desc << "/test" << "/" << 1; + path_desc_stream << path_desc << "/test" + << "/" << 1; FilePathDesc dest_path_desc = path_desc_stream.path_desc(); ASSERT_EQ("/local/test/1", dest_path_desc.filepath); ASSERT_EQ("/remote/test/1", dest_path_desc.remote_path); @@ -49,7 +50,6 @@ TEST_F(EnvPosixTest, file_path_desc) { TEST_F(EnvPosixTest, random_access) { std::string fname = "./ut_dir/env_posix/random_access"; - WritableFileOptions ops; std::unique_ptr wfile; auto env = Env::Default(); auto st = env->new_writable_file(fname, &wfile); @@ -68,7 +68,7 @@ TEST_F(EnvPosixTest, random_access) { ASSERT_TRUE(st.ok()); Slice abc("abc"); Slice bcd("bcd"); - Slice slices[2]{abc, bcd}; + Slice slices[2] {abc, bcd}; st = wfile->appendv(slices, 2); ASSERT_TRUE(st.ok()); st = wfile->flush(WritableFile::FLUSH_ASYNC); @@ -94,7 +94,7 @@ TEST_F(EnvPosixTest, random_access) { Slice slice2(mem + 9, 100); Slice slice3(mem + 9 + 100, 3); - Slice read_slices[3]{slice1, slice2, slice3}; + Slice read_slices[3] {slice1, slice2, slice3}; st = rfile->readv_at(0, read_slices, 3); ASSERT_TRUE(st.ok()); ASSERT_STREQ("123456789", std::string(slice1.data, slice1.size).c_str()); @@ -114,7 +114,6 @@ TEST_F(EnvPosixTest, random_access) { TEST_F(EnvPosixTest, random_rw) { std::string fname = "./ut_dir/env_posix/random_rw"; - WritableFileOptions ops; std::unique_ptr wfile; auto env = Env::Default(); auto st = env->new_random_rw_file(fname, &wfile); @@ -131,7 +130,7 @@ TEST_F(EnvPosixTest, random_rw) { ASSERT_TRUE(st.ok()); Slice abc("abc"); Slice bcd("bcd"); - Slice slices[2]{abc, bcd}; + Slice slices[2] {abc, bcd}; st = wfile->writev_at(0, slices, 2); ASSERT_TRUE(st.ok()); @@ -162,7 +161,7 @@ TEST_F(EnvPosixTest, random_rw) { Slice slice2(mem + 3, 3); Slice slice3(mem + 6, 3); - Slice read_slices[3]{slice1, slice2, slice3}; + Slice read_slices[3] {slice1, slice2, slice3}; st = rfile->readv_at(0, read_slices, 3); LOG(INFO) << st.to_string(); ASSERT_TRUE(st.ok()); @@ -171,11 +170,11 @@ TEST_F(EnvPosixTest, random_rw) { ASSERT_STREQ("789", std::string(slice3.data, slice3.size).c_str()); Slice slice4(mem, 100); - st = rfile->read_at(9, &slice4); + st = rfile->read_at(9, slice4); ASSERT_TRUE(st.ok()); // end of file - st = rfile->read_at(102, &slice4); + st = rfile->read_at(102, slice4); ASSERT_EQ(TStatusCode::END_OF_FILE, st.code()); LOG(INFO) << "st=" << st.to_string(); } diff --git a/be/test/exec/hash_table_test.cpp b/be/test/exec/hash_table_test.cpp index 3502d0b9d1..df5d29da52 100644 --- a/be/test/exec/hash_table_test.cpp +++ b/be/test/exec/hash_table_test.cpp @@ -353,8 +353,6 @@ TEST_F(HashTableTest, GrowTableTest) { // This test continues adding to the hash table to trigger the resize code paths TEST_F(HashTableTest, GrowTableTest2) { int build_row_val = 0; - int num_to_add = 1024; - int expected_size = 0; std::shared_ptr mem_tracker = MemTracker::CreateTracker(1024 * 1024 * 1024, "hash-table-grow2-tracker", _tracker); @@ -371,7 +369,6 @@ TEST_F(HashTableTest, GrowTableTest2) { for (int i = 0; i < test_size; ++i) { hash_table.insert(create_tuple_row(build_row_val++)); - expected_size += num_to_add; } LOG(INFO) << time(nullptr); diff --git a/be/test/gutil/strings/numbers_test.cpp b/be/test/gutil/strings/numbers_test.cpp index 49d80a1ff6..ceb0033dba 100644 --- a/be/test/gutil/strings/numbers_test.cpp +++ b/be/test/gutil/strings/numbers_test.cpp @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. +#include "gutil/strings/numbers.h" + +#include #include -#include -#include "gutil/strings/numbers.h" #include "util/mysql_global.h" namespace doris { @@ -47,7 +48,7 @@ TEST_F(NumbersTest, test_float_to_buffer) { EXPECT_EQ(std::string("20001230"), std::string(buffer1, len1)); EXPECT_EQ(std::string("20001230"), std::string(buffer2, len2)); - float v4 = 200012303131; + float v4 = static_cast(200012303131); len1 = FloatToBuffer(v4, MAX_FLOAT_STR_LENGTH, buffer1); len2 = FastFloatToBuffer(v4, buffer2); EXPECT_EQ(std::string("2.000123e+11"), std::string(buffer1, len1)); @@ -136,5 +137,3 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } - - diff --git a/be/test/http/http_client_test.cpp b/be/test/http/http_client_test.cpp index ee6e95918b..fa5850c872 100644 --- a/be/test/http/http_client_test.cpp +++ b/be/test/http/http_client_test.cpp @@ -111,7 +111,10 @@ TEST_F(HttpClientTest, get_normal) { client.set_basic_auth("test1", ""); st = client.execute(); ASSERT_TRUE(st.ok()); - ASSERT_EQ(5, client.get_content_length()); + uint64_t len = 0; + st = client.get_content_length(&len); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(5, len); } TEST_F(HttpClientTest, download) { diff --git a/be/test/http/stream_load_test.cpp b/be/test/http/stream_load_test.cpp index fc3435f67b..369af31834 100644 --- a/be/test/http/stream_load_test.cpp +++ b/be/test/http/stream_load_test.cpp @@ -33,7 +33,7 @@ #include "util/brpc_client_cache.h" #include "util/cpu_info.h" -class mg_connection; +struct mg_connection; namespace doris { diff --git a/be/test/olap/column_reader_test.cpp b/be/test/olap/column_reader_test.cpp index 9e48e60e28..1b9c641ee1 100644 --- a/be/test/olap/column_reader_test.cpp +++ b/be/test/olap/column_reader_test.cpp @@ -120,7 +120,7 @@ public: for (; it != _stream_factory->streams().end(); ++it) { StreamName stream_name = it->first; OutStream* out_stream = it->second; - std::vector* buffers; + std::vector* buffers = nullptr; if (out_stream->is_suppressed()) { continue; @@ -811,7 +811,6 @@ TEST_F(TestColumn, SeekShortColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -899,8 +898,6 @@ TEST_F(TestColumn, SkipShortColumnWithPresent) { &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -930,7 +927,6 @@ TEST_F(TestColumn, SkipShortColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -950,7 +946,6 @@ TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { &tablet_schema); create_column_writer(tablet_schema); - RowCursor write_row; write_row.init(tablet_schema); @@ -977,7 +972,6 @@ TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -999,7 +993,6 @@ TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { &tablet_schema); create_column_writer(tablet_schema); - RowCursor write_row; write_row.init(tablet_schema); @@ -1022,7 +1015,6 @@ TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1050,7 +1042,6 @@ TEST_F(TestColumn, VectorizedIntColumnWithPresent) { set_tablet_schema_with_one_column("IntColumn", "INT", "REPLACE", 4, true, true, &tablet_schema); create_column_writer(tablet_schema); - RowCursor write_row; write_row.init(tablet_schema); @@ -1077,7 +1068,6 @@ TEST_F(TestColumn, VectorizedIntColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1102,8 +1092,6 @@ TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1131,7 +1119,6 @@ TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1153,8 +1140,6 @@ TEST_F(TestColumn, VectorizedLongColumnWithPresent) { &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1182,7 +1167,6 @@ TEST_F(TestColumn, VectorizedLongColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1206,8 +1190,6 @@ TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1235,7 +1217,6 @@ TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1254,13 +1235,10 @@ TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1288,7 +1266,6 @@ TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1308,13 +1285,10 @@ TEST_F(TestColumn, SeekFloatColumnWithPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1346,7 +1320,6 @@ TEST_F(TestColumn, SeekFloatColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1378,13 +1351,10 @@ TEST_F(TestColumn, SkipFloatColumnWithPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1412,7 +1382,6 @@ TEST_F(TestColumn, SkipFloatColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1428,13 +1397,10 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DoubleColumnWithoutPresent", "DOUBLE", "REPLACE", 8, false, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1462,7 +1428,6 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1481,13 +1446,10 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DoubleColumnWithPresent", "DOUBLE", "REPLACE", 8, true, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1515,7 +1477,6 @@ TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1536,13 +1497,10 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, false, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1566,7 +1524,6 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1582,13 +1539,10 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, true, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1618,7 +1572,6 @@ TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1674,7 +1627,6 @@ TEST_F(TestColumn, VectorizedDatetimeColumnZero) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1699,13 +1651,10 @@ TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DateColumnWithoutoutPresent", "DATE", "REPLACE", 3, false, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1729,7 +1678,6 @@ TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1744,13 +1692,10 @@ TEST_F(TestColumn, VectorizedDateColumnWithPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DateColumnWithoutoutPresent", "DATE", "REPLACE", 3, true, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1782,7 +1727,6 @@ TEST_F(TestColumn, VectorizedDateColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1805,13 +1749,10 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DecimalColumnWithoutoutPresent", "DECIMAL", "REPLACE", 12, false, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1843,7 +1784,6 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1896,7 +1836,6 @@ TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1916,13 +1855,10 @@ TEST_F(TestColumn, SkipDecimalColumnWithPresent) { // write data TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DecimalColumnWithPresent", "DECIMAL", "REPLACE", 12, true, true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -1952,7 +1888,6 @@ TEST_F(TestColumn, SkipDecimalColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); @@ -1974,8 +1909,6 @@ TEST_F(TestColumn, SeekDecimalColumnWithPresent) { true, &tablet_schema); create_column_writer(tablet_schema); - - RowCursor write_row; write_row.init(tablet_schema); @@ -2011,7 +1944,6 @@ TEST_F(TestColumn, SeekDecimalColumnWithPresent) { create_column_reader(tablet_schema); - RowCursor read_row; read_row.init(tablet_schema); PositionEntryReader entry1; diff --git a/be/test/olap/delta_writer_test.cpp b/be/test/olap/delta_writer_test.cpp index 3e0272654f..bb3d740b8a 100644 --- a/be/test/olap/delta_writer_test.cpp +++ b/be/test/olap/delta_writer_test.cpp @@ -44,7 +44,6 @@ namespace doris { // This is DeltaWriter unit test which used by streaming load. // And also it should take schema change into account after streaming load. -static const uint32_t MAX_RETRY_TIMES = 10; static const uint32_t MAX_PATH_LEN = 1024; StorageEngine* k_engine = nullptr; @@ -492,9 +491,9 @@ TEST_F(TestDeltaWriter, write) { for (auto& tablet_rs : tablet_related_rs) { std::cout << "start to publish txn" << std::endl; RowsetSharedPtr rowset = tablet_rs.second; - res = k_engine->txn_manager()->publish_txn( - meta, write_req.partition_id, write_req.txn_id, write_req.tablet_id, - write_req.schema_hash, tablet_rs.first.tablet_uid, version); + res = k_engine->txn_manager()->publish_txn(meta, write_req.partition_id, write_req.txn_id, + write_req.tablet_id, write_req.schema_hash, + tablet_rs.first.tablet_uid, version); ASSERT_EQ(OLAP_SUCCESS, res); std::cout << "start to add inc rowset:" << rowset->rowset_id() << ", num rows:" << rowset->num_rows() << ", version:" << rowset->version().first @@ -570,9 +569,9 @@ TEST_F(TestDeltaWriter, sequence_col) { for (auto& tablet_rs : tablet_related_rs) { std::cout << "start to publish txn" << std::endl; RowsetSharedPtr rowset = tablet_rs.second; - res = k_engine->txn_manager()->publish_txn( - meta, write_req.partition_id, write_req.txn_id, write_req.tablet_id, - write_req.schema_hash, tablet_rs.first.tablet_uid, version); + res = k_engine->txn_manager()->publish_txn(meta, write_req.partition_id, write_req.txn_id, + write_req.tablet_id, write_req.schema_hash, + tablet_rs.first.tablet_uid, version); ASSERT_EQ(OLAP_SUCCESS, res); std::cout << "start to add inc rowset:" << rowset->rowset_id() << ", num rows:" << rowset->num_rows() << ", version:" << rowset->version().first diff --git a/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp b/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp index f79b068a51..e43f58c0ac 100644 --- a/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp +++ b/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp @@ -190,7 +190,7 @@ TEST_F(BitShufflePageTest, TestBitShuffleFloatBlockEncoderRandom) { std::unique_ptr floats(new float[size]); for (int i = 0; i < size; i++) { - floats.get()[i] = random() + static_cast(random()) / INT_MAX; + floats.get()[i] = random() + static_cast(random()) / static_cast(INT_MAX); } test_encode_decode_page_template floats(new float[size]); for (int i = 0; i < size; i++) { - floats.get()[i] = i + 100 + static_cast(random()) / INT_MAX; + floats.get()[i] = i + 100 + static_cast(random()) / static_cast(INT_MAX); } float small_than_smallest = 99.9; diff --git a/be/test/olap/rowset/segment_v2/plain_page_test.cpp b/be/test/olap/rowset/segment_v2/plain_page_test.cpp index 04eaf1af35..3691820853 100644 --- a/be/test/olap/rowset/segment_v2/plain_page_test.cpp +++ b/be/test/olap/rowset/segment_v2/plain_page_test.cpp @@ -229,7 +229,7 @@ TEST_F(PlainPageTest, TestPlainFloatBlockEncoderRandom) { std::unique_ptr floats(new float[size]); for (int i = 0; i < size; i++) { - floats.get()[i] = random() + static_cast(random()) / INT_MAX; + floats.get()[i] = random() + static_cast(random()) / static_cast(INT_MAX); } test_encode_decode_page_templatestreams().end(); ++it) { StreamName stream_name = it->first; OutStream* out_stream = it->second; - std::vector* buffers; + std::vector* buffers = nullptr; if (out_stream->is_suppressed()) { continue; diff --git a/be/test/olap/storage_types_test.cpp b/be/test/olap/storage_types_test.cpp index 78819cc92a..80de50f63a 100644 --- a/be/test/olap/storage_types_test.cpp +++ b/be/test/olap/storage_types_test.cpp @@ -126,7 +126,7 @@ void common_test(Slice src_val) { TEST(TypesTest, copy_and_equal) { common_test(true); common_test(112); - common_test(54321); + common_test(static_cast(54321)); common_test(-123454321); common_test(1234543212L); common_test(123454321123456789L); diff --git a/be/test/olap/timestamped_version_tracker_test.cpp b/be/test/olap/timestamped_version_tracker_test.cpp index 97a546bccb..b9f086145b 100644 --- a/be/test/olap/timestamped_version_tracker_test.cpp +++ b/be/test/olap/timestamped_version_tracker_test.cpp @@ -18,8 +18,8 @@ #include #include -#include #include +#include #include #include "gutil/strings/substitute.h" @@ -292,7 +292,6 @@ public: } private: - OlapMeta* _meta; std::string _json_rowset_meta; }; diff --git a/be/test/runtime/result_queue_mgr_test.cpp b/be/test/runtime/result_queue_mgr_test.cpp index d04040df90..a40e2faddd 100644 --- a/be/test/runtime/result_queue_mgr_test.cpp +++ b/be/test/runtime/result_queue_mgr_test.cpp @@ -77,9 +77,19 @@ TEST_F(ResultQueueMgrTest, fetch_result_normal) { std::shared_ptr k1_col; arrow::NumericBuilder builder; - builder.Reserve(1); - builder.Append(20); - builder.Finish(&k1_col); + + auto st = builder.Reserve(1); + if (!st.ok()) { + LOG(WARNING) << "Reserve error"; + } + st = builder.Append(20); + if (!st.ok()) { + LOG(WARNING) << "Append error"; + } + st = builder.Finish(&k1_col); + if (!st.ok()) { + LOG(WARNING) << "Finish error"; + } std::vector> arrays; arrays.push_back(k1_col); diff --git a/be/test/tools/benchmark_tool.cpp b/be/test/tools/benchmark_tool.cpp index 92fd034928..7a1f7087bc 100644 --- a/be/test/tools/benchmark_tool.cpp +++ b/be/test/tools/benchmark_tool.cpp @@ -135,7 +135,8 @@ private: class BinaryDictPageBenchmark : public BaseBenchmark { public: - BinaryDictPageBenchmark(const std::string& name, int iterations) : BaseBenchmark(name, iterations) {} + BinaryDictPageBenchmark(const std::string& name, int iterations) + : BaseBenchmark(name, iterations) {} virtual ~BinaryDictPageBenchmark() override {} virtual void init() override {} @@ -428,7 +429,8 @@ private: class SegmentWriteByFileBenchmark : public SegmentBenchmark { public: - SegmentWriteByFileBenchmark(const std::string& name, int iterations, const std::string& file_str) + SegmentWriteByFileBenchmark(const std::string& name, int iterations, + const std::string& file_str) : SegmentBenchmark(name + "/file_path:" + file_str, iterations) { std::ifstream file(file_str); assert(file.is_open()); @@ -458,7 +460,8 @@ private: class SegmentScanBenchmark : public SegmentBenchmark { public: - SegmentScanBenchmark(const std::string& name, int iterations, const std::string& column_type, int rows_number) + SegmentScanBenchmark(const std::string& name, int iterations, const std::string& column_type, + int rows_number) : SegmentBenchmark(name + "/rows_number:" + std::to_string(rows_number), iterations, column_type), _dataset(generate_dataset(rows_number)) {} @@ -473,13 +476,11 @@ public: RowBlockV2 block(get_schema(), 1024); int left = _dataset.size(); - int rowid = 0; while (left > 0) { int rows_read = std::min(left, 1024); block.clear(); iter->next_batch(&block); left -= rows_read; - rowid += rows_read; } } @@ -520,13 +521,11 @@ public: RowBlockV2 block(get_schema(), 1024); int left = _dataset.size(); - int rowid = 0; while (left > 0) { int rows_read = std::min(left, 1024); block.clear(); iter->next_batch(&block); left -= rows_read; - rowid += rows_read; } } diff --git a/be/test/util/broker_storage_backend_test.cpp b/be/test/util/broker_storage_backend_test.cpp index 548ce3109a..7ff0d05e65 100644 --- a/be/test/util/broker_storage_backend_test.cpp +++ b/be/test/util/broker_storage_backend_test.cpp @@ -37,7 +37,6 @@ static const std::string SK = "SK"; static const std::string ENDPOINT = "http://bj.bcebos.com"; static const std::string BUCKET = "bos://yang-repo/"; static const std::string BROKER_IP = "127.0.0.1"; -static const int BROKER_PORT = 8111; class StorageBackendTest : public testing::Test { public: StorageBackendTest() diff --git a/env.sh b/env.sh index 439de9e0ff..1eb5239125 100755 --- a/env.sh +++ b/env.sh @@ -91,6 +91,9 @@ elif [[ "${DORIS_TOOLCHAIN}" == "clang" ]]; then if test -x ${DORIS_CLANG_HOME}/bin/ld.lld; then export DORIS_BIN_UTILS=${DORIS_CLANG_HOME}/bin/ fi + if [[ -f ${DORIS_CLANG_HOME}/bin/llvm-symbolizer ]]; then + export ASAN_SYMBOLIZER_PATH=${DORIS_CLANG_HOME}/bin/llvm-symbolizer + fi else echo "Error: unknown DORIS_TOOLCHAIN=${DORIS_TOOLCHAIN}, currently only 'gcc' and 'clang' are supported" exit 1