diff --git a/be/src/agent/agent_server.h b/be/src/agent/agent_server.h index 11521eb09a..3aebc3d1f7 100644 --- a/be/src/agent/agent_server.h +++ b/be/src/agent/agent_server.h @@ -18,14 +18,11 @@ #pragma once #include -#include #include #include #include -#include "runtime/exec_env.h" - namespace doris { class TaskWorkerPool; diff --git a/be/src/agent/heartbeat_server.h b/be/src/agent/heartbeat_server.h index 65e228dd8c..e08603043f 100644 --- a/be/src/agent/heartbeat_server.h +++ b/be/src/agent/heartbeat_server.h @@ -24,8 +24,6 @@ #include #include "common/status.h" -#include "olap/olap_define.h" -#include "runtime/exec_env.h" namespace doris { class ExecEnv; @@ -34,7 +32,6 @@ class TMasterInfo; const uint32_t HEARTBEAT_INTERVAL = 10; class StorageEngine; -class Status; class ThriftServer; class HeartbeatServer : public HeartbeatServiceIf { diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index ab7877fbf2..f93cee6a6b 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -43,7 +43,7 @@ #include "common/logging.h" #include "common/status.h" #include "gutil/ref_counted.h" -#include "gutil/stringprintf.h" +#include "gutil/strings/numbers.h" #include "gutil/strings/substitute.h" #include "io/fs/file_system.h" #include "io/fs/local_file_system.h" diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h index cd9b274d98..2f90f7b685 100644 --- a/be/src/agent/task_worker_pool.h +++ b/be/src/agent/task_worker_pool.h @@ -19,7 +19,6 @@ #include #include -#include #include #include @@ -31,11 +30,8 @@ #include #include #include -#include -#include #include "common/status.h" -#include "olap/data_dir.h" #include "olap/tablet.h" #include "util/countdown_latch.h" #include "util/metrics.h" diff --git a/be/src/agent/topic_listener.h b/be/src/agent/topic_listener.h index 1ad66b31a6..0ef9c597f1 100644 --- a/be/src/agent/topic_listener.h +++ b/be/src/agent/topic_listener.h @@ -17,7 +17,7 @@ #pragma once -#include "gen_cpp/AgentService_types.h" +#include namespace doris { diff --git a/be/src/agent/topic_subscriber.h b/be/src/agent/topic_subscriber.h index 3151cbe54c..490bd35d2b 100644 --- a/be/src/agent/topic_subscriber.h +++ b/be/src/agent/topic_subscriber.h @@ -20,13 +20,9 @@ #include #include -#include #include -#include #include -#include "agent/topic_listener.h" - namespace doris { class TopicListener; diff --git a/be/src/agent/user_resource_listener.h b/be/src/agent/user_resource_listener.h index b26ae3f279..6fd25bef67 100644 --- a/be/src/agent/user_resource_listener.h +++ b/be/src/agent/user_resource_listener.h @@ -18,13 +18,9 @@ #pragma once #include -#include #include -#include - #include "agent/topic_listener.h" -#include "runtime/exec_env.h" namespace doris { diff --git a/be/src/agent/utils.cpp b/be/src/agent/utils.cpp index 0e91eeaa49..19792fd1f8 100644 --- a/be/src/agent/utils.cpp +++ b/be/src/agent/utils.cpp @@ -17,7 +17,8 @@ #include "agent/utils.h" -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include diff --git a/be/src/agent/utils.h b/be/src/agent/utils.h index 8cd1895d6a..eea8f9f8cf 100644 --- a/be/src/agent/utils.h +++ b/be/src/agent/utils.h @@ -18,15 +18,11 @@ #pragma once #include -#include -#include -#include #include #include #include "common/status.h" -#include "gutil/macros.h" namespace doris { class TConfirmUnusedRemoteFilesRequest; diff --git a/be/src/common/configbase.cpp b/be/src/common/configbase.cpp index 066aa0352d..d965687a23 100644 --- a/be/src/common/configbase.cpp +++ b/be/src/common/configbase.cpp @@ -19,7 +19,8 @@ #include #include -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include // IWYU pragma: keep @@ -36,6 +37,7 @@ #define __IN_CONFIGBASE_CPP__ #include "common/config.h" // IWYU pragma: keep + #undef __IN_CONFIGBASE_CPP__ #include "common/status.h" diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 6ef0d751f3..5c832cfd77 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -17,7 +17,8 @@ #include "common/daemon.h" -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include // IWYU pragma: keep // IWYU pragma: no_include @@ -31,7 +32,6 @@ // IWYU pragma: no_include #include // IWYU pragma: keep #include -#include #include #include #include @@ -41,6 +41,7 @@ #include "common/status.h" #include "olap/options.h" #include "olap/storage_engine.h" +#include "olap/tablet_manager.h" #include "runtime/block_spill_manager.h" #include "runtime/exec_env.h" #include "runtime/load_channel_mgr.h" @@ -53,8 +54,10 @@ #include "util/disk_info.h" #include "util/doris_metrics.h" #include "util/mem_info.h" +#include "util/metrics.h" #include "util/network_util.h" #include "util/perf_counters.h" +#include "util/system_metrics.h" #include "util/thrift_util.h" #include "util/time.h" diff --git a/be/src/common/daemon.h b/be/src/common/daemon.h index ef7ece4620..6fd287a537 100644 --- a/be/src/common/daemon.h +++ b/be/src/common/daemon.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include "gutil/ref_counted.h" @@ -27,7 +26,6 @@ namespace doris { struct StorePath; -class Thread; class Daemon { public: diff --git a/be/src/common/signal_handler.h b/be/src/common/signal_handler.h index 85da4e7096..3be54d0d39 100644 --- a/be/src/common/signal_handler.h +++ b/be/src/common/signal_handler.h @@ -33,14 +33,13 @@ #pragma once +#include #include #include #include #include #include - -#include "gen_cpp/version.h" #ifdef HAVE_UCONTEXT_H #include #endif diff --git a/be/src/common/status.h b/be/src/common/status.h index d989ba12c3..9f71f167ff 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -15,7 +15,8 @@ #include #include -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #ifdef ENABLE_STACKTRACE #include "util/stack_util.h" #endif diff --git a/be/src/exec/arrow/arrow_reader.cpp b/be/src/exec/arrow/arrow_reader.cpp index 6696d5489c..8e0559ca78 100644 --- a/be/src/exec/arrow/arrow_reader.cpp +++ b/be/src/exec/arrow/arrow_reader.cpp @@ -16,9 +16,7 @@ // under the License. #include "exec/arrow/arrow_reader.h" -#include #include -#include #include // IWYU pragma: no_include @@ -26,6 +24,8 @@ #include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" #include "io/fs/file_reader.h" #include "runtime/descriptors.h" diff --git a/be/src/exec/arrow/arrow_reader.h b/be/src/exec/arrow/arrow_reader.h index e685e7edaf..ad4c40d26b 100644 --- a/be/src/exec/arrow/arrow_reader.h +++ b/be/src/exec/arrow/arrow_reader.h @@ -17,20 +17,8 @@ #pragma once -#include -#include -#include -#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include #include #include #include @@ -47,7 +35,6 @@ #include "common/config.h" #include "common/status.h" -#include "io/fs/file_reader.h" #include "io/fs/file_reader_writer_fwd.h" #include "vec/exec/format/generic_reader.h" @@ -58,12 +45,8 @@ class RecordBatchReader; namespace doris { -class ExecEnv; -class TBrokerRangeDesc; -class TNetworkAddress; class RuntimeState; class SlotDescriptor; -class FileReader; class TupleDescriptor; namespace vectorized { diff --git a/be/src/exec/arrow/parquet_reader.cpp b/be/src/exec/arrow/parquet_reader.cpp index f0a1755324..4b924c0275 100644 --- a/be/src/exec/arrow/parquet_reader.cpp +++ b/be/src/exec/arrow/parquet_reader.cpp @@ -20,14 +20,12 @@ #include #include #include -#include #include #include #include #include #include -#include #include // IWYU pragma: no_include #include // IWYU pragma: keep @@ -38,6 +36,8 @@ #include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" #include "common/status.h" #include "util/string_util.h" diff --git a/be/src/exec/arrow/parquet_reader.h b/be/src/exec/arrow/parquet_reader.h index 4f57fdc7b9..ac60a25eae 100644 --- a/be/src/exec/arrow/parquet_reader.h +++ b/be/src/exec/arrow/parquet_reader.h @@ -17,34 +17,14 @@ #pragma once -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include #include -#include -#include #include -#include -#include -#include -#include #include -#include #include -#include #include -#include "common/config.h" #include "common/status.h" #include "exec/arrow/arrow_reader.h" #include "io/fs/file_reader_writer_fwd.h" @@ -58,13 +38,8 @@ class FileMetaData; namespace doris { -class ExecEnv; -class TBrokerRangeDesc; -class TNetworkAddress; class RuntimeState; class SlotDescriptor; -class FileReader; -class RowGroupReader; class TupleDescriptor; // Reader of parquet file diff --git a/be/src/exec/base_scanner.cpp b/be/src/exec/base_scanner.cpp index 20633d2c48..1f9b49a0f8 100644 --- a/be/src/exec/base_scanner.cpp +++ b/be/src/exec/base_scanner.cpp @@ -22,16 +22,18 @@ #include #include #include -#include #include #include +#include #include #include #include #include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/consts.h" #include "gutil/casts.h" #include "runtime/define_primitive_type.h" diff --git a/be/src/exec/base_scanner.h b/be/src/exec/base_scanner.h index 291f7d653d..ee4ae973ef 100644 --- a/be/src/exec/base_scanner.h +++ b/be/src/exec/base_scanner.h @@ -33,12 +33,9 @@ #include "vec/common/schema_util.h" #include "vec/core/block.h" #include "vec/exprs/vexpr.h" -#include "vec/exprs/vexpr_context.h" namespace doris { -class TupleDescriptor; -class RowDescriptor; class RuntimeState; class TBrokerRangeDesc; class TBrokerScanRangeParams; @@ -46,7 +43,6 @@ class TNetworkAddress; namespace vectorized { class VExprContext; -class IColumn; using MutableColumnPtr = IColumn::MutablePtr; } // namespace vectorized diff --git a/be/src/exec/data_sink.h b/be/src/exec/data_sink.h index 3ea5da0c3c..ee847da138 100644 --- a/be/src/exec/data_sink.h +++ b/be/src/exec/data_sink.h @@ -20,8 +20,6 @@ #pragma once -#include -#include #include #include // IWYU pragma: no_include @@ -30,8 +28,6 @@ #include #include "common/status.h" -#include "runtime/descriptors.h" -#include "runtime/query_statistics.h" #include "util/runtime_profile.h" #include "util/telemetry/telemetry.h" diff --git a/be/src/exec/es/es_scan_reader.h b/be/src/exec/es/es_scan_reader.h index 95b3e37980..a29dced15f 100644 --- a/be/src/exec/es/es_scan_reader.h +++ b/be/src/exec/es/es_scan_reader.h @@ -21,7 +21,6 @@ #include #include -#include "exec/es/es_scroll_parser.h" #include "http/http_client.h" namespace doris { diff --git a/be/src/exec/es/es_scroll_parser.h b/be/src/exec/es/es_scroll_parser.h index d4edcb9ed2..be3f9fa8c5 100644 --- a/be/src/exec/es/es_scroll_parser.h +++ b/be/src/exec/es/es_scroll_parser.h @@ -24,8 +24,6 @@ #include #include "rapidjson/document.h" -#include "runtime/descriptors.h" -#include "vec/core/block.h" #include "vec/data_types/data_type.h" namespace doris { diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp index aced73a595..68360adba2 100644 --- a/be/src/exec/exec_node.cpp +++ b/be/src/exec/exec_node.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include @@ -30,6 +29,8 @@ #include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" #include "common/logging.h" #include "common/object_pool.h" @@ -43,7 +44,6 @@ #include "util/uid_util.h" #include "vec/columns/column.h" #include "vec/columns/column_nullable.h" -#include "vec/common/pod_array_fwd.h" #include "vec/core/block.h" #include "vec/core/column_with_type_and_name.h" #include "vec/exec/join/vhash_join_node.h" @@ -71,6 +71,7 @@ #include "vec/exec/vunion_node.h" #include "vec/exprs/vexpr.h" #include "vec/exprs/vexpr_context.h" +#include "vec/runtime/vdatetime_value.h" #include "vec/utils/util.hpp" namespace doris { diff --git a/be/src/exec/exec_node.h b/be/src/exec/exec_node.h index d5b9688361..68f84934ad 100644 --- a/be/src/exec/exec_node.h +++ b/be/src/exec/exec_node.h @@ -35,31 +35,21 @@ #include "common/global_types.h" #include "common/status.h" #include "runtime/descriptors.h" -#include "runtime/query_statistics.h" -#include "service/backend_options.h" -#include "util/blocking_queue.hpp" #include "util/runtime_profile.h" #include "util/telemetry/telemetry.h" #include "vec/core/block.h" -#include "vec/exprs/vexpr_context.h" namespace doris { class ObjectPool; -class Counters; class RuntimeState; -class TPlan; class MemTracker; class QueryStatistics; namespace vectorized { -class Block; -class VExpr; class VExprContext; } // namespace vectorized namespace pipeline { -class PipelineFragmentContext; -class Pipeline; class OperatorBase; } // namespace pipeline diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h index e160d54fa3..2260dac25c 100644 --- a/be/src/exec/olap_common.h +++ b/be/src/exec/olap_common.h @@ -25,12 +25,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include diff --git a/be/src/exec/olap_utils.h b/be/src/exec/olap_utils.h index 610299cc0f..153d4f7963 100644 --- a/be/src/exec/olap_utils.h +++ b/be/src/exec/olap_utils.h @@ -17,10 +17,10 @@ #pragma once +#include #include #include "common/logging.h" -#include "gen_cpp/Opcodes_types.h" #include "olap/olap_tuple.h" #include "runtime/datetime_value.h" #include "runtime/primitive_type.h" diff --git a/be/src/exec/rowid_fetcher.h b/be/src/exec/rowid_fetcher.h index 19dabdf901..7f789b1c40 100644 --- a/be/src/exec/rowid_fetcher.h +++ b/be/src/exec/rowid_fetcher.h @@ -23,7 +23,6 @@ #include #include "common/status.h" -#include "vec/core/block.h" #include "vec/data_types/data_type.h" namespace doris { diff --git a/be/src/exec/scan_node.h b/be/src/exec/scan_node.h index aed701dbe9..6df382a975 100644 --- a/be/src/exec/scan_node.h +++ b/be/src/exec/scan_node.h @@ -20,8 +20,6 @@ #pragma once -#include - #include #include #include @@ -32,7 +30,6 @@ namespace doris { -class TScanRange; class DescriptorTbl; class ObjectPool; class RuntimeState; diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index bc600bdf58..bc2d7bcfc0 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include @@ -26,11 +25,9 @@ #include #include -#include "common/object_pool.h" #include "common/status.h" #include "runtime/define_primitive_type.h" #include "util/runtime_profile.h" -#include "vec/core/block.h" namespace doris { diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h index d6a340e1e4..72b7a9acf0 100644 --- a/be/src/exec/schema_scanner/schema_helper.h +++ b/be/src/exec/schema_scanner/schema_helper.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.h b/be/src/exec/schema_scanner/schema_rowsets_scanner.h index 0a8dabc270..7a6b528ec2 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.h @@ -19,14 +19,11 @@ #include #include -#include #include #include "common/status.h" #include "exec/schema_scanner.h" #include "olap/rowset/rowset.h" -#include "olap/rowset/segment_v2/segment.h" -#include "runtime/runtime_state.h" namespace doris { class RuntimeState; diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index c295fc6ba2..7cdb363774 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -23,7 +23,6 @@ #include "common/status.h" #include "exec/schema_scanner.h" -#include "vec/core/block.h" namespace doris { class RuntimeState; diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.h b/be/src/exec/schema_scanner/schema_variables_scanner.h index 2728b70530..d55f234df1 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.h +++ b/be/src/exec/schema_scanner/schema_variables_scanner.h @@ -20,8 +20,6 @@ #include #include -#include -#include #include #include "common/status.h" diff --git a/be/src/exec/table_connector.cpp b/be/src/exec/table_connector.cpp index ef0b9db8c3..0a469defca 100644 --- a/be/src/exec/table_connector.cpp +++ b/be/src/exec/table_connector.cpp @@ -17,14 +17,13 @@ #include "exec/table_connector.h" -#include -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include #include -#include #include #include #include diff --git a/be/src/exec/table_connector.h b/be/src/exec/table_connector.h index 27d3061e75..8a1cdff7e3 100644 --- a/be/src/exec/table_connector.h +++ b/be/src/exec/table_connector.h @@ -21,18 +21,14 @@ #include #include -#include -#include #include #include #include "common/status.h" -#include "runtime/descriptors.h" #include "runtime/types.h" #include "util/runtime_profile.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/data_types/data_type.h" -#include "vec/exprs/vexpr_context.h" namespace doris { class RuntimeState; diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h index 6c9272787d..4840187e56 100644 --- a/be/src/exec/tablet_info.h +++ b/be/src/exec/tablet_info.h @@ -32,12 +32,9 @@ #include "common/object_pool.h" #include "common/status.h" -#include "olap/tablet_schema.h" -#include "runtime/descriptors.h" #include "vec/columns/column.h" #include "vec/core/block.h" #include "vec/core/column_with_type_and_name.h" -#include "vec/exprs/vexpr_context.h" namespace doris { class MemTracker; diff --git a/be/src/exec/text_converter.cpp b/be/src/exec/text_converter.cpp index 8249b42a98..1dcec677bf 100644 --- a/be/src/exec/text_converter.cpp +++ b/be/src/exec/text_converter.cpp @@ -21,9 +21,11 @@ #include #include +#include #include -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "olap/hll.h" #include "runtime/decimalv2_value.h" #include "runtime/define_primitive_type.h" diff --git a/be/src/exec/text_converter.h b/be/src/exec/text_converter.h index 0bf3b3f16e..d70cfba982 100644 --- a/be/src/exec/text_converter.h +++ b/be/src/exec/text_converter.h @@ -24,9 +24,6 @@ namespace doris { class SlotDescriptor; -class Status; -struct StringRef; -class TupleDescriptor; // Helper class for dealing with text data, e.g., converting text data to // numeric types, etc. diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h index d127a4c19b..eb8ee79e0d 100644 --- a/be/src/exprs/bloom_filter_func.h +++ b/be/src/exprs/bloom_filter_func.h @@ -19,6 +19,7 @@ #include "exprs/block_bloom_filter.hpp" #include "exprs/runtime_filter.h" +#include "olap/rowset/segment_v2/bloom_filter.h" // IWYU pragma: keep namespace doris { @@ -473,4 +474,4 @@ private: typename BloomFilterTypeTraits::FindOp dummy; }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/exprs/function_filter.h b/be/src/exprs/function_filter.h index a9c2f264b4..9802f1b408 100644 --- a/be/src/exprs/function_filter.h +++ b/be/src/exprs/function_filter.h @@ -19,6 +19,7 @@ #include #include "udf/udf.h" +#include "vec/common/string_ref.h" // IWYU pragma: keep namespace doris { diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp index ebfc4b0ae8..107217ed4f 100644 --- a/be/src/exprs/json_functions.cpp +++ b/be/src/exprs/json_functions.cpp @@ -32,7 +32,8 @@ #include #include -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" namespace doris { diff --git a/be/src/exprs/json_functions.h b/be/src/exprs/json_functions.h index 2de0857219..fcec257142 100644 --- a/be/src/exprs/json_functions.h +++ b/be/src/exprs/json_functions.h @@ -17,10 +17,9 @@ #pragma once -#include #include #include -#include +#include // IWYU pragma: keep #include #include @@ -28,7 +27,6 @@ #include #include "common/status.h" -#include "udf/udf.h" namespace simdjson { namespace fallback { @@ -49,8 +47,6 @@ enum JsonFunctionType { JSON_FUN_UNKNOWN //The last }; -class OpcodeRegistry; - struct JsonPath { std::string key; // key of a json object int idx; // array index of a json array, -1 means not set, -2 means * diff --git a/be/src/exprs/math_functions.cpp b/be/src/exprs/math_functions.cpp index e76cdde2a5..bbfdc6053d 100644 --- a/be/src/exprs/math_functions.cpp +++ b/be/src/exprs/math_functions.cpp @@ -22,12 +22,14 @@ #include #include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep // IWYU pragma: no_include #include #include #include -#include "common/compiler_util.h" #include "udf/udf.h" #include "util/string_parser.hpp" diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h index 141df632c5..a76b02080f 100644 --- a/be/src/exprs/runtime_filter.h +++ b/be/src/exprs/runtime_filter.h @@ -50,15 +50,12 @@ class IOBufAsZeroCopyInputStream; namespace doris { class ObjectPool; class RuntimePredicateWrapper; -class MemTracker; class PPublishFilterRequest; class PMergeFilterRequest; class TRuntimeFilterDesc; class RowDescriptor; class PInFilter; class PMinMaxFilter; -class HashJoinNode; -class RuntimeProfile; class BloomFilterFuncBase; class BitmapFilterFuncBase; class TNetworkAddress; diff --git a/be/src/exprs/runtime_filter_slots.h b/be/src/exprs/runtime_filter_slots.h index 71084bacfb..2d3b0c2c73 100644 --- a/be/src/exprs/runtime_filter_slots.h +++ b/be/src/exprs/runtime_filter_slots.h @@ -23,6 +23,7 @@ #include "vec/columns/column_nullable.h" #include "vec/columns/columns_number.h" #include "vec/common/assert_cast.h" +#include "vec/core/block.h" // IWYU pragma: keep #include "vec/runtime/shared_hash_table_controller.h" namespace doris { diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index 5a64790032..8ed917b82d 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -22,14 +22,9 @@ #include -#include -#include #include -#include #include -#include -#include "gutil/strings/numbers.h" #include "vec/common/string_ref.h" namespace doris { diff --git a/be/src/geo/ByteOrderValues.h b/be/src/geo/ByteOrderValues.h index c78f6c8856..50915953e0 100644 --- a/be/src/geo/ByteOrderValues.h +++ b/be/src/geo/ByteOrderValues.h @@ -19,8 +19,6 @@ #include -#include - using int64_t = __int64_t; using int32_t = __int32_t; using uint32_t = __uint32_t; diff --git a/be/src/geo/geo_tobinary.h b/be/src/geo/geo_tobinary.h index 8b861a519a..ad4ee9d0a4 100644 --- a/be/src/geo/geo_tobinary.h +++ b/be/src/geo/geo_tobinary.h @@ -21,10 +21,6 @@ #include -#include "geo/geo_common.h" -#include "geo/geo_tobinary_type.h" -#include "geo/wkt_parse_type.h" - struct ToBinaryContext; namespace doris { diff --git a/be/src/geo/geo_types.h b/be/src/geo/geo_types.h index 7416dccb35..e3d16d67c1 100644 --- a/be/src/geo/geo_types.h +++ b/be/src/geo/geo_types.h @@ -21,7 +21,6 @@ #include #include -#include #include "geo/geo_common.h" #include "geo/wkt_parse_type.h" diff --git a/be/src/geo/wkb_parse.h b/be/src/geo/wkb_parse.h index c27537966b..966e290733 100644 --- a/be/src/geo/wkb_parse.h +++ b/be/src/geo/wkb_parse.h @@ -22,9 +22,7 @@ #include #include "geo/geo_common.h" -#include "geo/geo_types.h" #include "geo/wkt_parse_type.h" -#include "wkb_parse_ctx.h" struct WkbParseContext; diff --git a/be/src/geo/wkt_parse.h b/be/src/geo/wkt_parse.h index 0bb999af73..9dfcca98dd 100644 --- a/be/src/geo/wkt_parse.h +++ b/be/src/geo/wkt_parse.h @@ -19,8 +19,6 @@ #include -#include - #include "geo/geo_common.h" namespace doris { diff --git a/be/src/gutil/atomicops.h b/be/src/gutil/atomicops.h index 7873a39170..4c6d93e28c 100644 --- a/be/src/gutil/atomicops.h +++ b/be/src/gutil/atomicops.h @@ -72,13 +72,13 @@ // ThreadSanitizer provides own implementation of atomicops. #if defined(THREAD_SANITIZER) -#include "gutil/atomicops-internals-tsan.h" +#include "gutil/atomicops-internals-tsan.h" // IWYU pragma: export #elif defined(__GNUC__) && (defined(__i386) || defined(__x86_64__)) -#include "gutil/atomicops-internals-x86.h" +#include "gutil/atomicops-internals-x86.h" // IWYU pragma: export #elif defined(__GNUC__) && GCC_VERSION >= 40700 -#include "gutil/atomicops-internals-gcc.h" +#include "gutil/atomicops-internals-gcc.h" // IWYU pragma: export #elif defined(__clang__) && CLANG_VERSION >= 30400 -#include "gutil/atomicops-internals-gcc.h" +#include "gutil/atomicops-internals-gcc.h" // IWYU pragma: export #else #error You need to implement atomic operations for this architecture #endif diff --git a/be/src/http/action/check_tablet_segment_action.h b/be/src/http/action/check_tablet_segment_action.h index a4c6f422ee..0cc26e8199 100644 --- a/be/src/http/action/check_tablet_segment_action.h +++ b/be/src/http/action/check_tablet_segment_action.h @@ -20,7 +20,6 @@ #include #include "http/http_handler.h" -#include "util/easy_json.h" namespace doris { class HttpRequest; diff --git a/be/src/http/action/checksum_action.h b/be/src/http/action/checksum_action.h index 5afacbfe6d..4ec478acfc 100644 --- a/be/src/http/action/checksum_action.h +++ b/be/src/http/action/checksum_action.h @@ -23,7 +23,6 @@ namespace doris { -class ExecEnv; class HttpRequest; class ChecksumAction : public HttpHandler { diff --git a/be/src/http/action/download_action.h b/be/src/http/action/download_action.h index 86bee3e2b1..3965461844 100644 --- a/be/src/http/action/download_action.h +++ b/be/src/http/action/download_action.h @@ -21,9 +21,7 @@ #include #include "common/status.h" -#include "exec/scan_node.h" #include "http/http_handler.h" -#include "runtime/descriptors.h" namespace doris { diff --git a/be/src/http/action/health_action.h b/be/src/http/action/health_action.h index cdee0f12b4..6724ef0856 100644 --- a/be/src/http/action/health_action.h +++ b/be/src/http/action/health_action.h @@ -21,7 +21,6 @@ namespace doris { -class ExecEnv; class HttpRequest; // Get BE health state from http API. diff --git a/be/src/http/action/meta_action.h b/be/src/http/action/meta_action.h index c4809cd3b5..fe59ed2744 100644 --- a/be/src/http/action/meta_action.h +++ b/be/src/http/action/meta_action.h @@ -24,7 +24,6 @@ namespace doris { -class ExecEnv; class HttpRequest; // Get Meta Info diff --git a/be/src/http/action/metrics_action.h b/be/src/http/action/metrics_action.h index 25664a98aa..2051aed53a 100644 --- a/be/src/http/action/metrics_action.h +++ b/be/src/http/action/metrics_action.h @@ -21,8 +21,6 @@ namespace doris { -class Webserver; -class ExecEnv; class HttpRequest; class MetricRegistry; diff --git a/be/src/http/action/monitor_action.h b/be/src/http/action/monitor_action.h index 8d3f1c87d7..b4bbe7a09c 100644 --- a/be/src/http/action/monitor_action.h +++ b/be/src/http/action/monitor_action.h @@ -25,7 +25,6 @@ namespace doris { class HttpRequest; -class HttpChannel; class RestMonitorIface; class MonitorAction : public HttpHandler { diff --git a/be/src/http/action/pad_rowset_action.h b/be/src/http/action/pad_rowset_action.h index 13090dcd8c..26ff6b6854 100644 --- a/be/src/http/action/pad_rowset_action.h +++ b/be/src/http/action/pad_rowset_action.h @@ -19,7 +19,6 @@ #include "common/status.h" #include "http/http_handler.h" -#include "http/http_request.h" #include "olap/tablet.h" namespace doris { diff --git a/be/src/http/action/pprof_actions.cpp b/be/src/http/action/pprof_actions.cpp index 56ae5e49e9..1d71281273 100644 --- a/be/src/http/action/pprof_actions.cpp +++ b/be/src/http/action/pprof_actions.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include "common/config.h" #include "common/object_pool.h" diff --git a/be/src/http/action/reload_tablet_action.h b/be/src/http/action/reload_tablet_action.h index aa36a474dd..8ecc946bb0 100644 --- a/be/src/http/action/reload_tablet_action.h +++ b/be/src/http/action/reload_tablet_action.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include diff --git a/be/src/http/action/restore_tablet_action.cpp b/be/src/http/action/restore_tablet_action.cpp index d710a2b36b..7ff9aa37f3 100644 --- a/be/src/http/action/restore_tablet_action.cpp +++ b/be/src/http/action/restore_tablet_action.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include diff --git a/be/src/http/action/restore_tablet_action.h b/be/src/http/action/restore_tablet_action.h index a11d7783b7..2eccb22303 100644 --- a/be/src/http/action/restore_tablet_action.h +++ b/be/src/http/action/restore_tablet_action.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include diff --git a/be/src/http/action/snapshot_action.h b/be/src/http/action/snapshot_action.h index 0d1399604d..b1b58bee10 100644 --- a/be/src/http/action/snapshot_action.h +++ b/be/src/http/action/snapshot_action.h @@ -24,7 +24,6 @@ namespace doris { -class ExecEnv; class HttpRequest; // make snapshot diff --git a/be/src/http/action/stream_load.h b/be/src/http/action/stream_load.h index 3c44a77bb4..4319d722d1 100644 --- a/be/src/http/action/stream_load.h +++ b/be/src/http/action/stream_load.h @@ -17,15 +17,10 @@ #pragma once -#include - -#include #include #include #include "http/http_handler.h" -#include "runtime/client_cache.h" -#include "runtime/message_body_sink.h" #include "util/metrics.h" namespace doris { diff --git a/be/src/http/action/tablet_migration_action.h b/be/src/http/action/tablet_migration_action.h index aee9a65d12..11933e7fc7 100644 --- a/be/src/http/action/tablet_migration_action.h +++ b/be/src/http/action/tablet_migration_action.h @@ -17,7 +17,6 @@ #pragma once -#include #include #include @@ -29,10 +28,8 @@ #include #include "common/status.h" -#include "gutil/stringprintf.h" -#include "gutil/strings/numbers.h" +#include "gutil/strings/substitute.h" #include "http/http_handler.h" -#include "olap/data_dir.h" #include "olap/tablet.h" #include "util/threadpool.h" diff --git a/be/src/http/action/tablets_info_action.cpp b/be/src/http/action/tablets_info_action.cpp index 656fbdf158..374f7b2136 100644 --- a/be/src/http/action/tablets_info_action.cpp +++ b/be/src/http/action/tablets_info_action.cpp @@ -26,7 +26,7 @@ #include #include -#include "gutil/strings/numbers.h" +#include "gutil/strings/substitute.h" #include "http/http_channel.h" #include "http/http_headers.h" #include "http/http_request.h" diff --git a/be/src/http/action/version_action.h b/be/src/http/action/version_action.h index e78e19f37c..ed9a48ec61 100644 --- a/be/src/http/action/version_action.h +++ b/be/src/http/action/version_action.h @@ -22,7 +22,6 @@ namespace doris { -class ExecEnv; class HttpRequest; // Get BE version info from http API. diff --git a/be/src/http/default_path_handlers.h b/be/src/http/default_path_handlers.h index 1ff038350b..f9d022e793 100644 --- a/be/src/http/default_path_handlers.h +++ b/be/src/http/default_path_handlers.h @@ -17,13 +17,8 @@ #pragma once -#include - -#include - namespace doris { -class MemTracker; class WebPageHandler; // Adds a set of default path handlers to the webserver to display diff --git a/be/src/http/ev_http_server.cpp b/be/src/http/ev_http_server.cpp index ccea181a8f..b0743baee3 100644 --- a/be/src/http/ev_http_server.cpp +++ b/be/src/http/ev_http_server.cpp @@ -18,9 +18,10 @@ #include "http/ev_http_server.h" #include -#include #include #include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include diff --git a/be/src/http/ev_http_server.h b/be/src/http/ev_http_server.h index 5ba634b219..bf6b200bdc 100644 --- a/be/src/http/ev_http_server.h +++ b/be/src/http/ev_http_server.h @@ -20,7 +20,6 @@ #include #include #include -#include #include #include "common/status.h" diff --git a/be/src/http/http_channel.h b/be/src/http/http_channel.h index a12eddb535..478f013af8 100644 --- a/be/src/http/http_channel.h +++ b/be/src/http/http_channel.h @@ -19,13 +19,10 @@ #include -#include #include #include "http/http_status.h" -struct mg_connection; - namespace doris { class HttpRequest; diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h index 7ec714587a..c194c29c88 100644 --- a/be/src/http/http_client.h +++ b/be/src/http/http_client.h @@ -26,10 +26,7 @@ #include #include "common/status.h" -#include "http/http_headers.h" #include "http/http_method.h" -#include "http/http_response.h" -#include "http/utils.h" namespace doris { diff --git a/be/src/http/http_request.h b/be/src/http/http_request.h index a26be3a22c..a9286410af 100644 --- a/be/src/http/http_request.h +++ b/be/src/http/http_request.h @@ -19,17 +19,13 @@ #include -#include #include #include #include -#include "http/http_common.h" -#include "http/http_headers.h" #include "http/http_method.h" #include "util/string_util.h" -struct mg_connection; struct evhttp_request; namespace doris { diff --git a/be/src/http/utils.h b/be/src/http/utils.h index f9b09b3017..dd62a9b8a9 100644 --- a/be/src/http/utils.h +++ b/be/src/http/utils.h @@ -19,13 +19,10 @@ #include -#include "common/utils.h" -#include "http/http_common.h" #include "http/http_request.h" namespace doris { -class HttpRequest; struct AuthInfo; std::string encode_basic_auth(const std::string& user, const std::string& passwd); diff --git a/be/src/http/web_page_handler.h b/be/src/http/web_page_handler.h index 9ab304d96f..3fab082847 100644 --- a/be/src/http/web_page_handler.h +++ b/be/src/http/web_page_handler.h @@ -23,10 +23,8 @@ #include #include #include -#include #include "http/http_handler.h" -#include "util/easy_json.h" namespace doris { diff --git a/be/src/io/cache/block/block_file_cache.cpp b/be/src/io/cache/block/block_file_cache.cpp index 0b97facff5..9868c5fee3 100644 --- a/be/src/io/cache/block/block_file_cache.cpp +++ b/be/src/io/cache/block/block_file_cache.cpp @@ -20,7 +20,11 @@ #include "io/cache/block/block_file_cache.h" +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include +#include #include "io/cache/block/block_file_cache_fwd.h" #include "io/cache/block/block_file_cache_settings.h" diff --git a/be/src/io/cache/block/block_file_cache.h b/be/src/io/cache/block/block_file_cache.h index 81050c49b4..965a510717 100644 --- a/be/src/io/cache/block/block_file_cache.h +++ b/be/src/io/cache/block/block_file_cache.h @@ -20,20 +20,32 @@ #pragma once +#include +#include + +#include #include #include +#include +#include +#include #include +#include #include "common/config.h" +#include "common/status.h" #include "io/cache/block/block_file_cache_fwd.h" +#include "util/hash_util.hpp" +#include "vec/common/uint128.h" namespace doris { namespace io { class FileBlock; +struct FileCacheSettings; + using FileBlockSPtr = std::shared_ptr; using FileBlocks = std::list; struct FileBlocksHolder; -struct ReadSettings; /** * Local cache for remote filesystem files, represented as a set of non-overlapping non-empty file segments. diff --git a/be/src/io/cache/block/block_file_cache_factory.cpp b/be/src/io/cache/block/block_file_cache_factory.cpp index 2aee79fc50..5c6c569f21 100644 --- a/be/src/io/cache/block/block_file_cache_factory.cpp +++ b/be/src/io/cache/block/block_file_cache_factory.cpp @@ -20,14 +20,21 @@ #include "io/cache/block/block_file_cache_factory.h" -#include +#include + +#include +#include +#include #include "common/config.h" #include "io/cache/block/block_file_cache.h" +#include "io/cache/block/block_file_cache_settings.h" #include "io/cache/block/block_lru_file_cache.h" #include "io/fs/local_file_system.h" namespace doris { +class TUniqueId; + namespace io { FileCacheFactory& FileCacheFactory::instance() { diff --git a/be/src/io/cache/block/block_file_cache_factory.h b/be/src/io/cache/block/block_file_cache_factory.h index e37a6eebf0..62c23335b7 100644 --- a/be/src/io/cache/block/block_file_cache_factory.h +++ b/be/src/io/cache/block/block_file_cache_factory.h @@ -20,13 +20,18 @@ #pragma once +#include +#include #include +#include "common/status.h" #include "io/cache/block/block_file_cache.h" -#include "io/cache/block/block_file_cache_fwd.h" -#include "io/cache/block/block_file_cache_settings.h" + namespace doris { +class TUniqueId; + namespace io { +struct FileCacheSettings; enum FileCacheType { NORMAL, diff --git a/be/src/io/cache/block/block_file_cache_profile.cpp b/be/src/io/cache/block/block_file_cache_profile.cpp index 61b0ef72ce..ef245be350 100644 --- a/be/src/io/cache/block/block_file_cache_profile.cpp +++ b/be/src/io/cache/block/block_file_cache_profile.cpp @@ -17,7 +17,9 @@ #include "io/cache/block/block_file_cache_profile.h" +#include #include +#include namespace doris { namespace io { diff --git a/be/src/io/cache/block/block_file_cache_profile.h b/be/src/io/cache/block/block_file_cache_profile.h index 858a07aed5..7635413cd9 100644 --- a/be/src/io/cache/block/block_file_cache_profile.h +++ b/be/src/io/cache/block/block_file_cache_profile.h @@ -17,15 +17,15 @@ #pragma once +#include + #include #include -#include -#include #include #include -#include #include +#include "io/io_common.h" #include "olap/olap_common.h" #include "util/doris_metrics.h" #include "util/metrics.h" diff --git a/be/src/io/cache/block/block_file_segment.cpp b/be/src/io/cache/block/block_file_segment.cpp index 020fc85de1..46a1cedc53 100644 --- a/be/src/io/cache/block/block_file_segment.cpp +++ b/be/src/io/cache/block/block_file_segment.cpp @@ -20,16 +20,17 @@ #include "io/cache/block/block_file_segment.h" -#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include #include "common/status.h" +#include "io/fs/file_reader.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" -#include "olap/iterators.h" -#include "vec/common/hex.h" namespace doris { namespace io { diff --git a/be/src/io/cache/block/block_file_segment.h b/be/src/io/cache/block/block_file_segment.h index 47732cb70e..7cbc238f93 100644 --- a/be/src/io/cache/block/block_file_segment.h +++ b/be/src/io/cache/block/block_file_segment.h @@ -20,20 +20,28 @@ #pragma once +#include +#include + +#include #include #include #include #include +#include +#include #include "common/status.h" #include "io/cache/block/block_file_cache.h" -#include "io/fs/file_reader.h" #include "io/fs/file_writer.h" +#include "util/slice.h" namespace doris { namespace io { class FileBlock; +class FileReader; + using FileBlockSPtr = std::shared_ptr; using FileBlocks = std::list; diff --git a/be/src/io/cache/block/block_lru_file_cache.cpp b/be/src/io/cache/block/block_lru_file_cache.cpp index c13e74664f..431410c42e 100644 --- a/be/src/io/cache/block/block_lru_file_cache.cpp +++ b/be/src/io/cache/block/block_lru_file_cache.cpp @@ -20,19 +20,43 @@ #include "io/cache/block/block_lru_file_cache.h" +#include +#include +#include +#include + +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include #include +#include +#include +#include #include #include #include #include "common/status.h" #include "io/cache/block/block_file_cache.h" -#include "io/cache/block/block_file_cache_settings.h" +#include "io/cache/block/block_file_cache_fwd.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" -#include "olap/iterators.h" -#include "util/time.h" +#include "io/fs/path.h" +#include "util/slice.h" #include "vec/common/hex.h" -#include "vec/common/sip_hash.h" + +namespace doris { +namespace io { +struct FileCacheSettings; +} // namespace io +} // namespace doris namespace fs = std::filesystem; diff --git a/be/src/io/cache/block/block_lru_file_cache.h b/be/src/io/cache/block/block_lru_file_cache.h index ac6c09371e..6742a8c0d6 100644 --- a/be/src/io/cache/block/block_lru_file_cache.h +++ b/be/src/io/cache/block/block_lru_file_cache.h @@ -20,17 +20,25 @@ #pragma once -#include +#include #include #include +#include #include +#include #include +#include +#include +#include "common/status.h" #include "io/cache/block/block_file_cache.h" #include "io/cache/block/block_file_segment.h" namespace doris { +class TUniqueId; + namespace io { +struct FileCacheSettings; /** * Local cache for remote filesystem files, represented as a set of non-overlapping non-empty file segments. diff --git a/be/src/io/cache/block/cached_remote_file_reader.cpp b/be/src/io/cache/block/cached_remote_file_reader.cpp index e456811430..1c35375526 100644 --- a/be/src/io/cache/block/cached_remote_file_reader.cpp +++ b/be/src/io/cache/block/cached_remote_file_reader.cpp @@ -17,12 +17,26 @@ #include "io/cache/block/cached_remote_file_reader.h" +#include +#include +#include +#include + +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" #include "io/cache/block/block_file_cache.h" #include "io/cache/block/block_file_cache_factory.h" +#include "io/cache/block/block_file_segment.h" #include "io/fs/file_reader.h" -#include "olap/iterators.h" -#include "olap/olap_common.h" -#include "util/async_io.h" +#include "io/io_common.h" +#include "util/bit_util.h" +#include "util/doris_metrics.h" +#include "util/runtime_profile.h" namespace doris { namespace io { diff --git a/be/src/io/cache/block/cached_remote_file_reader.h b/be/src/io/cache/block/cached_remote_file_reader.h index 95fe00e936..6d9323f5c8 100644 --- a/be/src/io/cache/block/cached_remote_file_reader.h +++ b/be/src/io/cache/block/cached_remote_file_reader.h @@ -17,17 +17,25 @@ #pragma once -#include "gutil/macros.h" +#include +#include + +#include +#include +#include + +#include "common/status.h" #include "io/cache/block/block_file_cache.h" -#include "io/cache/block/block_file_cache_fwd.h" -#include "io/cache/block/block_file_cache_profile.h" -#include "io/cache/block/block_file_segment.h" #include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/file_system.h" #include "io/fs/path.h" -#include "io/fs/s3_file_system.h" +#include "util/slice.h" namespace doris { namespace io { +class IOContext; +struct FileCacheStatistics; class CachedRemoteFileReader final : public FileReader { public: diff --git a/be/src/io/cache/dummy_file_cache.cpp b/be/src/io/cache/dummy_file_cache.cpp index 5150fa1db7..14dda6898a 100644 --- a/be/src/io/cache/dummy_file_cache.cpp +++ b/be/src/io/cache/dummy_file_cache.cpp @@ -17,9 +17,12 @@ #include "io/cache/dummy_file_cache.h" -#include "gutil/strings/util.h" +#include + +#include +#include + #include "io/fs/local_file_system.h" -#include "util/string_util.h" namespace doris { namespace io { diff --git a/be/src/io/cache/dummy_file_cache.h b/be/src/io/cache/dummy_file_cache.h index 653d47b1d6..46d21d99a2 100644 --- a/be/src/io/cache/dummy_file_cache.h +++ b/be/src/io/cache/dummy_file_cache.h @@ -17,17 +17,23 @@ #pragma once -#include -#include +#include +#include + +#include #include +#include #include "common/status.h" #include "io/cache/file_cache.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" #include "io/fs/path.h" +#include "util/slice.h" namespace doris { namespace io { +class IOContext; // Only used for GC class DummyFileCache final : public FileCache { diff --git a/be/src/io/cache/file_cache.cpp b/be/src/io/cache/file_cache.cpp index 91379a2f1d..d37eba7724 100644 --- a/be/src/io/cache/file_cache.cpp +++ b/be/src/io/cache/file_cache.cpp @@ -17,12 +17,25 @@ #include "io/cache/file_cache.h" +#include +#include +#include + +#include +#include +#include +#include +#include +#include + #include "common/config.h" #include "common/status.h" #include "gutil/strings/util.h" +#include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" -#include "io/fs/local_file_writer.h" -#include "olap/iterators.h" +#include "runtime/exec_env.h" +#include "util/string_util.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/io/cache/file_cache.h b/be/src/io/cache/file_cache.h index 68ffc1a3a1..2b2156ea46 100644 --- a/be/src/io/cache/file_cache.h +++ b/be/src/io/cache/file_cache.h @@ -17,17 +17,23 @@ #pragma once +#include +#include +#include + #include -#include -#include #include +#include #include "common/status.h" #include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/path.h" +#include "util/slice.h" namespace doris { namespace io { +class IOContext; const std::string CACHE_DONE_FILE_SUFFIX = "_DONE"; diff --git a/be/src/io/cache/file_cache_manager.cpp b/be/src/io/cache/file_cache_manager.cpp index b14a02c1cb..f581b7d651 100644 --- a/be/src/io/cache/file_cache_manager.cpp +++ b/be/src/io/cache/file_cache_manager.cpp @@ -17,14 +17,27 @@ #include "io/cache/file_cache_manager.h" -#include "gutil/strings/util.h" +#include +#include +#include + +#include +#include +#include +#include + +#include "common/config.h" #include "io/cache/dummy_file_cache.h" #include "io/cache/sub_file_cache.h" #include "io/cache/whole_file_cache.h" +#include "io/fs/file_reader_options.h" +#include "io/fs/file_system.h" #include "io/fs/local_file_system.h" +#include "olap/data_dir.h" #include "olap/rowset/beta_rowset.h" #include "olap/storage_engine.h" -#include "util/string_util.h" +#include "olap/tablet.h" +#include "olap/tablet_manager.h" namespace doris { namespace io { diff --git a/be/src/io/cache/file_cache_manager.h b/be/src/io/cache/file_cache_manager.h index 6bf49c0dc7..7ac5e652d4 100644 --- a/be/src/io/cache/file_cache_manager.h +++ b/be/src/io/cache/file_cache_manager.h @@ -17,17 +17,22 @@ #pragma once -#include -#include -#include +#include + +#include +#include +#include +#include +#include +#include -#include "common/config.h" #include "common/status.h" #include "io/cache/file_cache.h" -#include "io/fs/file_reader_options.h" +#include "io/fs/file_reader_writer_fwd.h" namespace doris { namespace io { +enum class FileCachePolicy : uint8_t; class GCContextPerDisk { public: diff --git a/be/src/io/cache/sub_file_cache.cpp b/be/src/io/cache/sub_file_cache.cpp index fc075bf490..d7d09f3448 100644 --- a/be/src/io/cache/sub_file_cache.cpp +++ b/be/src/io/cache/sub_file_cache.cpp @@ -17,20 +17,27 @@ #include "io/cache/sub_file_cache.h" +#include #include -#include +#include #include #include +#include +#include +#include +#include +#include #include #include #include "common/config.h" #include "common/status.h" #include "io/fs/local_file_system.h" -#include "olap/iterators.h" -#include "util/async_io.h" +#include "io/io_common.h" +#include "runtime/exec_env.h" #include "util/string_util.h" +#include "util/threadpool.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/io/cache/sub_file_cache.h b/be/src/io/cache/sub_file_cache.h index 4766a4b6a6..96dc873a88 100644 --- a/be/src/io/cache/sub_file_cache.h +++ b/be/src/io/cache/sub_file_cache.h @@ -17,16 +17,28 @@ #pragma once -#include +#include +#include + +#include +#include #include +#include +#include +#include +#include #include "common/status.h" #include "io/cache/file_cache.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" #include "io/fs/path.h" +#include "util/slice.h" namespace doris { namespace io { +class IOContext; class SubFileCache final : public FileCache { public: diff --git a/be/src/io/cache/whole_file_cache.cpp b/be/src/io/cache/whole_file_cache.cpp index ef24295e68..59494d9cbd 100644 --- a/be/src/io/cache/whole_file_cache.cpp +++ b/be/src/io/cache/whole_file_cache.cpp @@ -17,9 +17,19 @@ #include "io/cache/whole_file_cache.h" +#include +#include + +#include +#include +#include +#include +#include + #include "io/fs/local_file_system.h" -#include "olap/iterators.h" -#include "util/async_io.h" +#include "io/io_common.h" +#include "runtime/exec_env.h" +#include "util/threadpool.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/io/cache/whole_file_cache.h b/be/src/io/cache/whole_file_cache.h index 0036678114..866e6e142e 100644 --- a/be/src/io/cache/whole_file_cache.h +++ b/be/src/io/cache/whole_file_cache.h @@ -17,16 +17,23 @@ #pragma once -#include +#include +#include + #include +#include #include "common/status.h" #include "io/cache/file_cache.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" #include "io/fs/path.h" +#include "util/slice.h" namespace doris { namespace io { +class IOContext; class WholeFileCache final : public FileCache { public: diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index c539f9ae50..e8fe4b2cec 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -17,24 +17,34 @@ #include "io/file_factory.h" +#include +#include +#include + +#include + #include "common/config.h" #include "common/status.h" #include "io/fs/broker_file_system.h" -#include "io/fs/broker_file_writer.h" #include "io/fs/file_reader_options.h" -#include "io/fs/file_system.h" #include "io/fs/hdfs_file_system.h" -#include "io/fs/hdfs_file_writer.h" #include "io/fs/local_file_system.h" #include "io/fs/s3_file_system.h" +#include "io/fs/stream_load_pipe.h" +#include "io/hdfs_builder.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" #include "runtime/stream_load/new_load_stream_mgr.h" #include "runtime/stream_load/stream_load_context.h" -#include "util/runtime_profile.h" #include "util/s3_uri.h" +#include "util/s3_util.h" +#include "util/uid_util.h" namespace doris { +namespace io { +class FileWriter; +} // namespace io + io::FileCachePolicy FileFactory::get_cache_policy(RuntimeState* state) { if (state != nullptr) { if (config::enable_file_cache && state->query_options().enable_file_cache) { diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h index 01b13cc521..2510b5e30d 100644 --- a/be/src/io/file_factory.h +++ b/be/src/io/file_factory.h @@ -16,19 +16,29 @@ // under the License. #pragma once -#include "gen_cpp/PlanNodes_types.h" -#include "gen_cpp/Types_types.h" -#include "io/fs/file_reader.h" -#include "io/fs/file_writer.h" +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader_options.h" +#include "io/fs/file_reader_writer_fwd.h" namespace doris { namespace io { class FileSystem; -class FileReaderOptions; +class FileWriter; } // namespace io class ExecEnv; -class TNetworkAddress; class RuntimeProfile; +class RuntimeState; struct FileSystemProperties { TFileType::type system_type; diff --git a/be/src/io/fs/broker_file_reader.cpp b/be/src/io/fs/broker_file_reader.cpp index 40ddfe6b44..ffd4c5154e 100644 --- a/be/src/io/fs/broker_file_reader.cpp +++ b/be/src/io/fs/broker_file_reader.cpp @@ -18,13 +18,26 @@ #include "io/fs/broker_file_reader.h" #include +#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/logging.h" #include "common/status.h" #include "io/fs/broker_file_system.h" #include "util/doris_metrics.h" namespace doris { namespace io { +class IOContext; BrokerFileReader::BrokerFileReader(const TNetworkAddress& broker_addr, const Path& path, size_t file_size, TBrokerFD fd, diff --git a/be/src/io/fs/broker_file_reader.h b/be/src/io/fs/broker_file_reader.h index e16e6e336f..a7f126ba0d 100644 --- a/be/src/io/fs/broker_file_reader.h +++ b/be/src/io/fs/broker_file_reader.h @@ -19,16 +19,23 @@ #include #include +#include #include +#include +#include "common/status.h" #include "io/fs/broker_file_system.h" #include "io/fs/file_reader.h" +#include "io/fs/file_system.h" +#include "io/fs/path.h" #include "runtime/client_cache.h" +#include "util/slice.h" + namespace doris { namespace io { -class BrokerFileSystem; +class IOContext; class BrokerFileReader : public FileReader { public: diff --git a/be/src/io/fs/broker_file_system.cpp b/be/src/io/fs/broker_file_system.cpp index d7e87e9930..cf7f34a9df 100644 --- a/be/src/io/fs/broker_file_system.cpp +++ b/be/src/io/fs/broker_file_system.cpp @@ -17,15 +17,33 @@ #include "io/fs/broker_file_system.h" +#include #include #include +#include +#include +#include +#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include + +#include "common/config.h" #include "io/fs/broker_file_reader.h" #include "io/fs/broker_file_writer.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "runtime/broker_mgr.h" #include "runtime/exec_env.h" -#include "util/defer_op.h" +#include "util/slice.h" namespace doris { namespace io { diff --git a/be/src/io/fs/broker_file_system.h b/be/src/io/fs/broker_file_system.h index fa39964878..6b04bbc61c 100644 --- a/be/src/io/fs/broker_file_system.h +++ b/be/src/io/fs/broker_file_system.h @@ -17,11 +17,25 @@ #pragma once +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/path.h" #include "io/fs/remote_file_system.h" #include "runtime/client_cache.h" + namespace doris { +class TNetworkAddress; namespace io { +struct FileInfo; + class BrokerFileSystem final : public RemoteFileSystem { public: static Status create(const TNetworkAddress& broker_addr, diff --git a/be/src/io/fs/broker_file_writer.cpp b/be/src/io/fs/broker_file_writer.cpp index 463c44946f..ac34f381d3 100644 --- a/be/src/io/fs/broker_file_writer.cpp +++ b/be/src/io/fs/broker_file_writer.cpp @@ -17,13 +17,17 @@ #include "io/fs/broker_file_writer.h" +#include +#include +#include +#include #include +#include #include +#include "common/config.h" #include "common/logging.h" -#include "gen_cpp/PaloBrokerService_types.h" -#include "gen_cpp/TPaloBrokerService.h" #include "runtime/broker_mgr.h" #include "runtime/client_cache.h" #include "runtime/exec_env.h" diff --git a/be/src/io/fs/broker_file_writer.h b/be/src/io/fs/broker_file_writer.h index 5705cc18df..95773f0a04 100644 --- a/be/src/io/fs/broker_file_writer.h +++ b/be/src/io/fs/broker_file_writer.h @@ -17,21 +17,23 @@ #pragma once +#include +#include +#include #include #include #include #include "common/status.h" -#include "gen_cpp/PaloBrokerService_types.h" -#include "gen_cpp/Types_types.h" +#include "io/fs/file_system.h" #include "io/fs/file_writer.h" +#include "util/hash_util.hpp" // IWYU pragma: keep +#include "util/slice.h" namespace doris { class ExecEnv; -class TBrokerRangeDesc; -class TNetworkAddress; namespace io { diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index 63d4f10f77..0ab588f68f 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -17,15 +17,22 @@ #include "io/fs/buffered_reader.h" -#include -#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" -#include "olap/iterators.h" -#include "olap/olap_define.h" +#include "runtime/exec_env.h" +#include "util/runtime_profile.h" +#include "util/threadpool.h" namespace doris { namespace io { +class IOContext; // add bvar to capture the download bytes per second by buffered reader bvar::Adder g_bytes_downloaded("buffered_reader", "bytes_downloaded"); diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h index 434058e32e..670f64a0db 100644 --- a/be/src/io/fs/buffered_reader.h +++ b/be/src/io/fs/buffered_reader.h @@ -17,21 +17,28 @@ #pragma once +#include #include #include #include +#include +#include +#include +#include -#include "common/config.h" #include "common/status.h" #include "io/fs/file_reader.h" -#include "olap/olap_define.h" -#include "util/runtime_profile.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/path.h" +#include "util/slice.h" namespace doris { namespace io { -class PrefetchBufferedReader; +class FileSystem; +class IOContext; + struct PrefetchBuffer : std::enable_shared_from_this { enum class BufferStatus { RESET, PENDING, PREFETCHED, CLOSED }; PrefetchBuffer() = default; diff --git a/be/src/io/fs/err_utils.cpp b/be/src/io/fs/err_utils.cpp index d01c7e7488..5530b25a0d 100644 --- a/be/src/io/fs/err_utils.cpp +++ b/be/src/io/fs/err_utils.cpp @@ -17,7 +17,10 @@ #include "io/fs/err_utils.h" +// IWYU pragma: no_include +#include // IWYU pragma: keep #include +#include #include diff --git a/be/src/io/fs/file_reader.cpp b/be/src/io/fs/file_reader.cpp index fba531630f..eff62210d5 100644 --- a/be/src/io/fs/file_reader.cpp +++ b/be/src/io/fs/file_reader.cpp @@ -17,8 +17,10 @@ #include "io/fs/file_reader.h" +#include +#include + #include "io/fs/file_system.h" -#include "io/io_common.h" #include "util/async_io.h" namespace doris { diff --git a/be/src/io/fs/file_reader.h b/be/src/io/fs/file_reader.h index c429b36ed3..d4f7c5d530 100644 --- a/be/src/io/fs/file_reader.h +++ b/be/src/io/fs/file_reader.h @@ -17,9 +17,12 @@ #pragma once +#include +#include + +#include + #include "common/status.h" -#include "gutil/macros.h" -#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/path.h" #include "util/slice.h" diff --git a/be/src/io/fs/file_reader_options.h b/be/src/io/fs/file_reader_options.h index c12ce727bf..ca50bcb1dd 100644 --- a/be/src/io/fs/file_reader_options.h +++ b/be/src/io/fs/file_reader_options.h @@ -17,7 +17,9 @@ #pragma once -#include "io/cache/block/block_file_cache.h" +#include + +#include namespace doris { namespace io { diff --git a/be/src/io/fs/file_system.cpp b/be/src/io/fs/file_system.cpp index 1f34500035..d399fbbc57 100644 --- a/be/src/io/fs/file_system.cpp +++ b/be/src/io/fs/file_system.cpp @@ -17,8 +17,7 @@ #include "io/fs/file_system.h" -#include "olap/olap_define.h" -#include "util/async_io.h" +#include "util/async_io.h" // IWYU pragma: keep namespace doris { namespace io { diff --git a/be/src/io/fs/file_system.h b/be/src/io/fs/file_system.h index 3dbf741a05..30ac5f5a4a 100644 --- a/be/src/io/fs/file_system.h +++ b/be/src/io/fs/file_system.h @@ -17,17 +17,26 @@ #pragma once +#include +#include +#include +#include + +#include +#include #include +#include +#include +#include #include "common/status.h" -#include "gutil/macros.h" #include "io/fs/file_reader_options.h" #include "io/fs/file_reader_writer_fwd.h" #include "io/fs/path.h" -#include "io/io_common.h" namespace doris { namespace io { +class FileSystem; #ifndef FILESYSTEM_M #define FILESYSTEM_M(stmt) \ diff --git a/be/src/io/fs/fs_utils.cpp b/be/src/io/fs/fs_utils.cpp index 066dbaa63a..7f12c6767e 100644 --- a/be/src/io/fs/fs_utils.cpp +++ b/be/src/io/fs/fs_utils.cpp @@ -17,7 +17,12 @@ #include "io/fs/fs_utils.h" +#include + +#include + #include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" namespace doris { diff --git a/be/src/io/fs/fs_utils.h b/be/src/io/fs/fs_utils.h index 866b3cfaea..39744d25a0 100644 --- a/be/src/io/fs/fs_utils.h +++ b/be/src/io/fs/fs_utils.h @@ -17,8 +17,15 @@ #pragma once +#include +#include + +#include +#include + #include "common/status.h" #include "io/fs/file_system.h" +#include "io/fs/path.h" namespace doris { namespace io { diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index ddd035213b..901d51bd9c 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -17,6 +17,16 @@ #include "io/fs/hdfs_file_reader.h" +#include + +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/logging.h" #include "io/fs/err_utils.h" #include "io/fs/hdfs_file_system.h" #include "service/backend_options.h" diff --git a/be/src/io/fs/hdfs_file_system.cpp b/be/src/io/fs/hdfs_file_system.cpp index 0d33bd30c9..c338390af4 100644 --- a/be/src/io/fs/hdfs_file_system.cpp +++ b/be/src/io/fs/hdfs_file_system.cpp @@ -17,16 +17,32 @@ #include "io/fs/hdfs_file_system.h" +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" #include "gutil/hash/hash.h" -#include "io/cache/block/cached_remote_file_reader.h" +#include "gutil/integral_types.h" #include "io/fs/err_utils.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "io/fs/hdfs_file_reader.h" #include "io/fs/hdfs_file_writer.h" #include "io/fs/local_file_system.h" #include "io/hdfs_builder.h" -#include "service/backend_options.h" #include "util/hdfs_util.h" -#include "util/stack_util.h" +#include "util/slice.h" namespace doris { namespace io { diff --git a/be/src/io/fs/hdfs_file_system.h b/be/src/io/fs/hdfs_file_system.h index 9cec56b86c..4928f193c3 100644 --- a/be/src/io/fs/hdfs_file_system.h +++ b/be/src/io/fs/hdfs_file_system.h @@ -17,15 +17,27 @@ #pragma once -#include +#include +#include #include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include "common/status.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/hdfs.h" +#include "io/fs/path.h" #include "io/fs/remote_file_system.h" namespace doris { +class THdfsParams; + namespace io { +struct FileInfo; class HdfsFileSystemHandle { public: diff --git a/be/src/io/fs/hdfs_file_writer.cpp b/be/src/io/fs/hdfs_file_writer.cpp index 723856fa25..5eaaa8a404 100644 --- a/be/src/io/fs/hdfs_file_writer.cpp +++ b/be/src/io/fs/hdfs_file_writer.cpp @@ -17,7 +17,13 @@ #include "io/fs/hdfs_file_writer.h" +#include +#include + #include +#include +#include +#include #include "common/logging.h" #include "io/fs/err_utils.h" diff --git a/be/src/io/fs/hdfs_file_writer.h b/be/src/io/fs/hdfs_file_writer.h index c9b3ff9def..371cae6e0c 100644 --- a/be/src/io/fs/hdfs_file_writer.h +++ b/be/src/io/fs/hdfs_file_writer.h @@ -19,16 +19,11 @@ #include -#include -#include - #include "common/status.h" -#include "gen_cpp/PlanNodes_types.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" #include "io/fs/hdfs.h" #include "io/fs/path.h" -#include "io/hdfs_builder.h" #include "util/slice.h" namespace doris { diff --git a/be/src/io/fs/local_file_reader.cpp b/be/src/io/fs/local_file_reader.cpp index c62633e8cb..bed46e0c28 100644 --- a/be/src/io/fs/local_file_reader.cpp +++ b/be/src/io/fs/local_file_reader.cpp @@ -17,14 +17,28 @@ #include "io/fs/local_file_reader.h" -#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "io/fs/err_utils.h" #include "util/async_io.h" #include "util/doris_metrics.h" namespace doris { namespace io { +class IOContext; LocalFileReader::LocalFileReader(Path path, size_t file_size, int fd, std::shared_ptr fs) diff --git a/be/src/io/fs/local_file_reader.h b/be/src/io/fs/local_file_reader.h index 1f74e7b4be..a42c0e3c07 100644 --- a/be/src/io/fs/local_file_reader.h +++ b/be/src/io/fs/local_file_reader.h @@ -17,14 +17,21 @@ #pragma once -#include +#include +#include +#include + +#include "common/status.h" #include "io/fs/file_reader.h" +#include "io/fs/file_system.h" #include "io/fs/local_file_system.h" #include "io/fs/path.h" +#include "util/slice.h" namespace doris { namespace io { +class IOContext; class LocalFileReader final : public FileReader { public: diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index 0b7747caf4..9b5c8eec85 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -17,18 +17,33 @@ #include "io/fs/local_file_system.h" +#include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include + +#include "gutil/macros.h" #include "io/fs/err_utils.h" #include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_reader.h" #include "io/fs/local_file_writer.h" #include "runtime/thread_context.h" -#include "util/async_io.h" +#include "util/async_io.h" // IWYU pragma: keep +#include "util/defer_op.h" namespace doris { namespace io { +class FileReaderOptions; std::shared_ptr LocalFileSystem::create(Path path, std::string id) { return std::shared_ptr(new LocalFileSystem(std::move(path), std::move(id))); diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h index 15b4c875c2..bcebd48a65 100644 --- a/be/src/io/fs/local_file_system.h +++ b/be/src/io/fs/local_file_system.h @@ -17,10 +17,22 @@ #pragma once +#include +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" +#include "io/fs/path.h" namespace doris { namespace io { +class FileReaderOptions; class LocalFileSystem final : public FileSystem { public: diff --git a/be/src/io/fs/local_file_writer.cpp b/be/src/io/fs/local_file_writer.cpp index c141471d98..62536dd687 100644 --- a/be/src/io/fs/local_file_writer.cpp +++ b/be/src/io/fs/local_file_writer.cpp @@ -17,12 +17,23 @@ #include "io/fs/local_file_writer.h" -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include #include +#include -#include +#include +#include +#include +#include +#include -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" #include "gutil/macros.h" #include "io/fs/file_writer.h" diff --git a/be/src/io/fs/local_file_writer.h b/be/src/io/fs/local_file_writer.h index e4745893dd..11b2f16434 100644 --- a/be/src/io/fs/local_file_writer.h +++ b/be/src/io/fs/local_file_writer.h @@ -19,9 +19,11 @@ #include +#include "common/status.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" -#include "io/fs/local_file_system.h" +#include "io/fs/path.h" +#include "util/slice.h" namespace doris { namespace io { diff --git a/be/src/io/fs/remote_file_system.cpp b/be/src/io/fs/remote_file_system.cpp index 9e4e777639..53f7ced5f9 100644 --- a/be/src/io/fs/remote_file_system.cpp +++ b/be/src/io/fs/remote_file_system.cpp @@ -17,11 +17,18 @@ #include "io/fs/remote_file_system.h" +#include + +#include + +#include "common/config.h" #include "gutil/strings/stringpiece.h" #include "io/cache/block/cached_remote_file_reader.h" +#include "io/cache/file_cache.h" #include "io/cache/file_cache_manager.h" +#include "io/fs/file_reader.h" #include "io/fs/file_reader_options.h" -#include "util/async_io.h" +#include "util/async_io.h" // IWYU pragma: keep namespace doris { namespace io { diff --git a/be/src/io/fs/remote_file_system.h b/be/src/io/fs/remote_file_system.h index 8626d0840a..44ecde4041 100644 --- a/be/src/io/fs/remote_file_system.h +++ b/be/src/io/fs/remote_file_system.h @@ -17,10 +17,21 @@ #pragma once +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" +#include "io/fs/path.h" namespace doris { namespace io { +class FileReaderOptions; class RemoteFileSystem : public FileSystem { public: diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index b60e038d39..77a3bfcd22 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -17,15 +17,25 @@ #include "io/fs/s3_file_reader.h" +#include #include +#include #include +#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "io/fs/s3_common.h" -#include "util/async_io.h" #include "util/doris_metrics.h" namespace doris { namespace io { +class IOContext; S3FileReader::S3FileReader(Path path, size_t file_size, std::string key, std::string bucket, std::shared_ptr fs) diff --git a/be/src/io/fs/s3_file_reader.h b/be/src/io/fs/s3_file_reader.h index c21084ae8d..1f53fb5814 100644 --- a/be/src/io/fs/s3_file_reader.h +++ b/be/src/io/fs/s3_file_reader.h @@ -17,13 +17,22 @@ #pragma once -#include "gutil/macros.h" +#include + +#include +#include +#include + +#include "common/status.h" #include "io/fs/file_reader.h" +#include "io/fs/file_system.h" #include "io/fs/path.h" #include "io/fs/s3_file_system.h" +#include "util/slice.h" namespace doris { namespace io { +class IOContext; class S3FileReader final : public FileReader { public: diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 3236e3f826..c7efcc414a 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -17,25 +17,57 @@ #include "io/fs/s3_file_system.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include +#include #include +#include +#include #include +#include #include +#include +#include +#include +#include #include +#include #include +#include +#include +#include #include +#include +#include #include -#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +// IWYU pragma: no_include +#include // IWYU pragma: keep #include -#include +#include // IWYU pragma: keep #include +#include #include "common/config.h" +#include "common/logging.h" #include "common/status.h" -#include "gutil/strings/stringpiece.h" -#include "io/cache/block/cached_remote_file_reader.h" +#include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "io/fs/remote_file_system.h" #include "io/fs/s3_file_reader.h" #include "io/fs/s3_file_writer.h" diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h index 0ae687e44d..0c7e3d17e1 100644 --- a/be/src/io/fs/s3_file_system.h +++ b/be/src/io/fs/s3_file_system.h @@ -17,8 +17,18 @@ #pragma once -#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/path.h" #include "io/fs/remote_file_system.h" #include "util/s3_util.h" @@ -31,6 +41,7 @@ class PooledThreadExecutor; namespace doris { namespace io { +struct FileInfo; // File system for S3 compatible object storage // When creating S3FileSystem, all required info should be set in S3Conf, diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index f613b14e17..989f9c0c2e 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -17,29 +17,41 @@ #include "io/fs/s3_file_writer.h" -#include +#include +#include #include +#include +#include #include +#include #include +#include #include +#include +#include +#include #include -#include -#include -#include +#include #include -#include -#include +#include +#include -#include +#include +#include -#include "common/compiler_util.h" #include "common/status.h" -#include "gutil/macros.h" #include "io/fs/file_writer.h" #include "io/fs/path.h" -#include "io/fs/s3_file_system.h" #include "util/doris_metrics.h" +namespace Aws { +namespace S3 { +namespace Model { +class DeleteObjectRequest; +} // namespace Model +} // namespace S3 +} // namespace Aws + using Aws::S3::Model::AbortMultipartUploadRequest; using Aws::S3::Model::CompletedPart; using Aws::S3::Model::CompletedMultipartUpload; diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 87aa58680f..d8dc9cc800 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -17,12 +17,19 @@ #pragma once +#include + #include #include +#include +#include +#include "common/status.h" +#include "io/fs/file_system.h" #include "io/fs/file_writer.h" -#include "io/fs/s3_file_system.h" +#include "io/fs/path.h" #include "util/s3_util.h" +#include "util/slice.h" namespace Aws::S3 { namespace Model { diff --git a/be/src/io/fs/stream_load_pipe.cpp b/be/src/io/fs/stream_load_pipe.cpp index f0e1be2c6e..25099c8796 100644 --- a/be/src/io/fs/stream_load_pipe.cpp +++ b/be/src/io/fs/stream_load_pipe.cpp @@ -17,12 +17,22 @@ #include "stream_load_pipe.h" -#include "olap/iterators.h" +#include + +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "runtime/exec_env.h" #include "runtime/thread_context.h" #include "util/bit_util.h" namespace doris { namespace io { +class IOContext; + StreamLoadPipe::StreamLoadPipe(size_t max_buffered_bytes, size_t min_chunk_size, int64_t total_length, bool use_proto) : _buffered_bytes(0), diff --git a/be/src/io/fs/stream_load_pipe.h b/be/src/io/fs/stream_load_pipe.h index 33e72c57a1..e06e7dfc6e 100644 --- a/be/src/io/fs/stream_load_pipe.h +++ b/be/src/io/fs/stream_load_pipe.h @@ -18,16 +18,26 @@ #pragma once #include +#include +#include #include #include +#include +#include +#include +#include "common/status.h" #include "io/fs/file_reader.h" #include "io/fs/file_system.h" +#include "io/fs/path.h" #include "runtime/message_body_sink.h" +#include "util/byte_buffer.h" +#include "util/slice.h" namespace doris { namespace io { +class IOContext; const size_t kMaxPipeBufferedBytes = 4 * 1024 * 1024; diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index 3ce48eede7..ca629e4401 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -18,14 +18,18 @@ #include "io/hdfs_builder.h" #include +#include +#include #include +#include +#include #include "agent/utils.h" #include "common/logging.h" +#include "io/fs/hdfs.h" #include "util/string_util.h" #include "util/uid_util.h" -#include "util/url_coding.h" namespace doris { diff --git a/be/src/io/hdfs_builder.h b/be/src/io/hdfs_builder.h index 7d448cb1cb..d3f7c35017 100644 --- a/be/src/io/hdfs_builder.h +++ b/be/src/io/hdfs_builder.h @@ -17,9 +17,15 @@ #pragma once +#include + +#include +#include + #include "common/status.h" -#include "gen_cpp/PlanNodes_types.h" -#include "io/fs/hdfs.h" +#include "io/fs/hdfs.h" // IWYU pragma: keep + +struct hdfsBuilder; namespace doris { diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index d99b4bb1fb..b3b95853f0 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -17,7 +17,7 @@ #pragma once -#include "gen_cpp/Types_types.h" +#include namespace doris { diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp index d67f8a2d72..c1fcbe3f33 100644 --- a/be/src/olap/base_compaction.cpp +++ b/be/src/olap/base_compaction.cpp @@ -17,7 +17,21 @@ #include "olap/base_compaction.h" +#include +#include +#include + +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "olap/olap_define.h" +#include "olap/rowset/rowset_meta.h" +#include "runtime/thread_context.h" #include "util/doris_metrics.h" +#include "util/thread.h" #include "util/trace.h" namespace doris { diff --git a/be/src/olap/base_compaction.h b/be/src/olap/base_compaction.h index da53a89d95..ecee304449 100644 --- a/be/src/olap/base_compaction.h +++ b/be/src/olap/base_compaction.h @@ -17,7 +17,16 @@ #pragma once +#include + +#include +#include + +#include "common/status.h" +#include "io/io_common.h" #include "olap/compaction.h" +#include "olap/rowset/rowset.h" +#include "olap/tablet.h" namespace doris { diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index ee2e9539ab..3af9615168 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -17,11 +17,18 @@ #include "olap/base_tablet.h" +#include +#include + +#include +#include + #include "gutil/strings/substitute.h" #include "olap/data_dir.h" +#include "olap/olap_define.h" +#include "olap/rowset/rowset_meta.h" #include "olap/tablet_schema_cache.h" #include "util/doris_metrics.h" -#include "util/path_util.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index 05707e81a6..164215989b 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -17,13 +17,17 @@ #pragma once +#include +#include +#include + #include #include -#include "olap/olap_define.h" +#include "common/status.h" +#include "olap/olap_common.h" #include "olap/tablet_meta.h" #include "olap/tablet_schema.h" -#include "olap/utils.h" #include "util/metrics.h" namespace doris { diff --git a/be/src/olap/block_column_predicate.cpp b/be/src/olap/block_column_predicate.cpp index ff7e0ba694..1bcde94de3 100644 --- a/be/src/olap/block_column_predicate.cpp +++ b/be/src/olap/block_column_predicate.cpp @@ -17,9 +17,17 @@ #include "block_column_predicate.h" -#include "olap/rowset/segment_v2/bloom_filter.h" +#include + +namespace roaring { +class Roaring; +} // namespace roaring namespace doris { +class WrapperField; +namespace segment_v2 { +class InvertedIndexIterator; +} // namespace segment_v2 uint16_t SingleColumnBlockPredicate::evaluate(vectorized::MutableColumns& block, uint16_t* sel, uint16_t selected_size) const { diff --git a/be/src/olap/block_column_predicate.h b/be/src/olap/block_column_predicate.h index 18c957e3c4..467962e809 100644 --- a/be/src/olap/block_column_predicate.h +++ b/be/src/olap/block_column_predicate.h @@ -17,15 +17,33 @@ #pragma once +#include +#include +#include + +#include +#include +#include +#include +#include #include +#include "common/status.h" #include "olap/column_predicate.h" +#include "olap/olap_common.h" +#include "vec/columns/column.h" + +namespace roaring { +class Roaring; +} // namespace roaring namespace doris { +class WrapperField; namespace segment_v2 { class BloomFilter; -} +class InvertedIndexIterator; +} // namespace segment_v2 // Block Column Predicate support do column predicate and support OR and AND predicate // Block Column Predicate will replace column predicate as a unified external vectorized interface diff --git a/be/src/olap/cold_data_compaction.cpp b/be/src/olap/cold_data_compaction.cpp index 2a27d92acb..6757691d84 100644 --- a/be/src/olap/cold_data_compaction.cpp +++ b/be/src/olap/cold_data_compaction.cpp @@ -17,10 +17,25 @@ #include "olap/cold_data_compaction.h" -#include "common/compiler_util.h" +#include + +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" #include "olap/compaction.h" -#include "olap/rowset/beta_rowset.h" +#include "olap/olap_common.h" #include "olap/rowset/rowset.h" +#include "olap/tablet_meta.h" +#include "runtime/thread_context.h" +#include "util/thread.h" +#include "util/uid_util.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/olap/cold_data_compaction.h b/be/src/olap/cold_data_compaction.h index b9e8465273..0b2eddf234 100644 --- a/be/src/olap/cold_data_compaction.h +++ b/be/src/olap/cold_data_compaction.h @@ -17,7 +17,13 @@ #pragma once +#include + +#include "common/status.h" +#include "io/io_common.h" #include "olap/compaction.h" +#include "olap/merger.h" +#include "olap/tablet.h" namespace doris { diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index cacf3a9b66..608c89f6ae 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -17,14 +17,39 @@ #include "olap/compaction.h" +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" #include "common/status.h" -#include "gutil/strings/substitute.h" +#include "io/fs/file_system.h" +#include "io/fs/remote_file_system.h" +#include "olap/cumulative_compaction_policy.h" +#include "olap/data_dir.h" +#include "olap/olap_define.h" #include "olap/rowset/beta_rowset.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_meta.h" +#include "olap/rowset/rowset_writer.h" #include "olap/rowset/rowset_writer_context.h" +#include "olap/storage_engine.h" +#include "olap/storage_policy.h" #include "olap/tablet.h" +#include "olap/tablet_meta.h" #include "olap/task/engine_checksum_task.h" +#include "olap/utils.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "util/time.h" #include "util/trace.h" diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index cf35ee8aa6..17acb45710 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -17,21 +17,27 @@ #pragma once +#include +#include + +#include +#include #include +#include "common/status.h" +#include "io/io_common.h" #include "olap/merger.h" #include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/storage_engine.h" +#include "olap/rowid_conversion.h" +#include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_reader.h" #include "olap/tablet.h" -#include "olap/tablet_meta.h" -#include "olap/utils.h" -#include "rowset/rowset_id_generator.h" +#include "olap/tablet_schema.h" namespace doris { -class DataDir; -class Merger; +class MemTrackerLimiter; +class RowsetWriter; // This class is a base class for compaction. // The entrance of this class is compact() diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp index 2d6309ae6c..0986730059 100644 --- a/be/src/olap/compaction_permit_limiter.cpp +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -17,6 +17,9 @@ #include "olap/compaction_permit_limiter.h" +#include "common/config.h" +#include "util/doris_metrics.h" + namespace doris { CompactionPermitLimiter::CompactionPermitLimiter() : _used_permits(0) {} diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h index 21c819830d..c00e0aae27 100644 --- a/be/src/olap/compaction_permit_limiter.h +++ b/be/src/olap/compaction_permit_limiter.h @@ -17,13 +17,12 @@ #pragma once +#include + +#include #include #include -#include "common/config.h" -#include "olap/utils.h" -#include "util/doris_metrics.h" - namespace doris { /* diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h index 0371b27282..c74b06eaf3 100644 --- a/be/src/olap/comparison_predicate.h +++ b/be/src/olap/comparison_predicate.h @@ -21,6 +21,7 @@ #include "olap/column_predicate.h" #include "olap/rowset/segment_v2/bloom_filter.h" +#include "olap/rowset/segment_v2/inverted_index_cache.h" // IWYU pragma: keep #include "olap/rowset/segment_v2/inverted_index_reader.h" #include "olap/wrapper_field.h" #include "vec/columns/column_dictionary.h" diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp index 745b316cab..303b58eefe 100644 --- a/be/src/olap/cumulative_compaction.cpp +++ b/be/src/olap/cumulative_compaction.cpp @@ -17,6 +17,19 @@ #include "olap/cumulative_compaction.h" +#include +#include + +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "olap/cumulative_compaction_policy.h" +#include "olap/olap_define.h" +#include "olap/rowset/rowset_meta.h" +#include "runtime/thread_context.h" #include "util/doris_metrics.h" #include "util/time.h" #include "util/trace.h" diff --git a/be/src/olap/cumulative_compaction.h b/be/src/olap/cumulative_compaction.h index fdfc9dbe46..d74542a2ff 100644 --- a/be/src/olap/cumulative_compaction.h +++ b/be/src/olap/cumulative_compaction.h @@ -17,10 +17,17 @@ #pragma once -#include +#include +#include +#include + +#include "common/status.h" +#include "io/io_common.h" #include "olap/compaction.h" -#include "olap/cumulative_compaction_policy.h" +#include "olap/olap_common.h" +#include "olap/rowset/rowset.h" +#include "olap/tablet.h" namespace doris { diff --git a/be/src/olap/cumulative_compaction_policy.cpp b/be/src/olap/cumulative_compaction_policy.cpp index b38271fd1a..d5d86ad142 100644 --- a/be/src/olap/cumulative_compaction_policy.cpp +++ b/be/src/olap/cumulative_compaction_policy.cpp @@ -17,10 +17,15 @@ #include "olap/cumulative_compaction_policy.h" -#include +#include +#include +#include #include -#include "util/time.h" +#include "common/logging.h" +#include "olap/olap_common.h" +#include "olap/tablet.h" +#include "olap/tablet_meta.h" namespace doris { diff --git a/be/src/olap/cumulative_compaction_policy.h b/be/src/olap/cumulative_compaction_policy.h index 20a9d64b61..a12d1d8e8b 100644 --- a/be/src/olap/cumulative_compaction_policy.h +++ b/be/src/olap/cumulative_compaction_policy.h @@ -17,17 +17,21 @@ #pragma once -#include +#include +#include +#include +#include +#include + +#include "common/config.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_meta.h" -#include "olap/tablet.h" -#include "olap/tablet_meta.h" -#include "olap/utils.h" namespace doris { class Tablet; +struct Version; const static std::string CUMULATIVE_SIZE_BASED_POLICY = "SIZE_BASED"; diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index c1f6e807d7..b83ce43635 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -17,42 +17,50 @@ #include "olap/data_dir.h" -#include "olap/storage_policy.h" - -#ifndef __APPLE__ -#include -#include -#endif - -#include +#include +#include #include #include -#include -#include #include -#include -#include -#include -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include #include +#include #include #include #include +#include +#include +#include "common/config.h" +#include "common/logging.h" #include "gutil/strings/substitute.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/file_writer.h" #include "io/fs/fs_utils.h" #include "io/fs/local_file_system.h" #include "io/fs/path.h" +#include "io/fs/remote_file_system.h" #include "olap/olap_define.h" +#include "olap/olap_meta.h" #include "olap/rowset/beta_rowset.h" +#include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_id_generator.h" +#include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_meta_manager.h" #include "olap/storage_engine.h" +#include "olap/storage_policy.h" +#include "olap/tablet.h" +#include "olap/tablet_manager.h" #include "olap/tablet_meta_manager.h" +#include "olap/txn_manager.h" #include "olap/utils.h" // for check_dir_existed #include "service/backend_options.h" -#include "util/errno.h" +#include "util/doris_metrics.h" #include "util/string_util.h" +#include "util/uid_util.h" using strings::Substitute; diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h index f9aa644b91..9c6adab9c5 100644 --- a/be/src/olap/data_dir.h +++ b/be/src/olap/data_dir.h @@ -17,28 +17,32 @@ #pragma once +#include +#include + +#include #include #include +#include #include #include #include #include +#include #include "common/status.h" -#include "gen_cpp/Types_types.h" -#include "gen_cpp/olap_file.pb.h" #include "io/fs/file_system.h" #include "io/fs/fs_utils.h" #include "olap/olap_common.h" -#include "olap/rowset/rowset_id_generator.h" #include "util/metrics.h" namespace doris { class Tablet; class TabletManager; -class TabletMeta; class TxnManager; +class OlapMeta; +class RowsetIdGenerator; // A DataDir used to manage data in same path. // Now, After DataDir was created, it will never be deleted for easy implementation. diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp index 655670e9a8..9beafe634f 100644 --- a/be/src/olap/delete_handler.cpp +++ b/be/src/olap/delete_handler.cpp @@ -17,20 +17,24 @@ #include "olap/delete_handler.h" -#include -#include +#include +#include #include +#include #include #include #include #include +#include #include -#include "gen_cpp/olap_file.pb.h" +#include "common/config.h" +#include "common/logging.h" +#include "olap/block_column_predicate.h" +#include "olap/column_predicate.h" #include "olap/olap_common.h" #include "olap/predicate_creator.h" -#include "olap/tablet.h" #include "olap/utils.h" using apache::thrift::ThriftDebugString; diff --git a/be/src/olap/delete_handler.h b/be/src/olap/delete_handler.h index f18b7c69bd..7239f4326b 100644 --- a/be/src/olap/delete_handler.h +++ b/be/src/olap/delete_handler.h @@ -17,23 +17,25 @@ #pragma once +#include +#include + +#include #include +#include #include -#include "gen_cpp/AgentService_types.h" -#include "gen_cpp/olap_file.pb.h" -#include "olap/block_column_predicate.h" -#include "olap/column_predicate.h" -#include "olap/olap_define.h" +#include "common/status.h" #include "olap/rowset/rowset_meta.h" #include "olap/tablet_schema.h" +#include "vec/common/arena.h" namespace doris { -class RowCursor; -class Tablet; -class TabletReader; -class TabletSchema; +class AndBlockColumnPredicate; +class ColumnPredicate; +class DeletePredicatePB; +class TCondition; // Represent a delete condition. struct DeleteConditions { diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index 5f339a5471..b846a5479f 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -17,21 +17,49 @@ #include "olap/delta_writer.h" +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "common/logging.h" #include "common/status.h" #include "exec/tablet_info.h" +#include "gutil/strings/numbers.h" +#include "io/fs/file_writer.h" // IWYU pragma: keep #include "olap/data_dir.h" #include "olap/memtable.h" #include "olap/memtable_flush_executor.h" +#include "olap/olap_define.h" #include "olap/rowset/beta_rowset.h" #include "olap/rowset/beta_rowset_writer.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/rowset/rowset_writer.h" #include "olap/rowset/rowset_writer_context.h" +#include "olap/rowset/segment_v2/segment.h" #include "olap/schema.h" #include "olap/schema_change.h" #include "olap/storage_engine.h" +#include "olap/tablet_manager.h" +#include "olap/txn_manager.h" +#include "runtime/exec_env.h" #include "runtime/load_channel_mgr.h" +#include "runtime/memory/mem_tracker.h" #include "service/backend_options.h" #include "util/brpc_client_cache.h" #include "util/ref_count_closure.h" +#include "util/stopwatch.hpp" +#include "util/time.h" +#include "vec/core/block.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/olap/delta_writer.h b/be/src/olap/delta_writer.h index 0b133e6ac9..03b30160fe 100644 --- a/be/src/olap/delta_writer.h +++ b/be/src/olap/delta_writer.h @@ -17,10 +17,26 @@ #pragma once -#include "gen_cpp/internal_service.pb.h" -#include "olap/rowset/rowset_writer.h" +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "olap/olap_common.h" +#include "olap/rowset/rowset.h" #include "olap/tablet.h" +#include "olap/tablet_meta.h" +#include "olap/tablet_schema.h" #include "util/spinlock.h" +#include "util/uid_util.h" namespace doris { @@ -31,6 +47,12 @@ class Schema; class StorageEngine; class TupleDescriptor; class SlotDescriptor; +class OlapTableSchemaParam; +class RowsetWriter; + +namespace vectorized { +class Block; +} // namespace vectorized enum WriteType { LOAD = 1, LOAD_DELETE = 2, DELETE = 3 }; diff --git a/be/src/olap/hll.cpp b/be/src/olap/hll.cpp index 8f7a61e712..bfbe954217 100644 --- a/be/src/olap/hll.cpp +++ b/be/src/olap/hll.cpp @@ -17,12 +17,14 @@ #include "olap/hll.h" -#include +#include #include +#include +#include #include "common/logging.h" #include "util/coding.h" -#include "vec/common/string_ref.h" +#include "util/slice.h" using std::map; using std::nothrow; diff --git a/be/src/olap/hll.h b/be/src/olap/hll.h index f1ad81100b..7c88abfbb5 100644 --- a/be/src/olap/hll.h +++ b/be/src/olap/hll.h @@ -17,18 +17,17 @@ #pragma once -#include -#include +#include +#include #include -#include #include +#include #ifdef __x86_64__ #include #endif -#include "gutil/macros.h" #include "vec/common/hash_table/phmap_fwd_decl.h" namespace doris { diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h index e207bf28d5..abecd69077 100644 --- a/be/src/olap/in_list_predicate.h +++ b/be/src/olap/in_list_predicate.h @@ -25,6 +25,7 @@ #include "olap/column_predicate.h" #include "olap/olap_common.h" #include "olap/rowset/segment_v2/bloom_filter.h" +#include "olap/rowset/segment_v2/inverted_index_cache.h" // IWYU pragma: keep #include "olap/rowset/segment_v2/inverted_index_reader.h" #include "olap/wrapper_field.h" #include "runtime/define_primitive_type.h" diff --git a/be/src/olap/itoken_extractor.cpp b/be/src/olap/itoken_extractor.cpp index 90ad29d291..fefe86ec27 100644 --- a/be/src/olap/itoken_extractor.cpp +++ b/be/src/olap/itoken_extractor.cpp @@ -17,6 +17,8 @@ #include "itoken_extractor.h" +#include + #include "util/simd/vstring_function.h" namespace doris { diff --git a/be/src/olap/key_coder.cpp b/be/src/olap/key_coder.cpp index 450c805a8d..803b353375 100644 --- a/be/src/olap/key_coder.cpp +++ b/be/src/olap/key_coder.cpp @@ -17,7 +17,9 @@ #include "olap/key_coder.h" +#include #include +#include namespace doris { diff --git a/be/src/olap/key_coder.h b/be/src/olap/key_coder.h index 411517bd05..87c2748688 100644 --- a/be/src/olap/key_coder.h +++ b/be/src/olap/key_coder.h @@ -17,13 +17,23 @@ #pragma once +#include +#include +#include +#include + +#include +#include #include #include #include "common/status.h" #include "gutil/endian.h" #include "gutil/strings/substitute.h" +#include "olap/decimal12.h" +#include "olap/olap_common.h" #include "olap/types.h" +#include "util/slice.h" namespace doris { diff --git a/be/src/olap/like_column_predicate.cpp b/be/src/olap/like_column_predicate.cpp index 049a419fbf..ef1374d445 100644 --- a/be/src/olap/like_column_predicate.cpp +++ b/be/src/olap/like_column_predicate.cpp @@ -17,8 +17,10 @@ #include "olap/like_column_predicate.h" -#include "olap/field.h" +#include "runtime/define_primitive_type.h" #include "udf/udf.h" +#include "vec/columns/columns_number.h" +#include "vec/columns/predicate_column.h" #include "vec/common/string_ref.h" namespace doris { diff --git a/be/src/olap/like_column_predicate.h b/be/src/olap/like_column_predicate.h index 11e4c9f07f..ddbe892303 100644 --- a/be/src/olap/like_column_predicate.h +++ b/be/src/olap/like_column_predicate.h @@ -16,14 +16,36 @@ // under the License. #pragma once +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" #include "olap/column_predicate.h" -#include "udf/udf.h" +#include "olap/rowset/segment_v2/bloom_filter.h" +#include "vec/columns/column.h" #include "vec/columns/column_dictionary.h" +#include "vec/columns/column_nullable.h" #include "vec/common/string_ref.h" #include "vec/core/types.h" #include "vec/functions/like.h" +namespace roaring { +class Roaring; +} // namespace roaring + namespace doris { +class FunctionContext; + +namespace segment_v2 { +class BitmapIndexIterator; +} // namespace segment_v2 class LikeColumnPredicate : public ColumnPredicate { public: diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp index 2132db5b2d..000315f773 100644 --- a/be/src/olap/lru_cache.cpp +++ b/be/src/olap/lru_cache.cpp @@ -4,17 +4,14 @@ #include "olap/lru_cache.h" -#include -#include #include +#include +#include #include #include #include "gutil/bits.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/utils.h" #include "runtime/thread_context.h" #include "util/doris_metrics.h" diff --git a/be/src/olap/lru_cache.h b/be/src/olap/lru_cache.h index 599f386e1b..654b9e8964 100644 --- a/be/src/olap/lru_cache.h +++ b/be/src/olap/lru_cache.h @@ -4,18 +4,22 @@ #pragma once +#include +#include +#include #include -#include #include +#include #include +#include #include -#include +#include +#include #include -#include +#include -#include "olap/olap_common.h" -#include "runtime/memory/mem_tracker.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" #include "util/doris_metrics.h" #include "util/lock.h" @@ -49,7 +53,6 @@ namespace doris { } while (0) class Cache; -class CacheKey; enum LRUCacheType { SIZE, // The capacity of cache is based on the size of cache entry. diff --git a/be/src/olap/match_predicate.cpp b/be/src/olap/match_predicate.cpp index 22d56cd1ff..6837d78f4e 100644 --- a/be/src/olap/match_predicate.cpp +++ b/be/src/olap/match_predicate.cpp @@ -17,14 +17,16 @@ #include "olap/match_predicate.h" -#include - -#include -#include +#include #include "exec/olap_utils.h" -#include "exprs/string_functions.h" +#include "olap/field.h" +#include "olap/olap_common.h" +#include "olap/rowset/segment_v2/inverted_index_cache.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" #include "olap/schema.h" +#include "olap/types.h" +#include "olap/utils.h" #include "vec/common/string_ref.h" namespace doris { diff --git a/be/src/olap/match_predicate.h b/be/src/olap/match_predicate.h index ff41fb00eb..ad793b90f8 100644 --- a/be/src/olap/match_predicate.h +++ b/be/src/olap/match_predicate.h @@ -18,17 +18,30 @@ #ifndef DORIS_BE_SRC_QUERY_EXPRS_MATCH_PREDICATE_H #define DORIS_BE_SRC_QUERY_EXPRS_MATCH_PREDICATE_H -#include +#include +#include + +#include #include -#include "gen_cpp/Exprs_types.h" +#include "common/status.h" #include "olap/column_predicate.h" -#include "runtime/string_search.hpp" +#include "olap/schema.h" + +namespace roaring { +class Roaring; +} // namespace roaring namespace doris { enum class MatchType; +namespace segment_v2 { +class BitmapIndexIterator; +class InvertedIndexIterator; +enum class InvertedIndexQueryType; +} // namespace segment_v2 + class MatchPredicate : public ColumnPredicate { public: MatchPredicate(uint32_t column_id, const std::string& value, MatchType match_type); diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index db18782dc3..9084cc617a 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -17,18 +17,41 @@ #include "olap/memtable.h" +#include +#include + +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/consts.h" #include "common/logging.h" +#include "olap/olap_define.h" #include "olap/rowset/beta_rowset.h" #include "olap/rowset/rowset_writer.h" +#include "olap/rowset/segment_v2/segment.h" #include "olap/schema.h" #include "olap/schema_change.h" +#include "olap/tablet_schema.h" +#include "runtime/descriptors.h" +#include "runtime/exec_env.h" #include "runtime/load_channel_mgr.h" +#include "runtime/thread_context.h" #include "util/doris_metrics.h" +#include "util/runtime_profile.h" +#include "util/stopwatch.hpp" #include "vec/aggregate_functions/aggregate_function_reader.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" +#include "vec/columns/column.h" #include "vec/columns/column_object.h" -#include "vec/core/columns_with_type_and_name.h" -#include "vec/core/field.h" +#include "vec/columns/column_string.h" +#include "vec/common/assert_cast.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/data_types/data_type.h" +#include "vec/json/path_in_data.h" #include "vec/jsonb/serialize.h" namespace doris { diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index e47e696bb0..f50b78c0b1 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -17,16 +17,22 @@ #pragma once +#include +#include + +#include #include +#include #include "common/object_pool.h" -#include "olap/olap_define.h" +#include "common/status.h" +#include "olap/olap_common.h" #include "olap/skiplist.h" #include "olap/tablet.h" +#include "olap/tablet_meta.h" #include "runtime/memory/mem_tracker.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/common/arena.h" -#include "vec/common/string_ref.h" #include "vec/core/block.h" namespace doris { @@ -36,6 +42,7 @@ class Schema; class SlotDescriptor; class TabletSchema; class TupleDescriptor; +enum KeysType : int; class MemTable { public: diff --git a/be/src/olap/memtable_flush_executor.cpp b/be/src/olap/memtable_flush_executor.cpp index b40ca93d9d..9c5b92ed10 100644 --- a/be/src/olap/memtable_flush_executor.cpp +++ b/be/src/olap/memtable_flush_executor.cpp @@ -17,11 +17,16 @@ #include "olap/memtable_flush_executor.h" -#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" #include "olap/memtable.h" -#include "runtime/thread_context.h" -#include "util/scoped_cleanup.h" +#include "util/stopwatch.hpp" #include "util/time.h" namespace doris { diff --git a/be/src/olap/memtable_flush_executor.h b/be/src/olap/memtable_flush_executor.h index ff5e1e08e8..7608fa4d19 100644 --- a/be/src/olap/memtable_flush_executor.h +++ b/be/src/olap/memtable_flush_executor.h @@ -17,21 +17,22 @@ #pragma once +#include + #include +#include #include +#include #include -#include "gen_cpp/olap_file.pb.h" -#include "olap/olap_define.h" +#include "common/status.h" #include "util/threadpool.h" namespace doris { class DataDir; -class DeltaWriter; -class ExecEnv; class MemTable; -class MemTrackerLimiter; +enum RowsetTypePB : int; // the statistic of a certain flush handler. // use atomic because it may be updated by multi threads diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index a12cc4cc16..437d9ba9eb 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -17,15 +17,39 @@ #include "olap/merger.h" +#include +#include +#include + +#include +#include #include +#include +#include +#include +#include +#include #include +#include "common/config.h" +#include "common/logging.h" +#include "olap/olap_common.h" #include "olap/olap_define.h" -#include "olap/row_cursor.h" +#include "olap/reader.h" +#include "olap/rowid_conversion.h" +#include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/rowset/rowset_writer.h" +#include "olap/rowset/segment_v2/segment_writer.h" #include "olap/storage_engine.h" #include "olap/tablet.h" +#include "olap/utils.h" +#include "util/slice.h" #include "util/trace.h" +#include "vec/core/block.h" #include "vec/olap/block_reader.h" +#include "vec/olap/vertical_block_reader.h" +#include "vec/olap/vertical_merge_iterator.h" namespace doris { diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h index 467c126304..37291c548c 100644 --- a/be/src/olap/merger.h +++ b/be/src/olap/merger.h @@ -17,20 +17,29 @@ #pragma once +#include + +#include + +#include "common/status.h" #include "io/io_common.h" -#include "olap/olap_define.h" -#include "olap/rowid_conversion.h" -#include "olap/rowset/rowset_writer.h" -#include "olap/rowset/segment_v2/segment_writer.h" +#include "olap/rowset/rowset_reader.h" #include "olap/tablet.h" -#include "vec/olap/vertical_block_reader.h" -#include "vec/olap/vertical_merge_iterator.h" +#include "olap/tablet_schema.h" namespace doris { +class KeyBoundsPB; +class RowIdConversion; +class RowsetWriter; + +namespace segment_v2 { +class SegmentWriter; +} // namespace segment_v2 namespace vectorized { class RowSourcesBuffer; -}; +class VerticalBlockReader; +}; // namespace vectorized class Merger { public: diff --git a/be/src/olap/null_predicate.cpp b/be/src/olap/null_predicate.cpp index e9a9bf9705..e4130f0d32 100644 --- a/be/src/olap/null_predicate.cpp +++ b/be/src/olap/null_predicate.cpp @@ -17,9 +17,16 @@ #include "olap/null_predicate.h" -#include "olap/field.h" +#include + +#include + +#include "olap/rowset/segment_v2/bitmap_index_reader.h" +#include "olap/rowset/segment_v2/inverted_index_cache.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" +#include "vec/columns/column.h" #include "vec/columns/column_nullable.h" -#include "vec/common/string_ref.h" +#include "vec/runtime/vdatetime_value.h" using namespace doris::vectorized; diff --git a/be/src/olap/null_predicate.h b/be/src/olap/null_predicate.h index e878907228..ed81cc6f1b 100644 --- a/be/src/olap/null_predicate.h +++ b/be/src/olap/null_predicate.h @@ -17,15 +17,31 @@ #pragma once +#include #include -#include +#include +#include +#include +#include "common/status.h" #include "olap/column_predicate.h" #include "olap/rowset/segment_v2/bloom_filter.h" +#include "olap/schema.h" #include "olap/wrapper_field.h" +namespace roaring { +class Roaring; +} // namespace roaring + namespace doris { +namespace segment_v2 { +class BitmapIndexIterator; +class InvertedIndexIterator; +} // namespace segment_v2 +namespace vectorized { +class IColumn; +} // namespace vectorized class NullPredicate : public ColumnPredicate { public: diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index afb2bdcf22..ad81578a6a 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include @@ -31,7 +32,6 @@ #include #include -#include "gen_cpp/Types_types.h" #include "io/io_common.h" #include "olap/olap_define.h" #include "util/hash_util.hpp" diff --git a/be/src/olap/olap_meta.cpp b/be/src/olap/olap_meta.cpp index 92b85b1d5b..35ea9b8d8e 100644 --- a/be/src/olap/olap_meta.cpp +++ b/be/src/olap/olap_meta.cpp @@ -17,9 +17,16 @@ #include "olap/olap_meta.h" +#include +#include +#include +#include + +#include #include #include +#include "common/config.h" #include "common/logging.h" #include "olap/olap_define.h" #include "rocksdb/db.h" diff --git a/be/src/olap/olap_meta.h b/be/src/olap/olap_meta.h index d2557415b3..a8f1df8d22 100644 --- a/be/src/olap/olap_meta.h +++ b/be/src/olap/olap_meta.h @@ -18,11 +18,15 @@ #pragma once #include -#include #include +#include -#include "olap/olap_define.h" -#include "rocksdb/db.h" +#include "common/status.h" + +namespace rocksdb { +class ColumnFamilyHandle; +class DB; +} // namespace rocksdb namespace doris { diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 55b6ef737b..3f21adb229 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -15,28 +15,55 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include +#include +#include -#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include +#include #include +#include +#include +#include +#include +#include #include #include +#include +#include +#include +#include #include "common/config.h" +#include "common/logging.h" #include "common/status.h" -#include "gutil/strings/substitute.h" +#include "gutil/ref_counted.h" #include "io/cache/file_cache_manager.h" +#include "io/fs/file_writer.h" // IWYU pragma: keep +#include "io/fs/path.h" #include "olap/cold_data_compaction.h" -#include "olap/cumulative_compaction.h" +#include "olap/compaction_permit_limiter.h" +#include "olap/cumulative_compaction_policy.h" +#include "olap/data_dir.h" #include "olap/olap_common.h" -#include "olap/olap_define.h" #include "olap/rowset/beta_rowset_writer.h" +#include "olap/rowset/segcompaction.h" #include "olap/storage_engine.h" +#include "olap/tablet.h" +#include "olap/tablet_manager.h" +#include "olap/tablet_meta.h" +#include "olap/tablet_schema.h" #include "service/point_query_executor.h" +#include "util/countdown_latch.h" +#include "util/doris_metrics.h" +#include "util/priority_thread_pool.hpp" +#include "util/thread.h" +#include "util/threadpool.h" #include "util/time.h" +#include "util/uid_util.h" using std::string; diff --git a/be/src/olap/options.cpp b/be/src/olap/options.cpp index 1226ecd415..80af2321e4 100644 --- a/be/src/olap/options.cpp +++ b/be/src/olap/options.cpp @@ -17,16 +17,23 @@ #include "olap/options.h" +#include #include +#include +#include +#include #include +#include +#include #include "common/config.h" #include "common/logging.h" #include "common/status.h" #include "gutil/strings/split.h" -#include "gutil/strings/substitute.h" +#include "gutil/strings/strip.h" #include "io/fs/local_file_system.h" +#include "olap/olap_define.h" #include "olap/utils.h" #include "util/path_util.h" diff --git a/be/src/olap/options.h b/be/src/olap/options.h index c6ed214ceb..84634a92cd 100644 --- a/be/src/olap/options.h +++ b/be/src/olap/options.h @@ -17,11 +17,15 @@ #pragma once +#include +#include + #include +#include #include +#include "common/status.h" #include "io/cache/block/block_file_cache_settings.h" -#include "olap/olap_define.h" #include "util/uid_util.h" namespace doris { diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp index 2813f85dd3..53f9378d17 100644 --- a/be/src/olap/page_cache.cpp +++ b/be/src/olap/page_cache.cpp @@ -17,7 +17,9 @@ #include "olap/page_cache.h" -#include "runtime/thread_context.h" +#include + +#include namespace doris { diff --git a/be/src/olap/page_cache.h b/be/src/olap/page_cache.h index 6313e931e0..cab0da2dd7 100644 --- a/be/src/olap/page_cache.h +++ b/be/src/olap/page_cache.h @@ -17,14 +17,17 @@ #pragma once +#include +#include +#include +#include + #include #include #include -#include "gen_cpp/segment_v2.pb.h" // for cache allocation -#include "gutil/macros.h" // for DISALLOW_COPY_AND_ASSIGN #include "olap/lru_cache.h" -#include "runtime/memory/mem_tracker.h" +#include "util/slice.h" namespace doris { diff --git a/be/src/olap/primary_key_index.cpp b/be/src/olap/primary_key_index.cpp index f9656ee897..e7f11aa3d1 100644 --- a/be/src/olap/primary_key_index.cpp +++ b/be/src/olap/primary_key_index.cpp @@ -17,9 +17,17 @@ #include "olap/primary_key_index.h" +#include + +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" -#include "io/fs/file_reader.h" +#include "olap/olap_common.h" +#include "olap/rowset/segment_v2/bloom_filter_index_reader.h" #include "olap/rowset/segment_v2/encoding_info.h" +#include "olap/types.h" namespace doris { diff --git a/be/src/olap/primary_key_index.h b/be/src/olap/primary_key_index.h index c80f0f132c..65cc64f0cd 100644 --- a/be/src/olap/primary_key_index.h +++ b/be/src/olap/primary_key_index.h @@ -17,11 +17,15 @@ #pragma once +#include +#include +#include + +#include + #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "io/fs/file_writer.h" +#include "io/fs/file_reader_writer_fwd.h" #include "olap/rowset/segment_v2/bloom_filter.h" -#include "olap/rowset/segment_v2/bloom_filter_index_reader.h" #include "olap/rowset/segment_v2/bloom_filter_index_writer.h" #include "olap/rowset/segment_v2/indexed_column_reader.h" #include "olap/rowset/segment_v2/indexed_column_writer.h" @@ -29,6 +33,14 @@ #include "util/slice.h" namespace doris { +class TypeInfo; + +namespace io { +class FileWriter; +} // namespace io +namespace segment_v2 { +class PrimaryKeyIndexMetaPB; +} // namespace segment_v2 // Build index for primary key. // The primary key index is designed in a similar way like RocksDB diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index 6ed44ea253..81a46753a3 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -17,20 +17,43 @@ #include "olap/push_handler.h" -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include -#include "common/object_pool.h" +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "common/logging.h" #include "common/status.h" -#include "olap/rowset/rowset_id_generator.h" -#include "olap/rowset/rowset_meta_manager.h" -#include "olap/schema_change.h" +#include "olap/delete_handler.h" +#include "olap/olap_define.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/rowset/rowset_writer.h" +#include "olap/rowset/rowset_writer_context.h" +#include "olap/schema.h" #include "olap/storage_engine.h" #include "olap/tablet.h" +#include "olap/tablet_manager.h" #include "olap/tablet_schema.h" +#include "olap/txn_manager.h" +#include "runtime/descriptors.h" #include "runtime/exec_env.h" +#include "util/runtime_profile.h" +#include "util/time.h" +#include "vec/core/block.h" #include "vec/exec/vparquet_scanner.h" namespace doris { diff --git a/be/src/olap/push_handler.h b/be/src/olap/push_handler.h index 8546fbf749..9f6a91a3d5 100644 --- a/be/src/olap/push_handler.h +++ b/be/src/olap/push_handler.h @@ -17,24 +17,36 @@ #pragma once -#include +#include +#include +#include +#include + +#include #include #include +#include "common/object_pool.h" +#include "common/status.h" #include "exec/base_scanner.h" -#include "gen_cpp/AgentService_types.h" -#include "gen_cpp/MasterService_types.h" -#include "gen_cpp/PaloInternalService_types.h" -#include "olap/merger.h" #include "olap/olap_common.h" -#include "olap/row_cursor.h" #include "olap/rowset/rowset.h" +#include "olap/tablet.h" #include "olap/tablet_schema.h" +#include "runtime/runtime_state.h" namespace doris { -struct ColumnMapping; -class RowCursor; +class DescriptorTbl; +class RuntimeProfile; +class Schema; +class TBrokerScanRange; +class TDescriptorTable; +class TTabletInfo; + +namespace vectorized { +class Block; +} // namespace vectorized class PushHandler { public: diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp index b21691af2d..9886db76d6 100644 --- a/be/src/olap/reader.cpp +++ b/be/src/olap/reader.cpp @@ -17,20 +17,42 @@ #include "olap/reader.h" +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "common/logging.h" #include "common/status.h" +#include "exprs/bitmapfilter_predicate.h" +#include "exprs/bloom_filter_func.h" #include "exprs/create_predicate_function.h" #include "exprs/hybrid_set.h" -#include "gen_cpp/segment_v2.pb.h" -#include "olap/bloom_filter_predicate.h" -#include "olap/comparison_predicate.h" -#include "olap/in_list_predicate.h" +#include "olap/column_predicate.h" #include "olap/itoken_extractor.h" #include "olap/like_column_predicate.h" #include "olap/olap_common.h" +#include "olap/olap_define.h" #include "olap/predicate_creator.h" #include "olap/row_cursor.h" +#include "olap/rowset/segment_v2/bloom_filter.h" #include "olap/schema.h" #include "olap/tablet.h" +#include "olap/tablet_meta.h" +#include "runtime/query_fragments_ctx.h" +#include "runtime/runtime_predicate.h" +#include "runtime/runtime_state.h" +#include "vec/common/arena.h" +#include "vec/core/block.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/olap/reader.h b/be/src/olap/reader.h index 5d0cc71126..394ca247b8 100644 --- a/be/src/olap/reader.h +++ b/be/src/olap/reader.h @@ -17,31 +17,50 @@ #pragma once -#include +#include +#include +#include +#include -#include "exprs/bitmapfilter_predicate.h" +#include +#include +#include +#include +#include +#include + +#include "common/status.h" #include "exprs/function_filter.h" -#include "exprs/hybrid_set.h" +#include "gutil/strings/substitute.h" #include "io/io_common.h" #include "olap/delete_handler.h" +#include "olap/iterators.h" +#include "olap/olap_common.h" +#include "olap/olap_tuple.h" #include "olap/row_cursor.h" +#include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_reader.h" +#include "olap/rowset/rowset_reader_context.h" #include "olap/tablet.h" #include "olap/tablet_schema.h" -#include "util/runtime_profile.h" -#include "vec/common/arena.h" -#include "vec/exprs/vexpr_context.h" namespace doris { -class Tablet; -class RowCursor; class RuntimeState; +class BitmapFilterFuncBase; +class BloomFilterFuncBase; +class ColumnPredicate; +class DeleteBitmap; +class HybridSetBase; +class RuntimeProfile; namespace vectorized { class VCollectIterator; class Block; class VExpr; +class Arena; +class VExprContext; } // namespace vectorized // Used to compare row with input scan key. Scan key only contains key columns, diff --git a/be/src/olap/row_cursor.cpp b/be/src/olap/row_cursor.cpp index ec0fd094ff..48756e2366 100644 --- a/be/src/olap/row_cursor.cpp +++ b/be/src/olap/row_cursor.cpp @@ -17,10 +17,18 @@ #include "olap/row_cursor.h" -#include -#include +#include +#include -#include "util/stack_util.h" +#include +#include +#include +#include + +#include "olap/field.h" +#include "olap/olap_common.h" +#include "olap/olap_define.h" +#include "util/slice.h" using std::min; using std::nothrow; diff --git a/be/src/olap/row_cursor.h b/be/src/olap/row_cursor.h index 0a0f5f2913..7c19e7ad55 100644 --- a/be/src/olap/row_cursor.h +++ b/be/src/olap/row_cursor.h @@ -17,15 +17,19 @@ #pragma once +#include +#include +#include + +#include #include #include -#include "olap/field.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" +#include "common/status.h" #include "olap/olap_tuple.h" #include "olap/row_cursor_cell.h" #include "olap/schema.h" +#include "olap/tablet_schema.h" namespace doris { class Field; diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 513a04a8d1..9e9769bcd4 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -17,16 +17,23 @@ #include "olap/rowset/beta_rowset.h" -#include -#include -#include // for remove() -#include // for link() +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" #include "common/status.h" -#include "gutil/strings/substitute.h" #include "io/cache/file_cache_manager.h" -#include "io/fs/fs_utils.h" -#include "io/fs/s3_file_system.h" +#include "io/fs/file_reader_options.h" +#include "io/fs/file_system.h" +#include "io/fs/local_file_system.h" +#include "io/fs/path.h" +#include "io/fs/remote_file_system.h" +#include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/rowset/beta_rowset_reader.h" #include "olap/rowset/segment_v2/inverted_index_cache.h" diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 5401c60860..0ef0c456ae 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -18,22 +18,29 @@ #ifndef DORIS_SRC_OLAP_ROWSET_BETA_ROWSET_H_ #define DORIS_SRC_OLAP_ROWSET_BETA_ROWSET_H_ -#include -#include +#include -#include "olap/olap_common.h" -#include "olap/olap_define.h" +#include +#include +#include +#include + +#include "common/status.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_reader.h" #include "olap/rowset/segment_v2/segment.h" +#include "olap/tablet_schema.h" namespace doris { -class BetaRowsetReader; -class RowsetFactory; - class BetaRowset; + +namespace io { +class RemoteFileSystem; +} // namespace io +struct RowsetId; + using BetaRowsetSharedPtr = std::shared_ptr; class BetaRowset : public Rowset { diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 0a4e7ccdb4..e54527e962 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -17,12 +17,30 @@ #include "beta_rowset_reader.h" +#include + +#include +#include +#include +#include +#include +#include #include +#include "common/logging.h" +#include "io/io_common.h" +#include "olap/block_column_predicate.h" +#include "olap/column_predicate.h" #include "olap/delete_handler.h" +#include "olap/olap_define.h" #include "olap/row_cursor.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/rowset/rowset_reader_context.h" +#include "olap/rowset/segment_v2/segment.h" #include "olap/schema.h" #include "olap/tablet_meta.h" +#include "olap/tablet_schema.h" +#include "util/runtime_profile.h" #include "vec/core/block.h" #include "vec/olap/vgeneric_iterators.h" diff --git a/be/src/olap/rowset/beta_rowset_reader.h b/be/src/olap/rowset/beta_rowset_reader.h index 10cfcb8b0f..de2ecb44d4 100644 --- a/be/src/olap/rowset/beta_rowset_reader.h +++ b/be/src/olap/rowset/beta_rowset_reader.h @@ -17,13 +17,27 @@ #pragma once +#include +#include + +#include +#include +#include + +#include "common/status.h" #include "olap/iterators.h" -#include "olap/row_cursor.h" +#include "olap/olap_common.h" #include "olap/rowset/beta_rowset.h" +#include "olap/rowset/rowset.h" #include "olap/rowset/rowset_reader.h" #include "olap/segment_loader.h" +#include "vec/core/block.h" namespace doris { +class RuntimeProfile; +class Schema; +struct RowLocation; +struct RowsetReaderContext; class BetaRowsetReader : public RowsetReader { public: diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 8b9b04a82c..744cf8066c 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -17,35 +17,46 @@ #include "olap/rowset/beta_rowset_writer.h" +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include + #include // time +#include #include #include +#include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" #include "common/logging.h" +#include "gutil/integral_types.h" #include "gutil/strings/substitute.h" +#include "io/fs/file_reader_options.h" +#include "io/fs/file_system.h" #include "io/fs/file_writer.h" -#include "olap/memtable.h" -#include "olap/merger.h" +#include "olap/data_dir.h" #include "olap/olap_define.h" -#include "olap/row_cursor.h" // RowCursor #include "olap/rowset/beta_rowset.h" #include "olap/rowset/rowset_factory.h" #include "olap/rowset/segment_v2/inverted_index_cache.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" +#include "olap/rowset/segment_v2/segment.h" #include "olap/rowset/segment_v2/segment_writer.h" #include "olap/storage_engine.h" -#include "runtime/exec_env.h" -#include "runtime/memory/mem_tracker_limiter.h" +#include "olap/tablet.h" +#include "olap/tablet_schema.h" #include "segcompaction.h" +#include "util/slice.h" +#include "util/time.h" #include "vec/common/schema_util.h" // LocalSchemaChangeRecorder -#include "vec/jsonb/serialize.h" +#include "vec/core/block.h" namespace doris { using namespace ErrorCode; -class StorageEngine; - BetaRowsetWriter::BetaRowsetWriter() : _rowset_meta(nullptr), _num_segment(0), diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h index 25c05318e9..c6fa740a58 100644 --- a/be/src/olap/rowset/beta_rowset_writer.h +++ b/be/src/olap/rowset/beta_rowset_writer.h @@ -17,22 +17,41 @@ #pragma once +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "olap/olap_common.h" +#include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_writer.h" +#include "olap/rowset/rowset_writer_context.h" #include "segcompaction.h" #include "segment_v2/segment.h" -#include "vec/columns/column.h" -#include "vec/olap/vertical_block_reader.h" -#include "vec/olap/vgeneric_iterators.h" +#include "util/spinlock.h" namespace doris { +namespace vectorized { +class Block; +} // namespace vectorized + namespace segment_v2 { class SegmentWriter; } // namespace segment_v2 -namespace io { -class FileWriter; -} // namespace io - using SegCompactionCandidates = std::vector; using SegCompactionCandidatesSharedPtr = std::shared_ptr; namespace vectorized::schema_util { diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp index d29c02ed95..d9aa8fb05e 100644 --- a/be/src/olap/rowset/rowset.cpp +++ b/be/src/olap/rowset/rowset.cpp @@ -17,8 +17,10 @@ #include "olap/rowset/rowset.h" +#include + +#include "olap/olap_define.h" #include "olap/tablet_schema.h" -#include "olap/tablet_schema_cache.h" #include "util/time.h" namespace doris { diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index c2ae5f1b05..198dd8a0cd 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -17,26 +17,36 @@ #pragma once +#include +#include +#include +#include +#include +#include + #include #include #include +#include +#include #include -#include "gen_cpp/olap_file.pb.h" -#include "gutil/macros.h" -#include "io/fs/remote_file_system.h" +#include "common/logging.h" +#include "common/status.h" +#include "olap/olap_common.h" #include "olap/rowset/rowset_meta.h" #include "olap/tablet_schema.h" #include "util/lock.h" namespace doris { -class DataDir; -class OlapTuple; -class RowCursor; class Rowset; + +namespace io { +class RemoteFileSystem; +} // namespace io + using RowsetSharedPtr = std::shared_ptr; -class RowsetFactory; class RowsetReader; // the rowset state transfer graph: diff --git a/be/src/olap/rowset/rowset_factory.cpp b/be/src/olap/rowset/rowset_factory.cpp index 78f6b2326c..e66004da70 100644 --- a/be/src/olap/rowset/rowset_factory.cpp +++ b/be/src/olap/rowset/rowset_factory.cpp @@ -17,12 +17,15 @@ #include "olap/rowset/rowset_factory.h" +#include + #include #include "beta_rowset.h" -#include "gen_cpp/olap_file.pb.h" +#include "io/fs/file_writer.h" // IWYU pragma: keep #include "olap/rowset/beta_rowset_writer.h" #include "olap/rowset/rowset_writer.h" +#include "olap/rowset/rowset_writer_context.h" #include "olap/rowset/vertical_beta_rowset_writer.h" namespace doris { diff --git a/be/src/olap/rowset/rowset_factory.h b/be/src/olap/rowset/rowset_factory.h index d0adc6f457..9c3543c184 100644 --- a/be/src/olap/rowset/rowset_factory.h +++ b/be/src/olap/rowset/rowset_factory.h @@ -18,9 +18,13 @@ #ifndef DORIS_BE_SRC_OLAP_ROWSET_FACTORY_H #define DORIS_BE_SRC_OLAP_ROWSET_FACTORY_H -#include "gen_cpp/olap_file.pb.h" -#include "olap/data_dir.h" +#include +#include + +#include "common/status.h" #include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/tablet_schema.h" namespace doris { diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index 9697055864..f9491e1553 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -18,12 +18,13 @@ #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H +#include + #include #include #include #include "common/logging.h" -#include "gen_cpp/olap_file.pb.h" #include "google/protobuf/util/message_differencer.h" #include "io/fs/file_system.h" #include "io/fs/local_file_system.h" diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp index 84e8be217f..3373db47d8 100644 --- a/be/src/olap/rowset/rowset_meta_manager.cpp +++ b/be/src/olap/rowset/rowset_meta_manager.cpp @@ -17,17 +17,18 @@ #include "olap/rowset/rowset_meta_manager.h" +#include + #include #include -#include +#include +#include #include #include #include "common/logging.h" -#include "json2pb/json_to_pb.h" -#include "json2pb/pb_to_json.h" #include "olap/olap_define.h" -#include "olap/storage_engine.h" +#include "olap/olap_meta.h" #include "olap/utils.h" namespace doris { diff --git a/be/src/olap/rowset/rowset_meta_manager.h b/be/src/olap/rowset/rowset_meta_manager.h index cca2304ce3..f6f193caae 100644 --- a/be/src/olap/rowset/rowset_meta_manager.h +++ b/be/src/olap/rowset/rowset_meta_manager.h @@ -18,11 +18,18 @@ #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_MANAGER_H #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_MANAGER_H +#include #include -#include "olap/olap_meta.h" +#include "common/status.h" +#include "olap/olap_common.h" #include "olap/rowset/rowset_meta.h" +namespace doris { +class OlapMeta; +class RowsetMetaPB; +} // namespace doris + using std::string; namespace doris { diff --git a/be/src/olap/rowset/rowset_reader.h b/be/src/olap/rowset/rowset_reader.h index 3d6b2c6fad..dadfd0e05c 100644 --- a/be/src/olap/rowset/rowset_reader.h +++ b/be/src/olap/rowset/rowset_reader.h @@ -18,10 +18,11 @@ #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_H #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_H +#include + #include #include -#include "gen_cpp/olap_file.pb.h" #include "olap/iterators.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_reader_context.h" diff --git a/be/src/olap/rowset/rowset_tree.cpp b/be/src/olap/rowset/rowset_tree.cpp index a9cc6a81e0..8ac153435f 100644 --- a/be/src/olap/rowset/rowset_tree.cpp +++ b/be/src/olap/rowset/rowset_tree.cpp @@ -21,8 +21,10 @@ #include "olap/rowset/rowset_tree.h" +#include #include +#include #include #include #include @@ -36,6 +38,7 @@ #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_meta.h" #include "util/interval_tree-inl.h" +#include "util/interval_tree.h" #include "util/slice.h" using std::shared_ptr; diff --git a/be/src/olap/rowset/rowset_tree.h b/be/src/olap/rowset/rowset_tree.h index 8228bc26d7..5ebabca0f1 100644 --- a/be/src/olap/rowset/rowset_tree.h +++ b/be/src/olap/rowset/rowset_tree.h @@ -21,15 +21,17 @@ #pragma once -#include #include #include #include +#include #include +#include #include #include "common/status.h" #include "gutil/map-util.h" +#include "olap/olap_common.h" #include "olap/rowset/rowset.h" #include "util/slice.h" @@ -37,7 +39,6 @@ namespace doris { template class IntervalTree; - struct RowsetIntervalTraits; struct RowsetWithBounds; diff --git a/be/src/olap/rowset/rowset_writer.h b/be/src/olap/rowset/rowset_writer.h index f4d55ac051..6249f01637 100644 --- a/be/src/olap/rowset/rowset_writer.h +++ b/be/src/olap/rowset/rowset_writer.h @@ -17,8 +17,9 @@ #pragma once -#include "gen_cpp/olap_file.pb.h" -#include "gen_cpp/types.pb.h" +#include +#include + #include "gutil/macros.h" #include "olap/column_mapping.h" #include "olap/rowset/rowset.h" diff --git a/be/src/olap/rowset/rowset_writer_context.h b/be/src/olap/rowset/rowset_writer_context.h index fc33a521bb..8d4fabfc88 100644 --- a/be/src/olap/rowset/rowset_writer_context.h +++ b/be/src/olap/rowset/rowset_writer_context.h @@ -17,7 +17,8 @@ #pragma once -#include "gen_cpp/olap_file.pb.h" +#include + #include "io/fs/file_system.h" #include "olap/tablet.h" #include "olap/tablet_schema.h" diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp index cc151afdac..8eb60eae7f 100644 --- a/be/src/olap/rowset/segcompaction.cpp +++ b/be/src/olap/rowset/segcompaction.cpp @@ -17,30 +17,50 @@ #include "segcompaction.h" -#include +#include +#include +#include +#include +#include +#include #include -#include +#include #include -#include +#include +#include +#include #include "beta_rowset_writer.h" -#include "common/config.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" +#include "gutil/stringprintf.h" #include "gutil/strings/substitute.h" +#include "io/fs/file_system.h" #include "io/fs/file_writer.h" -#include "olap/memtable.h" +#include "io/io_common.h" +#include "olap/data_dir.h" +#include "olap/iterators.h" #include "olap/merger.h" +#include "olap/olap_common.h" #include "olap/olap_define.h" -#include "olap/row_cursor.h" // RowCursor +#include "olap/reader.h" #include "olap/rowset/beta_rowset.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/rowset/rowset_writer_context.h" #include "olap/rowset/segment_v2/inverted_index_cache.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" +#include "olap/rowset/segment_v2/segment.h" +#include "olap/rowset/segment_v2/segment_writer.h" +#include "olap/schema.h" #include "olap/storage_engine.h" -#include "runtime/exec_env.h" +#include "olap/tablet_schema.h" #include "runtime/memory/mem_tracker_limiter.h" -#include "vec/common/schema_util.h" // LocalSchemaChangeRecorder -#include "vec/jsonb/serialize.h" +#include "runtime/thread_context.h" +#include "util/time.h" +#include "vec/olap/vertical_block_reader.h" +#include "vec/olap/vertical_merge_iterator.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/olap/rowset/segcompaction.h b/be/src/olap/rowset/segcompaction.h index 871f864b9b..33fcf4ba28 100644 --- a/be/src/olap/rowset/segcompaction.h +++ b/be/src/olap/rowset/segcompaction.h @@ -18,18 +18,27 @@ #pragma once #include -#include // time #include -#include +#include #include "common/status.h" +#include "io/fs/file_reader_writer_fwd.h" #include "olap/merger.h" -#include "olap/rowset/rowset_writer.h" +#include "olap/tablet.h" #include "segment_v2/segment.h" -#include "segment_v2/segment_writer.h" -#include "vec/olap/vertical_block_reader.h" namespace doris { +class Schema; + +namespace segment_v2 { +class SegmentWriter; +} // namespace segment_v2 +namespace vectorized { +class RowSourcesBuffer; +class VerticalBlockReader; +} // namespace vectorized +struct OlapReaderStatistics; + using SegCompactionCandidates = std::vector; using SegCompactionCandidatesSharedPtr = std::shared_ptr; diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp index 4ac002400e..94057fe5da 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp @@ -17,11 +17,26 @@ #include "olap/rowset/segment_v2/binary_dict_page.h" +#include + +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/logging.h" +#include "gutil/casts.h" +#include "gutil/port.h" #include "gutil/strings/substitute.h" // for Substitute -#include "util/slice.h" // for Slice +#include "olap/rowset/segment_v2/bitshuffle_page.h" +#include "util/coding.h" +#include "util/slice.h" // for Slice #include "vec/columns/column.h" namespace doris { +struct StringRef; + namespace segment_v2 { using strings::Substitute; diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.h b/be/src/olap/rowset/segment_v2/binary_dict_page.h index b570b042dc..0ab50ce634 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.h +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.h @@ -17,21 +17,33 @@ #pragma once -#include -#include -#include +#include +#include +#include -#include "gen_cpp/segment_v2.pb.h" +#include +#include + +#include "common/status.h" #include "gutil/hash/string_hash.h" #include "olap/olap_common.h" #include "olap/rowset/segment_v2/binary_plain_page.h" -#include "olap/rowset/segment_v2/bitshuffle_page.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/options.h" +#include "olap/rowset/segment_v2/page_builder.h" +#include "olap/rowset/segment_v2/page_decoder.h" +#include "util/faststring.h" +#include "util/slice.h" #include "vec/common/arena.h" +#include "vec/data_types/data_type.h" namespace doris { +struct StringRef; + namespace segment_v2 { +enum EncodingTypePB : int; +template +class BitShufflePageDecoder; enum { BINARY_DICT_PAGE_HEADER_SIZE = 4 }; diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp index 90029ea09c..e3e893a0dd 100644 --- a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp @@ -19,13 +19,11 @@ #include #include -#include -#include -#include +#include #include -#include "common/logging.h" +#include "gutil/port.h" #include "gutil/strings/substitute.h" #include "util/coding.h" #include "util/faststring.h" diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.h b/be/src/olap/rowset/segment_v2/binary_prefix_page.h index 2930a8e616..b6ca02190f 100644 --- a/be/src/olap/rowset/segment_v2/binary_prefix_page.h +++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.h @@ -17,20 +17,20 @@ #pragma once +#include #include #include -#include -#include -#include #include +#include "common/status.h" #include "olap/rowset/segment_v2/options.h" #include "olap/rowset/segment_v2/page_builder.h" #include "olap/rowset/segment_v2/page_decoder.h" #include "util/coding.h" #include "util/faststring.h" #include "util/slice.h" +#include "vec/columns/column.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp index c101e3e7da..0e2c26ff30 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp @@ -17,7 +17,16 @@ #include "olap/rowset/segment_v2/bitmap_index_reader.h" +#include +#include +#include + +#include + #include "olap/types.h" +#include "vec/columns/column.h" +#include "vec/common/string_ref.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_factory.hpp" namespace doris { diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h index e5fc28adcd..c448c9f5cc 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h +++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h @@ -17,23 +17,28 @@ #pragma once -#include +#include + +#include +#include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/indexed_column_reader.h" +#include "olap/types.h" + +namespace roaring { +class Roaring; +} // namespace roaring namespace doris { -class TypeInfo; - namespace segment_v2 { class BitmapIndexIterator; -class IndexedColumnReader; -class IndexedColumnIterator; +class BitmapIndexPB; class BitmapIndexReader { public: diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp index 957da3dc3d..8523740920 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp @@ -17,9 +17,15 @@ #include "olap/rowset/segment_v2/bitmap_index_writer.h" +#include + +#include #include #include +#include +#include +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/encoding_info.h" #include "olap/rowset/segment_v2/indexed_column_writer.h" diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.h b/be/src/olap/rowset/segment_v2/bitmap_index_writer.h index 93142728a4..cf7de5efe9 100644 --- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.h +++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.h @@ -17,12 +17,13 @@ #pragma once +#include +#include + #include #include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "gutil/macros.h" namespace doris { @@ -33,6 +34,7 @@ class FileWriter; } namespace segment_v2 { +class ColumnIndexMetaPB; class BitmapIndexWriter { public: diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.cpp b/be/src/olap/rowset/segment_v2/bitshuffle_page.cpp index 3afa0f4422..cd6f1f4e91 100644 --- a/be/src/olap/rowset/segment_v2/bitshuffle_page.cpp +++ b/be/src/olap/rowset/segment_v2/bitshuffle_page.cpp @@ -17,8 +17,6 @@ #include "olap/rowset/segment_v2/bitshuffle_page.h" -#include "olap/rowset/segment_v2/common.h" - namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.h b/be/src/olap/rowset/segment_v2/bitshuffle_page.h index 767ff35cfe..657a34e7c1 100644 --- a/be/src/olap/rowset/segment_v2/bitshuffle_page.h +++ b/be/src/olap/rowset/segment_v2/bitshuffle_page.h @@ -18,13 +18,15 @@ #pragma once #include -#include #include #include #include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" #include "gutil/port.h" #include "olap/olap_common.h" #include "olap/rowset/segment_v2/bitshuffle_wrapper.h" @@ -33,12 +35,12 @@ #include "olap/rowset/segment_v2/page_builder.h" #include "olap/rowset/segment_v2/page_decoder.h" #include "olap/types.h" -#include "runtime/memory/chunk_allocator.h" +#include "util/alignment.h" #include "util/coding.h" #include "util/faststring.h" #include "util/slice.h" -#include "vec/columns/column_nullable.h" -#include "vec/runtime/vdatetime_value.h" +#include "vec/columns/column.h" +#include "vec/data_types/data_type.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/block_split_bloom_filter.cpp b/be/src/olap/rowset/segment_v2/block_split_bloom_filter.cpp index da842d491b..871da736d5 100644 --- a/be/src/olap/rowset/segment_v2/block_split_bloom_filter.cpp +++ b/be/src/olap/rowset/segment_v2/block_split_bloom_filter.cpp @@ -17,7 +17,7 @@ #include "olap/rowset/segment_v2/block_split_bloom_filter.h" -#include "util/debug_util.h" +#include namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/block_split_bloom_filter.h b/be/src/olap/rowset/segment_v2/block_split_bloom_filter.h index 93591adc1d..f68ddd7e74 100644 --- a/be/src/olap/rowset/segment_v2/block_split_bloom_filter.h +++ b/be/src/olap/rowset/segment_v2/block_split_bloom_filter.h @@ -17,6 +17,8 @@ #pragma once +#include + #include "olap/rowset/segment_v2/bloom_filter.h" namespace doris { diff --git a/be/src/olap/rowset/segment_v2/bloom_filter.cpp b/be/src/olap/rowset/segment_v2/bloom_filter.cpp index e11fe6ca1d..12a7f2395e 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter.cpp +++ b/be/src/olap/rowset/segment_v2/bloom_filter.cpp @@ -15,14 +15,17 @@ // specific language governing permissions and limitations // under the License. -#include -#include +#include "olap/rowset/segment_v2/bloom_filter.h" -#include "gen_cpp/segment_v2.pb.h" -#include "gutil/strings/substitute.h" +#include +#include + +#include +#include + +#include "common/status.h" #include "olap/rowset/segment_v2/block_split_bloom_filter.h" #include "olap/rowset/segment_v2/ngram_bloom_filter.h" -#include "olap/utils.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/bloom_filter.h b/be/src/olap/rowset/segment_v2/bloom_filter.h index c235ec39b9..969d7b8aa9 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter.h +++ b/be/src/olap/rowset/segment_v2/bloom_filter.h @@ -17,14 +17,15 @@ #pragma once +#include +#include +#include + #include #include #include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "gutil/strings/substitute.h" -#include "olap/utils.h" #include "util/murmur_hash3.h" namespace doris { diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp index cecbd8c4ca..e3ae352dd7 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp @@ -17,8 +17,14 @@ #include "olap/rowset/segment_v2/bloom_filter_index_reader.h" +#include +#include + #include "olap/rowset/segment_v2/bloom_filter.h" #include "olap/types.h" +#include "vec/columns/column.h" +#include "vec/common/string_ref.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_factory.hpp" namespace doris { diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h index bb9a02c244..452a7ea2ae 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h +++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h @@ -17,26 +17,25 @@ #pragma once -#include +#include + #include +#include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/indexed_column_reader.h" -#include "olap/rowset/segment_v2/row_ranges.h" +#include "olap/types.h" namespace doris { -class TypeInfo; - namespace segment_v2 { class BloomFilterIndexIterator; -class IndexedColumnReader; -class IndexedColumnIterator; class BloomFilter; +class BloomFilterIndexPB; class BloomFilterIndexReader { public: diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp index 588f2eb997..34ff479733 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp @@ -17,16 +17,21 @@ #include "olap/rowset/segment_v2/bloom_filter_index_writer.h" -#include -#include +#include +#include +#include +#include +#include +#include + +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/bloom_filter.h" // for BloomFilterOptions, BloomFilter -#include "olap/rowset/segment_v2/common.h" -#include "olap/rowset/segment_v2/encoding_info.h" #include "olap/rowset/segment_v2/indexed_column_writer.h" #include "olap/types.h" -#include "util/faststring.h" +#include "runtime/decimalv2_value.h" #include "util/slice.h" +#include "util/types.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.h b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.h index 2cd0b8e340..c4f20d0f6b 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.h +++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.h @@ -17,13 +17,16 @@ #pragma once +#include +#include + #include #include +#include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "gutil/macros.h" #include "olap/itoken_extractor.h" +#include "olap/rowset/segment_v2/bloom_filter.h" #include "vec/common/arena.h" namespace doris { @@ -36,7 +39,7 @@ class FileWriter; namespace segment_v2 { -struct BloomFilterOptions; +class ColumnIndexMetaPB; class BloomFilterIndexWriter { public: diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index a4a437c353..c4219916c9 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -17,21 +17,51 @@ #include "olap/rowset/segment_v2/column_reader.h" +#include +#include + +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "io/fs/file_reader.h" +#include "olap/block_column_predicate.h" +#include "olap/column_predicate.h" +#include "olap/decimal12.h" +#include "olap/inverted_index_parser.h" #include "olap/rowset/segment_v2/binary_dict_page.h" // for BinaryDictPageDecoder +#include "olap/rowset/segment_v2/binary_plain_page.h" +#include "olap/rowset/segment_v2/bitmap_index_reader.h" +#include "olap/rowset/segment_v2/bloom_filter.h" #include "olap/rowset/segment_v2/bloom_filter_index_reader.h" #include "olap/rowset/segment_v2/encoding_info.h" // for EncodingInfo -#include "olap/rowset/segment_v2/page_handle.h" // for PageHandle +#include "olap/rowset/segment_v2/inverted_index_reader.h" +#include "olap/rowset/segment_v2/page_decoder.h" +#include "olap/rowset/segment_v2/page_handle.h" // for PageHandle #include "olap/rowset/segment_v2/page_io.h" #include "olap/rowset/segment_v2/page_pointer.h" // for PagePointer -#include "olap/types.h" // for TypeInfo +#include "olap/rowset/segment_v2/row_ranges.h" +#include "olap/rowset/segment_v2/zone_map_index.h" +#include "olap/tablet_schema.h" +#include "olap/types.h" // for TypeInfo #include "olap/wrapper_field.h" +#include "runtime/decimalv2_value.h" +#include "util/binary_cast.hpp" +#include "util/bitmap.h" #include "util/block_compression.h" #include "util/rle_encoding.h" // for RleDecoder +#include "util/slice.h" #include "vec/columns/column.h" #include "vec/columns/column_array.h" #include "vec/columns/column_map.h" +#include "vec/columns/column_nullable.h" #include "vec/columns/column_struct.h" +#include "vec/columns/column_vector.h" +#include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" +#include "vec/common/string_ref.h" #include "vec/core/types.h" #include "vec/runtime/vdatetime_value.h" //for VecDateTime diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index e0e7bc4a6f..19c43ed427 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -17,48 +17,61 @@ #pragma once +#include +#include + #include // for size_t #include // for uint32_t #include // for unique_ptr +#include +#include +#include +#include -#include "bloom_filter_index_reader.h" +#include "common/config.h" #include "common/logging.h" -#include "common/status.h" // for Status -#include "gen_cpp/segment_v2.pb.h" // for ColumnMetaPB -#include "io/fs/file_reader.h" +#include "common/status.h" // for Status +#include "io/fs/file_reader_writer_fwd.h" #include "io/io_common.h" -#include "olap/block_column_predicate.h" -#include "olap/column_predicate.h" -#include "olap/iterators.h" -#include "olap/rowset/segment_v2/bitmap_index_reader.h" // for BitmapIndexReader +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/common.h" -#include "olap/rowset/segment_v2/inverted_index_reader.h" // for InvertedIndexReader -#include "olap/rowset/segment_v2/ordinal_page_index.h" // for OrdinalPageIndexIterator -#include "olap/rowset/segment_v2/page_handle.h" // for PageHandle -#include "olap/rowset/segment_v2/parsed_page.h" // for ParsedPage -#include "olap/rowset/segment_v2/row_ranges.h" // for RowRanges -#include "olap/rowset/segment_v2/zone_map_index.h" -#include "olap/tablet_schema.h" +#include "olap/rowset/segment_v2/ordinal_page_index.h" // for OrdinalPageIndexIterator +#include "olap/rowset/segment_v2/page_handle.h" // for PageHandle +#include "olap/rowset/segment_v2/page_pointer.h" +#include "olap/rowset/segment_v2/parsed_page.h" // for ParsedPage +#include "olap/types.h" +#include "olap/utils.h" #include "util/once.h" +#include "vec/columns/column.h" #include "vec/columns/column_array.h" // ColumnArray +#include "vec/data_types/data_type.h" namespace doris { -class TypeInfo; class BlockCompressionCodec; class WrapperField; +class AndBlockColumnPredicate; +class ColumnPredicate; +class TabletIndex; -namespace fs { -class ReadableBlock; -} +namespace io { +class FileReader; +} // namespace io +struct Slice; +struct StringRef; namespace segment_v2 { class EncodingInfo; -class PageHandle; -struct PagePointer; class ColumnIterator; class BloomFilterIndexReader; +class BitmapIndexIterator; +class BitmapIndexReader; +class InvertedIndexIterator; +class InvertedIndexReader; +class PageDecoder; +class RowRanges; +class ZoneMapIndexReader; struct ColumnReaderOptions { // whether verify checksum when read page diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp index ab542a49dd..0ee3608f84 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/column_writer.cpp @@ -17,11 +17,17 @@ #include "olap/rowset/segment_v2/column_writer.h" -#include +#include +#include +#include +#include + +#include "common/config.h" #include "common/logging.h" #include "gutil/strings/substitute.h" #include "io/fs/file_writer.h" +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/bitmap_index_writer.h" #include "olap/rowset/segment_v2/bloom_filter.h" #include "olap/rowset/segment_v2/bloom_filter_index_writer.h" @@ -31,10 +37,15 @@ #include "olap/rowset/segment_v2/ordinal_page_index.h" #include "olap/rowset/segment_v2/page_builder.h" #include "olap/rowset/segment_v2/page_io.h" +#include "olap/rowset/segment_v2/page_pointer.h" #include "olap/rowset/segment_v2/zone_map_index.h" +#include "olap/tablet_schema.h" +#include "olap/types.h" +#include "runtime/collection_value.h" #include "util/block_compression.h" #include "util/faststring.h" #include "util/rle_encoding.h" +#include "vec/core/types.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h index 2aed982126..1c235ccda7 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.h +++ b/be/src/olap/rowset/segment_v2/column_writer.h @@ -17,22 +17,29 @@ #pragma once -#include // for unique_ptr +#include +#include +#include -#include "common/status.h" // for Status -#include "gen_cpp/segment_v2.pb.h" // for EncodingTypePB -#include "olap/field.h" // for Field -#include "olap/inverted_index_parser.h" +#include +#include // for unique_ptr +#include +#include +#include +#include + +#include "common/status.h" // for Status +#include "olap/field.h" // for Field #include "olap/rowset/segment_v2/common.h" -#include "olap/rowset/segment_v2/page_pointer.h" // for PagePointer -#include "olap/tablet_schema.h" // for TabletColumn -#include "util/bitmap.h" // for BitmapChange -#include "util/slice.h" // for OwnedSlice +#include "olap/rowset/segment_v2/inverted_index_writer.h" +#include "util/bitmap.h" // for BitmapChange +#include "util/slice.h" // for OwnedSlice namespace doris { -class TypeInfo; class BlockCompressionCodec; +class TabletColumn; +class TabletIndex; namespace io { class FileWriter; @@ -69,7 +76,6 @@ struct ColumnWriterOptions { }; class BitmapIndexWriter; -class InvertedIndexColumnWriter; class EncodingInfo; class NullBitmapBuilder; class OrdinalIndexWriter; diff --git a/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp b/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp index e9a51ed46a..95491196f9 100644 --- a/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/empty_segment_iterator.cpp @@ -18,6 +18,10 @@ #include "olap/rowset/segment_v2/empty_segment_iterator.h" namespace doris { +namespace vectorized { +class Block; +} // namespace vectorized + namespace segment_v2 { EmptySegmentIterator::EmptySegmentIterator(const doris::Schema& schema) : _schema(schema) {} diff --git a/be/src/olap/rowset/segment_v2/empty_segment_iterator.h b/be/src/olap/rowset/segment_v2/empty_segment_iterator.h index 0bd7014c61..46d4e9bb8f 100644 --- a/be/src/olap/rowset/segment_v2/empty_segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/empty_segment_iterator.h @@ -19,10 +19,13 @@ #include "common/status.h" #include "olap/iterators.h" -#include "olap/rowset/segment_v2/segment.h" #include "olap/schema.h" namespace doris { +namespace vectorized { +class Block; +} // namespace vectorized + namespace segment_v2 { class EmptySegmentIterator : public RowwiseIterator { diff --git a/be/src/olap/rowset/segment_v2/encoding_info.cpp b/be/src/olap/rowset/segment_v2/encoding_info.cpp index 0949e9c1b9..c500d7e902 100644 --- a/be/src/olap/rowset/segment_v2/encoding_info.cpp +++ b/be/src/olap/rowset/segment_v2/encoding_info.cpp @@ -17,9 +17,13 @@ #include "olap/rowset/segment_v2/encoding_info.h" -#include +#include + +#include +#include +#include +#include -#include "gutil/strings/substitute.h" #include "olap/olap_common.h" #include "olap/rowset/segment_v2/binary_dict_page.h" #include "olap/rowset/segment_v2/binary_plain_page.h" @@ -29,6 +33,7 @@ #include "olap/rowset/segment_v2/frame_of_reference_page.h" #include "olap/rowset/segment_v2/plain_page.h" #include "olap/rowset/segment_v2/rle_page.h" +#include "olap/types.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/encoding_info.h b/be/src/olap/rowset/segment_v2/encoding_info.h index 7783862462..4086f78294 100644 --- a/be/src/olap/rowset/segment_v2/encoding_info.h +++ b/be/src/olap/rowset/segment_v2/encoding_info.h @@ -17,15 +17,18 @@ #pragma once +#include + #include +#include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "olap/types.h" +#include "util/slice.h" namespace doris { class TypeInfo; +enum class FieldType; namespace segment_v2 { @@ -33,6 +36,7 @@ class PageBuilder; class PageDecoder; struct PageBuilderOptions; struct PageDecoderOptions; +enum EncodingTypePB : int; // For better performance, some encodings (like BitShuffle) need to be decoded before being added to the PageCache. class DataPagePreDecoder { diff --git a/be/src/olap/rowset/segment_v2/index_page.cpp b/be/src/olap/rowset/segment_v2/index_page.cpp index 604cab1f3d..3b8bc03977 100644 --- a/be/src/olap/rowset/segment_v2/index_page.cpp +++ b/be/src/olap/rowset/segment_v2/index_page.cpp @@ -17,9 +17,11 @@ #include "olap/rowset/segment_v2/index_page.h" -#include +#include + +#include +#include -#include "common/logging.h" #include "util/coding.h" namespace doris { diff --git a/be/src/olap/rowset/segment_v2/index_page.h b/be/src/olap/rowset/segment_v2/index_page.h index dcfcedafb1..106fc0ffc5 100644 --- a/be/src/olap/rowset/segment_v2/index_page.h +++ b/be/src/olap/rowset/segment_v2/index_page.h @@ -17,13 +17,15 @@ #pragma once +#include +#include +#include +#include + #include -#include #include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "gutil/macros.h" #include "olap/rowset/segment_v2/page_pointer.h" #include "util/faststring.h" #include "util/slice.h" @@ -77,7 +79,6 @@ private: uint32_t _count = 0; }; -class IndexPageIterator; class IndexPageReader { public: IndexPageReader() : _parsed(false) {} diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp index 0dc0bffca6..8c5997f99f 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp @@ -17,11 +17,19 @@ #include "olap/rowset/segment_v2/indexed_column_reader.h" +#include + +#include + #include "gutil/strings/substitute.h" // for Substitute -#include "io/fs/local_file_system.h" #include "olap/key_coder.h" +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/encoding_info.h" // for EncodingInfo +#include "olap/rowset/segment_v2/options.h" +#include "olap/rowset/segment_v2/page_decoder.h" #include "olap/rowset/segment_v2/page_io.h" +#include "olap/types.h" +#include "util/block_compression.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h b/be/src/olap/rowset/segment_v2/indexed_column_reader.h index 10b87050ae..3ee39e04c0 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h +++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h @@ -17,30 +17,33 @@ #pragma once -#include +#include +#include +#include +#include + #include +#include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "io/fs/file_reader.h" -#include "io/fs/file_system.h" +#include "io/fs/file_reader_writer_fwd.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/index_page.h" #include "olap/rowset/segment_v2/page_handle.h" #include "olap/rowset/segment_v2/page_pointer.h" #include "olap/rowset/segment_v2/parsed_page.h" -#include "util/block_compression.h" #include "util/slice.h" +#include "vec/data_types/data_type.h" namespace doris { class KeyCoder; class TypeInfo; +class BlockCompressionCodec; namespace segment_v2 { class EncodingInfo; -class IndexedColumnIterator; // thread-safe reader for IndexedColumn (see comments of `IndexedColumnWriter` to understand what IndexedColumn is) class IndexedColumnReader { diff --git a/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp b/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp index bd912c6eb8..0afbcc349c 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp @@ -17,10 +17,14 @@ #include "olap/rowset/segment_v2/indexed_column_writer.h" +#include + +#include #include #include "common/logging.h" #include "olap/key_coder.h" +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/encoding_info.h" #include "olap/rowset/segment_v2/index_page.h" #include "olap/rowset/segment_v2/options.h" @@ -29,7 +33,7 @@ #include "olap/rowset/segment_v2/page_pointer.h" #include "olap/types.h" #include "util/block_compression.h" -#include "util/coding.h" +#include "util/slice.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/indexed_column_writer.h b/be/src/olap/rowset/segment_v2/indexed_column_writer.h index e4aadb7c3f..ab6812b5e5 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_writer.h +++ b/be/src/olap/rowset/segment_v2/indexed_column_writer.h @@ -17,16 +17,17 @@ #pragma once +#include +#include +#include + #include #include -#include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "gutil/macros.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/page_pointer.h" -#include "util/slice.h" +#include "util/faststring.h" #include "vec/common/arena.h" namespace doris { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp index 00ae62c378..1c6a61003b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp @@ -17,9 +17,23 @@ #include "olap/rowset/segment_v2/inverted_index_cache.h" +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include + +#include "common/logging.h" +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/inverted_index_compound_directory.h" #include "olap/rowset/segment_v2/inverted_index_compound_reader.h" +#include "runtime/thread_context.h" #include "util/defer_op.h" +#include "util/runtime_profile.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.h b/be/src/olap/rowset/segment_v2/inverted_index_cache.h index 77d428aa42..3c225bb9d9 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_cache.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.h @@ -17,21 +17,36 @@ #pragma once -#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include -#include -#include +#include #include -#include #include -#include +#include +#include +#include "common/config.h" +#include "common/status.h" #include "io/fs/file_system.h" +#include "io/fs/path.h" #include "olap/lru_cache.h" #include "runtime/memory/mem_tracker.h" +#include "util/slice.h" #include "util/time.h" +namespace lucene { +namespace search { +class IndexSearcher; +} // namespace search +} // namespace lucene + namespace doris { +struct OlapReaderStatistics; namespace segment_v2 { using IndexSearcherPtr = std::shared_ptr; @@ -165,7 +180,6 @@ private: }; enum class InvertedIndexQueryType; - class InvertedIndexQueryCacheHandle; class InvertedIndexQueryCache { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp index c3b474dfe1..969623c780 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp @@ -18,10 +18,11 @@ #include "olap/rowset/segment_v2/inverted_index_compound_directory.h" #include "CLucene/SharedHeader.h" -#include "CLucene/StdHeader.h" +#include "common/status.h" +#include "io/fs/file_reader.h" #include "io/fs/file_writer.h" -#include "olap/iterators.h" -#include "util/md5.h" +#include "io/fs/path.h" +#include "util/slice.h" #ifdef _CL_HAVE_IO_H #include @@ -35,12 +36,29 @@ #ifdef _CL_HAVE_DIRECT_H #include #endif +#include +#include +#include +#include #include #include #include +#include #include #include -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include #define CL_MAX_PATH 4096 #define CL_MAX_DIR CL_MAX_PATH diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h index b87498a34e..b06f980f6b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h @@ -17,17 +17,28 @@ #pragma once -#include +#include // IWYU pragma: keep +#include +#include +#include +#include -#include -#include -#include -#include +#include #include +#include "CLucene/SharedHeader.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" #include "util/lock.h" +class CLuceneError; + +namespace lucene { +namespace store { +class LockFactory; +} // namespace store +} // namespace lucene + namespace doris { namespace segment_v2 { @@ -76,6 +87,7 @@ protected: public: class FSIndexOutput; class FSIndexInput; + friend class DorisCompoundDirectory::FSIndexOutput; friend class DorisCompoundDirectory::FSIndexInput; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp index 9ce20703b4..15646679cf 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp @@ -17,8 +17,28 @@ #include "olap/rowset/segment_v2/inverted_index_compound_reader.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "CLucene/SharedHeader.h" #include "olap/rowset/segment_v2/inverted_index_compound_directory.h" +namespace doris { +namespace io { +class FileWriter; +} // namespace io +} // namespace doris + #define BUFFER_LENGTH 16384 #define CL_MAX_PATH 4096 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h index d6e95d9d50..5c0d963247 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h @@ -17,14 +17,28 @@ #pragma once -#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include +#include -#include #include -#include #include +#include #include +class CLuceneError; + +namespace lucene { +namespace store { +class RAMDirectory; +} // namespace store +} // namespace lucene + namespace doris { namespace segment_v2 { @@ -32,6 +46,7 @@ namespace segment_v2 { class CLUCENE_EXPORT DorisCompoundReader : public lucene::store::Directory { private: class ReaderFileEntry; + friend class DorisCompoundReader::ReaderFileEntry; int32_t readBufferSize; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp index 170409cac4..4bdf44ead0 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp @@ -17,7 +17,10 @@ #include "olap/rowset/segment_v2/inverted_index_desc.h" +#include + #include "gutil/strings/strip.h" +#include "olap/olap_common.h" namespace doris::segment_v2 { const std::string segment_suffix = ".dat"; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.h b/be/src/olap/rowset/segment_v2/inverted_index_desc.h index 0da5dfddbb..1cf4636d80 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.h @@ -17,12 +17,13 @@ #pragma once -#include +#include + #include -#include "olap/olap_common.h" - namespace doris { +struct RowsetId; + namespace segment_v2 { class InvertedIndexDescriptor { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 9a7520dc57..6e94640255 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -17,22 +17,44 @@ #include "olap/rowset/segment_v2/inverted_index_reader.h" +#include +#include #include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include +#include +#include +#include + +#include +#include +#include +#include +#include #include "common/config.h" -#include "gutil/strings/strip.h" +#include "common/logging.h" #include "io/fs/file_system.h" #include "olap/key_coder.h" +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/inverted_index_cache.h" #include "olap/rowset/segment_v2/inverted_index_compound_directory.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" -#include "olap/tablet_schema.h" -#include "olap/utils.h" +#include "olap/types.h" #include "util/faststring.h" +#include "util/runtime_profile.h" #include "util/time.h" #include "vec/common/string_ref.h" diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h index 731b6ca6b2..9438d5a07d 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h @@ -17,30 +17,44 @@ #pragma once -#include -#include #include +#include -#include +#include +#include +#include +#include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "gutil/macros.h" #include "io/fs/file_system.h" +#include "io/fs/path.h" #include "olap/inverted_index_parser.h" -#include "olap/olap_common.h" -#include "olap/rowset/segment_v2/common.h" -#include "olap/rowset/segment_v2/inverted_index_cache.h" #include "olap/rowset/segment_v2/inverted_index_compound_reader.h" #include "olap/tablet_schema.h" +namespace lucene { +namespace store { +class Directory; +} // namespace store +namespace util { +namespace bkd { +class bkd_docid_set_iterator; +} // namespace bkd +} // namespace util +} // namespace lucene +namespace roaring { +class Roaring; +} // namespace roaring + namespace doris { class KeyCoder; class TypeInfo; +struct OlapReaderStatistics; namespace segment_v2 { class InvertedIndexIterator; +class InvertedIndexQueryCacheHandle; enum class InvertedIndexReaderType { UNKNOWN = -1, diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index ae95408648..7cfc1807c0 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -17,20 +17,33 @@ #include "olap/rowset/segment_v2/inverted_index_writer.h" -#include +#include // IWYU pragma: keep #include #include +#include +#include +#include +#include #include +#include +#include +#include +#include "common/config.h" #include "olap/field.h" +#include "olap/inverted_index_parser.h" +#include "olap/key_coder.h" #include "olap/olap_common.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/inverted_index_cache.h" #include "olap/rowset/segment_v2/inverted_index_compound_directory.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" #include "olap/tablet_schema.h" +#include "olap/types.h" +#include "runtime/collection_value.h" #include "util/faststring.h" +#include "util/slice.h" #include "util/string_util.h" #define FINALIZE_OUTPUT(x) \ @@ -531,4 +544,4 @@ Status InvertedIndexColumnWriter::create(const Field* field, } return Status::OK(); } -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.h b/be/src/olap/rowset/segment_v2/inverted_index_writer.h index 16678e6c93..a92ec6220b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.h @@ -17,14 +17,20 @@ #pragma once +#include +#include +#include + +#include +#include + #include "common/status.h" #include "io/fs/file_system.h" -#include "olap/inverted_index_parser.h" -#include "olap/olap_common.h" -#include "olap/tablet_schema.h" namespace doris { class CollectionValue; +class Field; +class TabletIndex; namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/ngram_bloom_filter.cpp b/be/src/olap/rowset/segment_v2/ngram_bloom_filter.cpp index a761c5a6e4..54f49b8bb6 100644 --- a/be/src/olap/rowset/segment_v2/ngram_bloom_filter.cpp +++ b/be/src/olap/rowset/segment_v2/ngram_bloom_filter.cpp @@ -17,8 +17,11 @@ #include "olap/rowset/segment_v2/ngram_bloom_filter.h" +#include +#include + +#include "gutil/strings/substitute.h" #include "util/cityhash102/city.h" -#include "util/debug_util.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/ngram_bloom_filter.h b/be/src/olap/rowset/segment_v2/ngram_bloom_filter.h index c7684a796f..b69e467831 100644 --- a/be/src/olap/rowset/segment_v2/ngram_bloom_filter.h +++ b/be/src/olap/rowset/segment_v2/ngram_bloom_filter.h @@ -17,10 +17,17 @@ #pragma once +#include +#include + +#include + +#include "common/status.h" #include "olap/rowset/segment_v2/bloom_filter.h" namespace doris { namespace segment_v2 { +enum HashStrategyPB : int; class NGramBloomFilter : public BloomFilter { public: diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp index 5a077ae80a..d977e29f11 100644 --- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp +++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp @@ -17,12 +17,19 @@ #include "olap/rowset/segment_v2/ordinal_page_index.h" -#include "common/logging.h" +#include + +#include +#include +#include +#include + #include "io/fs/file_writer.h" -#include "io/fs/local_file_system.h" #include "olap/key_coder.h" +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/page_handle.h" #include "olap/rowset/segment_v2/page_io.h" +#include "util/slice.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.h b/be/src/olap/rowset/segment_v2/ordinal_page_index.h index d399b8db23..17dba9e675 100644 --- a/be/src/olap/rowset/segment_v2/ordinal_page_index.h +++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.h @@ -17,18 +17,19 @@ #pragma once +#include +#include + #include #include -#include +#include +#include #include "common/status.h" -#include "gutil/macros.h" -#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/index_page.h" #include "olap/rowset/segment_v2/page_pointer.h" -#include "util/coding.h" -#include "util/slice.h" namespace doris { @@ -37,6 +38,8 @@ class FileWriter; } namespace segment_v2 { +class ColumnIndexMetaPB; +class OrdinalIndexPB; // Ordinal index is implemented by one IndexPage that stores the first value ordinal // and file pointer for each data page. diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp b/be/src/olap/rowset/segment_v2/page_io.cpp index 7eb3dd0549..ee8b761053 100644 --- a/be/src/olap/rowset/segment_v2/page_io.cpp +++ b/be/src/olap/rowset/segment_v2/page_io.cpp @@ -17,13 +17,24 @@ #include "olap/rowset/segment_v2/page_io.h" +#include +#include + +#include #include +#include +#include #include +#include #include "common/logging.h" #include "gutil/strings/substitute.h" +#include "io/fs/file_reader.h" #include "io/fs/file_writer.h" +#include "olap/olap_common.h" #include "olap/page_cache.h" +#include "olap/rowset/segment_v2/encoding_info.h" +#include "olap/rowset/segment_v2/page_handle.h" #include "util/block_compression.h" #include "util/coding.h" #include "util/crc32c.h" diff --git a/be/src/olap/rowset/segment_v2/page_io.h b/be/src/olap/rowset/segment_v2/page_io.h index 06e345c27a..5ccccc2a3d 100644 --- a/be/src/olap/rowset/segment_v2/page_io.h +++ b/be/src/olap/rowset/segment_v2/page_io.h @@ -17,15 +17,13 @@ #pragma once +#include + #include #include "common/logging.h" #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "io/fs/file_reader.h" -#include "olap/iterators.h" -#include "olap/rowset/segment_v2/encoding_info.h" -#include "olap/rowset/segment_v2/page_handle.h" +#include "io/io_common.h" #include "olap/rowset/segment_v2/page_pointer.h" #include "util/slice.h" @@ -34,16 +32,14 @@ namespace doris { class BlockCompressionCodec; struct OlapReaderStatistics; -namespace fs { -class ReadableBlock; -class WritableBlock; -} // namespace fs - namespace io { class FileWriter; +class FileReader; } // namespace io namespace segment_v2 { +class EncodingInfo; +class PageHandle; struct PageReadOptions { // block to read page diff --git a/be/src/olap/rowset/segment_v2/page_pointer.h b/be/src/olap/rowset/segment_v2/page_pointer.h index adddbbebaf..4693523986 100644 --- a/be/src/olap/rowset/segment_v2/page_pointer.h +++ b/be/src/olap/rowset/segment_v2/page_pointer.h @@ -17,10 +17,11 @@ #pragma once +#include + #include #include -#include "gen_cpp/segment_v2.pb.h" #include "util/coding.h" #include "util/faststring.h" diff --git a/be/src/olap/rowset/segment_v2/parsed_page.h b/be/src/olap/rowset/segment_v2/parsed_page.h index fe21f11dc3..159b0cca72 100644 --- a/be/src/olap/rowset/segment_v2/parsed_page.h +++ b/be/src/olap/rowset/segment_v2/parsed_page.h @@ -17,10 +17,11 @@ #pragma once +#include + #include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" #include "olap/rowset/segment_v2/binary_dict_page.h" #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/encoding_info.h" diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 4a86e04a0f..cf952c2856 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -17,30 +17,53 @@ #include "olap/rowset/segment_v2/segment.h" +#include #include +#include +#include #include #include -#include "common/config.h" -#include "common/logging.h" // LOG -#include "io/cache/file_cache_manager.h" -#include "io/fs/file_reader_options.h" +#include "io/fs/file_reader.h" #include "io/fs/file_system.h" +#include "io/io_common.h" +#include "olap/block_column_predicate.h" +#include "olap/column_predicate.h" #include "olap/iterators.h" +#include "olap/primary_key_index.h" #include "olap/rowset/segment_v2/empty_segment_iterator.h" +#include "olap/rowset/segment_v2/indexed_column_reader.h" #include "olap/rowset/segment_v2/page_io.h" +#include "olap/rowset/segment_v2/page_pointer.h" #include "olap/rowset/segment_v2/segment_iterator.h" #include "olap/rowset/segment_v2/segment_writer.h" // k_segment_magic_length +#include "olap/short_key_index.h" #include "olap/storage_engine.h" #include "olap/tablet_schema.h" +#include "olap/types.h" +#include "olap/utils.h" +#include "runtime/memory/mem_tracker.h" +#include "runtime/query_fragments_ctx.h" +#include "runtime/runtime_predicate.h" +#include "runtime/runtime_state.h" +#include "util/coding.h" #include "util/crc32c.h" #include "util/slice.h" // Slice +#include "vec/columns/column.h" +#include "vec/common/string_ref.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_factory.hpp" #include "vec/olap/vgeneric_iterators.h" namespace doris { +namespace io { +class FileCacheManager; +class FileReaderOptions; +} // namespace io + namespace segment_v2 { +class InvertedIndexIterator; using io::FileCacheManager; diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index c8414566fc..bfb09eb1fd 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -17,38 +17,47 @@ #pragma once +#include +#include +#include +#include + #include +#include #include // for unique_ptr #include -#include +#include #include "common/status.h" // Status -#include "gen_cpp/segment_v2.pb.h" -#include "gutil/macros.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_system.h" -#include "olap/iterators.h" -#include "olap/primary_key_index.h" +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/column_reader.h" // ColumnReader #include "olap/rowset/segment_v2/page_handle.h" -#include "olap/short_key_index.h" #include "olap/tablet_schema.h" -#include "util/faststring.h" #include "util/once.h" +#include "util/slice.h" namespace doris { -class TabletSchema; class ShortKeyIndexDecoder; class Schema; class StorageReadOptions; +class MemTracker; +class PrimaryKeyIndexReader; +class RowwiseIterator; + +namespace io { +class FileReaderOptions; +} // namespace io +struct RowLocation; namespace segment_v2 { class BitmapIndexIterator; -class ColumnReader; -class ColumnIterator; class Segment; -class SegmentIterator; +class InvertedIndexIterator; + using SegmentSharedPtr = std::shared_ptr; // A Segment is used to represent a segment in memory format. When segment is // generated, it won't be modified, so this struct aimed to help read operation. diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 5d3c375f54..a1c55855bf 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -17,25 +17,63 @@ #include "olap/rowset/segment_v2/segment_iterator.h" +#include +#include +#include + +#include +#include #include +#include #include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" #include "common/consts.h" +#include "common/logging.h" +#include "common/object_pool.h" #include "common/status.h" +#include "io/io_common.h" #include "olap/column_predicate.h" #include "olap/like_column_predicate.h" #include "olap/olap_common.h" +#include "olap/primary_key_index.h" +#include "olap/rowset/segment_v2/bitmap_index_reader.h" #include "olap/rowset/segment_v2/column_reader.h" +#include "olap/rowset/segment_v2/indexed_column_reader.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" +#include "olap/rowset/segment_v2/row_ranges.h" #include "olap/rowset/segment_v2/segment.h" #include "olap/short_key_index.h" +#include "olap/tablet_schema.h" +#include "olap/types.h" +#include "olap/utils.h" +#include "runtime/query_fragments_ctx.h" +#include "runtime/runtime_predicate.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "util/defer_op.h" #include "util/doris_metrics.h" #include "util/key_util.h" #include "util/simd/bits.h" #include "vec/columns/column.h" #include "vec/columns/column_const.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/column_string.h" +#include "vec/columns/column_vector.h" +#include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" +#include "vec/common/string_ref.h" +#include "vec/common/typeid_cast.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/core/field.h" +#include "vec/core/types.h" #include "vec/data_types/data_type_factory.hpp" #include "vec/data_types/data_type_number.h" +#include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" #include "vec/exprs/vliteral.h" #include "vec/exprs/vslot_ref.h" diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index 9650ad9e27..5e3485eee0 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -17,36 +17,54 @@ #pragma once +#include +#include +#include + +#include #include +#include #include +#include +#include +#include +#include #include #include "common/status.h" -#include "io/fs/file_reader.h" -#include "io/fs/file_system.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "olap/block_column_predicate.h" +#include "olap/column_predicate.h" +#include "olap/field.h" +#include "olap/iterators.h" #include "olap/olap_common.h" #include "olap/row_cursor.h" +#include "olap/row_cursor_cell.h" #include "olap/rowset/segment_v2/common.h" -#include "olap/rowset/segment_v2/inverted_index_reader.h" -#include "olap/rowset/segment_v2/row_ranges.h" #include "olap/rowset/segment_v2/segment.h" #include "olap/schema.h" -#include "vec/exprs/vexpr.h" +#include "util/runtime_profile.h" +#include "util/slice.h" +#include "vec/columns/column.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type.h" namespace doris { -class RowCursor; -class ShortKeyIndexIterator; +class ObjectPool; -namespace fs { -class ReadableBlock; -} +namespace vectorized { +class VExpr; +class VExprContext; +} // namespace vectorized +struct RowLocation; namespace segment_v2 { class BitmapIndexIterator; -class BitmapIndexReader; class ColumnIterator; +class InvertedIndexIterator; +class RowRanges; struct ColumnPredicateInfo { ColumnPredicateInfo() = default; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 3363f71cde..9f772e63e3 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -17,24 +17,40 @@ #include "olap/rowset/segment_v2/segment_writer.h" +#include +#include +#include + +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" -#include "common/consts.h" #include "common/logging.h" // LOG +#include "gutil/port.h" #include "io/fs/file_writer.h" #include "olap/data_dir.h" +#include "olap/key_coder.h" +#include "olap/olap_common.h" #include "olap/primary_key_index.h" -#include "olap/row_cursor.h" // RowCursor +#include "olap/row_cursor.h" // IWYU pragma: keep #include "olap/rowset/rowset_writer_context.h" // RowsetWriterContext #include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter #include "olap/rowset/segment_v2/page_io.h" -#include "olap/schema.h" +#include "olap/rowset/segment_v2/page_pointer.h" #include "olap/short_key_index.h" #include "runtime/memory/mem_tracker.h" #include "service/point_query_executor.h" +#include "util/coding.h" #include "util/crc32c.h" #include "util/faststring.h" #include "util/key_util.h" #include "vec/common/schema_util.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/olap/olap_data_convertor.h" namespace doris { namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h index 8ffd4eaa5b..c907a357df 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.h +++ b/be/src/olap/rowset/segment_v2/segment_writer.h @@ -17,29 +17,38 @@ #pragma once +#include +#include +#include +#include + #include +#include #include // unique_ptr #include +#include #include #include "common/status.h" // Status -#include "gen_cpp/segment_v2.pb.h" -#include "gutil/macros.h" +#include "gutil/strings/substitute.h" +#include "olap/olap_define.h" +#include "olap/rowset/segment_v2/column_writer.h" #include "olap/tablet_schema.h" #include "util/faststring.h" -#include "vec/core/block.h" -#include "vec/olap/olap_data_convertor.h" +#include "util/slice.h" namespace doris { +namespace vectorized { +class Block; +class IOlapColumnDataAccessor; +class OlapBlockDataConvertor; +} // namespace vectorized // TODO(lingbin): Should be a conf that can be dynamically adjusted, or a member in the context const uint32_t MAX_SEGMENT_SIZE = static_cast(OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE * OLAP_COLUMN_FILE_SEGMENT_SIZE_SCALE); class DataDir; class MemTracker; -class RowCursor; -class TabletSchema; -class TabletColumn; class ShortKeyIndexBuilder; class PrimaryKeyIndexBuilder; class KeyCoder; @@ -51,8 +60,6 @@ class FileWriter; namespace segment_v2 { -class ColumnWriter; - extern const char* k_segment_magic; extern const uint32_t k_segment_magic_length; diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp index 5b63bcda03..9a4ec628c2 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp +++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp @@ -17,12 +17,27 @@ #include "olap/rowset/segment_v2/zone_map_index.h" +#include +#include + +#include +#include + +#include "olap/olap_common.h" #include "olap/rowset/segment_v2/encoding_info.h" #include "olap/rowset/segment_v2/indexed_column_reader.h" #include "olap/rowset/segment_v2/indexed_column_writer.h" #include "olap/types.h" +#include "runtime/primitive_type.h" +#include "util/slice.h" +#include "vec/columns/column.h" +#include "vec/columns/column_string.h" +#include "vec/common/string_ref.h" +#include "vec/common/unaligned.h" +#include "vec/data_types/data_type.h" namespace doris { +struct uint24_t; namespace segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h b/be/src/olap/rowset/segment_v2/zone_map_index.h index 073ea2da31..ed3418dc48 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.h +++ b/be/src/olap/rowset/segment_v2/zone_map_index.h @@ -17,16 +17,19 @@ #pragma once +#include +#include +#include + #include #include +#include #include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" #include "olap/field.h" -#include "olap/rowset/segment_v2/binary_plain_page.h" -#include "util/slice.h" +#include "runtime/define_primitive_type.h" #include "vec/common/arena.h" namespace doris { diff --git a/be/src/olap/rowset/unique_rowset_id_generator.cpp b/be/src/olap/rowset/unique_rowset_id_generator.cpp index 9db7faf35e..0d7d77915e 100644 --- a/be/src/olap/rowset/unique_rowset_id_generator.cpp +++ b/be/src/olap/rowset/unique_rowset_id_generator.cpp @@ -17,9 +17,12 @@ #include "olap/rowset/unique_rowset_id_generator.h" +#include +#include + #include "util/doris_metrics.h" +#include "util/metrics.h" #include "util/spinlock.h" -#include "util/stack_util.h" #include "util/uid_util.h" namespace doris { diff --git a/be/src/olap/rowset/unique_rowset_id_generator.h b/be/src/olap/rowset/unique_rowset_id_generator.h index ef8d4cfde2..051eefd245 100644 --- a/be/src/olap/rowset/unique_rowset_id_generator.h +++ b/be/src/olap/rowset/unique_rowset_id_generator.h @@ -17,8 +17,12 @@ #pragma once -#include +#include +#include +#include + +#include "olap/olap_common.h" #include "olap/rowset/rowset_id_generator.h" #include "util/spinlock.h" #include "util/uid_util.h" diff --git a/be/src/olap/rowset/vertical_beta_rowset_writer.cpp b/be/src/olap/rowset/vertical_beta_rowset_writer.cpp index e1e3ab46de..6525407a91 100644 --- a/be/src/olap/rowset/vertical_beta_rowset_writer.cpp +++ b/be/src/olap/rowset/vertical_beta_rowset_writer.cpp @@ -17,7 +17,28 @@ #include "olap/rowset/vertical_beta_rowset_writer.h" +#include + +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/logging.h" +#include "gutil/strings/substitute.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "olap/rowset/beta_rowset.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/rowset/rowset_writer_context.h" +#include "util/slice.h" +#include "util/spinlock.h" +#include "vec/core/block.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/olap/rowset/vertical_beta_rowset_writer.h b/be/src/olap/rowset/vertical_beta_rowset_writer.h index 56e2fb4d6a..71b132b9b4 100644 --- a/be/src/olap/rowset/vertical_beta_rowset_writer.h +++ b/be/src/olap/rowset/vertical_beta_rowset_writer.h @@ -17,10 +17,20 @@ #pragma once +#include +#include + +#include +#include + +#include "common/status.h" #include "olap/rowset/beta_rowset_writer.h" #include "olap/rowset/segment_v2/segment_writer.h" namespace doris { +namespace vectorized { +class Block; +} // namespace vectorized // for vertical compaction class VerticalBetaRowsetWriter : public BetaRowsetWriter { diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp index ab75f1f93a..373773fffe 100644 --- a/be/src/olap/schema.cpp +++ b/be/src/olap/schema.cpp @@ -17,12 +17,24 @@ #include "olap/schema.h" -#include "olap/uint24.h" -#include "vec/columns/column_complex.h" +#include + +#include +#include +#include +#include + +#include "common/config.h" +#include "runtime/define_primitive_type.h" +#include "vec/columns/column_array.h" #include "vec/columns/column_dictionary.h" +#include "vec/columns/column_map.h" #include "vec/columns/column_nullable.h" +#include "vec/columns/column_struct.h" +#include "vec/columns/columns_number.h" #include "vec/columns/predicate_column.h" #include "vec/core/types.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_factory.hpp" namespace doris { diff --git a/be/src/olap/schema.h b/be/src/olap/schema.h index 4d9594b344..b592563bd0 100644 --- a/be/src/olap/schema.h +++ b/be/src/olap/schema.h @@ -17,14 +17,22 @@ #pragma once +#include +#include + +#include +#include +#include #include #include "common/consts.h" +#include "io/io_common.h" #include "olap/field.h" -#include "olap/row_cursor_cell.h" +#include "olap/olap_common.h" #include "olap/tablet_schema.h" -#include "olap/types.h" -#include "runtime/descriptors.h" +#include "olap/utils.h" +#include "vec/aggregate_functions/aggregate_function.h" +#include "vec/columns/column.h" namespace doris { diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 2089b7c62d..9742b4d095 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -17,37 +17,66 @@ #include "olap/schema_change.h" +#include +#include #include -#include "common/config.h" +#include +#include +#include +#include +#include +#include + +#include "common/logging.h" #include "common/status.h" +#include "gutil/hash/hash.h" #include "gutil/integral_types.h" +#include "gutil/strings/numbers.h" +#include "io/fs/file_system.h" +#include "io/io_common.h" +#include "olap/data_dir.h" +#include "olap/delete_handler.h" +#include "olap/field.h" +#include "olap/iterators.h" #include "olap/merger.h" #include "olap/olap_common.h" -#include "olap/row_cursor.h" +#include "olap/olap_define.h" #include "olap/rowset/beta_rowset.h" -#include "olap/rowset/rowset_id_generator.h" +#include "olap/rowset/rowset_meta.h" +#include "olap/rowset/rowset_reader_context.h" +#include "olap/rowset/rowset_writer_context.h" #include "olap/rowset/segment_v2/column_reader.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" #include "olap/rowset/segment_v2/inverted_index_writer.h" +#include "olap/rowset/segment_v2/segment.h" +#include "olap/schema.h" #include "olap/segment_loader.h" #include "olap/storage_engine.h" #include "olap/tablet.h" +#include "olap/tablet_manager.h" +#include "olap/tablet_meta.h" #include "olap/tablet_schema.h" #include "olap/types.h" #include "olap/utils.h" #include "olap/wrapper_field.h" #include "runtime/memory/mem_tracker.h" +#include "runtime/runtime_state.h" #include "util/defer_op.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/aggregate_functions/aggregate_function_reader.h" -#include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/columns/column.h" +#include "vec/columns/column_nullable.h" +#include "vec/common/assert_cast.h" #include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" #include "vec/exprs/vexpr.h" #include "vec/exprs/vexpr_context.h" +#include "vec/olap/olap_data_convertor.h" namespace doris { +class CollectionValue; + using namespace ErrorCode; constexpr int ALTER_TABLE_BATCH_SIZE = 4096; diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h index 16226dfead..b74113c8c7 100644 --- a/be/src/olap/schema_change.h +++ b/be/src/olap/schema_change.h @@ -17,24 +17,52 @@ #pragma once -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/object_pool.h" #include "common/status.h" -#include "gen_cpp/AgentService_types.h" #include "olap/column_mapping.h" -#include "olap/delete_handler.h" +#include "olap/olap_common.h" #include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_reader.h" #include "olap/rowset/rowset_writer.h" +#include "olap/rowset/segment_v2/inverted_index_writer.h" #include "olap/tablet.h" -#include "vec/columns/column.h" -#include "vec/core/block.h" -#include "vec/olap/olap_data_convertor.h" +#include "olap/tablet_schema.h" +#include "runtime/descriptors.h" +#include "runtime/memory/mem_tracker.h" +#include "vec/data_types/data_type.h" namespace doris { +class DeleteHandler; +class Field; +class TAlterInvertedIndexReq; +class TAlterTabletReqV2; +class TExpr; +enum AlterTabletType : int; +enum RowsetTypePB : int; +enum SegmentsOverlapPB : int; -namespace segment_v2 { -class InvertedIndexColumnWriter; -} +namespace vectorized { +class Block; +class OlapBlockDataConvertor; +} // namespace vectorized class BlockChanger { public: diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp index fa1e6c8751..eaf6651fc5 100644 --- a/be/src/olap/segment_loader.cpp +++ b/be/src/olap/segment_loader.cpp @@ -17,8 +17,9 @@ #include "olap/segment_loader.h" -#include "olap/rowset/rowset.h" -#include "olap/tablet_schema.h" +#include "common/config.h" +#include "olap/olap_define.h" +#include "olap/rowset/beta_rowset.h" #include "util/stopwatch.hpp" namespace doris { diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h index 03a54acf55..13da58a2e5 100644 --- a/be/src/olap/segment_loader.h +++ b/be/src/olap/segment_loader.h @@ -17,20 +17,28 @@ #pragma once +#include +#include +#include +#include + +#include #include +#include #include #include +#include -#include "gutil/macros.h" // for DISALLOW_COPY_AND_ASSIGN +#include "common/status.h" #include "olap/lru_cache.h" #include "olap/olap_common.h" // for rowset id -#include "olap/rowset/beta_rowset.h" -#include "olap/tablet_schema.h" +#include "olap/rowset/segment_v2/segment.h" #include "util/time.h" namespace doris { class SegmentCacheHandle; +class BetaRowset; // SegmentLoader is used to load the Segment of BetaRowset. // An LRUCache is encapsulated inside it, which is used to cache the opened segments. diff --git a/be/src/olap/short_key_index.cpp b/be/src/olap/short_key_index.cpp index 19055b3b12..69622cee45 100644 --- a/be/src/olap/short_key_index.cpp +++ b/be/src/olap/short_key_index.cpp @@ -17,7 +17,9 @@ #include "olap/short_key_index.h" -#include +#include + +#include #include "gutil/strings/substitute.h" #include "util/coding.h" diff --git a/be/src/olap/short_key_index.h b/be/src/olap/short_key_index.h index a217a928aa..a5ed018f72 100644 --- a/be/src/olap/short_key_index.h +++ b/be/src/olap/short_key_index.h @@ -17,14 +17,16 @@ #pragma once +#include +#include +#include + +#include #include #include -#include #include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" -#include "util/debug_util.h" #include "util/faststring.h" #include "util/slice.h" diff --git a/be/src/olap/skiplist.h b/be/src/olap/skiplist.h index 5c4a2b40d6..477a3e1e12 100644 --- a/be/src/olap/skiplist.h +++ b/be/src/olap/skiplist.h @@ -26,10 +26,11 @@ // // ... prev vs. next pointer ordering ... +#include + #include #include "common/logging.h" -#include "gen_cpp/olap_file.pb.h" #include "util/random.h" #include "vec/common/arena.h" diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index 687f76a0ec..cf498c0b7b 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -17,27 +17,44 @@ #include "olap/snapshot_manager.h" -#include -#include -#include +#include +#include +#include +#include #include #include +#include #include -#include +#include #include +#include +#include #include +#include +#include +#include +#include "common/config.h" +#include "common/logging.h" #include "common/status.h" -#include "gen_cpp/Types_constants.h" #include "io/fs/local_file_system.h" +#include "olap/data_dir.h" +#include "olap/olap_common.h" +#include "olap/olap_define.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_factory.h" +#include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_writer.h" +#include "olap/rowset/rowset_writer_context.h" #include "olap/storage_engine.h" +#include "olap/tablet_manager.h" #include "olap/tablet_meta.h" #include "olap/tablet_schema.h" +#include "olap/tablet_schema_cache.h" +#include "olap/utils.h" #include "runtime/thread_context.h" +#include "util/uid_util.h" using std::filesystem::path; using std::map; diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h index 982bf36375..3536cebd1a 100644 --- a/be/src/olap/snapshot_manager.h +++ b/be/src/olap/snapshot_manager.h @@ -17,27 +17,23 @@ #pragma once -#include -#include -#include -#include +#include + +#include #include -#include #include -#include #include #include "common/status.h" -#include "olap/data_dir.h" -#include "olap/field.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/push_handler.h" +#include "olap/rowset/rowset.h" #include "olap/tablet.h" -#include "olap/tablet_meta_manager.h" -#include "util/doris_metrics.h" +#include "olap/tablet_schema.h" +#include "runtime/memory/mem_tracker.h" namespace doris { +class RowsetMetaPB; +class TSnapshotRequest; +struct RowsetId; class SnapshotManager { public: diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 154145d823..722220d301 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -17,44 +17,61 @@ #include "olap/storage_engine.h" +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include #include -#include -#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include +#include +#include #include +#include +#include #include -#include +#include #include #include +#include +#include #include "agent/task_worker_pool.h" +#include "common/config.h" +#include "common/logging.h" +#include "gutil/strings/substitute.h" #include "io/fs/local_file_system.h" #include "olap/base_compaction.h" #include "olap/cumulative_compaction.h" #include "olap/data_dir.h" -#include "olap/lru_cache.h" #include "olap/memtable_flush_executor.h" -#include "olap/push_handler.h" -#include "olap/reader.h" +#include "olap/olap_define.h" +#include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_meta_manager.h" #include "olap/rowset/unique_rowset_id_generator.h" -#include "olap/schema_change.h" #include "olap/segment_loader.h" +#include "olap/tablet_manager.h" #include "olap/tablet_meta.h" -#include "olap/tablet_meta_manager.h" -#include "olap/utils.h" +#include "olap/task/engine_task.h" +#include "olap/txn_manager.h" +#include "runtime/memory/mem_tracker.h" +#include "runtime/stream_load/stream_load_recorder.h" #include "util/doris_metrics.h" -#include "util/pretty_printer.h" -#include "util/scoped_cleanup.h" -#include "util/time.h" +#include "util/metrics.h" +#include "util/priority_thread_pool.hpp" +#include "util/spinlock.h" +#include "util/stopwatch.hpp" +#include "util/thread.h" +#include "util/threadpool.h" #include "util/trace.h" +#include "util/uid_util.h" using apache::thrift::ThriftDebugString; using std::filesystem::canonical; @@ -69,7 +86,6 @@ using std::map; using std::nothrow; using std::pair; using std::set; -using std::set_difference; using std::string; using std::stringstream; using std::vector; diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index db113f1d93..481353ec0d 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -17,48 +17,54 @@ #pragma once -#include -#include +#include +#include +#include +#include +#include #include #include -#include #include +#include #include #include #include -#include +#include +#include #include #include "common/status.h" -#include "gen_cpp/AgentService_types.h" -#include "gen_cpp/BackendService_types.h" -#include "gen_cpp/MasterService_types.h" #include "gutil/ref_counted.h" #include "olap/compaction_permit_limiter.h" #include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/olap_meta.h" #include "olap/options.h" +#include "olap/rowset/rowset.h" #include "olap/rowset/rowset_id_generator.h" -#include "olap/tablet_manager.h" -#include "olap/task/engine_task.h" -#include "olap/txn_manager.h" +#include "olap/rowset/segment_v2/segment.h" +#include "olap/tablet.h" #include "runtime/heartbeat_flags.h" -#include "runtime/stream_load/stream_load_recorder.h" #include "util/countdown_latch.h" -#include "util/thread.h" -#include "util/threadpool.h" namespace doris { class DataDir; class EngineTask; -class BlockManager; class MemTableFlushExecutor; -class Tablet; class TaskWorkerPool; class BetaRowsetWriter; +class BaseCompaction; +class CumulativeCompaction; +class CumulativeCompactionPolicy; +class MemTracker; +class PriorityThreadPool; +class StreamLoadRecorder; +class TCloneReq; +class TCreateTabletReq; +class TabletManager; +class Thread; +class ThreadPool; +class TxnManager; using SegCompactionCandidates = std::vector; using SegCompactionCandidatesSharedPtr = std::shared_ptr; diff --git a/be/src/olap/storage_policy.cpp b/be/src/olap/storage_policy.cpp index 68f561397d..494c93cec1 100644 --- a/be/src/olap/storage_policy.cpp +++ b/be/src/olap/storage_policy.cpp @@ -17,6 +17,10 @@ #include "olap/storage_policy.h" +#include + +#include +#include #include #include diff --git a/be/src/olap/storage_policy.h b/be/src/olap/storage_policy.h index 04f05540ae..0a566d1002 100644 --- a/be/src/olap/storage_policy.h +++ b/be/src/olap/storage_policy.h @@ -17,6 +17,15 @@ #pragma once +#include +#include + +#include +#include +#include +#include + +#include "common/status.h" #include "io/fs/file_system.h" #include "io/fs/remote_file_system.h" diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 7603d6fbe7..96311c4c77 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -17,68 +17,113 @@ #include "olap/tablet.h" +#include +#include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include -#include -#include +#include +#include #include #include -#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include #include #include #include #include #include #include +#include +#include #include #include "agent/utils.h" #include "common/config.h" +#include "common/consts.h" #include "common/logging.h" #include "common/status.h" +#include "gutil/ref_counted.h" #include "gutil/strings/stringpiece.h" +#include "gutil/strings/substitute.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/file_system.h" +#include "io/fs/file_writer.h" #include "io/fs/path.h" #include "io/fs/remote_file_system.h" #include "olap/base_compaction.h" #include "olap/base_tablet.h" #include "olap/cumulative_compaction.h" +#include "olap/cumulative_compaction_policy.h" #include "olap/olap_common.h" #include "olap/olap_define.h" -#include "olap/reader.h" -#include "olap/row_cursor.h" +#include "olap/olap_meta.h" +#include "olap/primary_key_index.h" +#include "olap/rowid_conversion.h" +#include "olap/rowset/beta_rowset.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_factory.h" #include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_meta_manager.h" +#include "olap/rowset/rowset_writer.h" +#include "olap/rowset/rowset_writer_context.h" #include "olap/rowset/segment_v2/column_reader.h" +#include "olap/rowset/segment_v2/common.h" +#include "olap/rowset/segment_v2/indexed_column_reader.h" #include "olap/schema_change.h" #include "olap/storage_engine.h" #include "olap/storage_policy.h" +#include "olap/tablet_manager.h" #include "olap/tablet_meta.h" -#include "olap/tablet_meta_manager.h" #include "olap/tablet_schema.h" #include "olap/txn_manager.h" +#include "olap/types.h" +#include "olap/utils.h" #include "segment_loader.h" #include "service/point_query_executor.h" #include "util/defer_op.h" -#include "util/path_util.h" +#include "util/doris_metrics.h" #include "util/pretty_printer.h" #include "util/scoped_cleanup.h" +#include "util/stopwatch.hpp" +#include "util/threadpool.h" #include "util/time.h" #include "util/trace.h" #include "util/uid_util.h" +#include "vec/columns/column.h" +#include "vec/columns/column_string.h" +#include "vec/common/string_ref.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_factory.hpp" #include "vec/jsonb/serialize.h" namespace doris { +class TupleDescriptor; + +namespace vectorized { +class Block; +} // namespace vectorized + using namespace ErrorCode; using std::pair; diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index faa5a5c8e0..1fa6d1bc7f 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -17,48 +17,66 @@ #pragma once +#include +#include +#include +#include + #include #include +#include +#include #include #include +#include #include #include #include #include -#include +#include #include -#include "gen_cpp/AgentService_types.h" -#include "gen_cpp/MasterService_types.h" -#include "gen_cpp/olap_file.pb.h" -#include "io/fs/file_system.h" +#include "common/config.h" +#include "common/status.h" #include "olap/base_tablet.h" -#include "olap/cumulative_compaction_policy.h" #include "olap/data_dir.h" #include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/olap_tuple.h" #include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_reader.h" #include "olap/rowset/rowset_tree.h" #include "olap/rowset/segment_v2/segment.h" #include "olap/tablet_meta.h" -#include "olap/utils.h" +#include "olap/tablet_schema.h" #include "olap/version_graph.h" +#include "util/metrics.h" #include "util/once.h" +#include "util/slice.h" namespace doris { -class DataDir; class Tablet; -class TabletMeta; class CumulativeCompactionPolicy; class CumulativeCompaction; class BaseCompaction; class RowsetWriter; - struct TabletTxnInfo; struct RowsetWriterContext; +class RowIdConversion; +class TTabletInfo; +class TabletMetaPB; +class TupleDescriptor; +enum CompressKind : int; + +namespace io { +class RemoteFileSystem; +} // namespace io +namespace vectorized { +class Block; +} // namespace vectorized +struct RowLocation; +enum KeysType : int; +enum SortType : int; using TabletSharedPtr = std::shared_ptr; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 8ad1fe9d94..94986cdcfb 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -17,33 +17,55 @@ #include "olap/tablet_manager.h" +#include +#include +#include +#include +#include #include +#include #include +#include #include -#include -#include -#include +#include +#include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "common/logging.h" +#include "gutil/integral_types.h" #include "gutil/strings/strcat.h" -#include "olap/base_compaction.h" -#include "olap/cumulative_compaction.h" +#include "gutil/strings/substitute.h" +#include "io/fs/local_file_system.h" #include "olap/data_dir.h" #include "olap/olap_common.h" -#include "olap/push_handler.h" +#include "olap/olap_define.h" +#include "olap/rowset/rowset.h" +#include "olap/storage_engine.h" #include "olap/tablet.h" #include "olap/tablet_meta.h" #include "olap/tablet_meta_manager.h" -#include "rapidjson/prettywriter.h" -#include "rapidjson/stringbuffer.h" +#include "olap/tablet_schema.h" +#include "olap/txn_manager.h" +#include "runtime/exec_env.h" +#include "runtime/memory/mem_tracker.h" #include "runtime/thread_context.h" #include "service/backend_options.h" #include "util/doris_metrics.h" #include "util/histogram.h" +#include "util/metrics.h" #include "util/path_util.h" #include "util/scoped_cleanup.h" +#include "util/stopwatch.hpp" #include "util/time.h" #include "util/trace.h" +#include "util/uid_util.h" + +namespace doris { +class CumulativeCompactionPolicy; +} // namespace doris using std::list; using std::map; diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index dca1b635ad..0ddeb4aaac 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -17,30 +17,37 @@ #pragma once -#include -#include +#include +#include +#include +#include +#include + +#include #include +#include #include #include #include #include #include #include +#include #include #include "common/status.h" -#include "gen_cpp/AgentService_types.h" -#include "gen_cpp/BackendService_types.h" -#include "gen_cpp/MasterService_types.h" -#include "olap/olap_define.h" -#include "olap/olap_meta.h" -#include "olap/options.h" +#include "olap/olap_common.h" #include "olap/tablet.h" +#include "olap/tablet_meta.h" namespace doris { -class Tablet; class DataDir; +class CumulativeCompactionPolicy; +class MemTracker; +class TCreateTabletReq; +class TTablet; +class TTabletInfo; // TabletManager provides get, add, delete tablet method for storage engine // NOTE: If you want to add a method that needs to hold meta-lock before you can call it, diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index a318dc508b..f5548e8061 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -17,17 +17,29 @@ #include "olap/tablet_meta.h" -#include +#include +#include +#include +#include +#include +#include +#include +#include -#include "common/consts.h" +#include +#include + +#include "common/config.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_writer.h" +#include "olap/data_dir.h" #include "olap/file_header.h" #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/tablet_meta_manager.h" +#include "olap/utils.h" #include "util/string_util.h" #include "util/uid_util.h" -#include "util/url_coding.h" using std::string; using std::unordered_map; @@ -108,25 +120,25 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id // compression_type used to compress segment page switch (compression_type) { case TCompressionType::NO_COMPRESSION: - schema->set_compression_type(NO_COMPRESSION); + schema->set_compression_type(segment_v2::NO_COMPRESSION); break; case TCompressionType::SNAPPY: - schema->set_compression_type(SNAPPY); + schema->set_compression_type(segment_v2::SNAPPY); break; case TCompressionType::LZ4: - schema->set_compression_type(LZ4); + schema->set_compression_type(segment_v2::LZ4); break; case TCompressionType::LZ4F: - schema->set_compression_type(LZ4F); + schema->set_compression_type(segment_v2::LZ4F); break; case TCompressionType::ZLIB: - schema->set_compression_type(ZLIB); + schema->set_compression_type(segment_v2::ZLIB); break; case TCompressionType::ZSTD: - schema->set_compression_type(ZSTD); + schema->set_compression_type(segment_v2::ZSTD); break; default: - schema->set_compression_type(LZ4F); + schema->set_compression_type(segment_v2::LZ4F); break; } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 6410563bd3..d2870a8dfe 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -17,25 +17,39 @@ #pragma once +#include +#include +#include + #include #include +#include +#include #include +#include +#include #include #include +#include +#include #include #include "common/logging.h" -#include "gen_cpp/olap_file.pb.h" -#include "olap/delete_handler.h" +#include "common/status.h" +#include "gutil/stringprintf.h" +#include "io/fs/file_system.h" +#include "olap/lru_cache.h" #include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/rowid_conversion.h" -#include "olap/rowset/rowset.h" #include "olap/rowset/rowset_meta.h" #include "olap/tablet_schema.h" #include "util/uid_util.h" +namespace json2pb { +struct Pb2JsonOptions; +} // namespace json2pb + namespace doris { +class TColumn; // Lifecycle states that a Tablet can be in. Legal state transitions for a // Tablet object: @@ -64,11 +78,10 @@ enum TabletState { TABLET_SHUTDOWN }; -class RowsetMeta; -class Rowset; class DataDir; class TabletMeta; class DeleteBitmap; + using TabletMetaSharedPtr = std::shared_ptr; using DeleteBitmapPtr = std::shared_ptr; diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index 336cd1d592..ac4deacd14 100644 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -18,19 +18,32 @@ #include "olap/tablet_meta_manager.h" #include +#include +#include #include #include -#include +#include #include #include #include "common/logging.h" #include "json2pb/json_to_pb.h" #include "json2pb/pb_to_json.h" +#include "olap/data_dir.h" #include "olap/olap_define.h" #include "olap/olap_meta.h" -#include "olap/storage_engine.h" +#include "olap/utils.h" + +namespace rocksdb { +class Iterator; +class Slice; +class Status; +struct ColumnFamilyOptions; +struct DBOptions; +struct ReadOptions; +struct WriteOptions; +} // namespace rocksdb using rocksdb::DB; using rocksdb::DBOptions; diff --git a/be/src/olap/tablet_meta_manager.h b/be/src/olap/tablet_meta_manager.h index 2eba2bfd6c..15d6763477 100644 --- a/be/src/olap/tablet_meta_manager.h +++ b/be/src/olap/tablet_meta_manager.h @@ -17,13 +17,18 @@ #pragma once +#include + +#include #include -#include "olap/data_dir.h" -#include "olap/olap_define.h" +#include "common/status.h" +#include "gutil/stringprintf.h" #include "olap/tablet_meta.h" namespace doris { +class DataDir; +class OlapMeta; const std::string OLD_HEADER_PREFIX = "hdr_"; diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 2f6a54529d..174a1e13fb 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -17,14 +17,26 @@ #include "olap/tablet_schema.h" +#include #include +#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/consts.h" #include "common/status.h" #include "exec/tablet_info.h" -#include "gen_cpp/descriptors.pb.h" +#include "olap/olap_define.h" +#include "olap/types.h" #include "olap/utils.h" +#include "runtime/thread_context.h" #include "tablet_meta.h" -#include "vec/aggregate_functions/aggregate_function_reader.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/core/block.h" #include "vec/data_types/data_type.h" diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 74d8073031..a77bc03ebd 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -17,14 +17,25 @@ #pragma once +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include #include -#include "gen_cpp/olap_file.pb.h" -#include "gen_cpp/segment_v2.pb.h" -#include "olap/olap_define.h" -#include "olap/types.h" +#include "common/status.h" +#include "gutil/stringprintf.h" +#include "olap/olap_common.h" #include "vec/aggregate_functions/aggregate_function.h" -#include "vec/data_types/data_type.h" namespace doris { namespace vectorized { @@ -32,6 +43,8 @@ class Block; } struct OlapTableIndexSchema; +class TColumn; +class TOlapTableIndex; class TabletColumn { public: diff --git a/be/src/olap/task/engine_alter_tablet_task.cpp b/be/src/olap/task/engine_alter_tablet_task.cpp index a214ad1a1b..df801a6d1b 100644 --- a/be/src/olap/task/engine_alter_tablet_task.cpp +++ b/be/src/olap/task/engine_alter_tablet_task.cpp @@ -17,9 +17,18 @@ #include "olap/task/engine_alter_tablet_task.h" +#include +#include +#include + +#include +#include + +#include "common/config.h" #include "olap/schema_change.h" -#include "runtime/memory/mem_tracker.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" +#include "util/doris_metrics.h" namespace doris { diff --git a/be/src/olap/task/engine_alter_tablet_task.h b/be/src/olap/task/engine_alter_tablet_task.h index 8f7ba64b55..0c7851c98c 100644 --- a/be/src/olap/task/engine_alter_tablet_task.h +++ b/be/src/olap/task/engine_alter_tablet_task.h @@ -17,11 +17,15 @@ #pragma once -#include "gen_cpp/AgentService_types.h" -#include "olap/olap_define.h" +#include + +#include "common/status.h" #include "olap/task/engine_task.h" namespace doris { +class MemTrackerLimiter; +class TAlterInvertedIndexReq; +class TAlterTabletReqV2; // base class for storage engine // add "Engine" as task prefix to prevent duplicate name with agent task diff --git a/be/src/olap/task/engine_batch_load_task.cpp b/be/src/olap/task/engine_batch_load_task.cpp index cd8b73717d..afe3d0b366 100644 --- a/be/src/olap/task/engine_batch_load_task.cpp +++ b/be/src/olap/task/engine_batch_load_task.cpp @@ -17,27 +17,42 @@ #include "olap/task/engine_batch_load_task.h" +#include +#include +#include +#include #include #include #include #include +#include #include -#include -#include +#include #include +#include #include "boost/lexical_cast.hpp" -#include "gen_cpp/AgentService_types.h" +#include "common/config.h" +#include "common/logging.h" #include "http/http_client.h" +#include "olap/data_dir.h" #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/push_handler.h" #include "olap/storage_engine.h" #include "olap/tablet.h" +#include "olap/tablet_manager.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" #include "util/doris_metrics.h" #include "util/pretty_printer.h" +#include "util/runtime_profile.h" +#include "util/stopwatch.hpp" + +namespace doris { +class TTabletInfo; +} // namespace doris using apache::thrift::ThriftDebugString; using std::list; diff --git a/be/src/olap/task/engine_batch_load_task.h b/be/src/olap/task/engine_batch_load_task.h index e314b6faf7..036b49e26d 100644 --- a/be/src/olap/task/engine_batch_load_task.h +++ b/be/src/olap/task/engine_batch_load_task.h @@ -18,22 +18,23 @@ #ifndef DORIS_BE_SRC_OLAP_TASK_ENGINE_BATCH_LOAD_TASK_H #define DORIS_BE_SRC_OLAP_TASK_ENGINE_BATCH_LOAD_TASK_H -#include +#include + +#include +#include #include #include "common/status.h" -#include "gen_cpp/AgentService_types.h" -#include "gen_cpp/MasterService_types.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" #include "olap/task/engine_task.h" namespace doris { +class MemTrackerLimiter; +class TPushReq; +class TTabletInfo; const uint32_t PUSH_MAX_RETRY = 1; const uint32_t MAX_RETRY = 3; const uint32_t DEFAULT_DOWNLOAD_TIMEOUT = 3600; -class StorageEngine; class EngineBatchLoadTask : public EngineTask { public: diff --git a/be/src/olap/task/engine_checksum_task.cpp b/be/src/olap/task/engine_checksum_task.cpp index 1d6c3e52a3..fd98691415 100644 --- a/be/src/olap/task/engine_checksum_task.cpp +++ b/be/src/olap/task/engine_checksum_task.cpp @@ -17,7 +17,26 @@ #include "olap/task/engine_checksum_task.h" +#include +#include + +#include +#include +#include + +#include "io/io_common.h" +#include "olap/olap_common.h" +#include "olap/olap_define.h" +#include "olap/reader.h" +#include "olap/rowset/rowset.h" +#include "olap/storage_engine.h" +#include "olap/tablet.h" +#include "olap/tablet_manager.h" +#include "olap/utils.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" +#include "vec/common/sip_hash.h" +#include "vec/core/block.h" #include "vec/olap/block_reader.h" namespace doris { diff --git a/be/src/olap/task/engine_checksum_task.h b/be/src/olap/task/engine_checksum_task.h index 04afa1a5cd..58daba515c 100644 --- a/be/src/olap/task/engine_checksum_task.h +++ b/be/src/olap/task/engine_checksum_task.h @@ -18,11 +18,16 @@ #ifndef DORIS_BE_SRC_OLAP_TASK_ENGINE_CHECKSUM_TASK_H #define DORIS_BE_SRC_OLAP_TASK_ENGINE_CHECKSUM_TASK_H -#include "gen_cpp/AgentService_types.h" -#include "olap/olap_define.h" +#include +#include + +#include + +#include "common/status.h" #include "olap/task/engine_task.h" namespace doris { +class MemTrackerLimiter; // base class for storage engine // add "Engine" as task prefix to prevent duplicate name with agent task diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 1689d54512..dfb18ee376 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -17,25 +17,51 @@ #include "olap/task/engine_clone_task.h" -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include -#include "gen_cpp/BackendService.h" -#include "gen_cpp/Types_constants.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" #include "gutil/strings/split.h" #include "gutil/strings/stringpiece.h" +#include "gutil/strings/strip.h" #include "http/http_client.h" +#include "io/fs/file_system.h" #include "io/fs/local_file_system.h" +#include "io/fs/path.h" +#include "olap/data_dir.h" +#include "olap/olap_common.h" +#include "olap/olap_define.h" #include "olap/rowset/rowset.h" -#include "olap/rowset/rowset_factory.h" #include "olap/snapshot_manager.h" #include "olap/storage_engine.h" +#include "olap/tablet.h" +#include "olap/tablet_manager.h" #include "olap/tablet_meta.h" #include "runtime/client_cache.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" #include "util/defer_op.h" #include "util/network_util.h" +#include "util/stopwatch.hpp" #include "util/thrift_rpc_helper.h" using std::set; diff --git a/be/src/olap/task/engine_clone_task.h b/be/src/olap/task/engine_clone_task.h index 67a164b6c2..b3a055a500 100644 --- a/be/src/olap/task/engine_clone_task.h +++ b/be/src/olap/task/engine_clone_task.h @@ -18,15 +18,26 @@ #ifndef DORIS_BE_SRC_OLAP_TASK_ENGINE_CLONE_TASK_H #define DORIS_BE_SRC_OLAP_TASK_ENGINE_CLONE_TASK_H -#include "agent/utils.h" +#include +#include + +#include +#include +#include + #include "common/status.h" -#include "gen_cpp/AgentService_types.h" -#include "gen_cpp/HeartbeatService.h" -#include "gen_cpp/MasterService_types.h" -#include "olap/olap_define.h" +#include "gutil/strings/substitute.h" +#include "olap/tablet_meta.h" #include "olap/task/engine_task.h" namespace doris { +class DataDir; +class MemTrackerLimiter; +class TCloneReq; +class TMasterInfo; +class TTabletInfo; +class Tablet; +struct Version; // base class for storage engine // add "Engine" as task prefix to prevent duplicate name with agent task diff --git a/be/src/olap/task/engine_publish_version_task.cpp b/be/src/olap/task/engine_publish_version_task.cpp index 8fd438354a..91384ea89c 100644 --- a/be/src/olap/task/engine_publish_version_task.cpp +++ b/be/src/olap/task/engine_publish_version_task.cpp @@ -17,13 +17,26 @@ #include "olap/task/engine_publish_version_task.h" +#include +#include #include - +// IWYU pragma: no_include +#include // IWYU pragma: keep #include +#include +#include +#include +#include +#include +#include -#include "olap/data_dir.h" -#include "olap/rowset/rowset_meta_manager.h" +#include "common/logging.h" +#include "olap/storage_engine.h" #include "olap/tablet_manager.h" +#include "olap/tablet_meta.h" +#include "olap/txn_manager.h" +#include "olap/utils.h" +#include "util/threadpool.h" namespace doris { diff --git a/be/src/olap/task/engine_publish_version_task.h b/be/src/olap/task/engine_publish_version_task.h index 959584d116..daed805518 100644 --- a/be/src/olap/task/engine_publish_version_task.h +++ b/be/src/olap/task/engine_publish_version_task.h @@ -18,13 +18,25 @@ #ifndef DORIS_BE_SRC_OLAP_TASK_ENGINE_PUBLISH_VERSION_TASK_H #define DORIS_BE_SRC_OLAP_TASK_ENGINE_PUBLISH_VERSION_TASK_H -#include "gen_cpp/AgentService_types.h" -#include "olap/olap_define.h" +#include +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "olap/olap_common.h" +#include "olap/rowset/rowset.h" +#include "olap/tablet.h" #include "olap/task/engine_task.h" namespace doris { class EnginePublishVersionTask; +class TPublishVersionRequest; + class TabletPublishTxnTask { public: TabletPublishTxnTask(EnginePublishVersionTask* engine_task, TabletSharedPtr tablet, diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index 43c593b509..7bf512d407 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -17,10 +17,32 @@ #include "olap/task/engine_storage_migration_task.h" -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "gutil/strings/numbers.h" +#include "io/fs/local_file_system.h" +#include "olap/data_dir.h" +#include "olap/olap_common.h" +#include "olap/olap_define.h" +#include "olap/rowset/rowset_meta.h" #include "olap/snapshot_manager.h" -#include "olap/tablet_meta_manager.h" +#include "olap/storage_engine.h" +#include "olap/tablet_manager.h" +#include "olap/txn_manager.h" +#include "util/doris_metrics.h" +#include "util/uid_util.h" namespace doris { diff --git a/be/src/olap/task/engine_storage_migration_task.h b/be/src/olap/task/engine_storage_migration_task.h index c2c9b7a774..d392de9cf2 100644 --- a/be/src/olap/task/engine_storage_migration_task.h +++ b/be/src/olap/task/engine_storage_migration_task.h @@ -18,11 +18,21 @@ #ifndef DORIS_BE_SRC_OLAP_TASK_ENGINE_STORAGE_MIGRATION_TASK_H #define DORIS_BE_SRC_OLAP_TASK_ENGINE_STORAGE_MIGRATION_TASK_H -#include "gen_cpp/AgentService_types.h" -#include "olap/olap_define.h" +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "olap/rowset/rowset.h" +#include "olap/tablet.h" +#include "olap/tablet_meta.h" #include "olap/task/engine_task.h" namespace doris { +class DataDir; /// This task is used to migrate the specified tablet to the specified data directory. // Usually used for storage medium migration, or migration of tablets between disks. diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp index 474b4c8936..cfb1b676f0 100644 --- a/be/src/olap/txn_manager.cpp +++ b/be/src/olap/txn_manager.cpp @@ -17,39 +17,33 @@ #include "txn_manager.h" -#include -#include #include +#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include #include -#include #include +#include -#include "olap/base_compaction.h" -#include "olap/cumulative_compaction.h" +#include "common/config.h" +#include "common/logging.h" #include "olap/data_dir.h" #include "olap/delta_writer.h" -#include "olap/lru_cache.h" -#include "olap/push_handler.h" -#include "olap/reader.h" +#include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_meta_manager.h" -#include "olap/schema_change.h" #include "olap/storage_engine.h" +#include "olap/tablet_manager.h" #include "olap/tablet_meta.h" -#include "olap/tablet_meta_manager.h" -#include "olap/utils.h" -#include "rowset/beta_rowset.h" -#include "util/doris_metrics.h" -#include "util/pretty_printer.h" #include "util/time.h" +namespace doris { +class OlapMeta; +} // namespace doris + using apache::thrift::ThriftDebugString; using std::filesystem::canonical; using std::filesystem::directory_iterator; @@ -64,7 +58,6 @@ using std::nothrow; using std::pair; using std::priority_queue; using std::set; -using std::set_difference; using std::string; using std::stringstream; using std::vector; diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h index 7319234b27..ad986fbe9e 100644 --- a/be/src/olap/txn_manager.h +++ b/be/src/olap/txn_manager.h @@ -17,38 +17,33 @@ #pragma once -#include -#include +#include +#include +#include +#include +#include -#include -#include +#include #include +#include #include #include -#include -#include +#include #include #include +#include #include #include "common/status.h" -#include "gen_cpp/AgentService_types.h" -#include "gen_cpp/BackendService_types.h" -#include "gen_cpp/MasterService_types.h" -#include "olap/lru_cache.h" #include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/olap_meta.h" -#include "olap/options.h" #include "olap/rowset/rowset.h" -#include "olap/rowset/rowset_meta.h" -#include "olap/rowset/segment_v2/segment.h" #include "olap/tablet.h" #include "olap/tablet_meta.h" #include "util/time.h" namespace doris { class DeltaWriter; +class OlapMeta; struct TabletTxnInfo { PUniqueId load_id; diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp index da3fd4a255..e35725f07d 100644 --- a/be/src/olap/types.cpp +++ b/be/src/olap/types.cpp @@ -17,9 +17,12 @@ #include "olap/types.h" +#include + #include -#include "gen_cpp/segment_v2.pb.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "olap/tablet_schema.h" namespace doris { diff --git a/be/src/olap/types.h b/be/src/olap/types.h index 6b35729551..698b6548f7 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -17,29 +17,43 @@ #pragma once -#include +#include +#include +#include #include +#include +#include +#include +#include #include #include #include +#include #include #include +#include +#include +#include +#include "common/config.h" +#include "common/status.h" +#include "gutil/stringprintf.h" #include "gutil/strings/numbers.h" +#include "olap/decimal12.h" #include "olap/olap_common.h" #include "olap/olap_define.h" +#include "olap/uint24.h" #include "runtime/collection_value.h" -#include "runtime/jsonb_value.h" #include "runtime/map_value.h" #include "runtime/struct_value.h" -#include "util/jsonb_document.h" -#include "util/jsonb_utils.h" +#include "util/binary_cast.hpp" #include "util/mysql_global.h" #include "util/slice.h" #include "util/string_parser.hpp" #include "util/types.h" #include "vec/common/arena.h" +#include "vec/runtime/vdatetime_value.h" namespace doris { @@ -47,13 +61,12 @@ namespace segment_v2 { class ColumnMetaPB; } -struct uint24_t; -struct decimal12_t; class TabletColumn; extern bool is_olap_string_type(FieldType field_type); class TypeInfo; + using TypeInfoPtr = std::unique_ptr; TypeInfoPtr create_static_type_info_ptr(const TypeInfo* type_info); diff --git a/be/src/olap/utils.cpp b/be/src/olap/utils.cpp index 1f19881aee..188543b346 100644 --- a/be/src/olap/utils.cpp +++ b/be/src/olap/utils.cpp @@ -17,22 +17,24 @@ #include "olap/utils.h" -#include -#include -#include -#include -#include -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include +#include +#include -#include +#include #include -#include +#include #include +#include +#include #include #include +#include "util/sse_util.hpp" + #ifdef DORIS_WITH_LZO #include #include @@ -42,13 +44,11 @@ #include "common/logging.h" #include "common/status.h" -#include "gutil/strings/substitute.h" #include "io/fs/file_reader.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "util/errno.h" #include "util/string_parser.hpp" using std::string; diff --git a/be/src/olap/utils.h b/be/src/olap/utils.h index 71be36cb71..f91b625db3 100644 --- a/be/src/olap/utils.h +++ b/be/src/olap/utils.h @@ -17,26 +17,21 @@ #pragma once -#include -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include #include -#include #include #include -#include -#include #include #include -#include -#include -#include #include #include -#include "common/logging.h" +#include "common/status.h" #include "olap/olap_common.h" -#include "olap/olap_define.h" namespace doris { void write_log_info(char* buf, size_t buf_len, const char* fmt, ...); diff --git a/be/src/olap/version_graph.cpp b/be/src/olap/version_graph.cpp index bae1d289a2..0204ab783d 100644 --- a/be/src/olap/version_graph.cpp +++ b/be/src/olap/version_graph.cpp @@ -18,12 +18,17 @@ #include "olap/version_graph.h" #include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include #include -#include +#include +#include #include "common/logging.h" -#include "util/time.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/olap/version_graph.h b/be/src/olap/version_graph.h index 287d946fcf..25ca25cc5a 100644 --- a/be/src/olap/version_graph.h +++ b/be/src/olap/version_graph.h @@ -18,11 +18,16 @@ #pragma once #include -#include -#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" #include "olap/olap_common.h" -#include "olap/olap_define.h" #include "olap/rowset/rowset_meta.h" namespace doris { diff --git a/be/src/olap/wrapper_field.cpp b/be/src/olap/wrapper_field.cpp index 6f227e1775..cc5e756a31 100644 --- a/be/src/olap/wrapper_field.cpp +++ b/be/src/olap/wrapper_field.cpp @@ -17,6 +17,15 @@ #include "olap/wrapper_field.h" +#include +#include + +#include +#include + +#include "common/config.h" +#include "olap/olap_common.h" +#include "olap/olap_define.h" #include "olap/row_cursor.h" namespace doris { diff --git a/be/src/olap/wrapper_field.h b/be/src/olap/wrapper_field.h index b8ed3badfa..71afc2f2a4 100644 --- a/be/src/olap/wrapper_field.h +++ b/be/src/olap/wrapper_field.h @@ -17,13 +17,20 @@ #pragma once +#include +#include + +#include +#include + +#include "common/status.h" #include "olap/field.h" -#include "olap/olap_define.h" #include "olap/row_cursor_cell.h" #include "olap/tablet_schema.h" -#include "util/hash_util.hpp" +#include "util/slice.h" namespace doris { +enum class FieldType; class WrapperField { public: diff --git a/be/src/pipeline/exec/aggregation_sink_operator.cpp b/be/src/pipeline/exec/aggregation_sink_operator.cpp index ce53e4d6a4..70ca1219d4 100644 --- a/be/src/pipeline/exec/aggregation_sink_operator.cpp +++ b/be/src/pipeline/exec/aggregation_sink_operator.cpp @@ -17,7 +17,9 @@ #include "aggregation_sink_operator.h" -#include "vec/exec/vaggregation_node.h" +#include + +#include "pipeline/exec/operator.h" namespace doris::pipeline { diff --git a/be/src/pipeline/exec/aggregation_sink_operator.h b/be/src/pipeline/exec/aggregation_sink_operator.h index 23115fbe19..3bfc783ccc 100644 --- a/be/src/pipeline/exec/aggregation_sink_operator.h +++ b/be/src/pipeline/exec/aggregation_sink_operator.h @@ -17,14 +17,13 @@ #pragma once +#include + #include "operator.h" +#include "vec/exec/vaggregation_node.h" namespace doris { -namespace vectorized { -class AggregationNode; -class VExprContext; -class Block; -} // namespace vectorized +class ExecNode; namespace pipeline { diff --git a/be/src/pipeline/exec/aggregation_source_operator.cpp b/be/src/pipeline/exec/aggregation_source_operator.cpp index e3b2aa1f23..80a974c78d 100644 --- a/be/src/pipeline/exec/aggregation_source_operator.cpp +++ b/be/src/pipeline/exec/aggregation_source_operator.cpp @@ -17,7 +17,9 @@ #include "aggregation_source_operator.h" -#include "vec/exec/vaggregation_node.h" +#include + +#include "pipeline/exec/operator.h" namespace doris { namespace pipeline { diff --git a/be/src/pipeline/exec/aggregation_source_operator.h b/be/src/pipeline/exec/aggregation_source_operator.h index f1c93071da..19c64bc065 100644 --- a/be/src/pipeline/exec/aggregation_source_operator.h +++ b/be/src/pipeline/exec/aggregation_source_operator.h @@ -16,12 +16,15 @@ // under the License. #pragma once +#include + +#include "common/status.h" #include "operator.h" +#include "vec/exec/vaggregation_node.h" namespace doris { -namespace vectorized { -class AggregationNode; -} +class ExecNode; +class RuntimeState; namespace pipeline { diff --git a/be/src/pipeline/exec/analytic_sink_operator.cpp b/be/src/pipeline/exec/analytic_sink_operator.cpp index 4a17359ea7..a0634c81f6 100644 --- a/be/src/pipeline/exec/analytic_sink_operator.cpp +++ b/be/src/pipeline/exec/analytic_sink_operator.cpp @@ -18,6 +18,10 @@ #include "analytic_sink_operator.h" +#include + +#include "pipeline/exec/operator.h" + namespace doris::pipeline { OPERATOR_CODE_GENERATOR(AnalyticSinkOperator, StreamingOperator) diff --git a/be/src/pipeline/exec/analytic_sink_operator.h b/be/src/pipeline/exec/analytic_sink_operator.h index 793824ea75..2d16eb1c70 100644 --- a/be/src/pipeline/exec/analytic_sink_operator.h +++ b/be/src/pipeline/exec/analytic_sink_operator.h @@ -18,13 +18,13 @@ #pragma once +#include + #include "operator.h" #include "vec/exec/vanalytic_eval_node.h" namespace doris { -namespace vectorized { -class VAnalyticEvalNode; -} // namespace vectorized +class ExecNode; namespace pipeline { class AnalyticSinkOperatorBuilder final : public OperatorBuilder { diff --git a/be/src/pipeline/exec/analytic_source_operator.cpp b/be/src/pipeline/exec/analytic_source_operator.cpp index 9c9717d24a..20a4bfb9cc 100644 --- a/be/src/pipeline/exec/analytic_source_operator.cpp +++ b/be/src/pipeline/exec/analytic_source_operator.cpp @@ -17,6 +17,10 @@ #include "analytic_source_operator.h" +#include + +#include "pipeline/exec/operator.h" + namespace doris::pipeline { OPERATOR_CODE_GENERATOR(AnalyticSourceOperator, SourceOperator) diff --git a/be/src/pipeline/exec/analytic_source_operator.h b/be/src/pipeline/exec/analytic_source_operator.h index 34fd9cdfde..60f1a86158 100644 --- a/be/src/pipeline/exec/analytic_source_operator.h +++ b/be/src/pipeline/exec/analytic_source_operator.h @@ -17,14 +17,15 @@ #pragma once +#include + +#include "common/status.h" #include "operator.h" #include "vec/exec/vanalytic_eval_node.h" namespace doris { - -namespace vectorized { -class VAnalyticEvalNode; -} +class ExecNode; +class RuntimeState; namespace pipeline { diff --git a/be/src/pipeline/exec/data_queue.cpp b/be/src/pipeline/exec/data_queue.cpp index a6f2c43939..42a1fbf38e 100644 --- a/be/src/pipeline/exec/data_queue.cpp +++ b/be/src/pipeline/exec/data_queue.cpp @@ -17,14 +17,16 @@ #include "data_queue.h" -#include +#include +#include +#include +#include + +#include "gutil/integral_types.h" #include "vec/core/block.h" namespace doris { -namespace vectorized { -class Block; -} namespace pipeline { DataQueue::DataQueue(int child_count) { diff --git a/be/src/pipeline/exec/data_queue.h b/be/src/pipeline/exec/data_queue.h index 978fe6e629..299f7e6b4d 100644 --- a/be/src/pipeline/exec/data_queue.h +++ b/be/src/pipeline/exec/data_queue.h @@ -16,17 +16,18 @@ // under the License. #pragma once +#include + #include #include #include #include #include +#include "common/status.h" #include "vec/core/block.h" + namespace doris { -namespace vectorized { -class Block; -} namespace pipeline { class DataQueue { diff --git a/be/src/pipeline/exec/datagen_operator.cpp b/be/src/pipeline/exec/datagen_operator.cpp index fc4a32342c..7b5cf26032 100644 --- a/be/src/pipeline/exec/datagen_operator.cpp +++ b/be/src/pipeline/exec/datagen_operator.cpp @@ -17,8 +17,16 @@ #include "datagen_operator.h" +#include + +#include "pipeline/exec/operator.h" +#include "util/runtime_profile.h" #include "vec/exec/vdata_gen_scan_node.h" +namespace doris { +class RuntimeState; +} // namespace doris + namespace doris::pipeline { OPERATOR_CODE_GENERATOR(DataGenOperator, SourceOperator) diff --git a/be/src/pipeline/exec/datagen_operator.h b/be/src/pipeline/exec/datagen_operator.h index 7cea4d54df..48d525e80b 100644 --- a/be/src/pipeline/exec/datagen_operator.h +++ b/be/src/pipeline/exec/datagen_operator.h @@ -17,11 +17,16 @@ #pragma once -#include "operator.h" +#include -namespace doris::vectorized { -class VDataGenFunctionScanNode; -} // namespace doris::vectorized +#include "common/status.h" +#include "operator.h" +#include "vec/exec/vdata_gen_scan_node.h" + +namespace doris { +class ExecNode; +class RuntimeState; +} // namespace doris namespace doris::pipeline { diff --git a/be/src/pipeline/exec/empty_set_operator.cpp b/be/src/pipeline/exec/empty_set_operator.cpp index be1178e9a6..cc244e1426 100644 --- a/be/src/pipeline/exec/empty_set_operator.cpp +++ b/be/src/pipeline/exec/empty_set_operator.cpp @@ -17,7 +17,9 @@ #include "empty_set_operator.h" -#include "vec/exec/vempty_set_node.h" +#include + +#include "pipeline/exec/operator.h" namespace doris::pipeline { diff --git a/be/src/pipeline/exec/empty_set_operator.h b/be/src/pipeline/exec/empty_set_operator.h index a6324ca249..67cf23a230 100644 --- a/be/src/pipeline/exec/empty_set_operator.h +++ b/be/src/pipeline/exec/empty_set_operator.h @@ -17,10 +17,13 @@ #pragma once +#include + #include "operator.h" #include "vec/exec/vempty_set_node.h" namespace doris { +class ExecNode; namespace pipeline { diff --git a/be/src/pipeline/exec/empty_source_operator.cpp b/be/src/pipeline/exec/empty_source_operator.cpp index b5b140305b..5142c0c55a 100644 --- a/be/src/pipeline/exec/empty_source_operator.cpp +++ b/be/src/pipeline/exec/empty_source_operator.cpp @@ -17,6 +17,8 @@ #include "empty_source_operator.h" +#include "pipeline/exec/operator.h" + namespace doris::pipeline { OperatorPtr EmptySourceOperatorBuilder::build_operator() { return std::make_shared(this); diff --git a/be/src/pipeline/exec/empty_source_operator.h b/be/src/pipeline/exec/empty_source_operator.h index 0cfa3bcbf1..fd12b27dab 100644 --- a/be/src/pipeline/exec/empty_source_operator.h +++ b/be/src/pipeline/exec/empty_source_operator.h @@ -17,11 +17,21 @@ #pragma once -#include "operator.h" +#include -namespace doris::vectorized { -class VExchangeNode; -} +#include + +#include "common/status.h" +#include "operator.h" +#include "runtime/descriptors.h" + +namespace doris { +class RuntimeState; + +namespace vectorized { +class Block; +} // namespace vectorized +} // namespace doris namespace doris::pipeline { diff --git a/be/src/pipeline/exec/exchange_sink_buffer.cpp b/be/src/pipeline/exec/exchange_sink_buffer.cpp index da536d66ec..8f6927640b 100644 --- a/be/src/pipeline/exec/exchange_sink_buffer.cpp +++ b/be/src/pipeline/exec/exchange_sink_buffer.cpp @@ -17,14 +17,28 @@ #include "exchange_sink_buffer.h" -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include #include +#include +#include #include "common/status.h" #include "pipeline/pipeline_fragment_context.h" -#include "service/brpc.h" +#include "runtime/exec_env.h" +#include "runtime/thread_context.h" +#include "service/backend_options.h" #include "util/proto_util.h" #include "vec/sink/vdata_stream_sender.h" diff --git a/be/src/pipeline/exec/exchange_sink_buffer.h b/be/src/pipeline/exec/exchange_sink_buffer.h index 783457c165..17093bca87 100644 --- a/be/src/pipeline/exec/exchange_sink_buffer.h +++ b/be/src/pipeline/exec/exchange_sink_buffer.h @@ -17,18 +17,25 @@ #pragma once +#include +#include #include +#include +#include #include +#include +#include #include -#include +#include #include "common/global_types.h" #include "common/status.h" -#include "gen_cpp/Types_types.h" -#include "gen_cpp/internal_service.pb.h" namespace doris { +class PTransmitDataParams; +class TUniqueId; + namespace vectorized { class PipChannel; class BroadcastPBlockHolder; diff --git a/be/src/pipeline/exec/exchange_sink_operator.cpp b/be/src/pipeline/exec/exchange_sink_operator.cpp index 43242c8f3e..ab7a1ac07c 100644 --- a/be/src/pipeline/exec/exchange_sink_operator.cpp +++ b/be/src/pipeline/exec/exchange_sink_operator.cpp @@ -17,13 +17,20 @@ #include "exchange_sink_operator.h" +#include +#include +#include + #include "common/status.h" #include "exchange_sink_buffer.h" -#include "gen_cpp/internal_service.pb.h" -#include "util/brpc_client_cache.h" -#include "vec/exprs/vexpr.h" +#include "pipeline/exec/operator.h" +#include "runtime/runtime_state.h" #include "vec/sink/vdata_stream_sender.h" +namespace doris { +class DataSink; +} // namespace doris + namespace doris::pipeline { ExchangeSinkOperatorBuilder::ExchangeSinkOperatorBuilder(int32_t id, DataSink* sink, diff --git a/be/src/pipeline/exec/exchange_sink_operator.h b/be/src/pipeline/exec/exchange_sink_operator.h index c35e30f49b..034ceeffa2 100644 --- a/be/src/pipeline/exec/exchange_sink_operator.h +++ b/be/src/pipeline/exec/exchange_sink_operator.h @@ -17,11 +17,19 @@ #pragma once +#include + +#include + +#include "common/status.h" #include "exchange_sink_buffer.h" #include "operator.h" #include "vec/sink/vdata_stream_sender.h" namespace doris { +class DataSink; +class RuntimeState; +class TDataSink; namespace pipeline { class PipelineFragmentContext; diff --git a/be/src/pipeline/exec/exchange_source_operator.cpp b/be/src/pipeline/exec/exchange_source_operator.cpp index 88df3c57c7..1dd20f929c 100644 --- a/be/src/pipeline/exec/exchange_source_operator.cpp +++ b/be/src/pipeline/exec/exchange_source_operator.cpp @@ -17,7 +17,9 @@ #include "exchange_source_operator.h" -#include "common/status.h" +#include + +#include "pipeline/exec/operator.h" #include "vec/exec/vexchange_node.h" #include "vec/runtime/vdata_stream_recvr.h" diff --git a/be/src/pipeline/exec/exchange_source_operator.h b/be/src/pipeline/exec/exchange_source_operator.h index b6143a86ab..f80685f396 100644 --- a/be/src/pipeline/exec/exchange_source_operator.h +++ b/be/src/pipeline/exec/exchange_source_operator.h @@ -17,11 +17,14 @@ #pragma once -#include "operator.h" +#include -namespace doris::vectorized { -class VExchangeNode; -} +#include "operator.h" +#include "vec/exec/vexchange_node.h" + +namespace doris { +class ExecNode; +} // namespace doris namespace doris::pipeline { diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp index 73fdc69252..55aa25397c 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.cpp +++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp @@ -17,7 +17,9 @@ #include "hashjoin_build_sink.h" -#include "vec/exec/join/vhash_join_node.h" +#include + +#include "pipeline/exec/operator.h" namespace doris::pipeline { diff --git a/be/src/pipeline/exec/hashjoin_build_sink.h b/be/src/pipeline/exec/hashjoin_build_sink.h index 71579c2488..dfa077b83f 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.h +++ b/be/src/pipeline/exec/hashjoin_build_sink.h @@ -17,14 +17,13 @@ #pragma once +#include + #include "operator.h" +#include "vec/exec/join/vhash_join_node.h" namespace doris { -namespace vectorized { -class HashJoinNode; -class VExprContext; -class Block; -} // namespace vectorized +class ExecNode; namespace pipeline { diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp index 91cef6d915..e499d391ce 100644 --- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp +++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp @@ -17,7 +17,9 @@ #include "hashjoin_probe_operator.h" -#include "vec/exec/join/vhash_join_node.h" +#include + +#include "pipeline/exec/operator.h" namespace doris { namespace pipeline { diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.h b/be/src/pipeline/exec/hashjoin_probe_operator.h index 54105e7f8b..b59a6435bd 100644 --- a/be/src/pipeline/exec/hashjoin_probe_operator.h +++ b/be/src/pipeline/exec/hashjoin_probe_operator.h @@ -16,12 +16,15 @@ // under the License. #pragma once +#include + +#include "common/status.h" #include "operator.h" +#include "vec/exec/join/vhash_join_node.h" namespace doris { -namespace vectorized { -class HashJoinNode; -} +class ExecNode; +class RuntimeState; namespace pipeline { diff --git a/be/src/pipeline/exec/nested_loop_join_build_operator.cpp b/be/src/pipeline/exec/nested_loop_join_build_operator.cpp index 5e3245b983..c49e070ef1 100644 --- a/be/src/pipeline/exec/nested_loop_join_build_operator.cpp +++ b/be/src/pipeline/exec/nested_loop_join_build_operator.cpp @@ -17,7 +17,9 @@ #include "nested_loop_join_build_operator.h" -#include "vec/exec/join/vnested_loop_join_node.h" +#include + +#include "pipeline/exec/operator.h" namespace doris::pipeline { diff --git a/be/src/pipeline/exec/nested_loop_join_build_operator.h b/be/src/pipeline/exec/nested_loop_join_build_operator.h index 32bd55d8cf..33918f0a0c 100644 --- a/be/src/pipeline/exec/nested_loop_join_build_operator.h +++ b/be/src/pipeline/exec/nested_loop_join_build_operator.h @@ -17,14 +17,13 @@ #pragma once +#include + #include "operator.h" +#include "vec/exec/join/vnested_loop_join_node.h" namespace doris { -namespace vectorized { -class VNestedLoopJoinNode; -class VExprContext; -class Block; -} // namespace vectorized +class ExecNode; namespace pipeline { diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp index b839de0a98..cdc3630433 100644 --- a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp +++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp @@ -17,8 +17,16 @@ #include "nested_loop_join_probe_operator.h" +#include + +#include "pipeline/exec/operator.h" +#include "vec/core/block.h" #include "vec/exec/join/vnested_loop_join_node.h" +namespace doris { +class RuntimeState; +} // namespace doris + namespace doris::pipeline { OPERATOR_CODE_GENERATOR(NestLoopJoinProbeOperator, StatefulOperator) diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.h b/be/src/pipeline/exec/nested_loop_join_probe_operator.h index bcbf08111c..8fe9b1f883 100644 --- a/be/src/pipeline/exec/nested_loop_join_probe_operator.h +++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.h @@ -17,14 +17,16 @@ #pragma once +#include + +#include "common/status.h" #include "operator.h" +#include "vec/exec/join/vnested_loop_join_node.h" namespace doris { -namespace vectorized { -class VNestedLoopJoinNode; -class VExprContext; -class Block; -} // namespace vectorized +class ExecNode; +class RuntimeState; + namespace pipeline { class NestLoopJoinProbeOperatorBuilder final diff --git a/be/src/pipeline/exec/operator.cpp b/be/src/pipeline/exec/operator.cpp index 39cb8b8814..3fd4e34eba 100644 --- a/be/src/pipeline/exec/operator.cpp +++ b/be/src/pipeline/exec/operator.cpp @@ -17,6 +17,11 @@ #include "operator.h" +namespace doris { +class RowDescriptor; +class RuntimeState; +} // namespace doris + namespace doris::pipeline { OperatorBase::OperatorBase(OperatorBuilderBase* operator_builder) diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h index de02d4c73e..15afe49ef6 100644 --- a/be/src/pipeline/exec/operator.h +++ b/be/src/pipeline/exec/operator.h @@ -17,16 +17,30 @@ #pragma once +#include +#include +#include + +#include +#include +#include +#include +#include #include +#include #include "common/status.h" -#include "exec/data_sink.h" #include "exec/exec_node.h" -#include "runtime/runtime_state.h" +#include "runtime/memory/mem_tracker.h" +#include "util/runtime_profile.h" #include "vec/core/block.h" -#include "vec/exec/vdata_gen_scan_node.h" -#include "vec/exec/vselect_node.h" -#include "vec/exec/vunion_node.h" + +namespace doris { +class DataSink; +class RowDescriptor; +class RuntimeState; +class TDataSink; +} // namespace doris #define OPERATOR_CODE_GENERATOR(NAME, SUBCLASS) \ NAME##Builder::NAME##Builder(int32_t id, ExecNode* exec_node) \ diff --git a/be/src/pipeline/exec/repeat_operator.cpp b/be/src/pipeline/exec/repeat_operator.cpp index d2c9f0a1e2..3c0f819acb 100644 --- a/be/src/pipeline/exec/repeat_operator.cpp +++ b/be/src/pipeline/exec/repeat_operator.cpp @@ -17,8 +17,16 @@ #include "repeat_operator.h" +#include + +#include "pipeline/exec/operator.h" +#include "vec/core/block.h" #include "vec/exec/vrepeat_node.h" +namespace doris { +class RuntimeState; +} // namespace doris + namespace doris::pipeline { OPERATOR_CODE_GENERATOR(RepeatOperator, StatefulOperator) diff --git a/be/src/pipeline/exec/repeat_operator.h b/be/src/pipeline/exec/repeat_operator.h index b397ea05d3..ed982ad574 100644 --- a/be/src/pipeline/exec/repeat_operator.h +++ b/be/src/pipeline/exec/repeat_operator.h @@ -17,14 +17,16 @@ #pragma once +#include + +#include "common/status.h" #include "operator.h" +#include "vec/exec/vrepeat_node.h" namespace doris { -namespace vectorized { -class VRepeatNode; -class VExprContext; -class Block; -} // namespace vectorized +class ExecNode; +class RuntimeState; + namespace pipeline { class RepeatOperatorBuilder final : public OperatorBuilder { diff --git a/be/src/pipeline/exec/result_file_sink_operator.cpp b/be/src/pipeline/exec/result_file_sink_operator.cpp index 72f621ec8b..eeb4449ad6 100644 --- a/be/src/pipeline/exec/result_file_sink_operator.cpp +++ b/be/src/pipeline/exec/result_file_sink_operator.cpp @@ -17,8 +17,15 @@ #include "result_file_sink_operator.h" +#include + +#include "pipeline/exec/operator.h" #include "vec/sink/vresult_file_sink.h" +namespace doris { +class DataSink; +} // namespace doris + namespace doris::pipeline { ResultFileSinkOperatorBuilder::ResultFileSinkOperatorBuilder(int32_t id, DataSink* sink) diff --git a/be/src/pipeline/exec/result_file_sink_operator.h b/be/src/pipeline/exec/result_file_sink_operator.h index 5f21b108a7..beb04ff79c 100644 --- a/be/src/pipeline/exec/result_file_sink_operator.h +++ b/be/src/pipeline/exec/result_file_sink_operator.h @@ -17,12 +17,13 @@ #pragma once +#include + #include "operator.h" +#include "vec/sink/vresult_file_sink.h" namespace doris { -namespace vectorized { -class VResultFileSink; -} +class DataSink; namespace pipeline { diff --git a/be/src/pipeline/exec/result_sink_operator.cpp b/be/src/pipeline/exec/result_sink_operator.cpp index 971c459671..c55b3599ab 100644 --- a/be/src/pipeline/exec/result_sink_operator.cpp +++ b/be/src/pipeline/exec/result_sink_operator.cpp @@ -17,9 +17,16 @@ #include "result_sink_operator.h" +#include + +#include "pipeline/exec/operator.h" #include "runtime/buffer_control_block.h" #include "vec/sink/vresult_sink.h" +namespace doris { +class DataSink; +} // namespace doris + namespace doris::pipeline { ResultSinkOperatorBuilder::ResultSinkOperatorBuilder(int32_t id, DataSink* sink) diff --git a/be/src/pipeline/exec/result_sink_operator.h b/be/src/pipeline/exec/result_sink_operator.h index 483cd98261..45a324ac81 100644 --- a/be/src/pipeline/exec/result_sink_operator.h +++ b/be/src/pipeline/exec/result_sink_operator.h @@ -17,12 +17,13 @@ #pragma once +#include + #include "operator.h" +#include "vec/sink/vresult_sink.h" namespace doris { -namespace vectorized { -class VResultSink; -} +class DataSink; namespace pipeline { diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp index 10a550a371..83e41d93ba 100644 --- a/be/src/pipeline/exec/scan_operator.cpp +++ b/be/src/pipeline/exec/scan_operator.cpp @@ -17,6 +17,11 @@ #include "scan_operator.h" +#include + +#include + +#include "pipeline/exec/operator.h" #include "vec/exec/scan/scanner_context.h" #include "vec/exec/scan/vscan_node.h" diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h index a4810f9730..dbcf8f0ed7 100644 --- a/be/src/pipeline/exec/scan_operator.h +++ b/be/src/pipeline/exec/scan_operator.h @@ -17,13 +17,17 @@ #pragma once -#include "operator.h" +#include -namespace doris::vectorized { -class VScanNode; -class VScanner; -class ScannerContext; -} // namespace doris::vectorized +#include + +#include "common/status.h" +#include "operator.h" +#include "vec/exec/scan/vscan_node.h" + +namespace doris { +class ExecNode; +} // namespace doris namespace doris::pipeline { diff --git a/be/src/pipeline/exec/schema_scan_operator.cpp b/be/src/pipeline/exec/schema_scan_operator.cpp index 497d425bb8..c58d92392d 100644 --- a/be/src/pipeline/exec/schema_scan_operator.cpp +++ b/be/src/pipeline/exec/schema_scan_operator.cpp @@ -17,8 +17,16 @@ #include "schema_scan_operator.h" +#include + +#include "pipeline/exec/operator.h" +#include "util/runtime_profile.h" #include "vec/exec/vschema_scan_node.h" +namespace doris { +class RuntimeState; +} // namespace doris + namespace doris::pipeline { OPERATOR_CODE_GENERATOR(SchemaScanOperator, SourceOperator) diff --git a/be/src/pipeline/exec/schema_scan_operator.h b/be/src/pipeline/exec/schema_scan_operator.h index 34b56e3b13..df0eab8391 100644 --- a/be/src/pipeline/exec/schema_scan_operator.h +++ b/be/src/pipeline/exec/schema_scan_operator.h @@ -17,11 +17,16 @@ #pragma once -#include "operator.h" +#include -namespace doris::vectorized { -class VSchemaScanNode; -} // namespace doris::vectorized +#include "common/status.h" +#include "operator.h" +#include "vec/exec/vschema_scan_node.h" + +namespace doris { +class ExecNode; +class RuntimeState; +} // namespace doris namespace doris::pipeline { diff --git a/be/src/pipeline/exec/select_operator.cpp b/be/src/pipeline/exec/select_operator.cpp index c629c29978..b4eb27eace 100644 --- a/be/src/pipeline/exec/select_operator.cpp +++ b/be/src/pipeline/exec/select_operator.cpp @@ -17,7 +17,9 @@ #include "select_operator.h" -#include "vec/exec/vselect_node.h" +#include + +#include "pipeline/exec/operator.h" namespace doris::pipeline { diff --git a/be/src/pipeline/exec/select_operator.h b/be/src/pipeline/exec/select_operator.h index d71cb319cf..592d619246 100644 --- a/be/src/pipeline/exec/select_operator.h +++ b/be/src/pipeline/exec/select_operator.h @@ -17,14 +17,14 @@ #pragma once +#include + #include "operator.h" +#include "vec/exec/vselect_node.h" namespace doris { -namespace vectorized { -class VSelectNode; -class VExprContext; -class Block; -} // namespace vectorized +class ExecNode; + namespace pipeline { class SelectOperatorBuilder final : public OperatorBuilder { diff --git a/be/src/pipeline/exec/set_probe_sink_operator.cpp b/be/src/pipeline/exec/set_probe_sink_operator.cpp index 9ac62b08fb..eefea17aad 100644 --- a/be/src/pipeline/exec/set_probe_sink_operator.cpp +++ b/be/src/pipeline/exec/set_probe_sink_operator.cpp @@ -17,8 +17,22 @@ #include "set_probe_sink_operator.h" +#include + +#include + +#include "pipeline/exec/operator.h" #include "vec/exec/vset_operation_node.h" +namespace doris { +class ExecNode; +class RuntimeState; + +namespace vectorized { +class Block; +} // namespace vectorized +} // namespace doris + namespace doris::pipeline { template diff --git a/be/src/pipeline/exec/set_probe_sink_operator.h b/be/src/pipeline/exec/set_probe_sink_operator.h index 899bee2324..91ba62a199 100644 --- a/be/src/pipeline/exec/set_probe_sink_operator.h +++ b/be/src/pipeline/exec/set_probe_sink_operator.h @@ -17,14 +17,19 @@ #pragma once +#include + +#include "common/status.h" #include "operator.h" +#include "vec/exec/vset_operation_node.h" namespace doris { +class ExecNode; +class RuntimeState; namespace vectorized { -template -class VSetOperationNode; -} +class Block; +} // namespace vectorized namespace pipeline { diff --git a/be/src/pipeline/exec/set_sink_operator.cpp b/be/src/pipeline/exec/set_sink_operator.cpp index ea85b4f54e..c44f8d4db8 100644 --- a/be/src/pipeline/exec/set_sink_operator.cpp +++ b/be/src/pipeline/exec/set_sink_operator.cpp @@ -17,8 +17,15 @@ #include "set_sink_operator.h" +#include + +#include "pipeline/exec/operator.h" #include "vec/exec/vset_operation_node.h" +namespace doris { +class ExecNode; +} // namespace doris + namespace doris::pipeline { template diff --git a/be/src/pipeline/exec/set_sink_operator.h b/be/src/pipeline/exec/set_sink_operator.h index bd93592769..f622bef19a 100644 --- a/be/src/pipeline/exec/set_sink_operator.h +++ b/be/src/pipeline/exec/set_sink_operator.h @@ -17,14 +17,13 @@ #pragma once +#include + #include "operator.h" +#include "vec/exec/vset_operation_node.h" namespace doris { - -namespace vectorized { -template -class VSetOperationNode; -} +class ExecNode; namespace pipeline { diff --git a/be/src/pipeline/exec/set_source_operator.cpp b/be/src/pipeline/exec/set_source_operator.cpp index 078df0476e..79566d4d4a 100644 --- a/be/src/pipeline/exec/set_source_operator.cpp +++ b/be/src/pipeline/exec/set_source_operator.cpp @@ -17,8 +17,15 @@ #include "set_source_operator.h" +#include + +#include "pipeline/exec/operator.h" #include "vec/exec/vset_operation_node.h" +namespace doris { +class ExecNode; +} // namespace doris + namespace doris::pipeline { template diff --git a/be/src/pipeline/exec/set_source_operator.h b/be/src/pipeline/exec/set_source_operator.h index 14de0079fa..52108e6c08 100644 --- a/be/src/pipeline/exec/set_source_operator.h +++ b/be/src/pipeline/exec/set_source_operator.h @@ -17,14 +17,15 @@ #pragma once +#include + +#include "common/status.h" #include "operator.h" +#include "vec/exec/vset_operation_node.h" namespace doris { - -namespace vectorized { -template -class VSetOperationNode; -} +class ExecNode; +class RuntimeState; namespace pipeline { diff --git a/be/src/pipeline/exec/sort_sink_operator.cpp b/be/src/pipeline/exec/sort_sink_operator.cpp index cdcd75aaaa..5326b5d757 100644 --- a/be/src/pipeline/exec/sort_sink_operator.cpp +++ b/be/src/pipeline/exec/sort_sink_operator.cpp @@ -17,7 +17,9 @@ #include "sort_sink_operator.h" -#include "vec/exec/vsort_node.h" +#include + +#include "pipeline/exec/operator.h" namespace doris::pipeline { diff --git a/be/src/pipeline/exec/sort_sink_operator.h b/be/src/pipeline/exec/sort_sink_operator.h index c9fc6b5255..6ee8f4e170 100644 --- a/be/src/pipeline/exec/sort_sink_operator.h +++ b/be/src/pipeline/exec/sort_sink_operator.h @@ -17,13 +17,13 @@ #pragma once +#include + #include "operator.h" +#include "vec/exec/vsort_node.h" namespace doris { - -namespace vectorized { -class VSortNode; -} +class ExecNode; namespace pipeline { diff --git a/be/src/pipeline/exec/sort_source_operator.cpp b/be/src/pipeline/exec/sort_source_operator.cpp index a2b859d949..0c2b5f4b6e 100644 --- a/be/src/pipeline/exec/sort_source_operator.cpp +++ b/be/src/pipeline/exec/sort_source_operator.cpp @@ -17,7 +17,9 @@ #include "sort_source_operator.h" -#include "vec/exec/vsort_node.h" +#include + +#include "pipeline/exec/operator.h" namespace doris::pipeline { diff --git a/be/src/pipeline/exec/sort_source_operator.h b/be/src/pipeline/exec/sort_source_operator.h index 017e91f60c..389b86236f 100644 --- a/be/src/pipeline/exec/sort_source_operator.h +++ b/be/src/pipeline/exec/sort_source_operator.h @@ -17,13 +17,15 @@ #pragma once +#include + +#include "common/status.h" #include "operator.h" +#include "vec/exec/vsort_node.h" namespace doris { - -namespace vectorized { -class VSortNode; -} +class ExecNode; +class RuntimeState; namespace pipeline { diff --git a/be/src/pipeline/exec/streaming_aggregation_sink_operator.cpp b/be/src/pipeline/exec/streaming_aggregation_sink_operator.cpp index c896862702..29276f4112 100644 --- a/be/src/pipeline/exec/streaming_aggregation_sink_operator.cpp +++ b/be/src/pipeline/exec/streaming_aggregation_sink_operator.cpp @@ -17,8 +17,21 @@ #include "streaming_aggregation_sink_operator.h" +#include + +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "pipeline/exec/data_queue.h" +#include "pipeline/exec/operator.h" #include "vec/exec/vaggregation_node.h" +namespace doris { +class ExecNode; +class RuntimeState; +} // namespace doris + namespace doris::pipeline { StreamingAggSinkOperator::StreamingAggSinkOperator(OperatorBuilderBase* operator_builder, diff --git a/be/src/pipeline/exec/streaming_aggregation_sink_operator.h b/be/src/pipeline/exec/streaming_aggregation_sink_operator.h index 3d47f7fb37..02525c275e 100644 --- a/be/src/pipeline/exec/streaming_aggregation_sink_operator.h +++ b/be/src/pipeline/exec/streaming_aggregation_sink_operator.h @@ -17,17 +17,22 @@ #pragma once +#include + +#include + +#include "common/status.h" #include "operator.h" -#include "pipeline/exec/data_queue.h" +#include "util/runtime_profile.h" +#include "vec/core/block.h" +#include "vec/exec/vaggregation_node.h" namespace doris { -namespace vectorized { -class AggregationNode; -class VExprContext; -class Block; -} // namespace vectorized +class ExecNode; +class RuntimeState; namespace pipeline { +class DataQueue; class StreamingAggSinkOperatorBuilder final : public OperatorBuilder { public: diff --git a/be/src/pipeline/exec/streaming_aggregation_source_operator.cpp b/be/src/pipeline/exec/streaming_aggregation_source_operator.cpp index f048cd4b62..a293052232 100644 --- a/be/src/pipeline/exec/streaming_aggregation_source_operator.cpp +++ b/be/src/pipeline/exec/streaming_aggregation_source_operator.cpp @@ -17,9 +17,19 @@ #include "streaming_aggregation_source_operator.h" +#include + +#include "pipeline/exec/data_queue.h" +#include "pipeline/exec/operator.h" +#include "runtime/descriptors.h" +#include "util/runtime_profile.h" +#include "vec/core/block.h" #include "vec/exec/vaggregation_node.h" namespace doris { +class ExecNode; +class RuntimeState; + namespace pipeline { StreamingAggSourceOperator::StreamingAggSourceOperator(OperatorBuilderBase* templ, ExecNode* node, std::shared_ptr queue) diff --git a/be/src/pipeline/exec/streaming_aggregation_source_operator.h b/be/src/pipeline/exec/streaming_aggregation_source_operator.h index 74b4be5420..ef13e03c4b 100644 --- a/be/src/pipeline/exec/streaming_aggregation_source_operator.h +++ b/be/src/pipeline/exec/streaming_aggregation_source_operator.h @@ -16,16 +16,23 @@ // under the License. #pragma once -#include +#include +#include + +#include "common/status.h" #include "operator.h" -#include "pipeline/exec/data_queue.h" +#include "vec/exec/vaggregation_node.h" namespace doris { +class ExecNode; +class RuntimeState; + namespace vectorized { -class AggregationNode; -} +class Block; +} // namespace vectorized namespace pipeline { +class DataQueue; class StreamingAggSourceOperatorBuilder final : public OperatorBuilder { diff --git a/be/src/pipeline/exec/table_function_operator.cpp b/be/src/pipeline/exec/table_function_operator.cpp index 6d38726742..3f6133c3fc 100644 --- a/be/src/pipeline/exec/table_function_operator.cpp +++ b/be/src/pipeline/exec/table_function_operator.cpp @@ -17,6 +17,15 @@ #include "table_function_operator.h" +#include + +#include "pipeline/exec/operator.h" +#include "vec/core/block.h" + +namespace doris { +class RuntimeState; +} // namespace doris + namespace doris::pipeline { OPERATOR_CODE_GENERATOR(TableFunctionOperator, StatefulOperator) diff --git a/be/src/pipeline/exec/table_function_operator.h b/be/src/pipeline/exec/table_function_operator.h index 1d544e8834..a4b4e77141 100644 --- a/be/src/pipeline/exec/table_function_operator.h +++ b/be/src/pipeline/exec/table_function_operator.h @@ -17,9 +17,17 @@ #pragma once +#include + +#include "common/status.h" #include "operator.h" #include "vec/exec/vtable_function_node.h" +namespace doris { +class ExecNode; +class RuntimeState; +} // namespace doris + namespace doris::pipeline { class TableFunctionOperatorBuilder final : public OperatorBuilder { diff --git a/be/src/pipeline/exec/union_sink_operator.cpp b/be/src/pipeline/exec/union_sink_operator.cpp index 488d2dc58a..7ea31cdeaf 100644 --- a/be/src/pipeline/exec/union_sink_operator.cpp +++ b/be/src/pipeline/exec/union_sink_operator.cpp @@ -17,7 +17,19 @@ #include "union_sink_operator.h" +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" +#include "pipeline/exec/data_queue.h" +#include "pipeline/exec/operator.h" +#include "runtime/runtime_state.h" +#include "util/runtime_profile.h" + +namespace doris { +class ExecNode; +} // namespace doris namespace doris::pipeline { diff --git a/be/src/pipeline/exec/union_sink_operator.h b/be/src/pipeline/exec/union_sink_operator.h index 707a980cf4..dfe7085d5e 100644 --- a/be/src/pipeline/exec/union_sink_operator.h +++ b/be/src/pipeline/exec/union_sink_operator.h @@ -17,16 +17,21 @@ #pragma once +#include + +#include + +#include "common/status.h" #include "operator.h" -#include "pipeline/exec/data_queue.h" #include "vec/core/block.h" +#include "vec/exec/vunion_node.h" + namespace doris { -namespace vectorized { -class VUnionNode; -class Block; -} // namespace vectorized +class ExecNode; +class RuntimeState; namespace pipeline { +class DataQueue; class UnionSinkOperatorBuilder final : public OperatorBuilder { public: diff --git a/be/src/pipeline/exec/union_source_operator.cpp b/be/src/pipeline/exec/union_source_operator.cpp index b65e7b3f51..83069f3e90 100644 --- a/be/src/pipeline/exec/union_source_operator.cpp +++ b/be/src/pipeline/exec/union_source_operator.cpp @@ -17,15 +17,18 @@ #include "pipeline/exec/union_source_operator.h" -#include +#include +#include #include "common/status.h" #include "pipeline/exec/data_queue.h" +#include "pipeline/exec/operator.h" +#include "runtime/descriptors.h" +#include "vec/core/block.h" namespace doris { -namespace vectorized { -class Block; -} +class ExecNode; +class RuntimeState; namespace pipeline { diff --git a/be/src/pipeline/exec/union_source_operator.h b/be/src/pipeline/exec/union_source_operator.h index d330b45a82..33a6bef587 100644 --- a/be/src/pipeline/exec/union_source_operator.h +++ b/be/src/pipeline/exec/union_source_operator.h @@ -16,15 +16,24 @@ // under the License. #pragma once +#include + +#include + +#include "common/status.h" #include "operator.h" -#include "pipeline/exec/data_queue.h" +#include "vec/exec/vunion_node.h" namespace doris { +class ExecNode; +class RuntimeState; + namespace vectorized { -class VUnionNode; -} +class Block; +} // namespace vectorized namespace pipeline { +class DataQueue; class UnionSourceOperatorBuilder final : public OperatorBuilder { public: diff --git a/be/src/pipeline/pipeline.cpp b/be/src/pipeline/pipeline.cpp index c5a94ba640..ccc7786442 100644 --- a/be/src/pipeline/pipeline.cpp +++ b/be/src/pipeline/pipeline.cpp @@ -17,7 +17,11 @@ #include "pipeline.h" -#include "pipeline_fragment_context.h" +#include +#include +#include + +#include "pipeline/exec/operator.h" namespace doris::pipeline { diff --git a/be/src/pipeline/pipeline.h b/be/src/pipeline/pipeline.h index dc1277f721..4cf9b53086 100644 --- a/be/src/pipeline/pipeline.h +++ b/be/src/pipeline/pipeline.h @@ -17,16 +17,28 @@ #pragma once +#include +#include + +#include #include #include #include #include "common/status.h" #include "exec/operator.h" +#include "util/runtime_profile.h" + +namespace doris { +namespace pipeline { +class PipelineFragmentContext; +} // namespace pipeline +} // namespace doris namespace doris::pipeline { class Pipeline; + using PipelinePtr = std::shared_ptr; using Pipelines = std::vector; using PipelineId = uint32_t; diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp index a2b1afbe48..9a5c23fb6c 100644 --- a/be/src/pipeline/pipeline_fragment_context.cpp +++ b/be/src/pipeline/pipeline_fragment_context.cpp @@ -18,12 +18,28 @@ #include "pipeline_fragment_context.h" #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include "common/config.h" +#include "common/logging.h" #include "exec/data_sink.h" +#include "exec/exec_node.h" #include "exec/scan_node.h" -#include "gen_cpp/FrontendService.h" -#include "gen_cpp/HeartbeatService_types.h" +#include "io/fs/stream_load_pipe.h" #include "pipeline/exec/aggregation_sink_operator.h" #include "pipeline/exec/aggregation_source_operator.h" #include "pipeline/exec/analytic_sink_operator.h" @@ -38,7 +54,7 @@ #include "pipeline/exec/exchange_source_operator.h" #include "pipeline/exec/hashjoin_build_sink.h" #include "pipeline/exec/hashjoin_probe_operator.h" -#include "pipeline/exec/mysql_scan_operator.h" +#include "pipeline/exec/mysql_scan_operator.h" // IWYU pragma: keep #include "pipeline/exec/nested_loop_join_build_operator.h" #include "pipeline/exec/nested_loop_join_probe_operator.h" #include "pipeline/exec/olap_table_sink_operator.h" @@ -49,9 +65,9 @@ #include "pipeline/exec/scan_operator.h" #include "pipeline/exec/schema_scan_operator.h" #include "pipeline/exec/select_operator.h" -#include "pipeline/exec/set_probe_sink_operator.h" -#include "pipeline/exec/set_sink_operator.h" -#include "pipeline/exec/set_source_operator.h" +#include "pipeline/exec/set_probe_sink_operator.h" // IWYU pragma: keep +#include "pipeline/exec/set_sink_operator.h" // IWYU pragma: keep +#include "pipeline/exec/set_source_operator.h" // IWYU pragma: keep #include "pipeline/exec/sort_sink_operator.h" #include "pipeline/exec/sort_source_operator.h" #include "pipeline/exec/streaming_aggregation_sink_operator.h" @@ -61,32 +77,41 @@ #include "pipeline/exec/union_sink_operator.h" #include "pipeline/exec/union_source_operator.h" #include "pipeline_task.h" -#include "runtime/client_cache.h" +#include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" +#include "runtime/runtime_filter_mgr.h" #include "runtime/runtime_state.h" #include "runtime/stream_load/new_load_stream_mgr.h" #include "runtime/stream_load/stream_load_context.h" +#include "runtime/thread_context.h" +#include "service/backend_options.h" #include "task_scheduler.h" #include "util/container_util.hpp" +#include "util/debug_util.h" +#include "util/telemetry/telemetry.h" +#include "util/uid_util.h" +#include "vec/common/assert_cast.h" #include "vec/exec/join/vhash_join_node.h" -#include "vec/exec/join/vnested_loop_join_node.h" #include "vec/exec/scan/new_es_scan_node.h" #include "vec/exec/scan/new_file_scan_node.h" -#include "vec/exec/scan/new_jdbc_scan_node.h" #include "vec/exec/scan/new_odbc_scan_node.h" #include "vec/exec/scan/new_olap_scan_node.h" #include "vec/exec/scan/vmeta_scan_node.h" #include "vec/exec/scan/vscan_node.h" #include "vec/exec/vaggregation_node.h" #include "vec/exec/vexchange_node.h" -#include "vec/exec/vrepeat_node.h" -#include "vec/exec/vschema_scan_node.h" -#include "vec/exec/vset_operation_node.h" -#include "vec/exec/vsort_node.h" #include "vec/exec/vunion_node.h" #include "vec/runtime/vdata_stream_mgr.h" -#include "vec/sink/vresult_file_sink.h" -#include "vec/sink/vresult_sink.h" + +namespace apache { +namespace thrift { +class TException; + +namespace transport { +class TTransportException; +} // namespace transport +} // namespace thrift +} // namespace apache using apache::thrift::transport::TTransportException; using apache::thrift::TException; diff --git a/be/src/pipeline/pipeline_fragment_context.h b/be/src/pipeline/pipeline_fragment_context.h index b8db2728d2..5e23ab0e2b 100644 --- a/be/src/pipeline/pipeline_fragment_context.h +++ b/be/src/pipeline/pipeline_fragment_context.h @@ -17,27 +17,43 @@ #pragma once -#include +#include +#include +#include +#include -#include "io/fs/stream_load_pipe.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" #include "pipeline/pipeline.h" #include "pipeline/pipeline_task.h" +#include "runtime/query_fragments_ctx.h" #include "runtime/runtime_state.h" +#include "util/runtime_profile.h" +#include "util/stopwatch.hpp" namespace doris { class ExecNode; class DataSink; struct ReportStatusRequest; +class ExecEnv; +class RuntimeFilterMergeControllerEntity; +class TDataSink; +class TPipelineFragmentParams; -namespace vectorized { -template -class VSetOperationNode; -} +namespace taskgroup { +class TaskGroup; +} // namespace taskgroup namespace pipeline { -class PipelineTask; - class PipelineFragmentContext : public std::enable_shared_from_this { public: // Callback to report execution status of plan fragment. diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp index 8121414a6b..6d69ebfd89 100644 --- a/be/src/pipeline/pipeline_task.cpp +++ b/be/src/pipeline/pipeline_task.cpp @@ -17,8 +17,28 @@ #include "pipeline_task.h" +#include +#include +#include +#include + +#include + +#include "pipeline/exec/operator.h" +#include "pipeline/pipeline.h" #include "pipeline_fragment_context.h" +#include "runtime/descriptors.h" +#include "runtime/query_fragments_ctx.h" +#include "runtime/thread_context.h" #include "task_queue.h" +#include "util/defer_op.h" + +namespace doris { +class RuntimeState; +namespace taskgroup { +class TaskGroup; +} // namespace taskgroup +} // namespace doris namespace doris::pipeline { diff --git a/be/src/pipeline/pipeline_task.h b/be/src/pipeline/pipeline_task.h index 043da50d80..02eb8be992 100644 --- a/be/src/pipeline/pipeline_task.h +++ b/be/src/pipeline/pipeline_task.h @@ -17,9 +17,29 @@ #pragma once +#include + +#include +#include +#include + +#include "common/status.h" #include "exec/operator.h" #include "pipeline.h" +#include "util/runtime_profile.h" #include "util/stopwatch.hpp" +#include "vec/core/block.h" + +namespace doris { +class QueryFragmentsCtx; +class RuntimeState; +namespace pipeline { +class PipelineFragmentContext; +} // namespace pipeline +namespace taskgroup { +class TaskGroup; +} // namespace taskgroup +} // namespace doris namespace doris::pipeline { diff --git a/be/src/pipeline/task_queue.cpp b/be/src/pipeline/task_queue.cpp index dac6ca69ab..7ad69bfa21 100644 --- a/be/src/pipeline/task_queue.cpp +++ b/be/src/pipeline/task_queue.cpp @@ -17,6 +17,12 @@ #include "task_queue.h" +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include + +#include "common/logging.h" +#include "pipeline/pipeline_task.h" #include "runtime/task_group/task_group.h" namespace doris { diff --git a/be/src/pipeline/task_queue.h b/be/src/pipeline/task_queue.h index 2aa4881028..6b16b70f99 100644 --- a/be/src/pipeline/task_queue.h +++ b/be/src/pipeline/task_queue.h @@ -16,14 +16,23 @@ // under the License. #pragma once -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" #include "pipeline_task.h" +#include "runtime/task_group/task_group.h" namespace doris { -namespace taskgroup { -class TaskGroup; -} namespace pipeline { diff --git a/be/src/pipeline/task_scheduler.cpp b/be/src/pipeline/task_scheduler.cpp index 70d1bd518f..7362dd38b3 100644 --- a/be/src/pipeline/task_scheduler.cpp +++ b/be/src/pipeline/task_scheduler.cpp @@ -17,9 +17,30 @@ #include "task_scheduler.h" +#include +#include +#include +#include +#include + +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include + #include "common/signal_handler.h" +#include "pipeline/pipeline_task.h" +#include "pipeline/task_queue.h" #include "pipeline_fragment_context.h" +#include "runtime/query_fragments_ctx.h" +#include "util/sse_util.hpp" #include "util/thread.h" +#include "util/threadpool.h" +#include "util/uid_util.h" +#include "vec/runtime/vdatetime_value.h" namespace doris::pipeline { @@ -342,4 +363,4 @@ void TaskScheduler::try_update_task_group(const taskgroup::TaskGroupInfo& task_g _task_queue->update_task_group(task_group_info, task_group); } -} // namespace doris::pipeline \ No newline at end of file +} // namespace doris::pipeline diff --git a/be/src/pipeline/task_scheduler.h b/be/src/pipeline/task_scheduler.h index 777923dbd3..b2a665f034 100644 --- a/be/src/pipeline/task_scheduler.h +++ b/be/src/pipeline/task_scheduler.h @@ -17,11 +17,30 @@ #pragma once +#include + +#include +#include +#include +#include +#include +#include +#include + #include "common/status.h" -#include "pipeline.h" +#include "gutil/ref_counted.h" #include "pipeline_task.h" -#include "task_queue.h" -#include "util/threadpool.h" +#include "runtime/task_group/task_group.h" +#include "util/thread.h" + +namespace doris { +class ExecEnv; +class ThreadPool; + +namespace pipeline { +class TaskQueue; +} // namespace pipeline +} // namespace doris namespace doris::pipeline { diff --git a/be/src/runtime/block_spill_manager.cpp b/be/src/runtime/block_spill_manager.cpp index b627b5543f..b45e4c3ad6 100644 --- a/be/src/runtime/block_spill_manager.cpp +++ b/be/src/runtime/block_spill_manager.cpp @@ -17,10 +17,16 @@ #include "runtime/block_spill_manager.h" +#include +#include + +#include #include #include +#include #include +#include "io/fs/file_system.h" #include "io/fs/local_file_system.h" #include "util/time.h" #include "vec/core/block_spill_reader.h" diff --git a/be/src/runtime/block_spill_manager.h b/be/src/runtime/block_spill_manager.h index f406b8c9ac..5601d58aed 100644 --- a/be/src/runtime/block_spill_manager.h +++ b/be/src/runtime/block_spill_manager.h @@ -17,24 +17,28 @@ #pragma once +#include + +#include #include #include #include #include +#include "common/status.h" #include "olap/options.h" +#include "vec/core/block_spill_reader.h" +#include "vec/core/block_spill_writer.h" namespace doris { class RuntimeProfile; namespace vectorized { -class BlockSpillWriter; -class BlockSpillReader; + using BlockSpillWriterUPtr = std::unique_ptr; using BlockSpillReaderUPtr = std::unique_ptr; } // namespace vectorized -class ExecEnv; class BlockSpillManager { public: BlockSpillManager(const std::vector& paths); diff --git a/be/src/runtime/broker_mgr.cpp b/be/src/runtime/broker_mgr.cpp index 5e57922399..6102da5e40 100644 --- a/be/src/runtime/broker_mgr.cpp +++ b/be/src/runtime/broker_mgr.cpp @@ -17,16 +17,26 @@ #include "runtime/broker_mgr.h" +#include +#include +#include +#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include +#include #include "common/config.h" -#include "gen_cpp/PaloBrokerService_types.h" -#include "gen_cpp/TPaloBrokerService.h" +#include "common/status.h" #include "runtime/client_cache.h" #include "runtime/exec_env.h" #include "service/backend_options.h" #include "util/doris_metrics.h" -#include "util/thrift_util.h" +#include "util/hash_util.hpp" +#include "util/metrics.h" +#include "util/thread.h" namespace doris { diff --git a/be/src/runtime/broker_mgr.h b/be/src/runtime/broker_mgr.h index 7f55d78fff..fd7a67e43c 100644 --- a/be/src/runtime/broker_mgr.h +++ b/be/src/runtime/broker_mgr.h @@ -17,20 +17,20 @@ #pragma once +#include + #include #include -#include #include -#include "gen_cpp/Types_types.h" #include "gutil/ref_counted.h" #include "util/countdown_latch.h" -#include "util/hash_util.hpp" -#include "util/thread.h" +#include "util/hash_util.hpp" // IWYU pragma: keep namespace doris { class ExecEnv; +class Thread; class BrokerMgr { public: diff --git a/be/src/runtime/buffer_control_block.cpp b/be/src/runtime/buffer_control_block.cpp index 73de8a0852..7565e3edac 100644 --- a/be/src/runtime/buffer_control_block.cpp +++ b/be/src/runtime/buffer_control_block.cpp @@ -17,11 +17,20 @@ #include "runtime/buffer_control_block.h" -#include "gen_cpp/PaloInternalService_types.h" -#include "gen_cpp/internal_service.pb.h" +#include +#include +#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include + #include "runtime/exec_env.h" #include "runtime/thread_context.h" -#include "service/brpc.h" #include "util/thrift_util.h" namespace doris { diff --git a/be/src/runtime/buffer_control_block.h b/be/src/runtime/buffer_control_block.h index 46e6431f73..88fa8d2be4 100644 --- a/be/src/runtime/buffer_control_block.h +++ b/be/src/runtime/buffer_control_block.h @@ -17,14 +17,18 @@ #pragma once +#include +#include +#include + #include #include #include #include +#include #include #include "common/status.h" -#include "gen_cpp/Types_types.h" #include "runtime/query_statistics.h" namespace google { @@ -39,7 +43,6 @@ class Controller; namespace doris { -class TFetchDataResult; class PFetchDataResult; struct GetResultBatchCtx { diff --git a/be/src/runtime/cache/result_cache.cpp b/be/src/runtime/cache/result_cache.cpp index da4c1fc283..fec1d71f36 100644 --- a/be/src/runtime/cache/result_cache.cpp +++ b/be/src/runtime/cache/result_cache.cpp @@ -16,7 +16,15 @@ // under the License. #include "runtime/cache/result_cache.h" -#include "gen_cpp/internal_service.pb.h" +#include +#include + +#include +#include +#include + +#include "olap/olap_define.h" +#include "runtime/cache/cache_utils.h" #include "util/doris_metrics.h" namespace doris { diff --git a/be/src/runtime/cache/result_cache.h b/be/src/runtime/cache/result_cache.h index fc36f61b59..a629e7ed7d 100644 --- a/be/src/runtime/cache/result_cache.h +++ b/be/src/runtime/cache/result_cache.h @@ -17,22 +17,20 @@ #pragma once -#include #include -#include -#include -#include -#include -#include -#include #include -#include +#include -#include "common/config.h" -#include "runtime/cache/cache_utils.h" +#include "gutil/integral_types.h" #include "runtime/cache/result_node.h" +#include "util/uid_util.h" namespace doris { +class PCacheResponse; +class PClearCacheRequest; +class PFetchCacheRequest; +class PFetchCacheResult; +class PUpdateCacheRequest; typedef std::unordered_map ResultNodeMap; diff --git a/be/src/runtime/cache/result_node.cpp b/be/src/runtime/cache/result_node.cpp index 273b9cc5e6..dd00e75390 100644 --- a/be/src/runtime/cache/result_node.cpp +++ b/be/src/runtime/cache/result_node.cpp @@ -16,7 +16,16 @@ // under the License. #include "runtime/cache/result_node.h" -#include "gen_cpp/internal_service.pb.h" +#include +#include +#include + +#include +#include +#include + +#include "common/config.h" +#include "olap/olap_define.h" #include "runtime/cache/cache_utils.h" namespace doris { diff --git a/be/src/runtime/cache/result_node.h b/be/src/runtime/cache/result_node.h index 439ee64529..3edf48261b 100644 --- a/be/src/runtime/cache/result_node.h +++ b/be/src/runtime/cache/result_node.h @@ -17,34 +17,19 @@ #pragma once -#include +#include -#include -#include #include -#include -#include -#include #include -#include -#include +#include +#include -#include "common/config.h" -#include "gen_cpp/internal_service.pb.h" -#include "olap/olap_define.h" +#include "gutil/integral_types.h" #include "runtime/cache/cache_utils.h" #include "util/uid_util.h" namespace doris { -class PCacheParam; -class PCacheValue; -class PCacheResponse; -class PFetchCacheRequest; -class PFetchCacheResult; -class PUpdateCacheRequest; -class PClearCacheRequest; - /** * Cache one partition data, request param must match version and time of cache */ diff --git a/be/src/runtime/client_cache.cpp b/be/src/runtime/client_cache.cpp index d4edbca6c0..3da31caf5c 100644 --- a/be/src/runtime/client_cache.cpp +++ b/be/src/runtime/client_cache.cpp @@ -17,13 +17,10 @@ #include "runtime/client_cache.h" -#include - #include -#include +#include #include "common/logging.h" -#include "gen_cpp/FrontendService.h" #include "util/doris_metrics.h" #include "util/network_util.h" diff --git a/be/src/runtime/client_cache.h b/be/src/runtime/client_cache.h index c461a45082..7ba5cafabe 100644 --- a/be/src/runtime/client_cache.h +++ b/be/src/runtime/client_cache.h @@ -17,19 +17,31 @@ #pragma once -#include -#include -#include -#include -#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" #include "common/status.h" #include "util/hash_util.hpp" #include "util/metrics.h" #include "util/thrift_client.h" +#include "util/thrift_server.h" namespace doris { - // Helper class which implements the majority of the caching // functionality without using templates (i.e. pointers to the // superclass of all ThriftClients and a void* for the key). @@ -268,13 +280,12 @@ private: // Doris backend client cache, used by a backend to send requests // to any other backend. -class BackendServiceClient; using BackendServiceClientCache = ClientCache; using BackendServiceConnection = ClientConnection; -class FrontendServiceClient; + using FrontendServiceClientCache = ClientCache; using FrontendServiceConnection = ClientConnection; -class TPaloBrokerServiceClient; + using BrokerServiceClientCache = ClientCache; using BrokerServiceConnection = ClientConnection; diff --git a/be/src/runtime/collection_value.cpp b/be/src/runtime/collection_value.cpp index e052130bb0..1d50172069 100644 --- a/be/src/runtime/collection_value.cpp +++ b/be/src/runtime/collection_value.cpp @@ -17,12 +17,7 @@ #include "runtime/collection_value.h" -#include - -#include "common/object_pool.h" -#include "common/utils.h" -#include "runtime/types.h" -#include "vec/common/string_ref.h" +#include namespace doris { diff --git a/be/src/runtime/collection_value.h b/be/src/runtime/collection_value.h index 02a51e4bd1..82dfc78fe5 100644 --- a/be/src/runtime/collection_value.h +++ b/be/src/runtime/collection_value.h @@ -17,41 +17,16 @@ #pragma once -#include +#include -#include "runtime/primitive_type.h" +#include +#include namespace doris { -class FunctionContext; -} // namespace doris - -namespace doris { - -using doris::FunctionContext; using MemFootprint = std::pair; using GenMemFootprintFunc = std::function; -struct ArrayIteratorFunctionsBase; -class ArrayIterator; -class Status; -class ObjectPool; - -template -struct ArrayIteratorFunctions; -template -inline constexpr std::enable_if_t, bool> - IsTypeFixedWidth = true; - -template <> -inline constexpr bool IsTypeFixedWidth> = false; -template <> -inline constexpr bool IsTypeFixedWidth> = false; -template <> -inline constexpr bool IsTypeFixedWidth> = false; -template <> -inline constexpr bool IsTypeFixedWidth> = false; - /** * The format of array-typed slot. * A new array needs to be initialized before using it. diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h index d56f0e77cd..88279af1b0 100644 --- a/be/src/runtime/datetime_value.h +++ b/be/src/runtime/datetime_value.h @@ -20,6 +20,7 @@ #include #include +// IWYU pragma: no_include #include #include #include diff --git a/be/src/runtime/decimalv2_value.cpp b/be/src/runtime/decimalv2_value.cpp index 0256f6210f..d8552f3fa1 100644 --- a/be/src/runtime/decimalv2_value.cpp +++ b/be/src/runtime/decimalv2_value.cpp @@ -17,7 +17,10 @@ #include "runtime/decimalv2_value.h" -#include +#include + +#include +#include #include #include diff --git a/be/src/runtime/decimalv2_value.h b/be/src/runtime/decimalv2_value.h index ddce791e45..4a6e2cb6cf 100644 --- a/be/src/runtime/decimalv2_value.h +++ b/be/src/runtime/decimalv2_value.h @@ -17,16 +17,16 @@ #pragma once -#include -#include +#include +#include + +// IWYU pragma: no_include +#include // IWYU pragma: keep #include -#include #include -#include #include #include -#include "udf/udf.h" #include "util/hash_util.hpp" namespace doris { diff --git a/be/src/runtime/descriptor_helper.h b/be/src/runtime/descriptor_helper.h index ba3f17c57a..15c0d68599 100644 --- a/be/src/runtime/descriptor_helper.h +++ b/be/src/runtime/descriptor_helper.h @@ -17,10 +17,11 @@ #pragma once +#include +#include + #include -#include "gen_cpp/Descriptors_types.h" -#include "gen_cpp/Types_types.h" #include "runtime/define_primitive_type.h" #include "runtime/primitive_type.h" diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp index 1658a3bf6c..c13e2b53d5 100644 --- a/be/src/runtime/descriptors.cpp +++ b/be/src/runtime/descriptors.cpp @@ -20,19 +20,20 @@ #include "runtime/descriptors.h" +#include +#include #include +#include +#include +#include #include -#include -#include +#include #include "common/object_pool.h" -#include "gen_cpp/Descriptors_types.h" -#include "gen_cpp/descriptors.pb.h" #include "util/string_util.h" -#include "vec/columns/column_nullable.h" +#include "vec/aggregate_functions/aggregate_function.h" #include "vec/data_types/data_type_factory.hpp" -#include "vec/data_types/data_type_nullable.h" namespace doris { using boost::algorithm::join; diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index e03aaac7f4..8fd6654e13 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -20,38 +20,37 @@ #pragma once -#include -#include +#include +#include +#include +#include +#include #include +#include #include +#include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/global_types.h" #include "common/status.h" -#include "gen_cpp/Descriptors_types.h" // for TTupleId -#include "gen_cpp/FrontendService_types.h" // for TTupleId -#include "gen_cpp/Types_types.h" #include "runtime/types.h" #include "vec/data_types/data_type.h" -namespace doris::vectorized { -struct ColumnWithTypeAndName; -} +namespace google { +namespace protobuf { +template +class RepeatedField; +} // namespace protobuf +} // namespace google namespace doris { class ObjectPool; -class TDescriptorTable; -class TSlotDescriptor; -class TTupleDescriptor; -class RuntimeState; -class SchemaScanner; -class OlapTableSchemaParam; class PTupleDescriptor; class PSlotDescriptor; -class PInternalServiceImpl; -class TabletSchema; // Location information for null indicator bit for particular slot. // For non-nullable slots, the byte_offset will be 0 and the bit_mask will be 0. @@ -80,8 +79,6 @@ struct NullIndicatorOffset { std::ostream& operator<<(std::ostream& os, const NullIndicatorOffset& null_indicator); -class TupleDescriptor; - class SlotDescriptor { public: // virtual ~SlotDescriptor() {}; diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp index bde3de101d..8f63d6fe6a 100644 --- a/be/src/runtime/exec_env.cpp +++ b/be/src/runtime/exec_env.cpp @@ -17,7 +17,7 @@ #include "runtime/exec_env.h" -#include "gen_cpp/HeartbeatService_types.h" +#include namespace doris { diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 6862f191db..cc9cecad59 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -17,11 +17,14 @@ #pragma once -#include +#include +#include +#include +#include #include +#include -#include "common/config.h" #include "common/status.h" #include "olap/options.h" #include "util/threadpool.h" @@ -36,12 +39,8 @@ class TaskScheduler; } class BfdParser; class BrokerMgr; - template class BrpcClientCache; - -class DataStreamMgr; -class EvHttpServer; class ExternalScanContextMgr; class FragmentMgr; class ResultCache; @@ -50,28 +49,22 @@ class NewLoadStreamMgr; class MemTrackerLimiter; class MemTracker; class StorageEngine; -class PriorityThreadPool; -class PriorityWorkStealingThreadPool; class ResultBufferMgr; class ResultQueueMgr; class TMasterInfo; class LoadChannelMgr; -class WebPageHandler; class StreamLoadExecutor; class RoutineLoadTaskExecutor; class SmallFileMgr; -class StoragePolicyMgr; class BlockSpillManager; - class BackendServiceClient; class TPaloBrokerServiceClient; class PBackendService_Stub; class PFunctionService_Stub; - template class ClientCache; - class HeartbeatFlags; +class FrontendServiceClient; // Execution environment for queries/plan fragments. // Contains all required global structures, and handles to diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index e1c7cf39a9..b69b388ee1 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -15,15 +15,33 @@ // specific language governing permissions and limitations // under the License. +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + #include "common/config.h" #include "common/logging.h" -#include "gen_cpp/BackendService.h" -#include "gen_cpp/HeartbeatService_types.h" -#include "gen_cpp/TPaloBrokerService.h" +#include "common/status.h" +#include "olap/olap_define.h" +#include "olap/options.h" #include "olap/page_cache.h" #include "olap/rowset/segment_v2/inverted_index_cache.h" #include "olap/segment_loader.h" -#include "olap/storage_engine.h" +#include "pipeline/task_queue.h" #include "pipeline/task_scheduler.h" #include "runtime/block_spill_manager.h" #include "runtime/broker_mgr.h" @@ -31,28 +49,32 @@ #include "runtime/client_cache.h" #include "runtime/exec_env.h" #include "runtime/external_scan_context_mgr.h" -#include "runtime/fold_constant_executor.h" #include "runtime/fragment_mgr.h" #include "runtime/heartbeat_flags.h" #include "runtime/load_channel_mgr.h" #include "runtime/load_path_mgr.h" +#include "runtime/memory/chunk_allocator.h" #include "runtime/memory/mem_tracker.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" #include "runtime/result_buffer_mgr.h" #include "runtime/result_queue_mgr.h" #include "runtime/routine_load/routine_load_task_executor.h" #include "runtime/small_file_mgr.h" #include "runtime/stream_load/new_load_stream_mgr.h" #include "runtime/stream_load/stream_load_executor.h" +#include "runtime/thread_context.h" #include "service/point_query_executor.h" #include "util/bfd_parser.h" +#include "util/bit_util.h" #include "util/brpc_client_cache.h" +#include "util/cpu_info.h" #include "util/doris_metrics.h" #include "util/mem_info.h" #include "util/metrics.h" #include "util/parse_util.h" #include "util/pretty_printer.h" -#include "util/priority_thread_pool.hpp" -#include "util/priority_work_stealing_thread_pool.hpp" +#include "util/threadpool.h" #include "vec/exec/scan/scanner_scheduler.h" #include "vec/runtime/vdata_stream_mgr.h" @@ -62,6 +84,8 @@ #endif namespace doris { +class PBackendService_Stub; +class PFunctionService_Stub; DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(scanner_thread_pool_queue_size, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(send_batch_thread_pool_thread_num, MetricUnit::NOUNIT); diff --git a/be/src/runtime/export_task_mgr.h b/be/src/runtime/export_task_mgr.h index 51ea5a2d9c..90dd2017cb 100644 --- a/be/src/runtime/export_task_mgr.h +++ b/be/src/runtime/export_task_mgr.h @@ -17,12 +17,13 @@ #pragma once +#include + #include #include #include #include "common/status.h" -#include "gen_cpp/Types_types.h" #include "runtime/runtime_state.h" #include "util/lru_cache.hpp" diff --git a/be/src/runtime/external_scan_context_mgr.cpp b/be/src/runtime/external_scan_context_mgr.cpp index 8fa2183eba..9bb095d9a6 100644 --- a/be/src/runtime/external_scan_context_mgr.cpp +++ b/be/src/runtime/external_scan_context_mgr.cpp @@ -17,13 +17,19 @@ #include "runtime/external_scan_context_mgr.h" -#include -#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include #include +#include "common/config.h" +#include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/result_queue_mgr.h" #include "util/doris_metrics.h" +#include "util/metrics.h" +#include "util/thread.h" #include "util/uid_util.h" namespace doris { diff --git a/be/src/runtime/external_scan_context_mgr.h b/be/src/runtime/external_scan_context_mgr.h index 98ec60c4fe..93ae8d360a 100644 --- a/be/src/runtime/external_scan_context_mgr.h +++ b/be/src/runtime/external_scan_context_mgr.h @@ -17,23 +17,23 @@ #pragma once +#include +#include #include #include #include #include #include -#include #include #include "common/status.h" -#include "gen_cpp/Types_types.h" #include "gutil/ref_counted.h" -#include "runtime/exec_env.h" #include "util/countdown_latch.h" -#include "util/thread.h" namespace doris { +class ExecEnv; +class Thread; struct ScanContext { public: diff --git a/be/src/runtime/fold_constant_executor.cpp b/be/src/runtime/fold_constant_executor.cpp index aebda95c9f..2beb14b266 100644 --- a/be/src/runtime/fold_constant_executor.cpp +++ b/be/src/runtime/fold_constant_executor.cpp @@ -17,22 +17,47 @@ #include "runtime/fold_constant_executor.h" -#include +#include +#include +#include +#include +#include +#include +#include +#include -#include "common/object_pool.h" +#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" -#include "gen_cpp/PaloInternalService_types.h" -#include "gen_cpp/internal_service.pb.h" +#include "runtime/datetime_value.h" +#include "runtime/decimalv2_value.h" #include "runtime/define_primitive_type.h" +#include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/large_int_value.h" #include "runtime/memory/mem_tracker.h" +#include "runtime/primitive_type.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" +#include "util/binary_cast.hpp" +#include "util/defer_op.h" +#include "util/runtime_profile.h" +#include "vec/columns/column.h" +#include "vec/columns/column_vector.h" +#include "vec/columns/columns_number.h" #include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" #include "vec/data_types/data_type_number.h" #include "vec/exprs/vexpr.h" #include "vec/exprs/vexpr_context.h" +#include "vec/runtime/vdatetime_value.h" using std::string; using std::map; diff --git a/be/src/runtime/fold_constant_executor.h b/be/src/runtime/fold_constant_executor.h index edc1e4ada3..4a2ce4e96e 100644 --- a/be/src/runtime/fold_constant_executor.h +++ b/be/src/runtime/fold_constant_executor.h @@ -17,21 +17,27 @@ #pragma once +#include +#include + +#include +#include + #include "common/object_pool.h" #include "common/status.h" -#include "gen_cpp/PaloInternalService_types.h" -#include "gen_cpp/internal_service.pb.h" -#include "runtime/define_primitive_type.h" -#include "runtime/exec_env.h" +#include "runtime/memory/mem_tracker.h" #include "runtime/runtime_state.h" -#include "util/runtime_profile.h" +#include "runtime/types.h" +#include "vec/aggregate_functions/aggregate_function.h" #include "vec/data_types/data_type.h" namespace doris { class TFoldConstantParams; -class TExpr; class TQueryGlobals; +class PConstantExprResult; +class RuntimeProfile; +class TQueryOptions; // This class used to fold constant expr from fe class FoldConstantExecutor { diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 07ef9f3c0a..a113fa3792 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -18,44 +18,75 @@ #include "runtime/fragment_mgr.h" #include +#include +#include +#include +#include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include #include #include +#include +#include "common/config.h" +#include "common/logging.h" #include "common/object_pool.h" -#include "gen_cpp/FrontendService.h" -#include "gen_cpp/PaloInternalService_types.h" -#include "gen_cpp/PlanNodes_types.h" -#include "gen_cpp/QueryPlanExtra_types.h" -#include "gen_cpp/Types_types.h" +#include "common/utils.h" #include "gutil/strings/substitute.h" #include "io/fs/stream_load_pipe.h" #include "opentelemetry/trace/scope.h" #include "pipeline/pipeline_fragment_context.h" #include "pipeline/task_scheduler.h" #include "runtime/client_cache.h" -#include "runtime/datetime_value.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/plan_fragment_executor.h" +#include "runtime/primitive_type.h" +#include "runtime/query_fragments_ctx.h" #include "runtime/runtime_filter_mgr.h" +#include "runtime/runtime_state.h" #include "runtime/stream_load/new_load_stream_mgr.h" #include "runtime/stream_load/stream_load_context.h" +#include "runtime/stream_load/stream_load_executor.h" +#include "runtime/task_group/task_group.h" #include "runtime/task_group/task_group_manager.h" #include "runtime/thread_context.h" +#include "runtime/types.h" #include "service/backend_options.h" #include "util/doris_metrics.h" +#include "util/hash_util.hpp" #include "util/mem_info.h" #include "util/network_util.h" -#include "util/stopwatch.hpp" +#include "util/pretty_printer.h" +#include "util/runtime_profile.h" #include "util/telemetry/telemetry.h" +#include "util/thread.h" #include "util/threadpool.h" #include "util/thrift_util.h" #include "util/uid_util.h" #include "util/url_coding.h" +#include "vec/runtime/shared_hash_table_controller.h" +#include "vec/runtime/vdatetime_value.h" namespace doris { @@ -78,7 +109,6 @@ std::string to_load_error_http_path(const std::string& file_name) { using apache::thrift::TException; using apache::thrift::transport::TTransportException; -class RuntimeProfile; class FragmentExecState { public: using report_status_callback_impl = std::function; diff --git a/be/src/runtime/fragment_mgr.h b/be/src/runtime/fragment_mgr.h index 698986f13b..d5a724787e 100644 --- a/be/src/runtime/fragment_mgr.h +++ b/be/src/runtime/fragment_mgr.h @@ -17,23 +17,26 @@ #pragma once +#include +#include +#include + +#include #include +#include #include #include -#include +#include #include #include #include "common/status.h" -#include "gen_cpp/DorisExternalService_types.h" -#include "gen_cpp/Types_types.h" -#include "gen_cpp/internal_service.pb.h" #include "gutil/ref_counted.h" #include "http/rest_monitor_iface.h" #include "runtime_filter_mgr.h" #include "util/countdown_latch.h" +#include "util/hash_util.hpp" // IWYU pragma: keep #include "util/metrics.h" -#include "util/thread.h" namespace butil { class IOBufAsZeroCopyInputStream; @@ -44,20 +47,21 @@ namespace doris { namespace pipeline { class PipelineFragmentContext; } - -namespace io { -class StreamLoadPipe; -} - class QueryFragmentsCtx; class ExecEnv; class FragmentExecState; -class PlanFragmentExecutor; class ThreadPool; class TExecPlanFragmentParams; -class TExecPlanFragmentParamsList; -class TUniqueId; -class RuntimeFilterMergeController; +class PExecPlanFragmentStartRequest; +class PMergeFilterRequest; +class PPublishFilterRequest; +class RuntimeProfile; +class RuntimeState; +class TPipelineFragmentParams; +class TPipelineInstanceParams; +class TScanColumnDesc; +class TScanOpenParams; +class Thread; std::string to_load_error_http_path(const std::string& file_name); diff --git a/be/src/runtime/heartbeat_flags.h b/be/src/runtime/heartbeat_flags.h index 7412265096..77c6b942ec 100644 --- a/be/src/runtime/heartbeat_flags.h +++ b/be/src/runtime/heartbeat_flags.h @@ -17,9 +17,9 @@ #pragma once -#include +#include -#include "gen_cpp/HeartbeatService_constants.h" +#include namespace doris { diff --git a/be/src/runtime/jsonb_value.cpp b/be/src/runtime/jsonb_value.cpp index 93abcce640..62ab9102c5 100644 --- a/be/src/runtime/jsonb_value.cpp +++ b/be/src/runtime/jsonb_value.cpp @@ -17,9 +17,12 @@ #include "runtime/jsonb_value.h" -#include +#include #include "util/jsonb_error.h" +#include "util/jsonb_stream.h" +#include "util/jsonb_utils.h" +#include "util/jsonb_writer.h" namespace doris { diff --git a/be/src/runtime/jsonb_value.h b/be/src/runtime/jsonb_value.h index 997bca3e05..88abb379d9 100644 --- a/be/src/runtime/jsonb_value.h +++ b/be/src/runtime/jsonb_value.h @@ -18,14 +18,15 @@ #ifndef DORIS_BE_RUNTIME_JSON_VALUE_H #define DORIS_BE_RUNTIME_JSON_VALUE_H -#include "udf/udf.h" -#include "util/cpu_info.h" +#include + +#include +#include +#include + +#include "common/status.h" #include "util/hash_util.hpp" -#include "util/jsonb_error.h" #include "util/jsonb_parser_simd.h" -#include "util/jsonb_utils.h" -#include "util/sse_util.hpp" -#include "vec/common/string_ref.h" namespace doris { diff --git a/be/src/runtime/large_int_value.cpp b/be/src/runtime/large_int_value.cpp index f08eff26c8..b9f354b823 100644 --- a/be/src/runtime/large_int_value.cpp +++ b/be/src/runtime/large_int_value.cpp @@ -19,6 +19,7 @@ #include +#include "util/hash_util.hpp" #include "util/string_parser.hpp" namespace doris { diff --git a/be/src/runtime/large_int_value.h b/be/src/runtime/large_int_value.h index cd9fb0de53..7fa2e558ed 100644 --- a/be/src/runtime/large_int_value.h +++ b/be/src/runtime/large_int_value.h @@ -18,12 +18,10 @@ #pragma once #include -#include -#include -#include +#include +#include #include -#include #include namespace doris { diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp index de3d8d3365..3df55c9eff 100644 --- a/be/src/runtime/load_channel.cpp +++ b/be/src/runtime/load_channel.cpp @@ -17,11 +17,11 @@ #include "runtime/load_channel.h" -#include "olap/lru_cache.h" -#include "runtime/exec_env.h" +#include +#include + #include "runtime/memory/mem_tracker.h" #include "runtime/tablets_channel.h" -#include "runtime/thread_context.h" namespace doris { diff --git a/be/src/runtime/load_channel.h b/be/src/runtime/load_channel.h index 2fad48c79d..2abf822121 100644 --- a/be/src/runtime/load_channel.h +++ b/be/src/runtime/load_channel.h @@ -17,22 +17,31 @@ #pragma once +#include +#include + +#include +#include #include #include +#include #include #include +#include #include #include +#include +#include #include "common/status.h" -#include "gen_cpp/internal_service.pb.h" #include "runtime/memory/mem_tracker.h" #include "runtime/tablets_channel.h" +#include "util/spinlock.h" #include "util/uid_util.h" namespace doris { -class Cache; +class PTabletWriterOpenRequest; // A LoadChannel manages tablets channels for all indexes // corresponding to a certain load job diff --git a/be/src/runtime/load_channel_mgr.cpp b/be/src/runtime/load_channel_mgr.cpp index 413d7077cf..f280e87abc 100644 --- a/be/src/runtime/load_channel_mgr.cpp +++ b/be/src/runtime/load_channel_mgr.cpp @@ -17,20 +17,33 @@ #include "runtime/load_channel_mgr.h" +#include +#include + +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include #include #include #include +#include #include +#include #include #include +#include "common/config.h" +#include "common/logging.h" +#include "runtime/exec_env.h" #include "runtime/load_channel.h" #include "runtime/memory/mem_tracker.h" #include "util/doris_metrics.h" #include "util/mem_info.h" +#include "util/metrics.h" #include "util/perf_counters.h" -#include "util/stopwatch.hpp" -#include "util/time.h" +#include "util/pretty_printer.h" +#include "util/thread.h" namespace doris { diff --git a/be/src/runtime/load_channel_mgr.h b/be/src/runtime/load_channel_mgr.h index a825b74bca..69112724b5 100644 --- a/be/src/runtime/load_channel_mgr.h +++ b/be/src/runtime/load_channel_mgr.h @@ -17,24 +17,30 @@ #pragma once -#include +#include + +#include #include #include -#include #include +#include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" -#include "gen_cpp/internal_service.pb.h" #include "gutil/ref_counted.h" #include "olap/lru_cache.h" #include "runtime/load_channel.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/thread_context.h" #include "util/countdown_latch.h" -#include "util/thread.h" #include "util/uid_util.h" namespace doris { -class Cache; +class PTabletWriterCancelRequest; +class PTabletWriterOpenRequest; +class Thread; // LoadChannelMgr -> LoadChannel -> TabletsChannel -> DeltaWriter // All dispatched load data for this backend is routed from this class diff --git a/be/src/runtime/load_path_mgr.cpp b/be/src/runtime/load_path_mgr.cpp index 239aaffcd9..c9a5f18183 100644 --- a/be/src/runtime/load_path_mgr.cpp +++ b/be/src/runtime/load_path_mgr.cpp @@ -17,18 +17,31 @@ #include "runtime/load_path_mgr.h" +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include #include -#include -#include +#include #include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include #include -#include "gen_cpp/Types_types.h" +#include "common/config.h" +#include "io/fs/file_system.h" #include "io/fs/local_file_system.h" #include "olap/olap_define.h" -#include "olap/storage_engine.h" +#include "olap/options.h" #include "runtime/exec_env.h" +#include "util/thread.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/runtime/load_path_mgr.h b/be/src/runtime/load_path_mgr.h index 49a9457bb1..73703517c6 100644 --- a/be/src/runtime/load_path_mgr.h +++ b/be/src/runtime/load_path_mgr.h @@ -17,7 +17,8 @@ #pragma once -#include +#include +#include #include #include @@ -25,13 +26,13 @@ #include "common/status.h" #include "gutil/ref_counted.h" -#include "util/thread.h" -#include "util/uid_util.h" +#include "util/countdown_latch.h" namespace doris { class TUniqueId; class ExecEnv; +class Thread; // In every directory, '.trash' directory is used to save data need to delete // daemon thread is check no used directory to delete diff --git a/be/src/runtime/map_value.h b/be/src/runtime/map_value.h index d275316cf6..26c46692bc 100644 --- a/be/src/runtime/map_value.h +++ b/be/src/runtime/map_value.h @@ -17,9 +17,7 @@ #pragma once -#include - -#include "runtime/primitive_type.h" +#include namespace doris { diff --git a/be/src/runtime/memory/chunk_allocator.cpp b/be/src/runtime/memory/chunk_allocator.cpp index a99b8b0873..3c8a8d0946 100644 --- a/be/src/runtime/memory/chunk_allocator.cpp +++ b/be/src/runtime/memory/chunk_allocator.cpp @@ -17,18 +17,21 @@ #include "runtime/memory/chunk_allocator.h" +#include #include +#include -#include #include +#include "common/config.h" +#include "common/status.h" #include "runtime/memory/chunk.h" -#include "runtime/memory/mem_tracker.h" #include "runtime/memory/system_allocator.h" #include "runtime/thread_context.h" #include "util/bit_util.h" #include "util/cpu_info.h" #include "util/doris_metrics.h" +#include "util/metrics.h" #include "util/runtime_profile.h" #include "util/spinlock.h" diff --git a/be/src/runtime/memory/chunk_allocator.h b/be/src/runtime/memory/chunk_allocator.h index 8602815a57..467317c191 100644 --- a/be/src/runtime/memory/chunk_allocator.h +++ b/be/src/runtime/memory/chunk_allocator.h @@ -23,12 +23,13 @@ #include #include +#include "runtime/memory/mem_tracker_limiter.h" + namespace doris { struct Chunk; class ChunkArena; class MetricEntity; -class MemTrackerLimiter; class Status; // Used to allocate memory with power-of-two length. diff --git a/be/src/runtime/memory/jemalloc_hook.cpp b/be/src/runtime/memory/jemalloc_hook.cpp index bfc358c827..ec40879e4b 100644 --- a/be/src/runtime/memory/jemalloc_hook.cpp +++ b/be/src/runtime/memory/jemalloc_hook.cpp @@ -15,8 +15,15 @@ // specific language governing permissions and limitations // under the License. +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "jemalloc/jemalloc.h" #include "runtime/thread_context.h" +#include "util/sse_util.hpp" #ifndef __THROW #if __cplusplus diff --git a/be/src/runtime/memory/mem_tracker.cpp b/be/src/runtime/memory/mem_tracker.cpp index 78cb46d3bc..75e59385c2 100644 --- a/be/src/runtime/memory/mem_tracker.cpp +++ b/be/src/runtime/memory/mem_tracker.cpp @@ -22,9 +22,10 @@ #include +#include + +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" -#include "util/string_util.h" -#include "util/time.h" namespace doris { diff --git a/be/src/runtime/memory/mem_tracker.h b/be/src/runtime/memory/mem_tracker.h index 8553ce834e..60ac996a82 100644 --- a/be/src/runtime/memory/mem_tracker.h +++ b/be/src/runtime/memory/mem_tracker.h @@ -19,6 +19,20 @@ // and modified by Doris #pragma once +#include +#include + +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "util/pretty_printer.h" #include "util/runtime_profile.h" @@ -159,4 +173,4 @@ protected: std::list::iterator _tracker_group_it; }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp index 509f190358..d9974bef7f 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.cpp +++ b/be/src/runtime/memory/mem_tracker_limiter.cpp @@ -18,13 +18,17 @@ #include "runtime/memory/mem_tracker_limiter.h" #include +#include +#include -#include +#include +#include #include +#include +#include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/load_channel_mgr.h" -#include "runtime/runtime_state.h" #include "service/backend_options.h" #include "util/mem_info.h" #include "util/perf_counters.h" diff --git a/be/src/runtime/memory/mem_tracker_limiter.h b/be/src/runtime/memory/mem_tracker_limiter.h index e1e2dd0b14..7665a029f8 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.h +++ b/be/src/runtime/memory/mem_tracker_limiter.h @@ -17,16 +17,30 @@ #pragma once +#include +#include +#include +#include + #include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include #include "common/config.h" +#include "common/status.h" #include "runtime/memory/mem_tracker.h" +#include "util/runtime_profile.h" #include "util/string_util.h" +#include "util/uid_util.h" namespace doris { -class RuntimeState; - // Track and limit the memory usage of process and query. // Contains an limit, arranged into a tree structure. // diff --git a/be/src/runtime/memory/system_allocator.cpp b/be/src/runtime/memory/system_allocator.cpp index 34e379d9f1..fc72213cda 100644 --- a/be/src/runtime/memory/system_allocator.cpp +++ b/be/src/runtime/memory/system_allocator.cpp @@ -17,13 +17,17 @@ #include "runtime/memory/system_allocator.h" +#include #include #include -#include -#include "common/config.h" +#include +#include + #include "common/logging.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/thread_context.h" +#include "util/sse_util.hpp" namespace doris { diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp index b166163227..f0ffaebaad 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp @@ -17,12 +17,10 @@ #include "runtime/memory/thread_mem_tracker_mgr.h" -#include -#include +#include #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" -#include "service/backend_options.h" namespace doris { diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h b/be/src/runtime/memory/thread_mem_tracker_mgr.h index 3665ec3d53..1192bdb2a1 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.h +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h @@ -17,10 +17,18 @@ #pragma once -#include #include +#include +#include +#include -#include "gutil/macros.h" +#include +#include +#include +#include +#include + +#include "common/config.h" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker.h" #include "runtime/memory/mem_tracker_limiter.h" diff --git a/be/src/runtime/message_body_sink.cpp b/be/src/runtime/message_body_sink.cpp index 62aa309c24..36a498d27d 100644 --- a/be/src/runtime/message_body_sink.cpp +++ b/be/src/runtime/message_body_sink.cpp @@ -17,11 +17,14 @@ #include "runtime/message_body_sink.h" +// IWYU pragma: no_include +#include // IWYU pragma: keep #include +#include +#include +#include -#include - -#include "util/runtime_profile.h" +#include namespace doris { diff --git a/be/src/runtime/message_body_sink.h b/be/src/runtime/message_body_sink.h index 2c4df30976..a396cc3648 100644 --- a/be/src/runtime/message_body_sink.h +++ b/be/src/runtime/message_body_sink.h @@ -17,6 +17,11 @@ #pragma once +#include + +#include +#include + #include "common/status.h" #include "util/byte_buffer.h" diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp index 701a22601e..10747ea7d3 100644 --- a/be/src/runtime/plan_fragment_executor.cpp +++ b/be/src/runtime/plan_fragment_executor.cpp @@ -20,18 +20,33 @@ #include "runtime/plan_fragment_executor.h" +#include +#include +#include #include -#include - -#include +#include +#include +#include +#include +#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include "common/config.h" +#include "common/logging.h" #include "exec/data_sink.h" #include "exec/exec_node.h" #include "exec/scan_node.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" -#include "runtime/memory/mem_tracker.h" -#include "runtime/result_buffer_mgr.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/query_fragments_ctx.h" +#include "runtime/query_statistics.h" #include "runtime/result_queue_mgr.h" #include "runtime/runtime_filter_mgr.h" #include "runtime/thread_context.h" @@ -39,6 +54,8 @@ #include "util/defer_op.h" #include "util/pretty_printer.h" #include "util/telemetry/telemetry.h" +#include "util/threadpool.h" +#include "util/time.h" #include "util/uid_util.h" #include "vec/core/block.h" #include "vec/exec/scan/new_es_scan_node.h" @@ -47,6 +64,7 @@ #include "vec/exec/scan/new_odbc_scan_node.h" #include "vec/exec/scan/new_olap_scan_node.h" #include "vec/exec/scan/vmeta_scan_node.h" +#include "vec/exec/scan/vscan_node.h" #include "vec/exec/vexchange_node.h" #include "vec/runtime/vdata_stream_mgr.h" diff --git a/be/src/runtime/plan_fragment_executor.h b/be/src/runtime/plan_fragment_executor.h index 46521004fb..61f03b624d 100644 --- a/be/src/runtime/plan_fragment_executor.h +++ b/be/src/runtime/plan_fragment_executor.h @@ -20,17 +20,22 @@ #pragma once +#include +#include +#include + #include #include #include +#include +#include +#include +#include #include -#include "common/object_pool.h" #include "common/status.h" -#include "runtime/query_fragments_ctx.h" -#include "runtime/query_statistics.h" #include "runtime/runtime_state.h" -#include "vec/core/block.h" +#include "util/runtime_profile.h" namespace doris { @@ -38,14 +43,14 @@ class QueryFragmentsCtx; class ExecNode; class RowDescriptor; class DataSink; -class DataStreamMgr; -class RuntimeProfile; -class RuntimeState; -class TNetworkAddress; -class TPlanExecRequest; -class TPlanFragment; -class TPlanFragmentExecParams; -class TPlanExecParams; +class DescriptorTbl; +class ExecEnv; +class ObjectPool; +class QueryStatistics; + +namespace vectorized { +class Block; +} // namespace vectorized // PlanFragmentExecutor handles all aspects of the execution of a single plan fragment, // including setup and tear-down, both in the success and error case. diff --git a/be/src/runtime/primitive_type.cpp b/be/src/runtime/primitive_type.cpp index 6c3e9fb5f1..8f07934a91 100644 --- a/be/src/runtime/primitive_type.cpp +++ b/be/src/runtime/primitive_type.cpp @@ -17,13 +17,11 @@ #include "runtime/primitive_type.h" -#include "gen_cpp/Types_types.h" -#include "runtime/collection_value.h" +#include + +#include + #include "runtime/define_primitive_type.h" -#include "runtime/jsonb_value.h" -#include "runtime/map_value.h" -#include "runtime/struct_value.h" -#include "vec/common/string_ref.h" namespace doris { diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h index d261aa4582..dfc121c330 100644 --- a/be/src/runtime/primitive_type.h +++ b/be/src/runtime/primitive_type.h @@ -17,13 +17,20 @@ #pragma once +#include +#include +#include +#include + #include -#include "gen_cpp/PaloInternalService_types.h" +#include "olap/decimal12.h" #include "runtime/define_primitive_type.h" #include "vec/columns/column_decimal.h" +#include "vec/columns/column_vector.h" #include "vec/columns/columns_number.h" #include "vec/core/types.h" +#include "vec/runtime/vdatetime_value.h" namespace doris { diff --git a/be/src/runtime/query_fragments_ctx.h b/be/src/runtime/query_fragments_ctx.h index bbf5be6890..a5ff251dce 100644 --- a/be/src/runtime/query_fragments_ctx.h +++ b/be/src/runtime/query_fragments_ctx.h @@ -17,14 +17,15 @@ #pragma once +#include +#include + #include #include #include #include "common/config.h" #include "common/object_pool.h" -#include "gen_cpp/PaloInternalService_types.h" // for TQueryOptions -#include "gen_cpp/Types_types.h" // for TUniqueId #include "runtime/datetime_value.h" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker_limiter.h" diff --git a/be/src/runtime/query_statistics.cpp b/be/src/runtime/query_statistics.cpp index 4ef8e4a7ce..f9bd7867f4 100644 --- a/be/src/runtime/query_statistics.cpp +++ b/be/src/runtime/query_statistics.cpp @@ -17,6 +17,7 @@ #include "runtime/query_statistics.h" +#include #include namespace doris { diff --git a/be/src/runtime/query_statistics.h b/be/src/runtime/query_statistics.h index c1b2e4f028..6b5127cf4f 100644 --- a/be/src/runtime/query_statistics.h +++ b/be/src/runtime/query_statistics.h @@ -17,15 +17,20 @@ #pragma once -#include +#include + +#include +#include +#include +#include -#include "gen_cpp/data.pb.h" #include "util/spinlock.h" namespace doris { -class QueryStatistics; class QueryStatisticsRecvr; +class PNodeStatistics; +class PQueryStatistics; class NodeStatistics { public: diff --git a/be/src/runtime/record_batch_queue.cpp b/be/src/runtime/record_batch_queue.cpp index 46cac2e316..1f76102bb9 100644 --- a/be/src/runtime/record_batch_queue.cpp +++ b/be/src/runtime/record_batch_queue.cpp @@ -17,6 +17,8 @@ #include "runtime/record_batch_queue.h" +#include "util/spinlock.h" + namespace doris { void RecordBatchQueue::update_status(const Status& status) { diff --git a/be/src/runtime/record_batch_queue.h b/be/src/runtime/record_batch_queue.h index 008b8f10d3..7ababc9b61 100644 --- a/be/src/runtime/record_batch_queue.h +++ b/be/src/runtime/record_batch_queue.h @@ -17,8 +17,12 @@ #pragma once +#include #include +#include +#include + #include "common/status.h" #include "util/blocking_queue.hpp" diff --git a/be/src/runtime/result_buffer_mgr.cpp b/be/src/runtime/result_buffer_mgr.cpp index 02229efe23..14e844bfeb 100644 --- a/be/src/runtime/result_buffer_mgr.cpp +++ b/be/src/runtime/result_buffer_mgr.cpp @@ -17,12 +17,20 @@ #include "runtime/result_buffer_mgr.h" +#include +#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include +#include +#include -#include "gen_cpp/PaloInternalService_types.h" -#include "gen_cpp/types.pb.h" #include "runtime/buffer_control_block.h" #include "util/doris_metrics.h" +#include "util/metrics.h" +#include "util/thread.h" namespace doris { diff --git a/be/src/runtime/result_buffer_mgr.h b/be/src/runtime/result_buffer_mgr.h index 250551f42a..d8cf8e5787 100644 --- a/be/src/runtime/result_buffer_mgr.h +++ b/be/src/runtime/result_buffer_mgr.h @@ -17,24 +17,26 @@ #pragma once +#include + +#include #include +#include #include -#include #include #include #include "common/status.h" -#include "gen_cpp/Types_types.h" #include "gutil/ref_counted.h" +#include "util/countdown_latch.h" #include "util/hash_util.hpp" -#include "util/thread.h" namespace doris { -class TFetchDataResult; class BufferControlBlock; struct GetResultBatchCtx; class PUniqueId; +class Thread; // manage all result buffer control block in one backend class ResultBufferMgr { diff --git a/be/src/runtime/result_queue_mgr.cpp b/be/src/runtime/result_queue_mgr.cpp index 4f30f91345..8090a3e6ee 100644 --- a/be/src/runtime/result_queue_mgr.cpp +++ b/be/src/runtime/result_queue_mgr.cpp @@ -17,11 +17,16 @@ #include "runtime/result_queue_mgr.h" +#include + +#include + #include "common/config.h" #include "common/status.h" -#include "gen_cpp/Types_types.h" -#include "runtime/exec_env.h" +#include "runtime/record_batch_queue.h" #include "util/doris_metrics.h" +#include "util/hash_util.hpp" +#include "util/metrics.h" namespace doris { diff --git a/be/src/runtime/result_queue_mgr.h b/be/src/runtime/result_queue_mgr.h index 28dcabfaa5..d7e9f66898 100644 --- a/be/src/runtime/result_queue_mgr.h +++ b/be/src/runtime/result_queue_mgr.h @@ -17,15 +17,14 @@ #pragma once -#include +#include + #include #include #include #include "common/status.h" -#include "runtime/primitive_type.h" -#include "runtime/record_batch_queue.h" -#include "util/hash_util.hpp" +#include "util/hash_util.hpp" // IWYU pragma: keep namespace arrow { @@ -34,10 +33,9 @@ class RecordBatch; namespace doris { -class TUniqueId; -class TScanRowBatch; class RecordBatchQueue; -typedef std::shared_ptr BlockQueueSharedPtr; + +using BlockQueueSharedPtr = std::shared_ptr; class ResultQueueMgr { public: diff --git a/be/src/runtime/result_writer.h b/be/src/runtime/result_writer.h index bd5a11cc8f..7298cce13b 100644 --- a/be/src/runtime/result_writer.h +++ b/be/src/runtime/result_writer.h @@ -17,18 +17,16 @@ #pragma once +#include + +#include + #include "common/status.h" -#include "gen_cpp/PlanNodes_types.h" namespace doris { -class Status; class RuntimeState; -namespace vectorized { -class Block; -} - // abstract class of the result writer class ResultWriter { public: diff --git a/be/src/runtime/routine_load/data_consumer.cpp b/be/src/runtime/routine_load/data_consumer.cpp index 88c00044e1..cdbff1c1d1 100644 --- a/be/src/runtime/routine_load/data_consumer.cpp +++ b/be/src/runtime/routine_load/data_consumer.cpp @@ -17,18 +17,28 @@ #include "runtime/routine_load/data_consumer.h" +#include +#include +#include + #include -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include +#include +#include #include +#include "common/config.h" #include "common/status.h" -#include "gen_cpp/internal_service.pb.h" #include "gutil/strings/split.h" +#include "runtime/exec_env.h" #include "runtime/small_file_mgr.h" #include "service/backend_options.h" +#include "util/blocking_queue.hpp" #include "util/defer_op.h" #include "util/stopwatch.hpp" +#include "util/string_util.h" #include "util/uid_util.h" namespace doris { diff --git a/be/src/runtime/routine_load/data_consumer.h b/be/src/runtime/routine_load/data_consumer.h index 4ad3d6821f..2a3f65f199 100644 --- a/be/src/runtime/routine_load/data_consumer.h +++ b/be/src/runtime/routine_load/data_consumer.h @@ -17,19 +17,27 @@ #pragma once -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/logging.h" +#include "common/status.h" #include "librdkafka/rdkafkacpp.h" #include "runtime/stream_load/stream_load_context.h" -#include "util/blocking_queue.hpp" #include "util/uid_util.h" namespace doris { -class KafkaConsumerPipe; -class Status; +template +class BlockingQueue; class DataConsumer { public: @@ -77,6 +85,7 @@ protected: }; class PIntegerPair; + class KafkaEventCb : public RdKafka::EventCb { public: void event_cb(RdKafka::Event& event) { diff --git a/be/src/runtime/routine_load/data_consumer_group.cpp b/be/src/runtime/routine_load/data_consumer_group.cpp index 2d22fa93ab..d27c86fdf0 100644 --- a/be/src/runtime/routine_load/data_consumer_group.cpp +++ b/be/src/runtime/routine_load/data_consumer_group.cpp @@ -16,11 +16,20 @@ // under the License. #include "runtime/routine_load/data_consumer_group.h" +#include +#include + +#include +#include +#include +#include + +#include "common/logging.h" #include "io/fs/kafka_consumer_pipe.h" -#include "librdkafka/rdkafka.h" #include "librdkafka/rdkafkacpp.h" #include "runtime/routine_load/data_consumer.h" #include "runtime/stream_load/stream_load_context.h" +#include "util/stopwatch.hpp" namespace doris { diff --git a/be/src/runtime/routine_load/data_consumer_group.h b/be/src/runtime/routine_load/data_consumer_group.h index bd866f0468..6ede3d9f66 100644 --- a/be/src/runtime/routine_load/data_consumer_group.h +++ b/be/src/runtime/routine_load/data_consumer_group.h @@ -17,11 +17,25 @@ #pragma once +#include + +#include +#include +#include +#include + +#include "common/status.h" #include "runtime/routine_load/data_consumer.h" #include "util/blocking_queue.hpp" #include "util/priority_thread_pool.hpp" +#include "util/uid_util.h" + +namespace RdKafka { +class Message; +} // namespace RdKafka namespace doris { +class StreamLoadContext; // data consumer group saves a group of data consumers. // These data consumers share the same stream load pipe. diff --git a/be/src/runtime/routine_load/data_consumer_pool.cpp b/be/src/runtime/routine_load/data_consumer_pool.cpp index 579edfd201..c79e25634d 100644 --- a/be/src/runtime/routine_load/data_consumer_pool.cpp +++ b/be/src/runtime/routine_load/data_consumer_pool.cpp @@ -17,8 +17,24 @@ #include "runtime/routine_load/data_consumer_pool.h" +#include + +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include + #include "common/config.h" +#include "common/logging.h" +#include "common/status.h" +#include "runtime/routine_load/data_consumer.h" #include "runtime/routine_load/data_consumer_group.h" +#include "runtime/stream_load/stream_load_context.h" +#include "util/uid_util.h" namespace doris { diff --git a/be/src/runtime/routine_load/data_consumer_pool.h b/be/src/runtime/routine_load/data_consumer_pool.h index 6f8cf79f74..15baa2be6e 100644 --- a/be/src/runtime/routine_load/data_consumer_pool.h +++ b/be/src/runtime/routine_load/data_consumer_pool.h @@ -17,14 +17,14 @@ #pragma once -#include +#include + +#include #include #include #include "gutil/ref_counted.h" -#include "runtime/routine_load/data_consumer.h" #include "util/countdown_latch.h" -#include "util/lru_cache.hpp" #include "util/thread.h" namespace doris { @@ -32,6 +32,7 @@ namespace doris { class DataConsumer; class DataConsumerGroup; class Status; +class StreamLoadContext; // DataConsumerPool saves all available data consumer // to be reused diff --git a/be/src/runtime/routine_load/routine_load_task_executor.cpp b/be/src/runtime/routine_load/routine_load_task_executor.cpp index 3aa361d157..4db6d42e18 100644 --- a/be/src/runtime/routine_load/routine_load_task_executor.cpp +++ b/be/src/runtime/routine_load/routine_load_task_executor.cpp @@ -17,18 +17,43 @@ #include "runtime/routine_load/routine_load_task_executor.h" -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "common/logging.h" #include "common/status.h" -#include "gen_cpp/BackendService_types.h" -#include "gen_cpp/FrontendService_types.h" -#include "gen_cpp/Types_types.h" +#include "common/utils.h" #include "io/fs/kafka_consumer_pipe.h" -#include "olap/iterators.h" +#include "io/fs/stream_load_pipe.h" #include "runtime/exec_env.h" +#include "runtime/message_body_sink.h" +#include "runtime/routine_load/data_consumer.h" #include "runtime/routine_load/data_consumer_group.h" +#include "runtime/stream_load/new_load_stream_mgr.h" #include "runtime/stream_load/stream_load_context.h" +#include "runtime/stream_load/stream_load_executor.h" +#include "service/backend_options.h" #include "util/defer_op.h" +#include "util/doris_metrics.h" +#include "util/metrics.h" +#include "util/slice.h" +#include "util/time.h" #include "util/uid_util.h" namespace doris { diff --git a/be/src/runtime/routine_load/routine_load_task_executor.h b/be/src/runtime/routine_load/routine_load_task_executor.h index 8b46e7bd8d..b2ddfae2b7 100644 --- a/be/src/runtime/routine_load/routine_load_task_executor.h +++ b/be/src/runtime/routine_load/routine_load_task_executor.h @@ -17,14 +17,16 @@ #pragma once +#include + #include -#include #include #include +#include +#include +#include -#include "gen_cpp/internal_service.pb.h" #include "runtime/routine_load/data_consumer_pool.h" -#include "util/doris_metrics.h" #include "util/priority_thread_pool.hpp" #include "util/uid_util.h" @@ -34,6 +36,8 @@ class ExecEnv; class Status; class StreamLoadContext; class TRoutineLoadTask; +class PIntegerPair; +class PKafkaMetaProxyRequest; // A routine load task executor will receive routine load // tasks from FE, put it to a fixed thread pool. diff --git a/be/src/runtime/runtime_filter_mgr.cpp b/be/src/runtime/runtime_filter_mgr.cpp index b3ffd54032..187948d1a5 100644 --- a/be/src/runtime/runtime_filter_mgr.cpp +++ b/be/src/runtime/runtime_filter_mgr.cpp @@ -17,16 +17,26 @@ #include "runtime/runtime_filter_mgr.h" -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/logging.h" #include "exprs/bloom_filter_func.h" #include "exprs/runtime_filter.h" -#include "gen_cpp/internal_service.pb.h" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" -#include "service/brpc.h" #include "util/brpc_client_cache.h" namespace doris { diff --git a/be/src/runtime/runtime_filter_mgr.h b/be/src/runtime/runtime_filter_mgr.h index 8d2a19a5ff..6bf9454e32 100644 --- a/be/src/runtime/runtime_filter_mgr.h +++ b/be/src/runtime/runtime_filter_mgr.h @@ -17,19 +17,22 @@ #pragma once -#include +#include +#include +#include +#include + +#include #include #include #include -#include +#include #include +#include +#include #include "common/object_pool.h" #include "common/status.h" -#include "exprs/runtime_filter.h" -#include "gen_cpp/PaloInternalService_types.h" -#include "gen_cpp/PlanNodes_types.h" -#include "runtime/runtime_state.h" #include "util/uid_util.h" namespace butil { @@ -37,12 +40,12 @@ class IOBufAsZeroCopyInputStream; } namespace doris { -class TUniqueId; -class RuntimeFilter; -class FragmentExecState; -class PlanFragmentExecutor; class PPublishFilterRequest; class PMergeFilterRequest; +class IRuntimeFilter; +class MemTracker; +class RuntimeState; +enum class RuntimeFilterRole; /// producer: /// Filter filter; diff --git a/be/src/runtime/runtime_predicate.cpp b/be/src/runtime/runtime_predicate.cpp index 52be66dfe3..d2ee35c67e 100644 --- a/be/src/runtime/runtime_predicate.cpp +++ b/be/src/runtime/runtime_predicate.cpp @@ -17,7 +17,12 @@ #include "runtime/runtime_predicate.h" +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "olap/accept_null_predicate.h" +#include "olap/column_predicate.h" #include "olap/predicate_creator.h" namespace doris { diff --git a/be/src/runtime/runtime_predicate.h b/be/src/runtime/runtime_predicate.h index 0d2b8e23c2..9dd48279ac 100644 --- a/be/src/runtime/runtime_predicate.h +++ b/be/src/runtime/runtime_predicate.h @@ -17,30 +17,28 @@ #pragma once -#include +#include +#include #include #include #include -#include "common/config.h" -#include "common/object_pool.h" +#include "common/status.h" #include "exec/olap_common.h" -#include "gen_cpp/PaloInternalService_types.h" // for TQueryOptions -#include "gen_cpp/Types_types.h" // for TUniqueId -#include "olap/column_predicate.h" #include "olap/tablet_schema.h" -#include "runtime/datetime_value.h" -#include "runtime/exec_env.h" -#include "util/threadpool.h" +#include "runtime/define_primitive_type.h" +#include "runtime/primitive_type.h" +#include "util/binary_cast.hpp" +#include "vec/common/arena.h" #include "vec/core/field.h" #include "vec/core/types.h" +#include "vec/runtime/vdatetime_value.h" namespace doris { +class ColumnPredicate; namespace vectorized { -class Arena; - class RuntimePredicate { public: RuntimePredicate() = default; diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 052daa7515..69a1a560d9 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -21,22 +21,24 @@ #include "runtime/runtime_state.h" #include +#include +#include -#include -#include #include +#include "common/config.h" #include "common/logging.h" #include "common/object_pool.h" #include "common/status.h" -#include "exec/exec_node.h" #include "runtime/exec_env.h" #include "runtime/load_path_mgr.h" -#include "runtime/memory/mem_tracker.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/memory/thread_mem_tracker_mgr.h" #include "runtime/runtime_filter_mgr.h" -#include "util/pretty_printer.h" +#include "runtime/thread_context.h" #include "util/timezone_utils.h" #include "util/uid_util.h" +#include "vec/runtime/vdatetime_value.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index ae98d7e68a..105acf5918 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -20,14 +20,24 @@ #pragma once +#include +#include +#include +#include + +#include #include +#include +#include +#include +#include +#include +#include #include "cctz/time_zone.h" -#include "common/global_types.h" -#include "common/object_pool.h" -#include "gen_cpp/PaloInternalService_types.h" // for TQueryOptions -#include "gen_cpp/Types_types.h" // for TUniqueId -#include "runtime/query_fragments_ctx.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" #include "util/runtime_profile.h" #include "util/telemetry/telemetry.h" @@ -35,15 +45,10 @@ namespace doris { class DescriptorTbl; class ObjectPool; -class Status; class ExecEnv; -class DateTimeValue; -class MemTracker; -class DataStreamRecvr; -class ResultBufferMgr; -class BufferedBlockMgr; -class RowDescriptor; class RuntimeFilterMgr; +class MemTrackerLimiter; +class QueryFragmentsCtx; // A collection of items that are part of the global state of a // query and shared across all execution nodes of that query. diff --git a/be/src/runtime/small_file_mgr.cpp b/be/src/runtime/small_file_mgr.cpp index cfa367bbc4..19a05f9c9d 100644 --- a/be/src/runtime/small_file_mgr.cpp +++ b/be/src/runtime/small_file_mgr.cpp @@ -17,19 +17,29 @@ #include "runtime/small_file_mgr.h" +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include #include #include +#include +#include #include +#include +#include #include "common/status.h" -#include "gen_cpp/HeartbeatService.h" #include "gutil/strings/split.h" #include "http/http_client.h" +#include "io/fs/file_system.h" #include "io/fs/local_file_system.h" #include "runtime/exec_env.h" #include "util/doris_metrics.h" #include "util/md5.h" +#include "util/metrics.h" #include "util/string_util.h" namespace doris { diff --git a/be/src/runtime/small_file_mgr.h b/be/src/runtime/small_file_mgr.h index 89e0a85d89..74665a7655 100644 --- a/be/src/runtime/small_file_mgr.h +++ b/be/src/runtime/small_file_mgr.h @@ -22,12 +22,8 @@ #include #include #include -#include -#include #include "common/status.h" -#include "gen_cpp/Types_types.h" -#include "runtime/client_cache.h" namespace doris { diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index 50e0be587e..4db803b10e 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -17,25 +17,43 @@ #include "runtime/snapshot_loader.h" +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include +#include +#include #include +#include +#include + +#include +#include +#include +#include +#include #include "common/logging.h" -#include "gen_cpp/FrontendService.h" -#include "gen_cpp/FrontendService_types.h" -#include "gen_cpp/HeartbeatService_types.h" -#include "gen_cpp/PaloBrokerService_types.h" -#include "gen_cpp/TPaloBrokerService.h" #include "io/fs/broker_file_system.h" +#include "io/fs/file_system.h" #include "io/fs/hdfs_file_system.h" #include "io/fs/local_file_system.h" +#include "io/fs/path.h" +#include "io/fs/remote_file_system.h" #include "io/fs/s3_file_system.h" #include "io/hdfs_builder.h" +#include "olap/data_dir.h" #include "olap/snapshot_manager.h" #include "olap/storage_engine.h" #include "olap/tablet.h" -#include "runtime/broker_mgr.h" +#include "olap/tablet_manager.h" +#include "runtime/client_cache.h" #include "runtime/exec_env.h" #include "util/s3_uri.h" +#include "util/s3_util.h" #include "util/thrift_rpc_helper.h" namespace doris { diff --git a/be/src/runtime/snapshot_loader.h b/be/src/runtime/snapshot_loader.h index 3a5ea5925b..9e7a22d7a3 100644 --- a/be/src/runtime/snapshot_loader.h +++ b/be/src/runtime/snapshot_loader.h @@ -17,19 +17,21 @@ #pragma once +#include #include #include +#include #include #include #include "common/status.h" -#include "gen_cpp/Types_types.h" -#include "io/fs/remote_file_system.h" #include "olap/tablet.h" -#include "runtime/client_cache.h" namespace doris { +namespace io { +class RemoteFileSystem; +} // namespace io struct FileStat { std::string name; @@ -37,7 +39,6 @@ struct FileStat { int64_t size; }; class ExecEnv; -class StorageBackend; /* * Upload: diff --git a/be/src/runtime/stream_load/new_load_stream_mgr.cpp b/be/src/runtime/stream_load/new_load_stream_mgr.cpp index c7a57362fe..e5428bc21d 100644 --- a/be/src/runtime/stream_load/new_load_stream_mgr.cpp +++ b/be/src/runtime/stream_load/new_load_stream_mgr.cpp @@ -17,6 +17,9 @@ #include "runtime/stream_load/new_load_stream_mgr.h" +#include "util/doris_metrics.h" +#include "util/metrics.h" + namespace doris { DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(new_stream_load_pipe_count, MetricUnit::NOUNIT); diff --git a/be/src/runtime/stream_load/new_load_stream_mgr.h b/be/src/runtime/stream_load/new_load_stream_mgr.h index bb245232dc..65425a70cd 100644 --- a/be/src/runtime/stream_load/new_load_stream_mgr.h +++ b/be/src/runtime/stream_load/new_load_stream_mgr.h @@ -17,17 +17,21 @@ #pragma once +#include #include #include +#include #include +#include +#include "common/logging.h" #include "common/status.h" -#include "util/doris_metrics.h" #include "util/uid_util.h" namespace doris { class StreamLoadContext; + // used to register all streams in process so that other module can get this stream class NewLoadStreamMgr { public: diff --git a/be/src/runtime/stream_load/stream_load_context.cpp b/be/src/runtime/stream_load/stream_load_context.cpp index 1f3ecd4372..f381ba097d 100644 --- a/be/src/runtime/stream_load/stream_load_context.cpp +++ b/be/src/runtime/stream_load/stream_load_context.cpp @@ -17,8 +17,19 @@ #include "runtime/stream_load/stream_load_context.h" +#include +#include +#include +#include +#include +#include +#include + +#include #include +#include "common/logging.h" + namespace doris { using namespace ErrorCode; diff --git a/be/src/runtime/stream_load/stream_load_context.h b/be/src/runtime/stream_load/stream_load_context.h index d12148b527..0fa1d1f0e6 100644 --- a/be/src/runtime/stream_load/stream_load_context.h +++ b/be/src/runtime/stream_load/stream_load_context.h @@ -17,27 +17,31 @@ #pragma once -#include +#include +#include +#include +#include +#include +#include #include -#include +#include +#include +#include +#include +#include -#include "common/logging.h" #include "common/status.h" #include "common/utils.h" -#include "gen_cpp/BackendService_types.h" -#include "gen_cpp/FrontendService_types.h" -#include "io/fs/stream_load_pipe.h" #include "runtime/exec_env.h" -#include "runtime/message_body_sink.h" -#include "runtime/stream_load/new_load_stream_mgr.h" #include "runtime/stream_load/stream_load_executor.h" -#include "service/backend_options.h" -#include "util/string_util.h" #include "util/time.h" #include "util/uid_util.h" namespace doris { +namespace io { +class StreamLoadPipe; +} // namespace io // kafka related info class KafkaLoadInfo { diff --git a/be/src/runtime/stream_load/stream_load_executor.cpp b/be/src/runtime/stream_load/stream_load_executor.cpp index 9fe991f153..ba1c1f4517 100644 --- a/be/src/runtime/stream_load/stream_load_executor.cpp +++ b/be/src/runtime/stream_load/stream_load_executor.cpp @@ -18,16 +18,34 @@ #include "runtime/stream_load/stream_load_executor.h" #include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" #include "common/status.h" #include "common/utils.h" #include "runtime/client_cache.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" +#include "runtime/message_body_sink.h" +#include "runtime/runtime_state.h" +#include "runtime/stream_load/new_load_stream_mgr.h" #include "runtime/stream_load/stream_load_context.h" #include "util/doris_metrics.h" #include "util/thrift_rpc_helper.h" +#include "util/time.h" +#include "util/uid_util.h" namespace doris { using namespace ErrorCode; diff --git a/be/src/runtime/stream_load/stream_load_executor.h b/be/src/runtime/stream_load/stream_load_executor.h index b90cef0a06..fc9e6b17e4 100644 --- a/be/src/runtime/stream_load/stream_load_executor.h +++ b/be/src/runtime/stream_load/stream_load_executor.h @@ -19,14 +19,13 @@ #include -#include "gen_cpp/FrontendService_types.h" - namespace doris { class ExecEnv; class StreamLoadContext; class Status; class TTxnCommitAttachment; +class TLoadTxnCommitRequest; class StreamLoadExecutor { public: diff --git a/be/src/runtime/stream_load/stream_load_recorder.cpp b/be/src/runtime/stream_load/stream_load_recorder.cpp index e1d082ed18..bb055e43bd 100644 --- a/be/src/runtime/stream_load/stream_load_recorder.cpp +++ b/be/src/runtime/stream_load/stream_load_recorder.cpp @@ -17,12 +17,19 @@ #include "runtime/stream_load/stream_load_recorder.h" +#include +#include +#include + +#include +#include +#include + #include "common/config.h" #include "common/status.h" #include "rocksdb/db.h" #include "rocksdb/options.h" #include "rocksdb/slice.h" -#include "rocksdb/slice_transform.h" #include "rocksdb/utilities/db_ttl.h" #include "util/time.h" diff --git a/be/src/runtime/stream_load/stream_load_recorder.h b/be/src/runtime/stream_load/stream_load_recorder.h index 9d80360b91..5d446dd2af 100644 --- a/be/src/runtime/stream_load/stream_load_recorder.h +++ b/be/src/runtime/stream_load/stream_load_recorder.h @@ -15,11 +15,17 @@ // specific language governing permissions and limitations // under the License. +#include + +#include #include #include +#include -#include "olap/utils.h" -#include "rocksdb/utilities/db_ttl.h" +namespace rocksdb { +class ColumnFamilyHandle; +class DBWithTTL; +} // namespace rocksdb #pragma once diff --git a/be/src/runtime/struct_value.h b/be/src/runtime/struct_value.h index d02ddc994c..cb29d74b4c 100644 --- a/be/src/runtime/struct_value.h +++ b/be/src/runtime/struct_value.h @@ -17,9 +17,7 @@ #pragma once -#include - -#include "runtime/primitive_type.h" +#include namespace doris { diff --git a/be/src/runtime/tablets_channel.cpp b/be/src/runtime/tablets_channel.cpp index 57f795f912..b7f6f24953 100644 --- a/be/src/runtime/tablets_channel.cpp +++ b/be/src/runtime/tablets_channel.cpp @@ -17,14 +17,32 @@ #include "runtime/tablets_channel.h" +#include +#include +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include + +#include "common/logging.h" #include "exec/tablet_info.h" #include "olap/delta_writer.h" -#include "olap/memtable.h" #include "olap/storage_engine.h" +#include "olap/txn_manager.h" #include "runtime/load_channel.h" #include "util/doris_metrics.h" +#include "util/metrics.h" +#include "vec/core/block.h" namespace doris { +class SlotDescriptor; DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_writer_count, MetricUnit::NOUNIT); diff --git a/be/src/runtime/tablets_channel.h b/be/src/runtime/tablets_channel.h index 3a29ddd620..954f1543b0 100644 --- a/be/src/runtime/tablets_channel.h +++ b/be/src/runtime/tablets_channel.h @@ -17,23 +17,44 @@ #pragma once +#include + +#include #include #include #include +#include +#include #include +#include #include #include -#include #include -#include "gen_cpp/PaloInternalService_types.h" -#include "gen_cpp/Types_types.h" -#include "gen_cpp/internal_service.pb.h" -#include "runtime/descriptors.h" +#include "common/status.h" #include "util/bitmap.h" +#include "util/spinlock.h" #include "util/uid_util.h" +namespace google { +namespace protobuf { +template +class RepeatedField; +template +class Map; +template +class RepeatedPtrField; +} // namespace protobuf +} // namespace google + namespace doris { +class PSlaveTabletNodes; +class PSuccessSlaveTabletNodeIds; +class PTabletError; +class PTabletInfo; +class PTabletWriterOpenRequest; +class PUniqueId; +class TupleDescriptor; struct TabletsChannelKey { UniqueId id; diff --git a/be/src/runtime/task_group/task_group.cpp b/be/src/runtime/task_group/task_group.cpp index 783470be45..ea3a453d9c 100644 --- a/be/src/runtime/task_group/task_group.cpp +++ b/be/src/runtime/task_group/task_group.cpp @@ -17,10 +17,16 @@ #include "task_group.h" -#include +#include +#include -#include "gen_cpp/PaloInternalService_types.h" -#include "pipeline/pipeline_task.h" +#include +#include +#include +#include +#include + +#include "common/logging.h" namespace doris { namespace taskgroup { diff --git a/be/src/runtime/task_group/task_group.h b/be/src/runtime/task_group/task_group.h index d6eed42a19..9a3aa117c7 100644 --- a/be/src/runtime/task_group/task_group.h +++ b/be/src/runtime/task_group/task_group.h @@ -17,11 +17,16 @@ #pragma once +#include +#include + #include +#include #include #include +#include -#include "olap/olap_define.h" +#include "common/status.h" namespace doris { @@ -29,7 +34,6 @@ namespace pipeline { class PipelineTask; } -class QueryFragmentsCtx; class TPipelineResourceGroup; namespace taskgroup { diff --git a/be/src/runtime/task_group/task_group_manager.cpp b/be/src/runtime/task_group/task_group_manager.cpp index cb7de92a6e..d393264983 100644 --- a/be/src/runtime/task_group/task_group_manager.cpp +++ b/be/src/runtime/task_group/task_group_manager.cpp @@ -17,6 +17,11 @@ #include "task_group_manager.h" +#include +#include + +#include "runtime/task_group/task_group.h" + namespace doris::taskgroup { TaskGroupManager::TaskGroupManager() = default; diff --git a/be/src/runtime/task_group/task_group_manager.h b/be/src/runtime/task_group/task_group_manager.h index d5e32ac64e..0f415498a0 100644 --- a/be/src/runtime/task_group/task_group_manager.h +++ b/be/src/runtime/task_group/task_group_manager.h @@ -16,6 +16,8 @@ // under the License. #pragma once +#include + #include #include diff --git a/be/src/runtime/thread_context.cpp b/be/src/runtime/thread_context.cpp index 9e0fbf50f4..2e8845b206 100644 --- a/be/src/runtime/thread_context.cpp +++ b/be/src/runtime/thread_context.cpp @@ -19,8 +19,11 @@ #include "common/signal_handler.h" #include "runtime/runtime_state.h" +#include "util/doris_metrics.h" // IWYU pragma: keep +#include "util/uid_util.h" namespace doris { +class MemTracker; DEFINE_STATIC_THREAD_LOCAL(ThreadContext, ThreadContextPtr, _ptr); diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h index 04e8915378..983ad2fd0c 100644 --- a/be/src/runtime/thread_context.h +++ b/be/src/runtime/thread_context.h @@ -18,16 +18,22 @@ #pragma once #include +#include +#include +#include +#include +#include #include #include #include "common/logging.h" -// #include "gen_cpp/PaloInternalService_types.h" // for TQueryType #include "gutil/macros.h" +#include "runtime/exec_env.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "runtime/memory/thread_mem_tracker_mgr.h" #include "runtime/threadlocal.h" -#include "util/defer_op.h" +#include "util/defer_op.h" // IWYU pragma: keep // Used to observe the memory usage of the specified code segment #ifdef USE_MEM_TRACKER @@ -70,8 +76,9 @@ namespace doris { -class TUniqueId; class ThreadContext; +class MemTracker; +class RuntimeState; extern bthread_key_t btls_key; diff --git a/be/src/runtime/threadlocal.cc b/be/src/runtime/threadlocal.cc index 689931b8b6..69b5760773 100644 --- a/be/src/runtime/threadlocal.cc +++ b/be/src/runtime/threadlocal.cc @@ -19,7 +19,9 @@ #include #include +#include #include +#include #include "common/logging.h" #include "util/errno.h" diff --git a/be/src/runtime/threadlocal.h b/be/src/runtime/threadlocal.h index 4abbf14371..3842b2fcad 100644 --- a/be/src/runtime/threadlocal.h +++ b/be/src/runtime/threadlocal.h @@ -41,6 +41,8 @@ #pragma once +#include + #include "gutil/port.h" #define BLOCK_STATIC_THREAD_LOCAL(T, t, ...) \ diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp index 9ec81e9441..e0cf887b04 100644 --- a/be/src/runtime/types.cpp +++ b/be/src/runtime/types.cpp @@ -20,9 +20,16 @@ #include "runtime/types.h" -#include +#include +#include +#include +#include +#include + +#include "olap/olap_define.h" #include "runtime/primitive_type.h" + namespace doris { TypeDescriptor::TypeDescriptor(const std::vector& types, int* idx) diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h index 9cbd8d70d6..82df1d01d6 100644 --- a/be/src/runtime/types.h +++ b/be/src/runtime/types.h @@ -20,21 +20,21 @@ #pragma once +#include +#include +#include + +#include #include #include #include "common/config.h" -#include "gen_cpp/Types_types.h" -#include "gen_cpp/types.pb.h" -#include "olap/olap_define.h" #include "runtime/define_primitive_type.h" namespace doris { extern const int HLL_COLUMN_DEFAULT_LEN; -struct TPrimitiveType; - // Describes a type. Includes the enum, children types, and any type-specific metadata // (e.g. precision and scale for decimals). // TODO for 2.3: rename to TypeDescriptor diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp index 312e9ca1f2..ac87920cd8 100644 --- a/be/src/runtime/user_function_cache.cpp +++ b/be/src/runtime/user_function_cache.cpp @@ -17,17 +17,27 @@ #include "runtime/user_function_cache.h" +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include + #include +#include +#include #include +#include #include #include "common/config.h" #include "common/status.h" #include "gutil/strings/split.h" #include "http/http_client.h" +#include "io/fs/file_system.h" #include "io/fs/local_file_system.h" #include "util/dynamic_util.h" -#include "util/jni-util.h" #include "util/md5.h" #include "util/spinlock.h" #include "util/string_util.h" diff --git a/be/src/runtime/user_function_cache.h b/be/src/runtime/user_function_cache.h index b3c4aa7e80..1507642167 100644 --- a/be/src/runtime/user_function_cache.h +++ b/be/src/runtime/user_function_cache.h @@ -17,6 +17,8 @@ #pragma once +#include + #include #include #include diff --git a/be/src/service/backend_options.h b/be/src/service/backend_options.h index 87ca36b17a..9c6f0df484 100644 --- a/be/src/service/backend_options.h +++ b/be/src/service/backend_options.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include diff --git a/be/src/service/backend_service.h b/be/src/service/backend_service.h index 35a9ba8abc..727d2348a7 100644 --- a/be/src/service/backend_service.h +++ b/be/src/service/backend_service.h @@ -18,13 +18,8 @@ #pragma once #include -#include -#include #include -#include -#include -#include #include #include #include @@ -39,23 +34,10 @@ class ThriftServer; class TAgentResult; class TAgentTaskRequest; class TAgentPublishRequest; -class TPlanExecRequest; -class TPlanExecParams; class TExecPlanFragmentParams; class TExecPlanFragmentResult; -class TInsertResult; -class TReportExecStatusArgs; -class TReportExecStatusParams; -class TReportExecStatusResult; -class TCancelPlanFragmentArgs; class TCancelPlanFragmentResult; -class TTransmitDataArgs; class TTransmitDataResult; -class TNetworkAddress; -class TClientRequest; -class TExecRequest; -class TSessionState; -class TQueryOptions; class TExportTaskRequest; class TExportStatusResult; class TStreamLoadRecordResult; diff --git a/be/src/service/brpc_service.cpp b/be/src/service/brpc_service.cpp index 35989f29dc..80138b99ee 100644 --- a/be/src/service/brpc_service.cpp +++ b/be/src/service/brpc_service.cpp @@ -19,8 +19,9 @@ #include #include -#include #include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index be10089eae..c65f25da2e 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include #include -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include // IWYU pragma: keep #include diff --git a/be/src/service/internal_service.h b/be/src/service/internal_service.h index 1b1e56d0cf..6cfbae4fa7 100644 --- a/be/src/service/internal_service.h +++ b/be/src/service/internal_service.h @@ -23,7 +23,6 @@ #include #include "common/status.h" -#include "runtime/cache/result_cache.h" #include "util/priority_thread_pool.hpp" namespace google { @@ -33,10 +32,6 @@ class RpcController; } // namespace protobuf } // namespace google -namespace brpc { -class Controller; -} - namespace doris { class ExecEnv; diff --git a/be/src/service/point_query_executor.h b/be/src/service/point_query_executor.h index 69fa6850e9..bc8e770049 100644 --- a/be/src/service/point_query_executor.h +++ b/be/src/service/point_query_executor.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -53,8 +52,6 @@ namespace doris { -class RowCache; -class Cache; class PTabletKeyLookupRequest; class PTabletKeyLookupResponse; class RuntimeState; diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index 452402852f..e4ac6493da 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include +#include #include #include @@ -25,8 +27,6 @@ #include #include "common/status.h" -#include "gen_cpp/olap_file.pb.h" -#include "gen_cpp/segment_v2.pb.h" #include "gutil/strings/numbers.h" #include "gutil/strings/split.h" #include "gutil/strings/substitute.h" diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h index bec07e8363..5b429b85d1 100644 --- a/be/src/udf/udf.h +++ b/be/src/udf/udf.h @@ -20,11 +20,7 @@ #pragma once -#include - #include -#include -#include #include #include #include @@ -35,17 +31,7 @@ namespace doris { struct ColumnPtrWrapper; struct StringRef; -class BitmapValue; -class DecimalV2Value; -class CollectionValue; -struct TypeDescriptor; class RuntimeState; -// All input and output values will be one of the structs below. The struct is a simple -// object containing a boolean to store if the value is nullptr and the value itself. The -// value is unspecified if the nullptr boolean is set. -struct AnyVal; -struct StringRef; -struct DateTimeVal; // The FunctionContext is passed to every UDF/UDA and is the interface for the UDF to the // rest of the system. It contains APIs to examine the system state, report errors diff --git a/be/src/util/aligned_new.h b/be/src/util/aligned_new.h index f56a56eb51..2800b19e4f 100644 --- a/be/src/util/aligned_new.h +++ b/be/src/util/aligned_new.h @@ -22,7 +22,8 @@ #include -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" namespace doris { diff --git a/be/src/util/arrow/block_convertor.cpp b/be/src/util/arrow/block_convertor.cpp index 3fbfcbd0ed..2b426ca2f2 100644 --- a/be/src/util/arrow/block_convertor.cpp +++ b/be/src/util/arrow/block_convertor.cpp @@ -17,37 +17,49 @@ #include "util/arrow/block_convertor.h" -#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include #include #include #include -#include +#include #include #include +#include +#include -#include +#include #include #include +#include +#include +#include +#include -#include "gutil/strings/substitute.h" -#include "runtime/descriptor_helper.h" -#include "runtime/descriptors.h" -#include "runtime/jsonb_value.h" +#include "gutil/integral_types.h" #include "runtime/large_int_value.h" #include "util/arrow/utils.h" +#include "util/jsonb_utils.h" #include "util/types.h" +#include "vec/columns/column.h" #include "vec/columns/column_array.h" -#include "vec/columns/column_map.h" +#include "vec/columns/column_decimal.h" +#include "vec/columns/column_nullable.h" #include "vec/common/assert_cast.h" +#include "vec/common/string_ref.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_array.h" -#include "vec/data_types/data_type_map.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/runtime/vdatetime_value.h" + +namespace arrow { +class Array; +} // namespace arrow namespace doris { diff --git a/be/src/util/arrow/block_convertor.h b/be/src/util/arrow/block_convertor.h index c76ae73c9f..9fb710fc75 100644 --- a/be/src/util/arrow/block_convertor.h +++ b/be/src/util/arrow/block_convertor.h @@ -36,9 +36,6 @@ class Schema; namespace doris { -class ObjectPool; -class RowDescriptor; - Status convert_to_arrow_batch(const vectorized::Block& block, const std::shared_ptr& schema, arrow::MemoryPool* pool, std::shared_ptr* result); diff --git a/be/src/util/arrow/row_batch.cpp b/be/src/util/arrow/row_batch.cpp index 51c490c1ae..110977c6e6 100644 --- a/be/src/util/arrow/row_batch.cpp +++ b/be/src/util/arrow/row_batch.cpp @@ -17,30 +17,26 @@ #include "util/arrow/row_batch.h" -#include -#include #include -#include #include #include -#include #include +#include #include #include -#include -#include -#include +#include +#include +#include #include -#include #include +#include +#include #include "gutil/strings/substitute.h" -#include "runtime/descriptor_helper.h" +#include "runtime/define_primitive_type.h" #include "runtime/descriptors.h" -#include "runtime/large_int_value.h" -#include "util/arrow/utils.h" -#include "util/types.h" +#include "runtime/types.h" namespace doris { diff --git a/be/src/util/arrow/row_batch.h b/be/src/util/arrow/row_batch.h index 25203ec170..8946502f83 100644 --- a/be/src/util/arrow/row_batch.h +++ b/be/src/util/arrow/row_batch.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include "common/status.h" @@ -27,7 +28,6 @@ namespace arrow { -class MemoryPool; class RecordBatch; class Schema; @@ -35,7 +35,6 @@ class Schema; namespace doris { -class ObjectPool; class RowDescriptor; // Convert Doris RowDescriptor to Arrow Schema. diff --git a/be/src/util/arrow/utils.cpp b/be/src/util/arrow/utils.cpp index ed2bdfab2a..2ef5301673 100644 --- a/be/src/util/arrow/utils.cpp +++ b/be/src/util/arrow/utils.cpp @@ -18,9 +18,7 @@ #include "util/arrow/utils.h" #include -#include -#include -#include +#include #include "gutil/strings/substitute.h" diff --git a/be/src/util/arrow/utils.h b/be/src/util/arrow/utils.h index 6e8be0c625..f6ecff1a3a 100644 --- a/be/src/util/arrow/utils.h +++ b/be/src/util/arrow/utils.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include "common/status.h" diff --git a/be/src/util/bfd_parser.cpp b/be/src/util/bfd_parser.cpp index 98f0a46d62..8213d468c2 100644 --- a/be/src/util/bfd_parser.cpp +++ b/be/src/util/bfd_parser.cpp @@ -17,7 +17,12 @@ #include "util/bfd_parser.h" +#include +#include + +#include #include +#include #include "common/logging.h" diff --git a/be/src/util/bit_util.h b/be/src/util/bit_util.h index ede72dd1d5..0a60eb5429 100644 --- a/be/src/util/bit_util.h +++ b/be/src/util/bit_util.h @@ -24,7 +24,8 @@ #include #endif -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "gutil/bits.h" #include "util/cpu_info.h" #include "util/sse_util.hpp" diff --git a/be/src/util/bitmap.h b/be/src/util/bitmap.h index 931b3becc3..b523427bee 100644 --- a/be/src/util/bitmap.h +++ b/be/src/util/bitmap.h @@ -20,6 +20,15 @@ #pragma once +#include +#include +#include + +#include +#include +#include + +#include "gutil/port.h" #include "gutil/strings/fastmem.h" #include "util/bit_util.h" diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp index 4ff091b677..f909d4c452 100644 --- a/be/src/util/block_compression.cpp +++ b/be/src/util/block_compression.cpp @@ -17,15 +17,25 @@ #include "util/block_compression.h" +#include +#include +#include +#include #include #include #include #include +#include +#include #include #include #include +#include #include +#include +#include +#include #include "gutil/strings/substitute.h" #include "util/defer_op.h" diff --git a/be/src/util/block_compression.h b/be/src/util/block_compression.h index 60c0c22f8e..c4991a9191 100644 --- a/be/src/util/block_compression.h +++ b/be/src/util/block_compression.h @@ -17,15 +17,20 @@ #pragma once +#include + #include #include #include "common/status.h" -#include "gen_cpp/parquet_types.h" -#include "gen_cpp/segment_v2.pb.h" #include "util/slice.h" namespace doris { +class faststring; + +namespace segment_v2 { +enum CompressionTypePB : int; +} // namespace segment_v2 // This class is used to encapsulate Compression/Decompression algorithm. // This class only used to compress a block data, which means all data diff --git a/be/src/util/brpc_client_cache.cpp b/be/src/util/brpc_client_cache.cpp index df89585e73..b9135e8014 100644 --- a/be/src/util/brpc_client_cache.cpp +++ b/be/src/util/brpc_client_cache.cpp @@ -17,8 +17,13 @@ #include "util/brpc_client_cache.h" -namespace doris { +#include // IWYU pragma: keep +#include // IWYU pragma: keep +#include "util/doris_metrics.h" +#include "util/metrics.h" + +namespace doris { DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(brpc_endpoint_stub_count, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(brpc_function_endpoint_stub_count, MetricUnit::NOUNIT); diff --git a/be/src/util/brpc_client_cache.h b/be/src/util/brpc_client_cache.h index 364f11e6ea..9c48c15529 100644 --- a/be/src/util/brpc_client_cache.h +++ b/be/src/util/brpc_client_cache.h @@ -17,20 +17,37 @@ #pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include #include #include -#include +#include +#include +#include +#include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" -#include "gen_cpp/Types_types.h" // TNetworkAddress -#include "gen_cpp/function_service.pb.h" -#include "gen_cpp/internal_service.pb.h" -#include "service/brpc.h" -#include "util/doris_metrics.h" #include "util/network_util.h" +namespace doris { +class PBackendService_Stub; +class PFunctionService_Stub; +} // namespace doris + template using StubMap = phmap::parallel_flat_hash_map< std::string, std::shared_ptr, std::hash, std::equal_to, diff --git a/be/src/util/cgroup_util.cpp b/be/src/util/cgroup_util.cpp index 15e5e8791c..b1cb9cf8ea 100644 --- a/be/src/util/cgroup_util.cpp +++ b/be/src/util/cgroup_util.cpp @@ -21,9 +21,11 @@ #include #include #include -#include +#include #include +#include +#include "gutil/stringprintf.h" #include "gutil/strings/escaping.h" #include "gutil/strings/split.h" #include "gutil/strings/substitute.h" diff --git a/be/src/util/cidr.cpp b/be/src/util/cidr.cpp index 267998f51b..573074bae1 100644 --- a/be/src/util/cidr.cpp +++ b/be/src/util/cidr.cpp @@ -18,9 +18,15 @@ #include "util/cidr.h" #include +#include + +#include +#include +#include +#include +#include #include "common/logging.h" -#include "gutil/strings/split.h" namespace doris { diff --git a/be/src/util/cidr.h b/be/src/util/cidr.h index 599725554c..6204369bb1 100644 --- a/be/src/util/cidr.h +++ b/be/src/util/cidr.h @@ -17,9 +17,10 @@ #pragma once -#include +#include #include +#include #include namespace doris { diff --git a/be/src/util/cityhash102/city.cc b/be/src/util/cityhash102/city.cc index 765d2a88cd..8c3d449d14 100644 --- a/be/src/util/cityhash102/city.cc +++ b/be/src/util/cityhash102/city.cc @@ -29,9 +29,9 @@ #include "city.h" +// IWYU pragma: no_include #include // for memcpy and memset - -#include +#include #include "config.h" @@ -44,12 +44,14 @@ #ifdef _MSC_VER #include + #define bswap_32(x) _byteswap_ulong(x) #define bswap_64(x) _byteswap_uint64(x) #elif defined(__APPLE__) // Mac OS X / Darwin features #include + #define bswap_32(x) OSSwapInt32(x) #define bswap_64(x) OSSwapInt64(x) @@ -346,7 +348,7 @@ uint128 CityHash128(const char* s, size_t len) { #if defined(__SSE4_2__) || defined(__aarch64__) #include "citycrc.h" -#include "util/sse_util.hpp" +#include "util/sse_util.hpp" // IWYU pragma: keep namespace CityHash_v1_0_2 { diff --git a/be/src/util/coding.h b/be/src/util/coding.h index 16b89507b2..ee9ade4729 100644 --- a/be/src/util/coding.h +++ b/be/src/util/coding.h @@ -8,9 +8,12 @@ #pragma once -#include +#ifndef __APPLE__ +#include +#endif +#include +#include -#include "gutil/endian.h" #include "olap/olap_common.h" #include "util/slice.h" diff --git a/be/src/util/container_util.hpp b/be/src/util/container_util.hpp index 9f11c46983..3c99dc646d 100644 --- a/be/src/util/container_util.hpp +++ b/be/src/util/container_util.hpp @@ -20,10 +20,11 @@ #pragma once +#include + #include #include -#include "gen_cpp/Types_types.h" #include "util/hash_util.hpp" namespace doris { diff --git a/be/src/util/core_local.cpp b/be/src/util/core_local.cpp index 2077d87908..77efc18bfe 100644 --- a/be/src/util/core_local.cpp +++ b/be/src/util/core_local.cpp @@ -18,10 +18,15 @@ #include "util/core_local.h" #include +#include +#include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" #include "util/spinlock.h" +#include "util/sse_util.hpp" namespace doris { diff --git a/be/src/util/core_local.h b/be/src/util/core_local.h index 0ebfea0779..e23a6820d9 100644 --- a/be/src/util/core_local.h +++ b/be/src/util/core_local.h @@ -17,15 +17,20 @@ #pragma once +#include +#include +#include + #include #include -#include #include +#include #include +#include #include -#include "common/compiler_util.h" -#include "gutil/macros.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep namespace doris { diff --git a/be/src/util/countdown_latch.h b/be/src/util/countdown_latch.h index 4647f920fc..43db2bf189 100644 --- a/be/src/util/countdown_latch.h +++ b/be/src/util/countdown_latch.h @@ -17,6 +17,7 @@ #pragma once +// IWYU pragma: no_include #include #include #include diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp index 92badaed6b..60bdf1cd17 100644 --- a/be/src/util/cpu_info.cpp +++ b/be/src/util/cpu_info.cpp @@ -21,8 +21,6 @@ #include "util/cpu_info.h" #if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) -/* GCC-compatible compiler, targeting x86/x86-64 */ -#include #elif defined(__GNUC__) && defined(__ARM_NEON__) /* GCC-compatible compiler, targeting ARM with NEON */ #include @@ -43,24 +41,25 @@ #include #endif +#include #include #include -#include #include #include -#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include -#include -#include #include "common/config.h" #include "common/env_config.h" #include "gflags/gflags.h" +#include "gutil/stringprintf.h" #include "gutil/strings/substitute.h" #include "util/pretty_printer.h" -#include "util/string_parser.hpp" using boost::algorithm::contains; using boost::algorithm::trim; diff --git a/be/src/util/cpu_info.h b/be/src/util/cpu_info.h index d09c89775f..0e03ad3928 100644 --- a/be/src/util/cpu_info.h +++ b/be/src/util/cpu_info.h @@ -21,6 +21,7 @@ #pragma once #include +#include #include #include diff --git a/be/src/util/crc32c.cpp b/be/src/util/crc32c.cpp index 99884c3b5e..7ad31e7f22 100644 --- a/be/src/util/crc32c.cpp +++ b/be/src/util/crc32c.cpp @@ -18,8 +18,11 @@ // the following code are modified from RocksDB: // https://github.com/facebook/rocksdb/blob/master/util/crc32c.cc +// IWYU pragma: no_include +#include +#include + #include "util/coding.h" -#include "util/sse_util.hpp" namespace doris { namespace crc32c { diff --git a/be/src/util/date_func.cpp b/be/src/util/date_func.cpp index c77be510f8..e7ae30e003 100644 --- a/be/src/util/date_func.cpp +++ b/be/src/util/date_func.cpp @@ -17,7 +17,12 @@ #include "util/date_func.h" -#include +#include +#include +#include +#include + +#include #include "vec/runtime/vdatetime_value.h" diff --git a/be/src/util/date_func.h b/be/src/util/date_func.h index 4378fd32e1..cc9aa7b6da 100644 --- a/be/src/util/date_func.h +++ b/be/src/util/date_func.h @@ -18,12 +18,9 @@ #pragma once #include -#include #include -#include "olap/field.h" - namespace doris { uint64_t timestamp_from_datetime(const std::string& datetime_str); diff --git a/be/src/util/debug_util.cpp b/be/src/util/debug_util.cpp index 9179c70a60..bfb4826514 100644 --- a/be/src/util/debug_util.cpp +++ b/be/src/util/debug_util.cpp @@ -17,14 +17,14 @@ #include "util/debug_util.h" -#include -#include +#include +#include +#include -#include "common/logging.h" -#include "gen_cpp/Opcodes_types.h" -#include "gen_cpp/types.pb.h" -#include "gen_cpp/version.h" -#include "util/cpu_info.h" +#include +#include +#include // IWYU pragma: keep +#include namespace doris { diff --git a/be/src/util/debug_util.h b/be/src/util/debug_util.h index 956a38a403..fbc0c221f4 100644 --- a/be/src/util/debug_util.h +++ b/be/src/util/debug_util.h @@ -17,21 +17,14 @@ #pragma once -#include +#include +#include +#include + #include -#include "gen_cpp/AgentService_types.h" -#include "gen_cpp/Descriptors_types.h" -#include "gen_cpp/Exprs_types.h" -#include "gen_cpp/Opcodes_types.h" -#include "gen_cpp/PlanNodes_types.h" -#include "gen_cpp/RuntimeProfile_types.h" -#include "gen_cpp/Types_types.h" - namespace doris { -class PUniqueId; - std::string print_plan_node_type(const TPlanNodeType::type& type); std::string print_tstmt_type(const TStmtType::type& type); std::string print_query_state(const QueryState::type& type); diff --git a/be/src/util/disk_info.cpp b/be/src/util/disk_info.cpp index b3c70706bb..86ddb447f2 100644 --- a/be/src/util/disk_info.cpp +++ b/be/src/util/disk_info.cpp @@ -17,17 +17,23 @@ #include "util/disk_info.h" +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include -#include +#include +#include +#include #include "gutil/strings/split.h" #include "io/fs/local_file_system.h" diff --git a/be/src/util/disk_info.h b/be/src/util/disk_info.h index f284f55433..f415cb958d 100644 --- a/be/src/util/disk_info.h +++ b/be/src/util/disk_info.h @@ -17,9 +17,12 @@ #pragma once +#include + #include #include #include +#include #include "common/logging.h" #include "common/status.h" diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp index a40d1691cb..97ca3de938 100644 --- a/be/src/util/doris_metrics.cpp +++ b/be/src/util/doris_metrics.cpp @@ -17,14 +17,26 @@ #include "util/doris_metrics.h" -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include #include +#include +#include + +#include "common/status.h" #include "io/fs/local_file_system.h" -#include "util/debug_util.h" #include "util/system_metrics.h" namespace doris { +namespace io { +struct FileInfo; +} // namespace io DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(fragment_requests_total, MetricUnit::REQUESTS, "Total fragment requests received."); diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h index ec44f58e53..86b52e32be 100644 --- a/be/src/util/doris_metrics.h +++ b/be/src/util/doris_metrics.h @@ -17,9 +17,9 @@ #pragma once +#include #include #include -#include #include #include "util/metrics.h" diff --git a/be/src/util/dynamic_util.cpp b/be/src/util/dynamic_util.cpp index d6ff812493..1f09c11b6d 100644 --- a/be/src/util/dynamic_util.cpp +++ b/be/src/util/dynamic_util.cpp @@ -22,8 +22,6 @@ #include -#include - namespace doris { Status dynamic_lookup(void* handle, const char* symbol, void** fn_ptr) { diff --git a/be/src/util/easy_json.cc b/be/src/util/easy_json.cc index 96564b1d16..4df9e88fce 100644 --- a/be/src/util/easy_json.cc +++ b/be/src/util/easy_json.cc @@ -18,6 +18,7 @@ #include "util/easy_json.h" #include +#include #include #include #include diff --git a/be/src/util/encryption_util.cpp b/be/src/util/encryption_util.cpp index b490a30fc3..ab1ff3e857 100644 --- a/be/src/util/encryption_util.cpp +++ b/be/src/util/encryption_util.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include diff --git a/be/src/util/error_util.cc b/be/src/util/error_util.cc index f1b006b6fd..f13161eed8 100644 --- a/be/src/util/error_util.cc +++ b/be/src/util/error_util.cc @@ -20,7 +20,8 @@ #include "util/error_util.h" -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include diff --git a/be/src/util/faststring.h b/be/src/util/faststring.h index 354a8965ec..139b7bd92b 100644 --- a/be/src/util/faststring.h +++ b/be/src/util/faststring.h @@ -17,12 +17,13 @@ #pragma once +#include + #include #include #include #include "gutil/dynamic_annotations.h" -#include "gutil/macros.h" #include "gutil/port.h" #include "gutil/strings/fastmem.h" #include "util/slice.h" diff --git a/be/src/util/frame_of_reference_coding.cpp b/be/src/util/frame_of_reference_coding.cpp index 0e7bb57b64..d98726c6e9 100644 --- a/be/src/util/frame_of_reference_coding.cpp +++ b/be/src/util/frame_of_reference_coding.cpp @@ -17,8 +17,14 @@ #include "util/frame_of_reference_coding.h" +#include +#include + #include #include +#include +#include +#include #include "util/bit_util.h" #include "util/coding.h" diff --git a/be/src/util/frame_of_reference_coding.h b/be/src/util/frame_of_reference_coding.h index da3d716e75..06fd2a25f0 100644 --- a/be/src/util/frame_of_reference_coding.h +++ b/be/src/util/frame_of_reference_coding.h @@ -17,14 +17,13 @@ #pragma once +#include + #include -#include -#include +#include #include "olap/olap_common.h" #include "olap/uint24.h" -#include "util/bit_stream_utils.h" -#include "util/bit_stream_utils.inline.h" #include "util/faststring.h" namespace doris { diff --git a/be/src/util/hash_util.hpp b/be/src/util/hash_util.hpp index 2b9dbe0b2a..e8c70e290c 100644 --- a/be/src/util/hash_util.hpp +++ b/be/src/util/hash_util.hpp @@ -20,13 +20,14 @@ #pragma once +#include #include #include #include -#include "common/compiler_util.h" -#include "gen_cpp/Types_types.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "runtime/define_primitive_type.h" #include "util/cpu_info.h" #include "util/murmur_hash3.h" diff --git a/be/src/util/hdfs_util.cpp b/be/src/util/hdfs_util.cpp index d8be4b96c6..643053292e 100644 --- a/be/src/util/hdfs_util.cpp +++ b/be/src/util/hdfs_util.cpp @@ -17,11 +17,11 @@ #include "util/hdfs_util.h" -#include +#include -#include "common/config.h" #include "common/logging.h" #include "io/fs/err_utils.h" +#include "io/hdfs_builder.h" namespace doris { namespace io { diff --git a/be/src/util/hdfs_util.h b/be/src/util/hdfs_util.h index 2e56181df7..39dc8bc416 100644 --- a/be/src/util/hdfs_util.h +++ b/be/src/util/hdfs_util.h @@ -17,16 +17,14 @@ #pragma once -#include -#include #include -#include "common/status.h" #include "io/fs/hdfs.h" #include "io/fs/path.h" -#include "io/hdfs_builder.h" namespace doris { +class HDFSCommonBuilder; + namespace io { class HDFSHandle { diff --git a/be/src/util/histogram.cpp b/be/src/util/histogram.cpp index 21664eacd5..8cdcc42b7d 100644 --- a/be/src/util/histogram.cpp +++ b/be/src/util/histogram.cpp @@ -19,9 +19,11 @@ #include +#include #include #include #include +#include namespace doris { diff --git a/be/src/util/histogram.h b/be/src/util/histogram.h index c8dd6b8ed8..3055c75994 100644 --- a/be/src/util/histogram.h +++ b/be/src/util/histogram.h @@ -17,6 +17,9 @@ #pragma once +#include +#include + #include #include #include diff --git a/be/src/util/jni-util.cpp b/be/src/util/jni-util.cpp index 955efb78e7..411ef99dc2 100644 --- a/be/src/util/jni-util.cpp +++ b/be/src/util/jni-util.cpp @@ -17,20 +17,22 @@ #include "util/jni-util.h" +#include +#include #include #include -#include +#include #include #include +#include +#include #include #include #include #include -#include "common/config.h" #include "gutil/strings/substitute.h" -#include "util/defer_op.h" #include "util/jni_native_method.h" #include "util/libjvm_loader.h" diff --git a/be/src/util/jni-util.h b/be/src/util/jni-util.h index ec5f6abf6e..f43d5577d1 100644 --- a/be/src/util/jni-util.h +++ b/be/src/util/jni-util.h @@ -17,10 +17,16 @@ #pragma once +#include #include +#include +#include +#include + +#include #include "common/status.h" -#include "gutil/macros.h" +#include "jni_md.h" #include "util/thrift_util.h" #ifdef USE_HADOOP_HDFS @@ -29,6 +35,7 @@ extern "C" JNIEnv* getJNIEnv(void); #endif namespace doris { +class JniUtil; #define RETURN_ERROR_IF_EXC(env) \ do { \ diff --git a/be/src/util/jni_native_method.cpp b/be/src/util/jni_native_method.cpp index 185293440f..50f43c4d9a 100644 --- a/be/src/util/jni_native_method.cpp +++ b/be/src/util/jni_native_method.cpp @@ -17,6 +17,9 @@ #include "jni_native_method.h" +#include + +#include "jni.h" #include "vec/columns/column_string.h" namespace doris { diff --git a/be/src/util/jni_native_method.h b/be/src/util/jni_native_method.h index 75551870cf..d208cb0d95 100644 --- a/be/src/util/jni_native_method.h +++ b/be/src/util/jni_native_method.h @@ -19,6 +19,8 @@ #include +#include "jni_md.h" + namespace doris { /** diff --git a/be/src/util/jsonb_document.h b/be/src/util/jsonb_document.h index fb98175b60..19106cdea4 100644 --- a/be/src/util/jsonb_document.h +++ b/be/src/util/jsonb_document.h @@ -74,6 +74,9 @@ #include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep + // #include "util/string_parser.hpp" namespace doris { diff --git a/be/src/util/key_util.h b/be/src/util/key_util.h index 13cb5c1768..ccd67b9cdd 100644 --- a/be/src/util/key_util.h +++ b/be/src/util/key_util.h @@ -17,13 +17,14 @@ #pragma once +#include + #include #include #include #include #include "common/status.h" -#include "gen_cpp/segment_v2.pb.h" #include "util/debug_util.h" #include "util/faststring.h" #include "util/slice.h" diff --git a/be/src/util/libjvm_loader.cpp b/be/src/util/libjvm_loader.cpp index 6175da6081..02375f8142 100644 --- a/be/src/util/libjvm_loader.cpp +++ b/be/src/util/libjvm_loader.cpp @@ -24,6 +24,8 @@ #include #include "common/status.h" +#include "jni.h" +#include "jni_md.h" _JNI_IMPORT_OR_EXPORT_ jint JNICALL JNI_GetCreatedJavaVMs(JavaVM** vm_buf, jsize bufLen, jsize* numVMs) { diff --git a/be/src/util/libjvm_loader.h b/be/src/util/libjvm_loader.h index 2982cbea7f..cb55c22221 100644 --- a/be/src/util/libjvm_loader.h +++ b/be/src/util/libjvm_loader.h @@ -17,8 +17,7 @@ #pragma once -#include -#include +#include // IWYU pragma: keep #include #include diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index da52948dbc..918c3c85fc 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -24,21 +24,29 @@ #include #endif -#include -#include -#include +#include +#include +#include +#include +#include +#include #include -#include -#include +#include +#include #include "common/config.h" +#include "common/status.h" #include "gutil/strings/split.h" #include "olap/page_cache.h" #include "olap/segment_loader.h" +#include "runtime/memory/chunk_allocator.h" +#include "runtime/memory/mem_tracker_limiter.h" #include "util/cgroup_util.h" +#include "util/defer_op.h" #include "util/parse_util.h" #include "util/pretty_printer.h" +#include "util/stopwatch.hpp" #include "util/string_parser.hpp" namespace doris { diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index b13368222a..b25e194916 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -20,8 +20,10 @@ #pragma once -#include +#include +#include +#include #include #include "common/logging.h" @@ -31,7 +33,6 @@ #include #endif #include "util/perf_counters.h" -#include "util/pretty_printer.h" namespace doris { diff --git a/be/src/util/metrics.cpp b/be/src/util/metrics.cpp index a9cae294d6..de2d43c854 100644 --- a/be/src/util/metrics.cpp +++ b/be/src/util/metrics.cpp @@ -17,8 +17,15 @@ #include "util/metrics.h" +#include +#include #include #include + +#include + +#include "common/config.h" + namespace doris { std::ostream& operator<<(std::ostream& os, MetricType type) { diff --git a/be/src/util/metrics.h b/be/src/util/metrics.h index 648f96809f..6007562eb0 100644 --- a/be/src/util/metrics.h +++ b/be/src/util/metrics.h @@ -19,18 +19,20 @@ #include #include +#include +#include #include #include -#include +#include +#include #include -#include -#include #include #include #include +#include +#include -#include "common/config.h" #include "util/core_local.h" #include "util/histogram.h" #include "util/spinlock.h" @@ -39,8 +41,6 @@ namespace doris { namespace rj = RAPIDJSON_NAMESPACE; -class MetricRegistry; - enum class MetricType { COUNTER, GAUGE, HISTOGRAM, SUMMARY, UNTYPED }; enum class MetricUnit { diff --git a/be/src/util/mustache/mustache.cc b/be/src/util/mustache/mustache.cc index a3947e8463..e72967152d 100644 --- a/be/src/util/mustache/mustache.cc +++ b/be/src/util/mustache/mustache.cc @@ -12,10 +12,23 @@ #include "mustache.h" +#include +#include +#include #include +#include +#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // IWYU pragma: keep #include #include #include diff --git a/be/src/util/mustache/mustache.h b/be/src/util/mustache/mustache.h index e4b861f32f..90b2e0f3ee 100644 --- a/be/src/util/mustache/mustache.h +++ b/be/src/util/mustache/mustache.h @@ -13,6 +13,7 @@ #pragma once #include +#include #include "rapidjson/document.h" diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp index e86613713b..b3c0f0ccf2 100644 --- a/be/src/util/mysql_row_buffer.cpp +++ b/be/src/util/mysql_row_buffer.cpp @@ -17,16 +17,24 @@ #include "util/mysql_row_buffer.h" +#include #include -#include -#include +#include +#include + +#include +#include +#include +#include #include "common/logging.h" #include "date_func.h" #include "gutil/strings/numbers.h" +#include "olap/olap_common.h" +#include "runtime/decimalv2_value.h" #include "runtime/large_int_value.h" #include "util/mysql_global.h" -#include "vec/runtime/vdatetime_value.h" +#include "vec/runtime/vdatetime_value.h" // IWYU pragma: keep namespace doris { diff --git a/be/src/util/network_util.cpp b/be/src/util/network_util.cpp index 78e399a6a9..4b53e7f9b6 100644 --- a/be/src/util/network_util.cpp +++ b/be/src/util/network_util.cpp @@ -21,11 +21,17 @@ #include "util/network_util.h" #include -#include +// IWYU pragma: no_include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include #include -#include #include +#include +#include #include +#include #include diff --git a/be/src/util/network_util.h b/be/src/util/network_util.h index 27e97a23a0..195d909f40 100644 --- a/be/src/util/network_util.h +++ b/be/src/util/network_util.h @@ -20,10 +20,13 @@ #pragma once +#include +#include + +#include #include #include "common/status.h" -#include "gen_cpp/Types_types.h" namespace doris { diff --git a/be/src/util/os_info.cpp b/be/src/util/os_info.cpp index db14642806..0b42e29bbb 100644 --- a/be/src/util/os_info.cpp +++ b/be/src/util/os_info.cpp @@ -20,13 +20,7 @@ #include "util/os_info.h" -#include -#include -#include - #include -#include -#include namespace doris { diff --git a/be/src/util/os_util.cpp b/be/src/util/os_util.cpp index b59b12bee9..0b942c5f04 100644 --- a/be/src/util/os_util.cpp +++ b/be/src/util/os_util.cpp @@ -25,21 +25,16 @@ #include #include -#include -#include +#include #include -#include #include #include "gutil/macros.h" #include "gutil/strings/numbers.h" #include "gutil/strings/split.h" -#include "gutil/strings/stringpiece.h" #include "gutil/strings/substitute.h" -#include "gutil/strings/util.h" #include "io/fs/fs_utils.h" #include "io/fs/local_file_system.h" -#include "util/faststring.h" using std::string; using std::vector; diff --git a/be/src/util/os_util.h b/be/src/util/os_util.h index 42b3d61572..f830136b68 100644 --- a/be/src/util/os_util.h +++ b/be/src/util/os_util.h @@ -22,7 +22,6 @@ #include #include -#include #include "common/status.h" diff --git a/be/src/util/parse_util.h b/be/src/util/parse_util.h index 14c8cb85ba..22ad2940b3 100644 --- a/be/src/util/parse_util.h +++ b/be/src/util/parse_util.h @@ -20,6 +20,8 @@ #pragma once +#include + #include namespace doris { diff --git a/be/src/util/path_util.cpp b/be/src/util/path_util.cpp index 3de2f106c3..d3e860044f 100644 --- a/be/src/util/path_util.cpp +++ b/be/src/util/path_util.cpp @@ -17,12 +17,9 @@ #include "util/path_util.h" -#include -#include // Use the POSIX version of dirname(3). See `man 3 dirname` #include -#include "common/logging.h" #include "gutil/strings/split.h" #include "gutil/strings/stringpiece.h" #include "gutil/strings/strip.h" diff --git a/be/src/util/perf_counters.cpp b/be/src/util/perf_counters.cpp index d802cc77c8..e1b4bb7de9 100644 --- a/be/src/util/perf_counters.cpp +++ b/be/src/util/perf_counters.cpp @@ -21,19 +21,20 @@ #include "util/perf_counters.h" #include -#include #include #include #include +#include #include -#include +#include // IWYU pragma: keep #include #include -#include +#include +#include +#include "gutil/stringprintf.h" #include "gutil/strings/substitute.h" -#include "util/debug_util.h" #include "util/pretty_printer.h" #include "util/string_parser.hpp" #include "util/string_util.h" diff --git a/be/src/util/perf_counters.h b/be/src/util/perf_counters.h index 1a19f6025b..9f0936193d 100644 --- a/be/src/util/perf_counters.h +++ b/be/src/util/perf_counters.h @@ -20,12 +20,13 @@ #pragma once +#include +#include + #include #include #include -#include "util/debug_util.h" - // This is a utility class that aggregates counters from the kernel. These counters // come from different sources. // - perf counter syscall (/usr/include/linux/perf_event.h") diff --git a/be/src/util/pprof_utils.cpp b/be/src/util/pprof_utils.cpp index 2025a0a4a7..18f62fb1de 100644 --- a/be/src/util/pprof_utils.cpp +++ b/be/src/util/pprof_utils.cpp @@ -17,7 +17,12 @@ #include "util/pprof_utils.h" -#include +#include +#include + +#include +#include // IWYU pragma: keep +#include #include "agent/utils.h" #include "gutil/strings/substitute.h" diff --git a/be/src/util/pprof_utils.h b/be/src/util/pprof_utils.h index bef4c64dcb..461ec56280 100644 --- a/be/src/util/pprof_utils.h +++ b/be/src/util/pprof_utils.h @@ -17,8 +17,10 @@ #pragma once +#include + #include -#include +#include #include "common/status.h" diff --git a/be/src/util/pretty_printer.h b/be/src/util/pretty_printer.h index 501d15d42a..2f48eae33a 100644 --- a/be/src/util/pretty_printer.h +++ b/be/src/util/pretty_printer.h @@ -20,12 +20,13 @@ #pragma once +#include + #include #include #include #include -#include "gen_cpp/RuntimeProfile_types.h" #include "util/binary_cast.hpp" #include "util/cpu_info.h" diff --git a/be/src/util/progress_updater.cpp b/be/src/util/progress_updater.cpp index c3f3670324..320b37ed31 100644 --- a/be/src/util/progress_updater.cpp +++ b/be/src/util/progress_updater.cpp @@ -20,6 +20,8 @@ #include "util/progress_updater.h" +#include + #include "common/logging.h" namespace doris { diff --git a/be/src/util/progress_updater.h b/be/src/util/progress_updater.h index 5f241673af..f37de88335 100644 --- a/be/src/util/progress_updater.h +++ b/be/src/util/progress_updater.h @@ -20,6 +20,8 @@ #pragma once +#include + #include namespace doris { diff --git a/be/src/util/quantile_state.cpp b/be/src/util/quantile_state.cpp index d991f77953..40aba9006f 100644 --- a/be/src/util/quantile_state.cpp +++ b/be/src/util/quantile_state.cpp @@ -19,9 +19,13 @@ #include #include +#include +#include #include "common/logging.h" #include "util/coding.h" +#include "util/slice.h" +#include "util/tdigest.h" namespace doris { diff --git a/be/src/util/quantile_state.h b/be/src/util/quantile_state.h index c3af66c495..6f3a92b2a4 100644 --- a/be/src/util/quantile_state.h +++ b/be/src/util/quantile_state.h @@ -17,16 +17,17 @@ #pragma once +#include +#include + +#include #include -#include #include #include "slice.h" -#include "tdigest.h" namespace doris { -struct Slice; class TDigest; const static int QUANTILE_STATE_EXPLICIT_NUM = 2048; diff --git a/be/src/util/radix_sort.h b/be/src/util/radix_sort.h index a3738ac3d1..463af1c772 100644 --- a/be/src/util/radix_sort.h +++ b/be/src/util/radix_sort.h @@ -37,7 +37,8 @@ #include #include -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep namespace doris { diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp index 32e76c94eb..186c61fa56 100644 --- a/be/src/util/runtime_profile.cpp +++ b/be/src/util/runtime_profile.cpp @@ -20,17 +20,19 @@ #include "util/runtime_profile.h" +#include +#include +#include +#include +#include +#include +#include + #include #include -#include -#include "common/config.h" #include "common/object_pool.h" #include "util/container_util.hpp" -#include "util/cpu_info.h" -#include "util/debug_util.h" -#include "util/thrift_util.h" -#include "util/url_coding.h" namespace doris { diff --git a/be/src/util/runtime_profile.h b/be/src/util/runtime_profile.h index c80507110e..cd37f9fd43 100644 --- a/be/src/util/runtime_profile.h +++ b/be/src/util/runtime_profile.h @@ -20,24 +20,32 @@ #pragma once -#include -#include +#include +#include +#include +#include #include #include #include +#include +#include #include -#include +#include +#include +#include +#include -#include "gen_cpp/RuntimeProfile_types.h" -#include "rapidjson/stringbuffer.h" -#include "rapidjson/writer.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "util/binary_cast.hpp" #include "util/pretty_printer.h" #include "util/stopwatch.hpp" #include "util/telemetry/telemetry.h" namespace doris { +class TRuntimeProfileNode; +class TRuntimeProfileTree; // Some macro magic to generate unique ids using __COUNTER__ #define CONCAT_IMPL(x, y) x##y @@ -109,10 +117,6 @@ public: TUnit::type _type; }; - class DerivedCounter; - class EventSequence; - class HighWaterMarkCounter; - /// A counter that keeps track of the highest value seen (reporting that /// as value()) and the current value. class HighWaterMarkCounter : public Counter { diff --git a/be/src/util/s3_uri.cpp b/be/src/util/s3_uri.cpp index c8f5350d12..08041fece8 100644 --- a/be/src/util/s3_uri.cpp +++ b/be/src/util/s3_uri.cpp @@ -17,9 +17,9 @@ #include "util/s3_uri.h" +#include #include -#include "common/logging.h" #include "gutil/strings/split.h" #include "gutil/strings/strip.h" diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index 1c01297548..02508904aa 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -17,10 +17,21 @@ #include "util/s3_util.h" +#include #include +#include +#include +#include +#include #include #include +#include +#include +#include +#include +#include + #include "common/config.h" #include "common/logging.h" #include "s3_uri.h" diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 382a203d5b..6107fae4a7 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -18,10 +18,14 @@ #pragma once #include +#include +#include #include #include +#include #include +#include #include "common/status.h" #include "gutil/hash/hash.h" diff --git a/be/src/util/slice.h b/be/src/util/slice.h index 1a7ee78a01..8049de003d 100644 --- a/be/src/util/slice.h +++ b/be/src/util/slice.h @@ -25,6 +25,7 @@ #include #include #include +#include #include namespace doris { diff --git a/be/src/util/sm3.cpp b/be/src/util/sm3.cpp index 25f59c1f6b..2d4349abc8 100644 --- a/be/src/util/sm3.cpp +++ b/be/src/util/sm3.cpp @@ -17,6 +17,9 @@ #include "util/sm3.h" +#include +#include + #include "common/logging.h" namespace doris { diff --git a/be/src/util/sm3.h b/be/src/util/sm3.h index d2e93988ce..7f0ab12a72 100644 --- a/be/src/util/sm3.h +++ b/be/src/util/sm3.h @@ -19,7 +19,8 @@ #define SM3_DIGEST_LENGTH 32 -#include +#include +#include #include diff --git a/be/src/util/sort_heap.h b/be/src/util/sort_heap.h index 108017834b..f3f6e546b7 100644 --- a/be/src/util/sort_heap.h +++ b/be/src/util/sort_heap.h @@ -22,7 +22,8 @@ #include #include -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep namespace doris { diff --git a/be/src/util/sse_util.hpp b/be/src/util/sse_util.hpp index 1d00a54fba..a14c281c20 100644 --- a/be/src/util/sse_util.hpp +++ b/be/src/util/sse_util.hpp @@ -23,8 +23,10 @@ #if defined(__aarch64__) #include // IWYU pragma: export #elif defined(__x86_64__) +#include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export +#include // IWYU pragma: export #endif namespace doris { diff --git a/be/src/util/static_asserts.cpp b/be/src/util/static_asserts.cpp index 5258a3fecd..2493587fb8 100644 --- a/be/src/util/static_asserts.cpp +++ b/be/src/util/static_asserts.cpp @@ -18,9 +18,11 @@ // https://github.com/apache/impala/blob/branch-2.9.0/be/src/util/static-asserts.cpp // and modified by Doris -#include "runtime/datetime_value.h" +#include + #include "vec/common/string_ref.h" #include "vec/runtime/vdatetime_value.h" + namespace doris { // This class is unused. It contains static (compile time) asserts. // This is useful to validate struct sizes and other similar things diff --git a/be/src/util/string_parser.hpp b/be/src/util/string_parser.hpp index 4dbca73b23..2910cad9c9 100644 --- a/be/src/util/string_parser.hpp +++ b/be/src/util/string_parser.hpp @@ -21,20 +21,31 @@ #pragma once #include +#include +#include +#include -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include -#include #include +#include #include +#include #include +#include -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" -#include "runtime/primitive_type.h" #include "vec/data_types/data_type_decimal.h" namespace doris { +namespace vectorized { +struct Int128I; +template +struct Decimal; +} // namespace vectorized // Utility functions for doing atoi/atof on non-null terminated strings. On micro benchmarks, // this is significantly faster than libc (atoi/strtol and atof/strtod). diff --git a/be/src/util/string_util.h b/be/src/util/string_util.h index b0bfdbd7e9..c118e5ddab 100644 --- a/be/src/util/string_util.h +++ b/be/src/util/string_util.h @@ -20,7 +20,10 @@ #include #include +#include #include +#include +#include #include #include #include diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp index 2f259d343f..fa8f5a181a 100644 --- a/be/src/util/system_metrics.cpp +++ b/be/src/util/system_metrics.cpp @@ -17,13 +17,22 @@ #include "util/system_metrics.h" +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include #include +#include +#include #include +#include +#include +#include #include "gutil/strings/split.h" // for string split #include "gutil/strtoint.h" // for atoi64 -#include "util/doris_metrics.h" #include "util/mem_info.h" #include "util/perf_counters.h" diff --git a/be/src/util/system_metrics.h b/be/src/util/system_metrics.h index 19ccfe0726..830b11e04c 100644 --- a/be/src/util/system_metrics.h +++ b/be/src/util/system_metrics.h @@ -17,8 +17,14 @@ #pragma once +#include +#include + #include #include +#include +#include +#include #include "util/metrics.h" diff --git a/be/src/util/telemetry/brpc_carrier.cpp b/be/src/util/telemetry/brpc_carrier.cpp index 5d49b40fff..a3c9275009 100644 --- a/be/src/util/telemetry/brpc_carrier.cpp +++ b/be/src/util/telemetry/brpc_carrier.cpp @@ -17,6 +17,8 @@ #include "brpc_carrier.h" +#include + opentelemetry::nostd::string_view doris::telemetry::RpcServerCarrier::Get( opentelemetry::nostd::string_view key) const noexcept { auto it = cntl_->http_request().GetHeader(key.data()); diff --git a/be/src/util/telemetry/brpc_carrier.h b/be/src/util/telemetry/brpc_carrier.h index 60a7abf784..0e6b4d2d47 100644 --- a/be/src/util/telemetry/brpc_carrier.h +++ b/be/src/util/telemetry/brpc_carrier.h @@ -18,6 +18,16 @@ #pragma once #include +#include +#include +#include +#include +#include +#include +#include + +#include +#include #include "opentelemetry/context/propagation/global_propagator.h" #include "opentelemetry/context/propagation/text_map_propagator.h" @@ -27,6 +37,12 @@ #include "opentelemetry/trace/span_startoptions.h" #include "util/telemetry/telemetry.h" +namespace google { +namespace protobuf { +class RpcController; +} // namespace protobuf +} // namespace google + namespace doris::telemetry { class RpcServerCarrier : public opentelemetry::context::propagation::TextMapCarrier { diff --git a/be/src/util/telemetry/open_telemetry_scop_wrapper.hpp b/be/src/util/telemetry/open_telemetry_scop_wrapper.hpp index 4c30333476..2f1d8eb7b2 100644 --- a/be/src/util/telemetry/open_telemetry_scop_wrapper.hpp +++ b/be/src/util/telemetry/open_telemetry_scop_wrapper.hpp @@ -17,7 +17,8 @@ #pragma once -#include "common/compiler_util.h" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "opentelemetry/trace/provider.h" namespace doris { diff --git a/be/src/util/telemetry/telemetry.cpp b/be/src/util/telemetry/telemetry.cpp index 44904355f4..1b69da8356 100644 --- a/be/src/util/telemetry/telemetry.cpp +++ b/be/src/util/telemetry/telemetry.cpp @@ -17,7 +17,22 @@ #include "telemetry.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include +#include #include "common/config.h" #include "common/logging.h" diff --git a/be/src/util/telemetry/telemetry.h b/be/src/util/telemetry/telemetry.h index 5b20acc96e..c007269550 100644 --- a/be/src/util/telemetry/telemetry.h +++ b/be/src/util/telemetry/telemetry.h @@ -17,8 +17,19 @@ #pragma once +#include +#include +#include +#include +#include +#include +#include + +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "opentelemetry/trace/provider.h" -#include "util/telemetry/open_telemetry_scop_wrapper.hpp" /// A trace represents the execution process of a single request in the system, span represents a /// logical operation unit with start time and execution duration in the system, and multiple spans diff --git a/be/src/util/thread.cpp b/be/src/util/thread.cpp index aaee79fe7d..76b38195b3 100644 --- a/be/src/util/thread.cpp +++ b/be/src/util/thread.cpp @@ -21,6 +21,8 @@ #include "thread.h" #ifndef __APPLE__ +// IWYU pragma: no_include +#include #include #else #include @@ -28,22 +30,32 @@ #include #endif -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +#include +#include #include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include #include #include +#include +#include #include +#include #include "common/logging.h" #include "gutil/atomicops.h" #include "gutil/dynamic_annotations.h" #include "gutil/map-util.h" +#include "gutil/stringprintf.h" #include "gutil/strings/substitute.h" -#include "olap/olap_define.h" +#include "http/web_page_handler.h" #include "util/debug/sanitizer_scopes.h" #include "util/easy_json.h" #include "util/os_util.h" diff --git a/be/src/util/thread.h b/be/src/util/thread.h index 83a66fa71d..9514e4ef76 100644 --- a/be/src/util/thread.h +++ b/be/src/util/thread.h @@ -19,21 +19,20 @@ // and modified by Doris #pragma once - -#ifndef __APPLE__ -#include -#endif - +#include #include +#include -#include +#include +#include +#include #include "common/status.h" #include "gutil/ref_counted.h" -#include "http/web_page_handler.h" #include "util/countdown_latch.h" namespace doris { +class WebPageHandler; class Thread : public RefCountedThreadSafe { public: diff --git a/be/src/util/threadpool.cpp b/be/src/util/threadpool.cpp index adae72c623..93c14f4d61 100644 --- a/be/src/util/threadpool.cpp +++ b/be/src/util/threadpool.cpp @@ -20,14 +20,16 @@ #include "util/threadpool.h" +#include #include #include #include #include +#include #include "common/logging.h" -#include "gutil/macros.h" #include "gutil/map-util.h" +#include "gutil/port.h" #include "gutil/strings/substitute.h" #include "util/debug/sanitizer_scopes.h" #include "util/scoped_cleanup.h" diff --git a/be/src/util/threadpool.h b/be/src/util/threadpool.h index 89052274b0..d1b744b63e 100644 --- a/be/src/util/threadpool.h +++ b/be/src/util/threadpool.h @@ -20,26 +20,31 @@ #pragma once +#include +#include + +#include #include #include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include +#include #include +#include #include -#include #include -#include #include "common/status.h" -#include "gutil/ref_counted.h" -#include "util/priority_thread_pool.hpp" namespace doris { class Thread; class ThreadPool; class ThreadPoolToken; +class PriorityThreadPool; class Runnable { public: diff --git a/be/src/util/thrift_client.cpp b/be/src/util/thrift_client.cpp index 82bf291047..d133b45157 100644 --- a/be/src/util/thrift_client.cpp +++ b/be/src/util/thrift_client.cpp @@ -17,8 +17,12 @@ #include "util/thrift_client.h" -#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include +#include #include "gutil/strings/substitute.h" diff --git a/be/src/util/thrift_client.h b/be/src/util/thrift_client.h index 0bc7c1b95c..2328f29845 100644 --- a/be/src/util/thrift_client.h +++ b/be/src/util/thrift_client.h @@ -17,24 +17,26 @@ #pragma once -#include -#include -#include -#include -#include #include #include #include -#include +#include #include #include #include "common/logging.h" #include "common/status.h" -#include "gen_cpp/Types_types.h" #include "util/thrift_server.h" +namespace apache { +namespace thrift { +namespace transport { +class TTransport; +} // namespace transport +} // namespace thrift +} // namespace apache + namespace doris { // Super class for templatized thrift clients. class ThriftClientImpl { diff --git a/be/src/util/thrift_rpc_helper.cpp b/be/src/util/thrift_rpc_helper.cpp index d6889cd258..309d2fcedf 100644 --- a/be/src/util/thrift_rpc_helper.cpp +++ b/be/src/util/thrift_rpc_helper.cpp @@ -17,18 +17,33 @@ #include "util/thrift_rpc_helper.h" +#include +#include +#include +#include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include "common/status.h" -#include "gen_cpp/FrontendService.h" -#include "gen_cpp/FrontendService_types.h" #include "runtime/client_cache.h" -#include "runtime/exec_env.h" -#include "runtime/runtime_state.h" +#include "runtime/exec_env.h" // IWYU pragma: keep #include "util/network_util.h" -#include "util/runtime_profile.h" -#include "util/thrift_util.h" + +namespace apache { +namespace thrift { +namespace protocol { +class TProtocol; +} // namespace protocol +namespace transport { +class TBufferedTransport; +class TSocket; +class TTransport; +} // namespace transport +} // namespace thrift +} // namespace apache namespace doris { diff --git a/be/src/util/thrift_rpc_helper.h b/be/src/util/thrift_rpc_helper.h index 20765b5877..3b2bfb4551 100644 --- a/be/src/util/thrift_rpc_helper.h +++ b/be/src/util/thrift_rpc_helper.h @@ -17,14 +17,17 @@ #pragma once +#include + +#include +#include + #include "common/config.h" #include "common/status.h" -#include "gen_cpp/FrontendService_types.h" namespace doris { class ExecEnv; -class FrontendServiceClient; template class ClientConnection; diff --git a/be/src/util/thrift_server.cpp b/be/src/util/thrift_server.cpp index 7a11de2aa9..335a44a367 100644 --- a/be/src/util/thrift_server.cpp +++ b/be/src/util/thrift_server.cpp @@ -17,17 +17,22 @@ #include "util/thrift_server.h" -#include +#include +#include #include #include #include +#include #include #include #include +#include #include #include #include - +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include #include @@ -35,6 +40,16 @@ #include "util/doris_metrics.h" +namespace apache { +namespace thrift { +class TProcessor; + +namespace transport { +class TServerTransport; +} // namespace transport +} // namespace thrift +} // namespace apache + namespace doris { DEFINE_GAUGE_METRIC_PROTOTYPE_3ARG(thrift_current_connections, MetricUnit::CONNECTIONS, diff --git a/be/src/util/thrift_server.h b/be/src/util/thrift_server.h index dacde385e6..18dd5afd57 100644 --- a/be/src/util/thrift_server.h +++ b/be/src/util/thrift_server.h @@ -17,16 +17,23 @@ #pragma once -#include #include +#include #include +#include #include #include #include "common/status.h" #include "util/metrics.h" +namespace apache { +namespace thrift { +class TProcessor; +} // namespace thrift +} // namespace apache + namespace doris { // Utility class for all Thrift servers. Runs a TNonblockingServer(default) or a // TThreadPoolServer with, by default, 2 worker threads, that exposes the interface @@ -137,6 +144,7 @@ private: // Helper class which monitors starting servers. Needs access to internal members, and // is not used outside of this class. class ThriftServerEventProcessor; + friend class ThriftServerEventProcessor; std::shared_ptr _thrift_server_metric_entity; diff --git a/be/src/util/thrift_util.cpp b/be/src/util/thrift_util.cpp index f533eeb1c1..af0a206d4e 100644 --- a/be/src/util/thrift_util.cpp +++ b/be/src/util/thrift_util.cpp @@ -17,18 +17,27 @@ #include "util/thrift_util.h" -#include -#include -#include -#include -#include +#include +#include +#include #include +#include +// IWYU pragma: no_include +#include // IWYU pragma: keep -#include "gen_cpp/Data_types.h" -#include "gen_cpp/Types_types.h" -#include "util/hash_util.hpp" +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/logging.h" #include "util/thrift_server.h" +namespace apache { +namespace thrift { +namespace protocol { +class TProtocol; +} // namespace protocol +} // namespace thrift +} // namespace apache + // TCompactProtocol requires some #defines to work right. They also define UNLIKELY // so we need to undef this. // TODO: is there a better include to use? @@ -39,6 +48,9 @@ #define ARITHMETIC_RIGHT_SHIFT 1 #include +#include +#include + namespace doris { ThriftSerializer::ThriftSerializer(bool compact, int initial_buffer_size) diff --git a/be/src/util/thrift_util.h b/be/src/util/thrift_util.h index 3d9a82421d..6b40dd6906 100644 --- a/be/src/util/thrift_util.h +++ b/be/src/util/thrift_util.h @@ -17,16 +17,27 @@ #pragma once +#include +#include #include -#include -#include #include -#include +#include +#include +#include #include #include "common/status.h" +namespace apache { +namespace thrift { +namespace protocol { +class TProtocol; +class TProtocolFactory; +} // namespace protocol +} // namespace thrift +} // namespace apache + namespace doris { class TNetworkAddress; diff --git a/be/src/util/time.cpp b/be/src/util/time.cpp index b75ddeeb47..7a7f510905 100644 --- a/be/src/util/time.cpp +++ b/be/src/util/time.cpp @@ -20,9 +20,13 @@ #include "util/time.h" -#include +// IWYU pragma: no_include +#include // IWYU pragma: keep +// IWYU pragma: no_include +#include // IWYU pragma: keep #include #include +#include #include #include diff --git a/be/src/util/time_lut.h b/be/src/util/time_lut.h index 142700d079..3955d8c8ed 100644 --- a/be/src/util/time_lut.h +++ b/be/src/util/time_lut.h @@ -21,7 +21,6 @@ #include #include -#include namespace doris { diff --git a/be/src/util/timezone_utils.cpp b/be/src/util/timezone_utils.cpp index b40a61d129..2b22e52126 100644 --- a/be/src/util/timezone_utils.cpp +++ b/be/src/util/timezone_utils.cpp @@ -18,6 +18,9 @@ #include "util/timezone_utils.h" +#include +#include + namespace doris { RE2 TimezoneUtils::time_zone_offset_format_reg("^[+-]{1}\\d{2}\\:\\d{2}$"); diff --git a/be/src/util/timezone_utils.h b/be/src/util/timezone_utils.h index 7fbe0ef9e2..b74ce9e4b1 100644 --- a/be/src/util/timezone_utils.h +++ b/be/src/util/timezone_utils.h @@ -20,7 +20,11 @@ #include -#include "cctz/time_zone.h" +#include + +namespace cctz { +class time_zone; +} // namespace cctz namespace doris { diff --git a/be/src/util/trace.cpp b/be/src/util/trace.cpp index 20a8a9f5c0..01b396ee1d 100644 --- a/be/src/util/trace.cpp +++ b/be/src/util/trace.cpp @@ -18,18 +18,22 @@ #include "util/trace.h" #include -#include +#include +#include +#include -#include +#include #include #include #include +#include #include #include #include #include #include +#include "gutil/stringprintf.h" #include "gutil/strings/substitute.h" //#include "util/memory/arena.h" diff --git a/be/src/util/trace.h b/be/src/util/trace.h index 35bf9c2619..1ebd602b65 100644 --- a/be/src/util/trace.h +++ b/be/src/util/trace.h @@ -16,14 +16,16 @@ // under the License. #pragma once +#include +#include #include +#include #include -#include #include #include +#include -#include "gutil/macros.h" #include "gutil/ref_counted.h" #include "gutil/strings/stringpiece.h" #include "gutil/strings/substitute.h" diff --git a/be/src/util/trace_metrics.h b/be/src/util/trace_metrics.h index ea014ae77a..37f43b5924 100644 --- a/be/src/util/trace_metrics.h +++ b/be/src/util/trace_metrics.h @@ -16,12 +16,13 @@ // under the License. #pragma once +#include + #include #include #include #include -#include "gutil/macros.h" #include "gutil/map-util.h" #include "util/spinlock.h" diff --git a/be/src/util/uid_util.cpp b/be/src/util/uid_util.cpp index 1fc6c4d457..6743c05a84 100644 --- a/be/src/util/uid_util.cpp +++ b/be/src/util/uid_util.cpp @@ -17,6 +17,12 @@ #include "util/uid_util.h" +#include +#include +#include + +#include + #include "util/hash_util.hpp" namespace doris { diff --git a/be/src/util/uid_util.h b/be/src/util/uid_util.h index 498a1db503..c187bb9d6d 100644 --- a/be/src/util/uid_util.h +++ b/be/src/util/uid_util.h @@ -17,11 +17,19 @@ #pragma once +#include +#include +#include + +#include +#include +#include +#include +#include #include #include +#include -#include "gen_cpp/Types_types.h" // for TUniqueId -#include "gen_cpp/types.pb.h" // for PUniqueId #include "util/uuid_generator.h" namespace doris { diff --git a/be/src/util/url_coding.cpp b/be/src/util/url_coding.cpp index a3d64fb61b..0a5c4144c5 100644 --- a/be/src/util/url_coding.cpp +++ b/be/src/util/url_coding.cpp @@ -19,12 +19,9 @@ #include -#include #include #include -#include "common/logging.h" - namespace doris { static inline void url_encode(const char* in, int in_len, std::string* out) { diff --git a/be/src/util/url_coding.h b/be/src/util/url_coding.h index e1fd26e432..f4d63be5b1 100644 --- a/be/src/util/url_coding.h +++ b/be/src/util/url_coding.h @@ -17,7 +17,10 @@ #pragma once +#include + #include +#include #include #include diff --git a/be/src/util/url_parser.cpp b/be/src/util/url_parser.cpp index e54281eb2c..149398b39a 100644 --- a/be/src/util/url_parser.cpp +++ b/be/src/util/url_parser.cpp @@ -17,8 +17,13 @@ #include "util/url_parser.h" +#include +#include + +#include #include +#include "runtime/string_search.hpp" #include "vec/common/string_ref.h" namespace doris { diff --git a/be/src/util/url_parser.h b/be/src/util/url_parser.h index 882ae251ff..3c30a4ef3f 100644 --- a/be/src/util/url_parser.h +++ b/be/src/util/url_parser.h @@ -17,10 +17,10 @@ #pragma once -#include "runtime/string_search.hpp" #include "vec/common/string_ref.h" namespace doris { +class StringSearch; // TODO: For now, our parse_url may not behave exactly like Hive // when given malformed URLs. diff --git a/be/src/util/zlib.cpp b/be/src/util/zlib.cpp index c4143ca232..7de336e1ac 100644 --- a/be/src/util/zlib.cpp +++ b/be/src/util/zlib.cpp @@ -17,17 +17,14 @@ #include "util/zlib.h" -#include +#include #include -#include #include #include +#include #include -#include "gutil/macros.h" -#include "gutil/strings/substitute.h" - using std::ostream; using std::string; diff --git a/be/src/vec/aggregate_functions/aggregate_function.h b/be/src/vec/aggregate_functions/aggregate_function.h index 2d6b07e403..ea3e4897fe 100644 --- a/be/src/vec/aggregate_functions/aggregate_function.h +++ b/be/src/vec/aggregate_functions/aggregate_function.h @@ -22,6 +22,7 @@ #include "vec/columns/column_complex.h" #include "vec/common/hash_table/phmap_fwd_decl.h" +#include "vec/common/string_buffer.hpp" #include "vec/core/block.h" #include "vec/core/column_numbers.h" #include "vec/core/field.h" diff --git a/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.cpp b/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.cpp index 3083b9c7b1..3580ba72e8 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.cpp @@ -17,7 +17,16 @@ #include "vec/aggregate_functions/aggregate_function_approx_count_distinct.h" +#include "util/bitmap_value.h" #include "vec/aggregate_functions/helpers.h" +#include "vec/columns/column_array.h" +#include "vec/columns/column_decimal.h" +#include "vec/columns/column_map.h" +#include "vec/columns/column_string.h" +#include "vec/columns/column_struct.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/utils/template_helpers.hpp" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.h b/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.h index 79a8c23e34..19c0efb13a 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.h +++ b/be/src/vec/aggregate_functions/aggregate_function_approx_count_distinct.h @@ -17,15 +17,34 @@ #pragma once +#include +#include + +#include +#include +#include +#include + #include "olap/hll.h" -#include "udf/udf.h" +#include "util/hash_util.hpp" +#include "util/slice.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" -#include "vec/common/hash_table/hash.h" +#include "vec/columns/columns_number.h" #include "vec/common/string_ref.h" +#include "vec/core/types.h" #include "vec/data_types/data_type_number.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +class IColumn; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { struct AggregateFunctionApproxCountDistinctData { diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.cpp b/be/src/vec/aggregate_functions/aggregate_function_avg.cpp index f6fe08a9e3..2e58ccb56e 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_avg.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_avg.cpp @@ -22,6 +22,7 @@ #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/helpers.h" +#include "vec/core/field.h" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.h b/be/src/vec/aggregate_functions/aggregate_function_avg.h index 125ab1b8ee..1528881203 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_avg.h +++ b/be/src/vec/aggregate_functions/aggregate_function_avg.h @@ -20,14 +20,40 @@ #pragma once -#include "common/status.h" +#include +#include + +#include +#include +#include +#include +#include + +#include "runtime/decimalv2_value.h" #include "vec/aggregate_functions/aggregate_function.h" +#include "vec/columns/column.h" #include "vec/columns/column_fixed_length_object.h" +#include "vec/common/assert_cast.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_decimal.h" #include "vec/data_types/data_type_fixed_length_object.h" -#include "vec/data_types/data_type_number.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +template +class ColumnDecimal; +template +class DataTypeNumber; +template +class ColumnVector; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { template diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h index cc14f1e1b3..c99d5df8d6 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h +++ b/be/src/vec/aggregate_functions/aggregate_function_avg_weighted.h @@ -17,13 +17,34 @@ #pragma once +#include + +#include +#include #include +#include +#include #include "runtime/decimalv2_value.h" +#include "util/binary_cast.hpp" #include "vec/aggregate_functions/aggregate_function.h" +#include "vec/columns/column_vector.h" +#include "vec/common/assert_cast.h" +#include "vec/core/types.h" #include "vec/data_types/data_type_number.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +class IColumn; +template +class ColumnDecimal; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { template diff --git a/be/src/vec/aggregate_functions/aggregate_function_bit.h b/be/src/vec/aggregate_functions/aggregate_function_bit.h index 0714ca6f34..9538633721 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_bit.h +++ b/be/src/vec/aggregate_functions/aggregate_function_bit.h @@ -20,13 +20,27 @@ #pragma once -#include +#include + +#include #include "vec/aggregate_functions/aggregate_function.h" -#include "vec/columns/column_vector.h" -#include "vec/data_types/data_type_number.h" +#include "vec/core/types.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +class IColumn; +template +class DataTypeNumber; +template +class ColumnVector; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { template diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp b/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp index e896ab2996..35e4636dfa 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap.cpp @@ -19,6 +19,8 @@ #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/helpers.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_nullable.h" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h index c7c537d2e3..98d8bb69f4 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_bitmap.h +++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h @@ -17,17 +17,34 @@ #pragma once +#include + +#include +#include +#include +#include #include +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "util/bitmap_value.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/columns/column_complex.h" #include "vec/columns/column_nullable.h" +#include "vec/columns/column_vector.h" #include "vec/common/assert_cast.h" +#include "vec/core/types.h" #include "vec/data_types/data_type_bitmap.h" -#include "vec/data_types/data_type_nullable.h" #include "vec/data_types/data_type_number.h" -#include "vec/io/io_helper.h" + +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +class IColumn; +} // namespace vectorized +} // namespace doris namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.cpp b/be/src/vec/aggregate_functions/aggregate_function_collect.cpp index 957ce4f3fa..d6aab14411 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_collect.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_collect.cpp @@ -17,6 +17,11 @@ #include "vec/aggregate_functions/aggregate_function_collect.h" +#include + +#include +#include + #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/helpers.h" diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.h b/be/src/vec/aggregate_functions/aggregate_function_collect.h index f0ba7fbe77..206109e8b8 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_collect.h +++ b/be/src/vec/aggregate_functions/aggregate_function_collect.h @@ -17,19 +17,43 @@ #pragma once -#include +#include +#include +#include + +#include +#include +#include +#include -#include "common/status.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/aggregate_functions/key_holder_helpers.h" +#include "vec/columns/column.h" #include "vec/columns/column_array.h" -#include "vec/common/aggregation_common.h" +#include "vec/columns/column_decimal.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/column_string.h" +#include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" #include "vec/common/hash_table/hash_set.h" +#include "vec/common/hash_table/hash_table_key_holder.h" #include "vec/common/pod_array_fwd.h" +#include "vec/common/string_buffer.hpp" #include "vec/common/string_ref.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_array.h" -#include "vec/data_types/data_type_string.h" +#include "vec/data_types/data_type_nullable.h" #include "vec/io/io_helper.h" +#include "vec/io/var_int.h" + +namespace doris { +namespace vectorized { +class Arena; +} // namespace vectorized +} // namespace doris +template +struct DefaultHash; namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_combinator.h b/be/src/vec/aggregate_functions/aggregate_function_combinator.h index cc62e38aea..9f67df02db 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_combinator.h +++ b/be/src/vec/aggregate_functions/aggregate_function_combinator.h @@ -20,11 +20,11 @@ #pragma once -#include -#include - #include +#include "vec/aggregate_functions/aggregate_function.h" +#include "vec/data_types/data_type.h" + namespace doris::vectorized { /** Aggregate function combinator allows to take one aggregate function diff --git a/be/src/vec/aggregate_functions/aggregate_function_count.cpp b/be/src/vec/aggregate_functions/aggregate_function_count.cpp index 2fe3692d9e..8c54714b04 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_count.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_count.cpp @@ -20,6 +20,8 @@ #include "vec/aggregate_functions/aggregate_function_count.h" +#include + #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/factory_helpers.h" diff --git a/be/src/vec/aggregate_functions/aggregate_function_count.h b/be/src/vec/aggregate_functions/aggregate_function_count.h index 63808ac107..389e0f1df4 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_count.h +++ b/be/src/vec/aggregate_functions/aggregate_function_count.h @@ -20,14 +20,31 @@ #pragma once -#include +#include + +#include +#include +#include +#include -#include "common/logging.h" #include "vec/aggregate_functions/aggregate_function.h" +#include "vec/columns/column.h" #include "vec/columns/column_nullable.h" +#include "vec/columns/column_vector.h" +#include "vec/columns/columns_number.h" #include "vec/common/assert_cast.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_number.h" -#include "vec/io/io_helper.h" +#include "vec/io/var_int.h" + +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +} // namespace vectorized +} // namespace doris namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp b/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp index 4d57c689b7..5b2269a27d 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_distinct.cpp @@ -20,9 +20,12 @@ #include "vec/aggregate_functions/aggregate_function_distinct.h" +#include + #include "vec/aggregate_functions/aggregate_function_combinator.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/helpers.h" +#include "vec/data_types/data_type_nullable.h" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_distinct.h b/be/src/vec/aggregate_functions/aggregate_function_distinct.h index 3961e32447..f21a2c0acc 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_distinct.h +++ b/be/src/vec/aggregate_functions/aggregate_function_distinct.h @@ -20,15 +20,39 @@ #pragma once +#include +#include +#include + +#include +#include +#include +#include +#include + #include "vec/aggregate_functions/aggregate_function.h" #include "vec/aggregate_functions/key_holder_helpers.h" -#include "vec/common/aggregation_common.h" +#include "vec/columns/column.h" #include "vec/common/assert_cast.h" -#include "vec/common/field_visitors.h" #include "vec/common/hash_table/hash_set.h" -#include "vec/common/hash_table/hash_table.h" -#include "vec/common/sip_hash.h" +#include "vec/common/hash_table/hash_table_key_holder.h" +#include "vec/common/string_ref.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" #include "vec/io/io_helper.h" +#include "vec/io/var_int.h" + +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +template +class ColumnVector; +} // namespace vectorized +} // namespace doris +template +struct DefaultHash; namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_group_concat.cpp b/be/src/vec/aggregate_functions/aggregate_function_group_concat.cpp index 92a840949f..c43fb44333 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_group_concat.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_group_concat.cpp @@ -17,6 +17,9 @@ #include "vec/aggregate_functions/aggregate_function_group_concat.h" +#include +#include + #include "vec/aggregate_functions/helpers.h" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_group_concat.h b/be/src/vec/aggregate_functions/aggregate_function_group_concat.h index c4316fb2c7..6ddc600452 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_group_concat.h +++ b/be/src/vec/aggregate_functions/aggregate_function_group_concat.h @@ -17,13 +17,28 @@ #pragma once -#include "common/status.h" +#include + +#include +#include + #include "vec/aggregate_functions/aggregate_function.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" +#include "vec/columns/column_string.h" #include "vec/common/string_ref.h" +#include "vec/core/types.h" #include "vec/data_types/data_type_string.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +class IColumn; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { struct AggregateFunctionGroupConcatData { diff --git a/be/src/vec/aggregate_functions/aggregate_function_histogram.cpp b/be/src/vec/aggregate_functions/aggregate_function_histogram.cpp index b396862074..5b06af2839 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_histogram.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_histogram.cpp @@ -17,8 +17,15 @@ #include "vec/aggregate_functions/aggregate_function_histogram.h" +#include +#include + +#include + #include "vec/aggregate_functions/helpers.h" #include "vec/core/types.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_nullable.h" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_histogram.h b/be/src/vec/aggregate_functions/aggregate_function_histogram.h index 392d02693f..4c3f6548f1 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_histogram.h +++ b/be/src/vec/aggregate_functions/aggregate_function_histogram.h @@ -17,10 +17,40 @@ #pragma once +#include +#include + +#include +#include +#include +#include +#include +#include +#include + #include "vec/aggregate_functions/aggregate_function.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" +#include "vec/columns/column.h" +#include "vec/columns/column_decimal.h" +#include "vec/columns/column_string.h" +#include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" +#include "vec/common/string_ref.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_string.h" +#include "vec/io/io_helper.h" #include "vec/utils/histogram_helpers.hpp" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +template +class ColumnVector; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { template diff --git a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp index ec3488e4a1..4f2232d00d 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.cpp @@ -17,6 +17,8 @@ #include "vec/aggregate_functions/aggregate_function_hll_union_agg.h" +#include + #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/helpers.h" diff --git a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h index 3e8ee3cb29..7db9c06875 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h +++ b/be/src/vec/aggregate_functions/aggregate_function_hll_union_agg.h @@ -17,17 +17,35 @@ #pragma once -#include +#include +#include + +#include +#include +#include +#include #include "olap/hll.h" +#include "util/slice.h" #include "vec/aggregate_functions/aggregate_function.h" -#include "vec/columns/column_string.h" -#include "vec/columns/column_vector.h" +#include "vec/columns/column_complex.h" +#include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" +#include "vec/common/string_ref.h" +#include "vec/core/types.h" #include "vec/data_types/data_type_hll.h" #include "vec/data_types/data_type_number.h" -#include "vec/data_types/data_type_string.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +class IColumn; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { struct AggregateFunctionHLLData { diff --git a/be/src/vec/aggregate_functions/aggregate_function_max_by.cpp b/be/src/vec/aggregate_functions/aggregate_function_max_by.cpp index b2c44689bc..35b4a4400d 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_max_by.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_max_by.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include + #include "vec/aggregate_functions/aggregate_function_min_max_by.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_by.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_by.cpp index a860bea088..6e1d96ddf5 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_by.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_min_by.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include + #include "vec/aggregate_functions/aggregate_function_min_max_by.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp index d09beb1d49..27afac7789 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.cpp @@ -23,6 +23,7 @@ #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/factory_helpers.h" #include "vec/aggregate_functions/helpers.h" +#include "vec/data_types/data_type_nullable.h" namespace doris::vectorized { /// min, max, any diff --git a/be/src/vec/aggregate_functions/aggregate_function_min_max.h b/be/src/vec/aggregate_functions/aggregate_function_min_max.h index 1afd8e75f6..4d8a2ae598 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_min_max.h +++ b/be/src/vec/aggregate_functions/aggregate_function_min_max.h @@ -20,15 +20,37 @@ #pragma once +#include +#include + +#include +#include + #include "common/logging.h" #include "vec/aggregate_functions/aggregate_function.h" -#include "vec/columns/column_decimal.h" +#include "vec/columns/column.h" #include "vec/columns/column_fixed_length_object.h" -#include "vec/columns/column_vector.h" +#include "vec/columns/column_string.h" #include "vec/common/assert_cast.h" +#include "vec/common/bit_helpers.h" +#include "vec/common/string_buffer.hpp" +#include "vec/common/string_ref.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_fixed_length_object.h" +#include "vec/data_types/data_type_string.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +template +class ColumnDecimal; +template +class ColumnVector; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { /// For numeric values. diff --git a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp index 3f182a88c3..97269ced37 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp @@ -17,9 +17,17 @@ #include "vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h" +#include +#include + #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/helpers.h" -#include "vec/data_types/data_type_string.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_nullable.h" + +namespace doris { +struct StringRef; +} // namespace doris namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h index 16f16eaf98..737088de60 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h +++ b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h @@ -17,20 +17,38 @@ #pragma once +#include +#include +#include + +#include +#include +#include +#include +#include +#include + #include "util/bitmap_expr_calculation.h" #include "util/bitmap_intersect.h" #include "util/bitmap_value.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/columns/column_complex.h" -#include "vec/columns/column_nullable.h" #include "vec/columns/column_vector.h" -#include "vec/common/assert_cast.h" #include "vec/core/types.h" #include "vec/data_types/data_type_bitmap.h" -#include "vec/data_types/data_type_nullable.h" #include "vec/data_types/data_type_number.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +class ColumnString; +class IColumn; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { template diff --git a/be/src/vec/aggregate_functions/aggregate_function_percentile_approx.cpp b/be/src/vec/aggregate_functions/aggregate_function_percentile_approx.cpp index 1c160e7289..5c54fdf7da 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_percentile_approx.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_percentile_approx.cpp @@ -18,7 +18,6 @@ #include "vec/aggregate_functions/aggregate_function_percentile_approx.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" -#include "vec/aggregate_functions/factory_helpers.h" #include "vec/aggregate_functions/helpers.h" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_percentile_approx.h b/be/src/vec/aggregate_functions/aggregate_function_percentile_approx.h index dc3e232850..b94b922840 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_percentile_approx.h +++ b/be/src/vec/aggregate_functions/aggregate_function_percentile_approx.h @@ -17,16 +17,42 @@ #pragma once +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + #include "util/counts.h" #include "util/tdigest.h" #include "vec/aggregate_functions/aggregate_function.h" +#include "vec/columns/column.h" #include "vec/columns/column_array.h" #include "vec/columns/column_nullable.h" #include "vec/columns/column_vector.h" +#include "vec/common/assert_cast.h" +#include "vec/common/pod_array_fwd.h" +#include "vec/common/string_ref.h" +#include "vec/core/types.h" #include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_nullable.h" #include "vec/data_types/data_type_number.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { struct PercentileApproxState { diff --git a/be/src/vec/aggregate_functions/aggregate_function_quantile_state.cpp b/be/src/vec/aggregate_functions/aggregate_function_quantile_state.cpp index 4c8ec27296..10be5a6357 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_quantile_state.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_quantile_state.cpp @@ -17,7 +17,10 @@ #include "vec/aggregate_functions/aggregate_function_quantile_state.h" +#include + #include "vec/aggregate_functions//aggregate_function_simple_factory.h" +#include "vec/data_types/data_type.h" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_quantile_state.h b/be/src/vec/aggregate_functions/aggregate_function_quantile_state.h index 6b07f79648..00f93abb41 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_quantile_state.h +++ b/be/src/vec/aggregate_functions/aggregate_function_quantile_state.h @@ -17,15 +17,30 @@ #pragma once +#include + +#include +#include + +// IWYU pragma: no_include +#include "common/compiler_util.h" // IWYU pragma: keep #include "util/quantile_state.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/columns/column_complex.h" #include "vec/columns/column_nullable.h" #include "vec/common/assert_cast.h" -#include "vec/data_types/data_type_nullable.h" -#include "vec/data_types/data_type_number.h" +#include "vec/core/types.h" #include "vec/data_types/data_type_quantilestate.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +class IColumn; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { struct AggregateFunctionQuantileStateUnionOp { diff --git a/be/src/vec/aggregate_functions/aggregate_function_reader.cpp b/be/src/vec/aggregate_functions/aggregate_function_reader.cpp index 034eac8313..6f27a5dd12 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_reader.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_reader.cpp @@ -17,6 +17,18 @@ #include "vec/aggregate_functions/aggregate_function_reader.h" +#include +#include + +#include "vec/aggregate_functions/aggregate_function_bitmap.h" +#include "vec/aggregate_functions/aggregate_function_hll_union_agg.h" +#include "vec/aggregate_functions/aggregate_function_min_max.h" +#include "vec/aggregate_functions/aggregate_function_quantile_state.h" +#include "vec/aggregate_functions/aggregate_function_reader_first_last.h" +#include "vec/aggregate_functions/aggregate_function_simple_factory.h" +#include "vec/aggregate_functions/aggregate_function_sum.h" +#include "vec/aggregate_functions/helpers.h" + namespace doris::vectorized { // auto spread at nullable condition, null value do not participate aggregate diff --git a/be/src/vec/aggregate_functions/aggregate_function_reader.h b/be/src/vec/aggregate_functions/aggregate_function_reader.h index a062a7f496..ef35ea16fd 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_reader.h +++ b/be/src/vec/aggregate_functions/aggregate_function_reader.h @@ -17,13 +17,11 @@ #pragma once -#include "vec/aggregate_functions/aggregate_function_bitmap.h" -#include "vec/aggregate_functions/aggregate_function_hll_union_agg.h" -#include "vec/aggregate_functions/aggregate_function_min_max.h" -#include "vec/aggregate_functions/aggregate_function_quantile_state.h" -#include "vec/aggregate_functions/aggregate_function_reader_first_last.h" -#include "vec/aggregate_functions/aggregate_function_simple_factory.h" -#include "vec/aggregate_functions/aggregate_function_sum.h" +namespace doris { +namespace vectorized { +class AggregateFunctionSimpleFactory; +} // namespace vectorized +} // namespace doris namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_retention.h b/be/src/vec/aggregate_functions/aggregate_function_retention.h index 5a01e66810..8eb8db218a 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_retention.h +++ b/be/src/vec/aggregate_functions/aggregate_function_retention.h @@ -21,13 +21,35 @@ #pragma once +#include +#include + +#include +#include +#include + +#include "gutil/integral_types.h" #include "vec/aggregate_functions/aggregate_function.h" +#include "vec/columns/column.h" #include "vec/columns/column_array.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/column_vector.h" #include "vec/columns/columns_number.h" +#include "vec/common/assert_cast.h" +#include "vec/core/types.h" #include "vec/data_types/data_type_array.h" -#include "vec/data_types/data_type_decimal.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_number.h" #include "vec/io/var_int.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { struct RetentionState { static constexpr size_t MAX_EVENTS = 32; diff --git a/be/src/vec/aggregate_functions/aggregate_function_rpc.h b/be/src/vec/aggregate_functions/aggregate_function_rpc.h index eabfdf2fbc..40d314d2b2 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_rpc.h +++ b/be/src/vec/aggregate_functions/aggregate_function_rpc.h @@ -17,11 +17,13 @@ #pragma once +#include +#include + #include #include #include "common/status.h" -#include "gen_cpp/Exprs_types.h" #include "json2pb/json_to_pb.h" #include "json2pb/pb_to_json.h" #include "runtime/exec_env.h" @@ -389,4 +391,4 @@ private: DataTypePtr _return_type; }; -} // namespace doris::vectorized \ No newline at end of file +} // namespace doris::vectorized diff --git a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.cpp b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.cpp index 4f157fa107..bc5b3cbb77 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.cpp @@ -17,9 +17,23 @@ #include "vec/aggregate_functions/aggregate_function_sequence_match.h" +#include + #include "common/logging.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/helpers.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_nullable.h" + +namespace doris { +namespace vectorized { +class VecDateTimeValue; +struct DateTimeV2ValueType; +struct DateV2ValueType; +template +class DateV2Value; +} // namespace vectorized +} // namespace doris namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h index 4fe5b075f1..3e7e0b65e9 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h +++ b/be/src/vec/aggregate_functions/aggregate_function_sequence_match.h @@ -21,16 +21,43 @@ #pragma once +#include + +#include #include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include #include "common/logging.h" +#include "util/binary_cast.hpp" #include "vec/aggregate_functions/aggregate_function.h" -#include "vec/columns/column_array.h" +#include "vec/columns/column_string.h" +#include "vec/columns/column_vector.h" #include "vec/columns/columns_number.h" -#include "vec/data_types/data_type_decimal.h" +#include "vec/common/assert_cast.h" +#include "vec/common/pod_array_fwd.h" +#include "vec/common/string_ref.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_number.h" #include "vec/io/io_helper.h" -#include "vec/io/var_int.h" + +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +class IColumn; +} // namespace vectorized +} // namespace doris namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp index 5649438cdd..8c0ae92c07 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp @@ -20,12 +20,12 @@ #include "vec/aggregate_functions/aggregate_function_simple_factory.h" +#include + #include "vec/aggregate_functions/aggregate_function_reader.h" namespace doris::vectorized { -class AggregateFunctionSimpleFactory; - void register_aggregate_function_combinator_sort(AggregateFunctionSimpleFactory& factory); void register_aggregate_function_combinator_distinct(AggregateFunctionSimpleFactory& factory); diff --git a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h index 4ebc804d5d..bff49e9d9e 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h +++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.h @@ -21,13 +21,13 @@ #pragma once #include -#include #include -#include +#include #include +#include +#include #include "vec/aggregate_functions/aggregate_function.h" -#include "vec/core/field.h" #include "vec/data_types/data_type.h" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_sort.cpp b/be/src/vec/aggregate_functions/aggregate_function_sort.cpp index 8d14621709..c80d36aec5 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_sort.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_sort.cpp @@ -17,6 +17,8 @@ #include "vec/aggregate_functions/aggregate_function_sort.h" +#include + #include "vec/aggregate_functions/aggregate_function_simple_factory.h" namespace doris::vectorized { diff --git a/be/src/vec/aggregate_functions/aggregate_function_sort.h b/be/src/vec/aggregate_functions/aggregate_function_sort.h index 4abdb9c980..c9b192e030 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_sort.h +++ b/be/src/vec/aggregate_functions/aggregate_function_sort.h @@ -17,16 +17,37 @@ #pragma once +#include +#include +#include +#include +#include + +#include +#include +#include #include #include +#include #include "runtime/runtime_state.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/columns/column.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" #include "vec/core/sort_block.h" #include "vec/core/sort_description.h" +#include "vec/core/types.h" #include "vec/io/io_helper.h" +namespace doris { +namespace vectorized { +class Arena; +class BufferReadable; +class BufferWritable; +} // namespace vectorized +} // namespace doris + namespace doris::vectorized { struct AggregateFunctionSortData { diff --git a/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp b/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp index 7e96c2e5a4..2c4fa002d5 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp @@ -17,9 +17,16 @@ #include "vec/aggregate_functions/aggregate_function_stddev.h" +#include + +#include + #include "common/logging.h" #include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/aggregate_functions/helpers.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_nullable.h" + namespace doris::vectorized { template