From 298044a7f75225aba7f45dad9a201f565a4ae5e3 Mon Sep 17 00:00:00 2001 From: zzg19950727 <1071026277@qq.com> Date: Thu, 26 Sep 2024 05:19:26 +0000 Subject: [PATCH] [CP] add _partition_wise_plan_enabled tenant config --- src/share/parameter/ob_parameter_seed.ipp | 4 +++ src/sql/optimizer/ob_join_order.cpp | 11 +++++++ src/sql/optimizer/ob_log_plan.cpp | 8 +++-- src/sql/optimizer/ob_log_plan.h | 4 +-- src/sql/optimizer/ob_logical_operator.cpp | 11 ------- src/sql/optimizer/ob_logical_operator.h | 2 +- src/sql/optimizer/ob_optimizer.cpp | 12 +++++++ src/sql/optimizer/ob_optimizer_context.h | 6 +++- src/sql/optimizer/ob_select_log_plan.cpp | 31 +++++++++++++------ src/sql/resolver/dml/ob_hint.cpp | 5 +++ src/sql/resolver/dml/ob_hint.h | 7 +++-- .../all_virtual_sys_parameter_stat.result | 1 + 12 files changed, 72 insertions(+), 30 deletions(-) diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 3f2872c04..6659b83b4 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -2219,3 +2219,7 @@ DEF_STR_WITH_CHECKER(ob_storage_s3_url_encode_type, OB_CLUSTER_PARAMETER, "defau DEF_INT(ob_encoding_granularity, OB_TENANT_PARAMETER, "65536", "[8192, 1048576]", "Maximum rows for encoding in one micro block. Range:[8192,1048576]", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + +DEF_BOOL(_partition_wise_plan_enabled, OB_TENANT_PARAMETER, "True", + "enable/disable optimizer partition wise plan", + ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 006eb1983..a873e4c89 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -12587,6 +12587,11 @@ int ObJoinOrder::get_valid_path_info_from_hint(const ObRelIds &table_set, } else if (log_hint.is_outline_data_) { // outline data has no pq distributed hint path_info.distributed_methods_ &= DIST_BASIC_METHOD; + } else if (!get_plan()->get_optimizer_context().is_partition_wise_plan_enabled()) { + path_info.distributed_methods_ &= ~DIST_PARTITION_NONE; + path_info.distributed_methods_ &= ~DIST_NONE_PARTITION; + path_info.distributed_methods_ &= ~DIST_PARTITION_WISE; + path_info.distributed_methods_ &= ~DIST_EXT_PARTITION_WISE; } if (NULL != log_join_hint && both_access && !ignore_dist_hint @@ -12676,6 +12681,12 @@ int ObJoinOrder::get_valid_path_info(const ObJoinOrder &left_tree, DIST_EXT_PARTITION_WISE | DIST_BASIC_METHOD | DIST_NONE_ALL | DIST_ALL_NONE | DIST_RANDOM_ALL; + if (!get_plan()->get_optimizer_context().is_partition_wise_plan_enabled()) { + path_info.distributed_methods_ &= ~DIST_PARTITION_NONE; + path_info.distributed_methods_ &= ~DIST_NONE_PARTITION; + path_info.distributed_methods_ &= ~DIST_PARTITION_WISE; + path_info.distributed_methods_ &= ~DIST_EXT_PARTITION_WISE; + } } if (!ignore_hint && OB_FAIL(get_valid_path_info_from_hint(right_tree.get_tables(), both_access, diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 0f13e5146..3ae3a3032 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -5305,7 +5305,7 @@ int ObLogPlan::create_scala_group_plan(const ObIArray &aggr_it is_partition_wise))) { LOG_WARN("failed to check if sharding compatible with distinct expr", K(ret)); } else if (groupby_helper.can_three_stage_pushdown_ && - !(is_partition_wise && groupby_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()))) { + (!is_partition_wise || !get_optimizer_context().is_partition_wise_plan_enabled()) ) { OPT_TRACE("generate three stage group plan"); if (NULL == groupby_helper.aggr_code_expr_ && OB_FAIL(prepare_three_stage_info(dummy_exprs, dummy_exprs, groupby_helper))) { @@ -7966,7 +7966,11 @@ int ObLogPlan::get_valid_subplan_filter_dist_method(ObIArray &subpla if (OB_SUCC(ret) && !ignore_hint) { const bool implicit_hint_allowed = (subplans.count() == get_stmt()->get_subquery_expr_size()); - dist_methods &= get_log_plan_hint().get_valid_pq_subquery_dist_algo(sub_qb_names, implicit_hint_allowed); + dist_methods &= get_log_plan_hint().get_valid_pq_subquery_dist_algo(sub_qb_names, + implicit_hint_allowed); + } else if (ignore_hint && !get_optimizer_context().is_partition_wise_plan_enabled()) { + dist_methods &= ~DIST_PARTITION_WISE; + dist_methods &= ~DIST_PARTITION_NONE; } if (OB_FAIL(ret)) { diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index 40bc4f4cc..8a044a0d3 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -493,9 +493,9 @@ public: void clear_ignore_hint() { ignore_hint_ = false; } inline bool allow_basic() const { return ignore_hint_ || (!force_partition_wise_ && !force_dist_hash_); } inline bool allow_dist_hash() const { return ignore_hint_ || (!force_basic_ && !force_partition_wise_); } - inline bool allow_partition_wise(bool parallel_more_than_part_cnt) const + inline bool allow_partition_wise(bool enable_partition_wise_plan) const { - bool disable_by_rule = parallel_more_than_part_cnt && optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2; + bool disable_by_rule = !enable_partition_wise_plan && optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2; return ignore_hint_ ? !disable_by_rule : (disable_by_rule ? force_partition_wise_ : (!force_basic_ && !force_dist_hash_)); } diff --git a/src/sql/optimizer/ob_logical_operator.cpp b/src/sql/optimizer/ob_logical_operator.cpp index 6a834598f..fa2bb3217 100644 --- a/src/sql/optimizer/ob_logical_operator.cpp +++ b/src/sql/optimizer/ob_logical_operator.cpp @@ -6674,14 +6674,3 @@ int ObLogicalOperator::check_op_orderding_used_by_parent(bool &used) } return ret; } - -bool ObLogicalOperator::is_parallel_more_than_part_cnt() const -{ - if (NULL == strong_sharding_) { - return false; - } else if (strong_sharding_->get_part_cnt() < 1) { - return false; - } else { - return get_parallel() > strong_sharding_->get_part_cnt(); - } -} diff --git a/src/sql/optimizer/ob_logical_operator.h b/src/sql/optimizer/ob_logical_operator.h index 7dd0b4f97..62d41795a 100644 --- a/src/sql/optimizer/ob_logical_operator.h +++ b/src/sql/optimizer/ob_logical_operator.h @@ -1074,7 +1074,7 @@ public: { exchange_allocated_ = exchange_allocated; } - bool is_parallel_more_than_part_cnt() const; + virtual bool is_gi_above() const { return false; } inline void set_phy_plan_type(ObPhyPlanType phy_plan_type) { diff --git a/src/sql/optimizer/ob_optimizer.cpp b/src/sql/optimizer/ob_optimizer.cpp index 40713f16c..6443adbf3 100644 --- a/src/sql/optimizer/ob_optimizer.cpp +++ b/src/sql/optimizer/ob_optimizer.cpp @@ -565,6 +565,8 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession bool has_cursor_expr = false; int64_t link_stmt_count = 0; bool push_join_pred_into_view_enabled = true; + bool partition_wise_plan_enabled = true; + bool exists_partition_wise_plan_enabled_hint = false; omt::ObTenantConfigGuard tenant_config(TENANT_CONF(session.get_effective_tenant_id())); bool rowsets_enabled = tenant_config.is_valid() && tenant_config->_rowsets_enabled; ctx_.set_is_online_ddl(session.get_ddl_info().is_ddl()); // set is online ddl first, is used by other extract operations @@ -627,6 +629,10 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession LOG_WARN("failed to get opt param enable spf batch rescan", K(ret)); } else if (OB_FAIL(ctx_.get_global_hint().opt_params_.get_bool_opt_param(ObOptParamHint::_PUSH_JOIN_PREDICATE, push_join_pred_into_view_enabled))) { LOG_WARN("fail to check rowsets enabled", K(ret)); + } else if (OB_FAIL(opt_params.get_bool_opt_param(ObOptParamHint::PARTITION_WISE_PLAN_ENABLED, + partition_wise_plan_enabled, + exists_partition_wise_plan_enabled_hint))) { + LOG_WARN("failed to check partition wise plan enabled", K(ret)); } else { ctx_.set_storage_estimation_enabled(storage_estimation_enabled); ctx_.set_serial_set_order(force_serial_set_order); @@ -656,6 +662,12 @@ int ObOptimizer::extract_opt_ctx_basic_flags(const ObDMLStmt &stmt, ObSQLSession ctx_.set_merge_join_enabled(optimizer_sortmerge_join_enabled); ctx_.set_nested_join_enabled(nested_loop_join_enabled); } + if (tenant_config.is_valid()) { + ctx_.set_partition_wise_plan_enabled(tenant_config->_partition_wise_plan_enabled); + } + if (exists_partition_wise_plan_enabled_hint) { + ctx_.set_partition_wise_plan_enabled(partition_wise_plan_enabled); + } if (!session.is_inner() && stmt.get_query_ctx()->get_injected_random_status()) { ctx_.set_generate_random_plan(true); } diff --git a/src/sql/optimizer/ob_optimizer_context.h b/src/sql/optimizer/ob_optimizer_context.h index 2279b2ee2..d5a3b2bb1 100644 --- a/src/sql/optimizer/ob_optimizer_context.h +++ b/src/sql/optimizer/ob_optimizer_context.h @@ -259,7 +259,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, correlation_type_(ObEstCorrelationType::MAX), use_column_store_replica_(false), push_join_pred_into_view_enabled_(true), - table_access_policy_(ObTableAccessPolicy::AUTO) + table_access_policy_(ObTableAccessPolicy::AUTO), + partition_wise_plan_enabled_(true) { } inline common::ObOptStatManager *get_opt_stat_manager() { return opt_stat_manager_; } inline void set_opt_stat_manager(common::ObOptStatManager *sm) { opt_stat_manager_ = sm; } @@ -638,6 +639,8 @@ ObOptimizerContext(ObSQLSessionInfo *session_info, inline void set_has_multiple_link_stmt(bool v) { has_multiple_link_stmt_ = v; } inline bool is_hash_join_enabled() const { return hash_join_enabled_; } inline void set_hash_join_enabled(bool enabled) { hash_join_enabled_ = enabled; } + inline bool is_partition_wise_plan_enabled() const { return partition_wise_plan_enabled_; } + inline void set_partition_wise_plan_enabled(bool enabled) { partition_wise_plan_enabled_ = enabled; } inline bool is_merge_join_enabled() const { return optimizer_sortmerge_join_enabled_; } inline void set_merge_join_enabled(bool enabled) { optimizer_sortmerge_join_enabled_ = enabled; } inline bool is_nested_join_enabled() const { return nested_loop_join_enabled_; } @@ -760,6 +763,7 @@ private: bool use_column_store_replica_; bool push_join_pred_into_view_enabled_; ObTableAccessPolicy table_access_policy_; + bool partition_wise_plan_enabled_; }; } } diff --git a/src/sql/optimizer/ob_select_log_plan.cpp b/src/sql/optimizer/ob_select_log_plan.cpp index 15aeba623..1a1a30a2a 100644 --- a/src/sql/optimizer/ob_select_log_plan.cpp +++ b/src/sql/optimizer/ob_select_log_plan.cpp @@ -361,7 +361,7 @@ int ObSelectLogPlan::candi_allocate_three_stage_group_by(const ObIArrayis_distributed() && !groupby_helper.allow_basic()) { OPT_TRACE("ignore basic group by hint"); } else if (candidate_plan.plan_tree_->is_distributed() && !reduce_exprs.empty() - && groupby_helper.allow_partition_wise(candidate_plan.plan_tree_->is_parallel_more_than_part_cnt()) + && groupby_helper.allow_partition_wise(get_optimizer_context().is_partition_wise_plan_enabled()) && OB_FAIL(candidate_plan.plan_tree_->check_sharding_compatible_with_reduce_expr(reduce_exprs, is_partition_wise))) { LOG_WARN("failed to check if sharding compatible with distinct expr", K(ret)); } else if (!candidate_plan.plan_tree_->is_distributed() || is_partition_wise) { @@ -583,7 +583,7 @@ int ObSelectLogPlan::should_create_rollup_pushdown_plan(ObLogicalOperator *top, // do nothing } else if (!top->is_distributed() && !groupby_helper.allow_basic()) { // do nothing - } else if (top->is_distributed() && groupby_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + } else if (top->is_distributed() && groupby_helper.allow_partition_wise(get_optimizer_context().is_partition_wise_plan_enabled()) && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, is_partition_wise))) { LOG_WARN("failed to check is partition wise", K(ret)); @@ -747,7 +747,7 @@ int ObSelectLogPlan::create_hash_group_plan(const ObIArray &reduce_e } else if (!top->is_distributed() && !groupby_helper.allow_basic()) { top = NULL; OPT_TRACE("ignore basic hash group by hint"); - } else if (top->is_distributed() && groupby_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + } else if (top->is_distributed() && groupby_helper.allow_partition_wise(get_optimizer_context().is_partition_wise_plan_enabled()) && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, is_partition_wise))) { LOG_WARN("failed to check if sharding compatible", K(ret)); @@ -1098,7 +1098,7 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray &r } else if (!top->is_distributed() && !groupby_helper.allow_basic()) { top = NULL; OPT_TRACE("ignore basic group by hint"); - } else if (top->is_distributed() && groupby_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + } else if (top->is_distributed() && groupby_helper.allow_partition_wise(get_optimizer_context().is_partition_wise_plan_enabled()) && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, is_partition_wise))) { LOG_WARN("failed to check if sharding compatible with reduce expr", K(ret)); @@ -1612,7 +1612,7 @@ int ObSelectLogPlan::create_hash_distinct_plan(ObLogicalOperator *&top, } else if (!top->is_distributed() && !distinct_helper.allow_basic()) { top = NULL; OPT_TRACE("ignore basic distinct by hint"); - } else if (top->is_distributed() && distinct_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + } else if (top->is_distributed() && distinct_helper.allow_partition_wise(get_optimizer_context().is_partition_wise_plan_enabled()) && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, is_partition_wise))) { LOG_WARN("failed to check sharding compatible with reduce expr", K(ret)); @@ -1708,7 +1708,7 @@ int ObSelectLogPlan::create_merge_distinct_plan(ObLogicalOperator *&top, } else if (need_sort && can_ignore_merge_plan && OrderingFlag::NOT_MATCH == interesting_order_info) { // if no further order needed, not generate merge style distinct top = NULL; - } else if (top->is_distributed() && distinct_helper.allow_partition_wise(top->is_parallel_more_than_part_cnt()) + } else if (top->is_distributed() && distinct_helper.allow_partition_wise(get_optimizer_context().is_partition_wise_plan_enabled()) && OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs, is_partition_wise))) { LOG_WARN("failed to check sharding compatible with reduce exprs", K(ret)); @@ -2216,6 +2216,11 @@ int ObSelectLogPlan::create_union_all_plan(const ObIArray &c set_dist_methods &= hint_dist_methods; } else { random_none_idx = OB_INVALID_INDEX; + if (!get_optimizer_context().is_partition_wise_plan_enabled()) { + set_dist_methods &= ~DistAlgo::DIST_PARTITION_WISE; + set_dist_methods &= ~DistAlgo::DIST_SET_PARTITION_WISE; + set_dist_methods &= ~DistAlgo::DIST_EXT_PARTITION_WISE; + } } if (OB_FAIL(ret)) { //do nothing @@ -3002,6 +3007,12 @@ int ObSelectLogPlan::get_distributed_set_methods(const EqualSets &equal_sets, } else { OPT_TRACE("candi merge set dist method:basic, partition wise, none all, all none, "); } + if (!get_optimizer_context().is_partition_wise_plan_enabled()) { + set_dist_methods &= ~DistAlgo::DIST_PARTITION_WISE; + set_dist_methods &= ~DistAlgo::DIST_PARTITION_NONE; + set_dist_methods &= ~DistAlgo::DIST_NONE_PARTITION; + OPT_TRACE("tenant config disable partition wise plan"); + } } else { OPT_TRACE("use dist method with hint"); } @@ -6097,8 +6108,8 @@ int ObSelectLogPlan::create_none_dist_win_func(ObLogicalOperator *top, } else if (WinDistAlgo::WIN_DIST_NONE == win_func_helper.win_dist_method_ || !top->is_distributed() || (is_partition_wise && - !(top->is_parallel_more_than_part_cnt() && - get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2))) { + (get_optimizer_context().is_partition_wise_plan_enabled() || + get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ <= COMPAT_VERSION_4_3_2))) { LOG_TRACE("begin to create none dist window function", K(top->is_distributed()), K(single_part_parallel), K(is_partition_wise), K(need_sort), K(part_cnt), K(win_func_helper.force_hash_sort_), K(win_func_helper.force_normal_sort_)); @@ -6351,8 +6362,8 @@ int ObSelectLogPlan::create_hash_dist_win_func(ObLogicalOperator *top, is_partition_wise))) { LOG_WARN("failed to check if sharding compatible", K(ret)); } else if (!top->is_distributed() || (is_partition_wise && - !(top->is_parallel_more_than_part_cnt() && - get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ > COMPAT_VERSION_4_3_2))) { + (get_optimizer_context().is_partition_wise_plan_enabled() || + get_optimizer_context().get_query_ctx()->optimizer_features_enable_version_ <= COMPAT_VERSION_4_3_2))) { LOG_TRACE("ignore allocate hash window function for local or partition wise", K(top->is_distributed()), K(is_partition_wise)); } else { diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index 29d0b78d1..4766b9a6c 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -928,6 +928,11 @@ bool ObOptParamHint::is_param_val_valid(const OptParamType param_type, const ObO } break; } + case PARTITION_WISE_PLAN_ENABLED: { + is_valid = val.is_varchar() && (0 == val.get_varchar().case_compare("true") + || 0 == val.get_varchar().case_compare("false")); + break; + } default: LOG_TRACE("invalid opt param val", K(param_type), K(val)); break; diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index 9cb74f215..b7b0dce1b 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -176,11 +176,12 @@ struct ObOptParamHint DEF(RUNTIME_FILTER_TYPE,) \ DEF(BLOOM_FILTER_RATIO,) \ DEF(CORRELATION_FOR_CARDINALITY_ESTIMATION,) \ - DEF(CARDINALITY_ESTIMATION_MODEL,) \ + DEF(CARDINALITY_ESTIMATION_MODEL,) \ DEF(_PUSH_JOIN_PREDICATE,) \ - DEF(RANGE_INDEX_DIVE_LIMIT,) \ - DEF(PARTITION_INDEX_DIVE_LIMIT,) \ + DEF(RANGE_INDEX_DIVE_LIMIT,) \ + DEF(PARTITION_INDEX_DIVE_LIMIT,) \ DEF(OB_TABLE_ACCESS_POLICY,) \ + DEF(PARTITION_WISE_PLAN_ENABLED,) \ DECLARE_ENUM(OptParamType, opt_param, OPT_PARAM_TYPE_DEF, static); diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index 856cd2b4e..778eb4537 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -441,6 +441,7 @@ _parallel_max_active_sessions _parallel_min_message_pool _parallel_redo_logging_trigger _parallel_server_sleep_time +_partition_wise_plan_enabled _pdml_thread_cache_size _pipelined_table_function_memory_limit _preserve_order_for_pagination