From 17698731186a11f5350c09fe4fb2a03708e46623 Mon Sep 17 00:00:00 2001 From: wangt1xiuyi <13547954130@163.com> Date: Thu, 14 Sep 2023 05:40:21 +0000 Subject: [PATCH] fix some optimizer stat bug --- src/pl/sys_package/ob_dbms_stats.cpp | 12 ++- src/share/stat/ob_dbms_stats_executor.cpp | 28 +++---- .../stat/ob_dbms_stats_history_manager.cpp | 75 +---------------- .../stat/ob_dbms_stats_history_manager.h | 9 --- src/share/stat/ob_dbms_stats_utils.cpp | 81 +++++++++++++++++++ src/share/stat/ob_dbms_stats_utils.h | 13 +++ src/share/stat/ob_opt_stat_gather_stat.cpp | 39 +++++++-- src/share/stat/ob_opt_stat_gather_stat.h | 6 ++ .../optimizer/ob_access_path_estimation.cpp | 2 +- src/sql/optimizer/ob_access_path_estimation.h | 4 +- src/sql/optimizer/ob_opt_selectivity.cpp | 3 +- 11 files changed, 160 insertions(+), 112 deletions(-) diff --git a/src/pl/sys_package/ob_dbms_stats.cpp b/src/pl/sys_package/ob_dbms_stats.cpp index 240092a22..b093f7a80 100644 --- a/src/pl/sys_package/ob_dbms_stats.cpp +++ b/src/pl/sys_package/ob_dbms_stats.cpp @@ -3032,7 +3032,9 @@ int ObDbmsStats::update_stat_cache(const uint64_t rpc_tenant_id, LOG_WARN("memory is not enough", K(ret), K(tmp_str)); } else { MEMCPY(buf, tmp_str.ptr(), tmp_str.length()); - running_monitor->opt_stat_gather_stat_.set_stat_refresh_failed_list(buf, tmp_str.length()); + ObString tmp_failed_list(tmp_str.length(), buf); + ObOptStatGatherStatList::instance().update_gather_stat_refresh_failed_list(tmp_failed_list, + running_monitor->opt_stat_gather_stat_); } } } @@ -3990,7 +3992,8 @@ int ObDbmsStats::parse_granularity_and_method_opt(ObExecContext &ctx, { int ret = OB_SUCCESS; //virtual table(not include real agent table) doesn't gather histogram. - bool is_vt = is_virtual_table(param.table_id_); + bool is_vt = is_virtual_table(param.table_id_) && + !share::is_oracle_mapping_real_virtual_table(param.table_id_); bool use_size_auto = false; if (0 == param.method_opt_.case_compare("Z") && !is_vt) { if (OB_FAIL(set_default_column_params(param.column_params_))) { @@ -6117,6 +6120,11 @@ int ObDbmsStats::resovle_granularity(ObGranularityType granu_type, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected granularity type", K(granu_type)); } + //virtual table only gather global stats. + if (OB_SUCC(ret) && is_virtual_table(param.table_id_)) { + param.part_stat_param_.reset_gather_stat(); + param.subpart_stat_param_.reset_gather_stat(); + } LOG_TRACE("succeed to parse granularity", K(param.global_stat_param_), K(param.part_stat_param_), K(param.subpart_stat_param_)); return ret; diff --git a/src/share/stat/ob_dbms_stats_executor.cpp b/src/share/stat/ob_dbms_stats_executor.cpp index 350aacd2a..da14de6bf 100644 --- a/src/share/stat/ob_dbms_stats_executor.cpp +++ b/src/share/stat/ob_dbms_stats_executor.cpp @@ -812,10 +812,8 @@ int ObDbmsStatsExecutor::update_online_stat(ObExecContext &ctx, int ret = OB_SUCCESS; int64_t affected_rows = 0; - - //before write, we need record history stats. - ObSEArray history_tab_handles; - ObSEArray history_col_handles; + ObSEArray cur_tab_handles; + ObSEArray cur_col_handles; ObSEArray table_stats; ObSEArray column_stats; if (OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, param))) { @@ -828,30 +826,24 @@ int ObDbmsStatsExecutor::update_online_stat(ObExecContext &ctx, LOG_WARN("fail to check lock stat", K(ret)); } if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObDbmsStatsHistoryManager::get_history_stat_handles(ctx, - param, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to get history stat handles", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(param, + cur_tab_handles, + cur_col_handles))) { + LOG_WARN("failed to get current opt stats", K(ret)); } else if (OB_FAIL(ObDbmsStatsUtils::merge_tab_stats(param, online_table_stats, - history_tab_handles, + cur_tab_handles, table_stats))) { - LOG_WARN("fail to merge tab stats", K(ret), K(history_tab_handles)); + LOG_WARN("fail to merge tab stats", K(ret), K(cur_tab_handles)); } else if (OB_FAIL(ObDbmsStatsUtils::merge_col_stats(param, online_column_stats, - history_col_handles, + cur_col_handles, column_stats))) { - LOG_WARN("fail to merge col stats", K(ret), K(history_col_handles)); + LOG_WARN("fail to merge col stats", K(ret), K(cur_col_handles)); } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, table_stats, column_stats, false, false, true))) { LOG_WARN("fail to update stat", K(ret), K(table_stats), K(column_stats)); - } else if (OB_FAIL(ObDbmsStatsUtils::batch_write_history_stats(ctx, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to batch write history stats", K(ret)); } else if (OB_FAIL(ObBasicStatsEstimator::update_last_modified_count(ctx, param))) { - //update history LOG_WARN("failed to update last modified count", K(ret)); } else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), param))) { LOG_WARN("fail to update stat cache", K(ret)); diff --git a/src/share/stat/ob_dbms_stats_history_manager.cpp b/src/share/stat/ob_dbms_stats_history_manager.cpp index bd451b5cb..032d2d96e 100644 --- a/src/share/stat/ob_dbms_stats_history_manager.cpp +++ b/src/share/stat/ob_dbms_stats_history_manager.cpp @@ -69,8 +69,6 @@ int ObDbmsStatsHistoryManager::get_history_stat_handles(ObExecContext &ctx, ObIArray &history_col_handles) { int ret = OB_SUCCESS; - ObSEArray part_ids; - ObSEArray column_ids; int64_t retention_val = 0; if (param.is_index_stat_) { //do nothing @@ -78,21 +76,10 @@ int ObDbmsStatsHistoryManager::get_history_stat_handles(ObExecContext &ctx, LOG_WARN("failed to get stats history retention", K(ret)); } else if (retention_val == 0) { /*do nothing*/ - } else if (OB_FAIL(get_part_ids_and_column_ids(param, part_ids, column_ids))) { - LOG_WARN("failed to get part ids and column ids", K(ret)); - } else if (OB_FAIL(erase_stat_cache(param.tenant_id_, param.table_id_, part_ids, column_ids))) { - LOG_WARN("failed to erase stat cache", K(ret)); - } else if (OB_FAIL(ObOptStatManager::get_instance().get_table_stat(param.tenant_id_, - param.table_id_, - part_ids, - history_tab_handles))) { - LOG_WARN("failed to get table stat", K(ret)); - } else if (OB_FAIL(ObOptStatManager::get_instance().get_column_stat(param.tenant_id_, - param.table_id_, - part_ids, - column_ids, - history_col_handles))) { - LOG_WARN("failed to get column stat", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(param, + history_tab_handles, + history_col_handles))) { + LOG_WARN("failed to get current opt stats", K(ret)); } else if (OB_FAIL(set_col_stat_cs_type(param.column_params_, history_col_handles))) { LOG_WARN("failed to set col stat cs type", K(ret)); } else { @@ -695,60 +682,6 @@ int ObDbmsStatsHistoryManager::fill_bucket_stat_histroy(ObIAllocator &allocator, return ret; } -int ObDbmsStatsHistoryManager::get_part_ids_and_column_ids(const ObTableStatParam ¶m, - ObIArray &part_ids, - ObIArray &column_ids) -{ - int ret = OB_SUCCESS; - //get part ids - if (param.global_stat_param_.need_modify_) { - int64_t part_id = param.global_part_id_; - if (OB_FAIL(part_ids.push_back(part_id))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} - } - - if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) { - for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { - if (OB_FAIL(part_ids.push_back(param.part_infos_.at(i).part_id_))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} - } - } - - if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) { - for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { - if (OB_FAIL(part_ids.push_back(param.subpart_infos_.at(i).part_id_))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} - } - } - //get column ids - for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) { - if (OB_FAIL(column_ids.push_back(param.column_params_.at(i).column_id_))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} - } - return ret; -} - -int ObDbmsStatsHistoryManager::erase_stat_cache(const uint64_t tenant_id, - const uint64_t table_id, - const ObIArray &part_ids, - const ObIArray &column_ids) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(ObOptStatManager::get_instance().erase_table_stat(tenant_id, table_id, part_ids))) { - LOG_WARN("failed to erase table stats", K(ret)); - } else if (OB_FAIL(ObOptStatManager::get_instance().erase_column_stat(tenant_id, - table_id, - part_ids, - column_ids))) { - LOG_WARN("failed to erase column stats", K(ret)); - } else {/*do nothing*/} - return ret; -} - int ObDbmsStatsHistoryManager::get_stats_history_retention(ObExecContext &ctx, int64_t &retention_val) { diff --git a/src/share/stat/ob_dbms_stats_history_manager.h b/src/share/stat/ob_dbms_stats_history_manager.h index f204bb6b9..bd27e44f9 100644 --- a/src/share/stat/ob_dbms_stats_history_manager.h +++ b/src/share/stat/ob_dbms_stats_history_manager.h @@ -76,15 +76,6 @@ private: sqlclient::ObMySQLResult &result, ObOptColumnStat &stat); - static int get_part_ids_and_column_ids(const ObTableStatParam ¶m, - ObIArray &part_ids, - ObIArray &column_ids); - - static int erase_stat_cache(const uint64_t tenant_id, - const uint64_t table_id, - const ObIArray &part_ids, - const ObIArray &column_ids); - static int get_stats_history_retention(ObExecContext &ctx, int64_t &retention_val); static int set_col_stat_cs_type(const ObIArray &column_params, diff --git a/src/share/stat/ob_dbms_stats_utils.cpp b/src/share/stat/ob_dbms_stats_utils.cpp index db7c0033b..5bcb8fb3b 100644 --- a/src/share/stat/ob_dbms_stats_utils.cpp +++ b/src/share/stat/ob_dbms_stats_utils.cpp @@ -880,5 +880,86 @@ int64_t ObDbmsStatsUtils::get_truncated_str_len(const ObString &str, const ObCol return truncated_str_len; } +int ObDbmsStatsUtils::get_current_opt_stats(const ObTableStatParam ¶m, + ObIArray &cur_tab_handles, + ObIArray &cur_col_handles) +{ + int ret = OB_SUCCESS; + ObSEArray part_ids; + ObSEArray column_ids; + if (OB_FAIL(get_part_ids_and_column_ids(param, part_ids, column_ids))) { + LOG_WARN("failed to get part ids and column ids", K(ret)); + } else if (OB_FAIL(erase_stat_cache(param.tenant_id_, param.table_id_, + part_ids, column_ids))) { + LOG_WARN("failed to erase stat cache", K(ret)); + } else if (OB_FAIL(ObOptStatManager::get_instance().get_table_stat(param.tenant_id_, + param.table_id_, + part_ids, + cur_tab_handles))) { + LOG_WARN("failed to get table stat", K(ret)); + } else if (OB_FAIL(ObOptStatManager::get_instance().get_column_stat(param.tenant_id_, + param.table_id_, + part_ids, + column_ids, + cur_col_handles))) { + LOG_WARN("failed to get column stat", K(ret)); + } + return ret; +} + +int ObDbmsStatsUtils::get_part_ids_and_column_ids(const ObTableStatParam ¶m, + ObIArray &part_ids, + ObIArray &column_ids) +{ + int ret = OB_SUCCESS; + //get part ids + if (param.global_stat_param_.need_modify_) { + int64_t part_id = param.global_part_id_; + if (OB_FAIL(part_ids.push_back(part_id))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } + + if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) { + for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { + if (OB_FAIL(part_ids.push_back(param.part_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } + } + + if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) { + for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { + if (OB_FAIL(part_ids.push_back(param.subpart_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } + } + //get column ids + for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) { + if (OB_FAIL(column_ids.push_back(param.column_params_.at(i).column_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } + return ret; +} + +int ObDbmsStatsUtils::erase_stat_cache(const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &part_ids, + const ObIArray &column_ids) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObOptStatManager::get_instance().erase_table_stat(tenant_id, table_id, part_ids))) { + LOG_WARN("failed to erase table stats", K(ret)); + } else if (OB_FAIL(ObOptStatManager::get_instance().erase_column_stat(tenant_id, + table_id, + part_ids, + column_ids))) { + LOG_WARN("failed to erase column stats", K(ret)); + } else {/*do nothing*/} + return ret; +} + } } diff --git a/src/share/stat/ob_dbms_stats_utils.h b/src/share/stat/ob_dbms_stats_utils.h index 953d0abef..327eb8ee7 100644 --- a/src/share/stat/ob_dbms_stats_utils.h +++ b/src/share/stat/ob_dbms_stats_utils.h @@ -132,6 +132,19 @@ public: static int64_t get_truncated_str_len(const ObString &str, const ObCollationType cs_type); + static int get_current_opt_stats(const ObTableStatParam ¶m, + ObIArray &cur_tab_handles, + ObIArray &cur_col_handles); + + static int get_part_ids_and_column_ids(const ObTableStatParam ¶m, + ObIArray &part_ids, + ObIArray &column_ids); + + static int erase_stat_cache(const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &part_ids, + const ObIArray &column_ids); + private: static int batch_write(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, diff --git a/src/share/stat/ob_opt_stat_gather_stat.cpp b/src/share/stat/ob_opt_stat_gather_stat.cpp index c7ff3052b..f1ce5c542 100644 --- a/src/share/stat/ob_opt_stat_gather_stat.cpp +++ b/src/share/stat/ob_opt_stat_gather_stat.cpp @@ -184,19 +184,22 @@ int ObOptStatRunningMonitor::add_table_info(common::ObTableStatParam &table_para double stale_percent) { int ret = OB_SUCCESS; + ObString tmp_db_name; + ObString tmp_tab_name; + ObString tmp_properties_str; if (OB_FAIL(ob_write_string(allocator_, table_param.db_name_, - opt_stat_gather_stat_.get_database_name()))) { + tmp_db_name))) { LOG_WARN("failed to write string", K(ret)); } else if (OB_FAIL(ob_write_string(allocator_, table_param.tab_name_, - opt_stat_gather_stat_.get_table_name()))) { + tmp_tab_name))) { LOG_WARN("failed to write string", K(ret)); } else { opt_stat_gather_stat_.set_table_id(table_param.table_id_); - ObSqlString tmp_properties_str; + ObSqlString properties_sql_str; char *buf = NULL; - if (OB_FAIL(tmp_properties_str.append_fmt("GRANULARITY:%.*s;METHOD_OPT:%.*s;DEGREE:%ld;ESTIMATE_PERCENT:%lf;BLOCK_SAMPLE:%d;STALE_PERCENT:%lf;", + if (OB_FAIL(properties_sql_str.append_fmt("GRANULARITY:%.*s;METHOD_OPT:%.*s;DEGREE:%ld;ESTIMATE_PERCENT:%lf;BLOCK_SAMPLE:%d;STALE_PERCENT:%lf;", table_param.granularity_.length(), table_param.granularity_.ptr(), table_param.method_opt_.length(), @@ -206,12 +209,14 @@ int ObOptStatRunningMonitor::add_table_info(common::ObTableStatParam &table_para table_param.sample_info_.is_block_sample_, stale_percent))) { LOG_WARN("failed to append fmt", K(ret)); - } else if (OB_ISNULL(buf = static_cast(allocator_.alloc(tmp_properties_str.length())))) { + } else if (OB_ISNULL(buf = static_cast(allocator_.alloc(properties_sql_str.length())))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(tmp_properties_str)); + LOG_WARN("memory is not enough", K(ret), K(properties_sql_str)); } else { - MEMCPY(buf, tmp_properties_str.ptr(), tmp_properties_str.length()); - opt_stat_gather_stat_.set_properties(buf, static_cast(tmp_properties_str.length())); + MEMCPY(buf, properties_sql_str.ptr(), properties_sql_str.length()); + tmp_properties_str.assign_ptr(buf, static_cast(properties_sql_str.length())); + ObOptStatGatherStatList::instance().update_gather_stat_info(tmp_db_name, tmp_tab_name, + tmp_properties_str, opt_stat_gather_stat_); } } return ret; @@ -301,6 +306,24 @@ int ObOptStatGatherStatList::remove(ObOptStatGatherStat &stat_value) return ret; } +void ObOptStatGatherStatList::update_gather_stat_info(ObString &db_name, + ObString &tab_name, + ObString &properties, + ObOptStatGatherStat &stat_value) +{ + ObSpinLockGuard guard(lock_); + stat_value.set_database_name(db_name.ptr(), db_name.length()); + stat_value.set_table_name(tab_name.ptr(), tab_name.length()); + stat_value.set_properties(properties.ptr(), properties.length()); +} + +void ObOptStatGatherStatList::update_gather_stat_refresh_failed_list(ObString &failed_list, + ObOptStatGatherStat &stat_value) +{ + ObSpinLockGuard guard(lock_); + stat_value.set_stat_refresh_failed_list(failed_list.ptr(), failed_list.length()); +} + int ObOptStatGatherStatList::list_to_array(common::ObIAllocator &allocator, const uint64_t target_tenant_id, ObIArray &stat_array) diff --git a/src/share/stat/ob_opt_stat_gather_stat.h b/src/share/stat/ob_opt_stat_gather_stat.h index afffafd44..b1b0adaf9 100644 --- a/src/share/stat/ob_opt_stat_gather_stat.h +++ b/src/share/stat/ob_opt_stat_gather_stat.h @@ -215,6 +215,12 @@ public: static ObOptStatGatherStatList &instance(); int push(ObOptStatGatherStat &stat_value); int remove(ObOptStatGatherStat &stat_value); + void update_gather_stat_info(ObString &db_name, + ObString &tab_name, + ObString &properties, + ObOptStatGatherStat &stat_value); + void update_gather_stat_refresh_failed_list(ObString &failed_list, + ObOptStatGatherStat &stat_value); // param[in] tenant_id if tenant is sys, list all tenant stat, else list target tenant stat int list_to_array(common::ObIAllocator &allocator, const uint64_t target_tenant_id, diff --git a/src/sql/optimizer/ob_access_path_estimation.cpp b/src/sql/optimizer/ob_access_path_estimation.cpp index 0997ff639..bec084f4f 100644 --- a/src/sql/optimizer/ob_access_path_estimation.cpp +++ b/src/sql/optimizer/ob_access_path_estimation.cpp @@ -1399,7 +1399,7 @@ int ObAccessPathEstimation::process_dynamic_sampling_estimation(ObOptimizerConte int64_t start_time = ObTimeUtility::current_time(); bool throw_ds_error = false; if (OB_FAIL(dynamic_sampling.estimate_table_rowcount(ds_table_param, ds_result_items, throw_ds_error))) { - if (!throw_ds_error) { + if (!throw_ds_error && !is_retry_ret(ret)) { LOG_WARN("failed to estimate table rowcount caused by some reason, please check!!!", K(ret), K(start_time), K(ObTimeUtility::current_time() - start_time), K(ds_table_param), K(ctx.get_session_info()->get_current_query_string())); diff --git a/src/sql/optimizer/ob_access_path_estimation.h b/src/sql/optimizer/ob_access_path_estimation.h index 2d8e2982e..78a7cfbab 100644 --- a/src/sql/optimizer/ob_access_path_estimation.h +++ b/src/sql/optimizer/ob_access_path_estimation.h @@ -47,6 +47,8 @@ public: static int estimate_full_table_rowcount(ObOptimizerContext &ctx, const ObTablePartitionInfo &table_part_info, ObTableMetaInfo &meta); + + static bool is_retry_ret(int ret); private: static int process_common_estimate_rowcount(ObOptimizerContext &ctx, @@ -207,8 +209,6 @@ private: common::ObIArray &ds_paths, common::ObIArray &no_ds_paths, bool &all_path_is_get); - - static bool is_retry_ret(int ret); }; } diff --git a/src/sql/optimizer/ob_opt_selectivity.cpp b/src/sql/optimizer/ob_opt_selectivity.cpp index 0eb02d01e..856b34ce0 100644 --- a/src/sql/optimizer/ob_opt_selectivity.cpp +++ b/src/sql/optimizer/ob_opt_selectivity.cpp @@ -29,6 +29,7 @@ #include "sql/optimizer/ob_join_order.h" #include "common/ob_smart_call.h" #include "share/stat/ob_dbms_stats_utils.h" +#include "sql/optimizer/ob_access_path_estimation.h" using namespace oceanbase::common; using namespace oceanbase::share::schema; @@ -658,7 +659,7 @@ int ObOptSelectivity::calc_selectivity_by_dynamic_sampling(const OptSelectivityC int64_t start_time = ObTimeUtility::current_time(); bool throw_ds_error = false; if (OB_FAIL(dynamic_sampling.estimate_table_rowcount(ds_table_param, ds_result_items, throw_ds_error))) { - if (!throw_ds_error) { + if (!throw_ds_error && !ObAccessPathEstimation::is_retry_ret(ret)) { LOG_WARN("failed to estimate filter rowcount caused by some reason, please check!!!", K(ret), K(start_time), K(ObTimeUtility::current_time() - start_time), K(ds_table_param), K(ctx.get_session_info()->get_current_query_string()));