hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix some optimizer stat bug

Browse Source
This commit is contained in:
wangt1xiuyi 2023-09-14 05:40:21 +00:00 committed by ob-robot
parent 53824d0e67
commit 1769873118
11 changed files with 160 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/pl/sys_package/ob_dbms_stats.cpp
View File

@ -3032,7 +3032,9 @@ int ObDbmsStats::update_stat_cache(const uint64_t rpc_tenant_id,
LOG_WARN("memory is not enough", K(ret), K(tmp_str));
} else {
MEMCPY(buf, tmp_str.ptr(), tmp_str.length());
running_monitor->opt_stat_gather_stat_.set_stat_refresh_failed_list(buf, tmp_str.length());
ObString tmp_failed_list(tmp_str.length(), buf);
ObOptStatGatherStatList::instance().update_gather_stat_refresh_failed_list(tmp_failed_list,
running_monitor->opt_stat_gather_stat_);
}
}
}
@ -3990,7 +3992,8 @@ int ObDbmsStats::parse_granularity_and_method_opt(ObExecContext &ctx,
{
int ret = OB_SUCCESS;
//virtual table(not include real agent table) doesn't gather histogram.
bool is_vt = is_virtual_table(param.table_id_);
bool is_vt = is_virtual_table(param.table_id_) &&
!share::is_oracle_mapping_real_virtual_table(param.table_id_);
bool use_size_auto = false;
if (0 == param.method_opt_.case_compare("Z") && !is_vt) {
if (OB_FAIL(set_default_column_params(param.column_params_))) {
@ -6117,6 +6120,11 @@ int ObDbmsStats::resovle_granularity(ObGranularityType granu_type,
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected granularity type", K(granu_type));
}
//virtual table only gather global stats.
if (OB_SUCC(ret) && is_virtual_table(param.table_id_)) {
param.part_stat_param_.reset_gather_stat();
param.subpart_stat_param_.reset_gather_stat();
}
LOG_TRACE("succeed to parse granularity", K(param.global_stat_param_),
K(param.part_stat_param_), K(param.subpart_stat_param_));
return ret;

28
src/share/stat/ob_dbms_stats_executor.cpp
View File

@ -812,10 +812,8 @@ int ObDbmsStatsExecutor::update_online_stat(ObExecContext &ctx,
int ret = OB_SUCCESS;
int64_t affected_rows = 0;
//before write, we need record history stats.
ObSEArray<ObOptTableStatHandle, 4> history_tab_handles;
ObSEArray<ObOptColumnStatHandle, 4> history_col_handles;
ObSEArray<ObOptTableStatHandle, 4> cur_tab_handles;
ObSEArray<ObOptColumnStatHandle, 4> cur_col_handles;
ObSEArray<ObOptTableStat *, 4> table_stats;
ObSEArray<ObOptColumnStat *, 4> column_stats;
if (OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, param))) {
@ -828,30 +826,24 @@ int ObDbmsStatsExecutor::update_online_stat(ObExecContext &ctx,
LOG_WARN("fail to check lock stat", K(ret));
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObDbmsStatsHistoryManager::get_history_stat_handles(ctx,
param,
history_tab_handles,
history_col_handles))) {
LOG_WARN("failed to get history stat handles", K(ret));
} else if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(param,
cur_tab_handles,
cur_col_handles))) {
LOG_WARN("failed to get current opt stats", K(ret));
} else if (OB_FAIL(ObDbmsStatsUtils::merge_tab_stats(param,
online_table_stats,
history_tab_handles,
cur_tab_handles,
table_stats))) {
LOG_WARN("fail to merge tab stats", K(ret), K(history_tab_handles));
LOG_WARN("fail to merge tab stats", K(ret), K(cur_tab_handles));
} else if (OB_FAIL(ObDbmsStatsUtils::merge_col_stats(param,
online_column_stats,
history_col_handles,
cur_col_handles,
column_stats))) {
LOG_WARN("fail to merge col stats", K(ret), K(history_col_handles));
LOG_WARN("fail to merge col stats", K(ret), K(cur_col_handles));
} else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, table_stats, column_stats,
false, false, true))) {
LOG_WARN("fail to update stat", K(ret), K(table_stats), K(column_stats));
} else if (OB_FAIL(ObDbmsStatsUtils::batch_write_history_stats(ctx,
history_tab_handles,
history_col_handles))) {
LOG_WARN("failed to batch write history stats", K(ret));
} else if (OB_FAIL(ObBasicStatsEstimator::update_last_modified_count(ctx, param))) {
//update history
LOG_WARN("failed to update last modified count", K(ret));
} else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), param))) {
LOG_WARN("fail to update stat cache", K(ret));

75
src/share/stat/ob_dbms_stats_history_manager.cpp
View File

@ -69,8 +69,6 @@ int ObDbmsStatsHistoryManager::get_history_stat_handles(ObExecContext &ctx,
ObIArray<ObOptColumnStatHandle> &history_col_handles)
{
int ret = OB_SUCCESS;
ObSEArray<int64_t, 4> part_ids;
ObSEArray<uint64_t, 4> column_ids;
int64_t retention_val = 0;
if (param.is_index_stat_) {
//do nothing
@ -78,21 +76,10 @@ int ObDbmsStatsHistoryManager::get_history_stat_handles(ObExecContext &ctx,
LOG_WARN("failed to get stats history retention", K(ret));
} else if (retention_val == 0) {
/*do nothing*/
} else if (OB_FAIL(get_part_ids_and_column_ids(param, part_ids, column_ids))) {
LOG_WARN("failed to get part ids and column ids", K(ret));
} else if (OB_FAIL(erase_stat_cache(param.tenant_id_, param.table_id_, part_ids, column_ids))) {
LOG_WARN("failed to erase stat cache", K(ret));
} else if (OB_FAIL(ObOptStatManager::get_instance().get_table_stat(param.tenant_id_,
param.table_id_,
part_ids,
history_tab_handles))) {
LOG_WARN("failed to get table stat", K(ret));
} else if (OB_FAIL(ObOptStatManager::get_instance().get_column_stat(param.tenant_id_,
param.table_id_,
part_ids,
column_ids,
history_col_handles))) {
LOG_WARN("failed to get column stat", K(ret));
} else if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(param,
history_tab_handles,
history_col_handles))) {
LOG_WARN("failed to get current opt stats", K(ret));
} else if (OB_FAIL(set_col_stat_cs_type(param.column_params_, history_col_handles))) {
LOG_WARN("failed to set col stat cs type", K(ret));
} else {
@ -695,60 +682,6 @@ int ObDbmsStatsHistoryManager::fill_bucket_stat_histroy(ObIAllocator &allocator,
return ret;
}
int ObDbmsStatsHistoryManager::get_part_ids_and_column_ids(const ObTableStatParam &param,
ObIArray<int64_t> &part_ids,
ObIArray<uint64_t> &column_ids)
{
int ret = OB_SUCCESS;
//get part ids
if (param.global_stat_param_.need_modify_) {
int64_t part_id = param.global_part_id_;
if (OB_FAIL(part_ids.push_back(part_id))) {
LOG_WARN("failed to push back", K(ret));
} else {/*do nothing*/}
}
if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) {
for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) {
if (OB_FAIL(part_ids.push_back(param.part_infos_.at(i).part_id_))) {
LOG_WARN("failed to push back", K(ret));
} else {/*do nothing*/}
}
}
if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) {
for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) {
if (OB_FAIL(part_ids.push_back(param.subpart_infos_.at(i).part_id_))) {
LOG_WARN("failed to push back", K(ret));
} else {/*do nothing*/}
}
}
//get column ids
for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) {
if (OB_FAIL(column_ids.push_back(param.column_params_.at(i).column_id_))) {
LOG_WARN("failed to push back", K(ret));
} else {/*do nothing*/}
}
return ret;
}
int ObDbmsStatsHistoryManager::erase_stat_cache(const uint64_t tenant_id,
const uint64_t table_id,
const ObIArray<int64_t> &part_ids,
const ObIArray<uint64_t> &column_ids)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObOptStatManager::get_instance().erase_table_stat(tenant_id, table_id, part_ids))) {
LOG_WARN("failed to erase table stats", K(ret));
} else if (OB_FAIL(ObOptStatManager::get_instance().erase_column_stat(tenant_id,
table_id,
part_ids,
column_ids))) {
LOG_WARN("failed to erase column stats", K(ret));
} else {/*do nothing*/}
return ret;
}
int ObDbmsStatsHistoryManager::get_stats_history_retention(ObExecContext &ctx,
int64_t &retention_val)
{

9
src/share/stat/ob_dbms_stats_history_manager.h
View File

@ -76,15 +76,6 @@ private:
sqlclient::ObMySQLResult &result,
ObOptColumnStat &stat);
static int get_part_ids_and_column_ids(const ObTableStatParam &param,
ObIArray<int64_t> &part_ids,
ObIArray<uint64_t> &column_ids);
static int erase_stat_cache(const uint64_t tenant_id,
const uint64_t table_id,
const ObIArray<int64_t> &part_ids,
const ObIArray<uint64_t> &column_ids);
static int get_stats_history_retention(ObExecContext &ctx, int64_t &retention_val);
static int set_col_stat_cs_type(const ObIArray<ObColumnStatParam> &column_params,

81
src/share/stat/ob_dbms_stats_utils.cpp
View File

@ -880,5 +880,86 @@ int64_t ObDbmsStatsUtils::get_truncated_str_len(const ObString &str, const ObCol
return truncated_str_len;
}
int ObDbmsStatsUtils::get_current_opt_stats(const ObTableStatParam &param,
ObIArray<ObOptTableStatHandle> &cur_tab_handles,
ObIArray<ObOptColumnStatHandle> &cur_col_handles)
{
int ret = OB_SUCCESS;
ObSEArray<int64_t, 4> part_ids;
ObSEArray<uint64_t, 4> column_ids;
if (OB_FAIL(get_part_ids_and_column_ids(param, part_ids, column_ids))) {
LOG_WARN("failed to get part ids and column ids", K(ret));
} else if (OB_FAIL(erase_stat_cache(param.tenant_id_, param.table_id_,
part_ids, column_ids))) {
LOG_WARN("failed to erase stat cache", K(ret));
} else if (OB_FAIL(ObOptStatManager::get_instance().get_table_stat(param.tenant_id_,
param.table_id_,
part_ids,
cur_tab_handles))) {
LOG_WARN("failed to get table stat", K(ret));
} else if (OB_FAIL(ObOptStatManager::get_instance().get_column_stat(param.tenant_id_,
param.table_id_,
part_ids,
column_ids,
cur_col_handles))) {
LOG_WARN("failed to get column stat", K(ret));
}
return ret;
}
int ObDbmsStatsUtils::get_part_ids_and_column_ids(const ObTableStatParam &param,
ObIArray<int64_t> &part_ids,
ObIArray<uint64_t> &column_ids)
{
int ret = OB_SUCCESS;
//get part ids
if (param.global_stat_param_.need_modify_) {
int64_t part_id = param.global_part_id_;
if (OB_FAIL(part_ids.push_back(part_id))) {
LOG_WARN("failed to push back", K(ret));
} else {/*do nothing*/}
}
if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) {
for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) {
if (OB_FAIL(part_ids.push_back(param.part_infos_.at(i).part_id_))) {
LOG_WARN("failed to push back", K(ret));
} else {/*do nothing*/}
}
}
if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) {
for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) {
if (OB_FAIL(part_ids.push_back(param.subpart_infos_.at(i).part_id_))) {
LOG_WARN("failed to push back", K(ret));
} else {/*do nothing*/}
}
}
//get column ids
for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) {
if (OB_FAIL(column_ids.push_back(param.column_params_.at(i).column_id_))) {
LOG_WARN("failed to push back", K(ret));
} else {/*do nothing*/}
}
return ret;
}
int ObDbmsStatsUtils::erase_stat_cache(const uint64_t tenant_id,
const uint64_t table_id,
const ObIArray<int64_t> &part_ids,
const ObIArray<uint64_t> &column_ids)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObOptStatManager::get_instance().erase_table_stat(tenant_id, table_id, part_ids))) {
LOG_WARN("failed to erase table stats", K(ret));
} else if (OB_FAIL(ObOptStatManager::get_instance().erase_column_stat(tenant_id,
table_id,
part_ids,
column_ids))) {
LOG_WARN("failed to erase column stats", K(ret));
} else {/*do nothing*/}
return ret;
}
}
}

13
src/share/stat/ob_dbms_stats_utils.h
View File

@ -132,6 +132,19 @@ public:
static int64_t get_truncated_str_len(const ObString &str, const ObCollationType cs_type);
static int get_current_opt_stats(const ObTableStatParam &param,
ObIArray<ObOptTableStatHandle> &cur_tab_handles,
ObIArray<ObOptColumnStatHandle> &cur_col_handles);
static int get_part_ids_and_column_ids(const ObTableStatParam &param,
ObIArray<int64_t> &part_ids,
ObIArray<uint64_t> &column_ids);
static int erase_stat_cache(const uint64_t tenant_id,
const uint64_t table_id,
const ObIArray<int64_t> &part_ids,
const ObIArray<uint64_t> &column_ids);
private:
static int batch_write(share::schema::ObSchemaGetterGuard *schema_guard,
const uint64_t tenant_id,

39
src/share/stat/ob_opt_stat_gather_stat.cpp
View File

@ -184,19 +184,22 @@ int ObOptStatRunningMonitor::add_table_info(common::ObTableStatParam &table_para
double stale_percent)
{
int ret = OB_SUCCESS;
ObString tmp_db_name;
ObString tmp_tab_name;
ObString tmp_properties_str;
if (OB_FAIL(ob_write_string(allocator_,
table_param.db_name_,
opt_stat_gather_stat_.get_database_name()))) {
tmp_db_name))) {
LOG_WARN("failed to write string", K(ret));
} else if (OB_FAIL(ob_write_string(allocator_,
table_param.tab_name_,
opt_stat_gather_stat_.get_table_name()))) {
tmp_tab_name))) {
LOG_WARN("failed to write string", K(ret));
} else {
opt_stat_gather_stat_.set_table_id(table_param.table_id_);
ObSqlString tmp_properties_str;
ObSqlString properties_sql_str;
char *buf = NULL;
if (OB_FAIL(tmp_properties_str.append_fmt("GRANULARITY:%.*s;METHOD_OPT:%.*s;DEGREE:%ld;ESTIMATE_PERCENT:%lf;BLOCK_SAMPLE:%d;STALE_PERCENT:%lf;",
if (OB_FAIL(properties_sql_str.append_fmt("GRANULARITY:%.*s;METHOD_OPT:%.*s;DEGREE:%ld;ESTIMATE_PERCENT:%lf;BLOCK_SAMPLE:%d;STALE_PERCENT:%lf;",
table_param.granularity_.length(),
table_param.granularity_.ptr(),
table_param.method_opt_.length(),
@ -206,12 +209,14 @@ int ObOptStatRunningMonitor::add_table_info(common::ObTableStatParam &table_para
table_param.sample_info_.is_block_sample_,
stale_percent))) {
LOG_WARN("failed to append fmt", K(ret));
} else if (OB_ISNULL(buf = static_cast<char*>(allocator_.alloc(tmp_properties_str.length())))) {
} else if (OB_ISNULL(buf = static_cast<char*>(allocator_.alloc(properties_sql_str.length())))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("memory is not enough", K(ret), K(tmp_properties_str));
LOG_WARN("memory is not enough", K(ret), K(properties_sql_str));
} else {
MEMCPY(buf, tmp_properties_str.ptr(), tmp_properties_str.length());
opt_stat_gather_stat_.set_properties(buf, static_cast<int32_t>(tmp_properties_str.length()));
MEMCPY(buf, properties_sql_str.ptr(), properties_sql_str.length());
tmp_properties_str.assign_ptr(buf, static_cast<int32_t>(properties_sql_str.length()));
ObOptStatGatherStatList::instance().update_gather_stat_info(tmp_db_name, tmp_tab_name,
tmp_properties_str, opt_stat_gather_stat_);
}
}
return ret;
@ -301,6 +306,24 @@ int ObOptStatGatherStatList::remove(ObOptStatGatherStat &stat_value)
return ret;
}
void ObOptStatGatherStatList::update_gather_stat_info(ObString &db_name,
ObString &tab_name,
ObString &properties,
ObOptStatGatherStat &stat_value)
{
ObSpinLockGuard guard(lock_);
stat_value.set_database_name(db_name.ptr(), db_name.length());
stat_value.set_table_name(tab_name.ptr(), tab_name.length());
stat_value.set_properties(properties.ptr(), properties.length());
}
void ObOptStatGatherStatList::update_gather_stat_refresh_failed_list(ObString &failed_list,
ObOptStatGatherStat &stat_value)
{
ObSpinLockGuard guard(lock_);
stat_value.set_stat_refresh_failed_list(failed_list.ptr(), failed_list.length());
}
int ObOptStatGatherStatList::list_to_array(common::ObIAllocator &allocator,
const uint64_t target_tenant_id,
ObIArray<ObOptStatGatherStat> &stat_array)

6
src/share/stat/ob_opt_stat_gather_stat.h
View File

@ -215,6 +215,12 @@ public:
static ObOptStatGatherStatList &instance();
int push(ObOptStatGatherStat &stat_value);
int remove(ObOptStatGatherStat &stat_value);
void update_gather_stat_info(ObString &db_name,
ObString &tab_name,
ObString &properties,
ObOptStatGatherStat &stat_value);
void update_gather_stat_refresh_failed_list(ObString &failed_list,
ObOptStatGatherStat &stat_value);
// param[in] tenant_id if tenant is sys, list all tenant stat, else list target tenant stat
int list_to_array(common::ObIAllocator &allocator,
const uint64_t target_tenant_id,

2
src/sql/optimizer/ob_access_path_estimation.cpp
View File

@ -1399,7 +1399,7 @@ int ObAccessPathEstimation::process_dynamic_sampling_estimation(ObOptimizerConte
int64_t start_time = ObTimeUtility::current_time();
bool throw_ds_error = false;
if (OB_FAIL(dynamic_sampling.estimate_table_rowcount(ds_table_param, ds_result_items, throw_ds_error))) {
if (!throw_ds_error) {
if (!throw_ds_error && !is_retry_ret(ret)) {
LOG_WARN("failed to estimate table rowcount caused by some reason, please check!!!", K(ret),
K(start_time), K(ObTimeUtility::current_time() - start_time), K(ds_table_param),
K(ctx.get_session_info()->get_current_query_string()));

4
src/sql/optimizer/ob_access_path_estimation.h
View File

@ -47,6 +47,8 @@ public:
static int estimate_full_table_rowcount(ObOptimizerContext &ctx,
const ObTablePartitionInfo &table_part_info,
ObTableMetaInfo &meta);
static bool is_retry_ret(int ret);
private:
static int process_common_estimate_rowcount(ObOptimizerContext &ctx,
@ -207,8 +209,6 @@ private:
common::ObIArray<AccessPath *> &ds_paths,
common::ObIArray<AccessPath *> &no_ds_paths,
bool &all_path_is_get);
static bool is_retry_ret(int ret);
};
}

3
src/sql/optimizer/ob_opt_selectivity.cpp
View File

@ -29,6 +29,7 @@
#include "sql/optimizer/ob_join_order.h"
#include "common/ob_smart_call.h"
#include "share/stat/ob_dbms_stats_utils.h"
#include "sql/optimizer/ob_access_path_estimation.h"
using namespace oceanbase::common;
using namespace oceanbase::share::schema;
@ -658,7 +659,7 @@ int ObOptSelectivity::calc_selectivity_by_dynamic_sampling(const OptSelectivityC
int64_t start_time = ObTimeUtility::current_time();
bool throw_ds_error = false;
if (OB_FAIL(dynamic_sampling.estimate_table_rowcount(ds_table_param, ds_result_items, throw_ds_error))) {
if (!throw_ds_error) {
if (!throw_ds_error && !ObAccessPathEstimation::is_retry_ret(ret)) {
LOG_WARN("failed to estimate filter rowcount caused by some reason, please check!!!", K(ret),
K(start_time), K(ObTimeUtility::current_time() - start_time), K(ds_table_param),
K(ctx.get_session_info()->get_current_query_string()));