[dbms_stats]: bugfix for online optimizer stats gather.

This commit is contained in:
Monk-Liu 2023-02-14 04:42:08 +00:00 committed by ob-robot
parent 092b683a60
commit 5e0f5e4e12
13 changed files with 99 additions and 48 deletions

View File

@ -242,7 +242,7 @@ enum ObSysVarClassType
SYS_VAR_OB_SQL_PLAN_MEMORY_PERCENTAGE = 10135,
SYS_VAR_LOG_ROW_VALUE_OPTIONS = 10136,
SYS_VAR_OB_MAX_READ_STALE_TIME = 10137,
SYS_VAR_ONLINE_OPT_STAT_GATHER = 10138,
SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD = 10138,
SYS_VAR__SET_REVERSE_DBLINK_INFOS = 10139,
};

View File

@ -237,7 +237,7 @@ namespace share
static const char* const OB_SV_SQL_PLAN_MEMORY_PERCENTAGE = "ob_sql_plan_memory_percentage";
static const char* const OB_SV_LOG_ROW_VALUE_OPTIONS = "log_row_value_options";
static const char* const OB_SV_MAX_READ_STALE_TIME = "ob_max_read_stale_time";
static const char* const OB_SV_ONLINE_OPT_STAT_GATHER = "online_opt_stat_gather";
static const char* const OB_SV__OPTIMIZER_GATHER_STATS_ON_LOAD = "_optimizer_gather_stats_on_load";
static const char* const OB_SV__SET_REVERSE_DBLINK_INFOS = "_set_reverse_dblink_infos";
}

View File

@ -134,6 +134,7 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = {
"_ob_proxy_session_temporary_table_used",
"_ob_px_bcast_optimization",
"_ob_px_slave_mapping_threshold",
"_optimizer_gather_stats_on_load",
"_optimizer_null_aware_antijoin",
"_px_broadcast_fudge_factor",
"_px_dist_agg_partial_rollup_pushdown",
@ -270,7 +271,6 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_NAME[] = {
"ob_trx_idle_timeout",
"ob_trx_lock_timeout",
"ob_trx_timeout",
"online_opt_stat_gather",
"optimizer_capture_sql_plan_baselines",
"optimizer_use_sql_plan_baselines",
"parallel_servers_target",
@ -360,6 +360,7 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = {
SYS_VAR__OB_PROXY_SESSION_TEMPORARY_TABLE_USED,
SYS_VAR__OB_PX_BCAST_OPTIMIZATION,
SYS_VAR__OB_PX_SLAVE_MAPPING_THRESHOLD,
SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD,
SYS_VAR__OPTIMIZER_NULL_AWARE_ANTIJOIN,
SYS_VAR__PX_BROADCAST_FUDGE_FACTOR,
SYS_VAR__PX_DIST_AGG_PARTIAL_ROLLUP_PUSHDOWN,
@ -496,7 +497,6 @@ const ObSysVarClassType ObSysVarFactory::SYS_VAR_IDS_SORTED_BY_NAME[] = {
SYS_VAR_OB_TRX_IDLE_TIMEOUT,
SYS_VAR_OB_TRX_LOCK_TIMEOUT,
SYS_VAR_OB_TRX_TIMEOUT,
SYS_VAR_ONLINE_OPT_STAT_GATHER,
SYS_VAR_OPTIMIZER_CAPTURE_SQL_PLAN_BASELINES,
SYS_VAR_OPTIMIZER_USE_SQL_PLAN_BASELINES,
SYS_VAR_PARALLEL_SERVERS_TARGET,
@ -791,7 +791,7 @@ const char *ObSysVarFactory::SYS_VAR_NAMES_SORTED_BY_ID[] = {
"ob_sql_plan_memory_percentage",
"log_row_value_options",
"ob_max_read_stale_time",
"online_opt_stat_gather",
"_optimizer_gather_stats_on_load",
"_set_reverse_dblink_infos"
};
@ -1181,7 +1181,7 @@ int ObSysVarFactory::create_all_sys_vars()
+ sizeof(ObSysVarObSqlPlanMemoryPercentage)
+ sizeof(ObSysVarLogRowValueOptions)
+ sizeof(ObSysVarObMaxReadStaleTime)
+ sizeof(ObSysVarOnlineOptStatGather)
+ sizeof(ObSysVarOptimizerGatherStatsOnLoad)
+ sizeof(ObSysVarSetReverseDblinkInfos)
;
void *ptr = NULL;
@ -3181,12 +3181,12 @@ int ObSysVarFactory::create_all_sys_vars()
}
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOnlineOptStatGather())) {
if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerGatherStatsOnLoad())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to new ObSysVarOnlineOptStatGather", K(ret));
LOG_ERROR("fail to new ObSysVarOptimizerGatherStatsOnLoad", K(ret));
} else {
store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast<int64_t>(SYS_VAR_ONLINE_OPT_STAT_GATHER))] = sys_var_ptr;
ptr = (void *)((char *)ptr + sizeof(ObSysVarOnlineOptStatGather));
store_buf_[ObSysVarsToIdxMap::get_store_idx(static_cast<int64_t>(SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD))] = sys_var_ptr;
ptr = (void *)((char *)ptr + sizeof(ObSysVarOptimizerGatherStatsOnLoad));
}
}
if (OB_SUCC(ret)) {
@ -5655,14 +5655,14 @@ int ObSysVarFactory::create_sys_var(ObSysVarClassType sys_var_id, ObBasicSysVar
}
break;
}
case SYS_VAR_ONLINE_OPT_STAT_GATHER: {
case SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD: {
void *ptr = NULL;
if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOnlineOptStatGather)))) {
if (OB_ISNULL(ptr = allocator_.alloc(sizeof(ObSysVarOptimizerGatherStatsOnLoad)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOnlineOptStatGather)));
} else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOnlineOptStatGather())) {
LOG_ERROR("fail to alloc memory", K(ret), K(sizeof(ObSysVarOptimizerGatherStatsOnLoad)));
} else if (OB_ISNULL(sys_var_ptr = new (ptr)ObSysVarOptimizerGatherStatsOnLoad())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("fail to new ObSysVarOnlineOptStatGather", K(ret));
LOG_ERROR("fail to new ObSysVarOptimizerGatherStatsOnLoad", K(ret));
}
break;
}

View File

@ -1598,11 +1598,11 @@ public:
inline virtual ObSysVarClassType get_type() const { return SYS_VAR_OB_MAX_READ_STALE_TIME; }
inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(220); }
};
class ObSysVarOnlineOptStatGather : public ObBoolSysVar
class ObSysVarOptimizerGatherStatsOnLoad : public ObBoolSysVar
{
public:
ObSysVarOnlineOptStatGather() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {}
inline virtual ObSysVarClassType get_type() const { return SYS_VAR_ONLINE_OPT_STAT_GATHER; }
ObSysVarOptimizerGatherStatsOnLoad() : ObBoolSysVar(NULL, NULL, NULL, NULL, NULL) {}
inline virtual ObSysVarClassType get_type() const { return SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD; }
inline virtual const common::ObObj &get_global_default_value() const { return ObSysVariables::get_default_value(221); }
};
class ObSysVarSetReverseDblinkInfos : public ObVarcharSysVar

View File

@ -2907,14 +2907,14 @@ static struct VarsInit{
[&] (){
ObSysVars[221].info_ = "control wether we need to gather optimizer stats on insert into select/create table as select" ;
ObSysVars[221].name_ = "online_opt_stat_gather" ;
ObSysVars[221].name_ = "_optimizer_gather_stats_on_load" ;
ObSysVars[221].data_type_ = ObIntType ;
ObSysVars[221].value_ = "0" ;
ObSysVars[221].value_ = "1" ;
ObSysVars[221].flags_ = ObSysVarFlag::GLOBAL_SCOPE | ObSysVarFlag::SESSION_SCOPE ;
ObSysVars[221].id_ = SYS_VAR_ONLINE_OPT_STAT_GATHER ;
cur_max_var_id = MAX(cur_max_var_id, static_cast<int64_t>(SYS_VAR_ONLINE_OPT_STAT_GATHER)) ;
ObSysVarsIdToArrayIdx[SYS_VAR_ONLINE_OPT_STAT_GATHER] = 221 ;
ObSysVars[221].alias_ = "OB_SV_ONLINE_OPT_STAT_GATHER" ;
ObSysVars[221].id_ = SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD ;
cur_max_var_id = MAX(cur_max_var_id, static_cast<int64_t>(SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD)) ;
ObSysVarsIdToArrayIdx[SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD] = 221 ;
ObSysVars[221].alias_ = "OB_SV__OPTIMIZER_GATHER_STATS_ON_LOAD" ;
}();
[&] (){

View File

@ -2935,10 +2935,10 @@
"background_cn": "",
"ref_url": "https://yuque.antfin.com/ob/product_functionality_review/ns7okery0azyvlnp"
},
"online_opt_stat_gather": {
"_optimizer_gather_stats_on_load": {
"id": 10138,
"name": "online_opt_stat_gather",
"value": "0",
"name": "_optimizer_gather_stats_on_load",
"value": "1",
"data_type": "bool",
"info": "control wether we need to gather optimizer stats on insert into select/create table as select",
"flags": "GLOBAL | SESSION",

View File

@ -1929,13 +1929,13 @@ int ObLoadDataDirectImpl::init_execute_param()
if (OB_ISNULL(session = ctx_->get_my_session())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("session is null", KR(ret));
} else if (OB_FAIL(session->get_sys_variable(SYS_VAR_ONLINE_OPT_STAT_GATHER, obj))) {
} else if (OB_FAIL(session->get_sys_variable(SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD, obj))) {
LOG_WARN("fail to get sys variable", K(ret));
} else if (OB_FAIL(hint.get_value(ObLoadDataHint::APPEND, append))) {
LOG_WARN("fail to get value of APPEND", K(ret));
} else if (OB_FAIL(hint.get_value(ObLoadDataHint::GATHER_OPTIMIZER_STATISTICS, gather_optimizer_statistics))) {
LOG_WARN("fail to get value of APPEND", K(ret));
} else if ((append != 0) || (gather_optimizer_statistics != 0) || obj.get_bool()) {
} else if (((append != 0) || (gather_optimizer_statistics != 0)) && obj.get_bool()) {
execute_param_.online_opt_stat_gather_ = true;
} else {
execute_param_.online_opt_stat_gather_ = false;

View File

@ -605,7 +605,8 @@ int ObInsertValueGenerator::init(ObSQLSessionInfo &session,
int ObLoadDataSPImpl::gen_insert_columns_names_buff(ObExecContext &ctx,
const ObLoadArgument &load_args,
ObIArray<ObLoadTableColumnDesc> &insert_infos,
ObString &data_buff)
ObString &data_buff,
bool need_online_osg)
{
int ret = OB_SUCCESS;
@ -641,7 +642,8 @@ int ObLoadDataSPImpl::gen_insert_columns_names_buff(ObExecContext &ctx,
if (OB_FAIL(ObLoadDataUtils::build_insert_sql_string_head(load_args.dupl_action_,
load_args.combined_name_,
insert_column_names,
insert_stmt))) {
insert_stmt,
need_online_osg))) {
LOG_WARN("gen insert sql column_names failed", K(ret));
} else if (OB_FAIL(ob_write_string(ctx.get_allocator(), insert_stmt.string(), data_buff))) {
LOG_WARN("fail to write string", K(ret));
@ -2419,14 +2421,17 @@ int ObLoadDataSPImpl::ToolBox::build_calc_partid_expr(ObExecContext &ctx,
ObSqlString insert_sql;
ObSEArray<ObString, 16> column_names;
ObLoadArgument &load_args = load_stmt.get_load_arguments();
bool need_online_osg = false;
for (int i = 0; OB_SUCC(ret) && i < insert_infos.count(); ++i) {
OZ (column_names.push_back(insert_infos.at(i).column_name_));
}
OZ (ObLoadDataUtils::check_need_opt_stat_gather(ctx, load_stmt, need_online_osg));
OZ (ObLoadDataUtils::build_insert_sql_string_head(load_args.dupl_action_,
load_args.combined_name_,
column_names,
insert_sql));
insert_sql,
need_online_osg));
OZ (insert_sql.append(" VALUES("));
for (int i = 0; OB_SUCC(ret) && i < insert_infos.count(); ++i) {
if (i != 0) {
@ -2583,6 +2588,7 @@ int ObLoadDataSPImpl::ToolBox::init(ObExecContext &ctx, ObLoadDataStmt &load_stm
ObIODOpt opt;
ObIODOpts iod_opts;
ObBackupIoAdapter util;
bool need_online_osg = false;
iod_opts.opts_ = &opt;
iod_opts.opt_cnt_ = 0;
@ -2616,9 +2622,12 @@ int ObLoadDataSPImpl::ToolBox::init(ObExecContext &ctx, ObLoadDataStmt &load_stm
LOG_WARN("fail to init data_trimer", K(ret));
} else if (OB_FAIL(gen_load_table_column_desc(ctx, load_stmt, insert_infos))) {
LOG_WARN("fail to build load table column desc", K(ret));
} else if (OB_FAIL(ObLoadDataUtils::check_need_opt_stat_gather(ctx, load_stmt, need_online_osg))) {
LOG_WARN("fail to check need online stats gather", K(ret));
} else if (OB_FAIL(gen_insert_columns_names_buff(ctx, load_args,
insert_infos,
insert_stmt_head_buff))) {
insert_stmt_head_buff,
need_online_osg))) {
LOG_WARN("fail to gen insert column names buff", K(ret));
} else if (OB_FAIL(data_frag_mgr.init(ctx, load_args.table_id_))) {
LOG_WARN("fail to init data frag mgr", K(ret));

View File

@ -795,7 +795,8 @@ private:
static int gen_insert_columns_names_buff(ObExecContext &ctx,
const ObLoadArgument &load_args,
common::ObIArray<ObLoadTableColumnDesc> &insert_infos,
common::ObString &data_buff);
common::ObString &data_buff,
bool need_online_osg = false);
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObLoadDataSPImpl);
// function members

View File

@ -26,11 +26,13 @@ const char ObLoadDataUtils::NULL_VALUE_FLAG = '\xff';
int ObLoadDataUtils::build_insert_sql_string_head(ObLoadDupActionType insert_mode,
const ObString &table_name,
const ObIArray<ObString> &insert_keys,
ObSqlString &insertsql_keys)
ObSqlString &insertsql_keys,
bool need_gather_opt_stat)
{
int ret = OB_SUCCESS;
static const char *replace_stmt = "replace into ";
static const char *insert_stmt = "insert into ";
static const char *insert_stmt_gather_opt_stat = "insert /*+GATHER_OPTIMIZER_STATISTICS*/ into ";
static const char *insert_ignore_stmt = "insert ignore into ";
const char *stmt_head = NULL;
@ -41,9 +43,14 @@ int ObLoadDataUtils::build_insert_sql_string_head(ObLoadDupActionType insert_mod
case ObLoadDupActionType::LOAD_IGNORE:
stmt_head = insert_ignore_stmt;
break;
case ObLoadDupActionType::LOAD_STOP_ON_DUP:
stmt_head = insert_stmt;
case ObLoadDupActionType::LOAD_STOP_ON_DUP: {
if (need_gather_opt_stat) {
stmt_head = insert_stmt_gather_opt_stat;
} else {
stmt_head = insert_stmt;
}
break;
}
default:
ret = OB_NOT_SUPPORTED;
LOG_WARN("not suppport insert mode", K(insert_mode));
@ -365,6 +372,32 @@ int ObLoadDataUtils::check_session_status(ObSQLSessionInfo &session, int64_t res
return ret;
}
int ObLoadDataUtils::check_need_opt_stat_gather(ObExecContext &ctx,
ObLoadDataStmt &load_stmt,
bool &need_opt_stat_gather)
{
int ret = OB_SUCCESS;
ObSQLSessionInfo *session = nullptr;
const ObLoadDataHint &hint = load_stmt.get_hints();
ObObj obj;
int64_t append = 0;
int64_t gather_optimizer_statistics = 0;
need_opt_stat_gather = false;
if (OB_ISNULL(session = ctx.get_my_session())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("session is null", KR(ret));
} else if (OB_FAIL(session->get_sys_variable(share::SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD, obj))) {
LOG_WARN("fail to get sys variable", K(ret));
} else if (OB_FAIL(hint.get_value(ObLoadDataHint::APPEND, append))) {
LOG_WARN("fail to get value of APPEND", K(ret));
} else if (OB_FAIL(hint.get_value(ObLoadDataHint::GATHER_OPTIMIZER_STATISTICS, gather_optimizer_statistics))) {
LOG_WARN("fail to get value of APPEND", K(ret));
} else if (((append != 0) || (gather_optimizer_statistics != 0)) && obj.get_bool()) {
need_opt_stat_gather = true;
}
return ret;
}
/////////////////
ObGetAllJobStatusOp::ObGetAllJobStatusOp()

View File

@ -16,6 +16,8 @@
#include "lib/string/ob_sql_string.h"
#include "lib/hash/ob_hashmap.h"
#include "common/object/ob_object.h"
#include "sql/resolver/cmd/ob_load_data_stmt.h"
#include "sql/engine/ob_exec_context.h"
#ifndef OCEANBASE_SQL_ENGINE_CMD_LOAD_DATA_UTILS_H_
#define OCEANBASE_SQL_ENGINE_CMD_LOAD_DATA_UTILS_H_
@ -148,7 +150,13 @@ public:
static int build_insert_sql_string_head(ObLoadDupActionType insert_mode,
const common::ObString &table_name,
const common::ObIArray<common::ObString> &insert_keys,
common::ObSqlString &insertsql_keys);
common::ObSqlString &insertsql_keys,
bool need_gather_opt_stat = false);
static int check_need_opt_stat_gather(ObExecContext &ctx,
ObLoadDataStmt &load_stmt,
bool &need_opt_stat_gather);
static int append_values_in_remote_process(int64_t table_column_count,
int64_t append_values_count,
const ObExprValueBitSet &expr_bitset,

View File

@ -100,7 +100,6 @@ int ObCreateTableExecutor::prepare_ins_arg(ObCreateTableStmt &stmt,
bool is_set_subquery = false;
bool is_oracle_mode = lib::is_oracle_mode();
bool no_osg_hint = false;
bool osg_hint = false;
bool online_sys_var = false;
const ObString &db_name = stmt.get_database_name();
const ObString &tab_name = stmt.get_table_name();
@ -122,11 +121,10 @@ int ObCreateTableExecutor::prepare_ins_arg(ObCreateTableStmt &stmt,
} else {
//get hint
no_osg_hint = select_stmt->get_query_ctx()->get_global_hint().has_no_gather_opt_stat_hint();
osg_hint = select_stmt->get_query_ctx()->get_global_hint().has_gather_opt_stat_hint();
//get system variable
ObObj online_sys_var_obj;
if (OB_FAIL(OB_FAIL(my_session->get_sys_variable(SYS_VAR_ONLINE_OPT_STAT_GATHER, online_sys_var_obj)))) {
if (OB_FAIL(OB_FAIL(my_session->get_sys_variable(SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD, online_sys_var_obj)))) {
LOG_WARN("fail to get sys var", K(ret));
} else {
online_sys_var = online_sys_var_obj.get_bool();
@ -136,7 +134,7 @@ int ObCreateTableExecutor::prepare_ins_arg(ObCreateTableStmt &stmt,
if (OB_FAIL(ret)) {
} else if (OB_FAIL(databuff_printf(buf, buf_len, pos1,
(!no_osg_hint && (online_sys_var || osg_hint))
(!no_osg_hint && online_sys_var)
? "insert /*+GATHER_OPTIMIZER_STATISTICS*/ into %c%.*s%c.%c%.*s%c"
: "insert /*+NO_GATHER_OPTIMIZER_STATISTICS*/ into %c%.*s%c.%c%.*s%c",
sep_char,

View File

@ -164,12 +164,14 @@ int ObInsertLogPlan::check_need_online_stats_gather(bool &need_osg)
if (OB_ISNULL(ins_table = insert_stmt->get_table_item_by_id(insert_stmt->get_insert_table_info().table_id_))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null pointer", K(ret), K(insert_stmt->get_insert_table_info()));
} else if (OB_UNLIKELY(ins_table->is_system_table_ || ins_table->is_index_table_)) {
} else if (OB_UNLIKELY(ins_table->is_system_table_ || ins_table->is_index_table_)
|| insert_stmt->is_insert_up()
|| !get_optimizer_context().get_session_info()->is_user_session()) {
need_gathering = false;
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(get_optimizer_context().get_session_info()->get_sys_variable(share::SYS_VAR_ONLINE_OPT_STAT_GATHER,
} else if (OB_FAIL(get_optimizer_context().get_session_info()->get_sys_variable(share::SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD,
online_sys_var_obj))) {
LOG_WARN("fail to get sys var", K(ret));
} else {
@ -177,11 +179,11 @@ int ObInsertLogPlan::check_need_online_stats_gather(bool &need_osg)
// shouldn't gather stats if the stmt is insert update.
// if the online_opt_stat_gather is enable, should gather opt_stats even there is no hint.
// if the online_opt_stat_gather is disable, only gather opt_stats when there is hint.
need_osg = !insert_stmt->is_insert_up()
&& need_gathering
need_osg = need_gathering
&& !get_optimizer_context().get_query_ctx()->get_global_hint().has_no_gather_opt_stat_hint()
&& (online_sys_var
|| (get_optimizer_context().get_query_ctx()->get_global_hint().should_generate_osg_operator()));
&& online_sys_var
&& ((get_optimizer_context().get_query_ctx()->get_global_hint().should_generate_osg_operator())
|| (get_optimizer_context().use_pdml()));
LOG_TRACE("online insert stat", K(online_sys_var), K(need_osg));
}
return ret;