diff --git a/deps/oblib/src/lib/worker.h b/deps/oblib/src/lib/worker.h index 114a9cde00..4703ea5b06 100644 --- a/deps/oblib/src/lib/worker.h +++ b/deps/oblib/src/lib/worker.h @@ -109,6 +109,7 @@ public: { set_sql_throttle_current_priority(100); } OB_INLINE void set_session(sql::ObSQLSessionInfo* session) { session_ = session; } + OB_INLINE sql::ObSQLSessionInfo *get_session() { return session_; } public: static void set_compatibility_mode(CompatMode mode); diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 022dee5d71..64c1b70fc4 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -541,7 +541,7 @@ PCODE_DEF(OB_RECOMPILE_ALL_VIEWS_BATCH, 0x52A) //reset status for view during up PCODE_DEF(OB_DAS_ASYNC_ACCESS, 0x52B) //das async rpc PCODE_DEF(OB_TRY_ADD_DEP_INFOS_FOR_SYNONYM_BATCH, 0x52C) //add dependency for synonym during upgrade PCODE_DEF(OB_CLEAN_DTL_INTERM_RESULT, 0x52D) //add dependency for synonym during upgrade -//PCODE_DEF(OB_CANCEL_GATHER_STATS, 0x52E)//used to cancel gather stats by rpc, will patch later +PCODE_DEF(OB_CANCEL_GATHER_STATS, 0x52E)//used to cancel gather stats by rpc PCODE_DEF(OB_SQL_PCODE_END, 0x54F) // as a guardian diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 605ab2158b..597ac29785 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -1779,6 +1779,7 @@ typedef enum ObItemType T_ANALYZE_SAMPLE_INFO, T_MYSQL_UPDATE_HISTOGRAM, T_MYSQL_DROP_HISTOGRAM, + T_MYSQL_ANALYZE, T_ANALYZE_MYSQL_COLUMN_LIST, T_SWITCHOVER, T_SWITCHOVER_TO_PRIMARY, diff --git a/src/observer/mysql/ob_query_retry_ctrl.cpp b/src/observer/mysql/ob_query_retry_ctrl.cpp index 3036d5432a..a2200415fd 100644 --- a/src/observer/mysql/ob_query_retry_ctrl.cpp +++ b/src/observer/mysql/ob_query_retry_ctrl.cpp @@ -634,7 +634,8 @@ private: && parent_ctx->get_pl_stack_ctx() != nullptr && !parent_ctx->get_pl_stack_ctx()->in_autonomous()); bool is_fk_nested = (parent_ctx != nullptr && parent_ctx->get_das_ctx().is_fk_cascading_); - return is_pl_nested || is_fk_nested; + bool is_online_stat_gathering_nested = (parent_ctx != nullptr && parent_ctx->is_online_stats_gathering()); + return is_pl_nested || is_fk_nested || is_online_stat_gathering_nested; } }; diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index 254647081b..08cafccfbb 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -2955,5 +2955,18 @@ int ObTabletLocationReceiveP::process() return OB_SUCCESS; } +int ObCancelGatherStatsP::process() +{ + int ret = OB_SUCCESS; + if (!arg_.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", K(arg_), K(ret)); + } else if (OB_FAIL(ObOptStatGatherStatList::instance().cancel_gather_stats(arg_.tenant_id_, + arg_.task_id_))) { + LOG_WARN("failed to cancel gather stats", K(ret)); + } + return ret; +} + } // end of namespace observer } // end of namespace oceanbase diff --git a/src/observer/ob_rpc_processor_simple.h b/src/observer/ob_rpc_processor_simple.h index bdba722088..aa0b0085c4 100644 --- a/src/observer/ob_rpc_processor_simple.h +++ b/src/observer/ob_rpc_processor_simple.h @@ -259,6 +259,7 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_TABLET_MAJOR_FREEZE, ObRpcTabletMajorFreezeP); // OB_DEFINE_PROCESSOR_S(Srv, OB_KILL_CLIENT_SESSION, ObKillClientSessionP); // OB_DEFINE_PROCESSOR_S(Srv, OB_CLIENT_SESSION_CONNECT_TIME, ObClientSessionConnectTimeP); OB_DEFINE_PROCESSOR_S(Srv, OB_TABLET_LOCATION_BROADCAST, ObTabletLocationReceiveP); +OB_DEFINE_PROCESSOR_S(Srv, OB_CANCEL_GATHER_STATS, ObCancelGatherStatsP); OB_DEFINE_PROCESSOR_S(Srv, OB_KILL_CLIENT_SESSION, ObKillClientSessionP); OB_DEFINE_PROCESSOR_S(Srv, OB_CLIENT_SESSION_CONNECT_TIME, ObClientSessionConnectTimeP); diff --git a/src/observer/ob_srv_xlator_storage.cpp b/src/observer/ob_srv_xlator_storage.cpp index 1b2a893240..852a9eebe1 100644 --- a/src/observer/ob_srv_xlator_storage.cpp +++ b/src/observer/ob_srv_xlator_storage.cpp @@ -121,4 +121,5 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(ObRpcDDLCheckTabletMergeStatusP, gctx_); RPC_PROCESSOR(ObRpcCreateDuplicateLSP, gctx_); RPC_PROCESSOR(ObRpcTabletMajorFreezeP, gctx_); + RPC_PROCESSOR(ObCancelGatherStatsP, gctx_); } diff --git a/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.cpp b/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.cpp index d25596af6e..5a2165bfb0 100644 --- a/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.cpp +++ b/src/observer/virtual_table/ob_all_virtual_opt_stat_gather_monitor.cpp @@ -161,8 +161,14 @@ int ObAllVirtualOptStatGatherMonitor::inner_get_next_row(ObNewRow *&row) cells[cell_idx].set_null(); break; } - case SPARE2: { - cells[cell_idx].set_null(); + case SPARE2: {//running table progress + if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_2_0) { + cells[cell_idx].set_null(); + } else { + cells[cell_idx].set_varchar(stat_array_.at(index_).get_table_gather_progress()); + cells[cell_idx].set_collation_type( + ObCharset::get_default_collation(ObCharset::get_default_charset())); + } break; } default: { diff --git a/src/pl/ob_pl_interface_pragma.h b/src/pl/ob_pl_interface_pragma.h index 9bfa8d3668..46706400cd 100644 --- a/src/pl/ob_pl_interface_pragma.h +++ b/src/pl/ob_pl_interface_pragma.h @@ -339,6 +339,8 @@ INTERFACE_DEF(INTERFACE_DBMS_STATS_SET_INDEX_STATS, "SET_INDEX_STATS", (ObDbmsStats::set_index_stats)) INTERFACE_DEF(INTERFACE_DBMS_STATS_EXPORT_INDEX_STATS, "EXPORT_INDEX_STATS", (ObDbmsStats::export_index_stats)) INTERFACE_DEF(INTERFACE_DBMS_STATS_IMPORT_INDEX_STATS, "IMPORT_INDEX_STATS", (ObDbmsStats::import_index_stats)) + INTERFACE_DEF(INTERFACE_DBMS_STATS_COPY_TABLE_STATS, "COPY_TABLE_STATS", (ObDbmsStats::copy_table_stats)) + INTERFACE_DEF(INTERFACE_DBMS_STATS_CNACEL_GATHER_STATS, "CANCEL_GATHER_STATS", (ObDbmsStats::cancel_gather_stats)) INTERFACE_DEF(INTERFACE_DBMS_STATS_GATHER_SYSTEM_STATS, "GATHER_SYSTEM_STATS", (ObDbmsStats::gather_system_stats)) INTERFACE_DEF(INTERFACE_DBMS_STATS_DELETE_SYSTEM_STATS, "DELETE_SYSTEM_STATS", (ObDbmsStats::delete_system_stats)) INTERFACE_DEF(INTERFACE_DBMS_STATS_SET_SYSTEM_STATS, "SET_SYSTEM_STATS", (ObDbmsStats::set_system_stats)) diff --git a/src/pl/sys_package/ob_dbms_stats.cpp b/src/pl/sys_package/ob_dbms_stats.cpp index 636fa7841c..09eb51ea73 100644 --- a/src/pl/sys_package/ob_dbms_stats.cpp +++ b/src/pl/sys_package/ob_dbms_stats.cpp @@ -85,12 +85,14 @@ int ObDbmsStats::gather_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb ObOptStatGatherStat gather_stat(task_info); ObOptStatGatherStatList::instance().push(gather_stat); ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), start_time, stat_param.allocator_->used(), gather_stat); - if (OB_FAIL(parse_table_part_info(ctx, - params.at(0), - params.at(1), - params.at(2), - stat_param, - true))) { + if (OB_FAIL(running_monitor.add_monitor_info(ObOptStatRunningPhase::GATHER_PREPARE))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(parse_table_part_info(ctx, + params.at(0), + params.at(1), + params.at(2), + stat_param, + true))) { LOG_WARN("failed to parse owner", K(ret)); } else if (OB_FAIL(parse_gather_stat_options(ctx, params.at(3), @@ -113,7 +115,7 @@ int ObDbmsStats::gather_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb LOG_WARN("failed check stat locked", K(ret)); } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx, false, true))) { LOG_WARN("failed to do flush database monitoring info", K(ret)); - } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param))) { + } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param, running_monitor))) { LOG_WARN("failed to gather table stats", K(ret)); } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), stat_param, @@ -121,6 +123,8 @@ int ObDbmsStats::gather_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb LOG_WARN("failed to update stat cache", K(ret)); } else if (!need_gather_index_stats(stat_param)) { //not gather virtual table/external table index. + } else if (OB_FAIL(running_monitor.add_monitor_info(ObOptStatRunningPhase::GATHER_INDEX_STATS))) { + LOG_WARN("failed to add add monitor info", K(ret)); } else if (stat_param.cascade_ && OB_FAIL(fast_gather_index_stats(ctx, stat_param, is_all_fast_gather, no_gather_index_ids))) { @@ -135,8 +139,11 @@ int ObDbmsStats::gather_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb task_info.task_end_time_ = ObTimeUtility::current_time(); task_info.ret_code_ = ret; task_info.failed_count_ = ret == OB_SUCCESS ? 0 : 1; + sql::ObSQLSessionInfo *origin_session = THIS_WORKER.get_session(); + THIS_WORKER.set_session(NULL); ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); ObOptStatManager::get_instance().update_opt_stat_gather_stat(gather_stat); + THIS_WORKER.set_session(origin_session); ObOptStatGatherStatList::instance().remove(gather_stat); } return ret; @@ -176,6 +183,10 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO } else if (OB_ISNULL(ctx.get_my_session()) || OB_ISNULL(ctx.get_task_executor_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(ctx.get_my_session()), K(ctx.get_task_executor_ctx())); + } else if (ctx.get_my_session()->get_is_in_retry()) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("retry gather schema stats is not allowed", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry gather schema stats is not allowed"); } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx, false, true))) { LOG_WARN("failed to do flush database monitoring info", K(ret)); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { @@ -201,18 +212,22 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), start_time, stat_param.allocator_->used(), gather_stat); if (OB_FAIL(refresh_tenant_schema_guard(ctx, global_param.tenant_id_))) { LOG_WARN("refresh tenant schema guard failed", K(ret)); + } else if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); + } else if (OB_FAIL(running_monitor.add_monitor_info(ObOptStatRunningPhase::GATHER_PREPARE))) { + LOG_WARN("failed to add add monitor info", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, stat_table, stat_param, true))) { LOG_WARN("failed to parse table part info", K(ret)); } else if (OB_FAIL(parse_gather_stat_options(ctx, - params.at(1), - params.at(2), - params.at(3), - params.at(4), - params.at(5), - params.at(6), - params.at(10), - params.at(12), - stat_param))) { + params.at(1), + params.at(2), + params.at(3), + params.at(4), + params.at(5), + params.at(6), + params.at(10), + params.at(12), + stat_param))) { LOG_WARN("failed to parse stat optitions", K(ret)); } else if (OB_FAIL(running_monitor.add_table_info(stat_param))) { LOG_WARN("failed to add table info", K(ret)); @@ -230,7 +245,7 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO } } else if (share::schema::ObTableType::EXTERNAL_TABLE == stat_param.ref_table_type_) { // not allow gather external table in schema scope - } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param))) { + } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param, running_monitor))) { LOG_WARN("failed to gather table stats", K(ret)); } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), stat_param, @@ -238,6 +253,8 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO LOG_WARN("failed to update stat cache", K(ret)); } else if (is_virtual_table(stat_param.table_id_)) {//not gather virtual table index. //do nothing + } else if (OB_FAIL(running_monitor.add_monitor_info(ObOptStatRunningPhase::GATHER_INDEX_STATS))) { + LOG_WARN("failed to add add monitor info", K(ret)); } else if (stat_param.cascade_ && OB_FAIL(fast_gather_index_stats(ctx, stat_param, is_all_fast_gather, no_gather_index_ids))) { @@ -249,14 +266,20 @@ int ObDbmsStats::gather_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO LOG_TRACE("Succeed to gather table stats", K(stat_param), K(running_monitor)); } running_monitor.set_monitor_result(ret, ObTimeUtility::current_time(), stat_param.allocator_->used()); + sql::ObSQLSessionInfo *origin_session = THIS_WORKER.get_session(); + THIS_WORKER.set_session(NULL); ObOptStatManager::get_instance().update_opt_stat_gather_stat(gather_stat); + THIS_WORKER.set_session(origin_session); ObOptStatGatherStatList::instance().remove(gather_stat); task_info.completed_table_count_ ++; } task_info.task_end_time_ = ObTimeUtility::current_time(); task_info.ret_code_ = ret; task_info.failed_count_ = ret == OB_SUCCESS ? 0 : table_ids.count() - i + 1; + sql::ObSQLSessionInfo *origin_session = THIS_WORKER.get_session(); + THIS_WORKER.set_session(NULL); ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); + THIS_WORKER.set_session(origin_session); } return ret; } @@ -752,6 +775,7 @@ int ObDbmsStats::set_index_stats(ObExecContext &ctx, ParamStore ¶ms, ObObj & * 8. statown VARCHAR2 DEFAULT NULL, * 9. no_invalidate BOOLEAN DEFAULT FALSE, * 10. force BOOLEAN DEFAULT FALSE + * 11. degree NUMBER DEFAULT 1 * @param result * @return */ @@ -782,6 +806,9 @@ int ObDbmsStats::delete_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb LOG_WARN("failed to get no invalidate", K(ret)); } else if (!params.at(10).is_null() && OB_FAIL(params.at(10).get_bool(stat_param.force_))) { LOG_WARN("failed to get no invalidate", K(ret)); + } else if (GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_2_2_0 && + OB_FAIL(parse_degree_option(ctx, params.at(11), stat_param))) { + LOG_WARN("parse degree param failed", K(ret)); } else if (!cascade_columns) { stat_param.column_params_.reset(); } @@ -827,6 +854,7 @@ int ObDbmsStats::delete_table_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb * 8. no_invalidate BOOLEAN DEFAULT FALSE, * 9. force BOOLEAN DEFAULT FALSE, * 10. col_stat_type VARCHAR2 DEFAULT 'ALL' + * 11. degree NUMBER DEFAULT 1 * @param params * @param result * @return @@ -863,6 +891,9 @@ int ObDbmsStats::delete_column_stats(ObExecContext &ctx, ParamStore ¶ms, ObO ctx.get_my_session()->get_dtc_params(), col_stat_type))) { LOG_WARN("failed to convert vaild ident name", K(ret)); + } else if (GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_2_2_0 && + OB_FAIL(parse_degree_option(ctx, params.at(11), stat_param))) { + LOG_WARN("parse degree param failed", K(ret)); } else if (0 == col_stat_type.case_compare("ALL")) { only_histogram = false; } else if (0 == col_stat_type.case_compare("HISTOGRAM")) { @@ -901,6 +932,7 @@ int ObDbmsStats::delete_column_stats(ObExecContext &ctx, ParamStore ¶ms, ObO * 3. statown VARCHAR2 DEFAULT NULL, * 4. no_invalidate BOOLEAN DEFAULT FALSE, * 5. force BOOLEAN DEFAULT FALSE + * 6. degree NUMBER DEFAULT 1 * @param result * @return */ @@ -913,6 +945,10 @@ int ObDbmsStats::delete_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO ObSEArray table_ids; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (ctx.get_my_session()->get_is_in_retry()) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("retry delete schema stats is not allowed", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry delete schema stats is not allowed"); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { LOG_WARN("failed to get all table ids in database", K(ret)); } else { @@ -931,6 +967,9 @@ int ObDbmsStats::delete_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO LOG_WARN("failed to get no invalidate", K(ret)); } else if (!params.at(5).is_null() && OB_FAIL(params.at(5).get_bool(stat_param.force_))) { LOG_WARN("failed to get no invalidate", K(ret)); + } else if (GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_2_2_0 && + OB_FAIL(parse_degree_option(ctx, params.at(6), stat_param))) { + LOG_WARN("parse degree param failed", K(ret)); } else { stat_param.global_stat_param_.need_modify_ = true; stat_param.part_stat_param_.need_modify_ = true; @@ -979,6 +1018,7 @@ int ObDbmsStats::delete_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO 8.stattype VARCHAR2 DEFAULT 'ALL', 9.force BOOLEAN DEFAULT FALSE, 10.tabname VARCHAR2 DEFAULT NULL(for mysql mode only) + 11.degree NUMBER DEFAULT 1 * @param result * @return */ @@ -1014,6 +1054,9 @@ int ObDbmsStats::delete_index_stats(ObExecContext &ctx, ParamStore ¶ms, ObOb LOG_WARN("failed to get no invalidate", K(ret)); } else if (!params.at(9).is_null() && OB_FAIL(params.at(9).get_bool(index_stat_param.force_))) { LOG_WARN("failed to get force", K(ret)); + } else if (GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_2_2_0 && + OB_FAIL(parse_degree_option(ctx, params.at(11), index_stat_param))) { + LOG_WARN("parse degree param failed", K(ret)); } else { decide_modified_part(index_stat_param, cascade_parts); } @@ -1071,6 +1114,24 @@ int ObDbmsStats::delete_table_index_stats(sql::ObExecContext &ctx, return ret; } +int ObDbmsStats::parse_degree_option(ObExecContext &ctx, + const ObObjParam °ree, + ObTableStatParam &stat_param) +{ + int ret = OB_SUCCESS; + number::ObNumber num_degree; + if (degree.is_null()) { + stat_param.degree_ = 1; + } else if (OB_FAIL(degree.get_number(num_degree))) { + LOG_WARN("failed to get degree", K(ret), K(degree)); + } else if (OB_FAIL(num_degree.extract_valid_int64_with_trunc(stat_param.degree_))) { + LOG_WARN("extract_valid_int64_with_trunc failed", K(ret), K(num_degree)); + } else if (stat_param.degree_ < 1) { + stat_param.degree_ = 1; + } + return ret; +} + /** * @brief ObDbmsStat::create_stat_table * @param ctx @@ -1365,6 +1426,10 @@ int ObDbmsStats::export_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO ObString tmp_str; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (ctx.get_my_session()->get_is_in_retry()) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("retry export schema stats is not allowed", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry export schema stats is not allowed"); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { LOG_WARN("failed to get all table ids in database", K(ret)); } else if (OB_FAIL(parse_table_info(ctx, @@ -1712,6 +1777,10 @@ int ObDbmsStats::import_schema_stats(ObExecContext &ctx, ParamStore ¶ms, ObO ObSEArray table_ids; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (ctx.get_my_session()->get_is_in_retry()) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("retry import schema stats is not allowed", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry import schema stats is not allowed"); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { LOG_WARN("failed to get all table ids in database", K(ret)); } else if (OB_FAIL(parse_table_info(ctx, @@ -2022,6 +2091,10 @@ int ObDbmsStats::lock_schema_stats(sql::ObExecContext &ctx, ObSEArray table_ids; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (ctx.get_my_session()->get_is_in_retry()) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("retry lock schema stats is not allowed", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry lock schema stats is not allowed"); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { LOG_WARN("failed to get all table ids in database", K(ret)); } else if (OB_FAIL(params.at(1).get_varchar(stat_type_str))) { @@ -2212,7 +2285,11 @@ int ObDbmsStats::unlock_schema_stats(sql::ObExecContext &ctx, ObSEArray table_ids; global_param.stattype_ = StatTypeLocked::TABLE_ALL_TYPE; global_param.allocator_ = &ctx.get_allocator(); - if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { + if (ctx.get_my_session()->get_is_in_retry()) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("retry unlock schema stats is not allowed", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry unlock schema stats is not allowed"); + } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { LOG_WARN("failed to get all table ids in database", K(ret)); } else if (!params.at(1).is_null() && OB_FAIL(params.at(1).get_varchar(stat_type_str))) { LOG_WARN("failed to get stattype", K(ret)); @@ -2387,6 +2464,10 @@ int ObDbmsStats::restore_schema_stats(sql::ObExecContext &ctx, int64_t specify_time = 0; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (ctx.get_my_session()->get_is_in_retry()) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("retry restore schema stats is not allowed", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry restore schema stats is not allowed"); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { LOG_WARN("failed to get all table ids in database", K(ret)); } else if (lib::is_oracle_mode()) { @@ -2775,6 +2856,10 @@ int ObDbmsStats::set_schema_prefs(sql::ObExecContext &ctx, ObStatPrefs *stat_pref = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (ctx.get_my_session()->get_is_in_retry()) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("retry set schema stats is not allowed", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry set schema stats is not allowed"); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), global_param, table_ids))) { LOG_WARN("failed to get all table ids in database", K(ret)); } else if (!params.at(1).is_null() && OB_FAIL(params.at(1).get_string(opt_name))) { @@ -2894,6 +2979,10 @@ int ObDbmsStats::delete_schema_prefs(sql::ObExecContext &ctx, ObStatPrefs *stat_pref = NULL; if (OB_FAIL(check_statistic_table_writeable(ctx))) { LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (ctx.get_my_session()->get_is_in_retry()) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("retry delete schema stats is not allowed", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"retry delete schema stats is not allowed"); } else if (OB_FAIL(get_all_table_ids_in_database(ctx, params.at(0), dummy_param, table_ids))) { LOG_WARN("failed to get all table ids in database", K(ret)); } else if (params.at(1).is_null()) { @@ -2966,6 +3055,35 @@ int ObDbmsStats::delete_table_prefs(sql::ObExecContext &ctx, return ret; } +/** + * @brief ObDbmsStats::cancel_gather_stats + * @param ctx + * @param params + * 0. taskid VARCHAR2 + * @param result + * @return int + */ +int ObDbmsStats::cancel_gather_stats(sql::ObExecContext &ctx, + sql::ParamStore ¶ms, + common::ObObj &result) +{ + int ret = OB_SUCCESS; + UNUSED(result); + ObString task_id; + if (OB_UNLIKELY(params.at(0).is_null())) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("not a valid task id", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "not a valid task id"); + } else if (OB_FAIL(params.at(0).get_string(task_id))) { + LOG_WARN("failed to get table name", K(ret)); + } else if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_2_0) { + //do nothing + } else if (OB_FAIL(ObDbmsStatsExecutor::cancel_gather_stats(ctx, task_id))) { + LOG_WARN("failed to cancel gather stats", K(ret)); + } + return ret; +} + int ObDbmsStats::update_stat_cache(const uint64_t rpc_tenant_id, const ObTableStatParam ¶m, ObOptStatRunningMonitor *running_monitor/*default null*/) @@ -3072,6 +3190,18 @@ int ObDbmsStats::parse_table_part_info(ObExecContext &ctx, { int ret = OB_SUCCESS; const share::schema::ObTableSchema *table_schema = NULL; + return parse_table_part_info(ctx, owner, tab_name, part_name, param, table_schema, need_parse_col_group); +} + +int ObDbmsStats::parse_table_part_info(ObExecContext &ctx, + const ObObjParam &owner, + const ObObjParam &tab_name, + const ObObjParam &part_name, + ObTableStatParam ¶m, + const share::schema::ObTableSchema *&table_schema, + bool need_parse_col_group/*default false*/) +{ + int ret = OB_SUCCESS; share::schema::ObSchemaGetterGuard *schema_guard = ctx.get_virtual_table_ctx().schema_guard_; if (OB_ISNULL(schema_guard) || OB_ISNULL(param.allocator_)) { ret = OB_ERR_UNEXPECTED; @@ -3084,9 +3214,7 @@ int ObDbmsStats::parse_table_part_info(ObExecContext &ctx, LOG_USER_ERROR(OB_TABLE_NOT_EXIST, to_cstring(param.db_name_), to_cstring(param.tab_name_)); } else if (OB_FAIL(get_table_part_infos(table_schema, param.part_infos_, - param.subpart_infos_, - param.part_ids_, - param.subpart_ids_))) { + param.subpart_infos_))) { LOG_WARN("failed to get table part infos", K(ret)); } else if (OB_FAIL(param.all_part_infos_.assign(param.part_infos_)) || OB_FAIL(param.all_subpart_infos_.assign(param.subpart_infos_))) { @@ -3097,7 +3225,6 @@ int ObDbmsStats::parse_table_part_info(ObExecContext &ctx, param.table_id_ = table_schema->get_table_id(); param.ref_table_type_ = table_schema->get_table_type(); param.part_level_ = table_schema->get_part_level(); - param.total_part_cnt_ = table_schema->get_all_part_num(); // we can't get part/subpart type anyway, because default value of part_func_type is // PARTITION_FUNC_TYPE_HASH even table is not partitioned. if (share::schema::ObPartitionLevel::PARTITION_LEVEL_ONE == param.part_level_) { @@ -3138,9 +3265,7 @@ int ObDbmsStats::parse_table_part_info(ObExecContext &ctx, ret = OB_TABLE_NOT_EXIST; } else if (OB_FAIL(get_table_part_infos(table_schema, param.part_infos_, - param.subpart_infos_, - param.part_ids_, - param.subpart_ids_))) { + param.subpart_infos_))) { LOG_WARN("failed to get table part infos", K(ret)); } else if (OB_FAIL(param.all_part_infos_.assign(param.part_infos_)) || OB_FAIL(param.all_subpart_infos_.assign(param.subpart_infos_))) { @@ -3157,7 +3282,6 @@ int ObDbmsStats::parse_table_part_info(ObExecContext &ctx, param.table_id_ = table_schema->get_table_id(); param.ref_table_type_ = table_schema->get_table_type(); param.part_level_ = table_schema->get_part_level(); - param.total_part_cnt_ = table_schema->get_all_part_num(); } return ret; } @@ -3206,9 +3330,7 @@ int ObDbmsStats::parse_index_part_info(ObExecContext &ctx, LOG_WARN("failed to write string", K(ret)); } else if (OB_FAIL(get_table_part_infos(index_schema, param.part_infos_, - param.subpart_infos_, - param.part_ids_, - param.subpart_ids_))) { + param.subpart_infos_))) { LOG_WARN("failed to get table part infos", K(ret)); } else if (OB_FAIL(param.all_part_infos_.assign(param.part_infos_)) || OB_FAIL(param.all_subpart_infos_.assign(param.subpart_infos_))) { @@ -3222,7 +3344,6 @@ int ObDbmsStats::parse_index_part_info(ObExecContext &ctx, param.table_id_ = index_schema->get_table_id(); param.ref_table_type_ = index_schema->get_table_type(); param.part_level_ = index_schema->get_part_level(); - param.total_part_cnt_ = index_schema->get_all_part_num(); param.is_global_index_ = index_schema->is_global_index_table(); param.data_table_id_ = table_schema->get_table_id(); if (OB_FAIL(init_column_stat_params(*param.allocator_, @@ -3279,7 +3400,7 @@ int ObDbmsStats::init_column_stat_params(ObIAllocator &allocator, } else { col_param.column_id_ = col->get_column_id(); col_param.cs_type_ = col->get_collation_type(); - col_param.gather_flag_ = 0; + col_param.gather_flag_ = ColumnGatherFlag::NO_NEED_STAT; col_param.set_size_manual(); col_param.bucket_num_ = -1; col_param.column_attribute_ = 0; @@ -3294,11 +3415,6 @@ int ObDbmsStats::init_column_stat_params(ObIAllocator &allocator, if (ObColumnStatParam::is_valid_avglen_type(col->get_meta_type().get_type())) { col_param.set_need_avg_len(); } - //check can gather hist type, for longtext type, we disable gather hist - if (ob_obj_type_class(col->get_meta_type().get_type()) == ColumnTypeClass::ObTextTC && - col->get_meta_type().get_type() != ObTinyTextType) { - col_param.set_no_need_histogram(); - } } if (col->is_rowkey_column() && !table_schema.is_heap_table()) { col_param.set_is_index_column(); @@ -3533,9 +3649,7 @@ int ObDbmsStats::parse_set_partition_name(ObExecContext &ctx, LOG_WARN("the target table is not partitioned", K(ret)); } else if (OB_FAIL(get_table_part_infos(table_schema, param.part_infos_, - param.subpart_infos_, - param.part_ids_, - param.subpart_ids_))) { + param.subpart_infos_))) { LOG_WARN("failed to get table part infos", K(ret)); } else if (OB_FAIL(param.all_part_infos_.assign(param.part_infos_)) || OB_FAIL(param.all_subpart_infos_.assign(param.subpart_infos_))) { @@ -3579,8 +3693,8 @@ int ObDbmsStats::parse_partition_name(ObExecContext &ctx, ret = OB_ERR_NOT_PARTITIONED; LOG_WARN("the target table is not partitioned", K(ret)); } else if (OB_FAIL(find_selected_part_infos(param.part_name_, - param.part_infos_, - param.subpart_infos_, + param.all_part_infos_, + param.all_subpart_infos_, lib::is_oracle_mode(), part_infos, subpart_infos, @@ -3910,11 +4024,6 @@ int ObDbmsStats::use_default_gather_stat_options(ObExecContext &ctx, { int ret = OB_SUCCESS; int64_t stat_options = StatOptionFlags::OPT_STAT_OPTION_ALL; - const static char *incremental_granularity = "APPROX_GLOBAL AND PARTITION"; - if (stat_table.incremental_stat_) { - stat_options &= ~StatOptionFlags::OPT_GRANULARITY; - param.granularity_.assign_ptr(incremental_granularity, strlen(incremental_granularity)); - } if (OB_FAIL(get_default_stat_options(ctx, stat_options, param))) { LOG_WARN("failed to get default stat options", K(ret)); } else if (OB_FAIL(parse_granularity_and_method_opt(ctx, param))) { @@ -4062,6 +4171,21 @@ int ObDbmsStats::parse_granularity_and_method_opt(ObExecContext &ctx, LOG_WARN("failed to process not size manual column", K(ret)); } } + if (OB_SUCC(ret)) {//remove no gather columns + ObSEArray new_column_params; + for (int64_t i = 0 ; OB_SUCC(ret) && i < param.column_params_.count(); ++i) { + if (param.column_params_.at(i).need_col_stat()) { + if (OB_FAIL(new_column_params.push_back(param.column_params_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(param.column_params_.assign(new_column_params))) { + LOG_WARN("failed to assign", K(ret)); + } + } + } return ret; } @@ -4528,11 +4652,11 @@ int ObDbmsStats::check_is_valid_col(const ObString &src_str, int ObDbmsStats::get_table_part_infos(const share::schema::ObTableSchema *table_schema, ObIArray &part_infos, ObIArray &subpart_infos, - ObIArray &part_ids, - ObIArray &subpart_ids, OSGPartMap *part_map/*default NULL*/) { int ret = OB_SUCCESS; + ObSEArray part_ids; + ObSEArray subpart_ids; if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(table_schema)); @@ -4567,12 +4691,18 @@ int ObDbmsStats::get_part_ids_from_schema(const ObTableSchema *table_schema, ObSEArray dummy_subpart_infos; ObSEArray part_ids; ObSEArray subpart_ids; - if (OB_FAIL(get_table_part_infos(table_schema, - dummy_part_infos, - dummy_subpart_infos, - part_ids, - subpart_ids))) { - LOG_WARN("fail to get part infos", K(ret)); + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(table_schema)); + } else if (!table_schema->is_partitioned_table()) { + /*do notthing*/ + LOG_TRACE("table is not part table", K(table_schema->get_part_level())); + } else if (OB_FAIL(ObDbmsStatsUtils::get_part_infos(*table_schema, + dummy_part_infos, + dummy_subpart_infos, + part_ids, + subpart_ids))) { + LOG_WARN("failed to get partition infos", K(ret)); } else if (OB_FAIL(append(target_part_ids, part_ids))) { LOG_WARN("fail to append target part id", K(ret)); } else if (OB_FAIL(append(target_part_ids, subpart_ids))) { @@ -4765,68 +4895,46 @@ int ObDbmsStats::find_selected_part_infos(const ObString &part_name, bool &is_subpart_name) { int ret = OB_SUCCESS; - bool found = false; PartInfo part; is_subpart_name = false; - bool is_twopart = false; - for (int64_t i = 0; !found && i < part_infos.count(); ++i) { - if ((is_sensitive_compare && - ObCharset::case_sensitive_equal(part_name, part_infos.at(i).part_name_)) || - (!is_sensitive_compare && - ObCharset::case_insensitive_equal(part_name, part_infos.at(i).part_name_))) { - part = part_infos.at(i); - found = true; + if (ObDbmsStatsUtils::find_part(part_infos, part_name, is_sensitive_compare, part)) { + if (OB_FAIL(new_part_infos.push_back(part))) { + LOG_WARN("failed to push back part info", K(ret)); } - } - for (int64_t i = 0; !found && i < subpart_infos.count(); ++i) { - if ((is_sensitive_compare && - ObCharset::case_sensitive_equal(part_name, subpart_infos.at(i).part_name_)) || - (!is_sensitive_compare && - ObCharset::case_insensitive_equal(part_name, subpart_infos.at(i).part_name_))) { - part = subpart_infos.at(i); - found = true; - is_twopart = true; - } - } - if (OB_SUCC(ret)) { - if (!found) { - ret = OB_UNKNOWN_PARTITION; - LOG_WARN("the specified partition is not found", K(ret), K(part_name)); - } else if (is_twopart) { - if (OB_FAIL(new_subpart_infos.push_back(part))) { - LOG_WARN("failed to push back part info", K(ret)); - } else { - bool find_it = false; - int64_t cur_part_id = part.first_part_id_; - for (int64_t i = 0; OB_SUCC(ret) && !find_it && i < part_infos.count(); ++i) { - if (cur_part_id != part_infos.at(i).part_id_) { - // do nothing - } else if (OB_FAIL(new_part_infos.push_back(part_infos.at(i)))) { - LOG_WARN("failed to push back subpart infos", K(ret)); - } else { - find_it = true; - } - } - if (OB_SUCC(ret) && !find_it) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error, partition id isn't found", K(ret), K(cur_part_id)); - } else { - is_subpart_name = true; - } + for (int64_t i = 0; OB_SUCC(ret) && i < subpart_infos.count(); ++i) { + int64_t cur_part_id = subpart_infos.at(i).first_part_id_; + if (cur_part_id != part.part_id_) { + // do nothing + } else if (OB_FAIL(new_subpart_infos.push_back(subpart_infos.at(i)))) { + LOG_WARN("failed to push back subpart infos", K(ret)); } + } + } else if (ObDbmsStatsUtils::find_part(subpart_infos, part_name, is_sensitive_compare, part)) { + if (OB_FAIL(new_subpart_infos.push_back(part))) { + LOG_WARN("failed to push back part info", K(ret)); } else { - if (OB_FAIL(new_part_infos.push_back(part))) { - LOG_WARN("failed to push back part info", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < subpart_infos.count(); ++i) { - int64_t cur_part_id = subpart_infos.at(i).first_part_id_; - if (cur_part_id != part.part_id_) { + bool find_it = false; + int64_t cur_part_id = part.first_part_id_; + for (int64_t i = 0; OB_SUCC(ret) && !find_it && i < part_infos.count(); ++i) { + if (cur_part_id != part_infos.at(i).part_id_) { // do nothing - } else if (OB_FAIL(new_subpart_infos.push_back(subpart_infos.at(i)))) { + } else if (OB_FAIL(new_part_infos.push_back(part_infos.at(i)))) { LOG_WARN("failed to push back subpart infos", K(ret)); + } else { + find_it = true; } } + if (OB_SUCC(ret) && !find_it) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error, partition id isn't found", K(ret), K(cur_part_id)); + } else { + is_subpart_name = true; + } } + } else { + ret = OB_UNKNOWN_PARTITION; + LOG_WARN("the specified partition is not found", K(ret), + K(part_name), K(part_infos), K(subpart_infos)); } return ret; } @@ -4872,6 +4980,9 @@ int ObDbmsStats::process_not_size_manual_column(sql::ObExecContext &ctx, } if (flag > 0) { auto_columns.at(i)->bucket_num_ = ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM; + } else if (auto_columns.at(i)->is_index_column() + && !auto_columns.at(i)->is_unique_column()) { + auto_columns.at(i)->bucket_num_ = ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM; } } } @@ -5222,25 +5333,33 @@ int ObDbmsStats::gather_database_stats_job_proc(sql::ObExecContext &ctx, } else if (lib::is_mysql_mode() && !params.empty() && !params.at(0).is_null() && OB_FAIL(params.at(0).get_int(duration_time))) { LOG_WARN("failed to get duration", K(ret), K(params.at(0))); - } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx))) { - LOG_WARN("failed to flush database monitoring info", K(ret)); - } else if (OB_FAIL(init_gather_task_info(ctx, ObOptStatGatherType::AUTO_GATHER, start_time, 0, task_info))) { - LOG_WARN("failed to init gather task info", K(ret)); - } else if (OB_FAIL(gather_database_table_stats(ctx, duration_time, succeed_cnt, task_info))) { - LOG_WARN("failed to gather table stats", K(ret)); - } else {/*do nothing*/} - const int64_t exe_time = ObTimeUtility::current_time() - start_time; - LOG_INFO("have been gathered database stats job", - "the total used time:", exe_time, - "the duration time:", duration_time, - "the toatal gather table cnt:", task_info.task_table_count_, - "the succeed to gather table cnt:", succeed_cnt, - "the failed to gather table cnt:", task_info.failed_count_, K(ret)); - //reset the error code, the reason is that the total gather time is reach the duration time. - ret = ret == OB_TIMEOUT ? OB_SUCCESS : ret; - task_info.task_end_time_ = ObTimeUtility::current_time(); - task_info.ret_code_ = ret; - ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); + } else { + if (duration_time > 0) { + THIS_WORKER.set_timeout_ts(duration_time + ObTimeUtility::current_time()); + } + if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx))) { + LOG_WARN("failed to flush database monitoring info", K(ret)); + } else if (OB_FAIL(init_gather_task_info(ctx, ObOptStatGatherType::AUTO_GATHER, start_time, 0, task_info))) { + LOG_WARN("failed to init gather task info", K(ret)); + } else if (OB_FAIL(gather_database_table_stats(ctx, duration_time, succeed_cnt, task_info))) { + LOG_WARN("failed to gather table stats", K(ret)); + } else {/*do nothing*/} + const int64_t exe_time = ObTimeUtility::current_time() - start_time; + LOG_INFO("have been gathered database stats job", + "the total used time:", exe_time, + "the duration time:", duration_time, + "the toatal gather table cnt:", task_info.task_table_count_, + "the succeed to gather table cnt:", succeed_cnt, + "the failed to gather table cnt:", task_info.failed_count_, K(ret)); + //reset the error code, the reason is that the total gather time is reach the duration time. + ret = ret == OB_TIMEOUT ? OB_SUCCESS : ret; + task_info.task_end_time_ = ObTimeUtility::current_time(); + task_info.ret_code_ = ret; + sql::ObSQLSessionInfo *origin_session = THIS_WORKER.get_session(); + THIS_WORKER.set_session(NULL); + ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); + THIS_WORKER.set_session(origin_session); + } return ret; } @@ -5268,7 +5387,9 @@ int ObDbmsStats::gather_database_table_stats(sql::ObExecContext &ctx, do { table_ids.reuse(); tmp_succeed = succeed_cnt; - if (OB_FAIL(ObBasicStatsEstimator::get_need_stats_tables(ctx, tenant_id, table_ids, slice_cnt))) { + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); + } else if (OB_FAIL(ObBasicStatsEstimator::get_need_stats_tables(ctx, tenant_id, table_ids, slice_cnt))) { LOG_WARN("failed to get tables that need gather stats", K(ret)); } else if (OB_FAIL(do_gather_tables_stats(ctx, tenant_id, table_ids, duration_time, succeed_cnt, task_info))) { @@ -5362,7 +5483,7 @@ int ObDbmsStats::do_gather_table_stats(sql::ObExecContext &ctx, stale_percent_threshold))) { LOG_WARN("failed to get table stale percent threshold", K(ret)); } else if (OB_FAIL(get_table_stale_percent(ctx, tenant_id, *table_schema, - stale_percent_threshold, stat_table))) { + stale_percent_threshold, stat_table))) { LOG_WARN("failed to get table stale percent", K(ret)); } else if (stat_table.stale_percent_ < 0 || stat_table.stale_percent_ > stale_percent_threshold) { if (is_oceanbase_sys_database_id(stat_table.database_id_)) { @@ -5401,32 +5522,25 @@ int ObDbmsStats::get_table_stale_percent(sql::ObExecContext &ctx, { int ret = OB_SUCCESS; uint64_t table_id = table_schema.get_table_id(); - const int64_t part_id = PARTITION_LEVEL_ZERO == table_schema.get_part_level() ? table_id : -1; - ObSEArray partition_stat_infos; + const int64_t global_part_id = PARTITION_LEVEL_ZERO == table_schema.get_part_level() ? table_id : -1; bool is_locked = false; - bool dummy_is_big_table = false; if (OB_FAIL(ObBasicStatsEstimator::check_table_statistics_state(ctx, tenant_id, table_id, - part_id, + global_part_id, is_locked, - partition_stat_infos))) { + stat_table.partition_stat_infos_))) { LOG_WARN("failed to check table has any statistics", K(ret)); } else if (is_locked) { //if table is locked, don't gather stats. - stat_table.incremental_stat_ = false; stat_table.stale_percent_ = 0; - } else if (table_schema.is_user_table() && -1 == part_id) {//for partitioned user table + } else if (table_schema.is_user_table() && -1 == global_part_id) {//for partitioned user table if (OB_FAIL(get_user_partition_table_stale_percent(ctx, tenant_id, table_schema, stale_percent_threshold, - partition_stat_infos, - stat_table, - dummy_is_big_table))) { + stat_table))) { LOG_WARN("faild to get user partition table stale percent", K(ret)); } else {/*do nothing*/} - } else if (OB_FAIL(get_common_table_stale_percent(ctx, tenant_id, table_schema, - partition_stat_infos, stat_table, - dummy_is_big_table))) { + } else if (OB_FAIL(get_common_table_stale_percent(ctx, tenant_id, table_schema, stat_table))) { LOG_WARN("failed to get common table stale percent", K(ret)); } else {/*do nothing*/} return ret; @@ -5440,35 +5554,21 @@ int ObDbmsStats::get_table_stale_percent(sql::ObExecContext &ctx, int ObDbmsStats::get_common_table_stale_percent(sql::ObExecContext &ctx, const uint64_t tenant_id, const ObTableSchema &table_schema, - const ObIArray &partition_stat_infos, - StatTable &stat_table, - bool &is_big_table) + StatTable &stat_table) { int ret = OB_SUCCESS; - is_big_table = false; //if this is virtual table real agent, we need see the real table id modifed count uint64_t table_id = share::is_oracle_mapping_real_virtual_table(table_schema.get_table_id()) ? share::get_real_table_mappings_tid(table_schema.get_table_id()) : table_schema.get_table_id(); const int64_t part_id = PARTITION_LEVEL_ZERO == table_schema.get_part_level() ? table_id : -1; int64_t inc_modified_count = 0; - stat_table.incremental_stat_ = false; int64_t row_cnt = 0; - bool is_gather_global_stat = false; if (OB_UNLIKELY(table_schema.is_user_table() && -1 == part_id)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(table_schema.is_user_table()), K(part_id)); - } else if (!is_table_gather_global_stats(part_id, partition_stat_infos, row_cnt)) { + } else if (!is_table_gather_global_stats(part_id, stat_table.partition_stat_infos_, row_cnt)) { stat_table.stale_percent_ = -1.0; - if (OB_FAIL(ObBasicStatsEstimator::estimate_row_count(ctx, - tenant_id, - table_id, - row_cnt))) { - LOG_WARN("failed to estimate row count"); - } else { - // currently we only regard tables whose rows >= 1000w as big table - is_big_table = row_cnt >= OPT_STATS_BIG_TABLE_ROWS; - } } else if (is_virtual_table(table_id)) {//virtual table doesn't see the modfiy count, no need regather stat_table.stale_percent_ = 0.0; } else if (OB_FAIL(ObBasicStatsEstimator::estimate_modified_count(ctx, @@ -5483,158 +5583,63 @@ int ObDbmsStats::get_common_table_stale_percent(sql::ObExecContext &ctx, } else if (inc_modified_count == 0) { stat_table.stale_percent_ = 0.0; } else { - stat_table.stale_percent_ = row_cnt == 0 ? 1.0 : 1.0 * (inc_modified_count) / row_cnt; + stat_table.stale_percent_ = row_cnt <= 0 ? 1.0 : 1.0 * (inc_modified_count) / row_cnt; } - LOG_TRACE("succeed to get common table stale percent", K(stat_table), K(partition_stat_infos), K(is_big_table)); + LOG_TRACE("succeed to get common table stale percent", K(stat_table)); return ret; } -/*for partitioned user table: - * 1.if table do not have global statistics: - * i.if table do not have any part level statistics, then gather whole statistics; - * ii.if table have part level statistics: - * a.if less than %50 of all part have stale statistics or not have statistics, then use - * incremental gather statistics, it's mean to just regather part level statistics which - * statistics is stale and derive global statistics according to all part level statistics; - * b.if more than %50 of all part have stale statistics or not have statistics, then gather - * whole statistics; - * 2.if table have global statistics: - * i.if global statistics is stale: - * a.if less than %50 of all part have stale statistics or not have statistics, then use - * incremental gather statistics, it's mean to just regather part level statistics which - * statistics is stale and derive global statistics according to all part level statistics; - * b.if more than %50 of all part have stale statistics or not have statistics, gather whole - * statistics; - * ii.if global statistics is not stale: - * a.regather part statistics which have stale statistics or not have statistics. then use - * incremental gather statistics, it's mean to just regather part level statistics which - * statistics is stale and derive global statistics according to all part level statistics; - */ int ObDbmsStats::get_user_partition_table_stale_percent( sql::ObExecContext &ctx, const uint64_t tenant_id, const ObTableSchema &table_schema, const double stale_percent_threshold, - const ObIArray &partition_stat_infos, - StatTable &stat_table, - bool &is_big_table) + StatTable &stat_table) { int ret = OB_SUCCESS; - is_big_table = false; uint64_t table_id = table_schema.get_table_id(); - const int64_t part_id = PARTITION_LEVEL_ZERO == table_schema.get_part_level() ? table_id : -1; + const int64_t global_part_id = PARTITION_LEVEL_ZERO == table_schema.get_part_level() ? table_id : -1; int64_t inc_modified_count = 0; int64_t row_cnt = 0; - ObSEArray no_regather_partition_ids; - int64_t no_regather_first_part_cnt = 0; ObSEArray partition_infos; - if (OB_UNLIKELY(!table_schema.is_user_table() || -1 != part_id)) { + if (OB_UNLIKELY(!table_schema.is_user_table() || -1 != global_part_id)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(table_schema.is_user_table()), K(part_id)); - } else if (partition_stat_infos.empty()) { + LOG_WARN("get unexpected error", K(ret), K(table_schema.is_user_table()), K(global_part_id)); + } else if (stat_table.partition_stat_infos_.empty()) { // do not have any statistics stat_table.stale_percent_ = -1.0; - stat_table.incremental_stat_ = false; - if (OB_FAIL(ObBasicStatsEstimator::estimate_row_count(ctx, - tenant_id, - table_id, - row_cnt))) { - LOG_WARN("failed to estimate row count"); - } else { - is_big_table = row_cnt >= OPT_STATS_BIG_TABLE_ROWS; - } } else if (OB_FAIL(get_table_partition_infos(table_schema, partition_infos))) { LOG_WARN("failed to get table subpart infos", K(ret)); - } else if (!is_table_gather_global_stats(part_id, partition_stat_infos, row_cnt)) { - //do not have global statistics, but have part level statistics - stat_table.stale_percent_ = -1.0; - if (OB_FAIL(ObBasicStatsEstimator::estimate_stale_partition(ctx, - tenant_id, - table_id, - partition_infos, - stale_percent_threshold, - partition_stat_infos, - no_regather_partition_ids, - no_regather_first_part_cnt))) { - LOG_WARN("failed to get no regather partition", K(ret)); - } else if (no_regather_first_part_cnt > table_schema.get_first_part_num() / 2) { - if (OB_FAIL(append(stat_table.no_regather_partition_ids_, no_regather_partition_ids))) { - LOG_WARN("failed to append table ids"); - } else { - stat_table.incremental_stat_ = true; - stat_table.need_gather_subpart_ = PARTITION_LEVEL_TWO == table_schema.get_part_level(); - } - } else { - stat_table.incremental_stat_ = false; - } - //have global statistics - } else if (OB_FAIL(ObBasicStatsEstimator::estimate_modified_count(ctx, - tenant_id, - table_id, - inc_modified_count))) { - LOG_WARN("failed to estimate modified count", K(ret)); - } else if (inc_modified_count < 0) { - // if some server reboot, increment modified count may less than 0. In this scenario, - // we force gather table statistics and reset modified count. - stat_table.stale_percent_ = -1.0; - stat_table.incremental_stat_ = false; - } else if (inc_modified_count == 0) { - stat_table.stale_percent_ = 0.0; - stat_table.incremental_stat_ = false; + } else if (OB_FAIL(ObBasicStatsEstimator::estimate_stale_partition(ctx, + tenant_id, + table_id, + global_part_id, + partition_infos, + stale_percent_threshold, + stat_table.partition_stat_infos_))) { + LOG_WARN("failed to get no regather partition", K(ret)); } else { - stat_table.stale_percent_ = row_cnt == 0 ? 1.0 : 1.0 * (inc_modified_count) / row_cnt; - if (stat_table.stale_percent_ > stale_percent_threshold) {//global stat is stale - stat_table.incremental_stat_ = false; - if (OB_FAIL(ObBasicStatsEstimator::estimate_stale_partition(ctx, - tenant_id, - table_id, - partition_infos, - stale_percent_threshold, - partition_stat_infos, - no_regather_partition_ids, - no_regather_first_part_cnt))) { - LOG_WARN("failed to estimate stale partition", K(ret)); - } else if (no_regather_first_part_cnt > table_schema.get_first_part_num() / 2) { - if (OB_FAIL(append(stat_table.no_regather_partition_ids_, no_regather_partition_ids))) { - LOG_WARN("failed to append table ids"); + int64_t no_regather_part_cnt = 0; + for (int64_t i = 0; OB_SUCC(ret) && stat_table.stale_percent_ == 0 && i < stat_table.partition_stat_infos_.count(); ++i) { + if (!stat_table.partition_stat_infos_.at(i).is_regather()) { + if (stat_table.partition_stat_infos_.at(i).partition_id_ == global_part_id) { + //do nothing } else { - stat_table.incremental_stat_ = true; - stat_table.need_gather_subpart_ = PARTITION_LEVEL_TWO == table_schema.get_part_level(); + ++ no_regather_part_cnt; } - } else { - stat_table.incremental_stat_ = false; + } else if (stat_table.partition_stat_infos_.at(i).partition_id_ == global_part_id) { + stat_table.stale_percent_ = -1.0; } - } else if (OB_FAIL(ObBasicStatsEstimator::estimate_stale_partition(ctx, - tenant_id, - table_id, - partition_infos, - stale_percent_threshold, - partition_stat_infos, - no_regather_partition_ids, - no_regather_first_part_cnt))) { - LOG_WARN("failed to estimate stale partition", K(ret)); - } else { + } + if (OB_SUCC(ret) && stat_table.stale_percent_ == 0) { int64_t total_part_cnt = table_schema.get_all_part_num(); if (PARTITION_LEVEL_TWO == table_schema.get_part_level()) { total_part_cnt += table_schema.get_first_part_num(); } - if (no_regather_partition_ids.count() < total_part_cnt || - is_all_partition_locked(no_regather_partition_ids, partition_stat_infos)) { - if (OB_FAIL(append(stat_table.no_regather_partition_ids_, no_regather_partition_ids))) { - LOG_WARN("failed to append table ids"); - } else { - stat_table.stale_percent_ = -1.0; - stat_table.incremental_stat_ = true; - stat_table.need_gather_subpart_ = PARTITION_LEVEL_TWO == table_schema.get_part_level(); - } - } else {//no stale partition - stat_table.stale_percent_ = 0.0; - stat_table.incremental_stat_ = false; - } + stat_table.stale_percent_ = no_regather_part_cnt < total_part_cnt ? -1.0 : 0; } } - LOG_TRACE("succeed to get user partition table stale percent", - K(stat_table), K(partition_stat_infos), K(is_big_table)); + LOG_TRACE("succeed to get user partition table stale percent", K(stat_table)); return ret; } @@ -5654,54 +5659,55 @@ int ObDbmsStats::gather_table_stats_with_default_param(ObExecContext &ctx, gather_stat.set_table_id(stat_table.table_id_); ObOptStatGatherStatList::instance().push(gather_stat); ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), ObTimeUtility::current_time(), stat_param.allocator_->used(), gather_stat); - if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(task_info.task_start_time_, - duration_time, - stat_param.duration_time_))) { + if (OB_FAIL(running_monitor.add_monitor_info(ObOptStatRunningPhase::GATHER_PREPARE))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(task_info.task_start_time_, + duration_time, + stat_param.duration_time_))) { LOG_WARN("failed to get valid duration time", K(ret)); } else if (OB_FAIL(parse_table_part_info(ctx, stat_table, stat_param, true))) { LOG_WARN("failed to parse owner", K(ret)); } else if (OB_FAIL(use_default_gather_stat_options(ctx, stat_table, stat_param))) { LOG_WARN("failed to use default gather stat optitions", K(ret)); - } else if (stat_table.need_gather_subpart_) { - stat_param.subpart_stat_param_.set_gather_stat(); - } - if (OB_SUCC(ret)) { - if (OB_FAIL(running_monitor.add_table_info(stat_param, stat_table.stale_percent_))) { - LOG_WARN("failed to add table info", K(ret)); - } else if (OB_FAIL(ObDbmsStatsLockUnlock::adjust_table_stat_param(stat_table.no_regather_partition_ids_, - stat_param))) { - LOG_WARN("failed to adjust table stat param", K(ret)); - } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param))) { - LOG_WARN("failed to gather table stats", K(ret)); - } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), - stat_param, - &running_monitor))) { - LOG_WARN("failed to update stat cache", K(ret)); - //refresh duration time - } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(task_info.task_start_time_, - duration_time, - stat_param.duration_time_))) { - LOG_WARN("failed to get valid duration time", K(ret)); - } else if (!need_gather_index_stats(stat_param)) { - LOG_TRACE("Succeed to gather table stats", K(stat_param)); - } else if (stat_param.cascade_ && - OB_FAIL(fast_gather_index_stats(ctx, stat_param, - is_all_fast_gather, no_gather_index_ids))) { - LOG_WARN("failed to fast gather index stats", K(ret)); - //refresh duration time - } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(task_info.task_start_time_, - duration_time, - stat_param.duration_time_))) { - LOG_WARN("failed to get valid duration time", K(ret)); - } else if (stat_param.cascade_ && !is_all_fast_gather && - OB_FAIL(gather_table_index_stats(ctx, stat_param, no_gather_index_ids))) { - LOG_WARN("failed to gather table index stats", K(ret)); - } else { - LOG_TRACE("Succeed to gather table stats", K(stat_param)); - } + } else if (OB_FAIL(adjust_auto_gather_stat_option(stat_table.partition_stat_infos_, stat_param))) { + LOG_WARN("failed to use default gather stat optitions", K(ret)); + } else if (OB_FAIL(running_monitor.add_table_info(stat_param, stat_table.stale_percent_))) { + LOG_WARN("failed to add table info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, stat_param, running_monitor))) { + LOG_WARN("failed to gather table stats", K(ret)); + } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), + stat_param, + &running_monitor))) { + LOG_WARN("failed to update stat cache", K(ret)); + //refresh duration time + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(task_info.task_start_time_, + duration_time, + stat_param.duration_time_))) { + LOG_WARN("failed to get valid duration time", K(ret)); + } else if (!need_gather_index_stats(stat_param)) { + LOG_TRACE("Succeed to gather table stats", K(stat_param)); + } else if (stat_param.cascade_ && + OB_FAIL(fast_gather_index_stats(ctx, stat_param, + is_all_fast_gather, no_gather_index_ids))) { + LOG_WARN("failed to fast gather index stats", K(ret)); + //refresh duration time + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(task_info.task_start_time_, + duration_time, + stat_param.duration_time_))) { + LOG_WARN("failed to get valid duration time", K(ret)); + } else if (OB_FAIL(running_monitor.add_monitor_info(ObOptStatRunningPhase::GATHER_INDEX_STATS))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (stat_param.cascade_ && !is_all_fast_gather && + OB_FAIL(gather_table_index_stats(ctx, stat_param, no_gather_index_ids))) { + LOG_WARN("failed to gather table index stats", K(ret)); + } else { + LOG_TRACE("Succeed to gather table stats", K(stat_param)); } running_monitor.set_monitor_result(ret, ObTimeUtility::current_time(), stat_param.allocator_->used()); + sql::ObSQLSessionInfo *origin_session = THIS_WORKER.get_session(); + THIS_WORKER.set_session(NULL); ObOptStatManager::get_instance().update_opt_stat_gather_stat(gather_stat); + THIS_WORKER.set_session(origin_session); ObOptStatGatherStatList::instance().remove(gather_stat); task_info.completed_table_count_ ++; return ret; @@ -5901,26 +5907,6 @@ bool ObDbmsStats::is_table_gather_global_stats(const int64_t global_id, return is_gather; } -bool ObDbmsStats::is_all_partition_locked(const ObIArray &partition_ids, - const ObIArray &partition_stat_infos) -{ - bool is_all_locked = !partition_ids.empty(); - for (int64_t i = 0; is_all_locked && i < partition_ids.count(); ++i) { - is_all_locked = !partition_stat_infos.empty(); - bool find_it = false; - for (int64_t j = 0; is_all_locked && !find_it && j < partition_stat_infos.count(); ++j) { - if (partition_ids.at(i) == partition_stat_infos.at(j).partition_id_) { - is_all_locked = partition_stat_infos.at(j).is_stat_locked_; - find_it = true; - } else {/*do nothing*/} - } - if (!find_it) {//not found - is_all_locked = false; - } - } - return is_all_locked; -} - int ObDbmsStats::get_table_index_infos(sql::ObExecContext &ctx, const int64_t table_id, ObIArray &index_infos) @@ -6048,13 +6034,9 @@ int ObDbmsStats::get_table_partition_infos(const ObTableSchema &table_schema, int ret = OB_SUCCESS; ObSEArray part_infos; ObSEArray subpart_infos; - ObSEArray part_ids; - ObSEArray subpart_ids; if (OB_FAIL(get_table_part_infos(&table_schema, part_infos, - subpart_infos, - part_ids, - subpart_ids))) { + subpart_infos))) { LOG_WARN("failed to get table part infos", K(ret)); } else if (PARTITION_LEVEL_ONE == table_schema.get_part_level() && OB_FAIL(partition_infos.assign(part_infos))) { @@ -6075,13 +6057,9 @@ int ObDbmsStats::get_table_partition_map(const ObTableSchema &table_schema, } else { ObSEArray part_infos; ObSEArray subpart_infos; - ObSEArray part_ids; - ObSEArray subpart_ids; if (OB_FAIL(get_table_part_infos(&table_schema, part_infos, subpart_infos, - part_ids, - subpart_ids, &part_map))) { LOG_WARN("failed to get table part infos", K(ret)); } @@ -6129,8 +6107,8 @@ int ObDbmsStats::resovle_granularity(ObGranularityType granu_type, { int ret = OB_SUCCESS; if (ObGranularityType::GRANULARITY_AUTO == granu_type) { - param.global_stat_param_.set_gather_stat(false); - param.part_stat_param_.set_gather_stat(); + param.global_stat_param_.set_gather_stat(param.part_name_.empty()); + param.part_stat_param_.set_gather_stat(param.part_name_.empty() && param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO); param.subpart_stat_param_.set_gather_stat(); // refine auto granularity based on subpart type if (ObPartitionLevel::PARTITION_LEVEL_TWO == param.part_level_ && @@ -6146,7 +6124,7 @@ int ObDbmsStats::resovle_granularity(ObGranularityType granu_type, param.part_stat_param_.set_gather_stat(); param.subpart_stat_param_.reset_gather_stat(); } else if (ObGranularityType::GRANULARITY_APPROX_GLOBAL_AND_PARTITION == granu_type) { - bool gather_approx = param.part_level_ != ObPartitionLevel::PARTITION_LEVEL_ZERO; + bool gather_approx = param.part_level_ != ObPartitionLevel::PARTITION_LEVEL_ZERO && !param.is_subpart_name_; param.global_stat_param_.set_gather_stat(gather_approx); param.part_stat_param_.set_gather_stat(); param.subpart_stat_param_.reset_gather_stat(); @@ -6155,12 +6133,24 @@ int ObDbmsStats::resovle_granularity(ObGranularityType granu_type, param.part_stat_param_.reset_gather_stat(); param.subpart_stat_param_.reset_gather_stat(); } else if (ObGranularityType::GRANULARITY_PARTITION == granu_type) { - param.global_stat_param_.reset_gather_stat(); + if (param.part_name_.empty() && param.part_level_ != share::schema::ObPartitionLevel::PARTITION_LEVEL_ZERO) { + param.global_stat_param_.set_gather_stat(true); + } else { + param.global_stat_param_.reset_gather_stat(); + } param.part_stat_param_.set_gather_stat(); param.subpart_stat_param_.reset_gather_stat(); } else if (ObGranularityType::GRANULARITY_SUBPARTITION == granu_type) { param.global_stat_param_.reset_gather_stat(); param.part_stat_param_.reset_gather_stat(); + if (param.part_name_.empty() && param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { + param.global_stat_param_.set_gather_stat(true); + param.part_stat_param_.set_gather_stat(true); + } else if (!param.part_name_.empty() && + !param.is_subpart_name_ && + param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { + param.part_stat_param_.set_gather_stat(true); + } param.subpart_stat_param_.set_gather_stat(); } else { ret = OB_ERR_UNEXPECTED; @@ -6220,9 +6210,7 @@ int ObDbmsStats::init_gather_task_info(ObExecContext &ctx, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(ctx.get_my_session())); } else if (OB_FAIL(task_info.init(ctx.get_allocator(), - ctx.get_my_session()->get_effective_tenant_id(), - ctx.get_my_session()->get_sessid(), - ctx.get_my_session()->get_current_trace_id(), + ctx.get_my_session(), task_id, type, start_time, @@ -6235,6 +6223,10 @@ int ObDbmsStats::init_gather_task_info(ObExecContext &ctx, return ret; } +//} else if (OB_FAIL(schema->has_all_column_group(has_all_column_group))) { +// LOG_WARN("failed to check has row store", K(ret)); +// } else if (OB_FALSE_IT(has_normal_column_group = schema->is_normal_column_store_table())) { + int ObDbmsStats::init_column_group_stat_param(const share::schema::ObTableSchema &table_schema, ObIArray &column_group_params) { @@ -6504,6 +6496,230 @@ int ObDbmsStats::check_system_stat_table_ready(int64_t tenant_id) } return ret; } +/** + * @brief ObDbmsStats::copy_table_stats + * @param ctx + * @param params + * 0. ownname VARCHAR2, + * 1. tabname VARCHAR2, + * 2. srcpartname VARCHAR2, + * 3. dstpartname VARCHAR2, + * 4. scale_factor VARCHAR2, + * 5. flags DECIMAL DEFAULT NULL, + * 6. force BOOLEAN DEFAULT FALSE + * @param result + * @return int + */ +int ObDbmsStats::copy_table_stats(sql::ObExecContext &ctx, + sql::ParamStore ¶ms, + common::ObObj &result) +{ + int ret = OB_SUCCESS; + UNUSED(result); + CopyTableStatHelper copy_stat_helper(&ctx.get_allocator()); + const ObTableSchema *table_schema = NULL; + ObTableStatParam table_stat_param; + table_stat_param.allocator_ = &ctx.get_allocator(); + ObCopyLevel copy_level; + bool is_found = false; + ObObjParam dummy_part_name; + dummy_part_name.set_null(); + if (OB_FAIL(check_statistic_table_writeable(ctx))) { + LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_2_0) { + //do nothing + } else if (OB_FAIL(parse_table_part_info(ctx, + params.at(0), + params.at(1), + dummy_part_name, + table_stat_param, + table_schema))) { + LOG_WARN("failed to parse table part info", K(ret)); + } else if (OB_FAIL(extract_copy_stat_helper(params, ctx, table_schema, copy_stat_helper))) { + LOG_WARN("failed to extract copy table stats params", K(ret)); + } else if (OB_FAIL(ObDbmsStatsCopyTableStats::check_parts_valid(ctx, copy_stat_helper, table_stat_param, copy_level))) { + LOG_WARN("failed to check part valid", K(ret)); + } else if (params.at(4).is_null()) { + //scale_factor is null, do nothing + } else if (OB_FAIL(parse_partition_name(ctx, + table_schema, + params.at(3), + table_stat_param))) { + LOG_WARN("failed to parse partition name", K(ret)); + } else if (ObCharset::case_insensitive_equal(copy_stat_helper.srcpart_name_, + copy_stat_helper.dstpart_name_)) { + LOG_TRACE("src part and dst part is the same, no need to copy"); + } else if (OB_FAIL(ObDbmsStatsCopyTableStats::extract_partition_column_ids(copy_stat_helper, table_schema))) { + LOG_WARN("failed to classify partition column ids", K(ret)); + } else if (OB_FAIL(ObDbmsStatsCopyTableStats::get_dst_part_infos(table_stat_param, + copy_stat_helper, + table_schema, + copy_level, + is_found))) { + LOG_WARN("failed to get dst part infos", K(ret)); + } else if (OB_UNLIKELY(!is_found)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("failed to find dst part info", K(ret), K(copy_stat_helper), K(copy_level)); + } else if (!copy_stat_helper.force_copy_ && + OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, table_stat_param))) { + LOG_WARN("failed check stat locked", K(ret)); + } else if (OB_FAIL(parse_partition_name(ctx, + table_schema, + params.at(2), + table_stat_param))) { + LOG_WARN("failed to parse partition name", K(ret)); + } else if (OB_FAIL(ObDbmsStatsCopyTableStats::copy_tab_col_stats(ctx, table_stat_param, copy_stat_helper))) { + LOG_WARN("failed to copy table stats and column stats", K(ret)); + } else if (OB_FAIL(update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), + table_stat_param))) { + LOG_WARN("failed to update stat cache", K(ret)); + } + LOG_TRACE("succeed to copy table stat", K(copy_stat_helper)); + return ret; +} + +int ObDbmsStats::extract_copy_stat_helper(sql::ParamStore ¶ms, + sql::ObExecContext &ctx, + const share::schema::ObTableSchema *table_schema, + CopyTableStatHelper ©_stat_helper) +{ + int ret = OB_SUCCESS; + number::ObNumber scale_factor_num; + number::ObNumber flags_num; + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!params.at(0).is_null() && OB_FAIL(params.at(0).get_string(copy_stat_helper.owner_))) { + LOG_WARN("failed to get string", K(ret), K(params.at(0))); + } else if (!params.at(0).is_null() && OB_FAIL(convert_vaild_ident_name(ctx.get_allocator(), + ctx.get_my_session()->get_dtc_params(), + copy_stat_helper.owner_, + lib::is_oracle_mode()))) { + LOG_WARN("failed to extract table name", K(ret)); + } else if (!params.at(1).is_null() && OB_FAIL(params.at(1).get_string(copy_stat_helper.table_name_))) { + LOG_WARN("failed to get string", K(ret), K(params.at(1))); + } else if (!params.at(1).is_null() && OB_FAIL(convert_vaild_ident_name(ctx.get_allocator(), + ctx.get_my_session()->get_dtc_params(), + copy_stat_helper.table_name_, + lib::is_oracle_mode()))) { + LOG_WARN("failed to extract table name", K(ret)); + } else if (!params.at(2).is_null() && OB_FAIL(params.at(2).get_string(copy_stat_helper.srcpart_name_))) { + LOG_WARN("failed to get string", K(ret), K(params.at(2))); + } else if (!params.at(2).is_null() + && OB_FAIL(convert_vaild_ident_name(ctx.get_allocator(), + ctx.get_my_session()->get_dtc_params(), + copy_stat_helper.srcpart_name_, + lib::is_oracle_mode()))) { + LOG_WARN("failed to extract srcpart name", K(ret)); + } else if (!params.at(3).is_null() && OB_FAIL(params.at(3).get_string(copy_stat_helper.dstpart_name_))) { + LOG_WARN("failed to get string", K(ret), K(params.at(3))); + } else if (!params.at(3).is_null() + && OB_FAIL(convert_vaild_ident_name(ctx.get_allocator(), + ctx.get_my_session()->get_dtc_params(), + copy_stat_helper.dstpart_name_, + lib::is_oracle_mode()))) { + LOG_WARN("failed to extract dstpart name", K(ret)); + } else if (!params.at(4).is_null() && OB_FAIL(params.at(4).get_number(scale_factor_num))) { + LOG_WARN("failed to extract scale factor", K(ret)); + } else if (!params.at(5).is_null() && OB_FAIL(params.at(5).get_number(flags_num))) { + LOG_WARN("failed to extract flags", K(ret)); + } else if (!params.at(6).is_null() && OB_FAIL(params.at(6).get_bool(copy_stat_helper.force_copy_))) { + LOG_WARN("failed to extract force", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::cast_number_to_double(scale_factor_num, copy_stat_helper.scale_factor_))) { + LOG_WARN("failed to cast number to double" , K(ret), K(scale_factor_num)); + } else if (copy_stat_helper.scale_factor_ < 0) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("scale factor is less than 0", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"scale factor is less than 0"); + } else if (!params.at(4).is_null() && + OB_FAIL(ObDbmsStatsUtils::cast_number_to_double(scale_factor_num, copy_stat_helper.scale_factor_))) { + LOG_WARN("failed to cast number to double" , K(ret), K(scale_factor_num)); + } else { + copy_stat_helper.table_id_ = table_schema->get_table_id(); + copy_stat_helper.tenant_id_ = ctx.get_my_session()->get_effective_tenant_id(); + } + return ret; +} + +int ObDbmsStats::adjust_auto_gather_stat_option(const ObIArray &partition_stat_infos, + ObTableStatParam ¶m) +{ + int ret = OB_SUCCESS; + bool has_part_locked = false; + if (param.subpart_stat_param_.need_modify_) { + ObSEArray new_subpart_infos; + for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { + bool is_locked = false; + if (is_partition_no_regather(param.subpart_infos_.at(i).part_id_, partition_stat_infos, is_locked)) { + if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.subpart_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else { + has_part_locked |= is_locked; + } + } else if (OB_FAIL(new_subpart_infos.push_back(param.subpart_infos_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_FAIL(param.subpart_infos_.assign(new_subpart_infos))) { + LOG_WARN("failed to assign", K(ret)); + } else { + param.subpart_stat_param_.need_modify_ = !new_subpart_infos.empty(); + } + } + if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) { + ObSEArray new_part_infos; + for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { + bool is_locked = false; + if (is_partition_no_regather(param.part_infos_.at(i).part_id_, partition_stat_infos, is_locked)) { + if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.part_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else { + has_part_locked |= is_locked; + } + } else if (param.part_stat_param_.can_use_approx_ && + param.subpart_stat_param_.need_modify_ && + param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { + if (OB_FAIL(param.approx_part_infos_.push_back(param.part_infos_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } else if (OB_FAIL(new_part_infos.push_back(param.part_infos_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(param.part_infos_.assign(new_part_infos))) { + LOG_WARN("failed to assign", K(ret)); + } else { + param.part_stat_param_.can_use_approx_ = !param.approx_part_infos_.empty(); + param.part_stat_param_.need_modify_ = !new_part_infos.empty() || !param.approx_part_infos_.empty(); + } + } + } + if (OB_SUCC(ret) && + param.global_stat_param_.need_modify_ && + param.global_stat_param_.gather_approx_ && + (has_part_locked || !param.part_stat_param_.need_modify_)) { + param.global_stat_param_.gather_approx_ = false; + } + LOG_TRACE("succeed to adjust auto gather stat option", K(partition_stat_infos), K(param)); + return ret; +} + +bool ObDbmsStats::is_partition_no_regather(int64_t part_id, + const ObIArray &partition_stat_infos, + bool &is_locked) +{ + bool is_true = false; + bool found_it = false; + for (int64_t i = 0; !found_it && i < partition_stat_infos.count(); ++i) { + if (part_id == partition_stat_infos.at(i).partition_id_) { + is_true = !partition_stat_infos.at(i).is_regather(); + is_locked = partition_stat_infos.at(i).is_stat_locked_; + found_it = true; + } + } + return is_true; +} } } diff --git a/src/pl/sys_package/ob_dbms_stats.h b/src/pl/sys_package/ob_dbms_stats.h index 302250559e..acad1044ac 100644 --- a/src/pl/sys_package/ob_dbms_stats.h +++ b/src/pl/sys_package/ob_dbms_stats.h @@ -19,6 +19,7 @@ #include "pl/ob_pl_type.h" #include "share/stat/ob_dbms_stats_preferences.h" #include "share/stat/ob_opt_stat_gather_stat.h" +#include "share/stat/ob_dbms_stats_copy_table_stats.h" namespace oceanbase { @@ -237,6 +238,14 @@ public: sql::ParamStore ¶ms, common::ObObj &result); + static int copy_table_stats(sql::ObExecContext &ctx, + sql::ParamStore ¶ms, + common::ObObj &result); + + static int cancel_gather_stats(sql::ObExecContext &ctx, + sql::ParamStore ¶ms, + common::ObObj &result); + static int parse_method_opt(sql::ObExecContext &ctx, ObIAllocator *allocator, ObIArray &column_params, @@ -282,6 +291,14 @@ public: ObTableStatParam ¶m, bool need_parse_col_group = false); + static int parse_table_part_info(ObExecContext &ctx, + const ObObjParam &owner, + const ObObjParam &tab_name, + const ObObjParam &part_name, + ObTableStatParam ¶m, + const share::schema::ObTableSchema *&table_schema, + bool need_parse_col_group = false); + static int parse_table_part_info(ObExecContext &ctx, const StatTable stat_table, ObTableStatParam ¶m, @@ -368,8 +385,6 @@ public: static int get_table_part_infos(const share::schema::ObTableSchema *table_schema, common::ObIArray &part_infos, common::ObIArray &subpart_infos, - common::ObIArray &part_ids, - common::ObIArray &subpart_ids, OSGPartMap *part_map = NULL); static int get_part_ids_from_schema(const share::schema::ObTableSchema *table_schema, @@ -489,6 +504,10 @@ public: const uint64_t tenant_id, const uint64_t table_id, double &stale_percent_threshold); + static int extract_copy_stat_helper(sql::ParamStore ¶ms, + sql::ObExecContext &ctx, + const share::schema::ObTableSchema *table_schema, + CopyTableStatHelper ©_stat_helper); static int init_column_group_stat_param(const share::schema::ObTableSchema &table_schema, ObIArray &column_group_params); @@ -540,25 +559,18 @@ private: static int get_common_table_stale_percent(sql::ObExecContext &ctx, const uint64_t tenant_id, const share::schema::ObTableSchema &table_schema, - const ObIArray &partition_stat_infos, - StatTable &stat_table, - bool &is_big_table); + StatTable &stat_table); static int get_user_partition_table_stale_percent(sql::ObExecContext &ctx, const uint64_t tenant_id, const share::schema::ObTableSchema &table_schema, const double stale_percent_threshold, - const ObIArray &partition_stat_infos, - StatTable &stat_table, - bool &is_big_table); + StatTable &stat_table); static bool is_table_gather_global_stats(const int64_t global_id, const ObIArray &partition_stat_infos, int64_t &cur_row_cnt); - static bool is_all_partition_locked(const ObIArray &partition_ids, - const ObIArray &partition_stat_infos); - static int parse_index_part_info(ObExecContext &ctx, const ObObjParam &owner, const ObObjParam &index_name, @@ -591,8 +603,18 @@ private: static void decide_modified_part(ObTableStatParam ¶m, const bool cascade_parts); + static int parse_degree_option(ObExecContext &ctx, const ObObjParam °ree, + ObTableStatParam &stat_param); + static int refresh_tenant_schema_guard(ObExecContext &ctx, const uint64_t tenant_id); + static int adjust_auto_gather_stat_option(const ObIArray &partition_stat_infos, + ObTableStatParam ¶m); + + static bool is_partition_no_regather(int64_t part_id, + const ObIArray &partition_stat_infos, + bool &is_locked); + static int check_system_stats_name_valid(const ObString& name, bool &is_valid); static int check_modify_system_stats_pri(const ObSQLSessionInfo& session); diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp index edceaaf733..692dfe7d01 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp @@ -1862,7 +1862,7 @@ int ObDDLRedefinitionTask::sync_part_stats_info_accross_tenant(common::ObMySQLTr } if (FAILEDx(stat_svr.update_table_stat(dst_tenant_id_, - trans, + trans.get_connection(), target_part_stats, ObTimeUtility::current_time(), new_table_schema.is_index_table()))) { @@ -1949,7 +1949,7 @@ int ObDDLRedefinitionTask::sync_column_stats_info_accross_tenant(common::ObMySQL } else if (OB_FAIL(stat_svr.update_column_stat(dst_tenant_schema_guard, dst_tenant_id_, allocator, - trans, + trans.get_connection(), target_column_stats, ObTimeUtility::current_time(), false /* need update histogram table */))) { diff --git a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp index d11049de4a..1dfb48b78c 100755 --- a/src/rootserver/ddl_task/ob_table_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_table_redefinition_task.cpp @@ -844,7 +844,9 @@ int ObTableRedefinitionTask::take_effect(const ObDDLTaskStatus next_task_status) } else { LOG_WARN("sync auto increment position failed", K(ret), K(object_id_), K(target_object_id_)); } - } else if (OB_FAIL(sync_stats_info())) { + } else if (ObDDLType::DDL_DIRECT_LOAD != task_type_ && + ObDDLType::DDL_DIRECT_LOAD_INSERT != task_type_ && + OB_FAIL(sync_stats_info())) {//direct load no need sync stats info, because the stats have been regather LOG_WARN("fail to sync stats info", K(ret), K(object_id_), K(target_object_id_)); } else if (OB_FAIL(ObDDLUtil::get_ddl_rpc_timeout(dst_tenant_id_, target_object_id_, ddl_rpc_timeout))) { LOG_WARN("get ddl rpc timeout fail", K(ret)); diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index 95a55396e8..3a4610ce80 100644 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -454,6 +454,7 @@ ob_set_subtarget(ob_share schema ob_set_subtarget(ob_share stat stat/ob_basic_stats_estimator.cpp stat/ob_opt_stat_monitor_manager.cpp + stat/ob_dbms_stats_copy_table_stats.cpp stat/ob_dbms_stats_executor.cpp stat/ob_dbms_stats_export_import.cpp stat/ob_dbms_stats_history_manager.cpp @@ -479,6 +480,7 @@ ob_set_subtarget(ob_share stat stat/ob_opt_stat_gather_stat.cpp stat/ob_opt_ds_stat.cpp stat/ob_opt_ds_stat_cache.cpp + stat/ob_dbms_stats_gather.cpp stat/ob_opt_system_stat.cpp stat/ob_opt_system_stat_cache.cpp ) diff --git a/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp b/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp index 73c3949c71..dfbafea6e1 100644 --- a/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21351_21400.cpp @@ -1160,7 +1160,7 @@ int ObInnerTableSchema::gv_ob_opt_stat_gather_monitor_schema(ObTableSchema &tabl table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(SVR_IP AS CHAR(46)) AS SVR_IP, CAST(SVR_PORT AS SIGNED) AS SVR_PORT, CAST(SESSION_ID AS SIGNED) AS SESSION_ID, CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN TYPE = 0 THEN 'MANUAL GATHER' ELSE (CASE WHEN TYPE = 1 THEN 'AUTO GATHER' ELSE 'UNDEFINED GATHER' END) END) AS CHAR(16)) AS TYPE, CAST(TASK_START_TIME AS DATETIME(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS SIGNED) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS SIGNED) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS SIGNED) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS CHAR(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS CHAR(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS SIGNED) AS RUNNING_TABLE_DURATION_TIME FROM oceanbase.__all_virtual_opt_stat_gather_monitor )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(SVR_IP AS CHAR(46)) AS SVR_IP, CAST(SVR_PORT AS SIGNED) AS SVR_PORT, CAST(SESSION_ID AS SIGNED) AS SESSION_ID, CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN TYPE = 0 THEN 'MANUAL GATHER' ELSE (CASE WHEN TYPE = 1 THEN 'AUTO GATHER' ELSE 'UNDEFINED GATHER' END) END) AS CHAR(16)) AS TYPE, CAST(TASK_START_TIME AS DATETIME(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS SIGNED) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS SIGNED) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS SIGNED) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS CHAR(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS CHAR(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS SIGNED) AS RUNNING_TABLE_DURATION_TIME, CAST(SPARE2 AS CHAR(256)) AS RUNNING_TABLE_PROGRESS FROM oceanbase.__all_virtual_opt_stat_gather_monitor )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1260,7 +1260,7 @@ int ObInnerTableSchema::dba_ob_task_opt_stat_gather_history_schema(ObTableSchema table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS CHAR(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE 'FAILED' END) END) AS CHAR(8)) AS STATUS, CAST(TABLE_COUNT AS SIGNED) AS TABLE_COUNT, CAST(FAILED_COUNT AS SIGNED) AS FAILED_COUNT, CAST(START_TIME AS DATETIME(6)) AS START_TIME, CAST(END_TIME AS DATETIME(6)) AS END_TIME FROM oceanbase.__all_virtual_task_opt_stat_gather_history WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS SIGNED) AS TENANT_ID, CAST(TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS CHAR(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS CHAR(8)) AS STATUS, CAST(TABLE_COUNT AS SIGNED) AS TABLE_COUNT, CAST(FAILED_COUNT AS SIGNED) AS FAILED_COUNT, CAST(START_TIME AS DATETIME(6)) AS START_TIME, CAST(END_TIME AS DATETIME(6)) AS END_TIME FROM oceanbase.__all_virtual_task_opt_stat_gather_history WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1310,7 +1310,7 @@ int ObInnerTableSchema::dba_ob_table_opt_stat_gather_history_schema(ObTableSchem table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(DB.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(V.TABLE_NAME AS CHAR(256)) AS TABLE_NAME, CAST(STAT.TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE 'FAILED' END) END) AS CHAR(8)) AS STATUS, CAST(STAT.START_TIME AS DATETIME(6)) AS START_TIME, CAST(STAT.END_TIME AS DATETIME(6)) AS END_TIME, CAST(STAT.MEMORY_USED AS SIGNED) AS MEMORY_USED, CAST(STAT.STAT_REFRESH_FAILED_LIST AS CHAR(4096)) AS STAT_REFRESH_FAILED_LIST, CAST(STAT.PROPERTIES AS CHAR(4096)) AS PROPERTIES FROM ( (SELECT CAST(0 AS SIGNED) AS TENANT_ID, DATABASE_ID, TABLE_ID, TABLE_NAME FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID, DATABASE_ID, TABLE_ID, TABLE_NAME FROM oceanbase.__all_table T WHERE T.TABLE_TYPE IN (0,2,3,6)) ) V JOIN oceanbase.__all_database DB ON DB.TENANT_ID = V.TENANT_ID AND DB.DATABASE_ID = V.DATABASE_ID AND V.TENANT_ID = 0 LEFT JOIN oceanbase.__all_virtual_table_opt_stat_gather_history STAT ON STAT.TENANT_ID = EFFECTIVE_TENANT_ID() AND V.TABLE_ID = STAT.TABLE_ID )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(DB.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(V.TABLE_NAME AS CHAR(256)) AS TABLE_NAME, CAST(STAT.TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS CHAR(8)) AS STATUS, CAST(STAT.START_TIME AS DATETIME(6)) AS START_TIME, CAST(STAT.END_TIME AS DATETIME(6)) AS END_TIME, CAST(STAT.MEMORY_USED AS SIGNED) AS MEMORY_USED, CAST(STAT.STAT_REFRESH_FAILED_LIST AS CHAR(4096)) AS STAT_REFRESH_FAILED_LIST, CAST(STAT.PROPERTIES AS CHAR(4096)) AS PROPERTIES FROM ( (SELECT CAST(0 AS SIGNED) AS TENANT_ID, DATABASE_ID, TABLE_ID, TABLE_NAME FROM OCEANBASE.__ALL_VIRTUAL_CORE_ALL_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL SELECT TENANT_ID, DATABASE_ID, TABLE_ID, TABLE_NAME FROM oceanbase.__all_table T WHERE T.TABLE_TYPE IN (0,2,3,6)) ) V JOIN oceanbase.__all_database DB ON DB.TENANT_ID = V.TENANT_ID AND DB.DATABASE_ID = V.DATABASE_ID AND V.TENANT_ID = 0 LEFT JOIN oceanbase.__all_virtual_table_opt_stat_gather_history STAT ON STAT.TENANT_ID = EFFECTIVE_TENANT_ID() AND V.TABLE_ID = STAT.TABLE_ID )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp b/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp index 7f0cfd8005..79ba64edd0 100644 --- a/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25201_25250.cpp @@ -1310,7 +1310,7 @@ int ObInnerTableSchema::dba_ob_task_opt_stat_gather_history_ora_schema(ObTableSc table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS VARCHAR2(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE 'FAILED' END) END) AS VARCHAR2(8)) AS STATUS, CAST(TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(FAILED_COUNT AS NUMBER) AS FAILED_COUNT, CAST(START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(END_TIME AS TIMESTAMP(6)) AS TASK_END_TIME FROM SYS.ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN type = 0 THEN 'MANUAL GATHER' ELSE ( CASE WHEN type = 1 THEN 'AUTO GATHER' ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS VARCHAR2(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS VARCHAR2(8)) AS STATUS, CAST(TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(FAILED_COUNT AS NUMBER) AS FAILED_COUNT, CAST(START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(END_TIME AS TIMESTAMP(6)) AS TASK_END_TIME FROM SYS.ALL_VIRTUAL_TASK_OPT_STAT_GATHER_HISTORY WHERE TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1360,7 +1360,7 @@ int ObInnerTableSchema::dba_ob_table_opt_stat_gather_history_ora_schema(ObTableS table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(DB.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(V.TABLE_NAME AS VARCHAR2(256)) AS TABLE_NAME, CAST(STAT.TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE 'FAILED' END) END) AS VARCHAR2(8)) AS STATUS, CAST(STAT.START_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(STAT.END_TIME AS TIMESTAMP(6)) AS END_TIME, CAST(STAT.MEMORY_USED AS NUMBER) AS MEMORY_USED, CAST(STAT.STAT_REFRESH_FAILED_LIST AS VARCHAR2(4096)) AS STAT_REFRESH_FAILED_LIST, CAST(STAT.PROPERTIES AS VARCHAR2(4096)) AS PROPERTIES FROM ( (SELECT TENANT_ID, DATABASE_ID, TABLE_ID, TABLE_ID AS PARTITION_ID, TABLE_NAME, NULL AS PARTITION_NAME, NULL AS SUBPARTITION_NAME, NULL AS PARTITION_POSITION, NULL AS SUBPARTITION_POSITION, 'TABLE' AS OBJECT_TYPE FROM SYS.ALL_VIRTUAL_CORE_ALL_TABLE UNION ALL SELECT TENANT_ID, DATABASE_ID, TABLE_ID, CASE WHEN PART_LEVEL = 0 THEN TABLE_ID ELSE -1 END AS PARTITION_ID, TABLE_NAME, NULL AS PARTITION_NAME, NULL AS SUBPARTITION_NAME, NULL AS PARTITION_POSITION, NULL AS SUBPARTITION_POSITION, 'TABLE' AS OBJECT_TYPE FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T WHERE T.TABLE_TYPE IN (0,2,3,8,9)) ) V JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT db ON db.tenant_id = V.tenant_id AND db.database_id = V.database_id AND V.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY STAT ON V.TENANT_ID = STAT.TENANT_ID AND V.TABLE_ID = STAT.TABLE_ID AND STAT.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(DB.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(V.TABLE_NAME AS VARCHAR2(256)) AS TABLE_NAME, CAST(STAT.TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS VARCHAR2(8)) AS STATUS, CAST(STAT.START_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(STAT.END_TIME AS TIMESTAMP(6)) AS END_TIME, CAST(STAT.MEMORY_USED AS NUMBER) AS MEMORY_USED, CAST(STAT.STAT_REFRESH_FAILED_LIST AS VARCHAR2(4096)) AS STAT_REFRESH_FAILED_LIST, CAST(STAT.PROPERTIES AS VARCHAR2(4096)) AS PROPERTIES FROM ( (SELECT TENANT_ID, DATABASE_ID, TABLE_ID, TABLE_ID AS PARTITION_ID, TABLE_NAME, NULL AS PARTITION_NAME, NULL AS SUBPARTITION_NAME, NULL AS PARTITION_POSITION, NULL AS SUBPARTITION_POSITION, 'TABLE' AS OBJECT_TYPE FROM SYS.ALL_VIRTUAL_CORE_ALL_TABLE UNION ALL SELECT TENANT_ID, DATABASE_ID, TABLE_ID, CASE WHEN PART_LEVEL = 0 THEN TABLE_ID ELSE -1 END AS PARTITION_ID, TABLE_NAME, NULL AS PARTITION_NAME, NULL AS SUBPARTITION_NAME, NULL AS PARTITION_POSITION, NULL AS SUBPARTITION_POSITION, 'TABLE' AS OBJECT_TYPE FROM SYS.ALL_VIRTUAL_TABLE_REAL_AGENT T WHERE T.TABLE_TYPE IN (0,2,3,8,9)) ) V JOIN SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT db ON db.tenant_id = V.tenant_id AND db.database_id = V.database_id AND V.TENANT_ID = EFFECTIVE_TENANT_ID() AND DB.TENANT_ID = EFFECTIVE_TENANT_ID() LEFT JOIN SYS.ALL_VIRTUAL_TABLE_OPT_STAT_GATHER_HISTORY STAT ON V.TENANT_ID = STAT.TENANT_ID AND V.TABLE_ID = STAT.TABLE_ID AND STAT.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp index 1ae0359f79..5cd39e1b39 100644 --- a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp @@ -910,7 +910,7 @@ int ObInnerTableSchema::gv_ob_opt_stat_gather_monitor_ora_schema(ObTableSchema & table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(SVR_IP AS VARCHAR2(46)) AS SVR_IP, CAST(SVR_PORT AS NUMBER) AS SVR_PORT, CAST(SESSION_ID AS NUMBER) AS SESSION_ID, CAST(TRACE_ID AS VARCHAR2(64)) AS TRACE_ID, CAST(TASK_ID AS VARCHAR(36)) AS TASK_ID, CAST(DECODE(TYPE, 0, 'MANUAL GATHER', 1, 'AUTO GATHER', 'UNDEFINED GATHER') AS VARCHAR2(16)) AS TYPE, CAST(TASK_START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS NUMBER) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS NUMBER) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS VARCHAR2(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS VARCHAR2(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS NUMBER) AS RUNNING_TABLE_DURATION_TIME FROM SYS.ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(TENANT_ID AS NUMBER) AS TENANT_ID, CAST(SVR_IP AS VARCHAR2(46)) AS SVR_IP, CAST(SVR_PORT AS NUMBER) AS SVR_PORT, CAST(SESSION_ID AS NUMBER) AS SESSION_ID, CAST(TRACE_ID AS VARCHAR2(64)) AS TRACE_ID, CAST(TASK_ID AS VARCHAR(36)) AS TASK_ID, CAST(DECODE(TYPE, 0, 'MANUAL GATHER', 1, 'AUTO GATHER', 'UNDEFINED GATHER') AS VARCHAR2(16)) AS TYPE, CAST(TASK_START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, CAST(TASK_DURATION_TIME AS NUMBER) AS TASK_DURATION_TIME, CAST(TASK_TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(COMPLETED_TABLE_COUNT AS NUMBER) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS VARCHAR2(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS VARCHAR2(256)) AS RUNNING_TABLE_NAME, CAST(RUNNING_TABLE_DURATION_TIME AS NUMBER) AS RUNNING_TABLE_DURATION_TIME, CAST(SPARE2 AS VARCHAR2(256)) AS RUNNING_TABLE_PROGRESS FROM SYS.ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 85b461d8d7..daaf2b7553 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -27994,7 +27994,8 @@ def_table_schema( CAST(COMPLETED_TABLE_COUNT AS SIGNED) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS CHAR(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS CHAR(256)) AS RUNNING_TABLE_NAME, - CAST(RUNNING_TABLE_DURATION_TIME AS SIGNED) AS RUNNING_TABLE_DURATION_TIME + CAST(RUNNING_TABLE_DURATION_TIME AS SIGNED) AS RUNNING_TABLE_DURATION_TIME, + CAST(SPARE2 AS CHAR(256)) AS RUNNING_TABLE_PROGRESS FROM oceanbase.__all_virtual_opt_stat_gather_monitor """.replace("\n", " ") ) @@ -28031,7 +28032,8 @@ def_table_schema( ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS CHAR(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' - ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE 'FAILED' END) END) AS CHAR(8)) AS STATUS, + ELSE (CASE WHEN RET_CODE IS NULL THEN NULL + ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS CHAR(8)) AS STATUS, CAST(TABLE_COUNT AS SIGNED) AS TABLE_COUNT, CAST(FAILED_COUNT AS SIGNED) AS FAILED_COUNT, CAST(START_TIME AS DATETIME(6)) AS START_TIME, @@ -28057,7 +28059,8 @@ def_table_schema( CAST(V.TABLE_NAME AS CHAR(256)) AS TABLE_NAME, CAST(STAT.TASK_ID AS CHAR(36)) AS TASK_ID, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' - ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE 'FAILED' END) END) AS CHAR(8)) AS STATUS, + ELSE (CASE WHEN RET_CODE IS NULL THEN NULL + ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS CHAR(8)) AS STATUS, CAST(STAT.START_TIME AS DATETIME(6)) AS START_TIME, CAST(STAT.END_TIME AS DATETIME(6)) AS END_TIME, CAST(STAT.MEMORY_USED AS SIGNED) AS MEMORY_USED, @@ -47800,7 +47803,8 @@ def_table_schema( ELSE ( CASE WHEN type IS NULL THEN NULL ELSE 'UNDEFINED GATHER' END )END ) END ) AS VARCHAR2(16)) AS TYPE, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' - ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE 'FAILED' END) END) AS VARCHAR2(8)) AS STATUS, + ELSE (CASE WHEN RET_CODE IS NULL THEN NULL + ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS VARCHAR2(8)) AS STATUS, CAST(TABLE_COUNT AS NUMBER) AS TASK_TABLE_COUNT, CAST(FAILED_COUNT AS NUMBER) AS FAILED_COUNT, CAST(START_TIME AS TIMESTAMP(6)) AS TASK_START_TIME, @@ -47828,7 +47832,8 @@ def_table_schema( CAST(V.TABLE_NAME AS VARCHAR2(256)) AS TABLE_NAME, CAST(STAT.TASK_ID AS VARCHAR2(36)) AS TASK_ID, CAST((CASE WHEN RET_CODE = 0 THEN 'SUCCESS' - ELSE (CASE WHEN RET_CODE IS NULL THEN NULL ELSE 'FAILED' END) END) AS VARCHAR2(8)) AS STATUS, + ELSE (CASE WHEN RET_CODE IS NULL THEN NULL + ELSE (CASE WHEN RET_CODE = -5065 THEN 'CANCELED' ELSE 'FAILED' END) END) END) AS VARCHAR2(8)) AS STATUS, CAST(STAT.START_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(STAT.END_TIME AS TIMESTAMP(6)) AS END_TIME, CAST(STAT.MEMORY_USED AS NUMBER) AS MEMORY_USED, @@ -54860,7 +54865,8 @@ def_table_schema( CAST(COMPLETED_TABLE_COUNT AS NUMBER) AS COMPLETED_TABLE_COUNT, CAST(RUNNING_TABLE_OWNER AS VARCHAR2(128)) AS RUNNING_TABLE_OWNER, CAST(RUNNING_TABLE_NAME AS VARCHAR2(256)) AS RUNNING_TABLE_NAME, - CAST(RUNNING_TABLE_DURATION_TIME AS NUMBER) AS RUNNING_TABLE_DURATION_TIME + CAST(RUNNING_TABLE_DURATION_TIME AS NUMBER) AS RUNNING_TABLE_DURATION_TIME, + CAST(SPARE2 AS VARCHAR2(256)) AS RUNNING_TABLE_PROGRESS FROM SYS.ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR """.replace("\n", " ") ) diff --git a/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql b/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql index 1128b4a713..54c3292342 100644 --- a/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql +++ b/src/share/inner_table/sys_package/dbms_stats_body_mysql.sql @@ -139,7 +139,8 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats cascade_indexes BOOLEAN DEFAULT TRUE, statown VARCHAR(65535) DEFAULT NULL, no_invalidate BOOLEAN DEFAULT FALSE, - force BOOLEAN DEFAULT FALSE + force BOOLEAN DEFAULT FALSE, + degree DECIMAL DEFAULT 1 ); PRAGMA INTERFACE(C, DELETE_TABLE_STATS); @@ -154,7 +155,8 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats statown VARCHAR(65535) DEFAULT NULL, no_invalidate BOOLEAN DEFAULT FALSE, force BOOLEAN DEFAULT FALSE, - col_stat_type VARCHAR(65535) DEFAULT 'ALL' + col_stat_type VARCHAR(65535) DEFAULT 'ALL', + degree DECIMAL DEFAULT 1 ); PRAGMA INTERFACE(C, DELETE_COLUMN_STATS); @@ -169,7 +171,8 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats no_invalidate BOOLEAN DEFAULT FALSE, stattype VARCHAR(65535) DEFAULT 'ALL', force BOOLEAN DEFAULT FALSE, - tabname VARCHAR(65535) DEFAULT NULL + tabname VARCHAR(65535) DEFAULT NULL, + degree DECIMAL DEFAULT 1 ); PRAGMA INTERFACE(C, DELETE_INDEX_STATS); @@ -179,7 +182,8 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats statid VARCHAR(65535) DEFAULT NULL, statown VARCHAR(65535) DEFAULT NULL, no_invalidate BOOLEAN DEFAULT FALSE, - force BOOLEAN DEFAULT FALSE + force BOOLEAN DEFAULT FALSE, + degree DECIMAL DEFAULT 1 ); PRAGMA INTERFACE(C, DELETE_SCHEMA_STATS); @@ -431,6 +435,21 @@ CREATE OR REPLACE PACKAGE BODY dbms_stats ); PRAGMA INTERFACE(C, DELETE_TABLE_PREFS); + PROCEDURE copy_table_stats ( + ownname VARCHAR(65535), + tabname VARCHAR(65535), + srcpartname VARCHAR(65535), + dstpartname VARCHAR(65535), + scale_factor DECIMAL DEFAULT 1, + flags DECIMAL DEFAULT NULL, + force BOOLEAN DEFAULT FALSE + ); + PRAGMA INTERFACE(C, COPY_TABLE_STATS); + + PROCEDURE cancel_gather_stats ( + taskid VARCHAR(65535) + ); + PRAGMA INTERFACE(C, CANCEL_GATHER_STATS); PROCEDURE GATHER_SYSTEM_STATS(); PRAGMA INTERFACE(C, GATHER_SYSTEM_STATS); diff --git a/src/share/inner_table/sys_package/dbms_stats_mysql.sql b/src/share/inner_table/sys_package/dbms_stats_mysql.sql index b663905cc6..b4351fa0a4 100644 --- a/src/share/inner_table/sys_package/dbms_stats_mysql.sql +++ b/src/share/inner_table/sys_package/dbms_stats_mysql.sql @@ -139,7 +139,8 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER cascade_indexes BOOLEAN DEFAULT TRUE, statown VARCHAR(65535) DEFAULT NULL, no_invalidate BOOLEAN DEFAULT FALSE, - force BOOLEAN DEFAULT FALSE + force BOOLEAN DEFAULT FALSE, + degree DECIMAL DEFAULT 1 ); PROCEDURE delete_column_stats ( @@ -153,7 +154,8 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER statown VARCHAR(65535) DEFAULT NULL, no_invalidate BOOLEAN DEFAULT FALSE, force BOOLEAN DEFAULT FALSE, - col_stat_type VARCHAR(65535) DEFAULT 'ALL' + col_stat_type VARCHAR(65535) DEFAULT 'ALL', + degree DECIMAL DEFAULT 1 ); procedure delete_index_stats( @@ -167,7 +169,8 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER no_invalidate BOOLEAN DEFAULT FALSE, stattype VARCHAR(65535) DEFAULT 'ALL', force BOOLEAN DEFAULT FALSE, - tabname VARCHAR(65535) DEFAULT NULL + tabname VARCHAR(65535) DEFAULT NULL, + degree DECIMAL DEFAULT 1 ); PROCEDURE delete_schema_stats ( @@ -176,7 +179,8 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER statid VARCHAR(65535) DEFAULT NULL, statown VARCHAR(65535) DEFAULT NULL, no_invalidate BOOLEAN DEFAULT FALSE, - force BOOLEAN DEFAULT FALSE + force BOOLEAN DEFAULT FALSE, + degree DECIMAL DEFAULT 1 ); PROCEDURE FLUSH_DATABASE_MONITORING_INFO(); @@ -388,6 +392,19 @@ create or replace PACKAGE dbms_stats AUTHID CURRENT_USER pname VARCHAR(65535) ); + PROCEDURE copy_table_stats ( + ownname VARCHAR(65535), + tabname VARCHAR(65535), + srcpartname VARCHAR(65535), + dstpartname VARCHAR(65535), + scale_factor DECIMAL DEFAULT 1, + flags DECIMAL DEFAULT NULL, + force BOOLEAN DEFAULT FALSE + ); + + PROCEDURE cancel_gather_stats ( + taskid VARCHAR(65535) + ); PROCEDURE GATHER_SYSTEM_STATS(); PROCEDURE DELETE_SYSTEM_STATS(); diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index 2e1ef7dae8..5c2bd39bc4 100755 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -26165,9 +26165,9 @@ static const _error _error_OB_WRONG_PARTITION_NAME = { .sqlstate = "HY000", .str_error = "Incorrect partition name", .str_user_error = "Incorrect partition name '%.*s'", - .oracle_errno = 600, - .oracle_str_error = "ORA-00600: internal error code, arguments: -11002, Incorrect partition name", - .oracle_str_user_error = "ORA-00600: internal error code, arguments: -11002, Incorrect partition name '%.*s'" + .oracle_errno = 20000, + .oracle_str_error = "ORA-20000: invalid partition name", + .oracle_str_user_error = "ORA-20000: '%.*s' invalid partition name" }; static const _error _error_OB_ERR_PLUGIN_IS_NOT_LOADED = { .error_name = "OB_ERR_PLUGIN_IS_NOT_LOADED", diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index 065a6796d2..aff56a3960 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -2409,7 +2409,7 @@ DEFINE_ERROR(OB_TTL_INVALID_HBASE_MAXVERSIONS, -10508, -1, "HY000", "MaxVersions //////////////////////////////////////////////////////////////// DEFINE_ERROR(OB_ERR_VALUES_CLAUSE_NEED_HAVE_COLUMN, -11000, ER_VALUES_CLAUSE_NEED_HAVE_COLUMN, "HY000", "Each row of a VALUES clause must have at least one column, unless when used as source in an INSERT statement."); DEFINE_ERROR(OB_ERR_VALUES_CLAUSE_CANNOT_USE_DEFAULT_VALUES, -11001, ER_VALUES_CLAUSE_CANNOT_USE_DEFAULT_VALUES, "HY000", "A VALUES clause cannot use DEFAULT values, unless used as a source in an INSERT statement."); -DEFINE_ERROR_EXT(OB_WRONG_PARTITION_NAME, -11002, ER_WRONG_PARTITION_NAME, "HY000", "Incorrect partition name", "Incorrect partition name '%.*s'"); +DEFINE_ORACLE_ERROR_EXT(OB_WRONG_PARTITION_NAME, -11002, ER_WRONG_PARTITION_NAME, "HY000", "Incorrect partition name", "Incorrect partition name '%.*s'", 20000, "invalid partition name", "'%.*s' invalid partition name"); DEFINE_ERROR_EXT(OB_ERR_PLUGIN_IS_NOT_LOADED, -11003, ER_PLUGIN_IS_NOT_LOADED, "HY000", "Plugin is not loaded", "Plugin '%.*s' is not loaded"); //11004 : OB_EST_DEVIA_TOO_LARGE DEFINE_ORACLE_ERROR(OB_ERR_HAS_TYPE_OR_TABLE_DEPENDENT, -11005, -1, "HY000", "cannot drop or replace a type with type or table dependents", 2303, "cannot drop or replace a type with type or table dependents"); diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index c107da16ec..f69bba5ed1 100755 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -6215,7 +6215,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_TTL_INVALID_HBASE_MAXVERSIONS__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10508, MaxVersions of hbase table must be greater than 0" #define OB_ERR_VALUES_CLAUSE_NEED_HAVE_COLUMN__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -11000, Each row of a VALUES clause must have at least one column, unless when used as source in an INSERT statement." #define OB_ERR_VALUES_CLAUSE_CANNOT_USE_DEFAULT_VALUES__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -11001, A VALUES clause cannot use DEFAULT values, unless used as a source in an INSERT statement." -#define OB_WRONG_PARTITION_NAME__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -11002, Incorrect partition name '%.*s'" +#define OB_WRONG_PARTITION_NAME__ORA_USER_ERROR_MSG "ORA-20000: '%.*s' invalid partition name" #define OB_ERR_PLUGIN_IS_NOT_LOADED__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -11003, Plugin '%.*s' is not loaded" #define OB_ERR_HAS_TYPE_OR_TABLE_DEPENDENT__ORA_USER_ERROR_MSG "ORA-02303: cannot drop or replace a type with type or table dependents" #define OB_ERR_REPLACE_TYPE_WITH_TABLE_DEPENDENT__ORA_USER_ERROR_MSG "ORA-22866: cannot replace a type with table dependents" diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index fe667586a8..63a7dd973a 100755 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -10055,5 +10055,14 @@ int ObTabletLocationSendResult::get_ret() const OB_SERIALIZE_MEMBER(ObTabletLocationSendResult, ret_); +int ObCancelGatherStatsArg::assign(const ObCancelGatherStatsArg &other) +{ + int ret = OB_SUCCESS; + tenant_id_ = other.tenant_id_; + task_id_ = other.task_id_; + return ret; +} +OB_SERIALIZE_MEMBER(ObCancelGatherStatsArg, tenant_id_, task_id_); + }//end namespace obrpc }//end namepsace oceanbase diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 9738270d36..8735f0a6e8 100755 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -10436,6 +10436,19 @@ public: private: int ret_; }; + +struct ObCancelGatherStatsArg +{ + OB_UNIS_VERSION(1); +public: + ObCancelGatherStatsArg() : tenant_id_(0), task_id_() {} + int assign(const ObCancelGatherStatsArg &other); + bool is_valid() const { return tenant_id_ > 0; } + uint64_t tenant_id_; + common::ObString task_id_; + TO_STRING_KV(K(tenant_id_), K(task_id_)); +}; + }//end namespace obrpc }//end namespace oceanbase #endif diff --git a/src/share/ob_srv_rpc_proxy.h b/src/share/ob_srv_rpc_proxy.h index a18b3779f1..ed56ed6601 100755 --- a/src/share/ob_srv_rpc_proxy.h +++ b/src/share/ob_srv_rpc_proxy.h @@ -241,6 +241,7 @@ public: RPC_AP(PR5 kill_client_session, OB_KILL_CLIENT_SESSION, (ObKillClientSessionArg), ObKillClientSessionRes); RPC_S(PR5 client_session_create_time, OB_CLIENT_SESSION_CONNECT_TIME, (ObClientSessionCreateTimeAndAuthArg), ObClientSessionCreateTimeAndAuthRes); RPC_AP(PR5 tablet_location_send, OB_TABLET_LOCATION_BROADCAST, (obrpc::ObTabletLocationSendArg), obrpc::ObTabletLocationSendResult); + RPC_S(PR5 cancel_gather_stats, OB_CANCEL_GATHER_STATS, (ObCancelGatherStatsArg)); }; // end of class ObSrvRpcProxy } // end of namespace rpc diff --git a/src/share/stat/ob_basic_stats_estimator.cpp b/src/share/stat/ob_basic_stats_estimator.cpp index a4478cfd05..d9f40c2aae 100644 --- a/src/share/stat/ob_basic_stats_estimator.cpp +++ b/src/share/stat/ob_basic_stats_estimator.cpp @@ -17,6 +17,7 @@ #include "share/stat/ob_dbms_stats_utils.h" #include "sql/optimizer/ob_storage_estimator.h" #include "pl/sys_package/ob_dbms_stats.h" +#include "share/stat/ob_topk_hist_estimator.h" namespace oceanbase { namespace common @@ -42,8 +43,7 @@ int ObBasicStatsEstimator::add_stat_item(const T &item) return ret; } -int ObBasicStatsEstimator::estimate(const ObTableStatParam ¶m, - const ObExtraParam &extra, +int ObBasicStatsEstimator::estimate(const ObOptStatGatherParam ¶m, ObIArray &dst_opt_stats) { int ret = OB_SUCCESS; @@ -54,7 +54,7 @@ int ObBasicStatsEstimator::estimate(const ObTableStatParam ¶m, src_opt_stat.table_stat_ = &tab_stat; ObOptTableStat *src_tab_stat = src_opt_stat.table_stat_; ObIArray &src_col_stats = src_opt_stat.column_stats_; - ObArenaAllocator allocator("ObBasicStats"); + ObArenaAllocator allocator("ObBasicStatsEst", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); ObSqlString raw_sql; int64_t duration_time = -1; // Note that there are dependences between different kinds of statistics @@ -68,38 +68,61 @@ int ObBasicStatsEstimator::estimate(const ObTableStatParam ¶m, column_params.count(), src_col_stats))) { LOG_WARN("failed init col stats", K(ret)); - } else if (OB_FAIL(fill_hints(allocator, param.tab_name_))) { + } else if (OB_FAIL(fill_hints(allocator, param.tab_name_, param.gather_vectorize_))) { LOG_WARN("failed to fill hints", K(ret)); } else if (OB_FAIL(add_from_table(param.db_name_, param.tab_name_))) { LOG_WARN("failed to add from table", K(ret)); } else if (OB_FAIL(fill_parallel_info(allocator, param.degree_))) { LOG_WARN("failed to add query sql parallel info", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(extra.start_time_, - param.duration_time_, + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(param.gather_start_time_, + param.max_duration_time_, duration_time))) { LOG_WARN("failed to get valid duration time", K(ret)); - } else if (OB_FAIL(fill_query_timeout_info(*param.allocator_, duration_time))) { + } else if (OB_FAIL(fill_query_timeout_info(allocator, duration_time))) { LOG_WARN("failed to fill query timeout info", K(ret)); } else if (OB_FAIL(fill_sample_info(allocator, param.sample_info_))) { LOG_WARN("failed to fill sample info", K(ret)); - } else if (dst_opt_stats.count() > 1 && - OB_FAIL(fill_group_by_info(allocator, param, extra, calc_part_id_str))) { - LOG_WARN("failed to add group by info", K(ret)); - } else if (OB_FAIL(add_stat_item(ObStatRowCount(¶m, src_tab_stat)))) { + } else if (OB_FAIL(fill_specify_scn_info(allocator, param.sepcify_scn_))) { + LOG_WARN("failed to fill specify scn info", K(ret)); + } else if (OB_FAIL(add_stat_item(ObStatRowCount(src_tab_stat)))) { LOG_WARN("failed to add row count", K(ret)); - } else if (calc_part_id_str.empty()) { - if (!is_virtual_table(param.table_id_) && OB_FAIL(fill_partition_info(allocator, param, extra))) { - LOG_WARN("failed to add partition info", K(ret)); - } else if (OB_UNLIKELY(dst_opt_stats.count() != 1) || + } else if (!param.is_split_gather_) { + if (dst_opt_stats.count() > 1) { + if (OB_FAIL(fill_group_by_info(allocator, param, calc_part_id_str))) { + LOG_WARN("failed to add group by info", K(ret)); + } else if (OB_FAIL(add_stat_item(ObPartitionId(src_tab_stat, calc_part_id_str, -1)))) { + LOG_WARN("failed to add partition id", K(ret)); + } + } else if (OB_UNLIKELY(param.partition_infos_.count() > 1) || OB_ISNULL(dst_opt_stats.at(0).table_stat_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(dst_opt_stats.count())); + LOG_WARN("get unexpected error", K(ret), K(param.partition_infos_)); + } else if (!param.partition_infos_.empty() && + OB_FAIL(fill_partition_info(allocator, param.partition_infos_.at(0).part_name_))) { + LOG_WARN("failed to add partition info", K(ret)); } else { src_tab_stat->set_partition_id(dst_opt_stats.at(0).table_stat_->get_partition_id()); } - } else if (OB_FAIL(add_stat_item(ObPartitionId(¶m, src_tab_stat, calc_part_id_str, -1)))) { - LOG_WARN("failed to add partition id", K(ret)); - } else {/*do nothing*/} + } else {//table has been split gather because the system resource limit + if (dst_opt_stats.count() > 1) { + if (OB_FAIL(fill_group_by_info(allocator, param, calc_part_id_str))) { + LOG_WARN("failed to add group by info", K(ret)); + } else if (OB_FAIL(add_stat_item(ObPartitionId(src_tab_stat, calc_part_id_str, -1)))) { + LOG_WARN("failed to add partition id", K(ret)); + } else if (OB_FAIL(fill_partition_info(allocator, param.partition_infos_))) { + LOG_WARN("failed to add partition info", K(ret)); + } + } else if (OB_UNLIKELY(param.partition_infos_.count() > 1) || + OB_ISNULL(dst_opt_stats.at(0).table_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param.partition_infos_)); + } else if (!param.partition_infos_.empty() && + OB_FAIL(fill_partition_info(allocator, param.partition_infos_.at(0).part_name_))) { + LOG_WARN("failed to add partition info", K(ret)); + } else { + src_tab_stat->set_partition_id(dst_opt_stats.at(0).table_stat_->get_partition_id()); + } + } for (int64_t i = 0; OB_SUCC(ret) && i < column_params.count(); ++i) { const ObColumnStatParam *col_param = &column_params.at(i); if (OB_FAIL(add_stat_item(ObStatMaxValue(col_param, src_col_stats.at(i)))) || @@ -107,21 +130,18 @@ int ObBasicStatsEstimator::estimate(const ObTableStatParam ¶m, OB_FAIL(add_stat_item(ObStatNumNull(col_param, src_tab_stat, src_col_stats.at(i)))) || OB_FAIL(add_stat_item(ObStatNumDistinct(col_param, src_col_stats.at(i), param.need_approx_ndv_))) || OB_FAIL(add_stat_item(ObStatAvgLen(col_param, src_col_stats.at(i)))) || - OB_FAIL(add_stat_item(ObStatLlcBitmap(col_param, src_col_stats.at(i)))) || - (extra.need_histogram_ && - OB_FAIL(add_stat_item(ObStatTopKHist(col_param, src_tab_stat, src_col_stats.at(i)))))) { + OB_FAIL(add_stat_item(ObStatLlcBitmap(col_param, src_col_stats.at(i))))) { LOG_WARN("failed to add statistic item", K(ret)); } else {/*do nothing*/} } if (OB_SUCC(ret)) { - if (OB_FAIL(add_stat_item(ObStatAvgRowLen(¶m, src_tab_stat, src_col_stats)))) { + if (OB_FAIL(add_stat_item(ObStatAvgRowLen(src_tab_stat, src_col_stats)))) { LOG_WARN("failed to add avg row size estimator", K(ret)); } else if (OB_FAIL(pack(raw_sql))) { LOG_WARN("failed to pack raw sql", K(ret)); - } else if (OB_FAIL(do_estimate(param.tenant_id_, raw_sql.string(), COPY_ALL_STAT, - src_opt_stat, dst_opt_stats))) { + } else if (OB_FAIL(do_estimate(param.tenant_id_, raw_sql.string(), true, src_opt_stat, dst_opt_stats))) { LOG_WARN("failed to evaluate basic stats", K(ret)); - } else if (OB_FAIL(refine_basic_stats(param, extra, dst_opt_stats))) { + } else if (OB_FAIL(refine_basic_stats(param, dst_opt_stats))) { LOG_WARN("failed to refine basic stats", K(ret)); } else { LOG_TRACE("basic stats is collected", K(dst_opt_stats.count())); @@ -156,9 +176,9 @@ int ObBasicStatsEstimator::estimate_block_count(ObExecContext &ctx, LOG_WARN("failed to generate first part idx map", K(ret)); } else if (OB_FAIL(generate_column_group_ids(param, column_group_ids))) { LOG_WARN("failed to generate column group ids", K(ret), K(param)); - } else if (OB_FAIL(do_estimate_block_count_and_row_count(ctx, param.tenant_id_, table_id, tablet_ids, - partition_ids, column_group_ids, estimate_result))) { - LOG_WARN("failed to do estimate block count and row count", K(ret)); + } else if (OB_FAIL(do_estimate_block_count(ctx, param.tenant_id_, table_id, tablet_ids, + partition_ids, column_group_ids, estimate_result))) { + LOG_WARN("failed to do estimate block count", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < estimate_result.count(); ++i) { BlockNumStat *block_num_stat = NULL; @@ -268,6 +288,27 @@ int ObBasicStatsEstimator::estimate_block_count(ObExecContext &ctx, return ret; } +int ObBasicStatsEstimator::do_estimate_block_count(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &tablet_ids, + const ObIArray &partition_ids, + const ObIArray &column_group_ids, + ObIArray &estimate_res) +{ + int ret = OB_SUCCESS; + int64_t retry_cnt = 0; + do { + if (OB_FAIL(do_estimate_block_count_and_row_count(ctx, tenant_id, table_id, tablet_ids, + partition_ids, column_group_ids, estimate_res))) { + DAS_CTX(ctx).get_location_router().refresh_location_cache_by_errno(true, ret); + ++ retry_cnt; + LOG_WARN("failed to do estimate block count and row count", K(ret)); + } + } while (OB_FAIL(ret) && retry_cnt < 2);//retry one time if failed to estimate. + return ret; +} + int ObBasicStatsEstimator::do_estimate_block_count_and_row_count(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, @@ -387,11 +428,14 @@ int ObBasicStatsEstimator::stroage_estimate_block_count_and_row_count(ObExecCont } else { obrpc::ObSrvRpcProxy *rpc_proxy = NULL; const ObSQLSessionInfo *session_info = NULL; - int64_t timeout = 10 * 1000 * 1000; // 10s + int64_t timeout = std::min(MAX_OPT_STATS_PROCESS_RPC_TIMEOUT, THIS_WORKER.get_timeout_remain()); if (OB_ISNULL(session_info = ctx.get_my_session()) || OB_ISNULL(rpc_proxy = GCTX.srv_rpc_proxy_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("rpc_proxy or session is null", K(ret), K(rpc_proxy), K(session_info)); + } else if (0 >= timeout) { + ret = OB_TIMEOUT; + LOG_WARN("query timeout is reached", K(ret), K(timeout)); } else if (OB_FAIL(rpc_proxy->to(addr) .timeout(timeout) .by(session_info->get_rpc_tenant_id()) @@ -635,17 +679,18 @@ int ObBasicStatsEstimator::get_gather_table_duration(ObExecContext &ctx, int ObBasicStatsEstimator::estimate_stale_partition(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, + const int64_t global_part_id, const ObIArray &partition_infos, const double stale_percent_threshold, - const ObIArray &partition_stat_infos, - ObIArray &no_regather_partition_ids, - int64_t &no_regather_first_part_cnt) + ObIArray &partition_stat_infos) { int ret = OB_SUCCESS; ObSqlString select_sql; bool is_valid = true; - no_regather_first_part_cnt = 0; ObSEArray monitor_modified_part_ids; + bool is_check_global = false; + int64_t table_inc_modified = 0; + bool has_part_invalid_inc = false; if (OB_FAIL(ObDbmsStatsUtils::check_table_read_write_valid(tenant_id, is_valid))) { LOG_WARN("failed to check table read write valid", K(ret)); } else if (!is_valid) { @@ -671,6 +716,7 @@ int ObBasicStatsEstimator::estimate_stale_partition(ObExecContext &ctx, } else { int64_t cur_part_id = -1; //current partition for first part int64_t cur_inc_mod_count = 0;//current inc_mod_count for first part + bool has_subpart_invalid_inc = false; while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { int64_t idx1 = 0; int64_t idx2 = 1; @@ -695,36 +741,35 @@ int ObBasicStatsEstimator::estimate_stale_partition(ObExecContext &ctx, dst_partition))) { LOG_WARN("failed to get dst partition by tablet id", K(ret)); } else if (OB_FAIL(check_partition_stat_state(dst_partition, - partition_infos, inc_mod_count, stale_percent_threshold, - partition_stat_infos, - no_regather_partition_ids, - no_regather_first_part_cnt))) { + partition_stat_infos))) { LOG_WARN("failed to check partition stat state", K(ret)); } else if (OB_FAIL(monitor_modified_part_ids.push_back(dst_partition))) { LOG_WARN("failed to push back part ids occurred in monitor_modified", K(ret)); } else if (OB_FAIL(add_var_to_array_no_dup(monitor_modified_part_ids, cur_part_id))) { LOG_WARN("failed to push back part ids occurred in monitor_modified", K(ret)); } else if (ObDbmsStatsUtils::is_subpart_id(partition_infos, dst_partition, dst_part_id)) { + has_subpart_invalid_inc |= inc_mod_count < 0; if (cur_part_id == dst_part_id) { cur_inc_mod_count += inc_mod_count; } else if (cur_part_id == -1) { cur_part_id = dst_part_id; cur_inc_mod_count = inc_mod_count; } else if (OB_FAIL(check_partition_stat_state(cur_part_id, - partition_infos, - cur_inc_mod_count, + has_subpart_invalid_inc ? 0 : cur_inc_mod_count, stale_percent_threshold, - partition_stat_infos, - no_regather_partition_ids, - no_regather_first_part_cnt))) { + partition_stat_infos))) { LOG_WARN("failed to check partition stat state", K(ret)); } else { cur_part_id = dst_part_id; cur_inc_mod_count = inc_mod_count; + has_subpart_invalid_inc = false; } } + has_part_invalid_inc |= inc_mod_count < 0; + is_check_global = true; + table_inc_modified += inc_mod_count; } if (OB_FAIL(ret)) { if (OB_ITER_END != ret) { @@ -733,12 +778,15 @@ int ObBasicStatsEstimator::estimate_stale_partition(ObExecContext &ctx, ret = OB_SUCCESS; if (cur_part_id != -1 && OB_FAIL(check_partition_stat_state(cur_part_id, - partition_infos, - cur_inc_mod_count, + has_subpart_invalid_inc ? 0 : cur_inc_mod_count, stale_percent_threshold, - partition_stat_infos, - no_regather_partition_ids, - no_regather_first_part_cnt))) { + partition_stat_infos))) { + LOG_WARN("failed to check partition stat state", K(ret)); + } else if (is_check_global && + OB_FAIL(check_partition_stat_state(global_part_id, + has_part_invalid_inc ? 0 : table_inc_modified, + stale_percent_threshold, + partition_stat_infos))) { LOG_WARN("failed to check partition stat state", K(ret)); } else {/*do nothing*/} } @@ -757,39 +805,59 @@ int ObBasicStatsEstimator::estimate_stale_partition(ObExecContext &ctx, int64_t first_part_id = partition_infos.at(i).first_part_id_; // Partitions who not have dml infos are no need to regather stats if (!is_contain(monitor_modified_part_ids, partition_id)) { - int64_t part_id = -1; - if (OB_FAIL(no_regather_partition_ids.push_back(partition_id))) { - LOG_WARN("failed to push back part id that does not have dml info", K(ret)); - } else if (!ObDbmsStatsUtils::is_subpart_id(partition_infos, partition_id, part_id)) { - ++no_regather_first_part_cnt; - } + ObPartitionStatInfo partition_stat_info(partition_id, 0, false, true); + if (OB_FAIL(partition_stat_infos.push_back(partition_stat_info))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} } if (first_part_id != OB_INVALID_ID && !is_contain(monitor_modified_part_ids, first_part_id)) { - ret = no_regather_partition_ids.push_back(first_part_id); + ObPartitionStatInfo partition_stat_info(first_part_id, 0, false, true); + ret = partition_stat_infos.push_back(partition_stat_info); } } } LOG_INFO("succeed to estimate stale partition", K(stale_percent_threshold), - K(partition_stat_infos), - K(partition_infos), - K(monitor_modified_part_ids), - K(no_regather_partition_ids), - K(no_regather_first_part_cnt)); + K(partition_stat_infos), + K(partition_infos), + K(monitor_modified_part_ids)); return ret; } int ObBasicStatsEstimator::update_last_modified_count(ObExecContext &ctx, const ObTableStatParam ¶m) +{ + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + if (OB_FAIL(trans.start(ctx.get_sql_proxy(), param.tenant_id_))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(update_last_modified_count(trans.get_connection(), param))) { + LOG_WARN("failed to update last modified count", K(ret)); + } + //end gather trans + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } + return ret; +} + +int ObBasicStatsEstimator::update_last_modified_count(sqlclient::ObISQLConnection *conn, + const ObTableStatParam ¶m) { int ret = OB_SUCCESS; ObSqlString udpate_sql; ObSqlString tablet_list; - ObCommonSqlProxy *sql_proxy = ctx.get_sql_proxy(); int64_t affected_rows = 0; bool is_valid = true; - if (OB_ISNULL(ctx.get_sql_proxy())) { + if (OB_ISNULL(conn)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); + LOG_WARN("get unexpected null", K(ret), K(conn)); } else if (OB_FAIL(ObDbmsStatsUtils::check_table_read_write_valid(param.tenant_id_, is_valid))) { LOG_WARN("failed to check table read write valid", K(ret)); } else if (!is_valid) { @@ -806,7 +874,7 @@ int ObBasicStatsEstimator::update_last_modified_count(ObExecContext &ctx, share::schema::ObSchemaUtils::get_extract_schema_id(param.tenant_id_, param.table_id_), tablet_list.ptr()))) { LOG_WARN("failed to append fmt", K(ret)); - } else if (OB_FAIL(sql_proxy->write(param.tenant_id_, udpate_sql.ptr(), affected_rows))) { + } else if (OB_FAIL(conn->execute_write(param.tenant_id_, udpate_sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", K(ret), K(udpate_sql)); } else { LOG_TRACE("succeed to update last modified count", K(udpate_sql), K(affected_rows)); @@ -870,7 +938,7 @@ int ObBasicStatsEstimator::check_table_statistics_state(ObExecContext &ctx, } else if (global_part_id == part_val && lock_val > 0) { is_locked = true; } else { - ObPartitionStatInfo partition_stat_info(part_val, row_cnt, lock_val > 0); + ObPartitionStatInfo partition_stat_info(part_val, row_cnt, lock_val > 0, false); if (OB_FAIL(partition_stat_infos.push_back(partition_stat_info))) { LOG_WARN("failed to push back", K(ret)); } else {/*do nothing*/} @@ -898,33 +966,26 @@ int ObBasicStatsEstimator::check_table_statistics_state(ObExecContext &ctx, } int ObBasicStatsEstimator::check_partition_stat_state(const int64_t partition_id, - const ObIArray &partition_infos, const int64_t inc_mod_count, const double stale_percent_threshold, - const ObIArray &partition_stat_infos, - ObIArray &no_regather_partition_ids, - int64_t &no_regather_first_part_cnt) + ObIArray &partition_stat_infos) { int ret = OB_SUCCESS; bool find_it = false; - for (int64_t i = 0; OB_SUCC(ret) && !find_it && i < partition_stat_infos.count(); ++i) { + for (int64_t i = 0; !find_it && i < partition_stat_infos.count(); ++i) { if (partition_stat_infos.at(i).partition_id_ == partition_id) { //locked partition id or no arrived stale percent threshold no need regather stats. - double stale_percent = partition_stat_infos.at(i).row_cnt_ == 0 ? 1.0 : + double stale_percent = partition_stat_infos.at(i).row_cnt_ <= 0 ? 1.0 : 1.0 * inc_mod_count / partition_stat_infos.at(i).row_cnt_; - if (partition_stat_infos.at(i).is_stat_locked_ || stale_percent <= stale_percent_threshold) { - if (OB_FAIL(no_regather_partition_ids.push_back(partition_id))) { - LOG_WARN("failed to push back", K(ret)); - } else { - find_it = true; - int64_t part_id = -1; - if (!ObDbmsStatsUtils::is_subpart_id(partition_infos, partition_id, part_id)) { - ++ no_regather_first_part_cnt; - } - } - } else {/*do nothing*/} + partition_stat_infos.at(i).is_no_stale_ = stale_percent <= stale_percent_threshold; + find_it = true; } } + if (!find_it) { + ObPartitionStatInfo partition_stat_info(partition_id, 0, false, false); + partition_stat_info.is_no_stale_ = true; + ret = partition_stat_infos.push_back(partition_stat_info); + } return ret; } @@ -950,6 +1011,11 @@ int ObBasicStatsEstimator::gen_tablet_list(const ObTableStatParam ¶m, LOG_WARN("failed to push back", K(ret)); } } + for (int64_t i = 0; OB_SUCC(ret) && i < param.approx_part_infos_.count(); ++i) { + if (OB_FAIL(tablet_ids.push_back(param.approx_part_infos_.at(i).tablet_id_.id()))) { + LOG_WARN("failed to push back", K(ret)); + } + } } if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) { for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { @@ -1010,14 +1076,17 @@ int ObBasicStatsEstimator::get_need_stats_table_cnt(ObExecContext &ctx, int ret = OB_SUCCESS; ObSqlString select_sql; if (OB_FAIL(select_sql.append_fmt( - "select count(1) as cnt from (select m.table_id from " \ + "select count(1) as cnt from (select distinct m.table_id from " \ "%s m left join %s up on m.table_id = up.table_id and up.pname = 'STALE_PERCENT' join %s gp on gp.sname = 'STALE_PERCENT' " \ "where (case when (m.inserts+m.updates+m.deletes) = 0 then 0 " "else ((m.inserts+m.updates+m.deletes) - (m.last_inserts+m.last_updates+m.last_deletes)) * 1.0 / (m.inserts+m.updates+m.deletes) > " \ - "(CASE WHEN up.valchar IS NOT NULL THEN cast(up.valchar as signed) * 1.0 / 100 ELSE Cast(gp.spare4 AS signed) * 1.0 / 100 end) end)) ", + "(CASE WHEN up.valchar IS NOT NULL THEN cast(up.valchar as signed) * 1.0 / 100 ELSE Cast(gp.spare4 AS signed) * 1.0 / 100 end) end) " \ + "UNION select distinct table_id from %s where table_id not in (select table_id from %s)) ", share::OB_ALL_MONITOR_MODIFIED_TNAME, share::OB_ALL_OPTSTAT_USER_PREFS_TNAME, - share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME))) { + share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME, + share::OB_ALL_MONITOR_MODIFIED_TNAME, + share::OB_ALL_TABLE_STAT_TNAME))) { LOG_WARN("failed to append fmt", K(ret)); } else { ObCommonSqlProxy *sql_proxy = ctx.get_sql_proxy(); @@ -1064,13 +1133,16 @@ int ObBasicStatsEstimator::get_need_stats_tables(ObExecContext &ctx, if (OB_FAIL(select_sql.append_fmt( "select distinct table_id from (select m.table_id from " \ "%s m left join %s up on m.table_id = up.table_id and up.pname = 'STALE_PERCENT' join %s gp on gp.sname = 'STALE_PERCENT' " \ - "where (case when (m.inserts+m.updates+m.deletes) = 0 then 0 " + "where (case when (m.inserts+m.updates+m.deletes) = 0 then 0 "\ "else ((m.inserts+m.updates+m.deletes) - (m.last_inserts+m.last_updates+m.last_deletes)) * 1.0 / (m.inserts+m.updates+m.deletes) > " \ - "(CASE WHEN up.valchar IS NOT NULL THEN cast(up.valchar as signed) * 1.0 / 100 ELSE Cast(gp.spare4 AS signed) * 1.0 / 100 end) end)) " + "(CASE WHEN up.valchar IS NOT NULL THEN cast(up.valchar as signed) * 1.0 / 100 ELSE Cast(gp.spare4 AS signed) * 1.0 / 100 end) end) "\ + " UNION ALL select table_id from %s where table_id not in (select table_id from %s)) " "ORDER BY table_id DESC limit %ld", share::OB_ALL_MONITOR_MODIFIED_TNAME, share::OB_ALL_OPTSTAT_USER_PREFS_TNAME, share::OB_ALL_OPTSTAT_GLOBAL_PREFS_TNAME, + share::OB_ALL_MONITOR_MODIFIED_TNAME, + share::OB_ALL_TABLE_STAT_TNAME, slice_cnt))) { LOG_WARN("failed to append fmt", K(ret)); } else { @@ -1137,8 +1209,7 @@ int ObBasicStatsEstimator::generate_first_part_idx_map(const ObIArray * overall data distribution, So we need consider refine it, and reset the appropriate estimate_percent * to regather basic stats. */ -int ObBasicStatsEstimator::refine_basic_stats(const ObTableStatParam ¶m, - const ObExtraParam &extra, +int ObBasicStatsEstimator::refine_basic_stats(const ObOptStatGatherParam ¶m, ObIArray &dst_opt_stats) { int ret = OB_SUCCESS; @@ -1148,18 +1219,16 @@ int ObBasicStatsEstimator::refine_basic_stats(const ObTableStatParam ¶m, } else if (sample_value_ >= 0.000001 && sample_value_ < 100.0) { for (int64_t i = 0; OB_SUCC(ret) && i < dst_opt_stats.count(); ++i) { bool need_re_estimate = false; - ObExtraParam new_extra; - ObTableStatParam new_param; + ObOptStatGatherParam new_param; ObSEArray tmp_opt_stats; ObBasicStatsEstimator basic_re_est(ctx_, *param.allocator_); - if (OB_FAIL(check_stat_need_re_estimate(param, extra, dst_opt_stats.at(i), - need_re_estimate, new_param, new_extra))) { + if (OB_FAIL(check_stat_need_re_estimate(param, dst_opt_stats.at(i), need_re_estimate, new_param))) { LOG_WARN("failed to check stat need re-estimate", K(ret)); } else if (!need_re_estimate) { //do nothing } else if (OB_FAIL(tmp_opt_stats.push_back(dst_opt_stats.at(i)))) { LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(basic_re_est.estimate(new_param, new_extra, tmp_opt_stats))) { + } else if (OB_FAIL(basic_re_est.estimate(new_param, tmp_opt_stats))) { LOG_WARN("failed to estimate basic statistics", K(ret)); } else { LOG_TRACE("Suceed to re-estimate stats", K(new_param), K(param)); @@ -1169,12 +1238,10 @@ int ObBasicStatsEstimator::refine_basic_stats(const ObTableStatParam ¶m, return ret; } -int ObBasicStatsEstimator::check_stat_need_re_estimate(const ObTableStatParam &origin_param, - const ObExtraParam &origin_extra, +int ObBasicStatsEstimator::check_stat_need_re_estimate(const ObOptStatGatherParam &origin_param, ObOptStat &opt_stat, bool &need_re_estimate, - ObTableStatParam &new_param, - ObExtraParam &new_extra) + ObOptStatGatherParam &new_param) { int ret = OB_SUCCESS; need_re_estimate = false; @@ -1200,32 +1267,11 @@ int ObBasicStatsEstimator::check_stat_need_re_estimate(const ObTableStatParam &o new_param.sample_info_.sample_type_ = PercentSample; } //2.set partition info - new_extra.type_ = origin_extra.type_; - new_extra.start_time_ = origin_extra.start_time_; - new_extra.nth_part_ = origin_extra.nth_part_; - bool find_it = (new_extra.type_ == TABLE_LEVEL); - if (new_extra.type_ == PARTITION_LEVEL) { - for (int64_t i = 0; !find_it && i < new_param.part_infos_.count(); ++i) { - if (opt_stat.table_stat_->get_partition_id() == new_param.part_infos_.at(i).part_id_) { - find_it = true; - new_extra.nth_part_ = i; - new_param.part_name_ = new_param.part_infos_.at(i).part_name_; - new_param.is_subpart_name_ = false; - } + if (new_param.stat_level_ != TABLE_LEVEL) { + if (OB_FAIL(ObDbmsStatsUtils::remove_stat_gather_param_partition_info(opt_stat.table_stat_->get_partition_id(), + new_param))) { + LOG_WARN("failed to remove stat gather param partition info", K(ret)); } - } else if (new_extra.type_ == SUBPARTITION_LEVEL) { - for (int64_t i = 0; !find_it && i < new_param.subpart_infos_.count(); ++i) { - if (opt_stat.table_stat_->get_partition_id() == new_param.subpart_infos_.at(i).part_id_) { - find_it = true; - new_extra.nth_part_ = i; - new_param.part_name_ = new_param.subpart_infos_.at(i).part_name_; - new_param.is_subpart_name_ = true; - } - } - } - if (!find_it) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(new_param), KPC(opt_stat.table_stat_)); } //3.reset opt stat if (OB_SUCC(ret)) { @@ -1255,21 +1301,22 @@ int ObBasicStatsEstimator::check_stat_need_re_estimate(const ObTableStatParam &o } int ObBasicStatsEstimator::fill_hints(common::ObIAllocator &alloc, - const ObString &table_name) + const ObString &table_name, + int64_t gather_vectorize) { int ret = OB_SUCCESS; if (OB_UNLIKELY(table_name.empty())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(table_name)); } else { - const char *fmt_str = "NO_REWRITE USE_PLAN_CACHE(NONE) DBMS_STATS FULL(%.*s) OPT_PARAM('ROWSETS_MAX_ROWS', 256)"; + const char *fmt_str = "NO_REWRITE USE_PLAN_CACHE(NONE) DBMS_STATS FULL(%.*s) OPT_PARAM('ROWSETS_MAX_ROWS', %ld)"; int64_t buf_len = table_name.length() + strlen(fmt_str); char *buf = NULL; if (OB_ISNULL(buf = static_cast(alloc.alloc(buf_len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory", K(buf), K(buf_len)); } else { - int64_t real_len = sprintf(buf, fmt_str, table_name.length(), table_name.ptr()); + int64_t real_len = sprintf(buf, fmt_str, table_name.length(), table_name.ptr(), gather_vectorize); if (OB_UNLIKELY(real_len < 0 || real_len > buf_len)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(real_len)); diff --git a/src/share/stat/ob_basic_stats_estimator.h b/src/share/stat/ob_basic_stats_estimator.h index d8b6b016c6..6860eb8973 100644 --- a/src/share/stat/ob_basic_stats_estimator.h +++ b/src/share/stat/ob_basic_stats_estimator.h @@ -76,15 +76,17 @@ public: static int estimate_stale_partition(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, + const int64_t global_part_id, const ObIArray &partition_infos, const double stale_percent_threshold, - const ObIArray &partition_stat_infos, - ObIArray &no_regather_partition_ids, - int64_t &no_regather_first_part_cnt); + ObIArray &partition_stat_infos); static int update_last_modified_count(ObExecContext &ctx, const ObTableStatParam ¶m); + static int update_last_modified_count(sqlclient::ObISQLConnection *conn, + const ObTableStatParam ¶m); + static int check_table_statistics_state(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, @@ -93,16 +95,21 @@ public: ObIArray &partition_stat_infos); static int check_partition_stat_state(const int64_t partition_id, - const ObIArray &partition_infos, const int64_t inc_mod_count, const double stale_percent_threshold, - const ObIArray &partition_stat_infos, - ObIArray &no_regather_partition_ids, - int64_t &no_regather_first_part_cnt); + ObIArray &partition_stat_infos); static int gen_tablet_list(const ObTableStatParam ¶m, ObSqlString &tablet_list); + static int do_estimate_block_count(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &tablet_ids, + const ObIArray &partition_ids, + const ObIArray &column_group_ids, + ObIArray &estimate_res); + static int do_estimate_block_count_and_row_count(ObExecContext &ctx, const uint64_t tenant_id, const uint64_t table_id, @@ -135,28 +142,26 @@ public: const int64_t tenant_id, int64_t &task_table_count); - int estimate(const ObTableStatParam ¶m, - const ObExtraParam &extra, + int estimate(const ObOptStatGatherParam ¶m, ObIArray &dst_opt_stats); template int add_stat_item(const T &item); + int fill_hints(common::ObIAllocator &alloc, const ObString &table_name, int64_t gather_vectorize); + private: static int generate_first_part_idx_map(const ObIArray &all_part_infos, hash::ObHashMap &first_part_idx_map); - int refine_basic_stats(const ObTableStatParam ¶m, - const ObExtraParam &extra, + int refine_basic_stats(const ObOptStatGatherParam ¶m, ObIArray &dst_opt_stats); - int check_stat_need_re_estimate(const ObTableStatParam &origin_param, - const ObExtraParam &origin_extra, + int check_stat_need_re_estimate(const ObOptStatGatherParam &origin_param, ObOptStat &opt_stat, bool &need_re_estimate, - ObTableStatParam &new_param, - ObExtraParam &new_extra); + ObOptStatGatherParam &new_param); int fill_hints(common::ObIAllocator &alloc, const ObString &table_name); diff --git a/src/share/stat/ob_dbms_stats_copy_table_stats.cpp b/src/share/stat/ob_dbms_stats_copy_table_stats.cpp new file mode 100644 index 0000000000..677ce978b2 --- /dev/null +++ b/src/share/stat/ob_dbms_stats_copy_table_stats.cpp @@ -0,0 +1,834 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG +#include "share/stat/ob_opt_column_stat_cache.h" +#include "share/stat/ob_opt_table_stat.h" +#include "share/stat/ob_opt_column_stat.h" +#include "share/stat/ob_dbms_stats_utils.h" +#include "share/stat/ob_dbms_stats_copy_table_stats.h" + +int CopyTableStatHelper::copy_part_stat(ObIArray &table_stats) +{ + int ret = OB_SUCCESS; + ObOptTableStat *dst_part_stat = NULL; + if (OB_ISNULL(src_part_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_ISNULL(dst_part_stat = OB_NEWx(ObOptTableStat, allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate mem", K(ret)); + } else { + dst_part_stat->set_table_id(table_id_); + dst_part_stat->set_partition_id(dst_part_id_); + dst_part_stat->set_object_type(src_part_stat_->get_object_type()); + dst_part_stat->set_row_count(static_cast(src_part_stat_->get_row_count() * scale_factor_)); + dst_part_stat->set_avg_row_size(src_part_stat_->get_avg_row_size()); + dst_part_stat->set_sstable_row_count(static_cast(src_part_stat_->get_sstable_row_count() * scale_factor_)); + dst_part_stat->set_memtable_row_count(static_cast(src_part_stat_->get_memtable_row_count() * scale_factor_)); + dst_part_stat->set_data_size(src_part_stat_->get_data_size()); + dst_part_stat->set_macro_block_num(static_cast(src_part_stat_->get_macro_block_num() * scale_factor_)); + dst_part_stat->set_micro_block_num(static_cast(src_part_stat_->get_micro_block_num() * scale_factor_)); + dst_part_stat->set_sstable_avg_row_size(src_part_stat_->get_sstable_avg_row_size()); + dst_part_stat->set_memtable_avg_row_size(src_part_stat_->get_memtable_avg_row_size()); + dst_part_stat->set_data_version(src_part_stat_->get_data_version()); + dst_part_stat->set_last_analyzed(src_part_stat_->get_last_analyzed()); + // dst_part_stat->set_stattype_locked(src_part_stat_->get_stattype_locked()); no need to copy + dst_part_stat->set_sample_size(src_part_stat_->get_sample_size()); + dst_part_stat->set_tablet_id(src_part_stat_->get_tablet_id()); + dst_part_stat->set_stat_expired_time(src_part_stat_->get_stat_expired_time()); + //TODO:set stale_stats to true for hash/list partitions + if (OB_FAIL(table_stats.push_back(dst_part_stat))) { + LOG_WARN("failed to push back table stats", K(ret)); + } else { + LOG_TRACE("succeed to copy part stat", KPC(dst_part_stat), K(scale_factor_)); + } + } + return ret; +} + +int CopyTableStatHelper::copy_col_stat(bool is_subpart, + const ObIArray &col_handles, + ObIArray &column_stats) +{ + int ret = OB_SUCCESS; + bool is_src_part_equal_to_lower_bound = true; + if (OB_ISNULL(src_part_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(check_range_part(is_subpart, col_handles, is_src_part_equal_to_lower_bound))) { + LOG_WARN("failed to check range part", K(ret)); + } else { + for (int i = 0; OB_SUCC(ret) && i < col_handles.count(); ++i) { + const ObOptColumnStat *src_col_stat = col_handles.at(i).stat_; + ObOptColumnStat *dst_col_stat = NULL; + ObCopyPartInfo *dst_part_info = NULL; + if (OB_ISNULL(src_col_stat) || OB_ISNULL(allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), KP(allocator_), KP(src_col_stat)); + } else if (src_col_stat->get_partition_id() != src_part_stat_->get_partition_id()) { + LOG_TRACE("partition id not match, do nothing", K(src_col_stat->get_partition_id()), + K(src_part_stat_->get_partition_id())); + } else if (0 == src_col_stat->get_last_analyzed()) { + //do nothing + } else { + if (OB_ISNULL(dst_col_stat = ObOptColumnStat::malloc_new_column_stat(*allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate mem", K(ret)); + } else if ((!is_subpart && !is_contain(part_column_ids_, src_col_stat->get_column_id())) + || (is_subpart && !is_contain(subpart_column_ids_, src_col_stat->get_column_id()))) { + if (OB_FAIL(dst_col_stat->deep_copy(*src_col_stat))) { + LOG_WARN("failed to deep copy col stat", K(ret)); + } else { + dst_col_stat->set_partition_id(dst_part_id_); + } + LOG_TRACE("not partition key, directly copy", K(part_column_ids_), + K(subpart_column_ids_), + K(src_col_stat->get_column_id())); + } else if (OB_FAIL(dst_part_map_.get_refactored(src_col_stat->get_column_id(), dst_part_info))) { + LOG_WARN("failed to get dst part info", K(ret), K(src_col_stat->get_column_id())); + } else if (OB_ISNULL(dst_part_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else { + if (dst_part_info->is_normal_range_part_ && is_src_part_equal_to_lower_bound) { + dst_part_info->min_res_type_ = CopyPrePartUpperBound; + dst_part_info->max_res_type_ = CopyPrePartUpperBound; + } + dst_col_stat->set_table_id(table_id_); + dst_col_stat->set_partition_id(dst_part_id_); + dst_col_stat->set_column_id(src_col_stat->get_column_id()); + dst_col_stat->set_stat_level(src_col_stat->get_stat_level()); + dst_col_stat->set_version(src_col_stat->get_version()); + dst_col_stat->set_num_null(src_col_stat->get_num_null()); + dst_col_stat->set_num_not_null(src_part_stat_->get_row_count() - src_col_stat->get_num_null()); + dst_col_stat->set_num_distinct(src_col_stat->get_num_distinct()); + dst_col_stat->set_avg_len(src_col_stat->get_avg_len()); + dst_col_stat->set_last_analyzed(src_col_stat->get_last_analyzed()); + dst_col_stat->set_collation_type(src_col_stat->get_collation_type()); + dst_col_stat->set_total_col_len(src_col_stat->get_total_col_len()); + if (OB_FAIL(dst_col_stat->deep_copy_llc_bitmap(src_col_stat->get_llc_bitmap(), + src_col_stat->get_llc_bitmap_size()))) { + LOG_WARN("failed to deep copy llc bitmap", K(ret)); + } else if (0 == dst_col_stat->get_num_distinct()) { + dst_col_stat->get_min_value().set_null(); + dst_col_stat->get_max_value().set_null(); + } else if (OB_FAIL(copy_min_val(src_col_stat->get_min_value(), + dst_col_stat->get_min_value(), + dst_part_info))) { + LOG_WARN("failed to update min val", K(ret)); + } else if (OB_FAIL(copy_max_val(src_col_stat->get_max_value(), + dst_col_stat->get_max_value(), + dst_part_info))) { + LOG_WARN("failed to update min val", K(ret)); + } + if (OB_FAIL(ret)) { + } else if (dst_part_info->is_hash_part_) { + if (OB_FAIL(dst_col_stat->deep_copy_histogram(src_col_stat->get_histogram()))) { + LOG_WARN("failed to deep copy histogram", K(ret)); + } + } else { + dst_col_stat->get_histogram().set_sample_size(src_part_stat_->get_row_count() - src_col_stat->get_num_null()); + } + } + if (0 != dst_col_stat->get_max_value().compare(dst_col_stat->get_min_value()) + && dst_col_stat->get_num_distinct() <= 1) { + dst_col_stat->set_num_distinct(2); + } + LOG_TRACE("succeed to copy col stat", KPC(dst_col_stat), KPC(dst_part_info)); + if (OB_SUCC(ret) && OB_FAIL(column_stats.push_back(dst_col_stat))) { + LOG_WARN("failed to push back column stat", K(ret)); + } + } + } + } + return ret; +} + +int CopyTableStatHelper::check_range_part(bool is_subpart, + const ObIArray &col_handles, + bool &flag) +{ + int ret = OB_SUCCESS; + flag = true; + for (int64_t i = 0; OB_SUCC(ret) && flag && i < col_handles.count(); ++i) { + const ObOptColumnStat *src_col_stat = col_handles.at(i).stat_; + ObOptColumnStat *dst_col_stat = NULL; + ObCopyPartInfo *dst_part_info = NULL; + if (OB_ISNULL(allocator_) || OB_ISNULL(src_col_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), KP(allocator_), KP(src_col_stat)); + } else if (src_col_stat->get_partition_id() != src_part_stat_->get_partition_id()) { + } else if ((!is_subpart && !is_contain(part_column_ids_, src_col_stat->get_column_id())) + || (is_subpart && !is_contain(subpart_column_ids_, src_col_stat->get_column_id()))) { + //do nothing + } else { + if (OB_FAIL(dst_part_map_.get_refactored(src_col_stat->get_column_id(), dst_part_info))) { + LOG_WARN("failed to get dst part info", K(ret), K(src_col_stat->get_column_id())); + } else if (OB_ISNULL(dst_part_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (0 != src_col_stat->get_min_value().compare(dst_part_info->src_part_lower_bound_) || + 0 != src_col_stat->get_max_value().compare(dst_part_info->src_part_lower_bound_) || + 1 != src_col_stat->get_num_distinct()) { + flag = false; + } + } + } + return ret; +} + +int CopyTableStatHelper::copy_min_val(const common::ObObj &src_min_val, + common::ObObj &dst_min_val, + ObCopyPartInfo *dst_part_info) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(allocator_) || OB_ISNULL(dst_part_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), KP(allocator_), KP(dst_part_info)); + } else { + switch (dst_part_info->min_res_type_) { + case CopyDstPartLowerBound: { + if (OB_FAIL(ob_write_obj(*allocator_, dst_part_info->part_lower_bound_, dst_min_val))) { + LOG_WARN("failed to copy min val", K(ret)); + } + break; + } + case CopyDstPartUpperBound: { + if (OB_FAIL(ob_write_obj(*allocator_, dst_part_info->part_upper_bound_, dst_min_val))) { + LOG_WARN("failed to copy min val", K(ret)); + } + break; + } + case CopyPrePartUpperBound: { + if (OB_FAIL(ob_write_obj(*allocator_, dst_part_info->pre_part_upper_bound_, dst_min_val))) { + LOG_WARN("failed to copy max val", K(ret)); + } + break; + } + case CopySrcPartMinVal: { + if (OB_FAIL(ob_write_obj(*allocator_, src_min_val, dst_min_val))) { + LOG_WARN("failed to copy min val", K(ret)); + } + break; + } + case CopySrcPartMaxVal: + case CopyMaxSrcValDstBound: + default: { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid argument", K(ret)); + } + } + } + return ret; +} + +int CopyTableStatHelper::copy_max_val(const common::ObObj &src_max_val, + common::ObObj &dst_max_val, + ObCopyPartInfo *dst_part_info) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(allocator_) || OB_ISNULL(dst_part_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), KP(allocator_), KP(dst_part_info)); + } else { + switch (dst_part_info->max_res_type_) { + case CopyDstPartLowerBound: { + if (OB_FAIL(ob_write_obj(*allocator_, dst_part_info->part_lower_bound_, dst_max_val))) { + LOG_WARN("failed to copy min val", K(ret)); + } + break; + } + case CopyDstPartUpperBound: { + if (OB_FAIL(ob_write_obj(*allocator_, dst_part_info->part_upper_bound_, dst_max_val))) { + LOG_WARN("failed to copy min val", K(ret)); + } + break; + } + case CopyPrePartUpperBound: { + if (OB_FAIL(ob_write_obj(*allocator_, dst_part_info->pre_part_upper_bound_, dst_max_val))) { + LOG_WARN("failed to copy max val", K(ret)); + } + break; + } + case CopySrcPartMaxVal: { + if (OB_FAIL(ob_write_obj(*allocator_, src_max_val, dst_max_val))) { + LOG_WARN("failed to copy min val", K(ret)); + } + break; + } + case CopyMaxSrcValDstBound: { + if (src_max_val.compare(dst_part_info->part_upper_bound_) > 0) { + if (OB_FAIL(ob_write_obj(*allocator_, src_max_val, dst_max_val))) { + LOG_WARN("failed to copy min val", K(ret)); + } + } else { + if (OB_FAIL(ob_write_obj(*allocator_, dst_part_info->part_upper_bound_, dst_max_val))) { + LOG_WARN("failed to copy min val", K(ret)); + } + } + break; + } + case CopySrcPartMinVal: + default: { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid argument", K(ret)); + } + } + } + return ret; +} + +int CopyTableStatHelper::copy_part_col_stat(bool is_subpart, + const ObIArray &col_handles, + ObIArray &table_stats, + ObIArray &column_stats) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(copy_part_stat(table_stats))) { + LOG_WARN("failed to copy table stat", K(ret)); + } else if (OB_FAIL(copy_col_stat(is_subpart, col_handles, column_stats))) { + LOG_WARN("failed to copy column stat", K(ret)); + } + return ret; +} + +int ObDbmsStatsCopyTableStats::extract_partition_column_ids(CopyTableStatHelper ©_stat_helper, + const ObTableSchema *table_schema) +{ + int ret = OB_SUCCESS; + ObSEArray column_ids; + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(table_schema->get_column_ids(column_ids))) { + LOG_WARN("failed to get column ids", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < column_ids.count(); ++i) { + bool is_part = false; + if (table_schema->is_partitioned_table()) { + if (OB_FAIL(table_schema->get_partition_key_info().is_rowkey_column(column_ids.at(i), is_part))) { + LOG_WARN("check is partition key failed", K(ret), K(column_ids)); + } else if (is_part && + OB_FAIL(add_var_to_array_no_dup(copy_stat_helper.part_column_ids_, column_ids.at(i)))) { + LOG_WARN("failed to push back part column ids", K(ret)); + } + if (OB_SUCC(ret) && PARTITION_LEVEL_TWO == table_schema->get_part_level()) { + bool is_subpart = false; + if (OB_FAIL(table_schema->get_subpartition_key_info().is_rowkey_column(column_ids.at(i), is_subpart))) { + LOG_WARN("check is partition key failed", K(ret), K(column_ids)); + } else if (is_subpart && + OB_FAIL(add_var_to_array_no_dup(copy_stat_helper.subpart_column_ids_, column_ids.at(i)))) { + LOG_WARN("failed to push back part column ids", K(ret)); + } + } + } + } + } + return ret; +} + +int ObDbmsStatsCopyTableStats::check_parts_valid(sql::ObExecContext &ctx, + const CopyTableStatHelper &helper, + const ObTableStatParam &table_stat_param, + ObCopyLevel ©_level) +{ + int ret = OB_SUCCESS; + PartInfo dummy_part; + const ObString &srcpart_name = helper.srcpart_name_; + const ObString &dstpart_name = helper.dstpart_name_; + if (table_stat_param.part_infos_.empty()) { + ret = OB_WRONG_PARTITION_NAME; + LOG_WARN("The table is not partitioned, so the partition name is invalid", K(ret)); + LOG_ORACLE_USER_ERROR(OB_WRONG_PARTITION_NAME, srcpart_name.length(), srcpart_name.ptr()); + } else { + bool is_src_onepart = ObDbmsStatsUtils::find_part(table_stat_param.all_part_infos_, + srcpart_name, + lib::is_oracle_mode(), + dummy_part); + bool is_src_twopart = ObDbmsStatsUtils::find_part(table_stat_param.all_subpart_infos_, + srcpart_name, + lib::is_oracle_mode(), + dummy_part); + bool is_dst_onepart = ObDbmsStatsUtils::find_part(table_stat_param.all_part_infos_, + dstpart_name, + lib::is_oracle_mode(), + dummy_part); + bool is_dst_twopart = ObDbmsStatsUtils::find_part(table_stat_param.all_subpart_infos_, + dstpart_name, + lib::is_oracle_mode(), + dummy_part); + if ((!is_src_onepart && !is_src_twopart) || + (!is_dst_onepart && !is_dst_twopart)) { + ret = OB_WRONG_PARTITION_NAME; + LOG_WARN("invalid src/dst part name", K(ret), K(is_src_onepart), K(is_src_twopart), + K(is_dst_onepart), K(is_dst_twopart), K(srcpart_name), K(dstpart_name)); + if (!is_dst_onepart && !is_dst_twopart) { + LOG_ORACLE_USER_ERROR(OB_WRONG_PARTITION_NAME, dstpart_name.length(), dstpart_name.ptr()); + } else { + LOG_ORACLE_USER_ERROR(OB_WRONG_PARTITION_NAME, srcpart_name.length(), srcpart_name.ptr()); + } + } else if ((is_src_twopart && is_dst_onepart) + || (is_src_onepart && is_dst_twopart)) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("src partition and dst partition type are different", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "src partition and dst partition type are different"); + } else if (is_src_onepart && is_dst_onepart) { + copy_level = CopyOnePartLevel; + } else if (is_src_twopart && is_dst_twopart) { + copy_level = CopyTwoPartLevel; + } + } + return ret; +} + +int ObDbmsStatsCopyTableStats::copy_tab_col_stats(sql::ObExecContext &ctx, + ObTableStatParam &table_stat_param, + CopyTableStatHelper ©_stat_helper) +{ + int ret = OB_SUCCESS; + ObSEArray tab_handles; + ObSEArray col_handles; + ObSEArray table_stats; + ObSEArray column_stats; + if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(table_stat_param, + tab_handles, + col_handles))) { + LOG_WARN("failed to get history stat handles", K(ret)); + } else if (OB_FAIL(find_src_tab_stat(table_stat_param, tab_handles, copy_stat_helper.src_part_stat_))) { + LOG_WARN("failed to find src table stat", K(ret)); + } else if (OB_ISNULL(copy_stat_helper.src_part_stat_) + || 0 == copy_stat_helper.src_part_stat_->get_last_analyzed()) { + // the src part does not have stat, do nothing + LOG_WARN("src table stat is not analyzed", K(table_stat_param.part_infos_.at(0).part_id_)); + } else if (OB_FAIL(copy_stat_helper.copy_part_col_stat(table_stat_param.is_subpart_name_, col_handles, table_stats, column_stats))) { + LOG_WARN("failed to copy table column stat", K(ret), KPC(copy_stat_helper.src_part_stat_)); + } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, table_stats, column_stats))) { + LOG_WARN("failed to split batch write stat", K(ret)); + } + return ret; +} + +int ObDbmsStatsCopyTableStats::find_src_tab_stat(const ObTableStatParam &table_stat_param, + const ObIArray &tab_handles, + ObOptTableStat *&src_tab_stat) +{ + int ret = OB_SUCCESS; + src_tab_stat = NULL; + if (OB_UNLIKELY(table_stat_param.part_infos_.count() != 1)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("should only have one part info", K(ret), K(table_stat_param.part_infos_.count())); + } else if (table_stat_param.is_subpart_name_ + && OB_UNLIKELY(table_stat_param.subpart_infos_.count() != 1)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("should only have one subpart info", K(ret), K(table_stat_param.subpart_infos_.count())); + } else { + int64_t src_part_id = OB_INVALID_ID; + if (table_stat_param.is_subpart_name_) { + src_part_id = table_stat_param.subpart_infos_.at(0).part_id_; + } else { + src_part_id = table_stat_param.part_infos_.at(0).part_id_; + } + for (int64_t i = 0; OB_SUCC(ret) && src_tab_stat == NULL && i < tab_handles.count(); ++i) { + ObOptTableStat * cur_tab_stat = const_cast(tab_handles.at(i).stat_); + if (OB_ISNULL(cur_tab_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(cur_tab_stat), K(table_stat_param.part_infos_)); + } else if (src_part_id == cur_tab_stat->get_partition_id()) { + src_tab_stat = cur_tab_stat; + LOG_TRACE("succeed to find src part stat", KPC(src_tab_stat)); + } + } + } + return ret; +} + +int ObDbmsStatsCopyTableStats::get_dst_part_infos(const ObTableStatParam &table_stat_param, + CopyTableStatHelper &helper, + const ObTableSchema *table_schema, + const ObCopyLevel copy_level, + bool &is_found) +{ + int ret = OB_SUCCESS; + is_found = false; + bool found_dst = false; + bool found_src = false; + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (!table_schema->is_partitioned_table()) { + // do nothing + } else if (OB_FAIL(helper.dst_part_map_.create(7, "DstPartBucket", "DstPartNode", helper.tenant_id_))) { + LOG_WARN("failed to create dst part map", K(ret)); + } else if (copy_level == CopyOnePartLevel) { + const ObPartition *part = NULL; + const ObPartition *pre_part = NULL; + ObCheckPartitionMode check_partition_mode = CHECK_PARTITION_MODE_NORMAL; + ObPartIterator iter(*table_schema, check_partition_mode); + found_src = !table_schema->is_range_part(); + while (OB_SUCC(ret) && !is_found && OB_SUCC(iter.next(part))) { + if (OB_ISNULL(part)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null partition", K(ret), K(part)); + } else if (ObCharset::case_insensitive_equal(helper.dstpart_name_, part->get_part_name())) { + if (OB_FAIL(get_dst_part_info(table_schema, helper, part, pre_part, copy_level))) { + LOG_WARN("failed to get dst part info", K(ret)); + } else { + helper.dst_part_id_ = part->get_part_id(); + found_dst = true; + } + } else if (table_schema->is_range_part() && + ObCharset::case_insensitive_equal(helper.srcpart_name_, part->get_part_name())) { + if (OB_FAIL(get_src_part_info(table_schema, helper, part, pre_part, copy_level))) { + LOG_WARN("failed to get src part info", K(ret)); + } else { + found_src = true; + } + } + pre_part = part; + is_found = found_src && found_dst; + } + } else if (copy_level == CopyTwoPartLevel) { + const ObPartition *part = NULL; + ObCheckPartitionMode check_partition_mode = CHECK_PARTITION_MODE_NORMAL; + ObPartIterator iter(*table_schema, check_partition_mode); + found_src = !table_schema->is_range_subpart(); + while (OB_SUCC(ret) && !is_found && OB_SUCC(iter.next(part))) { + if (OB_ISNULL(part)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("part is null", KR(ret)); + } else { + ObSubPartIterator sub_iter(*table_schema, *part, check_partition_mode); + const ObSubPartition *subpart = NULL; + const ObSubPartition *pre_subpart = NULL; + while (OB_SUCC(ret) && !is_found && OB_SUCC(sub_iter.next(subpart))) { + if (OB_ISNULL(subpart)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null subpartition", K(ret)); + } else if (ObCharset::case_insensitive_equal(helper.dstpart_name_, subpart->get_part_name())) { + if (OB_FAIL(get_dst_part_info(table_schema, helper, subpart, pre_subpart, copy_level))) { + LOG_WARN("failed to get dst part info", K(ret)); + } else { + helper.dst_part_id_ = subpart->get_sub_part_id(); + found_dst = true; + } + } else if (table_schema->is_range_subpart() && + ObCharset::case_insensitive_equal(helper.srcpart_name_, subpart->get_part_name())) { + if (OB_FAIL(get_src_part_info(table_schema, helper, subpart, pre_subpart, copy_level))) { + LOG_WARN("failed to get src part info", K(ret)); + } else { + found_src = true; + } + } + pre_subpart = subpart; + is_found = found_src && found_dst; + } + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { + break; + } + } + } + } + ret = (ret == OB_ITER_END ? OB_SUCCESS : ret); + return ret; +} + +int ObDbmsStatsCopyTableStats::get_dst_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObBasePartition *part, + const ObBasePartition *pre_part, + const ObCopyLevel copy_level) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(part) || OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", KPC(part), K(table_schema)); + } else if ((copy_level == CopyOnePartLevel && + is_hash_like_part(table_schema->get_part_option().get_part_func_type())) || + (copy_level == CopyTwoPartLevel && + is_hash_like_part(table_schema->get_sub_part_option().get_part_func_type()))) { + if (OB_FAIL(get_hash_or_default_part_info(table_schema, + helper, + copy_level, + true))) { + LOG_WARN("failed to get hash or list default part info", K(ret)); + } + } else if ((copy_level == CopyOnePartLevel && table_schema->is_list_part()) + || (copy_level == CopyTwoPartLevel && table_schema->is_list_subpart())) { + int64_t list_val_cnt = part->get_list_row_values().count(); + if (OB_UNLIKELY(list_val_cnt == 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid argument", K(ret), KPC(part)); + } else if (list_val_cnt == 1 && part->get_list_row_values().at(0).get_cell(0).is_max_value()) { + // default part + if (OB_FAIL(get_hash_or_default_part_info(table_schema, + helper, + copy_level, + false))) { + LOG_WARN("failed to get hash or list default part info", K(ret)); + } + } else if (OB_FAIL(get_normal_list_part_info(table_schema, + helper, + part, + copy_level, + list_val_cnt))) { + LOG_WARN("failed to get normal list part info", K(ret)); + } + } else if ((copy_level == CopyOnePartLevel && table_schema->is_range_part()) + || (copy_level == CopyTwoPartLevel && table_schema->is_range_subpart())) { + const ObRowkey &high_bound_val = part->get_high_bound_val(); + int64_t rowkey_cnt = high_bound_val.get_obj_cnt(); + if (OB_UNLIKELY(rowkey_cnt == 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid rowkey cnt"); + } else if (OB_FAIL(get_range_part_info(table_schema, + helper, + part, + pre_part, + copy_level, + rowkey_cnt))) { + LOG_WARN("failed to get range part info", K(ret)); + } + } + return ret; +} + +int ObDbmsStatsCopyTableStats::get_hash_or_default_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObCopyLevel copy_level, + bool is_hash) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else { + int64_t sz = 0; + if (copy_level == CopyOnePartLevel) { + sz = table_schema->get_partition_key_info().get_size(); + } else if (copy_level == CopyTwoPartLevel) { + sz = table_schema->get_subpartition_key_info().get_size(); + } + for (int64_t i = 0; OB_SUCC(ret) && i < sz; ++i) { + ObCopyPartInfo *dst_part_info = NULL; + if (OB_FAIL(get_copy_part_info(table_schema, copy_level, i, helper, dst_part_info))) { + LOG_WARN("failed to get copy part info", K(ret)); + } else if (OB_ISNULL(dst_part_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else { + dst_part_info->min_res_type_ = CopySrcPartMinVal; + dst_part_info->max_res_type_ = CopySrcPartMaxVal; + dst_part_info->is_hash_part_ = is_hash; + } + LOG_TRACE("succeed to get hash or default list part info", KPC(dst_part_info), K(copy_level), K(i)); + } + } + return ret; +} + +int ObDbmsStatsCopyTableStats::get_normal_list_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObBasePartition *part, + const ObCopyLevel copy_level, + int64_t list_val_cnt) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_schema) || OB_ISNULL(part)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(table_schema), K(part)); + } else { + const common::ObNewRow &low_row = part->get_list_row_values().at(0); + const common::ObNewRow &high_row = part->get_list_row_values().at(list_val_cnt - 1); + for (int64_t i = 0; OB_SUCC(ret) && i < low_row.get_count(); ++i) { + ObCopyPartInfo *dst_part_info = NULL; + if (OB_FAIL(get_copy_part_info(table_schema, copy_level, i, helper, dst_part_info))) { + LOG_WARN("failed to get copy part info", K(ret)); + } else if (OB_ISNULL(dst_part_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(ob_write_obj(*helper.allocator_, + low_row.get_cell(i), + dst_part_info->part_lower_bound_))) { + LOG_WARN("failed to copy obj", K(ret)); + } else if (OB_FAIL(ob_write_obj(*helper.allocator_, + high_row.get_cell(i), + dst_part_info->part_upper_bound_))) { + LOG_WARN("failed to copy obj", K(ret)); + } else { + dst_part_info->min_res_type_ = CopyDstPartLowerBound; + dst_part_info->max_res_type_ = CopyDstPartUpperBound; + } + LOG_TRACE("succeed to get normal list part info", + KPC(dst_part_info), K(copy_level), K(i), K(part->get_list_row_values())); + } + } + return ret; +} + +int ObDbmsStatsCopyTableStats::get_range_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObBasePartition *part, + const ObBasePartition *pre_part, + const ObCopyLevel copy_level, + int64_t rowkey_cnt) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_schema) || OB_ISNULL(part)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(table_schema), K(part)); + } else { + const ObRowkey &high_bound_val = part->get_high_bound_val(); + const ObRowkey &low_bound_val = part->get_low_bound_val(); + + ObCopyPartInfo *dst_part_info = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; ++i) { + if (OB_FAIL(get_copy_part_info(table_schema, + copy_level, i, + helper, + dst_part_info))) { + LOG_WARN("failed to get copy part info", K(ret)); + } else if (OB_ISNULL(dst_part_info)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret)); + } else if (NULL != high_bound_val.get_obj_ptr() + && OB_FAIL(ob_write_obj(*helper.allocator_, + high_bound_val.get_obj_ptr()[i], + dst_part_info->part_upper_bound_))) { + LOG_WARN("failed to copy obj", K(ret)); + } else if (NULL != low_bound_val.get_obj_ptr() + && OB_FAIL(ob_write_obj(*helper.allocator_, + low_bound_val.get_obj_ptr()[i], + dst_part_info->part_lower_bound_))) { + LOG_WARN("failed to copy obj", K(ret)); + } else if (NULL != pre_part) { + const ObRowkey &pre_high_bound_val = pre_part->get_high_bound_val(); + if (NULL != pre_high_bound_val.get_obj_ptr() + && OB_FAIL(ob_write_obj(*helper.allocator_, + pre_high_bound_val.get_obj_ptr()[i], + dst_part_info->pre_part_upper_bound_))) { + LOG_WARN("failed to copy obj", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (NULL == pre_part) { + // first range part + dst_part_info->min_res_type_ = CopyDstPartUpperBound; + } else { + dst_part_info->min_res_type_ = CopyPrePartUpperBound; + dst_part_info->is_normal_range_part_ = true; + } + if (high_bound_val.get_obj_ptr()[i].is_max_value()) { + dst_part_info->max_res_type_ = CopyPrePartUpperBound; + } else { + dst_part_info->max_res_type_ = CopyDstPartUpperBound; + if (i != 0) { + dst_part_info->max_res_type_ = CopyMaxSrcValDstBound; + if (pre_part != NULL + && high_bound_val.get_obj_ptr()[i - 1] == + pre_part->get_high_bound_val().get_obj_ptr()[i - 1]) { + dst_part_info->max_res_type_ = CopyDstPartUpperBound; + } + } + } + } + LOG_TRACE("succeed to get range part info", KPC(dst_part_info), + K(copy_level), K(i), KP(pre_part), K(high_bound_val), K(low_bound_val)); + } + } + return ret; +} + +int ObDbmsStatsCopyTableStats::get_copy_part_info(const ObTableSchema *table_schema, + const ObCopyLevel copy_level, + const int64_t idx, + CopyTableStatHelper &helper, + ObCopyPartInfo *&dst_part_info) +{ + int ret = OB_SUCCESS; + ObRowkeyColumn column; + if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (copy_level == CopyOnePartLevel && + OB_FAIL(table_schema->get_partition_key_info().get_column(idx, column))) { + LOG_WARN("failed to get partition key info", K(ret)); + } else if (copy_level == CopyTwoPartLevel && + OB_FAIL(table_schema->get_subpartition_key_info().get_column(idx, column))) { + LOG_WARN("failed to get subpartition key info", K(ret)); + } else if (OB_FAIL(helper.dst_part_map_.get_refactored(column.column_id_, dst_part_info))) { + if (OB_UNLIKELY(ret != OB_HASH_NOT_EXIST)) { + LOG_WARN("failed to get copy part info", K(ret)); + } else { + ret = OB_SUCCESS; + if (OB_ISNULL(dst_part_info = OB_NEWx(ObCopyPartInfo, helper.allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate part info", K(ret)); + } else if (OB_FAIL(helper.dst_part_map_.set_refactored(column.column_id_, dst_part_info))) { + LOG_WARN("failed to set part info", K(ret)); + } else { + LOG_DEBUG("add dist part info", K(copy_level), K(column.column_id_)); + } + } + } + return ret; +} + +int ObDbmsStatsCopyTableStats::get_src_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObBasePartition *part, + const ObBasePartition *pre_part, + const ObCopyLevel copy_level) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_schema) || OB_ISNULL(part)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(part), K(table_schema)); + } else if (OB_UNLIKELY(!table_schema->is_partitioned_table() + || (CopyOnePartLevel == copy_level && !table_schema->is_range_part()) + || (CopyTwoPartLevel == copy_level && !table_schema->is_range_subpart()))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("only range partitioned table need to get src part info", K(ret)); + } else { + const ObRowkey &high_bound_val = part->get_high_bound_val(); + const ObRowkey &low_bound_val = NULL != pre_part ? pre_part->get_high_bound_val() + : part->get_low_bound_val(); + int64_t rowkey_cnt = high_bound_val.get_obj_cnt(); + if (OB_UNLIKELY(rowkey_cnt == 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid rowkey cnt", K(ret), K(high_bound_val)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt; ++i) { + ObCopyPartInfo *dst_part_info = NULL; + if (OB_FAIL(get_copy_part_info(table_schema, + copy_level, i, + helper, + dst_part_info))) { + LOG_WARN("failed to get copy part info", K(ret)); + } else if (OB_ISNULL(dst_part_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to allocate memory", K(ret)); + } else if (NULL != low_bound_val.get_obj_ptr() + && OB_FAIL(ob_write_obj(*helper.allocator_, + low_bound_val.get_obj_ptr()[i], + dst_part_info->src_part_lower_bound_))) { + LOG_WARN("failed to copy obj", K(ret)); + } + LOG_TRACE("succeed to get src part info", KPC(dst_part_info), + K(copy_level), K(i), K(high_bound_val), K(low_bound_val)); + } + } + } + return ret; +} diff --git a/src/share/stat/ob_dbms_stats_copy_table_stats.h b/src/share/stat/ob_dbms_stats_copy_table_stats.h new file mode 100644 index 0000000000..7d516649b2 --- /dev/null +++ b/src/share/stat/ob_dbms_stats_copy_table_stats.h @@ -0,0 +1,187 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_DBMS_STATS_COPY_TABLE_STATS_H +#define OB_DBMS_STATS_COPY_TABLE_STATS_H + +#include "share/stat/ob_opt_table_stat_cache.h" +#include "share/stat/ob_opt_column_stat_cache.h" +#include "share/stat/ob_stat_define.h" +namespace oceanbase +{ +using namespace sql; +namespace common +{ +enum ObCopyLevel { + InvalidCopyLevel = -1, + CopyOnePartLevel, + CopyTwoPartLevel +}; + +enum ObCopyResType { + InvalidCopyResType = -1, + CopyDstPartLowerBound, + CopyDstPartUpperBound, + CopyPrePartUpperBound, + CopySrcPartMinVal, + CopySrcPartMaxVal, + CopyMaxSrcValDstBound +}; + +struct ObCopyPartInfo { + ObCopyPartInfo(): + pre_part_upper_bound_(), + part_lower_bound_(), + part_upper_bound_(), + src_part_lower_bound_(), + min_res_type_(InvalidCopyResType), + max_res_type_(InvalidCopyResType), + is_normal_range_part_(false), + is_hash_part_(false) + { } + common::ObObj pre_part_upper_bound_; + common::ObObj part_lower_bound_; + common::ObObj part_upper_bound_; + common::ObObj src_part_lower_bound_; + ObCopyResType min_res_type_; + ObCopyResType max_res_type_; + bool is_normal_range_part_; + bool is_hash_part_; + TO_STRING_KV(K(pre_part_upper_bound_), K(part_lower_bound_), K(part_upper_bound_), + K(min_res_type_), K(max_res_type_), K(is_normal_range_part_), K(is_hash_part_)); +}; + +struct CopyTableStatHelper { + CopyTableStatHelper(ObIAllocator *alloc): + tenant_id_(OB_INVALID_ID), + table_id_(OB_INVALID_ID), + owner_(""), + table_name_(""), + srcpart_name_(""), + dstpart_name_(""), + scale_factor_(1), + flags_(0.0), + force_copy_(false), + src_part_stat_(NULL), + dst_part_id_(-1), + dst_part_map_(), + part_column_ids_(), + allocator_(alloc) + { } + int copy_part_col_stat(bool is_subpart, + const ObIArray &col_handles, + ObIArray &table_stats, + ObIArray &column_stats); + int copy_part_stat(ObIArray &table_stats); + int copy_col_stat(bool is_subpart, + const ObIArray &col_handles, + ObIArray &column_stats); + int copy_min_val(const common::ObObj &src_min_val, + common::ObObj &dst_min_val, + ObCopyPartInfo *dst_part_info); + int copy_max_val(const common::ObObj &src_max_val, + common::ObObj &dst_max_val, + ObCopyPartInfo *dst_part_info); + int check_range_part(bool is_subpart, const ObIArray &col_handles, bool &flag); + // int copy_histogram(const ObHistogram &src_hist, + // ObHistogram &dst_hist, + // ObCopyPartInfo *dst_part_info); + // int set_dst_hist_buckets(ObHistogram &dst_hist, + // const common::ObObj &endpoint_val_1, + // const common::ObObj &endpoint_val_2); + uint64_t tenant_id_; + uint64_t table_id_; + ObString owner_; + ObString table_name_; + ObString srcpart_name_; + ObString dstpart_name_; + double scale_factor_; + double flags_; // unused + bool force_copy_; + + ObOptTableStat *src_part_stat_; + int64_t dst_part_id_; + hash::ObHashMap dst_part_map_; + ObSEArray part_column_ids_; + ObSEArray subpart_column_ids_; + + ObIAllocator *allocator_; + TO_STRING_KV(K(tenant_id_), K(table_id_), K(owner_), K(table_name_), + K(srcpart_name_), K(dstpart_name_), K(scale_factor_), K(flags_), + K(force_copy_), K(part_column_ids_)); +}; + +class ObDbmsStatsCopyTableStats +{ +public: + static int check_parts_valid(sql::ObExecContext &ctx, + const CopyTableStatHelper &helper, + const ObTableStatParam &table_stat_param, + ObCopyLevel ©_level); + + static int extract_partition_column_ids(CopyTableStatHelper ©_stat_helper, + const ObTableSchema *table_schema); + + static int copy_tab_col_stats(sql::ObExecContext &ctx, + ObTableStatParam &table_stat_param, + CopyTableStatHelper ©_stat_helper); + + static int find_src_tab_stat(const ObTableStatParam &table_stat_param, + const ObIArray &history_tab_handles, + ObOptTableStat *&src_tab_stat); + + static int get_dst_part_infos(const ObTableStatParam &table_stat_param, + CopyTableStatHelper &helper, + const ObTableSchema *table_schema, + const ObCopyLevel copy_level, + bool &is_found); + + static int get_copy_part_info(const ObTableSchema *table_schema, + const ObCopyLevel copy_level, + const int64_t idx, + CopyTableStatHelper &helper, + ObCopyPartInfo *&dst_part_info); + + static int get_src_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObBasePartition *part, + const ObBasePartition *pre_part, + const ObCopyLevel copy_level); + + static int get_dst_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObBasePartition *part, + const ObBasePartition *pre_part, + const ObCopyLevel copy_level); + + static int get_hash_or_default_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObCopyLevel copy_level, + bool is_hash); + + static int get_normal_list_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObBasePartition *part, + const ObCopyLevel copy_level, + int64_t list_val_cnt); + + static int get_range_part_info(const ObTableSchema *table_schema, + CopyTableStatHelper &helper, + const ObBasePartition *part, + const ObBasePartition *pre_part, + const ObCopyLevel copy_level, + int64_t rowkey_idx); +}; + +} +} +#endif // OB_DBMS_STATS_COPY_TABLE_STATS_H \ No newline at end of file diff --git a/src/share/stat/ob_dbms_stats_executor.cpp b/src/share/stat/ob_dbms_stats_executor.cpp index 18717faf31..d3363fd42f 100644 --- a/src/share/stat/ob_dbms_stats_executor.cpp +++ b/src/share/stat/ob_dbms_stats_executor.cpp @@ -23,6 +23,10 @@ #include "share/stat/ob_index_stats_estimator.h" #include "pl/sys_package/ob_dbms_stats.h" #include "share/stat/ob_opt_stat_gather_stat.h" +#include "share/stat/ob_dbms_stats_gather.h" +#include "observer/omt/ob_tenant.h" +#include "observer/ob_inner_sql_connection_pool.h" +#include "observer/ob_inner_sql_connection.h" #include "src/observer/ob_server.h" namespace oceanbase { @@ -35,190 +39,747 @@ namespace common { * @return */ int ObDbmsStatsExecutor::gather_table_stats(ObExecContext &ctx, - const ObTableStatParam ¶m) + const ObTableStatParam ¶m, + ObOptStatRunningMonitor &running_monitor) { int ret = OB_SUCCESS; LOG_TRACE("begin to gather table stats", K(param)); + PartitionIdBlockMap partition_id_block_map; + GatherHelper gather_helper(running_monitor); + ObMySQLTransaction gather_trans; + ObMySQLTransaction backup_trans; + if (OB_FAIL(gather_trans.start(ctx.get_sql_proxy(), param.tenant_id_)) || + OB_FAIL(backup_trans.start(ctx.get_sql_proxy(), param.tenant_id_))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(prepare_gather_stats(ctx, gather_trans, param, + partition_id_block_map, + gather_helper))) { + LOG_WARN("failed to prepare gather stats", K(ret)); + } else if (gather_helper.is_split_gather_ && + OB_FAIL(split_gather_stats(ctx, gather_trans, param, &partition_id_block_map, gather_helper))) { + LOG_WARN("failed to split gather stats", K(ret)); + } else if (!gather_helper.is_split_gather_ && + OB_FAIL(no_split_gather_stats(ctx, gather_trans, param, &partition_id_block_map, gather_helper))) { + LOG_WARN("failed to do gather stats", K(ret)); + } else if (OB_FAIL(running_monitor.add_monitor_info(ObOptStatRunningPhase::BACKUP_HISTORY_STATS))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (!param.is_temp_table_ && + OB_FAIL(ObDbmsStatsHistoryManager::backup_opt_stats(ctx, backup_trans, param, + ObTimeUtility::current_time(), + true))) { + LOG_WARN("failed to backup opt stats", K(ret)); + } else if (share::schema::ObTableType::EXTERNAL_TABLE != param.ref_table_type_ && + OB_FAIL(ObBasicStatsEstimator::update_last_modified_count(gather_trans.get_connection(), param))) { + LOG_WARN("failed to update last modified count", K(ret)); + } + //end gather trans + if (OB_SUCC(ret)) { + if (OB_FAIL(gather_trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = gather_trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } + //end backup trans + if (OB_SUCC(ret)) { + if (OB_FAIL(backup_trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = backup_trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } + return ret; +} + +int ObDbmsStatsExecutor::split_gather_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const PartitionIdBlockMap *partition_id_block_map, + GatherHelper &gather_helper) + +{ + int ret = OB_SUCCESS; + if (param.subpart_stat_param_.need_modify_) {//process subpart stats + if (param.part_level_ != share::schema::PARTITION_LEVEL_TWO) { + /*do nothing*/ + } else if (OB_FAIL(split_gather_partition_stats(ctx, trans, param, SUBPARTITION_LEVEL, + partition_id_block_map, gather_helper))) { + LOG_WARN("failed to split gather partition stats", K(ret)); + } + } + if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) {//process part stats + gather_helper.is_approx_gather_ = param.part_stat_param_.can_use_approx_; + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); + } else if (param.part_level_ == share::schema::PARTITION_LEVEL_ZERO) { + /*do nothing*/ + } else if (OB_UNLIKELY(!param.part_stat_param_.can_use_approx_ && !param.approx_part_infos_.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param)); + } else if (!param.part_infos_.empty()) {//gather some part stats + if (OB_FAIL(split_gather_partition_stats(ctx, trans, param, PARTITION_LEVEL, + partition_id_block_map, gather_helper))) { + LOG_WARN("failed to split gather partition stats", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (param.part_stat_param_.can_use_approx_ && !param.approx_part_infos_.empty()) { + if (OB_FAIL(split_derive_part_stats_by_subpart_stats(ctx, trans, param, + partition_id_block_map, + gather_helper))) { + LOG_WARN("failed to split derive part stats by subpart stats", K(ret)); + } + } + } + } + if (OB_SUCC(ret) && param.global_stat_param_.need_modify_) {//process global stats + gather_helper.is_approx_gather_ = param.global_stat_param_.gather_approx_; + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); + } else if (OB_FAIL(split_gather_global_stats(ctx, trans, param, partition_id_block_map, gather_helper))) { + LOG_WARN("failed to split gather table stats", K(ret)); + } + } + return ret; +} + +int ObDbmsStatsExecutor::no_split_gather_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const PartitionIdBlockMap *partition_id_block_map, + GatherHelper &gather_helper) + +{ + int ret = OB_SUCCESS; ObArray all_tstats; ObArray all_cstats; - ObSEArray approx_opt_part_stats; - ObExtraParam extra; - extra.start_time_ = ObTimeUtility::current_time(); - if (OB_ISNULL(param.allocator_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(param.allocator_)); - } else if (OB_FAIL(extra.partition_id_block_map_.create(10000, - ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, - ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, - param.tenant_id_))) { + ObSEArray subpart_opt_stats; + ObSEArray part_opt_stats; + ObSEArray global_opt_stats; + if (param.subpart_stat_param_.need_modify_) {//process subpart stats + ObOptStatGatherParam gather_param; + if (param.part_level_ != share::schema::PARTITION_LEVEL_TWO || + param.subpart_infos_.empty()) { + /*do nothing*/ + } else if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(ObOptStatRunningPhase::GATHER_SUBPART_STATS))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::prepare_gather_stat_param(param, SUBPARTITION_LEVEL, partition_id_block_map, + false, gather_helper.gather_vectorize_, gather_param))) { + LOG_WARN("failed to prepare gather stat param", K(ret)); + } else if (OB_FAIL(do_gather_stats(ctx, trans, gather_param, + param.subpart_infos_, + param.column_params_, + true, + subpart_opt_stats, + all_tstats, + all_cstats))) { + LOG_WARN("failed to do gather stats", K(ret)); + } + } + if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) {//process part stats + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); + } else if (param.part_level_ == share::schema::PARTITION_LEVEL_ZERO) { + /*do nothing*/ + } else if (OB_UNLIKELY(!param.part_stat_param_.can_use_approx_ && !param.approx_part_infos_.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param)); + } else if (!param.part_infos_.empty()) {//gather some part stats + ObOptStatGatherParam gather_param; + if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(ObOptStatRunningPhase::GATHER_PART_STATS))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::prepare_gather_stat_param(param, PARTITION_LEVEL, partition_id_block_map, + false, gather_helper.gather_vectorize_, gather_param))) { + LOG_WARN("failed to prepare gather stat param", K(ret)); + } else if (OB_FAIL(do_gather_stats(ctx, trans, gather_param, + param.part_infos_, + param.column_params_, + true, + part_opt_stats, + all_tstats, + all_cstats))) { + LOG_WARN("failed to do gather stats", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (param.part_stat_param_.can_use_approx_ && !param.approx_part_infos_.empty()) {//approx some part stats base on subpart stats + if (OB_UNLIKELY(subpart_opt_stats.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param.approx_part_infos_), K(subpart_opt_stats)); + } else if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(ObOptStatRunningPhase::APPROX_GATHER_PART_STATS))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(ObIncrementalStatEstimator::derive_part_stats_from_subpart_stats(ctx, param, subpart_opt_stats, part_opt_stats))) { + LOG_WARN("failed to try derive global stat", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::calssify_opt_stat(part_opt_stats, + all_tstats, + all_cstats))) { + LOG_WARN("failed to calssify opt stat", K(ret)); + } + } + } + } + if (OB_SUCC(ret) && param.global_stat_param_.need_modify_) {//process global stats + ObOptStatGatherParam gather_param; + ObOptStat global_opt_stat; + ObArray dummy_part_infos; + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); + } else if (param.global_stat_param_.gather_approx_ && !part_opt_stats.empty()) {//approx global stats base on part stats + if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(ObOptStatRunningPhase::APPROX_GATHER_GLOBAL_STATS))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(ObIncrementalStatEstimator::derive_global_stat_from_part_stats(ctx, param, part_opt_stats, global_opt_stat))) { + LOG_WARN("failed to try derive global stat", K(ret)); + } else if (OB_FAIL(global_opt_stats.push_back(global_opt_stat))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::calssify_opt_stat(global_opt_stats, + all_tstats, + all_cstats))) { + LOG_WARN("failed to calssify opt stat", K(ret)); + } + } else if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(ObOptStatRunningPhase::GATHER_GLOBAL_STATS))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::prepare_gather_stat_param(param, TABLE_LEVEL, partition_id_block_map, + false, gather_helper.gather_vectorize_, gather_param))) { + LOG_WARN("failed to prepare gather stat param", K(ret)); + } else if (OB_FAIL(do_gather_stats(ctx, trans, gather_param, + dummy_part_infos, + param.column_params_, + true, + global_opt_stats, + all_tstats, + all_cstats))) { + LOG_WARN("failed to do gather stats", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, trans.get_connection(), all_tstats, all_cstats))) { + LOG_WARN("failed to split batch write", K(ret)); + } + } + return ret; +} + +/** @brief ObDbmsStatsExecutor::prepare_gather_stats used to prepare gather table stats, including: + * 1.estimate block count; + * 2.get the maximum num of partitions and columns for each stat gather. +*/ +int ObDbmsStatsExecutor::prepare_gather_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + PartitionIdBlockMap &partition_id_block_map, + GatherHelper &gather_helper) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(partition_id_block_map.create(10000, + ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, + ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, + param.tenant_id_))) { LOG_WARN("failed to create hash map", K(ret)); } else if (param.need_estimate_block_ && share::schema::ObTableType::EXTERNAL_TABLE != param.ref_table_type_ && OB_FAIL(ObBasicStatsEstimator::estimate_block_count(ctx, param, - extra.partition_id_block_map_))) { + partition_id_block_map))) { LOG_WARN("failed to estimate block count", K(ret)); + } else if (OB_FAIL(check_need_split_gather(param, gather_helper))) { + LOG_WARN("failed to check need split gather", K(ret)); + } else { + LOG_TRACE("succeed to prepare gather stats", K(param), K(gather_helper)); } - if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) { - extra.type_ = SUBPARTITION_LEVEL; - extra.nth_part_ = 0; - extra.need_histogram_ = param.subpart_stat_param_.gather_histogram_; - ObSEArray opt_stats; - if (param.part_level_ != share::schema::PARTITION_LEVEL_TWO) { - /*do nothing*/ - } else if (OB_FAIL(do_gather_stats(ctx, param, extra, approx_opt_part_stats, opt_stats))) { - LOG_WARN("failed to gather subpartition stats", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::calssify_opt_stat(opt_stats, - all_tstats, - all_cstats))) { - LOG_WARN("failed to calssify opt stat", K(ret)); - } - } - if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) { - extra.type_ = PARTITION_LEVEL; - extra.nth_part_ = 0; - extra.need_histogram_ = param.part_stat_param_.gather_histogram_; - ObSEArray opt_stats; - if (param.part_level_ == share::schema::PARTITION_LEVEL_ZERO) { - /*do nothing*/ - } else if (OB_FAIL(do_gather_stats(ctx, param, extra, approx_opt_part_stats, opt_stats))) { - LOG_WARN("failed to gather partition stats", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::calssify_opt_stat(opt_stats, - all_tstats, - all_cstats))) { - LOG_WARN("failed to calssify opt stat", K(ret)); - } - } - if (OB_SUCC(ret) && - param.global_stat_param_.need_modify_ && - !param.global_stat_param_.gather_approx_) { - extra.type_ = TABLE_LEVEL; - extra.nth_part_ = 0; - extra.need_histogram_ = param.global_stat_param_.gather_histogram_; - ObSEArray opt_stats; - if (OB_FAIL(do_gather_stats(ctx, param, extra, approx_opt_part_stats, opt_stats))) { - LOG_WARN("failed to gather table stats", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::calssify_opt_stat(opt_stats, - all_tstats, - all_cstats))) { - LOG_WARN("failed to calssify opt stat", K(ret)); - } - } - if (OB_SUCC(ret)) { - ObSEArray history_tab_handles; - ObSEArray history_col_handles; - //before write, we need record history stats. - if (!param.is_temp_table_ && - OB_FAIL(ObDbmsStatsHistoryManager::get_history_stat_handles(ctx, param, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to get history stat handles", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, all_tstats, all_cstats))) { - LOG_WARN("failed to split batch write", K(ret)); - } else if (!param.is_temp_table_ && - OB_FAIL(ObDbmsStatsUtils::batch_write_history_stats(ctx, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to batch write history stats", K(ret)); - } else if (share::schema::ObTableType::EXTERNAL_TABLE != param.ref_table_type_ && - OB_FAIL(ObBasicStatsEstimator::update_last_modified_count(ctx, param))) { - LOG_WARN("failed to update last modified count", K(ret)); - } else {/*do nothing*/} - } + return ret; +} - if (extra.partition_id_block_map_.created()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = extra.partition_id_block_map_.destroy())) { - LOG_WARN("failed to destroy hash map", K(ret), K(tmp_ret)); - ret = COVER_SUCC(tmp_ret); +int ObDbmsStatsExecutor::split_gather_partition_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + StatLevel stat_level, + const PartitionIdBlockMap *partition_id_block_map, + const GatherHelper &gather_helper) +{ + int ret = OB_SUCCESS; + ObOptStatGatherParam gather_param; + ObArenaAllocator allocator("SplitGatherStat", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); + const ObIArray &partition_infos = stat_level == PARTITION_LEVEL ? param.part_infos_ : param.subpart_infos_; + if (OB_UNLIKELY((stat_level != PARTITION_LEVEL && stat_level != SUBPARTITION_LEVEL) || + !gather_helper.is_split_gather_ || + gather_helper.maximum_gather_col_cnt_ < 1 || + gather_helper.maximum_gather_part_cnt_ < 1 || + partition_infos.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param), K(stat_level), K(gather_helper)); + } else if (OB_FAIL(ObDbmsStatsUtils::prepare_gather_stat_param(param, stat_level, partition_id_block_map, true, + gather_helper.gather_vectorize_, gather_param))) { + LOG_WARN("failed to prepare gather stat param", K(ret)); + } else {//need split gather + int64_t idx_part = 0; + int64_t total_split_cnt = std::ceil((1.0 * partition_infos.count() / gather_helper.maximum_gather_part_cnt_) * + (1.0 * param.column_params_.count() / gather_helper.maximum_gather_col_cnt_)); + ObOptStatRunningPhase current_phase = stat_level == SUBPARTITION_LEVEL ? ObOptStatRunningPhase::GATHER_SUBPART_STATS : ObOptStatRunningPhase::GATHER_PART_STATS; + ObOptStatRunningPhase next_phase = stat_level == SUBPARTITION_LEVEL ? ObOptStatRunningPhase::GATHER_PART_STATS : ObOptStatRunningPhase::GATHER_GLOBAL_STATS; + double split_extra_ratio = gather_helper.running_monitor_.get_monitor_extra_progress_ratio(current_phase, next_phase, total_split_cnt); + double extra_ratio = 0.0; + do { + ObSEArray gather_partition_infos; + for (int64_t i = 0; OB_SUCC(ret) && i < gather_helper.maximum_gather_part_cnt_ && idx_part < partition_infos.count(); ++i) { + if (OB_FAIL(gather_partition_infos.push_back(partition_infos.at(idx_part++)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (gather_helper.maximum_gather_col_cnt_ >= param.column_params_.count()) { + ObSEArray all_tstats; + ObSEArray all_cstats; + ObSEArray opt_stats; + gather_param.allocator_ = &allocator; + if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(current_phase, extra_ratio))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(do_gather_stats(ctx, trans, gather_param, + gather_partition_infos, + param.column_params_, + true, + opt_stats, + all_tstats, + all_cstats))) { + LOG_WARN("failed to do gather stats", K(ret)); + } else { + allocator.reuse();//Phased memory release for split gather, in order to reuse memory + } + } else { + int64_t idx_col = 0; + ObOptTableStat part_Stat; + ObSEArray all_tstats; + if (OB_FAIL(all_tstats.push_back(&part_Stat))) { + LOG_WARN("faile to push back", K(ret)); + } else if (OB_FAIL(fetch_gather_table_snapshot_read(trans.get_connection(), gather_param.tenant_id_, gather_param.sepcify_scn_))) { + LOG_WARN("failed to fetch gather table snapshot read", K(ret)); + ret = OB_SUCCESS;//if we failed to get the read snapshot, just skip, not specify the snapshot + } + if (OB_SUCC(ret)) { + do { + ObSEArray gather_column_params; + for (int64_t i = 0; OB_SUCC(ret) && i < gather_helper.maximum_gather_col_cnt_ && idx_col < param.column_params_.count(); ++i) { + if (OB_FAIL(gather_column_params.push_back(param.column_params_.at(idx_col++)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + ObSEArray all_cstats; + ObSEArray opt_stats; + gather_param.allocator_ = &allocator; + if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(current_phase, extra_ratio))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(do_gather_stats(ctx, trans, gather_param, + gather_partition_infos, + gather_column_params, + idx_col == param.column_params_.count(), + opt_stats, + all_tstats, + all_cstats))) { + if (gather_param.sepcify_scn_ > 0 && + (ret == OB_TABLE_DEFINITION_CHANGED || ret == OB_SNAPSHOT_DISCARDED)) { + LOG_WARN("failed to specify snapshot to gather stats, try no specify snapshot to gather stats", K(ret)); + gather_param.sepcify_scn_ = 0; + allocator.reuse(); + opt_stats.reset(); + all_cstats.reset(); + if (OB_FAIL(do_gather_stats(ctx, trans, gather_param, + gather_partition_infos, + gather_column_params, + idx_col == param.column_params_.count(), + opt_stats, + all_tstats, + all_cstats))) { + LOG_WARN("failed to do gather stats", K(ret)); + } else { + allocator.reuse();//Phased memory release for split gather, in order to reuse memory + } + } else { + LOG_WARN("failed to do gather stats", K(ret)); + } + } else { + allocator.reuse();//Phased memory release for split gather, in order to reuse memory + } + } + } while(OB_SUCC(ret) && idx_col < param.column_params_.count()); + } + gather_param.sepcify_scn_ = 0;//Try to ensure that the stat of a partition are collected in a snapshot + } + } + extra_ratio += split_extra_ratio; + } while (OB_SUCC(ret) && idx_part < partition_infos.count()); + } + return ret; +} + +int ObDbmsStatsExecutor::split_derive_part_stats_by_subpart_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const PartitionIdBlockMap *partition_id_block_map, + const GatherHelper &gather_helper) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator("SplitDeriveStat", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); + if (OB_UNLIKELY(!param.part_stat_param_.can_use_approx_ || + !param.subpart_stat_param_.need_modify_ || + param.part_level_ != share::schema::PARTITION_LEVEL_TWO || + param.approx_part_infos_.empty() || + !gather_helper.is_split_gather_ || + gather_helper.maximum_gather_col_cnt_ < 1 || + gather_helper.maximum_gather_part_cnt_ < 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param), K(gather_helper)); + } else {//need split gather + int64_t idx_part = 0; + int64_t total_split_cnt = std::ceil((1.0 * param.approx_part_infos_.count() / gather_helper.maximum_gather_part_cnt_) * + (1.0 * param.column_params_.count() / gather_helper.maximum_gather_col_cnt_)); + ObOptStatRunningPhase current_phase = ObOptStatRunningPhase::APPROX_GATHER_PART_STATS; + ObOptStatRunningPhase next_phase = ObOptStatRunningPhase::APPROX_GATHER_GLOBAL_STATS; + double split_extra_ratio = gather_helper.running_monitor_.get_monitor_extra_progress_ratio(current_phase, next_phase, total_split_cnt); + double extra_ratio = 0.0; + do { + ObSEArray approx_part_infos; + for (int64_t i = 0; OB_SUCC(ret) && i < gather_helper.maximum_gather_part_cnt_ && idx_part < param.approx_part_infos_.count(); ++i) { + if (OB_FAIL(approx_part_infos.push_back(param.approx_part_infos_.at(idx_part++)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (gather_helper.maximum_gather_col_cnt_ >= param.column_params_.count()) { + ObSEArray all_tstats; + ObTableStatParam derive_param; + if (OB_FAIL(derive_param.assign(param))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(derive_param.approx_part_infos_.assign(approx_part_infos))) { + LOG_WARN("failed to assign", K(ret)); + } else { + derive_param.allocator_ = &allocator; + if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(current_phase, extra_ratio))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(ObIncrementalStatEstimator::derive_split_gather_stats(ctx, trans, derive_param, true, true, all_tstats))) { + LOG_WARN("failed to derive split gather stats", K(ret)); + } else { + allocator.reuse();//Phased memory release for split gather, in order to reuse memory + } + } + } else { + int64_t idx_col = 0; + ObOptTableStat part_Stat; + ObSEArray all_tstats; + if (OB_FAIL(all_tstats.push_back(&part_Stat))) { + LOG_WARN("faile to push back", K(ret)); + } else { + do { + ObSEArray gather_column_params; + for (int64_t i = 0; OB_SUCC(ret) && i < gather_helper.maximum_gather_col_cnt_ && idx_col < param.column_params_.count(); ++i) { + if (OB_FAIL(gather_column_params.push_back(param.column_params_.at(idx_col++)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + ObTableStatParam derive_param; + if (OB_FAIL(derive_param.assign(param))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(derive_param.approx_part_infos_.assign(approx_part_infos))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(derive_param.column_params_.assign(gather_column_params))) { + LOG_WARN("failed to assign", K(ret)); + } else { + derive_param.allocator_ = &allocator; + if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(current_phase, extra_ratio))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(ObIncrementalStatEstimator::derive_split_gather_stats(ctx, trans, + derive_param, true, + idx_col == param.column_params_.count(), + all_tstats))) { + LOG_WARN("failed to derive split gather stats", K(ret)); + } else { + allocator.reuse();//Phased memory release for split gather, in order to reuse memory + } + } + } + } while(OB_SUCC(ret) && idx_col < param.column_params_.count()); + } + } + } + extra_ratio += split_extra_ratio; + } while (OB_SUCC(ret) && idx_part < param.approx_part_infos_.count()); + } + return ret; +} + +int ObDbmsStatsExecutor::split_gather_global_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const PartitionIdBlockMap *partition_id_block_map, + GatherHelper &gather_helper) +{ + int ret = OB_SUCCESS; + ObOptStatGatherParam gather_param; + ObArenaAllocator allocator("SplitGatherStat", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); + if (OB_UNLIKELY(gather_helper.maximum_gather_col_cnt_ < 1 || + !gather_helper.is_split_gather_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param), K(gather_helper)); + } else if (OB_FAIL(ObDbmsStatsUtils::prepare_gather_stat_param(param, TABLE_LEVEL, partition_id_block_map, true, + gather_helper.gather_vectorize_, gather_param))) { + LOG_WARN("failed to prepare gather stat param", K(ret)); + } else {//need split gather + int64_t idx_col = 0; + ObOptTableStat global_Stat; + ObSEArray all_tstats; + if (OB_FAIL(all_tstats.push_back(&global_Stat))) { + LOG_WARN("faile to push back", K(ret)); + } else if (OB_FAIL(fetch_gather_table_snapshot_read(trans.get_connection(), gather_param.tenant_id_, gather_param.sepcify_scn_))) { + LOG_WARN("failed to fetch gather table snapshot read", K(ret)); + ret = OB_SUCCESS;//if we failed to get the read snapshot, just skip, not specify the snapshot + } + if (OB_SUCC(ret)) { + int64_t total_split_cnt = std::ceil(1.0 * param.column_params_.count() / gather_helper.maximum_gather_col_cnt_); + ObOptStatRunningPhase current_phase = gather_helper.is_approx_gather_ && param.part_level_ != share::schema::PARTITION_LEVEL_ZERO ? + ObOptStatRunningPhase::APPROX_GATHER_GLOBAL_STATS : ObOptStatRunningPhase::GATHER_GLOBAL_STATS; + ObOptStatRunningPhase next_phase = ObOptStatRunningPhase::BACKUP_HISTORY_STATS; + double split_extra_ratio = gather_helper.running_monitor_.get_monitor_extra_progress_ratio(current_phase, next_phase, total_split_cnt); + double extra_ratio = 0.0; + do { + ObSEArray gather_column_params; + for (int64_t i = 0; OB_SUCC(ret) && i < gather_helper.maximum_gather_col_cnt_ && idx_col < param.column_params_.count(); ++i) { + if (OB_FAIL(gather_column_params.push_back(param.column_params_.at(idx_col++)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (gather_helper.is_approx_gather_ && param.part_level_ != share::schema::PARTITION_LEVEL_ZERO) { + ObTableStatParam derive_param; + if (OB_FAIL(derive_param.assign(param))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(derive_param.column_params_.assign(gather_column_params))) { + LOG_WARN("failed to assign", K(ret)); + } else { + derive_param.allocator_ = &allocator; + if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(current_phase, extra_ratio))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(ObIncrementalStatEstimator::derive_split_gather_stats(ctx, trans, + derive_param, + false, + idx_col == param.column_params_.count(), + all_tstats))) { + LOG_WARN("failed to derive split gather stats", K(ret)); + } else { + allocator.reuse();//Phased memory release for split gather, in order to reuse memory + } + } + } else { + ObArray dummy_part_infos; + ObSEArray all_cstats; + ObSEArray opt_stats; + gather_param.allocator_ = &allocator; + if (OB_FAIL(gather_helper.running_monitor_.add_monitor_info(current_phase, extra_ratio))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(do_gather_stats(ctx, trans, gather_param, + dummy_part_infos, + gather_column_params, + idx_col == param.column_params_.count(), + opt_stats, + all_tstats, + all_cstats))) { + if (gather_param.sepcify_scn_ > 0 && + (ret = OB_TABLE_DEFINITION_CHANGED || OB_SNAPSHOT_DISCARDED == ret)) { + LOG_WARN("failed to specify snapshot to gather stats, try no specify snapshot to gather stats", K(ret)); + gather_param.sepcify_scn_ = 0; + allocator.reuse(); + opt_stats.reset(); + all_cstats.reset(); + if (OB_FAIL(do_gather_stats(ctx, trans, gather_param, + dummy_part_infos, + gather_column_params, + idx_col == param.column_params_.count(), + opt_stats, + all_tstats, + all_cstats))) { + LOG_WARN("failed to do gather stats", K(ret)); + } else { + allocator.reuse();//Phased memory release for split gather, in order to reuse memory + } + } else { + LOG_WARN("failed to do gather stats", K(ret)); + } + } else { + allocator.reuse();//Phased memory release for split gather, in order to reuse memory + } + } + } + extra_ratio += split_extra_ratio; + } while(OB_SUCC(ret) && idx_col < param.column_params_.count()); } } return ret; } int ObDbmsStatsExecutor::do_gather_stats(ObExecContext &ctx, - const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &approx_part_opt_stats, - ObIArray &opt_stats) + ObMySQLTransaction &trans, + ObOptStatGatherParam ¶m, + const ObIArray &gather_partition_infos, + const ObIArray &gather_column_params, + bool is_all_columns_gather, + ObIArray &opt_stats, + ObIArray &all_tstats, + ObIArray &all_cstats) { int ret = OB_SUCCESS; - if (OB_ISNULL(param.allocator_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(param.allocator_)); - } else if (OB_FAIL(init_opt_stats(*param.allocator_, param, extra, opt_stats))) { - LOG_WARN("failed to init opt stats", K(ret)); - } else if (opt_stats.empty()) { - /*do nothing*/ - } else { - ObBasicStatsEstimator basic_est(ctx, *param.allocator_); - ObHybridHistEstimator hybrid_est(ctx, *param.allocator_); - if (OB_FAIL(basic_est.estimate(param, extra, opt_stats))) { - LOG_WARN("failed to estimate basic statistics", K(ret)); - } else if (extra.need_histogram_ && OB_FAIL(hybrid_est.estimate(param, extra, opt_stats))) { - LOG_WARN("failed to estimate hybrid histogram", K(ret)); - } else {/*do nothing*/} - } - if (OB_SUCC(ret)) { - if (OB_FAIL(ObIncrementalStatEstimator::try_derive_global_stat(ctx, param, - extra, approx_part_opt_stats, - opt_stats))) { - LOG_WARN("failed to try derive global stat", K(ret)); - } else if (OB_FAIL(check_all_cols_range_skew(param, opt_stats))) { - LOG_WARN("failed to check all cols range skew", K(ret)); + ObSEArray tmp_all_tstats; + if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); + } else if (OB_FAIL(param.partition_infos_.assign(gather_partition_infos))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(param.column_params_.assign(gather_column_params))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(ObDbmsStatsGather::gather_stats(ctx, param, opt_stats))) { + LOG_WARN("failed to gather stats", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::check_all_cols_range_skew(gather_column_params, opt_stats))) { + LOG_WARN("failed to check all cols range skew", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::calssify_opt_stat(opt_stats, + tmp_all_tstats, + all_cstats))) { + LOG_WARN("failed to calssify opt stat", K(ret)); + } else if (param.is_split_gather_) {//avoid memory use too much, write current gather stats. + if (OB_FAIL(ObDbmsStatsUtils::merge_split_gather_tab_stats(all_tstats, tmp_all_tstats))) { + LOG_WARN("failed to merge split gather tab stats", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, trans.get_connection(), + is_all_columns_gather ? all_tstats : tmp_all_tstats, + all_cstats))) { + LOG_WARN("failed to split batch write", K(ret)); } else {/*do nothing*/} + } else if (OB_FAIL(append(all_tstats,tmp_all_tstats))) { + LOG_WARN("failed to append", K(ret)); } return ret; } -int ObDbmsStatsExecutor::check_all_cols_range_skew(const ObTableStatParam ¶m, - ObIArray &opt_stats) +//Get the maximum number of partitions and columns for each stat gather and check need split gather +int ObDbmsStatsExecutor::check_need_split_gather(const ObTableStatParam ¶m, + GatherHelper &gather_helper) { int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < opt_stats.count(); ++i) { - ObIArray &col_stats = opt_stats.at(i).column_stats_; - if (OB_UNLIKELY(param.column_params_.count() != col_stats.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(param.column_params_.count()), - K(col_stats.count()), K(ret)); + int64_t column_cnt = param.column_params_.count(); + int64_t partition_cnt = param.subpart_stat_param_.need_modify_ ? param.subpart_infos_.count() : + (param.part_stat_param_.need_modify_ ? param.part_infos_.count() + param.approx_part_infos_.count() : 1); + bool need_histgoram = param.subpart_stat_param_.need_modify_ ? param.subpart_stat_param_.gather_histogram_ : + (param.part_stat_param_.need_modify_ ? param.part_stat_param_.gather_histogram_ : param.global_stat_param_.gather_histogram_); + partition_cnt = partition_cnt == 0 ? 1 : partition_cnt; + int64_t gather_vectorize = DEFAULT_STAT_GATHER_VECTOR_BATCH_SIZE; + //cache table stat size + int64_t tab_stat_size = sizeof(ObOptTableStat) * partition_cnt; + //cache histogram stat size + int64_t col_histogram_size = need_histgoram ? get_column_histogram_size(param.column_params_) : 0; + //cache column stat size + int64_t col_stat_size = (sizeof(ObOptColumnStat) + ObOptColumnStat::NUM_LLC_BUCKET) * partition_cnt * column_cnt + col_histogram_size * partition_cnt; + //calc stat size + int64_t calc_stat_size = gather_vectorize * 1000 * column_cnt * param.degree_; + //max memory used + int64_t max_memory_used = tab_stat_size + col_stat_size + calc_stat_size; + //get the max work arena size + int64_t max_wa_memory_size = MIN_GATHER_WORK_ARANA_SIZE; + if (OB_FAIL(get_max_work_area_size(param.tenant_id_, max_wa_memory_size))) { + LOG_WARN("failed to get max work area size", K(ret)); + } else if (OB_UNLIKELY(max_wa_memory_size < MIN_GATHER_WORK_ARANA_SIZE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(max_wa_memory_size), K(MIN_GATHER_WORK_ARANA_SIZE)); + } else if (OB_LIKELY(max_memory_used <= max_wa_memory_size)) { + gather_helper.maximum_gather_col_cnt_ = column_cnt; + gather_helper.maximum_gather_part_cnt_ = partition_cnt; + gather_helper.is_split_gather_ = false; + gather_helper.gather_vectorize_ = gather_vectorize; + } else { + //firstly, split according the partition + while (partition_cnt > 1 && max_memory_used > max_wa_memory_size) { + partition_cnt = partition_cnt / 2; + tab_stat_size = sizeof(ObOptTableStat) * partition_cnt; + col_stat_size = (sizeof(ObOptColumnStat) + ObOptColumnStat::NUM_LLC_BUCKET) * partition_cnt * column_cnt + + col_histogram_size * partition_cnt; + max_memory_used = tab_stat_size + col_stat_size + calc_stat_size; + } + if (max_memory_used <= max_wa_memory_size) { + gather_helper.maximum_gather_col_cnt_ = column_cnt; + gather_helper.maximum_gather_part_cnt_ = partition_cnt; + gather_helper.is_split_gather_ = true; + gather_helper.gather_vectorize_ = gather_vectorize; } else { - for (int64_t j = 0; OB_SUCC(ret) && j < param.column_params_.count(); ++j) { - const ObColumnStatParam &col_param = param.column_params_.at(j); - if (col_param.is_size_skewonly() || col_param.is_size_auto()) { - if (OB_ISNULL(col_stats.at(j))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(col_stats.at(j))); - } else { - ObHistogram &hist = col_stats.at(j)->get_histogram(); - if ((hist.get_type() == ObHistType::FREQUENCY && col_param.is_size_skewonly()) || - hist.get_type() == ObHistType::HYBIRD) { - if (OB_UNLIKELY(hist.get_bucket_size() < 1 || col_param.bucket_num_ < 1)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(hist.get_bucket_size()), - K(col_param.bucket_num_), K(*col_stats.at(j))); - } else { - bool is_even_dist = false; - int64_t standard_cnt = hist.get_type() == ObHistType::FREQUENCY ? - hist.get_buckets().at(0).endpoint_num_ : - hist.get_sample_size() / col_param.bucket_num_; - if (OB_FAIL(ObDbmsStatsUtils::check_range_skew(hist.get_type(), - hist.get_buckets(), - standard_cnt, - is_even_dist))) { - LOG_WARN("failed to check range skew", K(ret)); - } else if (is_even_dist) {//Evenly distributed, no need to build a histogram. - LOG_TRACE("check hist range skew is evenly distributed", K(hist.get_type())); - hist.reset(); - } - } - } else {/*do nothing*/} - } - } else {/*do nothing*/} + const int64_t MINIMUM_OF_VECTOR_SIZE = 8; + //secondly, split according the vector size + while (gather_vectorize > MINIMUM_OF_VECTOR_SIZE && max_memory_used > max_wa_memory_size) { + gather_vectorize = gather_vectorize / 2; + calc_stat_size = gather_vectorize * 1000 * column_cnt * param.degree_; + max_memory_used = tab_stat_size + col_stat_size + calc_stat_size; + } + if (max_memory_used <= max_wa_memory_size) { + gather_helper.maximum_gather_col_cnt_ = column_cnt; + gather_helper.maximum_gather_part_cnt_ = partition_cnt; + gather_helper.is_split_gather_ = true; + gather_helper.gather_vectorize_ = gather_vectorize; + } else { + //lastly, split according the column + while (column_cnt > 1 && max_memory_used > max_wa_memory_size) { + column_cnt = column_cnt / 2; + col_histogram_size = col_histogram_size / 2; + col_stat_size = (sizeof(ObOptColumnStat) + ObOptColumnStat::NUM_LLC_BUCKET) * partition_cnt * column_cnt + + col_histogram_size * partition_cnt; + max_memory_used = tab_stat_size + col_stat_size + calc_stat_size; + } + gather_helper.maximum_gather_col_cnt_ = column_cnt; + gather_helper.maximum_gather_part_cnt_ = partition_cnt; + gather_helper.is_split_gather_ = true; + gather_helper.gather_vectorize_ = gather_vectorize; } } } + LOG_TRACE("succeed to get the maximum num of part and column for stat gather", K(param), K(max_memory_used), + K(max_wa_memory_size), K(tab_stat_size), K(col_histogram_size), + K(col_stat_size), K(calc_stat_size), K(gather_helper)); + if (gather_helper.is_split_gather_) { + LOG_INFO("stat gather will use split gather", K(param.degree_), K(max_memory_used), + K(max_wa_memory_size), K(tab_stat_size), + K(col_histogram_size), K(col_stat_size), + K(calc_stat_size), K(gather_helper)); + } return ret; } +int ObDbmsStatsExecutor::get_max_work_area_size(uint64_t tenant_id, int64_t &max_wa_memory_size) +{ + int ret = OB_SUCCESS; + max_wa_memory_size = 0; + const ObTenantBase *tenant = NULL; + if (OB_ISNULL(tenant = MTL_CTX())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + int64_t worker_cnt = std::max(static_cast(tenant)->min_worker_cnt(), 4L); + max_wa_memory_size = lib::get_tenant_memory_limit(tenant_id) / worker_cnt; + if (lib::ObMallocAllocator::get_instance() != NULL) { + ObTenantCtxAllocatorGuard ta = lib::ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(tenant_id, common::ObCtxIds::WORK_AREA); + max_wa_memory_size = ta->get_limit() / worker_cnt; + } + max_wa_memory_size = std::max(MIN_GATHER_WORK_ARANA_SIZE, max_wa_memory_size); + } + return ret; +} + +int64_t ObDbmsStatsExecutor::get_column_histogram_size(const ObIArray &column_params) +{ + int64_t histogram_size = 0; + for (int64_t i = 0; i < column_params.count(); ++i) { + if (column_params.at(i).need_basic_stat() && column_params.at(i).bucket_num_ > 1) { + histogram_size += (sizeof(ObObj) + OPT_STATS_MAX_VALUE_CHAR_LEN) * column_params.at(i).bucket_num_; + } + } + return histogram_size; +} + /** * @brief ObDbmsStatsUtils::set_table_stats * set table stats @@ -229,7 +790,7 @@ int ObDbmsStatsExecutor::set_table_stats(ObExecContext &ctx, { int ret = OB_SUCCESS; UNUSED(ctx); - ObArenaAllocator alloc(ObModIds::OB_BUFFER); + ObArenaAllocator alloc("ObSetTableStats", OB_MALLOC_NORMAL_BLOCK_SIZE, param.table_param_.tenant_id_); ObOptTableStat table_stat; ObOptStatManager &mgr = ObOptStatManager::get_instance(); int64_t partition_id = param.table_param_.global_part_id_; @@ -255,27 +816,34 @@ int ObDbmsStatsExecutor::set_table_stats(ObExecContext &ctx, table_stat.set_last_analyzed(0); } if (OB_SUCC(ret)) { - ObSEArray history_tab_handles; - ObSEArray history_col_handles; - if (OB_FAIL(do_set_table_stats(param, &table_stat))) { + ObMySQLTransaction trans; + //begin trans + if (OB_FAIL(trans.start(ctx.get_sql_proxy(), param.table_param_.tenant_id_))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(do_set_table_stats(param, &table_stat))) { LOG_WARN("failed to do set table stats", K(ret)); ////before update, we need record history stats. } else if (!param.table_param_.is_temp_table_ && - OB_FAIL(ObDbmsStatsHistoryManager::get_history_stat_handles(ctx, param.table_param_, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to get history stat handles", K(ret)); + OB_FAIL(ObDbmsStatsHistoryManager::backup_opt_stats(ctx, trans, param.table_param_, ObTimeUtility::current_time()))) { + LOG_WARN("failed to backup opt stats", K(ret)); } else if (OB_FAIL(mgr.update_table_stat(param.table_param_.tenant_id_, + trans.get_connection(), &table_stat, param.table_param_.is_index_stat_))) { LOG_WARN("failed to update table stats", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::batch_write_history_stats(ctx, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to batch write history stats", K(ret)); } else { LOG_TRACE("end set table stats", K(param), K(table_stat)); } + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } } return ret; } @@ -343,7 +911,7 @@ int ObDbmsStatsExecutor::set_column_stats(ObExecContext &ctx, LOG_WARN("fail to start transaction", K(ret)); } else if (OB_FAIL(mgr.update_column_stat(ctx.get_virtual_table_ctx().schema_guard_, param.table_param_.tenant_id_, - trans, + trans.get_connection(), column_stats, true, CREATE_OBJ_PRINT_PARAM(ctx.get_my_session())))) { @@ -455,17 +1023,11 @@ int ObDbmsStatsExecutor::delete_table_stats(ObExecContext &ctx, ObSEArray part_stattypes; uint64_t table_id = param.table_id_; if (param.global_stat_param_.need_modify_) { - PartInfo part_info; - ObExtraParam extra; - extra.type_ = TABLE_LEVEL; - extra.nth_part_ = 0; - if (OB_FAIL(ObDbmsStatsUtils::get_part_info(param, extra, part_info))) { - LOG_WARN("failed to get part info", K(ret)); - } else if (OB_FAIL(part_ids.push_back(part_info.part_id_))) { + if (OB_FAIL(part_ids.push_back(param.global_part_id_))) { LOG_WARN("failed to push back partition id", K(ret)); } else if (param.stattype_ == NULL_TYPE) { /*do nothing*/ - } else if (OB_FAIL(no_stats_partition_ids.push_back(part_info.part_id_))) { + } else if (OB_FAIL(no_stats_partition_ids.push_back(param.global_part_id_))) { LOG_WARN("failed to push back", K(ret)); } else if (OB_FAIL(part_stattypes.push_back(param.stattype_))) { LOG_WARN("failed to push back", K(ret)); @@ -498,29 +1060,35 @@ int ObDbmsStatsExecutor::delete_table_stats(ObExecContext &ctx, } } if (OB_SUCC(ret)) { - ObSEArray history_tab_handles; - ObSEArray history_col_handles; int64_t affected_rows = 0; //before delete, we need record history stats. - if (!param.is_temp_table_ && - OB_FAIL(ObDbmsStatsHistoryManager::get_history_stat_handles(ctx, param, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to get history stat handles", K(ret)); + ObMySQLTransaction trans; + //begin trans + if (OB_FAIL(trans.start(ctx.get_sql_proxy(), param.tenant_id_))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (!param.is_temp_table_ && + OB_FAIL(ObDbmsStatsHistoryManager::backup_opt_stats(ctx, trans, param, ObTimeUtility::current_time()))) { + LOG_WARN("failed to backup opt stats", K(ret)); } else if (OB_FAIL(ObOptStatManager::get_instance().delete_table_stat(param.tenant_id_, table_id, part_ids, cascade_columns, + param.degree_, affected_rows))) { LOG_WARN("failed to delete table stats", K(ret)); - } else if (affected_rows != 0 && !param.is_temp_table_ && - OB_FAIL(ObDbmsStatsUtils::batch_write_history_stats(ctx, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to batch write history stats", K(ret)); } else if (OB_FAIL(reset_table_locked_state(ctx, param, no_stats_partition_ids, part_stattypes))) { LOG_WARN("failed to reset table locked state", K(ret)); - } else {/*do nothing*/} + } + if (OB_SUCC(ret) && affected_rows != 0) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } } return ret; } @@ -544,13 +1112,7 @@ int ObDbmsStatsExecutor::delete_column_stats(ObExecContext &ctx, } else {/*do nothing*/} } if (OB_SUCC(ret) && param.global_stat_param_.need_modify_) { - PartInfo part_info; - ObExtraParam extra; - extra.type_ = TABLE_LEVEL; - extra.nth_part_ = 0; - if (OB_FAIL(ObDbmsStatsUtils::get_part_info(param, extra, part_info))) { - LOG_WARN("failed to get part info", K(ret)); - } else if (OB_FAIL(part_ids.push_back(part_info.part_id_))) { + if (OB_FAIL(part_ids.push_back(param.global_part_id_))) { LOG_WARN("failed to push back partition id", K(ret)); } else {/*do nothing*/} } @@ -574,120 +1136,14 @@ int ObDbmsStatsExecutor::delete_column_stats(ObExecContext &ctx, table_id, column_ids, part_ids, - only_histogram))) { + only_histogram, + param.degree_))) { LOG_WARN("failed to delete column stat", K(ret)); } } return ret; } -int ObDbmsStatsExecutor::init_opt_stats(ObIAllocator &allocator, - const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &opt_stats) -{ - int ret = OB_SUCCESS; - if (TABLE_LEVEL == extra.type_) { - ObOptStat stat; - if (OB_FAIL(init_opt_stat(allocator, param, extra, stat))) { - LOG_WARN("failed to init opt stat"); - } else if (OB_FAIL(opt_stats.push_back(stat))) { - LOG_WARN("failed to push back stat"); - } else {/*do nothing*/} - } else if (PARTITION_LEVEL == extra.type_) { - for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { - extra.nth_part_ = i; - ObOptStat stat; - if (OB_FAIL(init_opt_stat(allocator, param, extra, stat))) { - LOG_WARN("failed to init opt stat"); - } else if (OB_FAIL(opt_stats.push_back(stat))) { - LOG_WARN("failed to push back stat"); - } else {/*do nothing*/} - } - } else if (SUBPARTITION_LEVEL == extra.type_) { - for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { - extra.nth_part_ = i; - ObOptStat stat; - if (OB_FAIL(init_opt_stat(allocator, param, extra, stat))) { - LOG_WARN("failed to init opt stat"); - } else if (OB_FAIL(opt_stats.push_back(stat))) { - LOG_WARN("failed to push back stat"); - } else {/*do nothing*/} - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected type", K(ret), K(extra.type_)); - } - return ret; -} - -int ObDbmsStatsExecutor::init_opt_stat(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra, - ObOptStat &stat) -{ - int ret = OB_SUCCESS; - void *ptr = NULL; - PartInfo part_info; - BlockNumStat *block_num_stat = NULL; - ObOptTableStat *&tab_stat = stat.table_stat_; - if (OB_FAIL(stat.column_stats_.prepare_allocate(param.column_params_.count()))) { - LOG_WARN("failed to prepare allocate column stat", K(ret)); - } else if (OB_ISNULL(ptr = allocator.alloc(sizeof(ObOptTableStat)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(ptr)); - } else if (OB_FAIL(ObDbmsStatsUtils::get_part_info(param, extra, part_info))) { - LOG_WARN("failed to get partition info", K(ret)); - } else { - LOG_TRACE("prepare opt stat", K(part_info), K(extra.type_), K(param.part_level_)); - tab_stat = new (ptr) ObOptTableStat(); - tab_stat->set_table_id(param.table_id_); - tab_stat->set_partition_id(part_info.part_id_); - tab_stat->set_object_type(extra.type_); - tab_stat->set_stattype_locked(part_info.part_stattype_); - if (OB_FAIL(extra.partition_id_block_map_.get_refactored(part_info.part_id_, block_num_stat))) { - if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) { - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to get refactored", K(ret)); - } - } else if (OB_ISNULL(block_num_stat)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(block_num_stat)); - } else { - tab_stat->set_macro_block_num(block_num_stat->tab_macro_cnt_); - tab_stat->set_micro_block_num(block_num_stat->tab_micro_cnt_); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) { - ObOptColumnStat *&col_stat = stat.column_stats_.at(i); - if (OB_ISNULL(col_stat = ObOptColumnStat::malloc_new_column_stat(allocator))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(ptr)); - } else { - col_stat->set_table_id(param.column_params_.at(i).need_basic_stat() ? param.table_id_: -1); - col_stat->set_partition_id(part_info.part_id_); - col_stat->set_stat_level(extra.type_); - col_stat->set_column_id(param.column_params_.at(i).column_id_); - col_stat->set_collation_type(param.column_params_.at(i).cs_type_); - if (block_num_stat != NULL) { - if (OB_LIKELY(param.column_group_params_.count() == block_num_stat->cg_macro_cnt_arr_.count() && - param.column_group_params_.count() == block_num_stat->cg_micro_cnt_arr_.count())) { - bool found_it = false; - for (int64_t j = 0; !found_it && j < param.column_group_params_.count(); ++j) { - if (param.column_group_params_.at(j).column_id_arr_.count() == 1 && - param.column_group_params_.at(j).column_id_arr_.at(0) == param.column_params_.at(i).column_id_) { - col_stat->set_cg_macro_blk_cnt(block_num_stat->cg_macro_cnt_arr_.at(j)); - col_stat->set_cg_micro_blk_cnt(block_num_stat->cg_micro_cnt_arr_.at(j)); - } - } - } - } - } - } - return ret; -} - ObOptStat::~ObOptStat() { if (NULL != table_stat_) { @@ -734,42 +1190,75 @@ int ObDbmsStatsExecutor::gather_index_stats(ObExecContext &ctx, { int ret = OB_SUCCESS; ObArray all_index_stats; + ObArray part_index_stats; ObArray empty_cstats; - ObExtraParam extra; - extra.start_time_ = ObTimeUtility::current_time(); + ObOptStatGatherParam gather_param; + PartitionIdBlockMap partition_id_block_map; LOG_TRACE("begin gather index stats", K(param)); - if (OB_FAIL(extra.partition_id_block_map_.create(10000, - ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, - ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, - param.tenant_id_))) { + if (OB_FAIL(partition_id_block_map.create(10000, + ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, + ObModIds::OB_HASH_BUCKET_TABLE_STATISTICS, + param.tenant_id_))) { LOG_WARN("failed to create hash map", K(ret)); } else if (OB_FAIL(ObBasicStatsEstimator::estimate_block_count(ctx, param, - extra.partition_id_block_map_))) { + partition_id_block_map))) { LOG_WARN("failed to estimate block count", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::prepare_gather_stat_param(param, INVALID_LEVEL, &partition_id_block_map, + false, DEFAULT_STAT_GATHER_VECTOR_BATCH_SIZE, gather_param))) { + LOG_WARN("failed to prepare gather stat param", K(ret)); + } else if (OB_FAIL(gather_param.column_params_.assign(param.column_params_))) { + LOG_WARN("failed to assign", K(ret)); } if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) { - extra.type_ = SUBPARTITION_LEVEL; - extra.nth_part_ = 0; + gather_param.stat_level_ = SUBPARTITION_LEVEL; if (param.part_level_ != share::schema::PARTITION_LEVEL_TWO) { /*do nothing*/ - } else if (OB_FAIL(do_gather_index_stats(ctx, param, extra, all_index_stats))) { - LOG_WARN("failed to gather subpartition stats", K(ret)); + } else if (OB_FAIL(gather_param.partition_infos_.assign(param.subpart_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(ObDbmsStatsGather::gather_index_stats(ctx, gather_param, all_index_stats))) { + LOG_WARN("failed to gather subpart index stats", K(ret)); } else {/*do nothing*/} } if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) { - extra.type_ = PARTITION_LEVEL; - extra.nth_part_ = 0; + gather_param.stat_level_ = PARTITION_LEVEL; if (param.part_level_ == share::schema::PARTITION_LEVEL_ZERO) { /*do nothing*/ - } else if (OB_FAIL(do_gather_index_stats(ctx, param, extra, all_index_stats))) { - LOG_WARN("failed to gather partition stats", K(ret)); - } else {/*do nothing*/} + } else if (param.part_stat_param_.can_use_approx_ && !param.approx_part_infos_.empty()) {//approx some part stats base on subpart stats + if (OB_UNLIKELY(all_index_stats.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param.approx_part_infos_), K(all_index_stats)); + } else if (OB_FAIL(ObIncrementalStatEstimator::derive_part_index_stat_by_subpart_index_stats(param, + all_index_stats, + part_index_stats))) { + LOG_WARN("failed to derive part index stat by subpart index stats", K(ret)); + } else if (OB_FAIL(append(all_index_stats, part_index_stats))) { + LOG_WARN("failed to append", K(ret)); + } + } else if (OB_UNLIKELY(!param.approx_part_infos_.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param)); + } + if (OB_SUCC(ret) && !param.part_infos_.empty()) { + if (OB_FAIL(gather_param.partition_infos_.assign(param.part_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(ObDbmsStatsGather::gather_index_stats(ctx, gather_param, part_index_stats))) { + LOG_WARN("failed to gather part index stats", K(ret)); + } else if (OB_FAIL(append(all_index_stats, part_index_stats))) { + LOG_WARN("failed to append", K(ret)); + } else {/*do nothing*/} + } } if (OB_SUCC(ret) && (param.global_stat_param_.need_modify_)) { - extra.type_ = TABLE_LEVEL; - extra.nth_part_ = 0; - if (OB_FAIL(do_gather_index_stats(ctx, param, extra, all_index_stats))) { - LOG_WARN("failed to gather table stats", K(ret)); + gather_param.stat_level_ = TABLE_LEVEL; + gather_param.partition_infos_.reset(); + if (param.global_stat_param_.gather_approx_ && !part_index_stats.empty()) {//approx global stats base on part stats + if (OB_FAIL(ObIncrementalStatEstimator::derive_global_index_stat_by_part_index_stats(param, + part_index_stats, + all_index_stats))) { + LOG_WARN("failed to derive global index stat by part index stats", K(ret)); + } + } else if (OB_FAIL(ObDbmsStatsGather::gather_index_stats(ctx, gather_param, all_index_stats))) { + LOG_WARN("failed to gather index stats", K(ret)); } } if (OB_SUCC(ret)) { @@ -778,9 +1267,9 @@ int ObDbmsStatsExecutor::gather_index_stats(ObExecContext &ctx, } else {/*do nothing*/} } - if (extra.partition_id_block_map_.created()) { + if (partition_id_block_map.created()) { int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = extra.partition_id_block_map_.destroy())) { + if (OB_SUCCESS != (tmp_ret = partition_id_block_map.destroy())) { LOG_WARN("failed to destroy hash map", K(ret), K(tmp_ret)); ret = COVER_SUCC(tmp_ret); } @@ -788,38 +1277,6 @@ int ObDbmsStatsExecutor::gather_index_stats(ObExecContext &ctx, return ret; } -int ObDbmsStatsExecutor::do_gather_index_stats(ObExecContext &ctx, - const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &all_index_stats) -{ - int ret = OB_SUCCESS; - ObSEArray opt_stats; - if (OB_ISNULL(param.allocator_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(param.allocator_)); - } else if (OB_FAIL(init_opt_stats(*param.allocator_, param, extra, opt_stats))) { - LOG_WARN("failed to init opt stats", K(ret)); - } else if (opt_stats.empty()) { - /*do nothing*/ - } else { - ObIndexStatsEstimator index_est(ctx, *param.allocator_); - if (OB_FAIL(index_est.estimate(param, extra, opt_stats))) { - LOG_WARN("failed to estimate basic statistics", K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < opt_stats.count(); ++i) { - if (OB_ISNULL(opt_stats.at(i).table_stat_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(opt_stats.at(i).table_stat_)); - } else if (OB_FAIL(all_index_stats.push_back(opt_stats.at(i).table_stat_))) { - LOG_WARN("failed to append", K(ret)); - } else {/*do nothing*/} - } - } - } - return ret; -} - int ObDbmsStatsExecutor::update_online_stat(ObExecContext &ctx, ObTableStatParam ¶m, share::schema::ObSchemaGetterGuard *schema_guard, @@ -827,12 +1284,12 @@ int ObDbmsStatsExecutor::update_online_stat(ObExecContext &ctx, const ColStatIndMap &online_column_stats) { int ret = OB_SUCCESS; - - int64_t affected_rows = 0; - ObSEArray cur_tab_handles; - ObSEArray cur_col_handles; + ObArenaAllocator allocator("ObOnlineStat", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); + ObSEArray cur_table_stats; + ObSEArray cur_column_stats; ObSEArray table_stats; ObSEArray column_stats; + bool succ_to_write_stats = false; if (OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, param))) { if (ret == OB_ERR_DBMS_STATS_PL) { param.global_stat_param_.reset_gather_stat(); @@ -842,32 +1299,385 @@ int ObDbmsStatsExecutor::update_online_stat(ObExecContext &ctx, } LOG_WARN("fail to check lock stat", K(ret)); } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(param, - cur_tab_handles, - cur_col_handles))) { - LOG_WARN("failed to get current opt stats", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::merge_tab_stats(param, - online_table_stats, - cur_tab_handles, - table_stats))) { - LOG_WARN("fail to merge tab stats", K(ret), K(cur_tab_handles)); - } else if (OB_FAIL(ObDbmsStatsUtils::merge_col_stats(param, - online_column_stats, - cur_col_handles, - column_stats))) { - LOG_WARN("fail to merge col stats", K(ret), K(cur_col_handles)); - } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, table_stats, column_stats, - false, false, true))) { - LOG_WARN("fail to update stat", K(ret), K(table_stats), K(column_stats)); - } else if (OB_FAIL(ObBasicStatsEstimator::update_last_modified_count(ctx, param))) { - LOG_WARN("failed to update last modified count", K(ret)); - } else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), param))) { - LOG_WARN("fail to update stat cache", K(ret)); - } else { - // should reuse stats out-side this function. + if (OB_SUCC(ret)) { + SMART_VAR(sql::ObSQLSessionInfo::StmtSavedValue, saved_value) { + int64_t nested_count = -1; + ObSqlString old_db_name; + int64_t old_db_id = -1; + int64_t old_trx_lock_timeout = -1; + bool need_restore_session = false; + bool need_reset_default_database = false; + bool need_reset_trx_lock_timeout = false; + common::sqlclient::ObISQLConnection *conn = NULL; + ctx.set_is_online_stats_gathering(true); + //lib::CompatModeGuard guard(lib::Worker::CompatMode::MYSQL); + if (OB_FAIL(prepare_conn_and_store_session_for_online_stats(ctx.get_my_session(), + ctx.get_sql_proxy(), + schema_guard, + saved_value, + nested_count, + old_db_name, + old_db_id, + old_trx_lock_timeout, + need_restore_session, + need_reset_default_database, + need_reset_trx_lock_timeout, + conn))) { + LOG_WARN("failed to prepare conn and store session for online stats", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(allocator, + conn, + param, + cur_table_stats, + cur_column_stats))) { + LOG_WARN("failed to get current opt stats", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::merge_tab_stats(param, + online_table_stats, + cur_table_stats, + table_stats))) { + LOG_WARN("fail to merge tab stats", K(ret), K(cur_table_stats)); + } else if (OB_FAIL(ObDbmsStatsUtils::merge_col_stats(param, + online_column_stats, + cur_column_stats, + column_stats))) { + LOG_WARN("fail to merge col stats", K(ret), K(cur_column_stats)); + } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, conn, table_stats, column_stats, false, true))) { + LOG_WARN("fail to update stat", K(ret), K(table_stats), K(column_stats)); + } else if (OB_FAIL(ObBasicStatsEstimator::update_last_modified_count(conn, param))) { + LOG_WARN("failed to update last modified count", K(ret)); + } else { + succ_to_write_stats = true; + } + if (ret == OB_ERR_EXCLUSIVE_LOCK_CONFLICT || ret == OB_ERR_SHARED_LOCK_CONFLICT) { + ret = OB_SUCCESS; + LOG_INFO("update online stats occur lock conflict, just skip"); + } + //release source + //guard.~CompatModeGuard(); + ctx.set_is_online_stats_gathering(false); + if (OB_NOT_NULL(conn)) { + int tmp_ret = OB_SUCCESS; + ctx.get_sql_proxy()->close(conn, tmp_ret); + ret = COVER_SUCC(tmp_ret); + } + //restore session + if (need_restore_session) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = restore_session_for_online_stat(ctx.get_my_session(), + saved_value, + nested_count, + old_db_name, + old_db_id, + old_trx_lock_timeout, + need_reset_default_database, + need_reset_trx_lock_timeout))) { + ret = COVER_SUCC(tmp_ret); + LOG_WARN("failed to restore session", K(tmp_ret)); + } + } + //update stat cache + if (succ_to_write_stats) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = pl::ObDbmsStats::update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), param))) { + ret = COVER_SUCC(tmp_ret); + LOG_WARN("fail to update stat cache", K(tmp_ret)); + } + } + } } + return ret; +} +int ObDbmsStatsExecutor::prepare_conn_and_store_session_for_online_stats(sql::ObSQLSessionInfo *session, + common::ObMySQLProxy *sql_proxy, + share::schema::ObSchemaGetterGuard *schema_guard, + sql::ObSQLSessionInfo::StmtSavedValue &saved_value, + int64_t &nested_count, + ObSqlString &old_db_name, + int64_t &old_db_id, + int64_t &old_trx_lock_timeout, + bool &need_restore_session, + bool &need_reset_default_database, + bool &need_reset_trx_lock_timeout, + sqlclient::ObISQLConnection *&conn) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(session) || OB_ISNULL(sql_proxy) || OB_ISNULL(schema_guard)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(session), K(sql_proxy)); + //1.save session info + } else if (OB_FAIL(session->save_session(saved_value))) { + LOG_WARN("failed to saved value", K(ret)); + } else { + need_restore_session = true; + nested_count = session->get_nested_count(); + //2.modify seesion info + //2.1 modify query start time + session->set_query_start_time(ObTimeUtility::current_time()); + session->set_inner_session(); + session->set_nested_count(-1); + if (OB_FAIL(old_db_name.append(session->get_database_name()))) { + LOG_WARN("failed to append", K(ret)); + } else { + old_db_id = session->get_database_id(); + //2.2 modify seesion compatible mode + ObObj mysql_mode; + mysql_mode.set_int(0); + if (OB_FAIL(session->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, mysql_mode))) { + LOG_WARN("failed to update sys variable for compatibility mode", K(ret)); + } else { + //2.3.modify session database name and database id + const share::schema::ObDatabaseSchema *db_schema = NULL; + if (OB_FAIL(schema_guard->get_database_schema(MTL_ID(), OB_SYS_DATABASE_ID, db_schema))) { + LOG_WARN("failed to get database schema", K(ret)); + } else if (OB_ISNULL(db_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(db_schema)); + } else if (OB_FAIL(session->set_default_database(db_schema->get_database_name_str()))) { + LOG_WARN("faile to set default database", K(ret)); + } else { + session->set_database_id(db_schema->get_database_id()); + need_reset_default_database = true; + //2.3 modify session trx lock timeout + old_trx_lock_timeout = session->get_trx_lock_timeout(); + ObObj trx_lock_timeout; + trx_lock_timeout.set_int(0); + if (OB_FAIL(session->update_sys_variable(share::SYS_VAR_OB_TRX_LOCK_TIMEOUT, trx_lock_timeout))) { + LOG_WARN("failed to update sys variable for trx lock timeout", K(ret)); + } else { + need_reset_trx_lock_timeout = true; + //3.get conn to update stats + observer::ObInnerSQLConnectionPool *pool = NULL; + if (OB_ISNULL(pool = static_cast(sql_proxy->get_pool()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(pool)); + } else if (OB_FAIL(pool->acquire(session, conn))) { + LOG_WARN("failed to acquire conn", K(ret)); + } + } + } + } + } + } + return ret; +} + +int ObDbmsStatsExecutor::restore_session_for_online_stat(sql::ObSQLSessionInfo *session, + sql::ObSQLSessionInfo::StmtSavedValue &saved_value, + int64_t nested_count, + ObSqlString &old_db_name, + int64_t old_db_id, + int64_t old_trx_lock_timeout, + bool need_reset_default_database, + bool need_reset_trx_lock_timeout) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(session)); + } else { + //1.restore sesssion + if (OB_FAIL(session->restore_session(saved_value))) { + LOG_WARN("failed to restore session", K(ret), K(session)); + } else { + session->set_nested_count(nested_count); + } + //2.restore seesion compatible oracle mode + if (lib::is_oracle_mode()) { + ObObj oracle_mode; + oracle_mode.set_int(1); + int tmp_ret = session->update_sys_variable(share::SYS_VAR_OB_COMPATIBILITY_MODE, oracle_mode); + if (tmp_ret != OB_SUCCESS) { + ret = COVER_SUCC(tmp_ret); + LOG_WARN("failed to update sys variable for compatibility mode", K(tmp_ret)); + } + } + //3.restore session database name and database id + if (need_reset_default_database) { + int tmp_ret = session->set_default_database(old_db_name.string()); + if (tmp_ret != OB_SUCCESS) { + ret = COVER_SUCC(tmp_ret); + LOG_WARN("failed to reset default database", K(tmp_ret), K(old_db_name)); + } else { + session->set_database_id(old_db_id); + } + } + //4.restore trx lock timeout + if (need_reset_trx_lock_timeout) { + ObObj trx_lock_timeout; + trx_lock_timeout.set_int(old_trx_lock_timeout); + int tmp_ret = session->update_sys_variable(share::SYS_VAR_OB_TRX_LOCK_TIMEOUT, trx_lock_timeout); + if (tmp_ret != OB_SUCCESS) { + ret = COVER_SUCC(tmp_ret); + LOG_WARN("failed to update sys variable for trx lock timeout", K(tmp_ret)); + } + } + } + return ret; +} + +int ObDbmsStatsExecutor::fetch_gather_table_snapshot_read(common::sqlclient::ObISQLConnection *conn, + uint64_t tenant_id, + uint64_t ¤t_scn) +{ + int ret = OB_SUCCESS; + const char *sql_str_fmt = "SELECT current_scn() FROM dual"; + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + sqlclient::ObMySQLResult *result = NULL; + if (OB_ISNULL(conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(conn)); + } else if (OB_FAIL(conn->execute_read(tenant_id, sql_str_fmt, res))) { + LOG_WARN("faield to exec read", K(ret)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + ObObj value; + int64_t col_idx = 0; + if (OB_FAIL(result->next())) { + LOG_WARN("failed to get next row", K(ret)); + } else if (OB_FAIL(result->get_obj(col_idx, value))) { + LOG_WARN("get obj failed", K(ret)); + } else if (value.is_number()) { + number::ObNumber scn_num; + if (OB_FAIL(value.get_number(scn_num))) { + LOG_WARN("get number failed", K(ret)); + } else if (OB_UNLIKELY(!scn_num.is_valid_uint64(current_scn))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected value value", K(ret), K(value)); + } + } else if (OB_LIKELY(value.is_uint64())) { + current_scn = value.get_uint64(); + } else if (value.is_int()) { + current_scn = value.get_int(); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected value type", K(ret), K(value)); + } + } + } + LOG_TRACE("succeed to fetch gather table snapshot read", K(current_scn)); + return ret; +} + +int ObDbmsStatsExecutor::cancel_gather_stats(ObExecContext &ctx, ObString &task_id) +{ + int ret = OB_SUCCESS; + ObSQLSessionInfo *session = ctx.get_my_session(); + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(session)); + } else { + const common::ObAddr &local_addr = ctx.get_addr(); + uint64_t tenant_id = session->get_effective_tenant_id(); + char *task_ip = NULL; + int32_t task_port = 0; + ObArenaAllocator allocator("CancelGather", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); + if (OB_FAIL(fetch_gather_task_addr(ctx.get_sql_proxy(), allocator, tenant_id, + task_id, task_ip, task_port))) { + LOG_WARN("failed to fetch gather task addr", K(ret)); + } else { + common::ObAddr rpc_addr(static_cast(local_addr.get_version()), task_ip, task_port); + if (local_addr == rpc_addr) {//local + if (OB_FAIL(ObOptStatGatherStatList::instance().cancel_gather_stats(tenant_id, + task_id))) { + LOG_WARN("failed to cancel gather stats", K(ret)); + } else {/*do nothing*/} + } else {//remote + int64_t timeout = std::min(10000000L, THIS_WORKER.get_timeout_remain()); + obrpc::ObCancelGatherStatsArg arg; + arg.tenant_id_ = tenant_id; + arg.task_id_ = task_id; + if (OB_UNLIKELY(0 >= timeout)) { + ret = OB_TIMEOUT; + LOG_WARN("query timeout is reached", K(ret), K(timeout)); + } else if (OB_FAIL(GCTX.srv_rpc_proxy_->to(rpc_addr) + .timeout(timeout) + .by(tenant_id) + .cancel_gather_stats(arg))) { + LOG_WARN("failed to cancel gather stats", K(ret), K(rpc_addr), K(arg)); + } else {/*do nothing*/} + } + } + } + return ret; +} + +int ObDbmsStatsExecutor::fetch_gather_task_addr(ObCommonSqlProxy *sql_proxy, + ObIAllocator &allcoator, + uint64_t tenant_id, + const ObString &task_id, + char *&svr_ip, + int32_t &svr_port) +{ + int ret = OB_SUCCESS; + ObSqlString raw_sql; + if (OB_FAIL(raw_sql.append_fmt("SELECT svr_ip, svr_port FROM %s WHERE task_id = \'%.*s\'", + share::OB_ALL_VIRTUAL_OPT_STAT_GATHER_MONITOR_TNAME, + task_id.length(), + task_id.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, raw_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(raw_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + bool got_result = false; + while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { + if (OB_UNLIKELY(got_result)) {//only one rows + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(task_id), K(task_id), K(svr_port)); + } else { + got_result = true; + int64_t idx1 = 0; + int64_t idx2 = 1; + ObObj obj; + ObString str; + int64_t tmp_val; + if (OB_FAIL(client_result->get_obj(idx1, obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(obj.get_string(str))) { + LOG_WARN("failed to get int", K(ret), K(obj)); + } else if (OB_FAIL(client_result->get_obj(idx2, obj))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(obj.get_int(tmp_val))) { + LOG_WARN("failed to get int", K(ret), K(obj)); + } else { + svr_port = static_cast(tmp_val); + } + if (OB_SUCC(ret) && !str.empty()) { + if (OB_ISNULL(svr_ip = static_cast(allcoator.alloc(str.length() + 1)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for saved session value", K(ret), K(svr_ip)); + } else { + MEMSET(svr_ip, 0, str.length() + 1); + MEMCPY(svr_ip, str.ptr(), str.length()); + LOG_TRACE("succeed to fetch gather task addr", K(str), K(svr_port)); + } + } + } + } + ret = OB_ITER_END == ret ? OB_SUCCESS : ret; + if (OB_SUCC(ret) && !got_result) {//invalid task id + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("The optimizer stats gather task has ended or the task doesn't exist", K(ret), K(task_id)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "The optimizer stats gather task has ended or the task doesn't exist"); + } + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + } return ret; } diff --git a/src/share/stat/ob_dbms_stats_executor.h b/src/share/stat/ob_dbms_stats_executor.h index eb13a48794..2be9eda5a6 100644 --- a/src/share/stat/ob_dbms_stats_executor.h +++ b/src/share/stat/ob_dbms_stats_executor.h @@ -18,16 +18,46 @@ #include "share/stat/ob_stat_item.h" namespace oceanbase { +namespace observer +{ +class ObInnerSQLConnection; +} using namespace sql; namespace common { +struct ObOptStatRunningMonitor; +struct GatherHelper +{ + explicit GatherHelper(ObOptStatRunningMonitor &running_monitor) : + is_split_gather_(false), + maximum_gather_part_cnt_(1), + maximum_gather_col_cnt_(1), + is_approx_gather_(false), + gather_vectorize_(DEFAULT_STAT_GATHER_VECTOR_BATCH_SIZE), + running_monitor_(running_monitor) + {} + bool is_split_gather_; + int64_t maximum_gather_part_cnt_; + int64_t maximum_gather_col_cnt_; + bool is_approx_gather_; + int64_t gather_vectorize_; + ObOptStatRunningMonitor &running_monitor_; + TO_STRING_KV(K(is_split_gather_), + K(maximum_gather_part_cnt_), + K(maximum_gather_col_cnt_), + K(is_approx_gather_), + K(gather_vectorize_), + K(running_monitor_)); +}; + class ObDbmsStatsExecutor { public: ObDbmsStatsExecutor(); static int gather_table_stats(ObExecContext &ctx, - const ObTableStatParam ¶m); + const ObTableStatParam ¶m, + ObOptStatRunningMonitor &running_monitor); static int gather_index_stats(ObExecContext &ctx, const ObTableStatParam ¶m); @@ -52,6 +82,8 @@ public: const TabStatIndMap &online_table_stats, const ColStatIndMap &online_column_stats); + static int cancel_gather_stats(ObExecContext &ctx, ObString &task_id); + static int gather_system_stats(ObExecContext &ctx, int64_t tenant_id); static int delete_system_stats(ObExecContext &ctx, int64_t tenant_id); @@ -60,11 +92,46 @@ public: private: + static int prepare_gather_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + PartitionIdBlockMap &partition_id_block_map, + GatherHelper &gather_helper); + + static int split_gather_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const PartitionIdBlockMap *partition_id_block_map, + GatherHelper &gather_helper); + + static int no_split_gather_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const PartitionIdBlockMap *partition_id_block_map, + GatherHelper &gather_helper); + + static int split_gather_partition_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + StatLevel stat_level, + const PartitionIdBlockMap *partition_id_block_map, + const GatherHelper &gather_helper); + + static int split_gather_global_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const PartitionIdBlockMap *partition_id_block_map, + GatherHelper &gather_helper); + static int do_gather_stats(ObExecContext &ctx, - const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &approx_part_opt_stats, - ObIArray &opt_stats); + ObMySQLTransaction &trans, + ObOptStatGatherParam ¶m, + const ObIArray &gather_partition_infos, + const ObIArray &gather_column_params, + bool is_all_columns_gather, + ObIArray &opt_stats, + ObIArray &all_tstats, + ObIArray &all_cstats); static int do_set_table_stats(const ObSetTableStatParam ¶m, ObOptTableStat *table_stat); @@ -72,30 +139,60 @@ private: static int do_set_column_stats(const ObSetColumnStatParam ¶m, ObOptColumnStat *&column_stat); - static int init_opt_stats(ObIAllocator &allocator, - const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &opt_stats); - - static int init_opt_stat(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra, - ObOptStat &stat); - - static int prepare_opt_stat(ObIArray &all_stats, ObOptStat *&opt_stat); - - static int check_all_cols_range_skew(const ObTableStatParam ¶m, - ObIArray &opt_stats); - static int reset_table_locked_state(ObExecContext &ctx, const ObTableStatParam ¶m, const ObIArray &no_stats_partition_ids, const ObIArray &part_stattypes); - static int do_gather_index_stats(ObExecContext &ctx, - const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &all_index_stats); + static int check_need_split_gather(const ObTableStatParam ¶m, + GatherHelper &gather_helper); + + static int prepare_conn_and_store_session_for_online_stats(sql::ObSQLSessionInfo *session, + common::ObMySQLProxy *sql_proxy, + share::schema::ObSchemaGetterGuard *schema_guard, + sql::ObSQLSessionInfo::StmtSavedValue &saved_value, + int64_t &nested_count, + ObSqlString &old_db_name, + int64_t &old_db_id, + int64_t &old_trx_lock_timeout, + bool &need_restore_session, + bool &need_reset_default_database, + bool &need_reset_trx_lock_timeout, + sqlclient::ObISQLConnection *&conn); + + static int restore_session_for_online_stat(sql::ObSQLSessionInfo *session, + sql::ObSQLSessionInfo::StmtSavedValue &saved_value, + int64_t nested_count, + ObSqlString &old_db_name, + int64_t old_db_id, + int64_t old_trx_lock_timeout, + bool need_reset_default_database, + bool need_reset_trx_lock_timeout); + + static int64_t get_column_histogram_size(const ObIArray &column_params); + + static int get_max_work_area_size(uint64_t tenant_id, int64_t &max_wa_memory_size); + + static int merge_split_gather_tab_stats(ObIArray &all_tstats, + ObIArray &cur_all_tstats); + + static int fetch_gather_table_snapshot_read(common::sqlclient::ObISQLConnection *conn, + uint64_t tenant_id, + uint64_t ¤t_scn); + + static int split_derive_part_stats_by_subpart_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const PartitionIdBlockMap *partition_id_block_map, + const GatherHelper &gather_helper); + + static int fetch_gather_task_addr(ObCommonSqlProxy *sql_proxy, + ObIAllocator &allcoator, + uint64_t tenant_id, + const ObString &task_id, + char *&svr_ip, + int32_t &svr_port); + }; diff --git a/src/share/stat/ob_dbms_stats_export_import.cpp b/src/share/stat/ob_dbms_stats_export_import.cpp index 4231d6e1ea..7d034063bd 100644 --- a/src/share/stat/ob_dbms_stats_export_import.cpp +++ b/src/share/stat/ob_dbms_stats_export_import.cpp @@ -71,7 +71,7 @@ namespace common { #define FETCH_COLUMN_STAT "select null statid, 'C' type, '8' version, case stat.histogram_type \ when 1 then 4096 + 4 when 3 then 8192 + 4 when 4 then 4 else 0 end flags,\ '%.*s' c1, %.*s c2, %.*s c3, %.*s c4, '%.*s' c5, null c6,\ - stat.distinct_cnt n1, stat.density n2, null n3, stat.sample_size n4, \ + stat.distinct_cnt n1, stat.density n2, stat.spare1 n3, stat.sample_size n4, \ stat.null_cnt n5, NULL n6, NULL n7, stat.avg_len n8, 1 n9, \ hist.endpoint_num n10, hist.endpoint_normalized_value n11, \ hist.endpoint_repeat_cnt n12, stat.bucket_cnt n13, last_analyzed d1, null t1, \ @@ -449,6 +449,7 @@ int ObDbmsStatsExportImport::import_column_stats(ObExecContext &ctx, const ObTab int ret = OB_SUCCESS; ObSqlString raw_sql; ObSqlString partition_list; + ObSqlString column_list; ObSqlString table_name_str; const char *histflag = lib::is_oracle_mode() ? "bitand(flags, 29700) histflg" : "cast(flags&29700 as decimal) histflg"; @@ -466,10 +467,9 @@ int ObDbmsStatsExportImport::import_column_stats(ObExecContext &ctx, const ObTab LOG_WARN("fail to append SQL stmt string.", K(raw_sql), K(ret)); } else if (OB_FAIL(raw_sql.append(" and type = 'C'"))) { LOG_WARN("fail to append SQL stmt string.", K(raw_sql), K(ret)); - } else if (param.column_params_.count() == 1 && - OB_FAIL(raw_sql.append_fmt(" and c4 = '%.*s'", - param.column_params_.at(0).column_name_.length(), - param.column_params_.at(0).column_name_.ptr()))) { + } else if (OB_FAIL(gen_import_column_list(param.column_params_, column_list))) { + LOG_WARN("failed to gen import partition list", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(" and c4 in %s", column_list.ptr()))) { LOG_WARN("fail to append SQL stmt string.", K(raw_sql), K(ret)); } else if (OB_FAIL(gen_import_partition_list(param, partition_list))) { LOG_WARN("failed to gen import partition list", K(ret)); @@ -604,21 +604,28 @@ int ObDbmsStatsExportImport::do_import_stats(ObExecContext &ctx, } else if (OB_FAIL(check_col_stat_validity(all_cstats))) { LOG_WARN("failed to check col stat validity", K(ret)); } else { - ObSEArray history_tab_handles; - ObSEArray history_col_handles; //before import, we need record history stats. - if (!is_index_stat && !all_tstats.empty() && !param.is_temp_table_ && - OB_FAIL(ObDbmsStatsHistoryManager::get_history_stat_handles(ctx, param, - history_tab_handles, - history_col_handles))) { + ObMySQLTransaction trans; + //begin trans + if (OB_FAIL(trans.start(ctx.get_sql_proxy(), param.tenant_id_))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (!is_index_stat && !all_tstats.empty() && !param.is_temp_table_ && + OB_FAIL(ObDbmsStatsHistoryManager::backup_opt_stats(ctx, trans, param, ObTimeUtility::current_time()))) { LOG_WARN("failed to get history stat handles", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, all_tstats, all_cstats, is_index_stat))) { + } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, trans.get_connection(), all_tstats, all_cstats, is_index_stat))) { LOG_WARN("failed to split batch write", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::batch_write_history_stats(ctx, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to batch write history stats", K(ret)); - } else {/*do nothing*/} + } + //end trans + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } } } int tmp_ret = OB_SUCCESS; @@ -661,7 +668,7 @@ static int get_nmb_val(const ObObj &obj, ObIAllocator &allocator, number::ObNumb * 9.C6 VARCHAR2(30) <==> NULL * 10.N1 NUMBER <==> Num rows('T') or Num distinct('C') * 11.N2 NUMBER <==> Blocks('T') or Density('C') - * 12.N3 NUMBER <==> Average row length('T') + * 12.N3 NUMBER <==> Average row length('T') or compress_type('C') * 13.N4 NUMBER <==> Sample size * 14.N5 NUMBER <==> Num nulls('C') * 15.N6 NUMBER <==> Lower value('C') @@ -707,6 +714,7 @@ int ObDbmsStatsExportImport::get_opt_stat(ObExecContext &ctx, ObString col_str; ObHistBucket hist_bucket; int64_t llc_bitmap_size = 0; + int64_t compress_type = ObOptStatCompressType::MAX_COMPRESS; for (int64_t i = 0; OB_SUCC(ret) && i < StatTableColumnName::MAX_COL; ++i) { if (OB_UNLIKELY(i >= StatTableColumnName::N1 && (stat_type == INVALID_STAT_TYPE || @@ -876,15 +884,17 @@ int ObDbmsStatsExportImport::get_opt_stat(ObExecContext &ctx, } break; } - case StatTableColumnName::N3: {//Average row length('T'/'I') + case StatTableColumnName::N3: {//Average row length('T'/'I') or compress_type('C') number::ObNumber num_val; int64_t int_val = 0; ObNumStackOnceAlloc tmp_alloc; - if (stat_type != TABLE_STAT && stat_type != INDEX_STAT) { - if (OB_UNLIKELY(!result_objs.at(i).is_null())) { - ret = OB_ERR_DBMS_STATS_PL; - LOG_WARN("Invalid or inconsistent input values", K(ret), K(result_objs.at(i))); - LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Invalid or inconsistent input values"); + if (stat_type == COLUMN_STAT) { + if (result_objs.at(i).is_null()) { + compress_type = ObOptStatCompressType::ZLIB_COMPRESS; + } else if (OB_FAIL(get_nmb_val(result_objs.at(i), tmp_alloc, num_val))) { + LOG_WARN("failed to get number", K(ret)); + } else if (OB_FAIL(num_val.extract_valid_int64_with_trunc(compress_type))) { + LOG_WARN("extract_valid_int64_with_trunc failed", K(ret), K(compress_type)); } } else if (!result_objs.at(i).is_null() && OB_FAIL(get_nmb_val(result_objs.at(i), tmp_alloc, num_val))) { @@ -1177,8 +1187,11 @@ int ObDbmsStatsExportImport::get_opt_stat(ObExecContext &ctx, LOG_WARN("failed to get varchar", K(ret)); } else if (stat_type == COLUMN_STAT && llc_bitmap_size > 0) { char *bitmap_buf = NULL; - if (OB_ISNULL(bitmap_buf = static_cast(param.allocator_->alloc( - hex_str.length())))) { + if (OB_UNLIKELY(compress_type < 0 || compress_type >= ObOptStatCompressType::MAX_COMPRESS)) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("Invalid or inconsistent input values", K(ret), K(compress_type)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Invalid or inconsistent input values"); + } else if (OB_ISNULL(bitmap_buf = static_cast(param.allocator_->alloc(hex_str.length())))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("allocate memory for llc_bitmap failed.", K(hex_str.length()), K(ret)); } else { @@ -1188,6 +1201,7 @@ int ObDbmsStatsExportImport::get_opt_stat(ObExecContext &ctx, int64_t decomp_size = ObOptColumnStat::NUM_LLC_BUCKET; const int64_t bitmap_size = hex_str.length() / 2; if (OB_FAIL(ObOptStatSqlService::get_decompressed_llc_bitmap(*param.allocator_, + bitmap_compress_lib_name[compress_type], bitmap_buf, bitmap_size, decomp_buf, @@ -1199,8 +1213,8 @@ int ObDbmsStatsExportImport::get_opt_stat(ObExecContext &ctx, } } else if (OB_UNLIKELY(!result_objs.at(i).is_null())) { ret = OB_ERR_DBMS_STATS_PL; - LOG_WARN("Invalid or inconsistent input values", K(ret), K(result_objs.at(i))); - LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Invalid or inconsistent input values"); + LOG_WARN("Invalid or inconsistent input values", K(ret), K(result_objs.at(i))); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "Invalid or inconsistent input values"); } break; } diff --git a/src/share/stat/ob_dbms_stats_gather.cpp b/src/share/stat/ob_dbms_stats_gather.cpp new file mode 100644 index 0000000000..9fe5c89929 --- /dev/null +++ b/src/share/stat/ob_dbms_stats_gather.cpp @@ -0,0 +1,249 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG +#include "share/stat/ob_dbms_stats_gather.h" +#include "share/stat/ob_basic_stats_estimator.h" +#include "share/stat/ob_hybrid_hist_estimator.h" +#include "share/stat/ob_opt_column_stat_cache.h" +#include "share/stat/ob_opt_stat_manager.h" +#include "share/stat/ob_dbms_stats_utils.h" +#include "share/stat/ob_incremental_stat_estimator.h" +#include "share/stat/ob_dbms_stats_history_manager.h" +#include "share/stat/ob_dbms_stats_lock_unlock.h" +#include "share/stat/ob_index_stats_estimator.h" +#include "pl/sys_package/ob_dbms_stats.h" +#include "share/stat/ob_opt_stat_gather_stat.h" +namespace oceanbase { +using namespace pl; +namespace common { + +/** + * @brief ObDbmsStatsUtils::gather_table_stats + * @return + */ +int ObDbmsStatsGather::gather_stats(ObExecContext &ctx, + const ObOptStatGatherParam ¶m, + ObIArray &opt_stats) +{ + int ret = OB_SUCCESS; + LOG_TRACE("begin to gather table stats", K(param)); + if (OB_ISNULL(param.allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param.allocator_)); + } else if (OB_FAIL(init_opt_stats(*param.allocator_, param, opt_stats))) { + LOG_WARN("failed to init opt stats", K(ret)); + } else if (!opt_stats.empty()) { + //1.firstly esimate basic stat + ObBasicStatsEstimator basic_est(ctx, *param.allocator_); + if (OB_FAIL(basic_est.estimate(param, opt_stats))) { + LOG_WARN("failed to estimate basic statistics", K(ret)); + } else if (param.need_histogram_) { + for (int64_t i = 0; OB_SUCC(ret) && i < opt_stats.count(); ++i) { + ObOptStatGatherParam new_param; + ObTopkHistEstimator topk_est(ctx, *param.allocator_); + ObHybridHistEstimator hybrid_est(ctx, *param.allocator_); + if (OB_ISNULL(opt_stats.at(i).table_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(opt_stats.at(i).table_stat_)); + } else if (OB_FAIL(THIS_WORKER.check_status())) { + LOG_WARN("check status failed", KR(ret)); + } else if (opt_stats.at(i).table_stat_->get_row_count() <= 0) { + //empty table or empty partition, no need gather histogram, just skip. + } else if (OB_FAIL(new_param.assign(param))) { + LOG_WARN("failed to assign", K(ret)); + } else if (new_param.stat_level_ != TABLE_LEVEL && + OB_FAIL(ObDbmsStatsUtils::remove_stat_gather_param_partition_info(opt_stats.at(i).table_stat_->get_partition_id(), + new_param))) { + LOG_WARN("failed to remove stat gather param partition info", K(ret)); + } else if (OB_FAIL(classfy_column_histogram(new_param, opt_stats.at(i)))) { + LOG_WARN("failed to classfy column histogram", K(ret)); + } else if (OB_FAIL(topk_est.estimate(new_param, opt_stats.at(i)))) { + LOG_WARN("failed to estimate topk histogram", K(ret)); + } else if (OB_FAIL(hybrid_est.estimate(new_param, opt_stats.at(i)))) { + LOG_WARN("failed to estimate hybrid histogram", K(ret)); + } + } + } else {/*do nothing*/} + } + return ret; +} + +int ObDbmsStatsGather::classfy_column_histogram(const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(opt_stat.table_stat_) || + OB_UNLIKELY(param.column_params_.count() != opt_stat.column_stats_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(opt_stat), K(param)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) { + const ObColumnStatParam &col_param = param.column_params_.at(i); + ObOptColumnStat *dst_col_stat = opt_stat.column_stats_.at(i); + if (OB_UNLIKELY(dst_col_stat->get_column_id() != col_param.column_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(dst_col_stat), K(col_param)); + } else if (col_param.need_basic_stat() && + col_param.bucket_num_ > 1 && + dst_col_stat->get_num_distinct() > 0) { + int64_t max_disuse_cnt = std::ceil(dst_col_stat->get_num_not_null() * 1.0 / col_param.bucket_num_); + //After testing, the error of using hyperloglog to estimate ndv is within %5. + const double MAX_LLC_NDV_ERR_RATE = !param.need_approx_ndv_ ? 0.0 : 0.05; + const int64_t fault_tolerance_cnt = std::ceil(dst_col_stat->get_num_distinct() * MAX_LLC_NDV_ERR_RATE); + if (dst_col_stat->get_num_distinct() >= col_param.bucket_num_ + max_disuse_cnt + fault_tolerance_cnt) { + //directly gather hybrid histogram + dst_col_stat->get_histogram().set_type(ObHistType::HYBIRD); + } else { + //otherwise, try gather top frequery histogram + dst_col_stat->get_histogram().set_type(ObHistType::TOP_FREQUENCY); + } + } + } + } + return ret; +} + +int ObDbmsStatsGather::init_opt_stats(ObIAllocator &allocator, + const ObOptStatGatherParam ¶m, + ObIArray &opt_stats) +{ + int ret = OB_SUCCESS; + if (TABLE_LEVEL == param.stat_level_) { + ObOptStat stat; + if (OB_FAIL(init_opt_stat(allocator, param, param.global_part_id_, param.stattype_, stat))) { + LOG_WARN("failed to init opt stat", K(ret)); + } else if (OB_FAIL(opt_stats.push_back(stat))) { + LOG_WARN("failed to push back stat", K(ret)); + } else {/*do nothing*/} + } else if (PARTITION_LEVEL == param.stat_level_ || + SUBPARTITION_LEVEL == param.stat_level_) { + for (int64_t i = 0; OB_SUCC(ret) && i < param.partition_infos_.count(); ++i) { + ObOptStat stat; + if (OB_FAIL(init_opt_stat(allocator, param, param.partition_infos_.at(i).part_id_, + param.partition_infos_.at(i).part_stattype_, stat))) { + LOG_WARN("failed to init opt stat"); + } else if (OB_FAIL(opt_stats.push_back(stat))) { + LOG_WARN("failed to push back stat"); + } else {/*do nothing*/} + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected stat level", K(ret), K(param.stat_level_)); + } + return ret; +} + +int ObDbmsStatsGather::init_opt_stat(ObIAllocator &allocator, + const ObOptStatGatherParam ¶m, + const int64_t part_id, + const int64_t part_stattype, + ObOptStat &stat) +{ + int ret = OB_SUCCESS; + void *ptr = NULL; + BlockNumStat *block_num_stat = NULL; + ObOptTableStat *&tab_stat = stat.table_stat_; + if (OB_FAIL(stat.column_stats_.prepare_allocate(param.column_params_.count()))) { + LOG_WARN("failed to prepare allocate column stat", K(ret)); + } else if (OB_ISNULL(ptr = allocator.alloc(sizeof(ObOptTableStat)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(ptr)); + } else { + tab_stat = new (ptr) ObOptTableStat(); + tab_stat->set_table_id(param.table_id_); + tab_stat->set_partition_id(part_id); + tab_stat->set_object_type(param.stat_level_); + tab_stat->set_stattype_locked(part_stattype); + if (OB_ISNULL(param.partition_id_block_map_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param.partition_id_block_map_)); + } else if (OB_FAIL(param.partition_id_block_map_->get_refactored(part_id, block_num_stat))) { + if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get refactored", K(ret)); + } + } else if (OB_ISNULL(block_num_stat)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(block_num_stat)); + } else { + tab_stat->set_macro_block_num(block_num_stat->tab_macro_cnt_); + tab_stat->set_micro_block_num(block_num_stat->tab_micro_cnt_); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) { + ObOptColumnStat *&col_stat = stat.column_stats_.at(i); + if (OB_UNLIKELY(!param.column_params_.at(i).need_col_stat())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(i), K(param)); + } else if (OB_ISNULL(col_stat = ObOptColumnStat::malloc_new_column_stat(allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(col_stat)); + } else { + col_stat->set_table_id(param.column_params_.at(i).need_basic_stat() ? param.table_id_: OB_INVALID_ID); + col_stat->set_partition_id(part_id); + col_stat->set_stat_level(param.stat_level_); + col_stat->set_column_id(param.column_params_.at(i).column_id_); + col_stat->set_collation_type(param.column_params_.at(i).cs_type_); + if (block_num_stat != NULL) { + if (OB_LIKELY(param.column_group_params_.count() == block_num_stat->cg_macro_cnt_arr_.count() && + param.column_group_params_.count() == block_num_stat->cg_micro_cnt_arr_.count())) { + bool found_it = false; + for (int64_t j = 0; !found_it && j < param.column_group_params_.count(); ++j) { + if (param.column_group_params_.at(j).column_id_arr_.count() == 1 && + param.column_group_params_.at(j).column_id_arr_.at(0) == param.column_params_.at(i).column_id_) { + col_stat->set_cg_macro_blk_cnt(block_num_stat->cg_macro_cnt_arr_.at(j)); + col_stat->set_cg_micro_blk_cnt(block_num_stat->cg_micro_cnt_arr_.at(j)); + } + } + } + } + } + } + return ret; +} + +int ObDbmsStatsGather::gather_index_stats(ObExecContext &ctx, + const ObOptStatGatherParam ¶m, + ObIArray &all_index_stats) +{ + int ret = OB_SUCCESS; + ObSEArray opt_stats; + LOG_TRACE("begin to gather index stats", K(param)); + if (OB_ISNULL(param.allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param.allocator_)); + } else if (OB_FAIL(init_opt_stats(*param.allocator_, param, opt_stats))) { + LOG_WARN("failed to init opt stats", K(ret)); + } else if (opt_stats.empty()) { + /*do nothing*/ + } else { + ObIndexStatsEstimator index_est(ctx, *param.allocator_); + if (OB_FAIL(index_est.estimate(param, opt_stats))) { + LOG_WARN("failed to estimate basic statistics", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < opt_stats.count(); ++i) { + if (OB_ISNULL(opt_stats.at(i).table_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(opt_stats.at(i).table_stat_)); + } else if (OB_FAIL(all_index_stats.push_back(opt_stats.at(i).table_stat_))) { + LOG_WARN("failed to append", K(ret)); + } else {/*do nothing*/} + } + } + } + return ret; +} + +} // namespace common +} // namespace oceanbase diff --git a/src/share/stat/ob_dbms_stats_gather.h b/src/share/stat/ob_dbms_stats_gather.h new file mode 100644 index 0000000000..22eca959cc --- /dev/null +++ b/src/share/stat/ob_dbms_stats_gather.h @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_DBMS_STATS_GATHER_H +#define OB_DBMS_STATS_GATHER_H + +#include "share/stat/ob_stat_define.h" +#include "sql/engine/ob_exec_context.h" +#include "share/stat/ob_stat_item.h" + +namespace oceanbase { +using namespace sql; +namespace common { + +class ObDbmsStatsGather +{ +public: + ObDbmsStatsGather(); + + static int gather_stats(ObExecContext &ctx, + const ObOptStatGatherParam ¶m, + ObIArray &opt_stats); + + static int gather_index_stats(ObExecContext &ctx, + const ObOptStatGatherParam ¶m, + ObIArray &all_index_stats); +private: + + static int init_opt_stats(ObIAllocator &allocator, + const ObOptStatGatherParam ¶m, + ObIArray &opt_stats); + + static int init_opt_stat(ObIAllocator &allocator, + const ObOptStatGatherParam ¶m, + const int64_t part_id, + const int64_t part_stattype, + ObOptStat &stat); + + static int classfy_column_histogram(const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat); + +}; + +} // end of sql +} // end of namespace + +#endif // OB_DBMS_STATS_GATHER_H diff --git a/src/share/stat/ob_dbms_stats_history_manager.cpp b/src/share/stat/ob_dbms_stats_history_manager.cpp index 91c5bec2aa..d50c0d6cd3 100644 --- a/src/share/stat/ob_dbms_stats_history_manager.cpp +++ b/src/share/stat/ob_dbms_stats_history_manager.cpp @@ -37,7 +37,7 @@ namespace common { #define FETCH_COL_STATS_HISTROY "SELECT table_id, partition_id, column_id, object_type stat_level,\ distinct_cnt num_distinct, null_cnt num_null, b_max_value,\ b_min_value, avg_len, distinct_cnt_synopsis, distinct_cnt_synopsis_size,\ - histogram_type, sample_size, bucket_cnt, density, last_analyzed%s\ + histogram_type, sample_size, bucket_cnt, density, last_analyzed, spare1 as compress_type %s\ FROM %s T WHERE tenant_id = %lu and table_id = %ld \ and partition_id in %s and savtime in (SELECT min(savtime) From \ %s TF where TF.tenant_id = T.tenant_id \ @@ -62,29 +62,801 @@ namespace common { #define DELETE_STAT_HISTORY "DELETE FROM %s %s;" +#define INSERT_TABLE_STAT_HISTORY "INSERT INTO %s(tenant_id, \ + table_id, \ + partition_id, \ + savtime, \ + object_type, \ + flags, \ + last_analyzed, \ + sstable_row_cnt, \ + sstable_avg_row_len, \ + macro_blk_cnt, \ + micro_blk_cnt, \ + memtable_row_cnt, \ + memtable_avg_row_len,\ + row_cnt, \ + avg_row_len, \ + index_type, \ + stattype_locked) %s" -int ObDbmsStatsHistoryManager::get_history_stat_handles(ObExecContext &ctx, - const ObTableStatParam ¶m, - ObIArray &history_tab_handles, - ObIArray &history_col_handles) +#define SELECT_TABLE_STAT "SELECT tenant_id, \ + table_id, \ + partition_id, \ + usec_to_time(%ld), \ + object_type, \ + 0, \ + last_analyzed, \ + sstable_row_cnt, \ + sstable_avg_row_len, \ + macro_blk_cnt, \ + micro_blk_cnt, \ + memtable_row_cnt, \ + memtable_avg_row_len,\ + row_cnt, \ + avg_row_len, \ + index_type, \ + stattype_locked \ + FROM %s \ + WHERE tenant_id = %lu and table_id = %lu %s" + +#define TABLE_STAT_MOCK_VALUE_PATTERN "(%lu, %lu, %ld, usec_to_time(%ld), 0, 0, usec_to_time(%ld), 0, -1, -1, 0, 0, -1, 0, 0, 0, 0)" + +#define INSERT_COLUMN_STAT_HISTORY "INSERT INTO %s(tenant_id, \ + table_id, \ + partition_id, \ + column_id, \ + savtime, \ + object_type, \ + flags, \ + last_analyzed, \ + distinct_cnt, \ + null_cnt, \ + max_value, \ + b_max_value, \ + min_value, \ + b_min_value, \ + avg_len, \ + distinct_cnt_synopsis, \ + distinct_cnt_synopsis_size,\ + sample_size, \ + density, \ + bucket_cnt, \ + histogram_type, \ + spare1%s) %s" + +#define SELECT_COLUMN_STAT "SELECT tenant_id, \ + table_id, \ + partition_id, \ + column_id, \ + usec_to_time(%ld), \ + object_type, \ + 0, \ + last_analyzed, \ + distinct_cnt, \ + null_cnt, \ + max_value, \ + b_max_value, \ + min_value, \ + b_min_value, \ + avg_len, \ + distinct_cnt_synopsis, \ + distinct_cnt_synopsis_size,\ + sample_size, \ + density, \ + bucket_cnt, \ + histogram_type, \ + spare1%s \ + FROM %s \ + WHERE %s" + +#define COLUMN_STAT_MOCK_VALUE_PATTERN "(%lu, %lu, %ld, %lu, usec_to_time(%ld), 0, 0, usec_to_time(%ld), 0, 0, \ + %s, '%.*s', %s, '%.*s', 0, '', 0, -1, 0.000000, 0, 0, NULL%s)" + +#define INSERT_HISTOGRAM_STAT_HISTORY "INSERT INTO %s(tenant_id, \ + table_id, \ + partition_id, \ + column_id, \ + endpoint_num, \ + savtime, \ + object_type, \ + endpoint_normalized_value,\ + endpoint_value, \ + b_endpoint_value, \ + endpoint_repeat_cnt) \ + SELECT tenant_id, \ + table_id, \ + partition_id, \ + column_id, \ + endpoint_num, \ + usec_to_time(%ld), \ + object_type, \ + endpoint_normalized_value,\ + endpoint_value, \ + b_endpoint_value, \ + endpoint_repeat_cnt \ + FROM %s \ + WHERE %s" + +#define CHECK_TABLE_STAT "SELECT partition_id FROM %s WHERE tenant_id = %lu \ + and table_id = %lu %s" + +#define CHECK_COLUMN_STAT "SELECT partition_id, column_id FROM %s WHERE tenant_id = %lu \ + and table_id = %lu %s" + +int ObDbmsStatsHistoryManager::backup_opt_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + int64_t saving_time, + bool is_backup_for_gather/*default false*/) { int ret = OB_SUCCESS; int64_t retention_val = 0; + ObSEArray part_ids; + ObSEArray column_ids; if (param.is_index_stat_) { //do nothing } else if (OB_FAIL(get_stats_history_retention(ctx, retention_val))) { LOG_WARN("failed to get stats history retention", K(ret)); } else if (retention_val == 0) { /*do nothing*/ - } else if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(param, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to get current opt stats", K(ret)); - } else if (OB_FAIL(set_col_stat_cs_type(param.column_params_, history_col_handles))) { - LOG_WARN("failed to set col stat cs type", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::get_part_ids_and_column_ids(param, part_ids, column_ids, is_backup_for_gather))) { + LOG_WARN("failed to get part ids and column ids", K(ret)); + } else if (OB_FAIL(backup_table_stats(ctx, trans, param, saving_time, part_ids))) { + LOG_WARN("faile to backup table stats", K(ret)); + } else if (OB_FAIL(backup_column_stats(ctx, trans, param, saving_time, part_ids, column_ids))) { + LOG_WARN("faile to backup column stats", K(ret)); + } else {/*do nothing*/} + return ret; +} + +int ObDbmsStatsHistoryManager::backup_table_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const int64_t saving_time, + ObIArray &part_ids) +{ + int ret = OB_SUCCESS; + ObSEArray no_stat_part_ids; + ObSEArray have_stat_part_ids; + bool is_specify_partition_gather = param.is_specify_partition_gather(); + if (part_ids.empty()) { + } else if (OB_FAIL(calssify_table_stat_part_ids(ctx, + param.tenant_id_, + param.table_id_, + is_specify_partition_gather, + part_ids, + no_stat_part_ids, + have_stat_part_ids))) { + LOG_WARN("failed to calssify table stat part ids", K(ret)); + } else if (OB_FAIL(backup_having_table_part_stats(trans, + param.tenant_id_, + param.table_id_, + (is_specify_partition_gather || have_stat_part_ids.count() != part_ids.count()), + have_stat_part_ids, + saving_time))) { + LOG_WARN("failed to backup having table part stats", K(ret)); + } else if (OB_FAIL(backup_no_table_part_stats(trans, param.tenant_id_, param.table_id_, no_stat_part_ids, saving_time))) { + LOG_WARN("failed to backup no table part stats", K(ret)); + } + return ret; +} + +int ObDbmsStatsHistoryManager::calssify_table_stat_part_ids(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_partition_gather, + const ObIArray &partition_ids, + ObIArray &no_stat_part_ids, + ObIArray &have_stat_part_ids) +{ + int ret = OB_SUCCESS; + ObSqlString partition_list; + ObSqlString extra_where_str; + ObSqlString raw_sql; + ObSQLSessionInfo *session = ctx.get_my_session(); + ObMySQLProxy *mysql_proxy = ctx.get_sql_proxy(); + if (OB_ISNULL(mysql_proxy) || OB_ISNULL(session) || OB_UNLIKELY(partition_ids.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(mysql_proxy), K(session), K(partition_ids)); + } else if (is_specify_partition_gather && + OB_FAIL(gen_partition_list(partition_ids, partition_list))) { + LOG_WARN("failed to gen partition list", K(ret)); + } else if (is_specify_partition_gather && + OB_FAIL(extra_where_str.append_fmt(" and partition_id in %s", partition_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(CHECK_TABLE_STAT, + share::OB_ALL_TABLE_STAT_TNAME, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), + is_specify_partition_gather ? extra_where_str.ptr() : " "))) { + LOG_WARN("failed to append fmt", K(ret)); } else { - LOG_TRACE("Succeed to get history stat handles", K(param), K(history_tab_handles.count()), - K(history_col_handles.count())); + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(mysql_proxy); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, raw_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(raw_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { + ObObj tmp; + int64_t idx = 0; + int64_t partition_id = 0; + if (OB_FAIL(client_result->get_obj(idx, tmp))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(tmp.get_int(partition_id))) { + LOG_WARN("failed to get int", K(ret), K(tmp)); + } else if (OB_FAIL(have_stat_part_ids.push_back(partition_id))) { + LOG_WARN("failed to write object", K(ret)); + } else {/*do nothing*/} + } + if (OB_ITER_END != ret) { + LOG_WARN("failed to get result", K(ret)); + } else { + ret = OB_SUCCESS; + } + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + if (OB_SUCC(ret)) { + if (have_stat_part_ids.count() == partition_ids.count()) { + //do nothing + } else if (have_stat_part_ids.empty()) { + if (OB_FAIL(no_stat_part_ids.assign(partition_ids))) { + LOG_WARN("failed to assign", K(ret)); + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + bool found_it = false; + for (int64_t j = 0; !found_it && j < have_stat_part_ids.count(); ++j) { + found_it = (partition_ids.at(i) == have_stat_part_ids.at(j)); + } + if (!found_it) { + if (OB_FAIL(no_stat_part_ids.push_back(partition_ids.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + } + } + } + LOG_TRACE("calssify table stat part ids", K(partition_ids), K(no_stat_part_ids), K(have_stat_part_ids)); + return ret; +} + +int ObDbmsStatsHistoryManager::backup_having_table_part_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_partition_gather, + const ObIArray &partition_ids, + const int64_t saving_time) +{ + int ret = OB_SUCCESS; + ObSqlString partition_list; + ObSqlString extra_where_str; + ObSqlString raw_sql; + ObSqlString select_sql; + int64_t affected_rows = 0; + if (partition_ids.empty()) { + } else if (is_specify_partition_gather && + OB_FAIL(gen_partition_list(partition_ids, partition_list))) { + LOG_WARN("failed to gen partition list", K(ret)); + } else if (is_specify_partition_gather && + OB_FAIL(extra_where_str.append_fmt(" and partition_id in %s", partition_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(select_sql.append_fmt(SELECT_TABLE_STAT, + saving_time, + share::OB_ALL_TABLE_STAT_TNAME, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), + is_specify_partition_gather ? extra_where_str.ptr() : " "))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(INSERT_TABLE_STAT_HISTORY, + share::OB_ALL_TABLE_STAT_HISTORY_TNAME, + select_sql.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(raw_sql), K(ret)); + } else { + LOG_TRACE("succeed to backup having table part stats", K(raw_sql), K(affected_rows)); + } + return ret; +} + +//mock the null stat info for no table part stats. +int ObDbmsStatsHistoryManager::backup_no_table_part_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &partition_ids, + const int64_t saving_time) +{ + int ret = OB_SUCCESS; + int64_t idx = 0; + //write 2000 part stats every time. + while (OB_SUCC(ret) && idx < partition_ids.count()) { + ObSqlString values_list; + if (OB_UNLIKELY(idx >= partition_ids.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpectd error", K(ret), K(idx), K(partition_ids)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < MAX_NUM_OF_WRITE_STATS && idx < partition_ids.count(); ++i) { + ObSqlString value; + if (OB_FAIL(value.append_fmt(TABLE_STAT_MOCK_VALUE_PATTERN, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), + partition_ids.at(idx++), + saving_time, + saving_time))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(values_list.append_fmt("%s%s", + i == 0 ? "VALUES " : ", ", + value.ptr()))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + ObSqlString raw_sql; + int64_t affected_rows = 0; + if (OB_FAIL(raw_sql.append_fmt(INSERT_TABLE_STAT_HISTORY, + share::OB_ALL_TABLE_STAT_HISTORY_TNAME, + values_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(raw_sql), K(ret)); + } else { + LOG_TRACE("succeed to backup no table part stats", K(raw_sql), K(affected_rows)); + } + } + } + return ret; +} + +int ObDbmsStatsHistoryManager::backup_column_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const int64_t saving_time, + const ObIArray &part_ids, + const ObIArray &column_ids) +{ + int ret = OB_SUCCESS; + hash::ObHashMap having_stat_part_col_map; + int64_t map_size = part_ids.count() * column_ids.count(); + bool is_specify_partition_gather = param.is_specify_partition_gather(); + bool is_specify_column_gather = param.is_specify_column_gather(); + if (part_ids.empty() || column_ids.empty()) { + } else if (OB_FAIL(having_stat_part_col_map.create(map_size, + "PartColHashMap", + "PartColNode", + param.tenant_id_))) { + LOG_WARN("fail to create hash map", K(ret)); + } else if (OB_FAIL(generate_having_stat_part_col_map(ctx, + param.tenant_id_, + param.table_id_, + is_specify_partition_gather, + is_specify_column_gather, + part_ids, + column_ids, + having_stat_part_col_map))) { + LOG_WARN("failed to calssify table stat part ids", K(ret)); + } else if (OB_FAIL(backup_having_column_stats(trans, param.tenant_id_, param.table_id_, + is_specify_partition_gather || is_specify_column_gather, + part_ids, column_ids, + having_stat_part_col_map, + saving_time))) { + LOG_WARN("failed to backup have column part stats", K(ret)); + } else if (OB_FAIL(backup_no_column_stats(trans, param.tenant_id_, param.table_id_, + part_ids, column_ids, + having_stat_part_col_map, + saving_time))) { + LOG_WARN("failed to backup column part stats", K(ret)); + } else if (OB_FAIL(backup_histogram_stats(trans, param.tenant_id_, param.table_id_, + is_specify_partition_gather, + is_specify_column_gather, + part_ids, column_ids, + having_stat_part_col_map, + saving_time))) { + LOG_WARN("faile to do backup histogram stats", K(ret)); + } else {/*do nothing*/} + return ret; +} + +int ObDbmsStatsHistoryManager::generate_having_stat_part_col_map(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_partition_gather, + const bool is_specify_column_gather, + const ObIArray &partition_ids, + const ObIArray &column_ids, + hash::ObHashMap &have_stat_part_col_map) +{ + int ret = OB_SUCCESS; + ObSqlString partition_list; + ObSqlString column_list; + ObSqlString extra_partition_str; + ObSqlString extra_column_str; + ObSqlString extra_where_str; + ObSqlString raw_sql; + ObSQLSessionInfo *session = ctx.get_my_session(); + ObMySQLProxy *mysql_proxy = ctx.get_sql_proxy(); + if (OB_ISNULL(mysql_proxy) || OB_ISNULL(session) || + OB_UNLIKELY(partition_ids.empty() || column_ids.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(mysql_proxy), K(session), K(partition_ids), K(column_ids)); + } else if (is_specify_partition_gather && + OB_FAIL(gen_partition_list(partition_ids, partition_list))) { + LOG_WARN("failed to gen partition list", K(ret)); + } else if (is_specify_partition_gather && + OB_FAIL(extra_partition_str.append_fmt(" and partition_id in %s", partition_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (is_specify_column_gather && + OB_FAIL(gen_column_list(column_ids, column_list))) { + LOG_WARN("failed to gen column list", K(ret)); + } else if (is_specify_column_gather && + OB_FAIL(extra_column_str.append_fmt(" and column_id in %s", column_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if ((is_specify_partition_gather || is_specify_column_gather) && + OB_FAIL(extra_where_str.append_fmt("%s%s", + is_specify_partition_gather ? extra_partition_str.ptr() : " ", + is_specify_column_gather ? extra_column_str.ptr() : " "))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(CHECK_COLUMN_STAT, + share::OB_ALL_COLUMN_STAT_TNAME, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), + (is_specify_partition_gather || is_specify_column_gather) ? extra_where_str.ptr() : " "))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, proxy_result) { + sqlclient::ObMySQLResult *client_result = NULL; + ObSQLClientRetryWeak sql_client_retry_weak(mysql_proxy); + if (OB_FAIL(sql_client_retry_weak.read(proxy_result, tenant_id, raw_sql.ptr()))) { + LOG_WARN("failed to execute sql", K(ret), K(raw_sql)); + } else if (OB_ISNULL(client_result = proxy_result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to execute sql", K(ret)); + } else { + while (OB_SUCC(ret) && OB_SUCC(client_result->next())) { + ObObj tmp; + int64_t idx1 = 0; + int64_t idx2 = 1; + int64_t partition_id = 0; + int64_t column_id = 0; + if (OB_FAIL(client_result->get_obj(idx1, tmp))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(tmp.get_int(partition_id))) { + LOG_WARN("failed to get int", K(ret), K(tmp)); + } else if (OB_FAIL(client_result->get_obj(idx2, tmp))) { + LOG_WARN("failed to get object", K(ret)); + } else if (OB_FAIL(tmp.get_int(column_id))) { + LOG_WARN("failed to get int", K(ret), K(tmp)); + } else { + ObOptColumnStat::Key key(tenant_id, table_id, partition_id, static_cast(column_id)); + if (OB_FAIL(have_stat_part_col_map.set_refactored(key, true))) { + LOG_WARN("failed to set refactored", K(ret), K(key)); + } + } + } + if (OB_ITER_END != ret) { + LOG_WARN("failed to get result", K(ret)); + } else { + ret = OB_SUCCESS; + } + } + int tmp_ret = OB_SUCCESS; + if (NULL != client_result) { + if (OB_SUCCESS != (tmp_ret = client_result->close())) { + LOG_WARN("close result set failed", K(ret), K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + } + } + } + LOG_TRACE("generate having stat part col map", K(have_stat_part_col_map.size()), K(partition_ids), K(column_ids)); + return ret; +} + +int ObDbmsStatsHistoryManager::backup_having_column_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_gather, + const ObIArray &partition_ids, + const ObIArray &column_ids, + hash::ObHashMap &having_stat_part_col_map, + const int64_t saving_time) +{ + int ret = OB_SUCCESS; + if (having_stat_part_col_map.size() != 0) { + ObSqlString raw_sql; + ObSqlString select_sql; + ObSqlString where_str; + uint64_t data_version = 0; + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret)); + } else if (OB_LIKELY(having_stat_part_col_map.size() == partition_ids.count() * column_ids.count())) { + if (!is_specify_gather) { + if (OB_FAIL(where_str.append_fmt(" tenant_id = %lu and table_id = %lu", + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id)))) { + LOG_WARN("failed to append fmt", K(ret)); + } + } else { + ObSqlString partition_list; + ObSqlString column_list; + if (OB_FAIL(gen_partition_list(partition_ids, partition_list))) { + LOG_WARN("failed to gen partition list", K(ret)); + } else if (OB_FAIL(gen_column_list(column_ids, column_list))) { + LOG_WARN("failed to gen partition list", K(ret)); + } else if (OB_FAIL(where_str.append_fmt(" tenant_id = %lu and table_id = %lu and partition_id in %s and column_id in %s", + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), + partition_list.ptr(), + column_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } + } + } else { + // tenant_id = xx and table_id = xx and ((partition_id in xx and column_id in xx) or ((partition_id, column_id) in ((xx)))) + ObSqlString partition_list; + ObSqlString part_col_list; + bool is_first_part_list = true; + bool is_first_part_col_list = true; + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + bool all_get = true; + ObSqlString tmp_part_col_list; + bool is_first = true; + for (int64_t j = 0; OB_SUCC(ret) && j < column_ids.count(); ++j) { + ObOptColumnStat::Key key(tenant_id, table_id, partition_ids.at(i), column_ids.at(j)); + bool val = false; + if (OB_FAIL(having_stat_part_col_map.get_refactored(key, val))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; // continue + all_get = false; + } else { + LOG_WARN("failed to get map", K(ret), K(key)); + } + } else if (OB_FAIL(tmp_part_col_list.append_fmt("%s(%ld, %lu)", is_first ? " " : ", ", + partition_ids.at(i), + column_ids.at(j)))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + is_first = false; + } + } + if (OB_SUCC(ret)) { + if (all_get) { + if (OB_FAIL(partition_list.append_fmt("%s%ld", + is_first_part_list ? " " : ", ", + partition_ids.at(i)))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + is_first_part_list = false; + } + } else if (tmp_part_col_list.empty()) { + //do nothing + } else if (OB_FAIL(part_col_list.append_fmt("%s%s", + is_first_part_col_list ? " " : ", ", + tmp_part_col_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + is_first_part_col_list = false; + } + } + } + if (OB_SUCC(ret)) { + ObSqlString all_column_list; + ObSqlString part_col_where1; + ObSqlString part_col_where2; + if (OB_FAIL(gen_column_list(column_ids, all_column_list))) { + LOG_WARN("failed to gen partition list", K(ret)); + } else if (!partition_list.empty() && + OB_FAIL(part_col_where1.append_fmt("(partition_id in (%s) and column_id in %s)", + partition_list.ptr(), + all_column_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (!part_col_list.empty() && + OB_FAIL(part_col_where2.append_fmt("((partition_id, column_id) in (%s))", + part_col_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(where_str.append_fmt(" tenant_id = %lu and table_id = %lu and (%s %s %s)", + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), + part_col_where1.empty() ? " " : part_col_where1.ptr(), + !part_col_where1.empty() && !part_col_where2.empty() ? "or" : " ", + part_col_where2.empty() ? " " : part_col_where2.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + int64_t affected_rows = 0; + if (OB_FAIL(select_sql.append_fmt(SELECT_COLUMN_STAT, + saving_time, + data_version < DATA_VERSION_4_3_0_0 ? " " : ",cg_macro_blk_cnt, cg_micro_blk_cnt", + share::OB_ALL_COLUMN_STAT_TNAME, + where_str.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(INSERT_COLUMN_STAT_HISTORY, + share::OB_ALL_COLUMN_STAT_HISTORY_TNAME, + data_version < DATA_VERSION_4_3_0_0 ? " " : ",cg_macro_blk_cnt, cg_micro_blk_cnt", + select_sql.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(raw_sql), K(ret)); + } else { + LOG_TRACE("succeed to backup having column stats", K(raw_sql), K(affected_rows)); + } + } + } + return ret; +} + +int ObDbmsStatsHistoryManager::backup_no_column_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &partition_ids, + const ObIArray &column_ids, + hash::ObHashMap &having_stat_part_col_map, + const int64_t saving_time) +{ + int ret = OB_SUCCESS; + int64_t total_cnt = partition_ids.count() * column_ids.count(); + if (having_stat_part_col_map.size() < total_cnt) { + int64_t idx_part = 0; + ObObj null_obj; + null_obj.set_null(); + ObArenaAllocator allocator("OptStatsHistory", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); + ObObjPrintParams tmp_obj_print_params; + ObString null_str; + ObSqlString null_sql_str; + ObString b_null_str; + uint64_t data_version = 0; + if (OB_FAIL(ObOptStatSqlService::get_obj_str(null_obj, allocator, null_str, tmp_obj_print_params))) { + LOG_WARN("failed to get obj str", K(ret)); + } else if (OB_FAIL(sql_append_hex_escape_str(null_str, null_sql_str))) { + LOG_WARN("failed to sql append hex escape str", K(ret)); + } else if (OB_FAIL(ObOptStatSqlService::get_obj_binary_hex_str(null_obj, allocator, b_null_str))) { + LOG_WARN("failed to convert obj to binary string", K(ret)); + } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret)); + } else { + ObSqlString values_list; + int64_t cur_cnt = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); ++i) { + for (int64_t j = 0; OB_SUCC(ret) && j < column_ids.count(); ++j) { + ObOptColumnStat::Key key(tenant_id, table_id, partition_ids.at(i), column_ids.at(j)); + bool val = false; + if (OB_FAIL(having_stat_part_col_map.get_refactored(key, val))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + ObSqlString value; + if (OB_FAIL(value.append_fmt(COLUMN_STAT_MOCK_VALUE_PATTERN, + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), + partition_ids.at(i), + column_ids.at(j), + saving_time, + saving_time, + null_sql_str.ptr(), + b_null_str.length(), + b_null_str.ptr(), + null_sql_str.ptr(), + b_null_str.length(), + b_null_str.ptr(), + data_version < DATA_VERSION_4_3_0_0 ? " " : ",0 ,0"))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(values_list.append_fmt("%s%s", + cur_cnt == 0 ? "VALUES " : ", ", + value.ptr()))) { + LOG_WARN("failed to push back", K(ret)); + } else { + ++ cur_cnt; + if (cur_cnt == MAX_NUM_OF_WRITE_STATS) { + if (OB_SUCC(ret)) { + ObSqlString raw_sql; + int64_t affected_rows = 0; + if (OB_FAIL(raw_sql.append_fmt(INSERT_COLUMN_STAT_HISTORY, + share::OB_ALL_COLUMN_STAT_HISTORY_TNAME, + data_version < DATA_VERSION_4_3_0_0 ? " " : ",cg_macro_blk_cnt, cg_micro_blk_cnt", + values_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(raw_sql), K(ret)); + } else { + cur_cnt = 0; + values_list.reuse(); + LOG_TRACE("succeed to backup no table part stats", K(raw_sql), K(affected_rows)); + } + } + } + } + } else { + LOG_WARN("failed to get map", K(ret), K(key)); + } + } + } + } + if (OB_SUCC(ret) && cur_cnt > 0) { + if (OB_SUCC(ret)) { + ObSqlString raw_sql; + int64_t affected_rows = 0; + if (OB_FAIL(raw_sql.append_fmt(INSERT_COLUMN_STAT_HISTORY, + share::OB_ALL_COLUMN_STAT_HISTORY_TNAME, + data_version < DATA_VERSION_4_3_0_0 ? " " : ",cg_macro_blk_cnt, cg_micro_blk_cnt", + values_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(raw_sql), K(ret)); + } else { + LOG_TRACE("succeed to backup no table part stats", K(raw_sql), K(affected_rows)); + } + } + } + } + } + return ret; +} + +int ObDbmsStatsHistoryManager::backup_histogram_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_partition_gather, + const bool is_specify_column_gather, + const ObIArray &partition_ids, + const ObIArray &column_ids, + hash::ObHashMap &having_stat_part_col_map, + const int64_t saving_time) +{ + int ret = OB_SUCCESS; + if (having_stat_part_col_map.size() != 0) {//only process have stat part col. + ObSqlString raw_sql; + ObSqlString where_str; + ObSqlString extra_partition_str; + ObSqlString extra_column_str; + ObSqlString extra_where_str; + ObSqlString partition_list; + ObSqlString column_list; + int64_t affected_rows = 0; + if (is_specify_partition_gather && OB_FAIL(gen_partition_list(partition_ids, partition_list))) { + LOG_WARN("failed to gen partition list", K(ret)); + } else if (is_specify_partition_gather && + OB_FAIL(extra_partition_str.append_fmt(" and partition_id in %s", partition_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (is_specify_column_gather && OB_FAIL(gen_column_list(column_ids, column_list))) { + LOG_WARN("failed to gen column list", K(ret)); + } else if (is_specify_column_gather && + OB_FAIL(extra_column_str.append_fmt(" and column_id in %s", column_list.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if ((is_specify_partition_gather || is_specify_column_gather) && + OB_FAIL(extra_where_str.append_fmt("%s%s", + is_specify_partition_gather ? extra_partition_str.ptr() : " ", + is_specify_column_gather ? extra_column_str.ptr() : " "))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(where_str.append_fmt(" tenant_id = %lu and table_id = %lu %s", + share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), + share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id), + (is_specify_partition_gather || is_specify_column_gather) ? extra_where_str.ptr() : " "))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(raw_sql.append_fmt(INSERT_HISTOGRAM_STAT_HISTORY, + share::OB_ALL_HISTOGRAM_STAT_HISTORY_TNAME, + saving_time, + share::OB_ALL_HISTOGRAM_STAT_TNAME, + where_str.ptr()))) { + LOG_WARN("failed to append fmt", K(ret)); + } else if (OB_FAIL(trans.write(tenant_id, raw_sql.ptr(), affected_rows))) { + LOG_WARN("fail to exec sql", K(raw_sql), K(ret)); + } else { + LOG_TRACE("succeed to backup having column stats", K(raw_sql), K(affected_rows)); + } } return ret; } @@ -302,23 +1074,35 @@ int ObDbmsStatsHistoryManager::restore_table_stats(ObExecContext &ctx, int ret = OB_SUCCESS; ObSEArray all_tstats; ObSEArray all_cstats; - ObSEArray history_tab_handles; - ObSEArray history_col_handles; - //before restore, we need record history stats. - if (OB_FAIL(get_history_stat_handles(ctx, param, history_tab_handles, history_col_handles))) { - LOG_WARN("failed to get history stat handles", K(ret)); - } else if (OB_FAIL(fetch_table_stat_histrory(ctx, param, specify_time, all_tstats))) { + //TODO, we need split it, avoiding the column stat is too many and the memory isn't enough. + if (OB_FAIL(fetch_table_stat_histrory(ctx, param, specify_time, all_tstats))) { LOG_WARN("failed to fetch table stat histrory", K(ret)); } else if (OB_FAIL(fetch_column_stat_history(ctx, param, specify_time, all_cstats))) { LOG_WARN("failed to fetch column stat history", K(ret)); } else if (all_tstats.empty() && all_cstats.empty()) { //do nothing - } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, all_tstats, all_cstats))) { - LOG_WARN("failed to split batch write", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::batch_write_history_stats(ctx, history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to batch write history stats", K(ret)); - } else {/*do nothing*/} + } else { + ObMySQLTransaction trans; + //begin trans + if (OB_FAIL(trans.start(ctx.get_sql_proxy(), param.tenant_id_))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(backup_opt_stats(ctx, trans, param, ObTimeUtility::current_time()))) { + LOG_WARN("failed to backup opt stats", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, trans.get_connection(), all_tstats, all_cstats))) { + LOG_WARN("failed to split batch write", K(ret)); + } else {/*do nothing*/} + //end trans + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } + } return ret; } @@ -584,21 +1368,28 @@ int ObDbmsStatsHistoryManager::fill_column_stat_history(ObIAllocator &allocator, EXTRACT_VARCHAR_FIELD_MYSQL(result, "distinct_cnt_synopsis", hex_str); char *bitmap_buf = NULL; if (OB_SUCC(ret) && llc_bitmap_size > 0) { - if (NULL == (bitmap_buf = static_cast(allocator.alloc(hex_str.length())))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("allocate memory for llc_bitmap failed.", K(hex_str.length()), K(ret)); - } else { - common::str_to_hex(hex_str.ptr(), hex_str.length(), bitmap_buf, hex_str.length()); - // decompress llc bitmap; - char *decomp_buf = NULL ; - int64_t decomp_size = ObOptColumnStat::NUM_LLC_BUCKET; - const int64_t bitmap_size = hex_str.length() / 2; - if (OB_FAIL(ObOptStatSqlService::get_decompressed_llc_bitmap(allocator, bitmap_buf, - bitmap_size, decomp_buf, - decomp_size))) { - COMMON_LOG(WARN, "decompress bitmap buffer failed.", K(ret)); + int64_t compress_type = ObOptStatCompressType::MAX_COMPRESS; + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "compress_type", compress_type, int64_t, true, false, ObOptStatCompressType::ZLIB_COMPRESS); + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(compress_type < 0 || compress_type >= ObOptStatCompressType::MAX_COMPRESS)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(compress_type)); + } else if (NULL == (bitmap_buf = static_cast(allocator.alloc(hex_str.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory for llc_bitmap failed.", K(hex_str.length()), K(ret)); } else { - col_stat->set_llc_bitmap(decomp_buf, decomp_size); + common::str_to_hex(hex_str.ptr(), hex_str.length(), bitmap_buf, hex_str.length()); + // decompress llc bitmap; + char *decomp_buf = NULL ; + int64_t decomp_size = ObOptColumnStat::NUM_LLC_BUCKET; + const int64_t bitmap_size = hex_str.length() / 2; + if (OB_FAIL(ObOptStatSqlService::get_decompressed_llc_bitmap(allocator, bitmap_compress_lib_name[compress_type], + bitmap_buf, bitmap_size, + decomp_buf, decomp_size))) { + COMMON_LOG(WARN, "decompress bitmap buffer failed.", K(ret)); + } else { + col_stat->set_llc_bitmap(decomp_buf, decomp_size); + } } } } @@ -778,6 +1569,11 @@ int ObDbmsStatsHistoryManager::gen_partition_list(const ObTableStatParam ¶m, LOG_WARN("failed to push back", K(ret), K(param)); } } + for (int64_t i = 0; OB_SUCC(ret) && i < param.approx_part_infos_.count(); ++i) { + if (OB_FAIL(partition_ids.push_back(param.approx_part_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret), K(param)); + } + } } if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) { for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { @@ -790,14 +1586,37 @@ int ObDbmsStatsHistoryManager::gen_partition_list(const ObTableStatParam ¶m, if (OB_UNLIKELY(partition_ids.empty())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(param), K(ret)); + } else if (OB_FAIL(gen_partition_list(partition_ids, partition_list))) { + LOG_WARN("failed to gen partition list", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); i++) { - char prefix = (i == 0 ? '(' : ' '); - char suffix = (i == partition_ids.count() - 1 ? ')' : ','); - if (OB_FAIL(partition_list.append_fmt("%c%ld%c", prefix, partition_ids.at(i), suffix))) { - LOG_WARN("failed to append sql", K(ret)); - } else {/*do nothing*/} - } + } + return ret; +} + +int ObDbmsStatsHistoryManager::gen_partition_list(const ObIArray &partition_ids, + ObSqlString &partition_list) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < partition_ids.count(); i++) { + char prefix = (i == 0 ? '(' : ' '); + char suffix = (i == partition_ids.count() - 1 ? ')' : ','); + if (OB_FAIL(partition_list.append_fmt("%c%ld%c", prefix, partition_ids.at(i), suffix))) { + LOG_WARN("failed to append sql", K(ret)); + } else {/*do nothing*/} + } + return ret; +} + +int ObDbmsStatsHistoryManager::gen_column_list(const ObIArray &column_ids, + ObSqlString &column_list) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < column_ids.count(); i++) { + char prefix = (i == 0 ? '(' : ' '); + char suffix = (i == column_ids.count() - 1 ? ')' : ','); + if (OB_FAIL(column_list.append_fmt("%c%lu%c", prefix, column_ids.at(i), suffix))) { + LOG_WARN("failed to append sql", K(ret)); + } else {/*do nothing*/} } return ret; } diff --git a/src/share/stat/ob_dbms_stats_history_manager.h b/src/share/stat/ob_dbms_stats_history_manager.h index d5c5b76169..0b14ff83dc 100644 --- a/src/share/stat/ob_dbms_stats_history_manager.h +++ b/src/share/stat/ob_dbms_stats_history_manager.h @@ -30,10 +30,12 @@ static int64_t MAX_HISTORY_RETENTION = 365001; class ObDbmsStatsHistoryManager { public: - static int get_history_stat_handles(ObExecContext &ctx, - const ObTableStatParam ¶m, - ObIArray &history_tab_handles, - ObIArray &history_col_handles); + + static int backup_opt_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + int64_t saving_time, + bool is_backup_for_gather = false); static int restore_table_stats(ObExecContext &ctx, const ObTableStatParam ¶m, @@ -88,8 +90,84 @@ private: static int gen_partition_list(const ObTableStatParam ¶m, ObSqlString &partition_list); + static int gen_partition_list(const ObIArray &partition_ids, + ObSqlString &partition_list); + + static int gen_column_list(const ObIArray &column_ids, + ObSqlString &column_list); + static int remove_useless_column_stats(ObMySQLTransaction &trans, uint64_t tenant_id); + static int backup_table_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const int64_t saving_time, + ObIArray &part_ids); + + static int calssify_table_stat_part_ids(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_partition_gather, + const ObIArray &partition_ids, + ObIArray &no_stat_part_ids, + ObIArray &have_stat_part_ids); + + static int backup_having_table_part_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_partition_gather, + const ObIArray &partition_ids, + const int64_t saving_time); + + static int backup_no_table_part_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &partition_ids, + const int64_t saving_time); + + static int backup_column_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + const int64_t saving_time, + const ObIArray &part_ids, + const ObIArray &column_ids); + + static int generate_having_stat_part_col_map(ObExecContext &ctx, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_partition_gather, + const bool is_specify_column_gather, + const ObIArray &partition_ids, + const ObIArray &column_ids, + hash::ObHashMap &have_stat_part_col_map); + + static int backup_having_column_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_gather, + const ObIArray &partition_ids, + const ObIArray &column_ids, + hash::ObHashMap &having_stat_part_col_map, + const int64_t saving_time); + + static int backup_no_column_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &partition_ids, + const ObIArray &column_ids, + hash::ObHashMap &having_stat_part_col_map, + const int64_t saving_time); + + static int backup_histogram_stats(ObMySQLTransaction &trans, + const uint64_t tenant_id, + const uint64_t table_id, + const bool is_specify_partition_gather, + const bool is_specify_column_gather, + const ObIArray &partition_ids, + const ObIArray &column_ids, + hash::ObHashMap &having_stat_part_col_map, + const int64_t saving_time); + }; diff --git a/src/share/stat/ob_dbms_stats_lock_unlock.cpp b/src/share/stat/ob_dbms_stats_lock_unlock.cpp index 001239c1bc..668e1a7700 100644 --- a/src/share/stat/ob_dbms_stats_lock_unlock.cpp +++ b/src/share/stat/ob_dbms_stats_lock_unlock.cpp @@ -69,19 +69,17 @@ int ObDbmsStatsLockUnlock::set_table_stats_lock(ObExecContext &ctx, ObSEArray no_stats_partition_ids;//used to save partition which have no stats ObSEArray part_stattypes; ObSEArray dummy_array; - ObSEArray history_tab_handles; - ObSEArray history_col_handles; uint64_t tenant_id = param.tenant_id_; uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id); uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, param.table_id_); if (OB_ISNULL(mysql_proxy = ctx.get_sql_proxy())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(mysql_proxy)); - } else if (OB_FAIL(get_stats_history_sql(ctx, param, set_locked, + } else if (OB_FAIL(trans.start(mysql_proxy, param.tenant_id_))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(get_stats_history_sql(ctx, trans, param, set_locked, need_update_lock, no_stats_partition_ids, - part_stattypes, - history_tab_handles, - history_col_handles))) { + part_stattypes))) { LOG_WARN("failed to get stats history sql", K(ret)); } else if (!need_update_lock) { LOG_TRACE("no need update lock", K(need_update_lock), K(param), K(set_locked)); @@ -98,8 +96,6 @@ int ObDbmsStatsLockUnlock::set_table_stats_lock(ObExecContext &ctx, } else if (OB_FAIL(get_insert_locked_type_sql(param, no_stats_partition_ids, part_stattypes, insert_sql))) { LOG_WARN("failed to get insert locked type sql", K(ret)); - } else if (OB_FAIL(trans.start(mysql_proxy, param.tenant_id_))) { - LOG_WARN("fail to start transaction", K(ret)); } else if (OB_FAIL(trans.write(param.tenant_id_, raw_sql.ptr(), affected_rows))) { LOG_WARN("fail to exec sql", K(raw_sql), K(ret)); } else if (!insert_sql.empty() && @@ -111,10 +107,6 @@ int ObDbmsStatsLockUnlock::set_table_stats_lock(ObExecContext &ctx, if (OB_SUCC(ret)) { if (OB_FAIL(trans.end(true))) { OB_LOG(WARN, "failed to commit", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::batch_write_history_stats(ctx, - history_tab_handles, - history_col_handles))) { - LOG_WARN("failed to batch write history stats", K(ret)); } else {/*do nothing*/} } else { int tmp_ret = OB_SUCCESS; @@ -126,13 +118,12 @@ int ObDbmsStatsLockUnlock::set_table_stats_lock(ObExecContext &ctx, } int ObDbmsStatsLockUnlock::get_stats_history_sql(ObExecContext &ctx, + ObMySQLTransaction &trans, const ObTableStatParam ¶m, bool set_locked, bool &need_update_lock, ObIArray &no_stats_partition_ids, - ObIArray &part_stattypes, - ObIArray &history_tab_handles, - ObIArray &history_col_handles) + ObIArray &part_stattypes) { int ret = OB_SUCCESS; ObSqlString raw_sql; @@ -189,9 +180,7 @@ int ObDbmsStatsLockUnlock::get_stats_history_sql(ObExecContext &ctx, if (OB_SUCC(ret) && need_update_lock) { //before lock, we need record history stats. - if (OB_FAIL(ObDbmsStatsHistoryManager::get_history_stat_handles(ctx, param, - history_tab_handles, - history_col_handles))) { + if (OB_FAIL(ObDbmsStatsHistoryManager::backup_opt_stats(ctx, trans, param, ObTimeUtility::current_time()))) { LOG_WARN("failed to get history stats", K(ret)); } else {/*do nothing*/} } @@ -229,11 +218,9 @@ int ObDbmsStatsLockUnlock::check_stat_locked(ObExecContext &ctx, locked_partition_ids, dummy_array))) { LOG_WARN("failed to get stat locked partition ids", K(ret)); - } else if (locked_partition_ids.empty()) {//no locked table - /*do nothing*/ - } else if (!param.is_index_param() - ? param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_ZERO - : is_partition_id_locked(param.global_data_part_id_, locked_partition_ids, dummy_idx)) { + } else if (!locked_partition_ids.empty() && + !param.is_index_param() ? param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_ZERO + : is_partition_id_locked(param.global_data_part_id_, locked_partition_ids, dummy_idx)) { //check the data table is locked for gather_index_stats ret = OB_ERR_DBMS_STATS_PL; LOG_WARN("object statistics are locked", K(ret), K(locked_partition_ids)); @@ -406,93 +393,101 @@ int ObDbmsStatsLockUnlock::adjust_table_stat_param(const ObIArray &lock ObTableStatParam ¶m) { int ret = OB_SUCCESS; - bool has_valid_partition_id = false; + bool has_valid_partition_id = locked_partition_ids.empty(); ObSEArray new_part_infos; - ObSEArray new_subpart_infos; int64_t idx = -1; - if (locked_partition_ids.empty()) { - /*do nothing*/ - } else { - if (param.global_stat_param_.need_modify_) { - int64_t part_id = param.global_part_id_; - if (is_partition_id_locked(part_id, locked_partition_ids, idx)) { - param.global_stat_param_.reset_gather_stat(); + bool has_part_locked = false; + ObSEArray new_subpart_infos; + for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { + if (!is_partition_id_locked(param.subpart_infos_.at(i).part_id_, locked_partition_ids, idx)) { + if (OB_FAIL(new_subpart_infos.push_back(param.subpart_infos_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); } else { - has_valid_partition_id = true; + has_valid_partition_id |= param.subpart_stat_param_.need_modify_; } + } else if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.subpart_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } + if (OB_SUCC(ret)) { + if (OB_FAIL(param.subpart_infos_.assign(new_subpart_infos))) { + LOG_WARN("failed to assign", K(ret)); + } else if (param.subpart_stat_param_.need_modify_) { + param.subpart_stat_param_.need_modify_ = !new_subpart_infos.empty(); } - if (param.subpart_stat_param_.need_modify_) { - for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { - if (!is_partition_id_locked(param.subpart_infos_.at(i).part_id_, locked_partition_ids, idx)) { - if (OB_FAIL(new_subpart_infos.push_back(param.subpart_infos_.at(i)))) { - LOG_WARN("failed to push back", K(ret)); - } else { - has_valid_partition_id = true; - } - } else if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.subpart_infos_.at(i).part_id_))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} - } - } - if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) { - for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { - if (!is_partition_id_locked(param.part_infos_.at(i).part_id_, locked_partition_ids, idx)) { - if (ObIncrementalStatEstimator::is_part_can_incremental_gather( - param, - param.part_infos_.at(i).part_id_, - param.part_infos_.at(i).subpart_cnt_, - true)) { - if (OB_FAIL(param.approx_part_infos_.push_back(param.part_infos_.at(i)))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} - } else if (OB_FAIL(new_part_infos.push_back(param.part_infos_.at(i)))) { - LOG_WARN("failed to push back", K(ret)); - } else { - has_valid_partition_id = true; - } - } else if (ObIncrementalStatEstimator::is_part_can_incremental_gather( - param, - param.part_infos_.at(i).part_id_, - param.part_infos_.at(i).subpart_cnt_, - false)) { + } + if (OB_SUCC(ret)) { + ObSEArray new_part_infos; + for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { + if (!is_partition_id_locked(param.part_infos_.at(i).part_id_, locked_partition_ids, idx)) { + has_valid_partition_id |= param.part_stat_param_.need_modify_; + if (param.part_stat_param_.can_use_approx_ && + param.subpart_stat_param_.need_modify_ && + param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { if (OB_FAIL(param.approx_part_infos_.push_back(param.part_infos_.at(i)))) { LOG_WARN("failed to push back", K(ret)); } else {/*do nothing*/} - } else if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.part_infos_.at(i).part_id_))) { + } else if (OB_FAIL(new_part_infos.push_back(param.part_infos_.at(i)))) { LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} + } + } else if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.part_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else { + has_part_locked = true; } } if (OB_SUCC(ret)) { - //1.lock all partition - //2.specify partition name and the partition is be locked; - if (!has_valid_partition_id || - (param.is_subpart_name_ && new_subpart_infos.empty()) || - (!param.part_name_.empty() && !param.is_subpart_name_ && - new_part_infos.empty() && param.approx_part_infos_.empty())) { - ret = OB_ERR_DBMS_STATS_PL; - LOG_WARN("object statistics are locked", K(ret), K(locked_partition_ids), K(param), - K(new_subpart_infos), K(new_part_infos)); - LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"object statistics are locked"); - } else if (OB_FAIL(param.subpart_infos_.assign(new_subpart_infos))) { - LOG_WARN("failed to assign", K(ret)); - } else if (OB_FAIL(param.part_infos_.assign(new_part_infos))) { + if (OB_FAIL(param.part_infos_.assign(new_part_infos))) { LOG_WARN("failed to assign", K(ret)); + } else if (param.part_stat_param_.need_modify_) { + param.part_stat_param_.need_modify_ = !new_part_infos.empty() || !param.approx_part_infos_.empty(); + param.part_stat_param_.can_use_approx_ = !param.approx_part_infos_.empty(); + } + } + } + if (OB_SUCC(ret)) { + int64_t part_id = param.global_part_id_; + if (is_partition_id_locked(part_id, locked_partition_ids, idx)) { + param.global_stat_param_.reset_gather_stat(); + } else if (param.global_stat_param_.need_modify_) { + has_valid_partition_id = true; + if (param.global_stat_param_.gather_approx_ && + (has_part_locked || !param.part_stat_param_.need_modify_)) { + param.global_stat_param_.gather_approx_ = false; + } + } + } + if (OB_SUCC(ret)) { + //1.lock all partition + //2.specify partition name and the partition is be locked; + if (!has_valid_partition_id || + (param.is_subpart_name_ && param.subpart_infos_.empty()) || + (!param.part_name_.empty() && !param.is_subpart_name_ && + param.part_infos_.empty() && param.approx_part_infos_.empty())) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("object statistics are locked", K(ret), K(locked_partition_ids), K(param)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL,"object statistics are locked"); + } else if (param.part_stat_param_.can_use_approx_ && + param.subpart_stat_param_.need_modify_ && + param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO && + !param.part_name_.empty()) { + if (OB_UNLIKELY(param.approx_part_infos_.count() != 1 || param.subpart_infos_.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param)); } else { - param.subpart_stat_param_.need_modify_ = !new_subpart_infos.empty(); - param.part_stat_param_.need_modify_ = !new_part_infos.empty(); - if (param.global_stat_param_.need_modify_ && - param.global_stat_param_.gather_approx_ && - !param.part_stat_param_.need_modify_) { - //if take approx global and partition to gather, but all partition are locked, should adjust - //approx global gather to global gather. - if (!param.subpart_stat_param_.need_modify_) { - param.global_stat_param_.gather_approx_ = false; - } else { //incremental partition gather stats from subpart stats. - param.part_stat_param_.need_modify_ = true; + for (int64_t i = 0; OB_SUCC(ret) && i < param.all_subpart_infos_.count(); ++i) { + if (param.all_subpart_infos_.at(i).first_part_id_ == param.approx_part_infos_.at(0).part_id_) { + bool found_it = false; + for (int64_t j = 0; !found_it && j < param.subpart_infos_.count(); ++j) { + found_it = param.all_subpart_infos_.at(i).part_id_ == param.subpart_infos_.at(j).part_id_; + } + if (!found_it) { + if (OB_FAIL(param.no_regather_partition_ids_.push_back(param.all_subpart_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } } } - LOG_TRACE("Succeed to adjust table stat param", K(param), K(locked_partition_ids)); } } } diff --git a/src/share/stat/ob_dbms_stats_lock_unlock.h b/src/share/stat/ob_dbms_stats_lock_unlock.h index 5b10deb43f..7061624cb6 100644 --- a/src/share/stat/ob_dbms_stats_lock_unlock.h +++ b/src/share/stat/ob_dbms_stats_lock_unlock.h @@ -65,13 +65,12 @@ private: int64_t &idx); static int get_stats_history_sql(ObExecContext &ctx, + ObMySQLTransaction &trans, const ObTableStatParam ¶m, bool set_locked, bool &need_update_lock, ObIArray &no_stats_partition_ids, - ObIArray &part_stattypes, - ObIArray &history_tab_handles, - ObIArray &history_col_handles); + ObIArray &part_stattypes); static int get_no_stats_partition_ids(const StatTypeLocked stattype, const ObIArray &all_partition_ids, diff --git a/src/share/stat/ob_dbms_stats_maintenance_window.cpp b/src/share/stat/ob_dbms_stats_maintenance_window.cpp index 71ca764502..32dc06e4cb 100644 --- a/src/share/stat/ob_dbms_stats_maintenance_window.cpp +++ b/src/share/stat/ob_dbms_stats_maintenance_window.cpp @@ -231,7 +231,7 @@ int ObDbmsStatsMaintenanceWindow::get_stats_history_manager_job_sql(const bool i OZ (dml.add_column("credential_name", ObHexEscapeSqlStr(ObString("")))); OZ (dml.add_column("destination_name", ObHexEscapeSqlStr(ObString("")))); OZ (dml.add_column("interval_ts", interval_ts)); - OZ (dml.add_column(true, "max_run_duration")); // add null column. + OZ (dml.add_column("max_run_duration", DEFAULT_HISTORY_MANAGER_DURATION_SEC)); OZ (dml.splice_values(raw_sql)); return ret; } diff --git a/src/share/stat/ob_dbms_stats_maintenance_window.h b/src/share/stat/ob_dbms_stats_maintenance_window.h index 51a824e4b8..ccca65545a 100644 --- a/src/share/stat/ob_dbms_stats_maintenance_window.h +++ b/src/share/stat/ob_dbms_stats_maintenance_window.h @@ -32,6 +32,7 @@ #define DEFAULT_NON_WORKING_DAY_DURATION_SEC (20 * 60 * 60) #define DEFAULT_NON_WORKING_DAY_DURATION_USEC (20 * 60 * 60 * 1000000LL) #define DEFAULT_DML_STATS_INTERVAL_USEC (15*60*1000000LL) +#define DEFAULT_HISTORY_MANAGER_DURATION_SEC (12 * 60 * 60) namespace oceanbase { diff --git a/src/share/stat/ob_dbms_stats_utils.cpp b/src/share/stat/ob_dbms_stats_utils.cpp index ca89627530..4e95c72ebe 100644 --- a/src/share/stat/ob_dbms_stats_utils.cpp +++ b/src/share/stat/ob_dbms_stats_utils.cpp @@ -31,44 +31,14 @@ namespace oceanbase { namespace common { -int ObDbmsStatsUtils::get_part_info(const ObTableStatParam ¶m, - const ObExtraParam &extra, - PartInfo &part_info) -{ - int ret = OB_SUCCESS; - if (extra.type_ == TABLE_LEVEL) { - part_info.part_name_ = param.tab_name_; - part_info.part_stattype_ = param.stattype_; - part_info.part_id_ = param.global_part_id_; - part_info.tablet_id_ = param.global_part_id_; - } else if (extra.type_ == PARTITION_LEVEL) { - if (OB_UNLIKELY(extra.nth_part_ >= param.part_infos_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid partition index", - K(ret), K(extra.nth_part_), K(param.part_infos_.count())); - } else { - part_info = param.part_infos_.at(extra.nth_part_); - } - } else if (extra.type_ == SUBPARTITION_LEVEL) { - if (OB_UNLIKELY(extra.nth_part_ >= param.subpart_infos_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid subpartition index", - K(ret), K(extra.nth_part_), K(param.subpart_infos_.count())); - } else { - part_info = param.subpart_infos_.at(extra.nth_part_); - } - } - return ret; -} int ObDbmsStatsUtils::init_col_stats(ObIAllocator &allocator, int64_t col_cnt, ObIArray &col_stats) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(col_cnt <= 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error, expected specify column cnt is great 0", K(ret), K(col_cnt)); + if (col_cnt <= 0) { + //do nothing } else if (OB_FAIL(col_stats.prepare_allocate(col_cnt))) { LOG_WARN("failed to prepare allocate column stat", K(ret)); } else { @@ -120,28 +90,25 @@ int ObDbmsStatsUtils::check_range_skew(ObHistType hist_type, int ObDbmsStatsUtils::batch_write(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, ObIArray &table_stats, ObIArray &column_stats, const int64_t current_time, const bool is_index_stat, - const bool is_history_stat, const bool is_online_stat, const ObObjPrintParams &print_params) { int ret = OB_SUCCESS; if (OB_FAIL(ObOptStatManager::get_instance().batch_write(schema_guard, tenant_id, - trans, + conn, table_stats, column_stats, current_time, is_index_stat, - is_history_stat, print_params))) { LOG_WARN("failed to batch write stats", K(ret)); - //histroy stat is from cache no need free. - } else if (!is_history_stat && !is_online_stat) { + } else if (!is_online_stat) { for (int64_t i = 0; OB_SUCC(ret) && i < table_stats.count(); ++i) { if (NULL != table_stats.at(i)) { table_stats.at(i)->~ObOptTableStat(); @@ -164,7 +131,7 @@ int ObDbmsStatsUtils::cast_number_to_double(const number::ObNumber &src_val, dou ObObj src_obj; ObObj dest_obj; src_obj.set_number(src_val); - ObArenaAllocator calc_buf(ObModIds::OB_SQL_PARSER); + ObArenaAllocator calc_buf("ObDbmsStatsUtil", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); ObCastCtx cast_ctx(&calc_buf, NULL, CM_NONE, ObCharset::get_system_collation()); if (OB_FAIL(ObObjCaster::to_type(ObDoubleType, cast_ctx, src_obj, dest_obj))) { LOG_WARN("failed to cast number to double type", K(ret)); @@ -283,6 +250,7 @@ bool ObDbmsStatsUtils::is_no_stat_virtual_table(const int64_t table_id) table_id == share::OB_ALL_VIRTUAL_TRANS_LOCK_STAT_TID || table_id == share::OB_ALL_VIRTUAL_TRANS_SCHEDULER_TID || table_id == share::OB_ALL_VIRTUAL_SQL_AUDIT_TID || + table_id == share::OB_TENANT_VIRTUAL_SHOW_RESTORE_PREVIEW_TID || table_id == share::OB_ALL_VIRTUAL_SESSTAT_ORA_TID || table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_TABLE_ORA_TID || table_id == share::OB_TENANT_VIRTUAL_SHOW_CREATE_PROCEDURE_ORA_TID || @@ -361,20 +329,35 @@ int ObDbmsStatsUtils::split_batch_write(sql::ObExecContext &ctx, ObIArray &table_stats, ObIArray &column_stats, const bool is_index_stat/*default false*/, - const bool is_history_stat/*default false*/, const bool is_online_stat /*default false*/) { int ret = OB_SUCCESS; - if (OB_FAIL(split_batch_write(ctx.get_virtual_table_ctx().schema_guard_, - ctx.get_my_session(), - ctx.get_sql_proxy(), - table_stats, - column_stats, - is_index_stat, - is_history_stat, - is_online_stat))) { + ObMySQLTransaction trans; + if (OB_ISNULL(ctx.get_my_session()) || OB_ISNULL(ctx.get_sql_proxy())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(ctx.get_my_session()), K(ctx.get_sql_proxy())); + } else if (OB_FAIL(trans.start(ctx.get_sql_proxy(), ctx.get_my_session()->get_effective_tenant_id()))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(split_batch_write(trans.get_connection(), + ctx.get_virtual_table_ctx().schema_guard_, + ctx.get_my_session(), + table_stats, + column_stats, + is_index_stat, + is_online_stat))) { LOG_WARN("failed to split batch write", K(ret)); } + //end trans after writing stats. + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } return ret; } @@ -384,7 +367,64 @@ int ObDbmsStatsUtils::split_batch_write(share::schema::ObSchemaGetterGuard *sche ObIArray &table_stats, ObIArray &column_stats, const bool is_index_stat/*default false*/, - const bool is_history_stat/*default false*/, + const bool is_online_stat /*default false*/) +{ + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + if (OB_ISNULL(session_info) || OB_ISNULL(sql_proxy)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(session_info), K(sql_proxy)); + } else if (OB_FAIL(trans.start(sql_proxy, session_info->get_effective_tenant_id()))) { + LOG_WARN("fail to start transaction", K(ret)); + } else if (OB_FAIL(split_batch_write(trans.get_connection(), + schema_guard, + session_info, + table_stats, + column_stats, + is_index_stat, + is_online_stat))) { + LOG_WARN("failed to split batch write", K(ret)); + } + //end trans after writing stats. + if (OB_SUCC(ret)) { + if (OB_FAIL(trans.end(true))) { + LOG_WARN("fail to commit transaction", K(ret)); + } + } else { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(false))) { + LOG_WARN("fail to roll back transaction", K(tmp_ret)); + } + } + return ret; +} + +int ObDbmsStatsUtils::split_batch_write(sql::ObExecContext &ctx, + sqlclient::ObISQLConnection *conn, + ObIArray &table_stats, + ObIArray &column_stats, + const bool is_index_stat/*default false*/, + const bool is_online_stat /*default false*/) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(split_batch_write(conn, + ctx.get_virtual_table_ctx().schema_guard_, + ctx.get_my_session(), + table_stats, + column_stats, + is_index_stat, + is_online_stat))) { + LOG_WARN("failed to split batch write", K(ret)); + } + return ret; +} + +int ObDbmsStatsUtils::split_batch_write(sqlclient::ObISQLConnection *conn, + share::schema::ObSchemaGetterGuard *schema_guard, + sql::ObSQLSessionInfo *session_info, + ObIArray &table_stats, + ObIArray &column_stats, + const bool is_index_stat/*default false*/, const bool is_online_stat /*default false*/) { int ret = OB_SUCCESS; @@ -393,15 +433,10 @@ int ObDbmsStatsUtils::split_batch_write(share::schema::ObSchemaGetterGuard *sche //avoid the write stat sql is too long, we split write table stats and column stats: // write 2000 tables and 2000 columns every time. LOG_DEBUG("dbms stats write stats", K(table_stats), K(column_stats)); - const int64_t MAX_NUM_OF_WRITE_STATS = 2000; int64_t current_time = ObTimeUtility::current_time(); - ObMySQLTransaction trans; - //begin trans before writing stats - if (OB_ISNULL(session_info) || OB_ISNULL(sql_proxy)) { + if (OB_ISNULL(session_info)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(session_info), K(sql_proxy)); - } else if (OB_FAIL(trans.start(sql_proxy, session_info->get_effective_tenant_id()))) { - LOG_WARN("fail to start transaction", K(ret)); + LOG_WARN("get unexpected null", K(ret), K(session_info)); } while (OB_SUCC(ret) && (idx_tab_stat < table_stats.count() || idx_col_stat < column_stats.count())) { @@ -441,54 +476,17 @@ int ObDbmsStatsUtils::split_batch_write(share::schema::ObSchemaGetterGuard *sche if (OB_SUCC(ret)) { if (OB_FAIL(ObDbmsStatsUtils::batch_write(schema_guard, session_info->get_effective_tenant_id(), - trans, + conn, write_table_stats, write_column_stats, current_time, is_index_stat, - is_history_stat, is_online_stat, CREATE_OBJ_PRINT_PARAM(session_info)))) { LOG_WARN("failed to batch write stats", K(ret), K(idx_tab_stat), K(idx_col_stat)); } else {/*do nothing*/} } } - //end trans after writing stats. - if (OB_SUCC(ret)) { - if (OB_FAIL(trans.end(true))) { - LOG_WARN("fail to commit transaction", K(ret)); - } - } else { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(false))) { - LOG_WARN("fail to roll back transaction", K(tmp_ret)); - } - } - return ret; -} - -int ObDbmsStatsUtils::batch_write_history_stats(sql::ObExecContext &ctx, - ObIArray &history_tab_handles, - ObIArray &history_col_handles) -{ - int ret = OB_SUCCESS; - ObSEArray table_stats; - ObSEArray column_stats; - for (int64_t i = 0; OB_SUCC(ret) && i < history_tab_handles.count(); ++i) { - if (OB_FAIL(table_stats.push_back(const_cast(history_tab_handles.at(i).stat_)))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} - } - for (int64_t i = 0; OB_SUCC(ret) && i < history_col_handles.count(); ++i) { - if (OB_FAIL(column_stats.push_back(const_cast(history_col_handles.at(i).stat_)))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} - } - if (OB_SUCC(ret)) { - if (OB_FAIL(split_batch_write(ctx, table_stats, column_stats, false, true))) { - LOG_WARN("failed to split batch write", K(ret)); - } else {/*do nothing*/} - } return ret; } @@ -578,28 +576,31 @@ int ObDbmsStatsUtils::calssify_opt_stat(const ObIArray &opt_stats, // else merge old and new stats; int ObDbmsStatsUtils::merge_tab_stats(const ObTableStatParam ¶m, const TabStatIndMap &online_table_stats, - common::ObIArray &old_tab_handles, + common::ObIArray &old_tab_stats, common::ObIArray &dst_tab_stats) { int ret = OB_SUCCESS; ObOptTableStat *tmp_tab_stat = NULL; // the map is faster than array traversal - for (int64_t i = 0; OB_SUCC(ret) && i < old_tab_handles.count(); i++) { - ObOptTableStat * old_tab_stat = const_cast(old_tab_handles.at(i).stat_); - ObOptTableStat::Key key(param.tenant_id_, old_tab_stat->get_table_id(), - old_tab_stat->get_partition_id()); - if (OB_FAIL(online_table_stats.get_refactored(key, tmp_tab_stat))) { - if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to find in hashmap", K(ret)); - } else { - if (OB_FAIL(dst_tab_stats.push_back(old_tab_stat))) { - LOG_WARN("fail to push back table stats", K(ret)); + for (int64_t i = 0; OB_SUCC(ret) && i < old_tab_stats.count(); i++) { + if (OB_ISNULL(old_tab_stats.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(old_tab_stats.at(i))); + } else { + ObOptTableStat::Key key(param.tenant_id_, + old_tab_stats.at(i)->get_table_id(), + old_tab_stats.at(i)->get_partition_id()); + if (OB_FAIL(online_table_stats.get_refactored(key, tmp_tab_stat))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("failed to find in hashmap", K(ret)); + } else { + ret = OB_SUCCESS; } + } else if (OB_FAIL(tmp_tab_stat->merge_table_stat(*old_tab_stats.at(i)))) { + //merge + LOG_WARN("fail to merge new table stat with old table stat", K(ret)); } - } else if (OB_FAIL(tmp_tab_stat->merge_table_stat(*old_tab_stat))) { - //merge - LOG_WARN("fail to merge new table stat with old table stat", K(ret)); } } @@ -611,32 +612,36 @@ int ObDbmsStatsUtils::merge_tab_stats(const ObTableStatParam ¶m, } } } - LOG_DEBUG("OSG debug", K(dst_tab_stats)); + LOG_TRACE("merge mtab stats", K(old_tab_stats), K(dst_tab_stats)); return ret; } int ObDbmsStatsUtils::merge_col_stats(const ObTableStatParam ¶m, const ColStatIndMap &online_column_stats, - common::ObIArray &old_col_handles, + common::ObIArray &old_col_stats, common::ObIArray &dst_col_stats) { int ret = OB_SUCCESS; ObOptColumnStat *tmp_col_stat = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < old_col_handles.count(); i++) { - ObOptColumnStat *old_col_stat = const_cast(old_col_handles.at(i).stat_); - ObOptColumnStat::Key key(param.tenant_id_, - old_col_stat->get_table_id(), - old_col_stat->get_partition_id(), - old_col_stat->get_column_id()); - if (OB_FAIL(online_column_stats.get_refactored(key, tmp_col_stat))) { - if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { - LOG_WARN("failed to find in hashmap", K(ret)); - } else if (OB_FAIL(dst_col_stats.push_back(old_col_stat))) { - LOG_WARN("fail to push back table stats", K(ret)); + for (int64_t i = 0; OB_SUCC(ret) && i < old_col_stats.count(); i++) { + if (OB_ISNULL(old_col_stats.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(old_col_stats.at(i))); + } else { + ObOptColumnStat::Key key(param.tenant_id_, + old_col_stats.at(i)->get_table_id(), + old_col_stats.at(i)->get_partition_id(), + old_col_stats.at(i)->get_column_id()); + if (OB_FAIL(online_column_stats.get_refactored(key, tmp_col_stat))) { + if (OB_UNLIKELY(OB_HASH_NOT_EXIST != ret)) { + LOG_WARN("failed to find in hashmap", K(ret)); + } else { + ret = OB_SUCCESS; + } + } else if (OB_FAIL(tmp_col_stat->merge_column_stat(*old_col_stats.at(i)))) { + LOG_WARN("fail to merge new table stat with old table stat", K(ret)); } - } else if (OB_FAIL(tmp_col_stat->merge_column_stat(*old_col_stat))) { - LOG_WARN("fail to merge new table stat with old table stat", K(ret)); } } @@ -654,7 +659,7 @@ int ObDbmsStatsUtils::merge_col_stats(const ObTableStatParam ¶m, } } } - LOG_DEBUG("OSG debug", K(dst_col_stats)); + LOG_TRACE("merge col stats", K(old_col_stats), K(dst_col_stats)); return ret; } @@ -684,32 +689,6 @@ bool ObDbmsStatsUtils::is_part_id_valid(const ObTableStatParam ¶m, return is_valid; } -int ObDbmsStatsUtils::get_part_ids_from_param(const ObTableStatParam ¶m, - common::ObIArray &part_ids) -{ - int ret = OB_SUCCESS; - if (param.global_stat_param_.need_modify_) { - if (OB_FAIL(part_ids.push_back(param.global_part_id_))) { - LOG_WARN("failed to push back partition id", K(ret)); - } - } - if (OB_SUCC(ret) && param.part_stat_param_.need_modify_) { - for (int64_t i = 0; OB_SUCC(ret) && i < param.part_infos_.count(); ++i) { - if (OB_FAIL(part_ids.push_back(param.part_infos_.at(i).part_id_))) { - LOG_WARN("failed to push back partition id", K(ret)); - } - } - } - if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) { - for (int64_t i = 0; OB_SUCC(ret) && i < param.subpart_infos_.count(); ++i) { - if (OB_FAIL(part_ids.push_back(param.subpart_infos_.at(i).part_id_))) { - LOG_WARN("failed to push back partition id", K(ret)); - } - } - } - return ret; -} - int ObDbmsStatsUtils::get_part_infos(const ObTableSchema &table_schema, ObIArray &part_infos, ObIArray &subpart_infos, @@ -870,7 +849,7 @@ int ObDbmsStatsUtils::truncate_string_for_opt_stats(const ObObj *old_obj, } -int ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(ObObj &obj, ObIAllocator &allocator) +int ObDbmsStatsUtils::truncate_string_for_opt_stats(ObObj &obj, ObIAllocator &allocator) { int ret = OB_SUCCESS; if (ObColumnStatParam::is_valid_opt_col_type(obj.get_type()) && obj.is_string_type()) { @@ -888,7 +867,7 @@ int ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(ObObj &obj, ObIAlloca LOG_WARN("failed to convert str to lob", K(ret)); } else { obj.set_type(ori_type); - LOG_TRACE("Succeed to shadow truncate text obj for opt stats", K(ret), K(obj), K(str)); + LOG_TRACE("Succeed to truncate text obj for opt stats", K(ret), K(obj), K(str)); } } else { ObString str = obj.get_string(); @@ -902,7 +881,7 @@ int ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(ObObj &obj, ObIAlloca str.assign_ptr(str.ptr(), static_cast(truncated_str_len)); obj.set_common_value(str); } - LOG_TRACE("Succeed to shadow truncate string obj for opt stats", K(ret), K(obj), K(str), K(truncated_str_len)); + LOG_TRACE("Succeed to truncate string obj for opt stats", K(ret), K(obj), K(str), K(truncated_str_len)); } } return ret; @@ -949,8 +928,7 @@ int ObDbmsStatsUtils::get_current_opt_stats(const ObTableStatParam ¶m, ObSEArray column_ids; if (OB_FAIL(get_part_ids_and_column_ids(param, part_ids, column_ids))) { LOG_WARN("failed to get part ids and column ids", K(ret)); - } else if (OB_FAIL(erase_stat_cache(param.tenant_id_, param.table_id_, - part_ids, column_ids))) { + } else if (OB_FAIL(erase_stat_cache(param.tenant_id_, param.table_id_, part_ids, column_ids))) { LOG_WARN("failed to erase stat cache", K(ret)); } else if (OB_FAIL(ObOptStatManager::get_instance().get_table_stat(param.tenant_id_, param.table_id_, @@ -969,7 +947,8 @@ int ObDbmsStatsUtils::get_current_opt_stats(const ObTableStatParam ¶m, int ObDbmsStatsUtils::get_part_ids_and_column_ids(const ObTableStatParam ¶m, ObIArray &part_ids, - ObIArray &column_ids) + ObIArray &column_ids, + bool need_stat_column /*default false*/) { int ret = OB_SUCCESS; //get part ids @@ -986,6 +965,11 @@ int ObDbmsStatsUtils::get_part_ids_and_column_ids(const ObTableStatParam ¶m, LOG_WARN("failed to push back", K(ret)); } else {/*do nothing*/} } + for (int64_t i = 0; OB_SUCC(ret) && i < param.approx_part_infos_.count(); ++i) { + if (OB_FAIL(part_ids.push_back(param.approx_part_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } } if (OB_SUCC(ret) && param.subpart_stat_param_.need_modify_) { @@ -997,9 +981,11 @@ int ObDbmsStatsUtils::get_part_ids_and_column_ids(const ObTableStatParam ¶m, } //get column ids for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) { - if (OB_FAIL(column_ids.push_back(param.column_params_.at(i).column_id_))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} + if (!need_stat_column || param.column_params_.at(i).need_basic_stat()) { + if (OB_FAIL(column_ids.push_back(param.column_params_.at(i).column_id_))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } } return ret; } @@ -1021,5 +1007,260 @@ int ObDbmsStatsUtils::erase_stat_cache(const uint64_t tenant_id, return ret; } +bool ObDbmsStatsUtils::find_part(const ObIArray &part_infos, + const ObString &part_name, + bool is_sensitive_compare, + PartInfo &part) +{ + bool found = false; + for (int64_t i = 0; !found && i < part_infos.count(); ++i) { + if ((is_sensitive_compare && + ObCharset::case_sensitive_equal(part_name, part_infos.at(i).part_name_)) || + (!is_sensitive_compare && + ObCharset::case_insensitive_equal(part_name, part_infos.at(i).part_name_))) { + part = part_infos.at(i); + found = true; + } + } + return found; +} + +int ObDbmsStatsUtils::remove_stat_gather_param_partition_info(int64_t reserved_partition_id, + ObOptStatGatherParam ¶m) +{ + int ret = OB_SUCCESS; + bool found_it = false; + for (int64_t i = 0; !found_it && i < param.partition_infos_.count(); ++i) { + if (reserved_partition_id == param.partition_infos_.at(i).part_id_) { + found_it = true; + PartInfo tmp_part_info = param.partition_infos_.at(i); + param.partition_infos_.reset(); + if (OB_FAIL(param.partition_infos_.push_back(tmp_part_info))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + if (!found_it) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param), K(reserved_partition_id)); + } + return ret; +} + +int ObDbmsStatsUtils::prepare_gather_stat_param(const ObTableStatParam ¶m, + StatLevel stat_level, + const PartitionIdBlockMap *partition_id_block_map, + bool is_split_gather, + int64_t gather_vectorize, + ObOptStatGatherParam &gather_param) +{ + int ret = OB_SUCCESS; + gather_param.tenant_id_ = param.tenant_id_; + gather_param.db_name_ = param.db_name_; + gather_param.tab_name_ = param.tab_name_; + gather_param.table_id_ = param.table_id_; + gather_param.stat_level_ = stat_level; + if (stat_level == SUBPARTITION_LEVEL) { + gather_param.need_histogram_ = param.subpart_stat_param_.gather_histogram_; + } else if (stat_level == PARTITION_LEVEL) { + gather_param.need_histogram_ = param.part_stat_param_.gather_histogram_; + } else if (stat_level == TABLE_LEVEL) { + gather_param.need_histogram_ = param.global_stat_param_.gather_histogram_; + } + gather_param.sample_info_.is_sample_ = param.sample_info_.is_sample_; + gather_param.sample_info_.is_block_sample_ = param.sample_info_.is_block_sample_; + gather_param.sample_info_.sample_type_ = param.sample_info_.sample_type_; + gather_param.sample_info_.sample_value_ = param.sample_info_.sample_value_; + gather_param.degree_ = param.degree_; + gather_param.max_duration_time_ = param.duration_time_; + gather_param.allocator_ = param.allocator_; + gather_param.partition_id_block_map_ = partition_id_block_map; + gather_param.gather_start_time_ = ObTimeUtility::current_time(); + gather_param.stattype_ = param.stattype_; + gather_param.is_split_gather_ = is_split_gather; + gather_param.need_approx_ndv_ = param.need_approx_ndv_; + gather_param.data_table_name_ = param.data_table_name_; + gather_param.global_part_id_ = param.global_part_id_; + gather_param.gather_vectorize_ = gather_vectorize; + return gather_param.column_group_params_.assign(param.column_group_params_); +} + +int ObDbmsStatsUtils::get_current_opt_stats(ObIAllocator &allocator, + sqlclient::ObISQLConnection *conn, + const ObTableStatParam ¶m, + ObIArray &table_stats, + ObIArray &column_stats) +{ + int ret = OB_SUCCESS; + ObSEArray part_ids; + ObSEArray column_ids; + if (OB_ISNULL(conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret)); + } else if (OB_FAIL(get_part_ids_and_column_ids(param, part_ids, column_ids))) { + LOG_WARN("failed to get part ids and column ids", K(ret)); + } else if (OB_FAIL(table_stats.prepare_allocate(part_ids.count()))) { + LOG_WARN("failed to prepare allocate table stats", K(ret)); + } else if (OB_FAIL(column_stats.prepare_allocate(part_ids.count() * column_ids.count()))) { + LOG_WARN("failed to prepare allocate column stats", K(ret)); + } else { + ObOptStatManager &stat_manager = ObOptStatManager::get_instance(); + ObSEArray key_column_stats; + for (int64_t i = 0; OB_SUCC(ret) && i < part_ids.count(); ++i) { + void *ptr = NULL; + if (OB_ISNULL(ptr = allocator.alloc(sizeof(ObOptTableStat)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(ptr)); + } else { + table_stats.at(i) = new (ptr) ObOptTableStat(); + table_stats.at(i)->set_table_id(param.table_id_); + table_stats.at(i)->set_partition_id(part_ids.at(i)); + for (int64_t j = 0; OB_SUCC(ret) && j < column_ids.count(); ++j) { + void *ptr1 = NULL; + int64_t idx = i * column_ids.count() + j; + if (OB_UNLIKELY(idx >= column_stats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(idx), K(column_stats.count())); + } else if (OB_ISNULL(ptr1 = allocator.alloc(sizeof(ObOptColumnStat::Key))) || + OB_ISNULL(column_stats.at(idx) = ObOptColumnStat::malloc_new_column_stat(allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(ptr1), K(column_stats.at(idx))); + } else { + ObOptColumnStat::Key *col_key = new (ptr1) ObOptColumnStat::Key(param.tenant_id_, + param.table_id_, + part_ids.at(i), + column_ids.at(j)); + column_stats.at(idx)->set_table_id(param.table_id_); + column_stats.at(idx)->set_partition_id(part_ids.at(i)); + column_stats.at(idx)->set_column_id(column_ids.at(j)); + if (OB_FAIL(key_column_stats.push_back(ObOptKeyColumnStat(col_key, column_stats.at(idx))))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(stat_manager.get_stat_service().get_sql_service().batch_fetch_table_stats(conn, + param.tenant_id_, + param.table_id_, + part_ids, + table_stats))) { + LOG_WARN("failed to batch fetch table stats", K(ret)); + } else if (OB_FAIL(stat_manager.get_stat_service().get_sql_service().fetch_column_stat(param.tenant_id_, + allocator, + key_column_stats, + false, + conn))) { + LOG_WARN("failed to fetch table stats", K(ret)); + } else { + for (int64_t i = 0; i < table_stats.count(); ++i) { + for (int64_t j = 0; j < column_ids.count(); ++j) { + int64_t idx = i * column_ids.count() + j; + if (OB_UNLIKELY(idx >= column_stats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(idx), K(column_stats.count())); + } else if (table_stats.at(i) != NULL && column_stats.at(idx) != NULL && column_stats.at(idx)->get_num_distinct() > 0) { + column_stats.at(idx)->set_num_not_null(table_stats.at(i)->get_row_count() - column_stats.at(idx)->get_num_null()); + column_stats.at(idx)->set_total_col_len(column_stats.at(idx)->get_num_not_null() * column_stats.at(idx)->get_avg_len()); + } + } + } + } + } + } + return ret; +} + +int ObDbmsStatsUtils::merge_split_gather_tab_stats(ObIArray &all_tstats, + ObIArray &cur_all_tstats) +{ + int ret = OB_SUCCESS; + if (all_tstats.empty()) { + if (OB_FAIL(all_tstats.assign(cur_all_tstats))) { + LOG_WARN("failed to assign"); + } else { + cur_all_tstats.reset(); + } + } else if (OB_UNLIKELY(all_tstats.count() != cur_all_tstats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(all_tstats), K(cur_all_tstats)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < all_tstats.count(); ++i) { + if (OB_ISNULL(all_tstats.at(i)) || + OB_ISNULL(cur_all_tstats.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(all_tstats.at(i)), K(cur_all_tstats.at(i))); + } else if (!all_tstats.at(i)->is_valid() && cur_all_tstats.at(i)->is_valid()) { + *all_tstats.at(i) = *cur_all_tstats.at(i); + cur_all_tstats.at(i)->~ObOptTableStat(); + cur_all_tstats.at(i) = NULL; + } else if (OB_UNLIKELY(all_tstats.at(i)->get_table_id() != cur_all_tstats.at(i)->get_table_id() || + all_tstats.at(i)->get_partition_id() != cur_all_tstats.at(i)->get_partition_id())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(all_tstats.at(i)), K(cur_all_tstats.at(i))); + } else { + all_tstats.at(i)->set_row_count(std::max(all_tstats.at(i)->get_row_count(), cur_all_tstats.at(i)->get_row_count())); + all_tstats.at(i)->set_avg_row_size(all_tstats.at(i)->get_avg_row_size() + cur_all_tstats.at(i)->get_avg_row_size()); + //free memory + cur_all_tstats.at(i)->~ObOptTableStat(); + cur_all_tstats.at(i) = NULL; + } + } + cur_all_tstats.reset(); + } + return ret; +} + +int ObDbmsStatsUtils::check_all_cols_range_skew(const ObIArray &column_params, + ObIArray &opt_stats) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < opt_stats.count(); ++i) { + ObIArray &col_stats = opt_stats.at(i).column_stats_; + if (OB_UNLIKELY(column_params.count() != col_stats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(column_params.count()), + K(col_stats.count()), K(ret)); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < column_params.count(); ++j) { + const ObColumnStatParam &col_param = column_params.at(j); + if (col_param.is_size_skewonly() || col_param.is_size_auto()) { + if (OB_ISNULL(col_stats.at(j))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(col_stats.at(j))); + } else { + ObHistogram &hist = col_stats.at(j)->get_histogram(); + if ((hist.get_type() == ObHistType::FREQUENCY && col_param.is_size_skewonly()) || + hist.get_type() == ObHistType::HYBIRD) { + if (OB_UNLIKELY(hist.get_bucket_size() < 1 || col_param.bucket_num_ < 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(hist.get_bucket_size()), + K(col_param.bucket_num_), K(*col_stats.at(j))); + } else { + bool is_even_dist = false; + int64_t standard_cnt = hist.get_type() == ObHistType::FREQUENCY ? + hist.get_buckets().at(0).endpoint_num_ : + hist.get_sample_size() / col_param.bucket_num_; + if (OB_FAIL(ObDbmsStatsUtils::check_range_skew(hist.get_type(), + hist.get_buckets(), + standard_cnt, + is_even_dist))) { + LOG_WARN("failed to check range skew", K(ret)); + } else if (is_even_dist) {//Evenly distributed, no need to build a histogram. + LOG_TRACE("check hist range skew is evenly distributed", K(hist.get_type())); + hist.reset(); + } + } + } else {/*do nothing*/} + } + } else {/*do nothing*/} + } + } + } + return ret; +} + + } } diff --git a/src/share/stat/ob_dbms_stats_utils.h b/src/share/stat/ob_dbms_stats_utils.h index 299de6f779..0a33b4b351 100644 --- a/src/share/stat/ob_dbms_stats_utils.h +++ b/src/share/stat/ob_dbms_stats_utils.h @@ -26,10 +26,6 @@ namespace common { class ObDbmsStatsUtils { public: - static int get_part_info(const ObTableStatParam ¶m, - const ObExtraParam &extra, - PartInfo &part_info); - static int init_col_stats(ObIAllocator &allocator, int64_t col_cnt, ObIArray &col_stats); @@ -43,7 +39,13 @@ public: ObIArray &table_stats, ObIArray &column_stats, const bool is_index_stat = false, - const bool is_history_stat = false, + const bool is_online_stat = false); + + static int split_batch_write(sql::ObExecContext &ctx, + sqlclient::ObISQLConnection *conn, + ObIArray &table_stats, + ObIArray &column_stats, + const bool is_index_stat = false, const bool is_online_stat = false); static int split_batch_write(share::schema::ObSchemaGetterGuard *schema_guard, @@ -52,12 +54,15 @@ public: ObIArray &table_stats, ObIArray &column_stats, const bool is_index_stat = false, - const bool is_history_stat = false, const bool is_online_stat = false); - static int batch_write_history_stats(sql::ObExecContext &ctx, - ObIArray &history_tab_handles, - ObIArray &history_col_handles); + static int split_batch_write(sqlclient::ObISQLConnection *conn, + share::schema::ObSchemaGetterGuard *schema_guard, + sql::ObSQLSessionInfo *session_info, + ObIArray &table_stats, + ObIArray &column_stats, + const bool is_index_stat = false, + const bool is_online_stat = false); static int cast_number_to_double(const number::ObNumber &src_val, double &dst_val); @@ -98,19 +103,17 @@ public: static int merge_tab_stats( const ObTableStatParam ¶m, const TabStatIndMap &table_stats, - common::ObIArray &history_tab_handles, - common::ObIArray &dst_table_stat); + common::ObIArray &old_tab_stats, + common::ObIArray &dst_tab_stats); static int merge_col_stats( const ObTableStatParam ¶m, const ColStatIndMap &column_stats, - common::ObIArray &history_col_handles, - common::ObIArray &dst_column_stat); + common::ObIArray &old_col_stats, + common::ObIArray &dst_col_stats); static bool is_part_id_valid(const ObTableStatParam ¶m, const ObObjectID part_id); - static int get_part_ids_from_param(const ObTableStatParam ¶m, common::ObIArray &part_ids); - static int get_part_infos(const ObTableSchema &table_schema, ObIArray &part_infos, ObIArray &subpart_infos, @@ -128,34 +131,61 @@ public: ObIAllocator &alloc, ObObj *&new_obj); - static int shadow_truncate_string_for_opt_stats(ObObj &obj, ObIAllocator &allocator); + static int truncate_string_for_opt_stats(ObObj &obj, ObIAllocator &allocator); static int64_t get_truncated_str_len(const ObString &str, const ObCollationType cs_type); + static int remove_stat_gather_param_partition_info(int64_t reserved_partition_id, + ObOptStatGatherParam ¶m); + static int64_t check_text_can_reuse(const ObObj &obj, bool &can_reuse); static int get_current_opt_stats(const ObTableStatParam ¶m, ObIArray &cur_tab_handles, ObIArray &cur_col_handles); + static int get_current_opt_stats(ObIAllocator &allocator, + sqlclient::ObISQLConnection *conn, + const ObTableStatParam ¶m, + ObIArray &table_stats, + ObIArray &column_stats); + static int get_part_ids_and_column_ids(const ObTableStatParam ¶m, ObIArray &part_ids, - ObIArray &column_ids); + ObIArray &column_ids, + bool need_stat_column = false); static int erase_stat_cache(const uint64_t tenant_id, const uint64_t table_id, const ObIArray &part_ids, const ObIArray &column_ids); + static bool find_part(const ObIArray &part_infos, + const ObString &part_name, + bool is_sensitive_compare, + PartInfo &part); + + static int prepare_gather_stat_param(const ObTableStatParam ¶m, + StatLevel stat_level, + const PartitionIdBlockMap *partition_id_block_map, + bool is_split_gather, + int64_t gather_vectorize, + ObOptStatGatherParam &gather_param); + + static int merge_split_gather_tab_stats(ObIArray &all_tstats, + ObIArray &cur_all_tstats); + + static int check_all_cols_range_skew(const ObIArray &column_params, + ObIArray &opt_stats); + private: static int batch_write(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, ObIArray &table_stats, ObIArray &column_stats, const int64_t current_time, const bool is_index_stat, - const bool is_history_stat, const bool is_online_stat = false, const ObObjPrintParams &print_params = ObObjPrintParams()); diff --git a/src/share/stat/ob_hybrid_hist_estimator.cpp b/src/share/stat/ob_hybrid_hist_estimator.cpp index fd5db27c30..a26411a68c 100644 --- a/src/share/stat/ob_hybrid_hist_estimator.cpp +++ b/src/share/stat/ob_hybrid_hist_estimator.cpp @@ -24,382 +24,228 @@ namespace common { ObHybridHistEstimator::ObHybridHistEstimator(ObExecContext &ctx, ObIAllocator &allocator) - : ObStatsEstimator(ctx, allocator) + : ObBasicStatsEstimator(ctx, allocator) {} -template -int ObHybridHistEstimator::add_stat_item(const T &item, ObIArray &stat_items) +int ObHybridHistEstimator::estimate(const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat) { int ret = OB_SUCCESS; - ObStatItem *cpy = NULL; - if (!item.is_needed()) { - // do nothing - } else if (OB_ISNULL(cpy = copy_stat_item(allocator_, item))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to copy stat item", K(ret)); - } else if (OB_FAIL(stat_items.push_back(cpy))) { - LOG_WARN("failed to push back stat item", K(ret)); - } - return ret; -} - -// select hybrid_hist(c1,20)，hybrid_hist(c2,20)，.... from t1 partition(p0) simple ... -// union all -// select hybrid_hist(c1,20)，hybrid_hist(c2,20)，.... from t1 partition(p1) simple ... -// union all -// select hybrid_hist(c1,20)，hybrid_hist(c2,20)，.... from t1 partition(p2) simple ... -// no need to hit plan cache or rewrite -int ObHybridHistEstimator::estimate(const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &dst_opt_stats) -{ - int ret = OB_SUCCESS; - ObOptTableStat tab_stat; - ObOptStat src_opt_stat; - src_opt_stat.table_stat_ = &tab_stat; - ObIArray &src_col_stats = src_opt_stat.column_stats_; - ObArenaAllocator allocator("ObHybridHist", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); - ObString raw_sql; - ObString refine_raw_sql; - int64_t refine_cnt = 0; - ObSEArray stat_items; - ObSEArray refine_stat_items; - ObSEArray refine_col_stats; + ObSEArray hybrid_col_params; + ObSEArray hybrid_col_stats; + bool need_sample = false; + bool is_block_sample = false; + double est_percent = 0.0; int64_t max_num_buckets = 0; - if (OB_FAIL(ObDbmsStatsUtils::init_col_stats(allocator, - param.column_params_.count(), - src_col_stats))) { - LOG_WARN("failed init col stats", K(ret)); - } else if (FALSE_IT(get_max_num_buckets(param.column_params_, max_num_buckets))) { - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < dst_opt_stats.count(); ++i) { - ObOptStat &dst_opt_stat = dst_opt_stats.at(i); - ObSEArray refine_col_ids; - ObString sample_hint; - if (OB_ISNULL(dst_opt_stat.table_stat_) || - OB_UNLIKELY(dst_opt_stat.column_stats_.count() != src_col_stats.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("column stat is null", K(ret), K(dst_opt_stat.table_stat_), - K(src_col_stats.count()), K(src_col_stats.count()), - K(dst_opt_stat.column_stats_.count())); - } else { - ObSEArray tmp_stat_items; - //1.calc sample size - bool need_sample = false; - bool is_block_sample = false; - double est_percent = 0.0; - if (OB_FAIL(compute_estimate_percent(dst_opt_stat.table_stat_->get_row_count(), - max_num_buckets, - param.sample_info_, - need_sample, - est_percent, - is_block_sample))) { - LOG_WARN("failed to compute estimate percent", K(ret)); - } else if (need_sample && OB_FAIL(fill_sample_info(allocator, - est_percent, - is_block_sample, - sample_hint))) { - LOG_WARN("failed to fill sample info", K(ret)); - } else {/*do nothing*/} - //2.add select item - bool need_gather_hybrid = false; - for (int64_t j = 0; OB_SUCC(ret) && j < param.column_params_.count(); ++j) { - const ObColumnStatParam &col_param = param.column_params_.at(j); - ObOptColumnStat *src_col_stat = src_col_stats.at(j); - ObOptColumnStat *dst_col_stat = dst_opt_stat.column_stats_.at(j); - bool is_need = false; - bool need_refine = false; - if (OB_FAIL(need_add_hybrid_hist_item(dst_col_stat, - dst_opt_stat.table_stat_, - col_param, - est_percent, - need_sample, - is_need, - need_refine))) { - LOG_WARN("failed to need add truly hybrid hist item", K(ret)); - } else if (is_need) { - if (OB_FAIL(add_stat_item(ObStatHybridHist(&col_param, src_col_stat), - tmp_stat_items))) { - LOG_WARN("failed to add statistic item", K(ret)); - } else { - dst_col_stat->get_histogram().reset(); - dst_col_stat->get_histogram().set_type(ObHistType::HYBIRD); - need_gather_hybrid = true; - } - } else if (OB_FAIL(add_stat_item(ObStatHybridHist(&col_param, src_col_stat, true), - tmp_stat_items))) { - LOG_WARN("failed to add statistic item", K(ret)); - } else if (need_refine) { - if (OB_FAIL(refine_col_ids.push_back(j))) { - LOG_WARN("failed to add statistic item", K(ret)); - } else { - dst_col_stat->get_histogram().reset(); - } - } - } - //3.add other base info - if (OB_SUCC(ret) && need_gather_hybrid) { - extra.nth_part_ = i; - ObSEArray dummy_col_stats; - if (OB_FAIL(gen_query_sql(allocator, param, extra, sample_hint, i + 1, src_opt_stat, - dst_opt_stat, dst_opt_stats, tmp_stat_items, dummy_col_stats, - raw_sql))) { - LOG_WARN("failed to gen query sql", K(ret)); - } else if (OB_FAIL(stat_items.assign(tmp_stat_items))) { - LOG_WARN("failed to assign", K(ret)); - } else {/*do nothing*/} - } - //4.refine hybrid hist - if (OB_SUCC(ret) && !refine_col_ids.empty()) { - if (OB_FAIL(refine_hybrid_hist(allocator, param, extra, refine_cnt + 1, src_opt_stat, - dst_opt_stat, dst_opt_stats, refine_col_ids, - refine_stat_items, refine_col_stats, refine_raw_sql))) { - LOG_WARN("failed to refine hybrid hist", K(ret)); - } else { - ++ refine_cnt; - } - } - } - } - //5.do estimate - if (OB_SUCC(ret) && !raw_sql.empty()) { - stat_items_.reset(); - if (OB_FAIL(append(stat_items_, stat_items))) { - LOG_WARN("failed to append stat items", K(ret)); - } else if (OB_FAIL(do_estimate(param.tenant_id_, raw_sql, COPY_HYBRID_HIST_STAT, - src_opt_stat, dst_opt_stats))) { - LOG_WARN("failed to evaluate basic stats", K(ret)); - } else { - LOG_TRACE("succeed to build hybrid histogram"); - } - } - if (OB_SUCC(ret) && !refine_raw_sql.empty()) { - stat_items_.reset(); - if (OB_FAIL(append(stat_items_, refine_stat_items))) { - LOG_WARN("failed to append stat items", K(ret)); - } else if (OB_FAIL(reset_histogram_for_refine_col_stats(refine_col_stats))) { - LOG_WARN("failed to reset histogram for refine col stats", K(ret)); - } else if (OB_FAIL(do_estimate(param.tenant_id_, refine_raw_sql, COPY_HYBRID_HIST_STAT, - src_opt_stat, dst_opt_stats))) { - LOG_WARN("failed to evaluate basic stats", K(ret)); - } else { - refine_col_stats.reset(); - LOG_TRACE("succeed to build hybrid histogram"); - } - } - } - return ret; -} - -void ObHybridHistEstimator::get_max_num_buckets(const ObIArray &col_params, - int64_t &max_num_buckets) -{ - max_num_buckets = 0; - for (int64_t i = 0; i < col_params.count(); ++i) { - max_num_buckets = std::max(max_num_buckets, col_params.at(i).bucket_num_); - } -} - -int ObHybridHistEstimator::need_add_hybrid_hist_item(ObOptColumnStat *dst_col_stat, - ObOptTableStat *dst_tab_stat, - const ObColumnStatParam &col_param, - double est_percent, - bool need_sample, - bool &is_need, - bool &need_refine) -{ - int ret = OB_SUCCESS; - is_need = false; - need_refine = false; - bool is_done = false; - if (OB_ISNULL(dst_col_stat) || OB_ISNULL(dst_tab_stat)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(dst_col_stat), K(dst_tab_stat), K(ret)); - } else if (!dst_col_stat->get_histogram().is_hybrid()) { - /*do nothing*/ - } else if (OB_FAIL(try_build_hybrid_hist(col_param, - *dst_col_stat, - *dst_tab_stat, - is_done))) { - LOG_WARN("failed to build hybrid hist from topk hist", K(ret)); - } else if (is_done) { - /*do nothing*/ - //if col not null cnt is less than expected sample row cnt, then choose full table scan. - } else if (need_sample && - dst_col_stat->get_num_not_null() <= dst_tab_stat->get_row_count() * est_percent / 100) { - need_refine = true; - } else { - is_need = true; - } - return ret; -} - -int ObHybridHistEstimator::refine_hybrid_hist(ObIAllocator &allocator, - const ObTableStatParam ¶m, - ObExtraParam &extra, - int64_t total_refine_cnt, - ObOptStat &src_opt_stat, - ObOptStat &dst_opt_stat, - ObIArray &dst_opt_stats, - ObIArray &refine_col_ids, - ObIArray &refine_stat_items, - ObIArray &refine_col_stats, - ObString &refine_raw_sql) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(refine_col_ids.empty())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected empty", K(ret), K(refine_col_ids.empty())); - } else { - //1.add select item - int64_t refine_id = 0; - refine_stat_items.reset(); - for (int64_t i = 0; OB_SUCC(ret) && i < param.column_params_.count(); ++i) { - const ObColumnStatParam &col_param = param.column_params_.at(i); - ObOptColumnStat *src_col_stat = src_opt_stat.column_stats_.at(i); - ObOptColumnStat *dst_col_stat = dst_opt_stat.column_stats_.at(i); - if (refine_id < refine_col_ids.count() && i == refine_col_ids.at(refine_id)) { - if (OB_FAIL(add_stat_item(ObStatHybridHist(&col_param, src_col_stat), refine_stat_items))) { - LOG_WARN("failed to add statistic item", K(ret)); - } else if (OB_FAIL(refine_col_stats.push_back(dst_col_stat))) { - LOG_WARN("failed to push back col stat", K(ret)); - } else { - ++ refine_id; - } - } else if (OB_FAIL(add_stat_item(ObStatHybridHist(&col_param, src_col_stat, true), - refine_stat_items))) { - LOG_WARN("failed to add statistic item", K(ret)); - } else {/*do nothing*/} - } - if (OB_SUCC(ret)) { - ObString dummy_simple_hint; - if (OB_FAIL(gen_query_sql(allocator, param, extra, dummy_simple_hint, total_refine_cnt, - src_opt_stat, dst_opt_stat, dst_opt_stats, refine_stat_items, - refine_col_stats, refine_raw_sql))) { - LOG_WARN("failed to gen query sql", K(ret)); - } else {/*do nothing*/} - } - } - return ret; -} - -int ObHybridHistEstimator::reset_histogram_for_refine_col_stats( - ObIArray &refine_col_stats) -{ - int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < refine_col_stats.count(); ++i) { - if (OB_ISNULL(refine_col_stats.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(refine_col_stats.at(i))); - } else { - refine_col_stats.at(i)->get_histogram().reset(); - refine_col_stats.at(i)->get_histogram().set_type(ObHistType::HYBIRD); - } - } - return ret; -} - -int ObHybridHistEstimator::gen_query_sql(ObIAllocator &allocator, - const ObTableStatParam ¶m, - ObExtraParam &extra, - const ObString &simple_hint, - int64_t part_cnt, - ObOptStat &src_opt_stat, - ObOptStat &dst_opt_stat, - ObIArray &dst_opt_stats, - ObIArray &stat_items, - ObIArray &refine_col_stats, - ObString &raw_sql) -{ - int ret = OB_SUCCESS; - ObSqlString single_raw_sql; - ObString empty_str; - reset_select_items(); - sample_hint_ = simple_hint; + ObSEArray no_sample_idx; + ObArenaAllocator allocator("ObHybridHistEst", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); + ObSqlString raw_sql; int64_t duration_time = -1; - ObString hint_str("NO_REWRITE USE_PLAN_CACHE(NONE) DBMS_STATS OPT_PARAM('ROWSETS_MAX_ROWS', 256)"); - //add select items - if (OB_ISNULL(dst_opt_stat.table_stat_)) { + ObSEArray tmp_opt_stats; + if (OB_FAIL(extract_hybrid_hist_col_info(param, opt_stat, + hybrid_col_params, + hybrid_col_stats, + max_num_buckets))) { + LOG_WARN("failed to extract hybrid hist col info", K(ret)); + } else if (hybrid_col_params.empty() || hybrid_col_stats.empty()) { + //do nothing + } else if (OB_UNLIKELY(hybrid_col_params.count() != hybrid_col_stats.count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(dst_opt_stat.table_stat_)); - } else if (OB_FAIL(add_hint(hint_str, allocator))) { - LOG_WARN("failed to add hint"); - } else if (OB_FAIL(add_stat_item(ObPartitionId(¶m, src_opt_stat.table_stat_, empty_str, - dst_opt_stat.table_stat_->get_partition_id()), - stat_items))) { - LOG_WARN("failed to add partition id item", K(ret)); - } else if (OB_FAIL(append(stat_items_, stat_items))) { - LOG_WARN("failed to append stat items", K(ret)); + LOG_WARN("get unexpected error", K(ret), K(hybrid_col_params.count()), K(hybrid_col_stats.count())); + } else if (OB_FAIL(compute_estimate_percent(opt_stat.table_stat_->get_row_count(), + max_num_buckets, + param.sample_info_, + need_sample, + est_percent, + is_block_sample))) { + LOG_WARN("failed to compute estimate percent", K(ret)); + } else if (need_sample && OB_FAIL(fill_sample_info(allocator, + est_percent, + is_block_sample))) { + LOG_WARN("failed to fill sample info", K(ret)); + } else if (OB_FAIL(add_hybrid_hist_stat_items(hybrid_col_params, + hybrid_col_stats, + opt_stat.table_stat_->get_row_count(), + need_sample, + est_percent, + no_sample_idx))) { + LOG_WARN("failed to add hybrid hist stat items", K(ret)); + } else if (OB_FAIL(fill_hints(allocator, param.tab_name_, param.gather_vectorize_))) { + LOG_WARN("failed to fill hints", K(ret)); } else if (OB_FAIL(add_from_table(param.db_name_, param.tab_name_))) { LOG_WARN("failed to add from table", K(ret)); } else if (OB_FAIL(fill_parallel_info(allocator, param.degree_))) { LOG_WARN("failed to fill parallel info", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(extra.start_time_, - param.duration_time_, + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(param.gather_start_time_, + param.max_duration_time_, duration_time))) { LOG_WARN("failed to get valid duration time", K(ret)); } else if (OB_FAIL(fill_query_timeout_info(allocator, duration_time))) { LOG_WARN("failed to fill query timeout info", K(ret)); - } else if (OB_FAIL(fill_partition_info(allocator, param, extra))) { - LOG_WARN("failed to fill partition info", K(ret)); - } else if (OB_FAIL(pack(single_raw_sql))) { - LOG_WARN("failed to pack sql", K(ret), K(single_raw_sql)); - } else if (raw_sql.empty()) { - char *ptr = NULL; - if (OB_ISNULL(ptr = static_cast(allocator.alloc(single_raw_sql.length() + 1)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(ptr)); - } else { - MEMSET(ptr, 0, single_raw_sql.length() + 1); - MEMCPY(ptr, single_raw_sql.ptr(), single_raw_sql.length()); - raw_sql.assign_ptr(ptr, single_raw_sql.length()); - } - } else if (OB_FAIL(gen_union_all_sql(allocator, single_raw_sql.string(), raw_sql, raw_sql))) { - LOG_WARN("failed to gen union all sql", K(ret)); - //Currently union all sql has 1000 queries at most each time, So do estimate. - } else if (part_cnt % 1000 == 0) { - if (OB_FAIL(reset_histogram_for_refine_col_stats(refine_col_stats))) { - LOG_WARN("failed to reset histogram for refine col stats", K(ret)); - } else if (OB_FAIL(do_estimate(param.tenant_id_, raw_sql, COPY_HYBRID_HIST_STAT, - src_opt_stat, dst_opt_stats))) { - LOG_WARN("failed to evaluate basic stats", K(ret)); - } else { - LOG_TRACE("succeed to build hybrid histogram", K(raw_sql)); - raw_sql.reset(); - stat_items.reset(); - refine_col_stats.reset(); - sample_hint_.reset(); - } - } - if (OB_SUCC(ret)) { - reset_select_items(); - other_hints_.reset(); + } else if (!param.partition_infos_.empty() && + OB_FAIL(fill_partition_info(allocator, param.partition_infos_.at(0).part_name_))) { + LOG_WARN("failed to add partition info", K(ret)); + } else if (OB_FAIL(fill_specify_scn_info(allocator, param.sepcify_scn_))) { + LOG_WARN("failed to fill specify scn info", K(ret)); + } else if (OB_FAIL(pack(raw_sql))) { + LOG_WARN("failed to pack", K(ret)); + } else if (OB_FAIL(tmp_opt_stats.push_back(opt_stat))) { + LOG_WARN("failed to push back", K(ret)); + } else if (get_item_size() > 0 && + OB_FAIL(do_estimate(param.tenant_id_, raw_sql.string(), false, + opt_stat, tmp_opt_stats))) { + LOG_WARN("failed to do estimate", K(ret)); + } else if (!no_sample_idx.empty() && + OB_FAIL(estimate_no_sample_col_hydrid_hist(allocator, param, opt_stat, + hybrid_col_params, hybrid_col_stats, + no_sample_idx))) { + LOG_WARN("failed to estimate no sample col hydrid_hist", K(ret)); + } else { + LOG_TRACE("succeed to build hybrid histogram", K(hybrid_col_stats)); } return ret; } -int ObHybridHistEstimator::gen_union_all_sql(ObIAllocator &allocator, - const ObString &raw_sql1, - const ObString &raw_sql2, - ObString &merge_raw_sql) +int ObHybridHistEstimator::extract_hybrid_hist_col_info(const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat, + ObIArray &hybrid_col_params, + ObIArray &hybrid_col_stats, + int64_t &max_num_buckets) { int ret = OB_SUCCESS; - ObSqlString tmp_sql_str; - char *ptr = NULL; - if (OB_FAIL(tmp_sql_str.append_fmt("%.*s union all %.*s", - raw_sql1.length(), - raw_sql1.ptr(), - raw_sql2.length(), - raw_sql2.ptr()))) { - LOG_WARN("failed to build query sql stmt", K(ret)); - } else if (OB_ISNULL(ptr = static_cast(allocator.alloc(tmp_sql_str.length() + 1)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory is not enough", K(ret), K(ptr)); + max_num_buckets = 0; + if (OB_UNLIKELY(param.column_params_.count() != opt_stat.column_stats_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param.column_params_), K(opt_stat.column_stats_)); } else { - MEMSET(ptr, 0, tmp_sql_str.length() + 1); - MEMCPY(ptr, tmp_sql_str.ptr(), tmp_sql_str.length()); - merge_raw_sql.assign_ptr(ptr, tmp_sql_str.length()); - LOG_TRACE("Succeed to gen union all sql", K(merge_raw_sql)); + for (int64_t i = 0; OB_SUCC(ret) && i < opt_stat.column_stats_.count(); ++i) { + bool is_done = false; + if (OB_ISNULL(opt_stat.table_stat_) || + OB_ISNULL(opt_stat.column_stats_.at(i)) || + OB_UNLIKELY(opt_stat.column_stats_.at(i)->get_column_id() != param.column_params_.at(i).column_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(opt_stat.column_stats_.at(i)), + K(opt_stat.table_stat_), K(param.column_params_.at(i))); + } else if (opt_stat.table_stat_->get_row_count() <=0 || + !opt_stat.column_stats_.at(i)->get_histogram().is_hybrid()) { + //do nothing + } else if (OB_FAIL(try_build_hybrid_hist(param.column_params_.at(i), + *opt_stat.column_stats_.at(i), + *opt_stat.table_stat_, + is_done))) { + LOG_WARN("failed to try build hybrid hist", K(ret)); + } else if (is_done) { + //do nothing + } else if (OB_FAIL(hybrid_col_params.push_back(¶m.column_params_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(hybrid_col_stats.push_back(opt_stat.column_stats_.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } else { + opt_stat.column_stats_.at(i)->get_histogram().reset(); + opt_stat.column_stats_.at(i)->get_histogram().set_type(ObHistType::HYBIRD); + max_num_buckets = std::max(max_num_buckets, param.column_params_.at(i).bucket_num_); + } + } + LOG_TRACE("succeed to extract hybrid hist col info", K(param), K(hybrid_col_params), K(hybrid_col_stats)); + } + return ret; +} + +int ObHybridHistEstimator::add_hybrid_hist_stat_items(ObIArray &hybrid_col_params, + ObIArray &hybrid_col_stats, + const int64_t table_row_cnt, + const bool need_sample, + const double est_percent, + ObIArray &no_sample_idx) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(hybrid_col_params.count() != hybrid_col_stats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(hybrid_col_params), K(hybrid_col_stats)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < hybrid_col_stats.count(); ++i) { + if (OB_ISNULL(hybrid_col_params.at(i)) || OB_ISNULL(hybrid_col_stats.at(i)) || + OB_UNLIKELY(hybrid_col_params.at(i)->column_id_ != hybrid_col_stats.at(i)->get_column_id())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(hybrid_col_stats.at(i)), KPC(hybrid_col_params.at(i))); + } else if (!need_sample || hybrid_col_stats.at(i)->get_num_not_null() > table_row_cnt * est_percent / 100) { + if (OB_FAIL(add_stat_item(ObStatHybridHist(hybrid_col_params.at(i), + hybrid_col_stats.at(i))))) { + LOG_WARN("failed to add stat item", K(ret)); + } + //if col not null cnt is less than expected sample row cnt, then no need sample. + } else if (OB_FAIL(no_sample_idx.push_back(i))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + return ret; +} + +int ObHybridHistEstimator::estimate_no_sample_col_hydrid_hist(ObIAllocator &allocator, + const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat, + ObIArray &hybrid_col_params, + ObIArray &hybrid_col_stats, + ObIArray &no_sample_idx) +{ + int ret = OB_SUCCESS; + ObSqlString raw_sql; + int64_t duration_time = -1; + ObSEArray tmp_opt_stats; + if (no_sample_idx.empty()) { + //do nothing + //reset select item if exists; + } else if (FALSE_IT(reset_select_items())) { + //reset sample hint if exists; + } else if (FALSE_IT(reset_sample_hint())) { + //reset other hint + } else if (FALSE_IT(reset_other_hint())) { + //add stat items + } else if (OB_FAIL(add_no_sample_hybrid_hist_stat_items(hybrid_col_params, + hybrid_col_stats, + no_sample_idx))) { + LOG_WARN("failed to add no sample hybrid hist stat items", K(ret)); + } else if (OB_FAIL(fill_hints(allocator, param.tab_name_, param.gather_vectorize_))) { + LOG_WARN("failed to fill hints", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(param.gather_start_time_, + param.max_duration_time_, + duration_time))) { + LOG_WARN("failed to get valid duration time", K(ret)); + } else if (OB_FAIL(fill_query_timeout_info(allocator, duration_time))) { + LOG_WARN("failed to fill query timeout info", K(ret)); + } else if (OB_FAIL(pack(raw_sql))) { + LOG_WARN("failed to pack", K(ret)); + } else if (OB_FAIL(tmp_opt_stats.push_back(opt_stat))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(do_estimate(param.tenant_id_, raw_sql.string(), false, + opt_stat, tmp_opt_stats))) { + LOG_WARN("failed to do estimate", K(ret)); + } else {/*do nothing*/} + return ret; +} + +int ObHybridHistEstimator::add_no_sample_hybrid_hist_stat_items(ObIArray &hybrid_col_params, + ObIArray &hybrid_col_stats, + ObIArray &no_sample_idx) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(no_sample_idx.empty() || hybrid_col_params.count() != hybrid_col_stats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(no_sample_idx), K(hybrid_col_params), K(hybrid_col_stats)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < no_sample_idx.count(); ++i) { + int64_t idx = no_sample_idx.at(i); + if (OB_UNLIKELY(idx >= hybrid_col_params.count()) || + OB_ISNULL(hybrid_col_params.at(idx)) || OB_ISNULL(hybrid_col_stats.at(idx)) || + OB_UNLIKELY(hybrid_col_params.at(idx)->column_id_ != hybrid_col_stats.at(idx)->get_column_id())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(idx), K(hybrid_col_stats), K(hybrid_col_params)); + } else if (OB_FAIL(add_stat_item(ObStatHybridHist(hybrid_col_params.at(idx), + hybrid_col_stats.at(idx))))) { + LOG_WARN("failed to add stat item", K(ret)); + } + } } return ret; } @@ -707,7 +553,14 @@ int ObHybridHistograms::build_hybrid_hist(ObAggregateProcessor::HybridHistExtraR if (bucket_rows > bucket_size || 0 == i || extra->get_material_row_count() - 1 == i) { bucket_rows = 0; ObObj ep_val; - if (OB_FAIL(row->cells()[0].to_obj(ep_val, obj_meta))) { + common::ObArenaAllocator tmp_alloctor("BulidHybridHist", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObDatum new_datum = row->cells()[0]; + if (obj_meta.is_lob_storage() && + OB_FAIL(build_prefix_str_datum_for_lob(tmp_alloctor, + obj_meta, row->cells()[0], + new_datum))) { + LOG_WARN("failed to build prefix str datum for lob", K(ret)); + } else if (OB_FAIL(new_datum.to_obj(ep_val, obj_meta))) { LOG_WARN("failed to obj", K(ret)); } else if (OB_FAIL(ob_write_obj(*alloc, ep_val, ep_val))) { LOG_WARN("failed to write obj", K(ret), K(ep_val)); @@ -752,6 +605,42 @@ int ObHybridHistograms::build_hybrid_hist(ObAggregateProcessor::HybridHistExtraR return ret; } +int ObHybridHistograms::build_prefix_str_datum_for_lob(ObIAllocator &allocator, + const ObObjMeta &obj_meta, + const ObDatum &old_datum, + ObDatum &new_datum) +{ + int ret = OB_SUCCESS; + if (!obj_meta.is_lob_storage()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(obj_meta)); + } else { + ObObj obj; + ObString str; + bool can_reuse = false; + if (OB_FAIL(old_datum.to_obj(obj, obj_meta))) { + LOG_WARN("failed to obj", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::check_text_can_reuse(obj, can_reuse))) { + LOG_WARN("failed to check text obj can reuse", K(ret), K(obj)); + } else if (can_reuse) { + new_datum = old_datum; + } else if (OB_FAIL(sql::ObTextStringHelper::read_prefix_string_data(&allocator, obj, str, OPT_STATS_MAX_VALUE_CHAR_LEN))) { + LOG_WARN("failed to read prefix string data", K(ret)); + } else { + ObTextStringDatumResult text_result(obj_meta.get_type(), obj_meta.has_lob_header(), &new_datum); + if (OB_FAIL(text_result.init(str.length(), &allocator))) { + LOG_WARN("init lob result failed"); + } else if (OB_FAIL(text_result.append(str.ptr(), str.length()))) { + LOG_WARN("failed to append realdata", K(ret), K(str), K(text_result)); + } else { + text_result.set_result(); + LOG_TRACE("Succeed to build_prefix_str_datum_for_lob", K(obj), K(str)); + } + } + } + return ret; +} + OB_DEF_SERIALIZE(ObHybridHistograms) { int ret = OB_SUCCESS; diff --git a/src/share/stat/ob_hybrid_hist_estimator.h b/src/share/stat/ob_hybrid_hist_estimator.h index b082e4e64a..3c2fd7008d 100644 --- a/src/share/stat/ob_hybrid_hist_estimator.h +++ b/src/share/stat/ob_hybrid_hist_estimator.h @@ -77,6 +77,10 @@ public: int64_t pop_count, int64_t pop_freq, const ObObjMeta &obj_meta); + static int build_prefix_str_datum_for_lob(ObIAllocator &allocator, + const ObObjMeta &obj_meta, + const ObDatum &old_datum, + ObDatum &new_datum); TO_STRING_KV(K_(total_count), K_(num_distinct), K_(pop_count), @@ -90,18 +94,13 @@ private: }; -class ObHybridHistEstimator : public ObStatsEstimator +class ObHybridHistEstimator : public ObBasicStatsEstimator { public: explicit ObHybridHistEstimator(ObExecContext &ctx, ObIAllocator &allocator); - int estimate(const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &dst_opt_stats); - - template - int add_stat_item(const T &item, ObIArray &stat_items); - + int estimate(const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat); private: int try_build_hybrid_hist(const ObColumnStatParam ¶m, @@ -116,51 +115,31 @@ private: double &est_percent, bool &is_block_sample); - int gen_union_all_sql(ObIAllocator &allocator, - const ObString &raw_sql1, - const ObString &raw_sql2, - ObString &merge_raw_sql); - - int refine_hybrid_hist(ObIAllocator &allocator, - const ObTableStatParam ¶m, - ObExtraParam &extra, - int64_t total_refine_cnt, - ObOptStat &src_opt_stat, - ObOptStat &dst_opt_stat, - ObIArray &dst_opt_stats, - ObIArray &refine_col_ids, - ObIArray &refine_stat_items, - ObIArray &refine_col_stats, - ObString &refine_raw_sql); - - int gen_query_sql(ObIAllocator &allocator, - const ObTableStatParam ¶m, - ObExtraParam &extra, - const ObString &simple_hint, - int64_t part_cnt, - ObOptStat &src_opt_stat, - ObOptStat &dst_opt_stat, - ObIArray &dst_opt_stats, - ObIArray &stat_items, - ObIArray &refine_col_stats, - ObString &raw_sql); - int reset_histogram_for_refine_col_stats(ObIArray &refine_col_stats); - int need_add_hybrid_hist_item(ObOptColumnStat *dst_col_stat, - ObOptTableStat *dst_tab_stat, - const ObColumnStatParam &col_param, - double est_percent, - bool need_sample, - bool &is_need, - bool &need_refine); + int extract_hybrid_hist_col_info(const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat, + ObIArray &hybrid_col_params, + ObIArray &hybrid_col_stats, + int64_t &max_num_buckets); - void get_max_num_buckets(const ObIArray &col_params, - int64_t &max_num_buckets); + int add_hybrid_hist_stat_items(ObIArray &hybrid_col_params, + ObIArray &hybrid_col_stats, + const int64_t table_row_cnt, + const bool need_sample, + const double est_percent, + ObIArray &no_sample_idx); -private: - /// a magic number - static const int64_t AUTO_SAMPLE_SIZE; + int estimate_no_sample_col_hydrid_hist(ObIAllocator &allocator, + const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat, + ObIArray &hybrid_col_params, + ObIArray &hybrid_col_stats, + ObIArray &no_sample_idx); + + int add_no_sample_hybrid_hist_stat_items(ObIArray &hybrid_col_params, + ObIArray &hybrid_col_stats, + ObIArray &no_sample_idx); }; } diff --git a/src/share/stat/ob_incremental_stat_estimator.cpp b/src/share/stat/ob_incremental_stat_estimator.cpp index d009b12841..dd451b9f04 100644 --- a/src/share/stat/ob_incremental_stat_estimator.cpp +++ b/src/share/stat/ob_incremental_stat_estimator.cpp @@ -27,38 +27,6 @@ namespace oceanbase { using namespace pl; namespace common { -/* Now, the following scenarios will try derive global stat: - * if granularity specify 'APPROX_GLOBAL AND PARTITION': - * if in auto gather stats, choose gather subpartition stats and incremental gather global stats: - * then we choose incremental gather global stats from subpartition stats. - * else we choose incremental gather global stats from partition stats. - * -*/ -int ObIncrementalStatEstimator::try_derive_global_stat(ObExecContext &ctx, - const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &approx_part_opt_stats, - ObIArray &opt_stats) -{ - int ret = OB_SUCCESS; - if (extra.type_ == INVALID_LEVEL || - extra.type_ == TABLE_LEVEL || - !(param.global_stat_param_.need_modify_ && param.global_stat_param_.gather_approx_) || - param.part_level_ == share::schema::PARTITION_LEVEL_ZERO) { - LOG_TRACE("not fullfill derive global stat", K(extra.type_),K(param.global_stat_param_), - K(param.part_level_)); - } else if (extra.type_ == SUBPARTITION_LEVEL) { - if (OB_FAIL(derive_part_stats_from_subpart_stats(ctx, param, opt_stats, - approx_part_opt_stats))) { - LOG_WARN("failed to derive part stats from subpart stats", K(ret)); - } - } else if (OB_FAIL(derive_global_stat_from_part_stats(ctx, param, - approx_part_opt_stats, opt_stats))) { - LOG_WARN("failed to derive global stat from part stats", K(ret)); - } - return ret; -} - int ObIncrementalStatEstimator::derive_global_stat_by_direct_load(ObExecContext &ctx, const uint64_t table_id) { @@ -82,14 +50,13 @@ int ObIncrementalStatEstimator::derive_global_stat_by_direct_load(ObExecContext } else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(ctx.get_my_session()->get_rpc_tenant_id(), param))) { LOG_WARN("fail to update stat cache", K(ret)); } - } else if (OB_FAIL(get_all_part_opt_stat_by_direct_load(param, - part_tab_stats, - part_col_handles, - part_opt_stats))) { - LOG_WARN("failed to get all part opt stat by direct load", K(ret)); - } else if (OB_FAIL(param.all_part_infos_.assign(param.part_infos_))|| - OB_FAIL(param.all_subpart_infos_.assign(param.subpart_infos_))) { - LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(get_all_part_opt_stats(param, + param.part_level_ == share::schema::PARTITION_LEVEL_ONE ? + param.all_part_infos_ : param.all_subpart_infos_, + part_tab_stats, + part_col_handles, + part_opt_stats))) { + LOG_WARN("failed to get all part opt stats", K(ret)); } else { bool nee_derive_part = param.part_level_ == share::schema::PARTITION_LEVEL_TWO; //derive part stat first @@ -97,18 +64,17 @@ int ObIncrementalStatEstimator::derive_global_stat_by_direct_load(ObExecContext ObSEArray empty_opt_stats; if (OB_FAIL(do_derive_part_stats_from_subpart_stats(ctx, alloc, param, empty_opt_stats, part_opt_stats, all_derive_opt_stats))) { - LOG_WARN("failed to derive part stat by direct load", K(ret)); + LOG_WARN("failed to do derive part stats from subpart stats", K(ret)); } } //derive global stat if (OB_SUCC(ret)) { ObOptStat global_opt_stat; bool need_derive_hist = false; - bool need_gather_hybrid_hist = false; if (OB_FAIL(do_derive_global_stat(ctx, alloc, param, nee_derive_part ? all_derive_opt_stats : part_opt_stats, need_derive_hist, - TABLE_LEVEL, param.global_part_id_, need_gather_hybrid_hist, + TABLE_LEVEL, param.global_part_id_, global_opt_stat))) { LOG_WARN("Failed to derive global stat from part stat", K(ret)); } else if (OB_FAIL(all_derive_opt_stats.push_back(global_opt_stat))) { @@ -141,25 +107,21 @@ int ObIncrementalStatEstimator::derive_global_stat_by_direct_load(ObExecContext int ObIncrementalStatEstimator::derive_global_stat_from_part_stats( ObExecContext &ctx, const ObTableStatParam ¶m, - const ObIArray &approx_part_opt_stats, - ObIArray &opt_stats) + const ObIArray &part_opt_stats, + ObOptStat &global_opt_stat) { int ret = OB_SUCCESS; - ObOptStat global_opt_stat; ObSEArray table_stats; ObSEArray col_handles; ObSEArray tmp_opt_stats; ObSEArray no_regather_part_ids; bool need_derive_hist = true; - bool need_gather_hybrid_hist = false; if (OB_ISNULL(param.allocator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(param)); - } else if (OB_FAIL(append(opt_stats, approx_part_opt_stats))) { - LOG_WARN("failed to append", K(ret)); - } else if (opt_stats.empty()) { + } else if (part_opt_stats.empty()) { /*do nothing*/ - } else if (OB_FAIL(tmp_opt_stats.assign(opt_stats))) { + } else if (OB_FAIL(tmp_opt_stats.assign(part_opt_stats))) { LOG_WARN("failed to assign", K(ret)); } else { //if specify part/subpart name, need get other already existed part stats, but only derive base @@ -207,13 +169,11 @@ int ObIncrementalStatEstimator::derive_global_stat_from_part_stats( } if (OB_SUCC(ret)) { if (OB_FAIL(do_derive_global_stat(ctx, *param.allocator_, param, tmp_opt_stats, need_derive_hist, - TABLE_LEVEL,param.global_part_id_, need_gather_hybrid_hist, + TABLE_LEVEL,param.global_part_id_, global_opt_stat))) { LOG_WARN("Failed to derive global stat from part stat", K(ret)); - } else if (OB_FAIL(opt_stats.push_back(global_opt_stat))) { - LOG_WARN("failed to push back global stat", K(ret)); } else { - LOG_TRACE("Succeed to derive global stat from part stats", K(opt_stats.count()), K(param)); + LOG_TRACE("Succeed to derive global stat from part stats", K(global_opt_stat), K(param)); } } } @@ -223,7 +183,7 @@ int ObIncrementalStatEstimator::derive_global_stat_from_part_stats( int ObIncrementalStatEstimator::derive_part_stats_from_subpart_stats( ObExecContext &ctx, const ObTableStatParam ¶m, - const ObIArray &gather_opt_stats, + const ObIArray &gather_subpart_opt_stats, ObIArray &approx_part_opt_stats) { int ret = OB_SUCCESS; @@ -231,7 +191,6 @@ int ObIncrementalStatEstimator::derive_part_stats_from_subpart_stats( ObSEArray no_regather_table_stats; ObSEArray no_regather_col_handles; ObSEArray no_regather_subpart_opt_stats; - bool need_gather_hybrid_hist = false; if (OB_ISNULL(param.allocator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(param)); @@ -243,28 +202,100 @@ int ObIncrementalStatEstimator::derive_part_stats_from_subpart_stats( LOG_WARN("failed to get locked partition stats", K(ret)); } else if (OB_FAIL(do_derive_part_stats_from_subpart_stats(ctx, *param.allocator_, param, no_regather_subpart_opt_stats, - gather_opt_stats, + gather_subpart_opt_stats, approx_part_opt_stats))) { LOG_WARN("failed to do derive part stats from subpart stats", K(ret)); } else { LOG_TRACE("Succeed to derive part stats from subpart stats", K(approx_part_opt_stats.count()), - K(gather_opt_stats.count()), + K(gather_subpart_opt_stats.count()), K(param)); } return ret; } +int ObIncrementalStatEstimator::derive_split_gather_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + bool derive_part_stat, + bool is_all_columns_gather, + ObIArray &all_tstats) +{ + int ret = OB_SUCCESS; + if (param.part_level_ != share::schema::PARTITION_LEVEL_ONE && + param.part_level_ != share::schema::PARTITION_LEVEL_TWO) { + //do nothing + } else if (OB_ISNULL(param.allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param)); + } else { + ObArenaAllocator allocator("IncrementStats", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); + ObSEArray cur_table_stats; + ObSEArray cur_column_stats; + ObSEArray part_opt_stats; + ObSEArray derive_opt_stats; + ObTableStatParam new_param; + if (OB_FAIL(prepare_get_opt_stats_param(param, derive_part_stat, new_param))) { + LOG_WARN("failed to prepare get opt stats param", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::get_current_opt_stats(allocator, + trans.get_connection(), + new_param, + cur_table_stats, + cur_column_stats))) { + LOG_WARN("failed to get current opt stats", K(ret)); + } else if (OB_FAIL(generate_all_opt_stat(cur_table_stats, + cur_column_stats, + param.column_params_.count(), + part_opt_stats))) { + LOG_WARN("failed to generate all opt stat", K(ret)); + } else if (derive_part_stat) { + ObSEArray empty_opt_stats; + if (OB_FAIL(do_derive_part_stats_from_subpart_stats(ctx, *param.allocator_, param, empty_opt_stats, + part_opt_stats, derive_opt_stats))) { + LOG_WARN("failed to do derive part stats from subpart stats", K(ret)); + } + } else { + ObOptStat global_opt_stat; + bool need_derive_hist = true; + if (OB_FAIL(do_derive_global_stat(ctx, *param.allocator_, param, + part_opt_stats, + need_derive_hist, + TABLE_LEVEL, param.global_part_id_, + global_opt_stat))) { + LOG_WARN("Failed to derive global stat from part stat", K(ret)); + } else if (OB_FAIL(derive_opt_stats.push_back(global_opt_stat))) { + LOG_WARN("faield to push back", K(ret)); + } + } + //write all stat + if (OB_SUCC(ret)) { + ObSEArray tmp_all_tstats; + ObSEArray all_cstats; + if (OB_FAIL(ObDbmsStatsUtils::calssify_opt_stat(derive_opt_stats, + tmp_all_tstats, + all_cstats))) { + LOG_WARN("failed to calssify opt stat", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::merge_split_gather_tab_stats(all_tstats, tmp_all_tstats))) { + LOG_WARN("failed to merge split gather tab stats", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::split_batch_write(ctx, trans.get_connection(), + is_all_columns_gather ? all_tstats : tmp_all_tstats, + all_cstats))) { + LOG_WARN("failed to split batch write", K(ret)); + } + } + LOG_TRACE("succeed to derive split gather stats", K(param), K(derive_part_stat), K(part_opt_stats), K(derive_opt_stats)); + } + return ret; +} + int ObIncrementalStatEstimator::do_derive_part_stats_from_subpart_stats( ObExecContext &ctx, ObIAllocator &alloc, const ObTableStatParam ¶m, const ObIArray &no_regather_subpart_opt_stats, - const ObIArray &gather_opt_stats, + const ObIArray &gather_subpart_opt_stats, ObIArray &approx_part_opt_stats) { int ret = OB_SUCCESS; - bool need_gather_hybrid_hist = false; - ObSEArray gather_hybrid_hist_opt_stats; int64_t cur_part_id = OB_INVALID_ID; for (int64_t i = 0; OB_SUCC(ret) && i < param.approx_part_infos_.count(); ++i) { ObOptStat opt_part_stat; @@ -285,15 +316,15 @@ int ObIncrementalStatEstimator::do_derive_part_stats_from_subpart_stats( } else {/*do nothing*/} } //get regather subpart stats - for (int64_t j = 0; OB_SUCC(ret) && j < gather_opt_stats.count(); ++j) { - if (OB_ISNULL(gather_opt_stats.at(j).table_stat_)) { + for (int64_t j = 0; OB_SUCC(ret) && j < gather_subpart_opt_stats.count(); ++j) { + if (OB_ISNULL(gather_subpart_opt_stats.at(j).table_stat_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(gather_opt_stats.at(j).table_stat_)); + LOG_WARN("get unexpected error", K(ret), K(gather_subpart_opt_stats.at(j).table_stat_)); } else if (ObDbmsStatsUtils::is_subpart_id(param.all_subpart_infos_, - gather_opt_stats.at(j).table_stat_->get_partition_id(), + gather_subpart_opt_stats.at(j).table_stat_->get_partition_id(), cur_part_id)) { if (param.approx_part_infos_.at(i).part_id_ == cur_part_id) { - if (OB_FAIL(subpart_opt_stats.push_back(gather_opt_stats.at(j)))) { + if (OB_FAIL(subpart_opt_stats.push_back(gather_subpart_opt_stats.at(j)))) { LOG_WARN("failed to push back", K(ret)); } else {/*do nothing*/} } else {/*do nothing*/} @@ -307,34 +338,13 @@ int ObIncrementalStatEstimator::do_derive_part_stats_from_subpart_stats( K(param.approx_part_infos_.at(i))); } else if (OB_FAIL(do_derive_global_stat(ctx, alloc, param, subpart_opt_stats, true, PARTITION_LEVEL, param.approx_part_infos_.at(i).part_id_, - need_gather_hybrid_hist, opt_part_stat))) { + opt_part_stat))) { LOG_WARN("Failed to derive global stat from part stat", K(ret)); } else if (OB_FAIL(approx_part_opt_stats.push_back(opt_part_stat))) { LOG_WARN("faield to push back", K(ret)); - } else if (!need_gather_hybrid_hist) { - /*do nothing*/ - } else if (OB_FAIL(gather_hybrid_hist_opt_stats.push_back(opt_part_stat))) { - LOG_WARN("failed to push back", K(ret)); } else {/*do nothing*/} } } - if (OB_SUCC(ret) && !gather_hybrid_hist_opt_stats.empty()) { - //gather partition hybird hist togather. - ObHybridHistEstimator hybrid_est(ctx, *param.allocator_); - ObExtraParam extra; - extra.type_ = PARTITION_LEVEL; - extra.nth_part_ = 0; - extra.start_time_ = ObTimeUtility::current_time(); - ObTableStatParam part_param; - if (OB_FAIL(gen_part_param(param, gather_hybrid_hist_opt_stats, part_param))) { - LOG_WARN("failed to gen part param", K(ret)); - } else if (OB_FAIL(hybrid_est.estimate(part_param, extra, - gather_hybrid_hist_opt_stats))) { - LOG_WARN("failed to estimate hybrid histogram", K(ret)); - } else { - LOG_TRACE("succeed to gather partition hybrid hist", K(gather_hybrid_hist_opt_stats.count())); - } - } return ret; } @@ -370,6 +380,41 @@ int ObIncrementalStatEstimator::generate_all_opt_stat(ObIArray & return ret; } +int ObIncrementalStatEstimator::generate_all_opt_stat(ObIArray &table_stats, + ObIArray &col_stats, + int64_t col_cnt, + ObIArray &all_opt_stats) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < table_stats.count(); ++i) { + ObOptStat opt_stat; + opt_stat.table_stat_ = table_stats.at(i); + for (int64_t j = 0; OB_SUCC(ret) && j < col_cnt; ++j) { + int64_t idx = i * col_cnt + j; + if (OB_UNLIKELY(idx >= col_stats.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(idx), K(col_stats.count())); + } else if (OB_ISNULL(col_stats.at(idx)) || OB_ISNULL(opt_stat.table_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(table_stats), K(col_stats), K(i), K(idx)); + } else if (opt_stat.table_stat_->get_partition_id() == col_stats.at(idx)->get_partition_id()) { + if (OB_FAIL(opt_stat.column_stats_.push_back(col_stats.at(idx)))) { + LOG_WARN("failed to push back col stat", K(ret)); + } else {/*do nothing*/} + } + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(opt_stat.column_stats_.count() != col_cnt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(table_stats), K(col_cnt), K(col_stats)); + } else if (OB_FAIL(all_opt_stats.push_back(opt_stat))) { + LOG_WARN("failed to push back opt stat", K(ret)); + } else {/*do nothing*/} + } + } + return ret; +} + int ObIncrementalStatEstimator::get_table_and_column_stats( ObOptStat &src_opt_stat, const ObTableStatParam ¶m, @@ -426,14 +471,14 @@ int ObIncrementalStatEstimator::get_part_ids_and_column_ids_info(ObOptStat &src_ if (OB_UNLIKELY(no_gather_id == -1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error, expected valid id", K(ret), K(no_gather_id)); - } else if (OB_UNLIKELY(param.part_ids_.empty())) { + } else if (OB_UNLIKELY(param.all_part_infos_.empty())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error, expected isn't empty", K(ret), K(param.part_ids_)); + LOG_WARN("get unexpected error, expected isn't empty", K(ret), K(param.all_part_infos_)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < param.part_ids_.count(); ++i) { - if (param.part_ids_.at(i) == no_gather_id) { + for (int64_t i = 0; OB_SUCC(ret) && i < param.all_part_infos_.count(); ++i) { + if (param.all_part_infos_.at(i).part_id_ == no_gather_id) { /*do nothing*/ - } else if (OB_FAIL(part_ids.push_back(param.part_ids_.at(i)))) { + } else if (OB_FAIL(part_ids.push_back(param.all_part_infos_.at(i).part_id_))) { LOG_WARN("failed to push back id", K(ret)); } else {/*do nothing*/} } @@ -450,25 +495,22 @@ int ObIncrementalStatEstimator::do_derive_global_stat(ObExecContext &ctx, bool need_derive_hist, const StatLevel &approx_level, const int64_t partition_id, - bool &need_gather_hybrid_hist, ObOptStat &global_opt_stat) { int ret = OB_SUCCESS; - if (OB_FAIL(derive_global_tbl_stat(ctx, alloc, param, approx_level, partition_id, + if (OB_FAIL(derive_global_tbl_stat(alloc, param, approx_level, partition_id, part_opt_stats, global_opt_stat))) { LOG_WARN("failed to derive global tbl stat from part tbl stat", K(ret)); } else if (OB_FAIL(derive_global_col_stat(ctx, alloc, param, part_opt_stats, need_derive_hist, - approx_level, partition_id, need_gather_hybrid_hist, - global_opt_stat))) { + approx_level, partition_id, global_opt_stat))) { LOG_WARN("failed to derive global col stat from part col stat", K(ret)); } else { - LOG_TRACE("Succeed to derive global stat", K(part_opt_stats.count())); + LOG_TRACE("Succeed to derive global stat", K(part_opt_stats)); } return ret; } -int ObIncrementalStatEstimator::derive_global_tbl_stat(ObExecContext &ctx, - ObIAllocator &alloc, +int ObIncrementalStatEstimator::derive_global_tbl_stat(ObIAllocator &alloc, const ObTableStatParam ¶m, const StatLevel &approx_level, const int64_t partition_id, @@ -511,6 +553,7 @@ int ObIncrementalStatEstimator::derive_global_tbl_stat(ObExecContext &ctx, table_stat->set_partition_id(partition_id); table_stat->set_object_type(approx_level); table_stat->set_row_count(global_tstat.get_row_count()); + table_stat->set_sample_size(global_tstat.get_row_count()); table_stat->set_avg_row_size(global_tstat.get_avg_row_size()); table_stat->set_data_size(global_tstat.get_avg_data_size()); table_stat->set_macro_block_num(global_tstat.get_macro_block_count()); @@ -530,13 +573,12 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, bool need_derive_hist, const StatLevel &approx_level, const int64_t partition_id, - bool &need_gather_hybrid_hist, ObOptStat &global_opt_stat) { int ret = OB_SUCCESS; int64_t part_cnt = part_opt_stats.count(); int64_t column_cnt = param.column_params_.count(); - need_gather_hybrid_hist = false; + bool need_gather_hist = false; ObIArray &col_stats = global_opt_stat.column_stats_; if (OB_UNLIKELY(part_cnt <= 0 || column_cnt <= 0 || approx_level == INVALID_LEVEL || approx_level == SUBPARTITION_LEVEL) || @@ -559,7 +601,7 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, ObGlobalNdvEval ndv_eval; ObGlobalAvglenEval avglen_eval; ObSEArray all_part_histograms; - int64_t total_avg_len = 0; + bool can_try_derive_hist = need_derive_hist; int64_t max_bucket_num = param.column_params_.at(i).bucket_num_; for (int64_t j = 0; OB_SUCC(ret) && j < part_cnt; ++j) { ObOptColumnStat *opt_col_stat = NULL; @@ -585,7 +627,7 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, OB_FAIL(all_part_histograms.push_back(opt_col_stat->get_histogram()))) { LOG_WARN("failed to push back histogram", K(ret)); } else { - need_derive_hist &= opt_col_stat->get_histogram().is_valid(); + can_try_derive_hist &= (opt_col_stat->get_num_distinct() == 0 || opt_col_stat->get_histogram().is_valid()); null_eval.add(opt_col_stat->get_num_null()); if (opt_col_stat->get_num_distinct() != 0) { min_eval.add(opt_col_stat->get_min_value()); @@ -599,12 +641,13 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, } } if (OB_SUCC(ret)) { - col_stat->set_table_id(param.table_id_); + col_stat->set_table_id(param.column_params_.at(i).need_basic_stat() ? param.table_id_ : OB_INVALID_ID); col_stat->set_partition_id(partition_id); col_stat->set_column_id(param.column_params_.at(i).column_id_); col_stat->set_stat_level(approx_level); col_stat->set_num_null(null_eval.get()); col_stat->set_num_not_null(not_null_eval.get()); + col_stat->get_histogram().set_sample_size(not_null_eval.get()); col_stat->set_num_distinct(ndv_eval.get()); col_stat->set_avg_len(avglen_eval.get()); ndv_eval.get_llc_bitmap(col_stat->get_llc_bitmap(), col_stat->get_llc_bitmap_size()); @@ -618,39 +661,70 @@ int ObIncrementalStatEstimator::derive_global_col_stat(ObExecContext &ctx, } else if (OB_FAIL(ob_write_obj(alloc, max_eval.get(), new_max_obj)) || FALSE_IT(col_stat->set_max_value(new_max_obj))) { LOG_WARN("failed to set max value", K(ret), K(max_eval.get()), K(new_max_obj)); - } else if (need_derive_hist && !all_part_histograms.empty() && - OB_FAIL(derive_global_histogram(all_part_histograms, - alloc, - max_bucket_num, - global_opt_stat.table_stat_->get_row_count(), - col_stat->get_num_not_null(), - col_stat->get_num_distinct(), - col_stat->get_histogram(), - need_gather_hybrid_hist))) { - LOG_WARN("failed to derive global histogram from part histogram", K(ret)); } else if (OB_FAIL(col_stats.push_back(col_stat))) { LOG_WARN("failed to push back", K(ret)); - } else { + } else if (need_derive_hist) { + if (can_try_derive_hist && col_stat->get_num_distinct() > 0) { + if (OB_FAIL(derive_global_histogram(all_part_histograms, + alloc, + max_bucket_num, + global_opt_stat.table_stat_->get_row_count(), + col_stat->get_num_not_null(), + col_stat->get_num_distinct(), + col_stat->get_histogram(), + need_gather_hist))) { + LOG_WARN("failed to derive global histogram from part histogram", K(ret)); + } + } else if (max_bucket_num > 1 && + param.column_params_.at(i).need_basic_stat() && + col_stat->get_num_distinct() > 0) { + need_gather_hist = true; + int64_t max_disuse_cnt = std::ceil(col_stat->get_num_not_null() * 1.0 / max_bucket_num); + //After testing, the error of using hyperloglog to estimate ndv is within %5. + const double MAX_LLC_NDV_ERR_RATE = !param.need_approx_ndv_ ? 0.0 : 0.05; + const int64_t fault_tolerance_cnt = std::ceil(col_stat->get_num_distinct() * MAX_LLC_NDV_ERR_RATE); + if (col_stat->get_num_distinct() >= max_bucket_num + max_disuse_cnt + fault_tolerance_cnt) { + //directly gather hybrid histogram + col_stat->get_histogram().set_type(ObHistType::HYBIRD); + } else { + //otherwise, try gather top frequery histogram + col_stat->get_histogram().set_type(ObHistType::TOP_FREQUENCY); + } + } LOG_TRACE("succeed to derive global col stat", K(*col_stat)); } } } } - if (OB_SUCC(ret) && need_gather_hybrid_hist) { - if (approx_level == TABLE_LEVEL) {//for partition level derive put together do it. - ObHybridHistEstimator hybrid_est(ctx, *param.allocator_); - ObExtraParam extra; - extra.type_ = TABLE_LEVEL; - extra.nth_part_ = 0; - extra.start_time_ = ObTimeUtility::current_time(); - ObSEArray opt_stats; - if (OB_FAIL(opt_stats.push_back(global_opt_stat))) { - LOG_WARN("failed to push back opt stat", K(ret)); - } else if (OB_FAIL(hybrid_est.estimate(param, extra, opt_stats))) { - LOG_WARN("failed to estimate hybrid histogram", K(ret)); - } else { - LOG_TRACE("succeed to gather hybrid hist"); - } + if (OB_SUCC(ret) && need_gather_hist) { + ObOptStatGatherParam gather_param; + ObTopkHistEstimator topk_est(ctx, *param.allocator_); + ObHybridHistEstimator hybrid_est(ctx, *param.allocator_); + ObSEArray tmp_opt_stats; + if (OB_FAIL(ObDbmsStatsUtils::prepare_gather_stat_param(param, approx_level, NULL, false, + DEFAULT_STAT_GATHER_VECTOR_BATCH_SIZE, + gather_param))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(gather_param.column_params_.assign(param.column_params_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (gather_param.stat_level_ != TABLE_LEVEL && + OB_FAIL(gather_param.partition_infos_.assign(param.approx_part_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (gather_param.stat_level_ != TABLE_LEVEL && + OB_FAIL(ObDbmsStatsUtils::remove_stat_gather_param_partition_info(global_opt_stat.table_stat_->get_partition_id(), + gather_param))) { + LOG_WARN("failed to remove stat gather param partition info", K(ret)); + } else if (OB_FAIL(topk_est.estimate(gather_param, global_opt_stat))) { + LOG_WARN("failed to estimate topk histogram", K(ret)); + } else if (OB_FAIL(hybrid_est.estimate(gather_param, global_opt_stat))) { + LOG_WARN("failed to estimate hybrid histogram", K(ret)); + } else if (OB_FAIL(tmp_opt_stats.push_back(global_opt_stat))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::check_all_cols_range_skew(gather_param.column_params_, + tmp_opt_stats))) { + LOG_WARN("failed to check all cols range skew", K(ret)); + } else { + LOG_TRACE("succeed to gather hist", K(gather_param), K(global_opt_stat)); } } } @@ -664,13 +738,15 @@ int ObIncrementalStatEstimator::derive_global_histogram(ObIArray &a int64_t not_null_count, int64_t num_distinct, ObHistogram &histogram, - bool &need_gather_hybrid_hist) + bool &need_gather_hist) { int ret = OB_SUCCESS; ObSEArray all_bkts; histogram.reset(); void *ptr = NULL; - if (OB_ISNULL(ptr = allocator.alloc(sizeof(ObTopKFrequencyHistograms)))) { + if (max_bucket_num <= 1) { + //do nothing + } else if (OB_ISNULL(ptr = allocator.alloc(sizeof(ObTopKFrequencyHistograms)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to allocate memory", K(ret), K(ptr)); } else { @@ -678,6 +754,7 @@ int ObIncrementalStatEstimator::derive_global_histogram(ObIArray &a top_k_fre_hist->set_window_size(1000); top_k_fre_hist->set_item_size(256); top_k_fre_hist->set_is_topk_hist_need_des_row(true); + top_k_fre_hist->set_max_disuse_cnt(std::ceil(not_null_count * 1.0 / max_bucket_num)); for (int64_t i = 0; OB_SUCC(ret) && i < all_part_histograms.count(); ++i) { if (all_part_histograms.at(i).is_valid()) { if (all_part_histograms.at(i).get_type() == ObHistType::FREQUENCY || @@ -685,11 +762,9 @@ int ObIncrementalStatEstimator::derive_global_histogram(ObIArray &a all_part_histograms.at(i).get_type() == ObHistType::HYBIRD) { const ObHistogram::Buckets &part_bkts = all_part_histograms.at(i).get_buckets(); for (int64_t j = 0; OB_SUCC(ret) && j < part_bkts.count(); ++j) { - for (int64_t k = 0; OB_SUCC(ret) && k < part_bkts.at(j).endpoint_repeat_count_; ++k) { - if (OB_FAIL(top_k_fre_hist->add_top_k_frequency_item( - part_bkts.at(j).endpoint_value_))) { - LOG_WARN("failed to add topk frequency item", K(ret)); - } else {/*do nothing*/} + if (OB_FAIL(top_k_fre_hist->add_top_k_frequency_item(part_bkts.at(j).endpoint_value_, + part_bkts.at(j).endpoint_repeat_count_))) { + LOG_WARN("failed to add topk frequency item", K(ret)); } } } else { @@ -706,7 +781,9 @@ int ObIncrementalStatEstimator::derive_global_histogram(ObIArray &a if (OB_FAIL(top_k_fre_hist->create_topk_fre_items())) { LOG_WARN("failed to adjust frequency sort", K(ret)); } else if (top_k_fre_hist->get_buckets().count() == 0) { - /*do nothing*/ + //directly to gather hybrid histogram + histogram.set_type(ObHistType::HYBIRD); + need_gather_hist = true; } else if (OB_FAIL(ObStatTopKHist::build_histogram_from_topk_items( allocator, top_k_fre_hist->get_buckets(), @@ -719,7 +796,7 @@ int ObIncrementalStatEstimator::derive_global_histogram(ObIArray &a } else if (OB_FAIL(histogram.deep_copy(allocator, tmp_histogram))) { LOG_WARN("failed to deep copy", K(ret)); } else { - need_gather_hybrid_hist |= histogram.is_hybrid(); + need_gather_hist |= histogram.is_hybrid(); } } if (top_k_fre_hist != NULL) { @@ -776,81 +853,37 @@ int ObIncrementalStatEstimator::get_column_ids(const ObIArray return ret; } -int ObIncrementalStatEstimator::gen_part_param(const ObTableStatParam ¶m, - const ObIArray &need_hybrid_hist_opt_stats, - ObTableStatParam &part_param) +int ObIncrementalStatEstimator::get_need_hybrid_part_infos(const ObTableStatParam ¶m, + const ObIArray &need_hybrid_hist_opt_stats, + ObIArray &hybrid_part_infos) { int ret = OB_SUCCESS; - if (OB_FAIL(part_param.assign(param))) { - LOG_WARN("failed to assign", K(ret)); - } else { - part_param.part_infos_.reset(); - for (int64_t i = 0; OB_SUCC(ret) && i < need_hybrid_hist_opt_stats.count(); ++i) { - if (OB_ISNULL(need_hybrid_hist_opt_stats.at(i).table_stat_)) { + for (int64_t i = 0; OB_SUCC(ret) && i < need_hybrid_hist_opt_stats.count(); ++i) { + if (OB_ISNULL(need_hybrid_hist_opt_stats.at(i).table_stat_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(need_hybrid_hist_opt_stats.at(i).table_stat_)); + } else { + bool find_it = false; + for (int64_t j = 0; OB_SUCC(ret) && !find_it && j < param.approx_part_infos_.count(); ++j) { + if (need_hybrid_hist_opt_stats.at(i).table_stat_->get_partition_id() == + param.approx_part_infos_.at(j).part_id_) { + if (OB_FAIL(hybrid_part_infos.push_back(param.approx_part_infos_.at(j)))) { + LOG_WARN("failed to push back", K(ret)); + } else { + find_it = true; + } + } else {/*do noting*/} + } + if (OB_SUCC(ret) && !find_it) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(need_hybrid_hist_opt_stats.at(i).table_stat_)); - } else { - bool find_it = false; - for (int64_t j = 0; OB_SUCC(ret) && !find_it && j < param.approx_part_infos_.count(); ++j) { - if (need_hybrid_hist_opt_stats.at(i).table_stat_->get_partition_id() == - param.approx_part_infos_.at(j).part_id_) { - if (OB_FAIL(part_param.part_infos_.push_back(param.approx_part_infos_.at(j)))) { - LOG_WARN("failed to push back", K(ret)); - } else { - find_it = true; - } - } else {/*do noting*/} - } - if (OB_SUCC(ret) && !find_it) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(param.approx_part_infos_), - K(*need_hybrid_hist_opt_stats.at(i).table_stat_)); - } + LOG_WARN("get unexpected error", K(ret), K(param.approx_part_infos_), + K(*need_hybrid_hist_opt_stats.at(i).table_stat_)); } } } return ret; } - -/*we take following strategy to judge a part can incremental gather wheather or not: - * if we need approx global and gather subpart: - * 1.if the part need gather stats: - * if more than 50% of subpart is regather stats, then not choose incremental gather stats, else - * choose incremental gather stats - * 2.if the part don't need gather stats, and some subpart is regather stats, then the part - * stats need incremental gather. -*/ -bool ObIncrementalStatEstimator::is_part_can_incremental_gather(const ObTableStatParam ¶m, - int64_t part_id, - int64_t subpart_cnt, - bool is_gather_part) -{ - bool can_be = false; - int64_t no_regather_subpart_cnt = 0; - int64_t cur_part_id = OB_INVALID_ID; - if (param.global_stat_param_.need_modify_ && - param.global_stat_param_.gather_approx_ && - param.subpart_stat_param_.need_modify_ && - param.part_level_ == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { - for (int64_t i = 0; i < param.no_regather_partition_ids_.count(); ++i) { - if (ObDbmsStatsUtils::is_subpart_id(param.all_subpart_infos_, - param.no_regather_partition_ids_.at(i), - cur_part_id)) { - if (cur_part_id == part_id) { - ++ no_regather_subpart_cnt; - } - } - } - if (!is_gather_part) { - can_be = (subpart_cnt != no_regather_subpart_cnt); - } else { - can_be = subpart_cnt > 0 ? no_regather_subpart_cnt <= subpart_cnt / 2 : false; - } - } - return can_be; -} - int ObIncrementalStatEstimator::get_no_regather_subpart_stats( const ObTableStatParam ¶m, ObIArray &no_regather_table_stats, @@ -900,6 +933,8 @@ int ObIncrementalStatEstimator::gen_opt_stat_param_by_direct_load(ObExecContext const share::schema::ObTableSchema *table_schema = NULL; share::schema::ObSchemaGetterGuard *schema_guard = ctx.get_virtual_table_ctx().schema_guard_; param.table_id_ = table_id; + ObSEArray part_ids; + ObSEArray subpart_ids; if (OB_ISNULL(schema_guard = ctx.get_virtual_table_ctx().schema_guard_) || OB_ISNULL(ctx.get_my_session())) { ret = OB_ERR_UNEXPECTED; @@ -914,11 +949,11 @@ int ObIncrementalStatEstimator::gen_opt_stat_param_by_direct_load(ObExecContext } else if (OB_FAIL(ObDbmsStatsUtils::get_part_infos(*table_schema, param.part_infos_, param.subpart_infos_, - param.part_ids_, - param.subpart_ids_))) { + part_ids, + subpart_ids))) { LOG_WARN("failed to get partition infos", K(ret)); - } else if (table_schema->get_part_level() == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO - && OB_FAIL(param.approx_part_infos_.assign(param.part_infos_))) { + } else if (OB_FAIL(param.all_part_infos_.assign(param.part_infos_)) || + OB_FAIL(param.all_subpart_infos_.assign(param.subpart_infos_))) { LOG_WARN("failed to assign", K(ret)); } else if (OB_FAIL(pl::ObDbmsStats::init_column_stat_params(alloc, *schema_guard, @@ -926,34 +961,46 @@ int ObIncrementalStatEstimator::gen_opt_stat_param_by_direct_load(ObExecContext param.column_params_))) { LOG_WARN("failed to init column stat params", K(ret)); } else { + for (int64_t i = 0; i < param.column_params_.count(); ++i) { + if (param.column_params_.at(i).is_valid_opt_col()) { + param.column_params_.at(i).set_need_basic_stat(); + } + } param.tenant_id_ = ctx.get_my_session()->get_effective_tenant_id(); param.part_level_ = table_schema->get_part_level(); - param.total_part_cnt_ = table_schema->get_all_part_num(); - param.global_stat_param_.need_modify_ = true; - param.part_stat_param_.need_modify_ = true; - param.subpart_stat_param_.need_modify_ = true; + param.global_stat_param_.set_gather_stat(true); + param.part_stat_param_.set_gather_stat(table_schema->get_part_level() == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO); + param.subpart_stat_param_.set_gather_stat(); if (OB_FAIL(pl::ObDbmsStats::set_param_global_part_id(ctx, param))) { LOG_WARN("failed to set param globa part id", K(ret)); + } else if (table_schema->get_part_level() == share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO) { + if (OB_FAIL(param.approx_part_infos_.assign(param.part_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } else { + param.part_infos_.reset(); + } } - LOG_TRACE("succeed to gen opt stat param by direct load", K(ret)); + LOG_TRACE("succeed to gen opt stat param by direct load", K(param)); } return ret; } -int ObIncrementalStatEstimator::get_all_part_opt_stat_by_direct_load( +int ObIncrementalStatEstimator::get_all_part_opt_stats( const ObTableStatParam param, + const ObIArray &partition_infos, ObIArray &part_tab_stats, ObIArray &part_col_handles, ObIArray &part_opt_stats) { int ret = OB_SUCCESS; + ObSEArray partition_ids; ObSEArray column_ids; - const ObIArray &partition_ids = param.part_level_ == share::schema::PARTITION_LEVEL_ONE ? - param.part_ids_ : param.subpart_ids_; if (OB_UNLIKELY(param.part_level_ != share::schema::PARTITION_LEVEL_ONE && param.part_level_ != share::schema::PARTITION_LEVEL_TWO)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(param)); + } else if (OB_FAIL(get_partition_ids(partition_infos, partition_ids))) { + LOG_WARN("failed to get column ids", K(ret)); } else if (OB_FAIL(get_column_ids(param.column_params_, column_ids))) { LOG_WARN("failed to get column ids", K(ret)); } else if (OB_FAIL(ObOptStatManager::get_instance().get_table_stat(param.tenant_id_, @@ -973,7 +1020,148 @@ int ObIncrementalStatEstimator::get_all_part_opt_stat_by_direct_load( part_opt_stats))) { LOG_WARN("failed to generate all opt stat", K(ret)); } else { - LOG_TRACE("Succeed get all part opt stat by direct load", K(param), K(part_tab_stats)); + LOG_TRACE("Succeed get all part opt stats", K(param), K(part_tab_stats)); + } + return ret; +} + +int ObIncrementalStatEstimator::get_partition_ids(const ObIArray &partition_infos, + ObIArray &partition_ids) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < partition_infos.count(); ++i) { + if (OB_FAIL(partition_ids.push_back(partition_infos.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } + return ret; +} + +int ObIncrementalStatEstimator::derive_part_index_stat_by_subpart_index_stats( + const ObTableStatParam ¶m, + const ObIArray &subpart_index_stats, + ObIArray &part_index_stats) +{ + int ret = OB_SUCCESS; + int64_t cur_part_id = OB_INVALID_ID; + for (int64_t i = 0; OB_SUCC(ret) && i < param.approx_part_infos_.count(); ++i) { + ObSEArray subpart_opt_stats; + for (int64_t j = 0; OB_SUCC(ret) && j < subpart_index_stats.count(); ++j) { + if (OB_ISNULL(subpart_index_stats.at(j))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(subpart_index_stats.at(j))); + } else if (ObDbmsStatsUtils::is_subpart_id(param.all_subpart_infos_, + subpart_index_stats.at(j)->get_partition_id(), + cur_part_id)) { + if (param.approx_part_infos_.at(i).part_id_ == cur_part_id) { + ObOptStat tmp_opt_stat; + tmp_opt_stat.table_stat_ = subpart_index_stats.at(j); + if (OB_FAIL(subpart_opt_stats.push_back(tmp_opt_stat))) { + LOG_WARN("failed to push back", K(ret)); + } else {/*do nothing*/} + } else {/*do nothing*/} + } else {/*do nothing*/} + } + //derive part stat from subpart stats + if (OB_SUCC(ret)) { + ObOptStat opt_part_stat; + if (OB_ISNULL(param.allocator_) || + OB_UNLIKELY(subpart_opt_stats.count() != param.approx_part_infos_.at(i).subpart_cnt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param), K(subpart_opt_stats.count()), + K(param.approx_part_infos_.at(i))); + } else if (OB_FAIL(derive_global_tbl_stat(*param.allocator_, param, + PARTITION_LEVEL, + param.approx_part_infos_.at(i).part_id_, + subpart_opt_stats, + opt_part_stat))) { + LOG_WARN("failed to derive global tbl stat from part tbl stat", K(ret)); + } else if (OB_FAIL(part_index_stats.push_back(opt_part_stat.table_stat_))) { + LOG_WARN("failed to push back", K(ret)); + } else { + LOG_TRACE("succeed to derive part index stat by subpart index stats", K(param), + K(subpart_index_stats), K(part_index_stats)); + } + } + } + return ret; +} + +int ObIncrementalStatEstimator::derive_global_index_stat_by_part_index_stats( + const ObTableStatParam ¶m, + const ObIArray &part_index_stats, + ObIArray &all_index_stats) +{ + int ret = OB_SUCCESS; + ObSEArray part_opt_stats; + for (int64_t i = 0; OB_SUCC(ret) && i < part_index_stats.count(); ++i) { + ObOptStat tmp_opt_stat; + tmp_opt_stat.table_stat_ = part_index_stats.at(i); + ret = part_opt_stats.push_back(tmp_opt_stat); + } + if (OB_SUCC(ret)) { + ObOptStat global_opt_stat; + if (OB_ISNULL(param.allocator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param.allocator_)); + } else if (OB_FAIL(derive_global_tbl_stat(*param.allocator_, param, + TABLE_LEVEL, + param.global_part_id_, + part_opt_stats, + global_opt_stat))) { + LOG_WARN("failed to derive global tbl stat from part tbl stat", K(ret)); + } else if (OB_FAIL(all_index_stats.push_back(global_opt_stat.table_stat_))) { + LOG_WARN("failed to push back", K(ret)); + } else { + LOG_TRACE("succeed to derive global index stat by part index stats", K(param), K(all_index_stats)); + } + } + return ret; +} + +int ObIncrementalStatEstimator::prepare_get_opt_stats_param(const ObTableStatParam ¶m, + bool derive_part_stat, + ObTableStatParam &new_param) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(new_param.assign(param))) { + LOG_WARN("failed to assign", K(ret)); + } else { + //reset relation param + new_param.global_stat_param_.reset_gather_stat(); + new_param.part_stat_param_.reset_gather_stat(); + new_param.subpart_stat_param_.reset_gather_stat(); + new_param.part_infos_.reset(); + new_param.approx_part_infos_.reset(); + new_param.subpart_infos_.reset(); + new_param.all_part_infos_.reset(); + new_param.all_subpart_infos_.reset(); + if (!derive_part_stat) { + new_param.part_stat_param_.set_gather_stat(); + if (OB_FAIL(new_param.part_infos_.assign(param.all_part_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } + } else { + new_param.subpart_stat_param_.set_gather_stat(); + for (int64_t i = 0; OB_SUCC(ret) && i < param.approx_part_infos_.count(); ++i) { + int64_t subpart_cnt = 0; + for (int64_t j = 0; + OB_SUCC(ret) && subpart_cnt < param.approx_part_infos_.at(i).subpart_cnt_ && j < param.all_subpart_infos_.count(); + ++j) { + if (param.approx_part_infos_.at(i).part_id_ == param.all_subpart_infos_.at(j).first_part_id_) { + if (OB_FAIL(new_param.subpart_infos_.push_back(param.all_subpart_infos_.at(j)))) { + LOG_WARN("failed to push back", K(ret)); + } else { + ++ subpart_cnt; + } + } + } + if (OB_UNLIKELY(subpart_cnt != param.approx_part_infos_.at(i).subpart_cnt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(subpart_cnt), K(param)); + } + } + } } return ret; } diff --git a/src/share/stat/ob_incremental_stat_estimator.h b/src/share/stat/ob_incremental_stat_estimator.h index cea2f29820..5ac975c449 100644 --- a/src/share/stat/ob_incremental_stat_estimator.h +++ b/src/share/stat/ob_incremental_stat_estimator.h @@ -26,36 +26,39 @@ class ObIncrementalStatEstimator { public: - static int try_derive_global_stat(ObExecContext &ctx, - const ObTableStatParam ¶m, - ObExtraParam &extra, - ObIArray &approx_part_opt_stats, - ObIArray &opt_stats); - - static bool is_part_can_incremental_gather(const ObTableStatParam ¶m, - int64_t part_id, - int64_t subpart_cnt, - bool is_gather_part); - - static int derive_global_stat_by_direct_load(ObExecContext &ctx, const uint64_t table_id); -private: - static int derive_global_stat_from_part_stats(ObExecContext &ctx, const ObTableStatParam ¶m, const ObIArray &approx_part_opt_stats, - ObIArray &opt_stats); + ObOptStat &global_opt_stat); static int derive_part_stats_from_subpart_stats(ObExecContext &ctx, - const ObTableStatParam ¶m, - const ObIArray &gather_opt_stats, - ObIArray &approx_part_opt_stats); + const ObTableStatParam ¶m, + const ObIArray &gather_subpart_opt_stats, + ObIArray &approx_part_opt_stats); + static int derive_global_stat_by_direct_load(ObExecContext &ctx, const uint64_t table_id); + + static int derive_split_gather_stats(ObExecContext &ctx, + ObMySQLTransaction &trans, + const ObTableStatParam ¶m, + bool derive_part_stat, + bool is_all_columns_gather, + ObIArray &all_tstats); + + static int derive_part_index_stat_by_subpart_index_stats(const ObTableStatParam ¶m, + const ObIArray &subpart_index_stats, + ObIArray &part_index_stats); + + static int derive_global_index_stat_by_part_index_stats(const ObTableStatParam ¶m, + const ObIArray &part_index_stats, + ObIArray &all_index_stats); +private: static int do_derive_part_stats_from_subpart_stats( ObExecContext &ctx, ObIAllocator &alloc, const ObTableStatParam ¶m, const ObIArray &no_regather_subpart_opt_stats, - const ObIArray &gather_opt_stats, + const ObIArray &gather_subpart_opt_stats, ObIArray &approx_part_opt_stats); static int get_table_and_column_stats(ObOptStat &src_opt_stat, @@ -73,6 +76,11 @@ private: int64_t col_cnt, ObIArray &all_opt_stats); + static int generate_all_opt_stat(ObIArray &table_stats, + ObIArray &col_stats, + int64_t col_cnt, + ObIArray &all_opt_stats); + static int do_derive_global_stat(ObExecContext &ctx, ObIAllocator &alloc, const ObTableStatParam ¶m, @@ -80,11 +88,9 @@ private: bool need_derive_hist, const StatLevel &approx_level, const int64_t partition_id, - bool &need_gather_hybrid_hist, ObOptStat &global_opt_stat); - static int derive_global_tbl_stat(ObExecContext &ctx, - ObIAllocator &alloc, + static int derive_global_tbl_stat(ObIAllocator &alloc, const ObTableStatParam ¶m, const StatLevel &approx_level, const int64_t partition_id, @@ -98,7 +104,6 @@ private: bool need_derive_hist, const StatLevel &approx_level, const int64_t partition_id, - bool &need_gather_hybrid_hist, ObOptStat &global_opt_stat); static int derive_global_histogram(ObIArray &all_part_histogram, @@ -108,7 +113,7 @@ private: int64_t not_null_count, int64_t num_distinct, ObHistogram &histogram, - bool &need_gather_hybrid_hist); + bool &need_gather_hist); static int get_no_regather_partition_stats(const uint64_t tenant_id, const uint64_t table_id, @@ -121,9 +126,9 @@ private: static int get_column_ids(const ObIArray &column_params, ObIArray &column_ids); - static int gen_part_param(const ObTableStatParam ¶m, - const ObIArray &need_hybrid_hist_opt_stats, - ObTableStatParam &part_param); + static int get_need_hybrid_part_infos(const ObTableStatParam ¶m, + const ObIArray &need_hybrid_hist_opt_stats, + ObIArray &hybrid_part_infos); static int get_no_regather_subpart_stats(const ObTableStatParam ¶m, ObIArray &no_regather_table_stats, @@ -135,10 +140,18 @@ private: const uint64_t table_id, ObTableStatParam ¶m); - static int get_all_part_opt_stat_by_direct_load(const ObTableStatParam param, - ObIArray &part_tab_stats, - ObIArray &part_col_handles, - ObIArray &part_opt_stats); + static int get_all_part_opt_stats(const ObTableStatParam param, + const ObIArray &partition_infos, + ObIArray &part_tab_stats, + ObIArray &part_col_handles, + ObIArray &part_opt_stats); + + static int get_partition_ids(const ObIArray &partition_infos, + ObIArray &partition_ids); + + static int prepare_get_opt_stats_param(const ObTableStatParam ¶m, + bool derive_part_stat, + ObTableStatParam &new_param); }; diff --git a/src/share/stat/ob_index_stats_estimator.cpp b/src/share/stat/ob_index_stats_estimator.cpp index 5f9fbd94b0..8c4866379b 100644 --- a/src/share/stat/ob_index_stats_estimator.cpp +++ b/src/share/stat/ob_index_stats_estimator.cpp @@ -25,13 +25,12 @@ ObIndexStatsEstimator::ObIndexStatsEstimator(ObExecContext &ctx, ObIAllocator &a : ObBasicStatsEstimator(ctx, allocator) {} -int ObIndexStatsEstimator::estimate(const ObTableStatParam ¶m, - const ObExtraParam &extra, +int ObIndexStatsEstimator::estimate(const ObOptStatGatherParam ¶m, ObIArray &dst_opt_stats) { int ret = OB_SUCCESS; const ObIArray &column_params = param.column_params_; - ObString no_rewrite("NO_REWRITE USE_PLAN_CACHE(NONE) DBMS_STATS"); + ObString no_rewrite("NO_REWRITE USE_PLAN_CACHE(NONE) DBMS_STATS OPT_PARAM('ROWSETS_MAX_ROWS', 256)"); ObString calc_part_id_str; ObOptTableStat tab_stat; ObOptStat src_opt_stat; @@ -63,20 +62,19 @@ int ObIndexStatsEstimator::estimate(const ObTableStatParam ¶m, LOG_WARN("failed to add from table", K(ret)); } else if (OB_FAIL(fill_parallel_info(ctx_.get_allocator(), param.degree_))) { LOG_WARN("failed to add query sql parallel info", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(extra.start_time_, - param.duration_time_, + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(param.gather_start_time_, + param.max_duration_time_, duration_time))) { LOG_WARN("failed to get valid duration time", K(ret)); } else if (OB_FAIL(fill_query_timeout_info(ctx_.get_allocator(), duration_time))) { LOG_WARN("failed to fill query timeout info", K(ret)); } else if (dst_opt_stats.count() > 1 && - OB_FAIL(fill_index_group_by_info(ctx_.get_allocator(), param, - extra, calc_part_id_str))) { + OB_FAIL(fill_index_group_by_info(ctx_.get_allocator(), param, calc_part_id_str))) { LOG_WARN("failed to group by info", K(ret)); - } else if (OB_FAIL(add_stat_item(ObStatRowCount(¶m, src_tab_stat)))) { + } else if (OB_FAIL(add_stat_item(ObStatRowCount(src_tab_stat)))) { LOG_WARN("failed to add row count", K(ret)); } else if (calc_part_id_str.empty()) { - if (OB_FAIL(fill_partition_condition(ctx_.get_allocator(), param, extra, + if (OB_FAIL(fill_partition_condition(ctx_.get_allocator(), param, dst_opt_stats.at(0).table_stat_->get_partition_id()))) { LOG_WARN("failed to add fill partition condition", K(ret)); } else if (OB_UNLIKELY(dst_opt_stats.count() != 1) || @@ -86,7 +84,7 @@ int ObIndexStatsEstimator::estimate(const ObTableStatParam ¶m, } else { src_tab_stat->set_partition_id(dst_opt_stats.at(0).table_stat_->get_partition_id()); } - } else if (OB_FAIL(add_stat_item(ObPartitionId(¶m, src_tab_stat, calc_part_id_str, -1)))) { + } else if (OB_FAIL(add_stat_item(ObPartitionId(src_tab_stat, calc_part_id_str, -1)))) { LOG_WARN("failed to add partition id", K(ret)); } else {/*do nothing*/} for (int64_t i = 0; OB_SUCC(ret) && i < column_params.count(); ++i) { @@ -96,12 +94,11 @@ int ObIndexStatsEstimator::estimate(const ObTableStatParam ¶m, } else {/*do nothing*/} } if (OB_SUCC(ret)) { - if (OB_FAIL(add_stat_item(ObStatAvgRowLen(¶m, src_tab_stat, src_col_stats)))) { + if (OB_FAIL(add_stat_item(ObStatAvgRowLen(src_tab_stat, src_col_stats)))) { LOG_WARN("failed to add avg row size estimator", K(ret)); } else if (OB_FAIL(pack(raw_sql))) { LOG_WARN("failed to pack raw sql", K(ret)); - } else if (OB_FAIL(do_estimate(param.tenant_id_, raw_sql.string(), COPY_ALL_STAT, - src_opt_stat, dst_opt_stats))) { + } else if (OB_FAIL(do_estimate(param.tenant_id_, raw_sql.string(), true, src_opt_stat, dst_opt_stats))) { LOG_WARN("failed to evaluate basic stats", K(ret)); } else { LOG_TRACE("index stats is collected", K(dst_opt_stats.count())); @@ -147,8 +144,7 @@ int ObIndexStatsEstimator::fill_index_info(common::ObIAllocator &alloc, //Specify the partition name or non-partition table don't use group by. int ObIndexStatsEstimator::fill_index_group_by_info(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra, + const ObOptStatGatherParam ¶m, ObString &calc_part_id_str) { int ret = OB_SUCCESS; @@ -156,13 +152,13 @@ int ObIndexStatsEstimator::fill_index_group_by_info(ObIAllocator &allocator, : "GROUP BY CALC_PARTITION_ID(`%.*s`, `%.*s`, %.*s)"; char *buf = NULL; ObString type_str; - if (extra.type_ == PARTITION_LEVEL) { + if (param.stat_level_ == PARTITION_LEVEL) { type_str = ObString(4, "PART"); - } else if (extra.type_ == SUBPARTITION_LEVEL) { + } else if (param.stat_level_ == SUBPARTITION_LEVEL) { type_str = ObString(7, "SUBPART"); } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected type", K(extra.type_), K(ret)); + LOG_WARN("get unexpected type", K(param.stat_level_), K(ret)); } if (OB_SUCC(ret)) { const int64_t len = strlen(fmt_str) + @@ -195,19 +191,18 @@ int ObIndexStatsEstimator::fill_index_group_by_info(ObIAllocator &allocator, //Specify the partition name need add filter. int ObIndexStatsEstimator::fill_partition_condition(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra, + const ObOptStatGatherParam ¶m, const int64_t dst_partition_id) { int ret = OB_SUCCESS; - if (extra.type_ == PARTITION_LEVEL || extra.type_ == SUBPARTITION_LEVEL) { + if (param.stat_level_ == PARTITION_LEVEL || param.stat_level_ == SUBPARTITION_LEVEL) { const char *fmt_str = lib::is_oracle_mode() ? "WHERE CALC_PARTITION_ID(\"%.*s\", \"%.*s\", %.*s) = %ld" : "WHERE CALC_PARTITION_ID(`%.*s`, `%.*s`, %.*s) = %ld"; char *buf = NULL; ObString type_str; - if (extra.type_ == PARTITION_LEVEL) { + if (param.stat_level_ == PARTITION_LEVEL) { type_str = ObString(4, "PART"); - } else if (extra.type_ == SUBPARTITION_LEVEL) { + } else if (param.stat_level_ == SUBPARTITION_LEVEL) { type_str = ObString(7, "SUBPART"); } const int64_t len = strlen(fmt_str) + @@ -331,7 +326,7 @@ int ObIndexStatsEstimator::fast_gather_index_stats(ObExecContext &ctx, if (OB_FAIL(trans.start(ctx.get_sql_proxy(), index_param.tenant_id_))) { LOG_WARN("fail to start transaction", K(ret)); } else if (OB_FAIL(mgr.update_table_stat(index_param.tenant_id_, - trans, + trans.get_connection(), index_table_stats, index_param.is_index_stat_))) { LOG_WARN("failed to update table stats", K(ret)); @@ -434,6 +429,11 @@ int ObIndexStatsEstimator::get_all_need_gather_partition_ids(const ObTableStatPa LOG_WARN("failed to push back", K(ret)); } } + for (int64_t i = 0; OB_SUCC(ret) && i < data_param.approx_part_infos_.count(); ++i) { + if (OB_FAIL(gather_part_ids.push_back(data_param.approx_part_infos_.at(i).part_id_))) { + LOG_WARN("failed to push back", K(ret)); + } + } } if (OB_SUCC(ret) && index_param.subpart_stat_param_.need_modify_) { diff --git a/src/share/stat/ob_index_stats_estimator.h b/src/share/stat/ob_index_stats_estimator.h index eae151f47c..0629fa8349 100644 --- a/src/share/stat/ob_index_stats_estimator.h +++ b/src/share/stat/ob_index_stats_estimator.h @@ -27,8 +27,7 @@ class ObIndexStatsEstimator : public ObBasicStatsEstimator public: explicit ObIndexStatsEstimator(ObExecContext &ctx, ObIAllocator &allocator); - int estimate(const ObTableStatParam ¶m, - const ObExtraParam &extra, + int estimate(const ObOptStatGatherParam ¶m, ObIArray &dst_opt_stats); static int fast_gather_index_stats(ObExecContext &ctx, @@ -41,13 +40,11 @@ private: const ObString &index_name); int fill_index_group_by_info(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra, + const ObOptStatGatherParam ¶m, ObString &calc_part_id_str); int fill_partition_condition(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra, + const ObOptStatGatherParam ¶m, const int64_t dst_partition_id); static int fast_get_index_avg_len(const int64_t data_partition_id, diff --git a/src/share/stat/ob_opt_column_stat.cpp b/src/share/stat/ob_opt_column_stat.cpp index 00c8c833fd..658956438d 100644 --- a/src/share/stat/ob_opt_column_stat.cpp +++ b/src/share/stat/ob_opt_column_stat.cpp @@ -177,7 +177,7 @@ int ObHistogram::assign(const ObHistogram &other) } ObOptColumnStat::ObOptColumnStat() - : table_id_(0), + : table_id_(OB_INVALID_ID), partition_id_(0), column_id_(0), object_type_(StatLevel::INVALID_LEVEL), @@ -205,7 +205,7 @@ ObOptColumnStat::ObOptColumnStat() } ObOptColumnStat::ObOptColumnStat(ObIAllocator &allocator) - : table_id_(0), + : table_id_(OB_INVALID_ID), partition_id_(0), column_id_(0), object_type_(StatLevel::INVALID_LEVEL), @@ -293,44 +293,63 @@ int ObOptColumnStat::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue int ObOptColumnStat::deep_copy(const ObOptColumnStat &src) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!src.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments.", K(src)); - } else { - table_id_ = src.table_id_; - partition_id_ = src.partition_id_; - column_id_ = src.column_id_; - object_type_ = src.object_type_; - version_ = src.version_; - num_null_ = src.num_null_; - num_not_null_ = src.num_not_null_; - num_distinct_ = src.num_distinct_; - avg_length_ = src.avg_length_; - last_analyzed_ = src.last_analyzed_; - cs_type_ = src.cs_type_; - llc_bitmap_size_ = src.llc_bitmap_size_; - total_col_len_ = src.total_col_len_; - cg_macro_blk_cnt_ = src.cg_macro_blk_cnt_; - cg_micro_blk_cnt_ = src.cg_micro_blk_cnt_; - cg_skip_rate_ = src.cg_skip_rate_; - if (OB_FAIL(ob_write_obj(allocator_, src.min_value_, min_value_))) { - LOG_WARN("deep copy min_value_ failed.", K_(src.min_value), K(ret)); - } else if (OB_FAIL(ob_write_obj(allocator_, src.max_value_, max_value_))) { - LOG_WARN("deep copy max_value_ failed.", K_(src.max_value), K(ret)); - } else if (OB_FAIL(histogram_.deep_copy(allocator_, src.histogram_))) { - LOG_WARN("failed to deep copy histogram", K(ret)); - } else if (src.llc_bitmap_size_ != 0 && src.llc_bitmap_ != NULL) { - if (OB_ISNULL(llc_bitmap_ = static_cast(allocator_.alloc(src.llc_bitmap_size_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memory for llc_bitmap_"); - } else { - MEMCPY(llc_bitmap_, src.llc_bitmap_, src.llc_bitmap_size_); - } + table_id_ = src.table_id_; + partition_id_ = src.partition_id_; + column_id_ = src.column_id_; + object_type_ = src.object_type_; + version_ = src.version_; + num_null_ = src.num_null_; + num_not_null_ = src.num_not_null_; + num_distinct_ = src.num_distinct_; + avg_length_ = src.avg_length_; + last_analyzed_ = src.last_analyzed_; + cs_type_ = src.cs_type_; + llc_bitmap_size_ = src.llc_bitmap_size_; + total_col_len_ = src.total_col_len_; + cg_macro_blk_cnt_ = src.cg_macro_blk_cnt_; + cg_micro_blk_cnt_ = src.cg_micro_blk_cnt_; + cg_skip_rate_ = src.cg_skip_rate_; + if (OB_FAIL(ob_write_obj(allocator_, src.min_value_, min_value_))) { + LOG_WARN("deep copy min_value_ failed.", K_(src.min_value), K(ret)); + } else if (OB_FAIL(ob_write_obj(allocator_, src.max_value_, max_value_))) { + LOG_WARN("deep copy max_value_ failed.", K_(src.max_value), K(ret)); + } else if (OB_FAIL(histogram_.deep_copy(allocator_, src.histogram_))) { + LOG_WARN("failed to deep copy histogram", K(ret)); + } else if (src.llc_bitmap_size_ != 0 && src.llc_bitmap_ != NULL) { + if (OB_ISNULL(llc_bitmap_ = static_cast(allocator_.alloc(src.llc_bitmap_size_)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for llc_bitmap_"); + } else { + MEMCPY(llc_bitmap_, src.llc_bitmap_, src.llc_bitmap_size_); } } return ret; } +int ObOptColumnStat::deep_copy_llc_bitmap(const char *bitmap, const int64_t size) +{ + int ret = OB_SUCCESS; + if (size != 0 && bitmap != NULL) { + if (OB_ISNULL(llc_bitmap_ = static_cast(allocator_.alloc(size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for llc_bitmap_"); + } else { + MEMCPY(llc_bitmap_, bitmap, size); + llc_bitmap_size_ = size; + } + } + return ret; +} + +int ObOptColumnStat::deep_copy_histogram(const ObHistogram &hist) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(histogram_.deep_copy(allocator_, hist))) { + LOG_WARN("failed to deep copy histogram", K(ret)); + } + return ret; +} + int ObOptColumnStat::deep_copy(const ObOptColumnStat &src, char *buf, const int64_t size, int64_t &pos) { int ret = OB_SUCCESS; @@ -371,6 +390,32 @@ int ObOptColumnStat::deep_copy(const ObOptColumnStat &src, char *buf, const int6 return ret; } +int ObOptColumnStat::assign(const ObOptColumnStat &other) +{ + int ret = OB_SUCCESS; + table_id_ = other.table_id_; + partition_id_ = other.partition_id_; + column_id_ = other.column_id_; + object_type_ = other.object_type_; + version_ = other.version_; + num_null_ = other.num_null_; + num_not_null_ = other.num_not_null_; + num_distinct_ = other.num_distinct_; + avg_length_ = other.avg_length_; + min_value_ = other.min_value_; + max_value_ = other.max_value_; + llc_bitmap_size_ = other.llc_bitmap_size_; + llc_bitmap_ = other.llc_bitmap_; + if (OB_FAIL(histogram_.assign(other.histogram_))) { + LOG_WARN("failed to assign", K(ret)); + } else { + last_analyzed_ = other.last_analyzed_; + cs_type_ = other.cs_type_; + total_col_len_ = other.total_col_len_; + } + return ret; +} + /** * @brief ObHistogram::calc_density * basically, we assume non-popular values are uniformly distributed diff --git a/src/share/stat/ob_opt_column_stat.h b/src/share/stat/ob_opt_column_stat.h index 0d724fbcf1..f2331ed750 100644 --- a/src/share/stat/ob_opt_column_stat.h +++ b/src/share/stat/ob_opt_column_stat.h @@ -285,6 +285,10 @@ public: virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const override; int deep_copy(const ObOptColumnStat &src, char *buf, const int64_t size, int64_t &pos); int deep_copy(const ObOptColumnStat &value); + int deep_copy_llc_bitmap(const char *bitmap, const int64_t size); + int deep_copy_histogram(const ObHistogram &hist); + + int assign(const ObOptColumnStat &other); int64_t get_last_analyzed() const { return last_analyzed_; } void set_last_analyzed(int64_t last) { last_analyzed_ = last; } @@ -300,6 +304,8 @@ public: void set_llc_bitmap(char *bitmap, const int64_t size) { llc_bitmap_ = bitmap; llc_bitmap_size_ = size; } + void set_version(const int64_t version) { version_ = version; } + int64_t get_version() const { return version_; } bool is_valid() const { return common::OB_INVALID_ID != table_id_ @@ -310,6 +316,7 @@ public: } void add_col_len(int64_t len) { total_col_len_ += len; } + void set_total_col_len(int64_t len) { total_col_len_ = len; } int64_t get_total_col_len() const { return total_col_len_; } int merge_column_stat(const ObOptColumnStat &other); diff --git a/src/share/stat/ob_opt_osg_column_stat.cpp b/src/share/stat/ob_opt_osg_column_stat.cpp index 4d5971c43a..2aee9988ab 100644 --- a/src/share/stat/ob_opt_osg_column_stat.cpp +++ b/src/share/stat/ob_opt_osg_column_stat.cpp @@ -117,9 +117,9 @@ int ObOptOSGColumnStat::set_min_max_datum_to_obj() LOG_WARN("failed to get min obj"); } else if (OB_FAIL(max_val_.get_obj(*max_obj))) { LOG_WARN("failed to get max obj"); - } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(*min_obj, allocator_))) { + } else if (OB_FAIL(ObDbmsStatsUtils::truncate_string_for_opt_stats(*min_obj, allocator_))) { LOG_WARN("fail to truncate string", K(ret)); - } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(*max_obj, allocator_))) { + } else if (OB_FAIL(ObDbmsStatsUtils::truncate_string_for_opt_stats(*max_obj, allocator_))) { LOG_WARN("fail to truncate string", K(ret)); } else { const ObObj &min_val = col_stat_->get_min_value(); diff --git a/src/share/stat/ob_opt_stat_gather_stat.cpp b/src/share/stat/ob_opt_stat_gather_stat.cpp index f1ce5c542c..641031aa43 100644 --- a/src/share/stat/ob_opt_stat_gather_stat.cpp +++ b/src/share/stat/ob_opt_stat_gather_stat.cpp @@ -13,14 +13,14 @@ #define USING_LOG_PREFIX SQL_OPT #include "share/stat/ob_opt_stat_gather_stat.h" #include "lib/string/ob_sql_string.h" +#include "sql/session/ob_sql_session_info.h" +#include "sql/session/ob_sql_session_mgr.h" using namespace oceanbase::common; using namespace oceanbase::sql; int ObOptStatTaskInfo::init(common::ObIAllocator &allocator, - uint64_t tenant_id, - uint64_t session_id, - const common::ObCurTraceId::TraceId &trace_id, + sql::ObSQLSessionInfo *session, ObString &task_id, ObOptStatGatherType type, uint64_t task_start_time, @@ -29,18 +29,22 @@ int ObOptStatTaskInfo::init(common::ObIAllocator &allocator, int ret = OB_SUCCESS; char *trace_id_buf = NULL; const int32_t max_trace_id_len = 64; - if (OB_ISNULL(trace_id_buf = static_cast(allocator.alloc(max_trace_id_len)))) { + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(session)); + } else if (OB_ISNULL(trace_id_buf = static_cast(allocator.alloc(max_trace_id_len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc memory failed", K(ret), K(trace_id_buf)); } else { - tenant_id_ = tenant_id; - session_id_ = session_id; - int64_t len = trace_id.to_string(trace_id_buf, max_trace_id_len); + tenant_id_ = session->get_effective_tenant_id(); + session_id_ = session->get_sessid(); + int64_t len = session->get_current_trace_id().to_string(trace_id_buf, max_trace_id_len); trace_id_.assign_ptr(trace_id_buf, static_cast(len)); task_id_ = task_id; type_ = type; task_start_time_ = task_start_time; task_table_count_ = task_table_cnt; + session_ = session; } return ret; } @@ -81,7 +85,8 @@ ObOptStatGatherStat::ObOptStatGatherStat() : end_time_(0), memory_used_(0), stat_refresh_failed_list_(), - properties_() + properties_(), + table_gather_progress_() { } @@ -95,7 +100,8 @@ ObOptStatGatherStat::ObOptStatGatherStat(ObOptStatTaskInfo &task_info) : end_time_(0), memory_used_(0), stat_refresh_failed_list_(), - properties_() + properties_(), + table_gather_progress_() { } @@ -116,6 +122,7 @@ int ObOptStatGatherStat::assign(const ObOptStatGatherStat &other) memory_used_ = other.memory_used_; stat_refresh_failed_list_ = other.stat_refresh_failed_list_; properties_ = other.properties_; + table_gather_progress_ = other.table_gather_progress_; return ret; } @@ -127,6 +134,7 @@ int64_t ObOptStatGatherStat::size() const base_size += table_name_.length(); base_size += stat_refresh_failed_list_.length(); base_size += properties_.length(); + base_size += table_gather_progress_.length(); return base_size; } @@ -163,14 +171,18 @@ int ObOptStatGatherStat::deep_copy(common::ObIAllocator &allocator, ObOptStatGat new_stat->set_end_time(end_time_); //set memory_used_ new_stat->set_memory_used(memory_used_); - ////set stat_refresh_failed_list_ + //set stat_refresh_failed_list_ MEMCPY(buf + pos, stat_refresh_failed_list_.ptr(), stat_refresh_failed_list_.length()); new_stat->set_stat_refresh_failed_list(buf + pos, stat_refresh_failed_list_.length()); pos += stat_refresh_failed_list_.length(); - ////set properties_ + //set properties_ MEMCPY(buf + pos, properties_.ptr(), properties_.length()); new_stat->set_properties(buf + pos, properties_.length()); pos += properties_.length(); + //set table_gather_progress_ + MEMCPY(buf + pos, table_gather_progress_.ptr(), table_gather_progress_.length()); + new_stat->set_table_gather_progress(buf + pos, table_gather_progress_.length()); + pos += table_gather_progress_.length(); if (OB_UNLIKELY(pos != buf_len)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(pos), K(buf_len)); @@ -222,43 +234,40 @@ int ObOptStatRunningMonitor::add_table_info(common::ObTableStatParam &table_para return ret; } -// int ObOptStatRunningMonitor::add_monitor_info(ObOptStatRunningPhase current_phase, -// int64_t current_memory_used) -// { -// int ret = OB_SUCCESS; -// if (current_phase > ObOptStatRunningPhase::GATHER_BEGIN && -// current_phase <= ObOptStatRunningPhase::GATHER_END) { -// int64_t current_time = ObTimeUtility::current_time(); -// ObSqlString tmp_str; -// if (current_phase != ObOptStatRunningPhase::GATHER_END && -// OB_FAIL(tmp_str.append_fmt("%s:cost_time=%ldus,cost_memory=%ldbyte;", running_phase_name[current_phase], -// current_time - last_start_time_, -// current_memory_used - last_memory_used_))) { -// LOG_WARN("failed to append fmt", K(ret)); -// } else if (current_phase == ObOptStatRunningPhase::GATHER_END && -// OB_FAIL(tmp_str.append_fmt("%s;", running_phase_name[current_phase]))) { -// LOG_WARN("failed to append fmt", K(ret)); -// } else { -// char *buf = NULL; -// ObString &running_detailed = opt_stat_gather_stat_.get_running_detailed(); -// opt_stat_gather_stat_.add_memory_used(current_memory_used - last_memory_used_); -// opt_stat_gather_stat_.add_duration_time(current_time - last_start_time_); -// int32_t buf_len = static_cast(tmp_str.length() + running_detailed.length()); -// if (OB_ISNULL(buf = static_cast(allocator_.alloc(tmp_str.length() + running_detailed.length())))) { -// ret = OB_ALLOCATE_MEMORY_FAILED; -// LOG_WARN("memory is not enough", K(ret), K(buf)); -// } else { -// MEMCPY(buf, running_detailed.ptr(), running_detailed.length()); -// MEMCPY(buf + running_detailed.length() , tmp_str.ptr(), tmp_str.length()); -// running_detailed.assign_ptr(buf, buf_len); -// running_phase_ = current_phase; -// last_start_time_ = current_time; -// last_memory_used_ = current_memory_used; -// } -// } -// } -// return ret; -// } +int ObOptStatRunningMonitor::add_monitor_info(ObOptStatRunningPhase current_phase, + double extra_progress_ratio/*default 0*/) +{ + int ret = OB_SUCCESS; + ObSqlString tmp_str; + if (OB_FAIL(tmp_str.append_fmt("%.2f%%(%s)", running_progress_ratio[current_phase] + extra_progress_ratio, + running_phase_name[current_phase]))) { + LOG_WARN("failed to append fmt", K(ret)); + } else { + char *buf = NULL; + if (OB_ISNULL(buf = static_cast(allocator_.alloc(tmp_str.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory is not enough", K(ret), K(buf)); + } else { + ObString tmp_tab_gather_progress; + MEMCPY(buf , tmp_str.ptr(), tmp_str.length()); + tmp_tab_gather_progress.assign_ptr(buf, tmp_str.length()); + ObOptStatGatherStatList::instance().update_table_gather_progress(tmp_tab_gather_progress, + opt_stat_gather_stat_); + } + } + return ret; +} + +double ObOptStatRunningMonitor::get_monitor_extra_progress_ratio(ObOptStatRunningPhase current_phase, + ObOptStatRunningPhase next_phase, + int64_t total_split_cnt) +{ + double res = 0.0; + if (total_split_cnt > 0) { + res = (running_progress_ratio[next_phase] - running_progress_ratio[current_phase]) * 1.0 / total_split_cnt; + } + return res > 0.0 ? res : 0.0; +} void ObOptStatRunningMonitor::set_monitor_result(int ret_code, int64_t end_time, @@ -317,6 +326,13 @@ void ObOptStatGatherStatList::update_gather_stat_info(ObString &db_name, stat_value.set_properties(properties.ptr(), properties.length()); } +void ObOptStatGatherStatList::update_table_gather_progress(ObString &table_gather_progress, + ObOptStatGatherStat &stat_value) +{ + ObSpinLockGuard guard(lock_); + stat_value.set_table_gather_progress(table_gather_progress.ptr(), table_gather_progress.length()); +} + void ObOptStatGatherStatList::update_gather_stat_refresh_failed_list(ObString &failed_list, ObOptStatGatherStat &stat_value) { @@ -349,3 +365,31 @@ int ObOptStatGatherStatList::list_to_array(common::ObIAllocator &allocator, } return ret; } + +int ObOptStatGatherStatList::cancel_gather_stats(const uint64_t tenant_id, + const ObString &task_id) +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(lock_); + bool is_cancel = false; + DLIST_FOREACH(cur, stat_list_) { + // sys tenant list all tenant stat + // non-sys tennat list self tenant stat + ObOptStatGatherStat *tmp_stat = NULL; + if (!is_sys_tenant(tenant_id) && cur->get_tenant_id() != tenant_id) { + //do nothing + } else if (0 != cur->get_task_id().case_compare(task_id) || OB_ISNULL(cur->get_session())) { + //do nothing + } else if (OB_FAIL(sql::ObSQLSessionMgr::kill_query(*cur->get_session(), ObSQLSessionState::QUERY_KILLED))) { + LOG_WARN("kill query failed", K(ret)); + } else { + is_cancel = true; + } + } + if (OB_SUCC(ret) && !is_cancel) { + ret = OB_ERR_DBMS_STATS_PL; + LOG_WARN("The optimizer stats gather task has ended or the task doesn't exist", K(ret)); + LOG_USER_ERROR(OB_ERR_DBMS_STATS_PL, "The optimizer stats gather task has ended or the task doesn't exist"); + } + return ret; +} \ No newline at end of file diff --git a/src/share/stat/ob_opt_stat_gather_stat.h b/src/share/stat/ob_opt_stat_gather_stat.h index b1b0adaf9a..3bfe8ef697 100644 --- a/src/share/stat/ob_opt_stat_gather_stat.h +++ b/src/share/stat/ob_opt_stat_gather_stat.h @@ -29,30 +29,43 @@ enum ObOptStatGatherType { AUTO_GATHER }; -// enum ObOptStatRunningPhase { -// GATHER_BEGIN = 0, -// GATHER_SUBPART_STATS, -// GATHER_PART_STATS, -// DERIVE_PART_STATS, -// GATHER_GLOBAL_STATS, -// DERIVE_GLOBAL_STATS, -// WRITE_STATS, -// GATHER_INDEX_STATS, -// GATHER_END -// }; +enum ObOptStatRunningPhase { + GATHER_PREPARE, + GATHER_SUBPART_STATS, + GATHER_PART_STATS, + APPROX_GATHER_PART_STATS, + GATHER_GLOBAL_STATS, + APPROX_GATHER_GLOBAL_STATS, + BACKUP_HISTORY_STATS, + GATHER_INDEX_STATS, + GATHER_END +}; -// static const char *running_phase_name[ObOptStatRunningPhase::GATHER_END + 1] = -// { -// "GATHER BEGIN", -// "GATHER SUBPART STATS", -// "GATHER PART STATS", -// "DERIVE PART STATS", -// "GATHER GLOBAL STATS", -// "DERIVE GLOBAL STATS", -// "WRITE STATS", -// "GATHER INDEX STATS", -// "GATHER END" -// }; +static const char *running_phase_name[ObOptStatRunningPhase::GATHER_END + 1] = +{ + "GATHER PREPARE", + "GATHER SUBPART STATS", + "GATHER PART STATS", + "APPROX GATHER PART STATS", + "GATHER GLOBAL STATS", + "APPROX GATHER GLOBAL STATS", + "BACKUP HISTORY STATS", + "GATHER INDEX STATS", + "GATHER END" +}; + +static const int64_t running_progress_ratio[ObOptStatRunningPhase::GATHER_END + 1] = +{ + 0,//GATHER_PREPARE + 5,//GATHER_SUBPART_STATS + 45,//GATHER_PART_STATS + 65,//APPROX_GATHER_PART_STATS + 70,//GATHER_GLOBAL_STATS + 80,//APPROX_GATHER_GLOBAL_STATS + 90,//BACKUP_HISTORY_STATS + 95,//GATHER_INDEX_STATS + 100//GATHER_END +}; struct ObOptStatTaskInfo { @@ -67,13 +80,12 @@ struct ObOptStatTaskInfo task_end_time_(0), ret_code_(0), failed_count_(0), - completed_table_count_(0) + completed_table_count_(0), + session_(NULL) {} int64_t size() const { return trace_id_.length() + task_id_.length(); } int init(common::ObIAllocator &allocator, - uint64_t tenant_id, - uint64_t session_id, - const common::ObCurTraceId::TraceId &trace_id, + sql::ObSQLSessionInfo *session, ObString &task_id, ObOptStatGatherType type, uint64_t task_start_time, @@ -101,6 +113,7 @@ struct ObOptStatTaskInfo int ret_code_; int64_t failed_count_; int64_t completed_table_count_; + sql::ObSQLSessionInfo *session_;//no deep copy }; class ObOptStatGatherStat : public common::ObDLinkBase @@ -154,6 +167,9 @@ public: inline void set_properties(const char *ptr, int32_t len) { properties_.assign_ptr(ptr, len); } inline int64_t get_running_table_duration_time() const { return ObTimeUtility::current_time() - start_time_; } inline int64_t get_completed_table_count() { return task_info_.completed_table_count_; } + inline const ObString &get_table_gather_progress() const { return table_gather_progress_; } + inline void set_table_gather_progress(const char *ptr, int32_t len) { table_gather_progress_.assign_ptr(ptr, len); } + sql::ObSQLSessionInfo *get_session() { return task_info_.session_; } TO_STRING_KV(K(task_info_), K(database_name_), K(table_id_), @@ -163,7 +179,8 @@ public: K(end_time_), K(memory_used_), K(stat_refresh_failed_list_), - K(properties_)); + K(properties_), + K(table_gather_progress_)); private: ObOptStatTaskInfo task_info_; @@ -176,6 +193,7 @@ private: int64_t memory_used_; ObString stat_refresh_failed_list_; ObString properties_; + ObString table_gather_progress_; }; struct ObOptStatRunningMonitor @@ -197,6 +215,10 @@ struct ObOptStatRunningMonitor ObOptStatGatherStat &opt_stat_gather_stat); int add_table_info(common::ObTableStatParam &table_param, double stale_percent = -1.0); + int add_monitor_info(ObOptStatRunningPhase current_phase, double extra_progress_ratio = 0); + double get_monitor_extra_progress_ratio(ObOptStatRunningPhase current_phase, + ObOptStatRunningPhase next_phase, + int64_t total_split_cnt); void set_monitor_result(int ret_code, int64_t current_time, int64_t current_memory_used); @@ -216,9 +238,12 @@ public: int push(ObOptStatGatherStat &stat_value); int remove(ObOptStatGatherStat &stat_value); void update_gather_stat_info(ObString &db_name, - ObString &tab_name, - ObString &properties, - ObOptStatGatherStat &stat_value); + ObString &tab_name, + ObString &properties, + ObOptStatGatherStat &stat_value); + void update_table_gather_progress(ObString &table_gather_progress, + ObOptStatGatherStat &stat_value); + int cancel_gather_stats(const uint64_t tenant_id, const ObString &task_id); void update_gather_stat_refresh_failed_list(ObString &failed_list, ObOptStatGatherStat &stat_value); // param[in] tenant_id if tenant is sys, list all tenant stat, else list target tenant stat diff --git a/src/share/stat/ob_opt_stat_manager.cpp b/src/share/stat/ob_opt_stat_manager.cpp index a8ddb57ef1..8b08b6b741 100644 --- a/src/share/stat/ob_opt_stat_manager.cpp +++ b/src/share/stat/ob_opt_stat_manager.cpp @@ -130,9 +130,10 @@ int ObOptStatManager::get_column_stat(const uint64_t tenant_id, ret = OB_NOT_INIT; LOG_WARN("stat manager has not been initialized.", K(ret)); } else { - ObArenaAllocator arena(ObModIds::OB_SQL_PARSER); - ObSEArray keys; + ObArenaAllocator arena("ObGetColStat", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); for (int64_t i = 0; OB_SUCC(ret) && i < part_ids.count(); ++i) { + ObSEArray tmp_handles; + ObSEArray keys; for (int64_t j = 0; OB_SUCC(ret) && j < column_ids.count(); ++j) { void *ptr = NULL; if (OB_ISNULL(ptr = arena.alloc(sizeof(ObOptColumnStat::Key)))) { @@ -148,12 +149,14 @@ int ObOptStatManager::get_column_stat(const uint64_t tenant_id, } else {/*do nothing*/} } } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(stat_service_.get_column_stat(tenant_id, keys, handles))) { - LOG_WARN("get column stat failed.", K(ret)); - } else { - LOG_TRACE("succeed to get get column stat", K(keys), K(handles)); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(stat_service_.get_column_stat(tenant_id, keys, tmp_handles))) { + LOG_WARN("get column stat failed.", K(ret)); + } else if (OB_FAIL(append(handles, tmp_handles))) { + LOG_WARN("failed to append", K(ret)); + } else { + arena.reuse(); + } } } return ret; @@ -243,25 +246,24 @@ int ObOptStatManager::get_table_stat(const uint64_t tenant_id, int ObOptStatManager::update_column_stat(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, const ObIArray &column_stats, bool only_update_col_stat /*default false*/, const ObObjPrintParams &print_params) { int ret = OB_SUCCESS; - ObArenaAllocator allocator("UpdateColStat", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); int64_t current_time = ObTimeUtility::current_time(); + ObArenaAllocator allocator("UpdateColStat", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("optimizer statistics manager has not been initialized.", K(ret)); } else if (OB_FAIL(stat_service_.get_sql_service().update_column_stat(schema_guard, tenant_id, allocator, - trans, + conn, column_stats, current_time, only_update_col_stat, - false, print_params))) { LOG_WARN("failed to update column stat.", K(ret)); } else { /*do nothing*/ } @@ -269,6 +271,7 @@ int ObOptStatManager::update_column_stat(share::schema::ObSchemaGetterGuard *sch } int ObOptStatManager::update_table_stat(const uint64_t tenant_id, + sqlclient::ObISQLConnection *conn, const ObOptTableStat *table_stats, const bool is_index_stat) { @@ -277,6 +280,7 @@ int ObOptStatManager::update_table_stat(const uint64_t tenant_id, ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); } else if (OB_FAIL(stat_service_.get_sql_service().update_table_stat(tenant_id, + conn, table_stats, is_index_stat))) { LOG_WARN("failed to update table stats", K(ret)); @@ -285,7 +289,7 @@ int ObOptStatManager::update_table_stat(const uint64_t tenant_id, } int ObOptStatManager::update_table_stat(const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, const ObIArray &table_stats, const bool is_index_stat) { @@ -295,7 +299,7 @@ int ObOptStatManager::update_table_stat(const uint64_t tenant_id, ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); } else if (OB_FAIL(stat_service_.get_sql_service().update_table_stat(tenant_id, - trans, + conn, table_stats, current_time, is_index_stat))) { @@ -311,13 +315,15 @@ int ObOptStatManager::delete_table_stat(uint64_t tenant_id, int ret = OB_SUCCESS; ObSEArray part_ids; bool cascade_column = true; - return delete_table_stat(tenant_id, ref_id, part_ids, cascade_column, affected_rows); + int64_t degree = 1; + return delete_table_stat(tenant_id, ref_id, part_ids, cascade_column, degree, affected_rows); } int ObOptStatManager::delete_table_stat(uint64_t tenant_id, const uint64_t ref_id, const ObIArray &part_ids, const bool cascade_column, + const int64_t degree, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -328,6 +334,7 @@ int ObOptStatManager::delete_table_stat(uint64_t tenant_id, ref_id, part_ids, cascade_column, + degree, affected_rows))) { LOG_WARN("failed to delete table stat", K(ret)); } @@ -338,14 +345,15 @@ int ObOptStatManager::delete_column_stat(const uint64_t tenant_id, const uint64_t ref_id, const ObIArray &column_ids, const ObIArray &part_ids, - const bool only_histogram) + const bool only_histogram, + const int64_t degree) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret)); } else if (OB_FAIL(stat_service_.get_sql_service().delete_column_stat( - tenant_id, ref_id, column_ids, part_ids, only_histogram))) { + tenant_id, ref_id, column_ids, part_ids, only_histogram, degree))) { LOG_WARN("failed to delete column stat", K(ret)); } return ret; @@ -381,12 +389,11 @@ int ObOptStatManager::erase_table_stat(const ObOptTableStat::Key &key) int ObOptStatManager::batch_write(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, ObIArray &table_stats, ObIArray &column_stats, const int64_t current_time, const bool is_index_stat, - const bool is_history_stat, const ObObjPrintParams &print_params) { int ret = OB_SUCCESS; @@ -397,21 +404,19 @@ int ObOptStatManager::batch_write(share::schema::ObSchemaGetterGuard *schema_gua } else if (!table_stats.empty() && OB_FAIL(stat_service_.get_sql_service().update_table_stat( tenant_id, - trans, + conn, table_stats, current_time, - is_index_stat, - is_history_stat))) { + is_index_stat))) { LOG_WARN("failed to update table stats", K(ret)); } else if (!column_stats.empty() && OB_FAIL(stat_service_.get_sql_service().update_column_stat(schema_guard, tenant_id, allocator, - trans, + conn, column_stats, current_time, false, - is_history_stat, print_params))) { LOG_WARN("failed to update coumn stats", K(ret)); } diff --git a/src/share/stat/ob_opt_stat_manager.h b/src/share/stat/ob_opt_stat_manager.h index 2ae63d0d97..1f95786feb 100644 --- a/src/share/stat/ob_opt_stat_manager.h +++ b/src/share/stat/ob_opt_stat_manager.h @@ -62,11 +62,12 @@ public: bool &is_opt_stat_valid); int update_table_stat(const uint64_t tenant_id, + sqlclient::ObISQLConnection *conn, const ObOptTableStat *table_stats, const bool is_index_stat); int update_table_stat(const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, const ObIArray &table_stats, const bool is_index_stat); @@ -125,7 +126,7 @@ public: ObOptColumnStatHandle &handle); virtual int update_column_stat(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, const common::ObIArray &column_stats, bool only_update_col_stat = false, const ObObjPrintParams &print_params = ObObjPrintParams()); @@ -138,13 +139,15 @@ public: const uint64_t ref_id, const ObIArray &part_ids, const bool cascade_column, + const int64_t degree, int64_t &affected_rows); int delete_column_stat(const uint64_t tenant_id, const uint64_t ref_id, const ObIArray &column_ids, const ObIArray &part_ids, - const bool only_histogram = false); + const bool only_histogram = false, + const int64_t degree = 1); int erase_column_stat(const ObOptColumnStat::Key &key); int erase_table_stat(const ObOptTableStat::Key &key); @@ -160,12 +163,11 @@ public: int batch_write(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, ObIArray &table_stats, ObIArray &column_stats, const int64_t current_time, const bool is_index_stat, - const bool is_history_stat, const ObObjPrintParams &print_params); /** @brief 外部获取行统计信息的接口 */ @@ -206,6 +208,8 @@ public: ObOptDSStatHandle &ds_stat_handle); int update_opt_stat_gather_stat(const ObOptStatGatherStat &gather_stat); int update_opt_stat_task_stat(const ObOptStatTaskInfo &task_info); + ObOptStatService &get_stat_service() { return stat_service_; } + int get_system_stat(const uint64_t tenant_id, ObOptSystemStat &stat); int update_system_stats(const uint64_t tenant_id, diff --git a/src/share/stat/ob_opt_stat_monitor_manager.cpp b/src/share/stat/ob_opt_stat_monitor_manager.cpp index 4e81766c83..ead2f44574 100644 --- a/src/share/stat/ob_opt_stat_monitor_manager.cpp +++ b/src/share/stat/ob_opt_stat_monitor_manager.cpp @@ -175,12 +175,13 @@ int ObOptStatMonitorManager::flush_database_monitoring_info(sql::ObExecContext & } else { ObSEArray failed_server_arr; for (int64_t i = 0; OB_SUCC(ret) && i < all_server_arr.count(); i++) { + timeout = std::min(MAX_OPT_STATS_PROCESS_RPC_TIMEOUT, THIS_WORKER.get_timeout_remain()); if (!all_server_arr.at(i).is_active() || ObServerStatus::OB_SERVER_ACTIVE != all_server_arr.at(i).get_server_status() || 0 == all_server_arr.at(i).get_start_service_time() || 0 != all_server_arr.at(i).get_server_stop_time()) { //server may not serving - } else if (0 >= (timeout = THIS_WORKER.get_timeout_remain())) { + } else if (0 >= timeout) { ret = OB_TIMEOUT; LOG_WARN("query timeout is reached", K(ret), K(timeout)); } else if (OB_FAIL(GCTX.srv_rpc_proxy_->to(all_server_arr.at(i).get_addr()) @@ -190,8 +191,11 @@ int ObOptStatMonitorManager::flush_database_monitoring_info(sql::ObExecContext & LOG_WARN("failed to flush opt stat monitoring info caused by unknow error", K(ret), K(all_server_arr.at(i).get_addr()), K(arg)); //ignore flush cache failed, TODO @jiangxiu.wt can aduit it and flush cache manually later. - if (ignore_failed && OB_FAIL(failed_server_arr.push_back(all_server_arr.at(i)))) { - LOG_WARN("failed to push back", K(ret)); + if (ignore_failed) { + LOG_USER_WARN(OB_ERR_DBMS_STATS_PL, "failed to flush opt stat monitoring info"); + if (OB_FAIL(failed_server_arr.push_back(all_server_arr.at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } } } } @@ -640,8 +644,8 @@ int ObOptStatMonitorManager::clean_useless_dml_stat_info() share::OB_ALL_MONITOR_MODIFIED_TNAME, all_table_name, share::OB_ALL_DATABASE_TNAME, OB_MAX_INNER_TABLE_ID))) { LOG_WARN("failed to append fmt", K(ret)); - } else if (OB_FAIL(delete_part_sql.append_fmt("DELETE /*+use_nl(m1)*/FROM %s m1 WHERE (tenant_id, table_id, tablet_id) IN ( "\ - "SELECT /*+leading(db, t, m, view1, view2) use_hash(m) use_hash(view1) use_hash(view2)*/ "\ + } else if (OB_FAIL(delete_part_sql.append_fmt("DELETE /*+leading(view3, m1) use_nl(view3, m1)*/FROM %s m1 WHERE (tenant_id, table_id, tablet_id) IN ( "\ + "SELECT /*+leading(m, view1, view2, t, db) use_hash(m,view1) use_hash((m,view1),view2) use_nl((m,view1,view2),t), use_nl((m,view1,view2,t),db)*/ "\ "m.tenant_id, m.table_id, m.tablet_id FROM %s m, %s t, %s db WHERE t.table_id = m.table_id AND t.tenant_id = m.tenant_id AND t.part_level > 0 "\ "AND t.tenant_id = db.tenant_id AND t.database_id = db.database_id AND db.database_name != '__recyclebin' "\ "AND NOT EXISTS (SELECT 1 FROM %s p WHERE p.table_id = m.table_id AND p.tenant_id = m.tenant_id AND p.tablet_id = m.tablet_id) "\ diff --git a/src/share/stat/ob_opt_stat_service.cpp b/src/share/stat/ob_opt_stat_service.cpp index 054ce934d5..783b6d66c9 100644 --- a/src/share/stat/ob_opt_stat_service.cpp +++ b/src/share/stat/ob_opt_stat_service.cpp @@ -189,7 +189,7 @@ int ObOptStatService::load_column_stat_and_put_cache(const uint64_t tenant_id, ObIArray &handles) { int ret = OB_SUCCESS; - ObArenaAllocator arena(ObModIds::OB_SQL_PARSER); + ObArenaAllocator arena("ObOptColStatGet", OB_MALLOC_NORMAL_BLOCK_SIZE, tenant_id); LOG_TRACE("begin load column stat and put cache", K(keys)); ObSEArray key_column_stats; // generate new entrys and load from global statistics table and store it in cache. diff --git a/src/share/stat/ob_opt_stat_sql_service.cpp b/src/share/stat/ob_opt_stat_sql_service.cpp index ee7bff854c..f35348d4cf 100644 --- a/src/share/stat/ob_opt_stat_sql_service.cpp +++ b/src/share/stat/ob_opt_stat_sql_service.cpp @@ -99,7 +99,8 @@ "bucket_cnt," \ "histogram_type," \ "global_stats," \ - "user_stats%s) VALUES "\ + "user_stats,"\ + "spare1%s) VALUES " #define INSERT_HISTOGRAM_STAT_SQL "INSERT INTO __all_histogram_stat(tenant_id," \ @@ -113,29 +114,10 @@ "b_endpoint_value," \ "endpoint_repeat_cnt) VALUES " -#define DELETE_HISTOGRAM_STAT_SQL "DELETE /*+opt_param('enable_in_range_optimization','true')*/ FROM __all_histogram_stat WHERE " -#define DELETE_COL_STAT_SQL "DELETE FROM __all_column_stat WHERE " -#define DELETE_TAB_STAT_SQL "DELETE FROM __all_table_stat WHERE " -#define UPDATE_HISTOGRAM_TYPE_SQL "UPDATE __all_column_stat SET histogram_type = 0, bucket_cnt = 0 WHERE" - -#define INSERT_TAB_STAT_HISTORY_SQL "INSERT INTO %s(tenant_id, table_id,\ - partition_id, savtime, index_type, object_type, flags,\ - last_analyzed, sstable_row_cnt, sstable_avg_row_len,\ - macro_blk_cnt, micro_blk_cnt, memtable_row_cnt,\ - memtable_avg_row_len, row_cnt, avg_row_len, stattype_locked)\ - VALUES" - -#define INSERT_COL_STAT_HISTORY_SQL "INSERT INTO %s(tenant_id, table_id,\ - partition_id, column_id, savtime, object_type, flags, \ - last_analyzed, distinct_cnt, null_cnt, max_value, b_max_value,\ - min_value, b_min_value, avg_len, distinct_cnt_synopsis,\ - distinct_cnt_synopsis_size, sample_size, density, bucket_cnt,\ - histogram_type%s) VALUES" - -#define INSERT_HISTOGRAM_STAT_HISTORY_SQL "INSERT INTO %s(tenant_id,\ - table_id, partition_id, column_id, endpoint_num,savtime,\ - object_type, endpoint_normalized_value, endpoint_value,\ - b_endpoint_value, endpoint_repeat_cnt) VALUES " +#define DELETE_HISTOGRAM_STAT_SQL "DELETE /*+%.*s*/ FROM __all_histogram_stat WHERE %.*s" +#define DELETE_COL_STAT_SQL "DELETE /*+%.*s*/ FROM __all_column_stat WHERE %.*s" +#define DELETE_TAB_STAT_SQL "DELETE /*+%.*s*/ FROM __all_table_stat WHERE %.*s" +#define UPDATE_HISTOGRAM_TYPE_SQL "UPDATE /*+%.*s*/ __all_column_stat SET histogram_type = 0, bucket_cnt = 0 WHERE %.*s" // not used yet. #define INSERT_ONLINE_TABLE_STAT_SQL "INSERT INTO oceanbase.__all_table_stat(tenant_id," \ @@ -258,6 +240,7 @@ "col_stat.bucket_cnt as bucket_cnt," \ "col_stat.density as density," \ "col_stat.last_analyzed as last_analyzed,"\ + "col_stat.spare1 as compress_type,"\ "hist_stat.endpoint_num as endpoint_num, " \ "hist_stat.b_endpoint_value as b_endpoint_value," \ "hist_stat.endpoint_repeat_cnt as endpoint_repeat_cnt "\ @@ -286,6 +269,7 @@ "col_stat.bucket_cnt as bucket_cnt," \ "col_stat.density as density," \ "col_stat.last_analyzed as last_analyzed,"\ + "col_stat.spare1 as compress_type,"\ "col_stat.cg_macro_blk_cnt as cg_macro_blk_cnt,"\ "col_stat.cg_micro_blk_cnt as cg_micro_blk_cnt,"\ "hist_stat.endpoint_num as endpoint_num, " \ @@ -316,8 +300,6 @@ using namespace common::sqlclient; namespace common { -const char *ObOptStatSqlService::bitmap_compress_lib_name = "zlib_1.0"; - ObOptStatSqlService::ObOptStatSqlService() : inited_(false), mysql_proxy_(nullptr), mutex_(ObLatchIds::DEFAULT_MUTEX), config_(nullptr) { @@ -399,7 +381,89 @@ int ObOptStatSqlService::fetch_table_stat(const uint64_t tenant_id, return ret; } +int ObOptStatSqlService::batch_fetch_table_stats(sqlclient::ObISQLConnection *conn, + const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &part_ids, + ObIArray &all_part_stats) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret)); + } else { + ObSQLClientRetryWeak sql_client_retry_weak(mysql_proxy_, false, OB_INVALID_TIMESTAMP, false); + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + sqlclient::ObMySQLResult *result = NULL; + ObSqlString sql; + ObSqlString part_list; + ObSqlString part_str; + uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("sql service has not been initialized.", K(ret)); + } else if (OB_FAIL(sql.append_fmt("SELECT partition_id, " + "object_type, " + "row_cnt as row_count, " + "avg_row_len as avg_row_size, " + "macro_blk_cnt as macro_block_num, " + "micro_blk_cnt as micro_block_num, " + "stattype_locked as stattype_locked," + "last_analyzed FROM %s", share::OB_ALL_TABLE_STAT_TNAME))) { + LOG_WARN("fail to append SQL stmt string.", K(sql), K(ret)); + } else if (OB_FAIL(generate_in_list(part_ids, part_list))) { + LOG_WARN("failed to generate in list", K(ret)); + } else if (!part_list.empty() && + OB_FAIL(part_str.append_fmt(" AND partition_id in %s", part_list.ptr()))) { + LOG_WARN("fail to append partition string.", K(ret)); + } else if (OB_FAIL(sql.append_fmt(" WHERE TENANT_ID = %ld AND TABLE_ID=%ld %s", + ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id), + ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), + !part_str.empty() ? part_str.ptr() : " "))) { + LOG_WARN("fail to append SQL where string.", K(ret)); + } else if (OB_FAIL(conn->execute_read(exec_tenant_id, sql.ptr(), res))) { + LOG_WARN("execute sql failed", "sql", sql.ptr(), K(ret)); + } else if (NULL == (result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to execute ", "sql", sql.ptr(), K(ret)); + } + while (OB_SUCC(ret)) { + ObOptTableStat stat; + stat.set_table_id(table_id); + if (OB_FAIL(result->next())) { + if (OB_ITER_END != ret) { + LOG_WARN("get next row failed", K(ret)); + } else { + ret = OB_SUCCESS; + break; + } + } else if (OB_FAIL(fill_table_stat(*result, stat))) { + LOG_WARN("failed to fill table stat", K(ret)); + } else { + bool found_it = false; + for (int64_t i = 0; OB_SUCC(ret) && i < all_part_stats.count(); ++i) { + if (OB_ISNULL(all_part_stats.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret)); + } else if (all_part_stats.at(i)->get_table_id() == stat.get_table_id() && + all_part_stats.at(i)->get_partition_id() == stat.get_partition_id()) { + found_it = true; + *all_part_stats.at(i) = stat; + } + } + if (OB_SUCC(ret) && !found_it) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(all_part_stats), K(stat)); + } + } + } + } + } + return ret; +} + int ObOptStatSqlService::update_table_stat(const uint64_t tenant_id, + sqlclient::ObISQLConnection *conn, const ObOptTableStat *table_stat, const bool is_index_stat) { @@ -408,53 +472,35 @@ int ObOptStatSqlService::update_table_stat(const uint64_t tenant_id, ObSqlString tmp; int64_t current_time = ObTimeUtility::current_time(); int64_t affected_rows = 0; - if (OB_ISNULL(table_stat)) { + if (OB_ISNULL(table_stat) || OB_ISNULL(conn)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("table stat is null", K(ret), K(table_stat)); + LOG_WARN("table stat or conn is null", K(ret), K(table_stat), K(conn)); } else if (OB_FAIL(table_stat_sql.append(INSERT_TABLE_STAT_SQL))) { LOG_WARN("failed to append sql", K(ret)); } else if (OB_FAIL(get_table_stat_sql(tenant_id, *table_stat, current_time, is_index_stat, tmp))) { LOG_WARN("failed to get table stat sql", K(ret)); } else if (OB_FAIL(table_stat_sql.append_fmt("(%s);", tmp.ptr()))) { LOG_WARN("failed to append table stat sql", K(ret)); - } else { - ObMySQLTransaction trans; - LOG_TRACE("sql string of table stat update", K(table_stat_sql)); - if (OB_FAIL(trans.start(mysql_proxy_, tenant_id))) { - LOG_WARN("fail to start transaction", K(ret), K(tenant_id)); - } else if (OB_FAIL(trans.write(tenant_id, table_stat_sql.ptr(), affected_rows))) { - LOG_WARN("failed to exec sql", K(ret)); - } else {/*do nothing*/} - if (OB_SUCC(ret)) { - if (OB_FAIL(trans.end(true))) { - LOG_WARN("fail to commit transaction", K(ret)); - } - } else { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(false))) { - LOG_WARN("fail to roll back transaction", K(tmp_ret)); - } - } - } + } else if (OB_FAIL(conn->execute_write(tenant_id, table_stat_sql.ptr(), affected_rows))) { + LOG_WARN("failed to exec sql", K(ret)); + } else {/*do nothing*/} return ret; } int ObOptStatSqlService::update_table_stat(const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, const common::ObIArray &table_stats, const int64_t current_time, - const bool is_index_stat, - const bool is_history_stat /*default false*/) + const bool is_index_stat) { int ret = OB_SUCCESS; ObSqlString table_stat_sql; ObSqlString tmp; int64_t affected_rows = 0; - if (!is_history_stat && OB_FAIL(table_stat_sql.append(INSERT_TABLE_STAT_SQL))) { - LOG_WARN("failed to append sql", K(ret)); - } else if (is_history_stat && - OB_FAIL(table_stat_sql.append_fmt(INSERT_TAB_STAT_HISTORY_SQL, - share::OB_ALL_TABLE_STAT_HISTORY_TNAME))) { + if (OB_ISNULL(conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("conn is is null", K(ret), K(conn)); + } else if (OB_FAIL(table_stat_sql.append(INSERT_TABLE_STAT_SQL))) { LOG_WARN("failed to append sql", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < table_stats.count(); ++i) { @@ -463,11 +509,7 @@ int ObOptStatSqlService::update_table_stat(const uint64_t tenant_id, if (OB_ISNULL(table_stats.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table stat is null", K(ret)); - } else if (!is_history_stat && - OB_FAIL(get_table_stat_sql(tenant_id, *table_stats.at(i), current_time, is_index_stat, tmp))) { - LOG_WARN("failed to get table stat sql", K(ret)); - } else if (is_history_stat && - OB_FAIL(get_table_stat_history_sql(tenant_id, *table_stats.at(i), current_time, tmp))) { + } else if (OB_FAIL(get_table_stat_sql(tenant_id, *table_stats.at(i), current_time, is_index_stat, tmp))) { LOG_WARN("failed to get table stat sql", K(ret)); } else if (OB_FAIL(table_stat_sql.append_fmt("(%s)%c",tmp.ptr(), (is_last? ';' : ',')))) { LOG_WARN("failed to append table stat sql", K(ret)); @@ -475,8 +517,8 @@ int ObOptStatSqlService::update_table_stat(const uint64_t tenant_id, } if (OB_SUCC(ret)) { LOG_TRACE("sql string of table stat update", K(table_stat_sql)); - if (OB_FAIL(trans.write(tenant_id, table_stat_sql.ptr(), affected_rows))) { - LOG_WARN("failed to exec sql", K(ret)); + if (OB_FAIL(conn->execute_write(tenant_id, table_stat_sql.ptr(), affected_rows))) { + LOG_WARN("failed to write", K(ret)); } } return ret; @@ -485,11 +527,10 @@ int ObOptStatSqlService::update_table_stat(const uint64_t tenant_id, int ObOptStatSqlService::update_column_stat(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t exec_tenant_id, ObIAllocator &allocator, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, const ObIArray &column_stats, const int64_t current_time, bool only_update_col_stat /*default false*/, - bool is_history_stat/*default false*/, const ObObjPrintParams &print_params) { int ret = OB_SUCCESS; @@ -498,9 +539,9 @@ int ObOptStatSqlService::update_column_stat(share::schema::ObSchemaGetterGuard * ObSqlString delete_histogram; ObSqlString column_stats_sql; bool need_histogram = false; - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("sql service not inited", K(ret)); + if (!inited_ || OB_ISNULL(conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(conn), K(inited_)); } else if (OB_UNLIKELY(column_stats.empty()) || OB_ISNULL(column_stats.at(0))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column stats is empty", K(ret)); @@ -510,12 +551,11 @@ int ObOptStatSqlService::update_column_stat(share::schema::ObSchemaGetterGuard * allocator, column_stats, current_time, - is_history_stat, column_stats_sql, print_params))) { LOG_WARN("failed to construct column stat sql", K(ret)); // construct histogram delete column - } else if (!only_update_col_stat && !is_history_stat && + } else if (!only_update_col_stat && construct_delete_column_histogram_sql(exec_tenant_id, column_stats, delete_histogram)) { LOG_WARN("failed to construc delete column histogram sql", K(ret)); // construct histogram insert sql @@ -525,19 +565,18 @@ int ObOptStatSqlService::update_column_stat(share::schema::ObSchemaGetterGuard * allocator, column_stats, current_time, - is_history_stat, insert_histogram, need_histogram, print_params))) { LOG_WARN("failed to construct histogram insert sql", K(ret)); - } else if (!only_update_col_stat && !is_history_stat && - OB_FAIL(trans.write(exec_tenant_id, delete_histogram.ptr(), affected_rows))) { - LOG_WARN("fail to exec sql", K(delete_histogram), K(ret)); - } else if (need_histogram && - OB_FAIL(trans.write(exec_tenant_id, insert_histogram.ptr(), affected_rows))) { - LOG_WARN("failed to exec sql", K(insert_histogram), K(ret)); - } else if (OB_FAIL(trans.write(exec_tenant_id, column_stats_sql.ptr(), affected_rows))) { - LOG_WARN("failed to exec sql", K(column_stats_sql), K(ret)); + } else if (!only_update_col_stat && + OB_FAIL(conn->execute_write(exec_tenant_id, delete_histogram.ptr(), affected_rows))) { + LOG_WARN("failed to execute write", K(delete_histogram)); + } else if (OB_FAIL(need_histogram && + conn->execute_write(exec_tenant_id, insert_histogram.ptr(), affected_rows))) { + LOG_WARN("failed to execute write", K(insert_histogram)); + } else if (OB_FAIL(conn->execute_write(exec_tenant_id, column_stats_sql.ptr(), affected_rows))) { + LOG_WARN("failed to execute write", K(column_stats_sql)); } return ret; } @@ -547,7 +586,6 @@ int ObOptStatSqlService::construct_column_stat_sql(share::schema::ObSchemaGetter ObIAllocator &allocator, const ObIArray &column_stats, const int64_t current_time, - bool is_history_stat, ObSqlString &column_stats_sql, const ObObjPrintParams &print_params) { @@ -557,8 +595,7 @@ int ObOptStatSqlService::construct_column_stat_sql(share::schema::ObSchemaGetter ObObjMeta max_meta; uint64_t data_version = 0; if (OB_FAIL(get_column_stat_min_max_meta(schema_guard, tenant_id, - is_history_stat ? share::OB_ALL_COLUMN_STAT_HISTORY_TID : - share::OB_ALL_COLUMN_STAT_TID, + share::OB_ALL_COLUMN_STAT_TID, min_meta, max_meta))) { LOG_WARN("failed to get column stat min max meta", K(ret)); @@ -570,26 +607,13 @@ int ObOptStatSqlService::construct_column_stat_sql(share::schema::ObSchemaGetter if (OB_ISNULL(column_stats.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column stat is null", K(ret)); - } else if (is_history_stat) { - if (i == 0 && OB_FAIL(column_stats_sql.append_fmt(INSERT_COL_STAT_HISTORY_SQL, - share::OB_ALL_COLUMN_STAT_HISTORY_TNAME, - data_version < DATA_VERSION_4_3_0_0 ? " " : ",cg_macro_blk_cnt, cg_micro_blk_cnt"))) { - LOG_WARN("failed to append fmt sql", K(ret)); - } else if (OB_FAIL(get_column_stat_history_sql(tenant_id, allocator, - *column_stats.at(i), current_time, - min_meta, max_meta, tmp, print_params))) { - LOG_WARN("failed to get column stat history sql", K(ret)); - } else if (OB_FAIL(column_stats_sql.append_fmt("(%s)%s", tmp.ptr(), - (i == column_stats.count() - 1 ? ";" : ",")))) { - LOG_WARN("failed to append sql", K(ret)); - } else {/*do nothing*/} } else if (i == 0 && OB_FAIL(column_stats_sql.append_fmt(REPLACE_COL_STAT_SQL, data_version < DATA_VERSION_4_3_0_0 ? " " : ",cg_macro_blk_cnt, cg_micro_blk_cnt"))) { LOG_WARN("failed to append sql", K(ret)); } else if (OB_FAIL(get_column_stat_sql(tenant_id, allocator, *column_stats.at(i), current_time, min_meta, max_meta, tmp, print_params))) { - LOG_WARN("failed to get column stat", K(ret)); + LOG_WARN("failed to get column stat", K(ret), K(*column_stats.at(i))); } else if (OB_FAIL(column_stats_sql.append_fmt("(%s)%s", tmp.ptr(), (i == column_stats.count() - 1 ? ";" : ",")))) { LOG_WARN("failed to append sql", K(ret)); @@ -605,6 +629,7 @@ int ObOptStatSqlService::construct_delete_column_histogram_sql(const uint64_t te { int ret = OB_SUCCESS; ObSqlString where_str; + ObSqlString hint_str; const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); for (int64_t i = 0; OB_SUCC(ret) && i < column_stats.count(); ++i) { if (OB_ISNULL(column_stats.at(i))) { @@ -621,9 +646,13 @@ int ObOptStatSqlService::construct_delete_column_histogram_sql(const uint64_t te } } if (OB_SUCC(ret) && !where_str.empty()) { - if (OB_FAIL(delete_histogram_sql.append_fmt(" %s %.*s;", DELETE_HISTOGRAM_STAT_SQL, - where_str.string().length(), - where_str.string().ptr()))) { + if (OB_FAIL(hint_str.append("opt_param('enable_in_range_optimization','true')"))) { + LOG_WARN("fail to append hint", K(ret)); + } else if (OB_FAIL(delete_histogram_sql.append_fmt(DELETE_HISTOGRAM_STAT_SQL, + hint_str.string().length(), + hint_str.string().ptr(), + where_str.string().length(), + where_str.string().ptr()))) { LOG_WARN("fail to append SQL where string.", K(ret)); } else { LOG_TRACE("Succeed to construct delete column histogram sql", K(delete_histogram_sql)); @@ -637,7 +666,6 @@ int ObOptStatSqlService::construct_histogram_insert_sql(share::schema::ObSchemaG ObIAllocator &allocator, const ObIArray &column_stats, const int64_t current_time, - bool is_history_stat, ObSqlString &insert_histogram_sql, bool &need_histogram, const ObObjPrintParams &print_params) @@ -647,8 +675,7 @@ int ObOptStatSqlService::construct_histogram_insert_sql(share::schema::ObSchemaG need_histogram = false; ObObjMeta endpoint_meta; if (OB_FAIL(get_histogram_endpoint_meta(schema_guard, tenant_id, - is_history_stat ? share::OB_ALL_HISTOGRAM_STAT_HISTORY_TID : - share::OB_ALL_HISTOGRAM_STAT_TID, + share::OB_ALL_HISTOGRAM_STAT_TID, endpoint_meta))) { LOG_WARN("failed to get histogram endpoint meta", K(ret)); } @@ -660,25 +687,7 @@ int ObOptStatSqlService::construct_histogram_insert_sql(share::schema::ObSchemaG ObHistogram &hist = column_stats.at(i)->get_histogram(); for (int64_t j = 0; OB_SUCC(ret) && hist.is_valid() && j < hist.get_bucket_size(); ++j) { tmp.reset(); - if (is_history_stat) { - if (!need_histogram && OB_FAIL(insert_histogram_sql.append_fmt(INSERT_HISTOGRAM_STAT_HISTORY_SQL, - share::OB_ALL_HISTOGRAM_STAT_HISTORY_TNAME))) { - LOG_WARN("failed to append fmt sql", K(ret)); - } else if (OB_FAIL(get_histogram_stat_history_sql(tenant_id, - *column_stats.at(i), - allocator, - hist.get(j), - current_time, - endpoint_meta, - tmp, - print_params))) { - LOG_WARN("failed to get histogram stat history sql", K(ret)); - } else if (OB_FAIL(insert_histogram_sql.append_fmt("%s (%s)", (!need_histogram ? "" : ","), tmp.ptr()))) { - LOG_WARN("failed to append sql", K(ret)); - } else { - need_histogram = true; - } - } else if (!need_histogram && OB_FAIL(insert_histogram_sql.append(INSERT_HISTOGRAM_STAT_SQL))) { + if (!need_histogram && OB_FAIL(insert_histogram_sql.append(INSERT_HISTOGRAM_STAT_SQL))) { LOG_WARN("failed to append sql", K(ret)); } else if (OB_FAIL(get_histogram_stat_sql(tenant_id, *column_stats.at(i), allocator, hist.get(j), endpoint_meta, tmp, print_params))) { @@ -705,6 +714,7 @@ int ObOptStatSqlService::delete_table_stat(const uint64_t exec_tenant_id, const uint64_t table_id, const ObIArray &part_ids, const bool cascade_column, + int64_t degree, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -713,6 +723,8 @@ int ObOptStatSqlService::delete_table_stat(const uint64_t exec_tenant_id, ObSqlString delete_tstat; ObSqlString delete_hist; ObSqlString in_list; + ObSqlString where_str; + ObSqlString hint_str; bool has_part = !part_ids.empty(); int64_t tmp_affected_rows1 = 0; int64_t tmp_affected_rows2 = 0; @@ -722,31 +734,43 @@ int ObOptStatSqlService::delete_table_stat(const uint64_t exec_tenant_id, LOG_WARN("sql service not inited", K(ret)); } else if (OB_FAIL(generate_in_list(part_ids, in_list))) { LOG_WARN("failed to generate in list", K(ret)); + } else if (degree > 1 && + OB_FAIL(hint_str.append_fmt( + "ENABLE_PARALLEL_DML parallel(%ld)", + degree + ))) { + LOG_WARN("failed to append sql", K(ret)); + } else if (OB_FAIL(where_str.append_fmt( + "tenant_id = %lu and table_id = %ld %s%s;", + ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, exec_tenant_id), + ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), + has_part ? "AND partition_id in " : "", + has_part ? in_list.ptr() : ""))) { + LOG_WARN("failed to append sql", K(ret)); } else if (OB_FAIL(delete_tstat.append_fmt( - "%s tenant_id = %lu and table_id = %ld %s%s;", - DELETE_TAB_STAT_SQL, - ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, exec_tenant_id), - ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), - has_part ? "AND partition_id in " : "", - has_part ? in_list.ptr() : ""))) { + DELETE_TAB_STAT_SQL, + hint_str.string().length(), + hint_str.string().ptr(), + where_str.string().length(), + where_str.string().ptr()))) { LOG_WARN("failed to append sql", K(ret)); } else if (!cascade_column) { // do nothing } else if (OB_FAIL(delete_cstat.append_fmt( - "%s tenant_id = %lu and table_id = %ld %s%s;", - DELETE_COL_STAT_SQL, - ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, exec_tenant_id), - ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), - has_part ? "AND partition_id in " : "", - has_part ? in_list.ptr() : ""))) { + DELETE_COL_STAT_SQL, + hint_str.string().length(), + hint_str.string().ptr(), + where_str.string().length(), + where_str.string().ptr()))) { LOG_WARN("failed to append sql", K(ret)); - } else if (OB_FAIL(delete_hist.append_fmt( - "%s tenant_id = %lu and table_id = %ld %s%s;", - DELETE_HISTOGRAM_STAT_SQL, - ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, exec_tenant_id), - ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), - has_part ? "AND partition_id in " : "", - has_part ? in_list.ptr() : ""))) { + } else if (OB_FAIL(hint_str.append("opt_param('enable_in_range_optimization','true')"))) { + LOG_WARN("failed to append sql", K(ret)); + } else if (OB_FAIL(delete_hist.append_fmt( + DELETE_HISTOGRAM_STAT_SQL, + hint_str.string().length(), + hint_str.string().ptr(), + where_str.string().length(), + where_str.string().ptr()))) { LOG_WARN("failed to append sql", K(ret)); } if (OB_SUCC(ret)) { @@ -787,7 +811,8 @@ int ObOptStatSqlService::delete_column_stat(const uint64_t exec_tenant_id, const uint64_t table_id, const ObIArray &column_ids, const ObIArray &partition_ids, - const bool only_histogram /*=false*/) + const bool only_histogram /*=false*/, + const int64_t degree) { int ret = OB_SUCCESS; ObMySQLTransaction trans; @@ -796,6 +821,8 @@ int ObOptStatSqlService::delete_column_stat(const uint64_t exec_tenant_id, ObSqlString delete_histogram; ObSqlString partition_list; ObSqlString column_list; + ObSqlString hint_str; + ObSqlString where_str; bool has_part = !partition_ids.empty(); if (!inited_) { ret = OB_NOT_INIT; @@ -807,25 +834,37 @@ int ObOptStatSqlService::delete_column_stat(const uint64_t exec_tenant_id, LOG_WARN("failed to generate in list", K(ret)); } else if (OB_FAIL(generate_in_list(column_ids, column_list))) { LOG_WARN("failed to generate in list", K(ret)); + } else if (degree > 1 && + OB_FAIL(hint_str.append_fmt("ENABLE_PARALLEL_DML parallel(%ld)",degree))) { + LOG_WARN("failed to append sql", K(ret)); + } else if (OB_FAIL(where_str.append_fmt( + "tenant_id = %lu and table_id = %ld and column_id in %s %s%s;", + ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, exec_tenant_id), + ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), + column_list.ptr(), + has_part ? "AND partition_id in " : "", + has_part ? partition_list.ptr() : ""))) { + LOG_WARN("failed to append sql", K(ret)); + } else if (OB_FAIL(hint_str.append_fmt("opt_param('enable_in_range_optimization','true')"))) { + LOG_WARN("failed to append sql", K(ret)); } else if (OB_FAIL(delete_histogram.append_fmt( - "%s tenant_id = %lu and table_id = %ld and column_id in %s %s%s;", DELETE_HISTOGRAM_STAT_SQL, - ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, exec_tenant_id), - ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), - column_list.ptr(), - has_part ? "AND partition_id in " : "", - has_part ? partition_list.ptr() : ""))) { + hint_str.string().length(), + hint_str.string().ptr(), + where_str.string().length(), + where_str.string().ptr()))) { LOG_WARN("failed to append sql", K(ret)); - } else if (OB_FAIL(write_cstat.append_fmt( - "%s tenant_id = %lu and table_id = %ld and column_id in %s %s%s;", - (only_histogram ? UPDATE_HISTOGRAM_TYPE_SQL : DELETE_COL_STAT_SQL), - ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, exec_tenant_id), - ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), - column_list.ptr(), - has_part ? "AND partition_id in " : "", - has_part ? partition_list.ptr() : ""))) { + } else if (FALSE_IT(hint_str.reuse())) { + } else if (degree > 1 && + OB_FAIL(hint_str.append_fmt("ENABLE_PARALLEL_DML parallel(%ld)",degree))) { + LOG_WARN("failed to append sql", K(ret)); + } else if (OB_FAIL(write_cstat.append_fmt( + (only_histogram ? UPDATE_HISTOGRAM_TYPE_SQL : DELETE_COL_STAT_SQL), + hint_str.string().length(), + hint_str.string().ptr(), + where_str.string().length(), + where_str.string().ptr()))) { LOG_WARN("failed to append sql", K(ret)); - } if (OB_SUCC(ret)) { @@ -889,41 +928,6 @@ int ObOptStatSqlService::get_table_stat_sql(const uint64_t tenant_id, return ret; } -int ObOptStatSqlService::get_table_stat_history_sql(const uint64_t tenant_id, - const ObOptTableStat &stat, - const int64_t saving_time, - ObSqlString &sql_string) -{ - int ret = OB_SUCCESS; - share::ObDMLSqlSplicer dml_splicer; - uint64_t table_id = stat.get_table_id(); - uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id); - uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id); - if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", ext_tenant_id)) || - OB_FAIL(dml_splicer.add_pk_column("table_id", pure_table_id)) || - OB_FAIL(dml_splicer.add_pk_column("partition_id", stat.get_partition_id())) || - OB_FAIL(dml_splicer.add_time_column("savtime", saving_time, true)) || - OB_FAIL(dml_splicer.add_column("index_type", false)) || - OB_FAIL(dml_splicer.add_column("object_type", stat.get_object_type())) || - OB_FAIL(dml_splicer.add_column("flags", 0)) || - OB_FAIL(dml_splicer.add_time_column("last_analyzed", stat.get_last_analyzed() == 0 ? - saving_time : stat.get_last_analyzed())) || - OB_FAIL(dml_splicer.add_column("sstable_row_count", -1)) || - OB_FAIL(dml_splicer.add_column("sstable_avg_row_len", -1)) || - OB_FAIL(dml_splicer.add_column("macro_blk_cnt", stat.get_macro_block_num())) || - OB_FAIL(dml_splicer.add_column("micro_blk_cnt", stat.get_micro_block_num())) || - OB_FAIL(dml_splicer.add_column("memtable_row_cnt", -1)) || - OB_FAIL(dml_splicer.add_column("memtable_avg_row_len", -1)) || - OB_FAIL(dml_splicer.add_column("row_cnt", stat.get_row_count())) || - OB_FAIL(dml_splicer.add_column("avg_row_len", stat.get_avg_row_size())) || - OB_FAIL(dml_splicer.add_column("stattype_locked", stat.get_stattype_locked()))) { - LOG_WARN("failed to add dml splicer column", K(ret)); - } else if (OB_FAIL(dml_splicer.splice_values(sql_string))) { - LOG_WARN("failed to get sql string", K(ret)); - } else { /*do nothing*/ } - return ret; -} - int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id, ObIAllocator &allocator, const ObOptColumnStat &stat, @@ -945,8 +949,10 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id, int64_t llc_comp_size = 0; int64_t llc_hex_size = 0; uint64_t data_version = 0; - if (OB_UNLIKELY(ObHistType::INVALID_TYPE != stat.get_histogram().get_type() && - stat.get_histogram().get_bucket_cnt() == 0)) { + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { + LOG_WARN("fail to get tenant data version", KR(ret), K(tenant_id), K(data_version)); + } else if (OB_UNLIKELY(ObHistType::INVALID_TYPE != stat.get_histogram().get_type() && + stat.get_histogram().get_bucket_cnt() == 0)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(stat)); } else if (OB_FAIL(get_valid_obj_str(stat.get_min_value(), min_meta, allocator, min_str, print_params)) || @@ -960,6 +966,8 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id, } else if (stat.get_llc_bitmap_size() <= 0) { // do nothing } else if (OB_FAIL(get_compressed_llc_bitmap(allocator, + data_version < DATA_VERSION_4_2_2_0 ? bitmap_compress_lib_name[ObOptStatCompressType::ZLIB_COMPRESS] : + bitmap_compress_lib_name[ObOptStatCompressType::ZSTD_1_3_8_COMPRESS], stat.get_llc_bitmap(), stat.get_llc_bitmap_size(), llc_comp_buf, @@ -998,87 +1006,8 @@ int ObOptStatSqlService::get_column_stat_sql(const uint64_t tenant_id, OB_FAIL(dml_splicer.add_column("histogram_type", stat.get_histogram().get_type())) || OB_FAIL(dml_splicer.add_column("global_stats", 0)) || OB_FAIL(dml_splicer.add_column("user_stats", 0)) || - (data_version >= DATA_VERSION_4_3_0_0 && - OB_FAIL(dml_splicer.add_column("cg_macro_blk_cnt", stat.get_cg_macro_blk_cnt()))) || - (data_version >= DATA_VERSION_4_3_0_0 && - OB_FAIL(dml_splicer.add_column("cg_micro_blk_cnt", stat.get_cg_micro_blk_cnt())))) { - LOG_WARN("failed to add dml splicer column", K(ret)); - } else if (OB_FAIL(dml_splicer.splice_values(sql_string))) { - LOG_WARN("failed to get sql string", K(ret)); - } else { /*do nothing*/ } - } - return ret; -} - -int ObOptStatSqlService::get_column_stat_history_sql(const uint64_t tenant_id, - ObIAllocator &allocator, - const ObOptColumnStat &stat, - const int64_t saving_time, - ObObjMeta min_meta, - ObObjMeta max_meta, - ObSqlString &sql_string, - const ObObjPrintParams &print_params) -{ - int ret = OB_SUCCESS; - share::ObDMLSqlSplicer dml_splicer; - ObString min_str, b_min_str; - ObString max_str, b_max_str; - uint64_t table_id = stat.get_table_id(); - uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id); - uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id); - char *llc_comp_buf = NULL; - char *llc_hex_buf = NULL; - int64_t llc_comp_size = 0; - int64_t llc_hex_size = 0; - uint64_t data_version = 0; - if (OB_FAIL(ObOptStatSqlService::get_valid_obj_str(stat.get_min_value(), min_meta, allocator, min_str, print_params)) || - OB_FAIL(ObOptStatSqlService::get_valid_obj_str(stat.get_max_value(), max_meta, allocator, max_str, print_params)) || - OB_FAIL(ObOptStatSqlService::get_obj_binary_hex_str(stat.get_min_value(), allocator, b_min_str)) || - OB_FAIL(ObOptStatSqlService::get_obj_binary_hex_str(stat.get_max_value(), allocator, b_max_str))) { - LOG_WARN("failed to convert obj to str", K(ret)); - } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, data_version))) { - LOG_WARN("fail to get tenant data version", KR(ret)); - } else if (stat.get_llc_bitmap_size() <= 0) { - // do nothing - } else if (OB_FAIL(ObOptStatSqlService::get_compressed_llc_bitmap(allocator, - stat.get_llc_bitmap(), - stat.get_llc_bitmap_size(), - llc_comp_buf, - llc_comp_size))) { - LOG_WARN("failed to get compressed llc bit map", K(ret)); - } else if (FALSE_IT(llc_hex_size = llc_comp_size * 2 + 2)) { - // 1 bytes are represented by 2 hex char (2 bytes) - // 1 bytes for '\0', and 1 bytes just safe - } else if (OB_ISNULL(llc_hex_buf = static_cast(allocator.alloc(llc_hex_size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to allocate memory", K(ret), K(llc_hex_buf), K(llc_hex_size)); - } else if (OB_FAIL(common::to_hex_cstr(llc_comp_buf, llc_comp_size, llc_hex_buf, llc_hex_size))) { - LOG_WARN("failed to convert to hex cstr", K(ret)); - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", ext_tenant_id)) || - OB_FAIL(dml_splicer.add_pk_column("table_id", pure_table_id)) || - OB_FAIL(dml_splicer.add_pk_column("partition_id", stat.get_partition_id())) || - OB_FAIL(dml_splicer.add_pk_column("column_id", stat.get_column_id())) || - OB_FAIL(dml_splicer.add_time_column("savtime", saving_time, true)) || - OB_FAIL(dml_splicer.add_column("object_type", stat.get_stat_level())) || - OB_FAIL(dml_splicer.add_column("flags", 0)) || - OB_FAIL(dml_splicer.add_time_column("last_analyzed", stat.get_last_analyzed() == 0 ? - saving_time : stat.get_last_analyzed())) || - OB_FAIL(dml_splicer.add_column("distinct_cnt", stat.get_num_distinct())) || - OB_FAIL(dml_splicer.add_column("null_cnt", stat.get_num_null())) || - OB_FAIL(dml_splicer.add_column("max_value", ObHexEscapeSqlStr(max_str))) || - OB_FAIL(dml_splicer.add_column("b_max_value", b_max_str)) || - OB_FAIL(dml_splicer.add_column("min_value", ObHexEscapeSqlStr(min_str))) || - OB_FAIL(dml_splicer.add_column("b_min_value", b_min_str)) || - OB_FAIL(dml_splicer.add_column("avg_len", stat.get_avg_len())) || - OB_FAIL(dml_splicer.add_column("distinct_cnt_synopsis", llc_hex_buf == NULL ? "" : llc_hex_buf)) || - OB_FAIL(dml_splicer.add_column("distinct_cnt_synopsis_size", llc_comp_size * 2)) || - OB_FAIL(dml_splicer.add_column("sample_size", stat.get_histogram().get_sample_size())) || - OB_FAIL(dml_splicer.add_column("density", stat.get_histogram().get_density())) || - OB_FAIL(dml_splicer.add_column("bucket_cnt", stat.get_histogram().get_bucket_cnt())) || - OB_FAIL(dml_splicer.add_column("histogram_type", stat.get_histogram().get_type())) || + OB_FAIL(dml_splicer.add_column("spare1", data_version < DATA_VERSION_4_2_2_0 ? ObOptStatCompressType::ZLIB_COMPRESS : + ObOptStatCompressType::ZSTD_1_3_8_COMPRESS)) || (data_version >= DATA_VERSION_4_3_0_0 && OB_FAIL(dml_splicer.add_column("cg_macro_blk_cnt", stat.get_cg_macro_blk_cnt()))) || (data_version >= DATA_VERSION_4_3_0_0 && @@ -1131,49 +1060,6 @@ int ObOptStatSqlService::get_histogram_stat_sql(const uint64_t tenant_id, return ret; } -int ObOptStatSqlService::get_histogram_stat_history_sql(const uint64_t tenant_id, - const ObOptColumnStat &stat, - ObIAllocator &allocator, - const ObHistBucket &bucket, - const int64_t saving_time, - ObObjMeta endpoint_meta, - ObSqlString &sql_string, - const ObObjPrintParams &print_params) -{ - int ret = OB_SUCCESS; - ObString endpoint_value; - ObString b_endpoint_value; - share::ObDMLSqlSplicer dml_splicer; - uint64_t table_id = stat.get_table_id(); - uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id); - uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id); - if (OB_FAIL(ObOptStatSqlService::get_valid_obj_str(bucket.endpoint_value_, - endpoint_meta, - allocator, - endpoint_value, - print_params))) { - LOG_WARN("failed to convert obj to string", K(ret)); - } else if (OB_FAIL(ObOptStatSqlService::get_obj_binary_hex_str(bucket.endpoint_value_, - allocator, b_endpoint_value))) { - LOG_WARN("failed to convert obj to binary string", K(ret)); - } else if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", ext_tenant_id)) || - OB_FAIL(dml_splicer.add_pk_column("table_id", pure_table_id)) || - OB_FAIL(dml_splicer.add_pk_column("partition_id", stat.get_partition_id())) || - OB_FAIL(dml_splicer.add_pk_column("column_id", stat.get_column_id())) || - OB_FAIL(dml_splicer.add_pk_column("endpoint_num", bucket.endpoint_num_)) || - OB_FAIL(dml_splicer.add_time_column("savtime", saving_time, true)) || - OB_FAIL(dml_splicer.add_column("object_type", stat.get_stat_level())) || - OB_FAIL(dml_splicer.add_column("endpoint_normalized_value", -1)) || - OB_FAIL(dml_splicer.add_column("endpoint_value", ObHexEscapeSqlStr(endpoint_value))) || - OB_FAIL(dml_splicer.add_column("b_endpoint_value", b_endpoint_value)) || - OB_FAIL(dml_splicer.add_column("endpoint_repeat_cnt", bucket.endpoint_repeat_count_))) { - LOG_WARN("failed to add dml splice values", K(ret)); - } else if (OB_FAIL(dml_splicer.splice_values(sql_string))) { - LOG_WARN("failed to get sql string", K(ret)); - } else { /*do nothing*/ } - return ret; -} - int ObOptStatSqlService::hex_str_to_obj(const char *buf, int64_t buf_len, ObIAllocator &allocator, @@ -1307,7 +1193,8 @@ int ObOptStatSqlService::fill_table_stat(common::sqlclient::ObMySQLResult &resul int ObOptStatSqlService::fetch_column_stat(const uint64_t tenant_id, ObIAllocator &allocator, ObIArray &key_col_stats, - bool is_accross_tenant_query) + bool is_accross_tenant_query, + sqlclient::ObISQLConnection *conn/*default null*/) { int ret = OB_SUCCESS; ObSqlString keys_list_str; @@ -1346,7 +1233,11 @@ int ObOptStatSqlService::fetch_column_stat(const uint64_t tenant_id, keys_list_str.string().length(), keys_list_str.string().ptr()))) { LOG_WARN("fail to append SQL stmt string.", K(ret)); - } else if (OB_FAIL(sql_client_retry_weak.read(res, exec_tenant_id, sql.ptr()))) { + } else if (conn != NULL && + OB_FAIL(conn->execute_read(exec_tenant_id, sql.ptr(), res))) { + LOG_WARN("execute sql failed", "sql", sql.ptr(), K(ret)); + } else if (conn == NULL && + OB_FAIL(sql_client_retry_weak.read(res, exec_tenant_id, sql.ptr()))) { LOG_WARN("execute sql failed", "sql", sql.ptr(), K(ret)); } else if (NULL == (result = res.get_result())) { ret = OB_ERR_UNEXPECTED; @@ -1490,20 +1381,27 @@ int ObOptStatSqlService::fill_column_stat(ObIAllocator &allocator, EXTRACT_VARCHAR_FIELD_MYSQL(result, "distinct_cnt_synopsis", hex_str); char *bitmap_buf = NULL; if (OB_SUCC(ret) && llc_bitmap_size > 0) { - if (NULL == (bitmap_buf = static_cast(allocator.alloc(hex_str.length())))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("allocate memory for llc_bitmap failed.", K(hex_str.length()), K(ret)); - } else { - common::str_to_hex(hex_str.ptr(), hex_str.length(), bitmap_buf, hex_str.length()); - // decompress llc bitmap; - char *decomp_buf = NULL ; - int64_t decomp_size = ObOptColumnStat::NUM_LLC_BUCKET; - const int64_t bitmap_size = hex_str.length() / 2; - if (OB_FAIL(get_decompressed_llc_bitmap(allocator, bitmap_buf, - bitmap_size, decomp_buf, decomp_size))) { - COMMON_LOG(WARN, "decompress bitmap buffer failed.", K(ret)); + int64_t compress_type = ObOptStatCompressType::MAX_COMPRESS; + EXTRACT_INT_FIELD_MYSQL_WITH_DEFAULT_VALUE(result, "compress_type", compress_type, int64_t, true, false, ObOptStatCompressType::ZLIB_COMPRESS); + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(compress_type < 0 || compress_type >= ObOptStatCompressType::MAX_COMPRESS)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(compress_type)); + } else if (NULL == (bitmap_buf = static_cast(allocator.alloc(hex_str.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("allocate memory for llc_bitmap failed.", K(hex_str.length()), K(ret)); } else { - stat->set_llc_bitmap(decomp_buf, decomp_size); + common::str_to_hex(hex_str.ptr(), hex_str.length(), bitmap_buf, hex_str.length()); + // decompress llc bitmap; + char *decomp_buf = NULL ; + int64_t decomp_size = ObOptColumnStat::NUM_LLC_BUCKET; + const int64_t bitmap_size = hex_str.length() / 2; + if (OB_FAIL(get_decompressed_llc_bitmap(allocator, bitmap_compress_lib_name[compress_type], bitmap_buf, + bitmap_size, decomp_buf, decomp_size))) { + COMMON_LOG(WARN, "decompress bitmap buffer failed.", K(ret)); + } else { + stat->set_llc_bitmap(decomp_buf, decomp_size); + } } } } @@ -1545,6 +1443,7 @@ int ObOptStatSqlService::fill_column_stat(ObIAllocator &allocator, } int ObOptStatSqlService::get_compressed_llc_bitmap(ObIAllocator &allocator, + const char *bitmap_compress_name, const char *bitmap_buf, int64_t bitmap_size, char *&comp_buf, @@ -1553,20 +1452,20 @@ int ObOptStatSqlService::get_compressed_llc_bitmap(ObIAllocator &allocator, int ret = OB_SUCCESS; ObCompressor *compressor = NULL; int64_t max_comp_size = 0; - if (NULL == bitmap_buf || bitmap_size <= 0) { + if (NULL == bitmap_buf || bitmap_size <= 0 || bitmap_compress_name == NULL) { ret = common::OB_INVALID_ARGUMENT; - COMMON_LOG(WARN, "invalid arguments.", KP(bitmap_buf), K(bitmap_size), K(ret)); + COMMON_LOG(WARN, "invalid arguments.", KP(bitmap_buf), K(bitmap_size), K(bitmap_compress_name), K(ret)); } else if (OB_FAIL(ObCompressorPool::get_instance().get_compressor( - bitmap_compress_lib_name, compressor))) { + bitmap_compress_name, compressor))) { COMMON_LOG(WARN, "cannot create compressor, do not compress data.", - K(bitmap_compress_lib_name), K(ret)); + K(bitmap_compress_name), K(ret)); } else if (NULL == compressor) { ret = OB_ERR_UNEXPECTED; COMMON_LOG(WARN, "compressor is NULL, do not compress data.", - K(bitmap_compress_lib_name), K(ret)); + K(bitmap_compress_name), K(ret)); } else if (OB_FAIL(compressor->get_max_overflow_size(bitmap_size, max_comp_size))) { COMMON_LOG(WARN, "get max overflow size failed.", - K(bitmap_compress_lib_name), K(bitmap_size), K(ret)); + K(bitmap_compress_name), K(bitmap_size), K(ret)); } else { max_comp_size += bitmap_size; if (NULL == (comp_buf = static_cast(allocator.alloc(max_comp_size)))) { @@ -1592,6 +1491,7 @@ int ObOptStatSqlService::get_compressed_llc_bitmap(ObIAllocator &allocator, } int ObOptStatSqlService::get_decompressed_llc_bitmap(ObIAllocator &allocator, + const char *bitmap_compress_name, const char *comp_buf, int64_t comp_size, char *&bitmap_buf, @@ -1601,7 +1501,10 @@ int ObOptStatSqlService::get_decompressed_llc_bitmap(ObIAllocator &allocator, const int64_t max_bitmap_size = ObOptColumnStat::NUM_LLC_BUCKET; // max size of uncompressed buffer. ObCompressor* compressor = NULL; - if (comp_size >= ObOptColumnStat::NUM_LLC_BUCKET) { + if (OB_ISNULL(bitmap_compress_name)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(bitmap_compress_name)); + } else if (comp_size >= ObOptColumnStat::NUM_LLC_BUCKET) { // not compressed bitmap, use directly; bitmap_buf = const_cast(comp_buf); bitmap_size = comp_size; @@ -1609,13 +1512,13 @@ int ObOptStatSqlService::get_decompressed_llc_bitmap(ObIAllocator &allocator, ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("allocate memory for uncompressed data failed.", K(max_bitmap_size), K(ret)); } else if (OB_FAIL(ObCompressorPool::get_instance().get_compressor( - bitmap_compress_lib_name, compressor))) { + bitmap_compress_name, compressor))) { LOG_WARN("cannot create compressor, do not uncompress data.", - K(bitmap_compress_lib_name), K(ret)); + K(bitmap_compress_name), K(ret)); } else if (NULL == compressor) { ret = OB_ERR_UNEXPECTED; LOG_WARN("compressor is NULL, do not compress data.", - K(bitmap_compress_lib_name), K(ret)); + K(bitmap_compress_name), K(ret)); } else if (OB_FAIL(compressor->decompress(comp_buf, comp_size, bitmap_buf, diff --git a/src/share/stat/ob_opt_stat_sql_service.h b/src/share/stat/ob_opt_stat_sql_service.h index 233b222023..37576f7a87 100644 --- a/src/share/stat/ob_opt_stat_sql_service.h +++ b/src/share/stat/ob_opt_stat_sql_service.h @@ -61,6 +61,12 @@ struct ObOptKeyInfo struct ObOptKeyColumnStat { ObOptKeyColumnStat() : key_(NULL), stat_(NULL), only_histogram_stat_(false) {} + ObOptKeyColumnStat(const ObOptColumnStat::Key *key, + ObOptColumnStat *stat) : + key_(key), + stat_(stat), + only_histogram_stat_(false) + {} const ObOptColumnStat::Key *key_; ObOptColumnStat *stat_; bool only_histogram_stat_; @@ -68,6 +74,15 @@ struct ObOptKeyColumnStat KPC_(stat), K(only_histogram_stat_)); }; + +enum ObOptStatCompressType +{ + ZLIB_COMPRESS = 0, + ZSTD_1_3_8_COMPRESS = 1, + MAX_COMPRESS = 2 +}; + +static const char *bitmap_compress_lib_name[ObOptStatCompressType::MAX_COMPRESS] = {"zlib_1.0", "zstd_1.3.8"}; /** * SQL Service for fetching/updating table level statistics and column level statistics */ @@ -83,6 +98,12 @@ public: const ObOptTableStat::Key &key, ObIArray &all_part_stats); + int batch_fetch_table_stats(sqlclient::ObISQLConnection *conn, + const uint64_t tenant_id, + const uint64_t table_id, + const ObIArray &part_ids, + ObIArray &all_part_stats); + int fill_table_stat(sqlclient::ObMySQLResult &result, ObOptTableStat &stat); int fill_column_stat(ObIAllocator &allocator, @@ -93,46 +114,50 @@ public: int fetch_column_stat(const uint64_t tenant_id, ObIAllocator &allocator, ObIArray &key_col_stats, - bool is_accross_tenant_query = false); + bool is_accross_tenant_query = false, + sqlclient::ObISQLConnection *conn = NULL); int update_table_stat(const uint64_t tenant_id, + sqlclient::ObISQLConnection *conn, const ObOptTableStat *tab_stat, const bool is_index_stat); int update_table_stat(const uint64_t tenant_id, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, const common::ObIArray &table_stats, const int64_t current_time, - const bool is_index_stat, - const bool is_history_stat = false); + const bool is_index_stat); int update_column_stat(share::schema::ObSchemaGetterGuard *schema_guard, const uint64_t exec_tenant_id, ObIAllocator &allocator, - ObMySQLTransaction &trans, + sqlclient::ObISQLConnection *conn, const common::ObIArray &column_stats, const int64_t current_time, bool only_update_col_stat = false, - bool is_history_stat = false, const ObObjPrintParams &print_params = ObObjPrintParams()); int delete_table_stat(const uint64_t exec_tenant_id, const uint64_t table_id, const ObIArray &part_ids, const bool cascade_column, + int64_t degree, int64_t &affected_rows); int delete_column_stat(const uint64_t exec_tenant_id, const uint64_t table_id, const ObIArray &column_ids, const ObIArray &partition_ids, - const bool only_histogram /*=false*/); + const bool only_histogram /*=false*/, + const int64_t degree); static int get_decompressed_llc_bitmap(ObIAllocator &allocator, + const char *bitmap_compress_name, const char *comp_buf, int64_t comp_size, char *&bitmap_buf, int64_t &bitmap_size); static int get_compressed_llc_bitmap(ObIAllocator &allocator, + const char *bitmap_compress_name, const char *bitmap_buf, int64_t bitmap_size, char *&comp_buf, @@ -185,10 +210,6 @@ private: const int64_t current_time, const bool is_index, ObSqlString &sql_string); - int get_table_stat_history_sql(const uint64_t tenant_id, - const ObOptTableStat &stat, - const int64_t saving_time, - ObSqlString &sql_string); int get_column_stat_sql(const uint64_t tenant_id, ObIAllocator &allocator, const ObOptColumnStat &stat, @@ -197,14 +218,6 @@ private: ObObjMeta max_meta, ObSqlString &sql_string, const ObObjPrintParams &print_params); - int get_column_stat_history_sql(const uint64_t tenant_id, - ObIAllocator &allocator, - const ObOptColumnStat &stat, - const int64_t saving_time, - ObObjMeta min_meta, - ObObjMeta max_meta, - ObSqlString &sql_string, - const ObObjPrintParams &print_params); int get_histogram_stat_sql(const uint64_t tenant_id, const ObOptColumnStat &stat, common::ObIAllocator &allocator, @@ -212,14 +225,6 @@ private: ObObjMeta endpoint_meta, ObSqlString &sql_string, const ObObjPrintParams &print_params); - int get_histogram_stat_history_sql(const uint64_t tenant_id, - const ObOptColumnStat &stat, - ObIAllocator &allocator, - const ObHistBucket &bucket, - const int64_t saving_time, - ObObjMeta endpoint_meta, - ObSqlString &sql_string, - const ObObjPrintParams &print_params); int generate_in_list(const ObIArray &list, ObSqlString &sql_string); @@ -229,7 +234,6 @@ private: ObIAllocator &allocator, const ObIArray &column_stats, const int64_t current_time, - bool is_history_stat, ObSqlString &column_stats_sql, const ObObjPrintParams &print_params); @@ -242,7 +246,6 @@ private: ObIAllocator &allocator, const ObIArray &column_stats, const int64_t current_time, - bool is_history_stat, ObSqlString &insert_histogram_sql, bool &need_histogram, const ObObjPrintParams &print_params); @@ -289,8 +292,6 @@ private: int fill_system_stat(sqlclient::ObMySQLResult &result, ObOptSystemStat &stat); - static const char *bitmap_compress_lib_name; - bool inited_; ObMySQLProxy *mysql_proxy_; lib::ObMutex mutex_; diff --git a/src/share/stat/ob_opt_table_stat.h b/src/share/stat/ob_opt_table_stat.h index f32ba5da45..bea08d4bc9 100644 --- a/src/share/stat/ob_opt_table_stat.h +++ b/src/share/stat/ob_opt_table_stat.h @@ -234,6 +234,7 @@ public: bool is_arrived_expired_time() const { return stat_expired_time_ != -1 && stat_expired_time_ <= ObTimeUtility::current_time(); } + int64_t get_stat_expired_time() const { return stat_expired_time_; } void set_stat_expired_time(int64_t expired_time) { stat_expired_time_ = expired_time; } bool is_locked() const { return stattype_locked_ > 0; } diff --git a/src/share/stat/ob_stat_define.cpp b/src/share/stat/ob_stat_define.cpp index e50b77cbd4..ded92ea5c9 100644 --- a/src/share/stat/ob_stat_define.cpp +++ b/src/share/stat/ob_stat_define.cpp @@ -84,10 +84,8 @@ int StatTable::assign(const StatTable &other) int ret = OB_SUCCESS; database_id_ = other.database_id_; table_id_ = other.table_id_; - incremental_stat_ = other.incremental_stat_; stale_percent_ = other.stale_percent_; - need_gather_subpart_ = other.need_gather_subpart_; - return no_regather_partition_ids_.assign(other.no_regather_partition_ids_); + return partition_stat_infos_.assign(other.partition_stat_infos_); } /** @@ -149,7 +147,6 @@ int ObTableStatParam::assign(const ObTableStatParam &other) tab_name_ = other.tab_name_; table_id_ = other.table_id_; part_level_ = other.part_level_; - total_part_cnt_ = other.total_part_cnt_; part_name_ = other.part_name_; sample_info_.is_sample_ = other.sample_info_.is_sample_; sample_info_.is_block_sample_ = other.sample_info_.is_block_sample_; @@ -192,10 +189,6 @@ int ObTableStatParam::assign(const ObTableStatParam &other) LOG_WARN("failed to assign", K(ret)); } else if (OB_FAIL(column_params_.assign(other.column_params_))) { LOG_WARN("failed to assign", K(ret)); - } else if (OB_FAIL(part_ids_.assign(other.part_ids_))) { - LOG_WARN("failed to assign", K(ret)); - } else if (OB_FAIL(subpart_ids_.assign(other.subpart_ids_))) { - LOG_WARN("failed to assign", K(ret)); } else if (OB_FAIL(no_regather_partition_ids_.assign(other.no_regather_partition_ids_))) { LOG_WARN("failed to assign", K(ret)); } else if (OB_FAIL(all_part_infos_.assign(other.all_part_infos_))) { @@ -235,5 +228,61 @@ int ObTableStatParam::assign_common_property(const ObTableStatParam &other) return ret; } +int ObOptStatGatherParam::assign(const ObOptStatGatherParam &other) +{ + int ret = OB_SUCCESS; + tenant_id_ = other.tenant_id_; + db_name_ = other.db_name_; + tab_name_ = other.tab_name_; + table_id_ = other.table_id_; + stat_level_ = other.stat_level_; + need_histogram_ = other.need_histogram_; + sample_info_.is_sample_ = other.sample_info_.is_sample_; + sample_info_.is_block_sample_ = other.sample_info_.is_block_sample_; + sample_info_.sample_type_ = other.sample_info_.sample_type_; + sample_info_.sample_value_ = other.sample_info_.sample_value_; + degree_ = other.degree_; + allocator_ = other.allocator_; + partition_id_block_map_ = other.partition_id_block_map_; + gather_start_time_ = other.gather_start_time_; + stattype_ = other.stattype_; + is_split_gather_ = other.is_split_gather_; + max_duration_time_ = other.max_duration_time_; + need_approx_ndv_ = other.need_approx_ndv_; + data_table_name_ = other.data_table_name_; + global_part_id_ = other.global_part_id_; + gather_vectorize_ = other.gather_vectorize_; + sepcify_scn_ = other.sepcify_scn_; + if (OB_FAIL(partition_infos_.assign(other.partition_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(column_params_.assign(other.column_params_))) { + LOG_WARN("failed to assign", K(ret)); + } else {/*do nothing*/} + return ret; +} + +bool ObTableStatParam::is_specify_partition_gather() const +{ + bool is_specify = false; + if (part_level_ == share::schema::PARTITION_LEVEL_ZERO) { + //do nothing + } else if (part_level_ == share::schema::PARTITION_LEVEL_ONE) { + is_specify = part_infos_.count() != all_part_infos_.count(); + } else if (part_level_ == share::schema::PARTITION_LEVEL_TWO) { + is_specify = (part_infos_.count() + approx_part_infos_.count() != all_part_infos_.count()) || + (subpart_infos_.count() != all_subpart_infos_.count()); + } + return is_specify; +} + +bool ObTableStatParam::is_specify_column_gather() const +{ + bool is_specify = false; + for (int64_t i = 0; !is_specify && i < column_params_.count(); ++i) { + is_specify = column_params_.at(i).is_valid_opt_col() && !column_params_.at(i).need_basic_stat(); + } + return is_specify; +} + } } diff --git a/src/share/stat/ob_stat_define.h b/src/share/stat/ob_stat_define.h index b379549ccf..5b5cd20356 100644 --- a/src/share/stat/ob_stat_define.h +++ b/src/share/stat/ob_stat_define.h @@ -62,6 +62,10 @@ const int64_t MAGIC_MAX_AUTO_SAMPLE_SIZE = 22000; const int64_t MAGIC_MIN_SAMPLE_SIZE = 2500; const int64_t MAGIC_BASE_SAMPLE_SIZE = 1000; const double MAGIC_SAMPLE_CUT_RATIO = 0.00962; +const static int64_t MAX_NUM_OF_WRITE_STATS = 2000; +const static int64_t DEFAULT_STAT_GATHER_VECTOR_BATCH_SIZE = 256; +const static int64_t MIN_GATHER_WORK_ARANA_SIZE = 10 * 1024L * 1024L; //10M +const int64_t MAX_OPT_STATS_PROCESS_RPC_TIMEOUT = 300000000;//one optimizer stats processing rpc time should not exceed 300 seconds enum StatLevel { @@ -113,8 +117,7 @@ enum ColumnGatherFlag NO_NEED_STAT = 0, VALID_OPT_COL = 1, NEED_BASIC_STAT = 1 << 1, - NEED_AVG_LEN = 1 << 2, - NO_NEED_HISTOGRAM = 1 << 3 + NEED_AVG_LEN = 1 << 2 }; enum ObGranularityType @@ -150,15 +153,6 @@ struct BlockNumStat K(cg_micro_cnt_arr_)) }; -struct ObExtraParam -{ - StatLevel type_; - int64_t nth_part_; - PartitionIdBlockMap partition_id_block_map_; - int64_t start_time_; - bool need_histogram_; -}; - //TODO@jiangxiu.wt: improve the expression of PartInfo, use the map is better. struct PartInfo { @@ -186,38 +180,35 @@ struct PartInfo K_(first_part_id)); }; - struct StatTable +struct ObPartitionStatInfo; +struct StatTable { StatTable() : database_id_(OB_INVALID_ID), table_id_(OB_INVALID_ID), - incremental_stat_(false), stale_percent_(0.0), - need_gather_subpart_(false), - no_regather_partition_ids_() - {} + partition_stat_infos_() + { + partition_stat_infos_.set_attr(lib::ObMemAttr(MTL_ID(), "StatTable")); + } StatTable(uint64_t database_id, uint64_t table_id) : database_id_(database_id), table_id_(table_id), - incremental_stat_(false), stale_percent_(0.0), - need_gather_subpart_(false), - no_regather_partition_ids_() - {} + partition_stat_infos_() + { + partition_stat_infos_.set_attr(lib::ObMemAttr(MTL_ID(), "StatTable")); + } bool operator<(const StatTable &other) const; int assign(const StatTable &other); TO_STRING_KV(K_(database_id), K_(table_id), - K_(incremental_stat), K_(stale_percent), - K_(need_gather_subpart), - K_(no_regather_partition_ids)); + K_(partition_stat_infos)); uint64_t database_id_; uint64_t table_id_; - bool incremental_stat_; double stale_percent_; - bool need_gather_subpart_; - ObSEArray no_regather_partition_ids_; + ObArray partition_stat_infos_; }; enum ObStatTableType { @@ -291,6 +282,7 @@ struct ObPartitionStatParam ObPartitionStatParam(share::schema::ObPartitionFuncType type) : part_type_(type), need_modify_(true), + can_use_approx_(false), gather_histogram_(true) {} ObPartitionStatParam& operator=(const ObPartitionStatParam& other) @@ -305,21 +297,25 @@ struct ObPartitionStatParam need_modify_ = other.need_modify_; gather_histogram_ = other.gather_histogram_; } - void set_gather_stat() + void set_gather_stat(const bool can_use_approx = false) { need_modify_ = true; + can_use_approx_ = can_use_approx; gather_histogram_ = true; } void reset_gather_stat() { need_modify_ = false; + can_use_approx_ = false; gather_histogram_ = false; } TO_STRING_KV(K_(part_type), K_(need_modify), + K_(can_use_approx), K_(gather_histogram)); share::schema::ObPartitionFuncType part_type_; bool need_modify_; + bool can_use_approx_; bool gather_histogram_; }; @@ -375,11 +371,10 @@ struct ObColumnStatParam { inline void set_valid_opt_col() { gather_flag_ |= ColumnGatherFlag::VALID_OPT_COL; } inline void set_need_basic_stat() { gather_flag_ |= ColumnGatherFlag::NEED_BASIC_STAT; } inline void set_need_avg_len() { gather_flag_ |= ColumnGatherFlag::NEED_AVG_LEN; } - inline void set_no_need_histogram() { gather_flag_ |= ColumnGatherFlag::NO_NEED_HISTOGRAM; } inline bool is_valid_opt_col() const { return gather_flag_ & ColumnGatherFlag::VALID_OPT_COL; } inline bool need_basic_stat() const { return gather_flag_ & ColumnGatherFlag::NEED_BASIC_STAT; } inline bool need_avg_len() const { return gather_flag_ & ColumnGatherFlag::NEED_AVG_LEN; } - inline bool is_no_need_histogram() const { return gather_flag_ & ColumnGatherFlag::NO_NEED_HISTOGRAM; } + inline bool need_col_stat() const { return gather_flag_ != ColumnGatherFlag::NO_NEED_STAT; } ObString column_name_; uint64_t column_id_; @@ -417,7 +412,7 @@ struct ObTableStatParam { static const int64_t DEFAULT_DATA_PART_ID = -1; ObTableStatParam() : tenant_id_(0), - db_name_(0), + db_name_(), db_id_(OB_INVALID_ID), tab_name_(), table_id_(OB_INVALID_ID), @@ -425,7 +420,6 @@ struct ObTableStatParam { global_stat_param_(), part_stat_param_(share::schema::ObPartitionFuncType::PARTITION_FUNC_TYPE_MAX), subpart_stat_param_(share::schema::ObPartitionFuncType::PARTITION_FUNC_TYPE_MAX), - total_part_cnt_(0), part_name_(), part_infos_(), subpart_infos_(), @@ -441,8 +435,6 @@ struct ObTableStatParam { column_params_(), no_invalidate_(false), force_(false), - part_ids_(), - subpart_ids_(), no_regather_partition_ids_(), is_subpart_name_(false), stat_category_(), @@ -479,6 +471,10 @@ struct ObTableStatParam { return global_data_part_id_ != INVALID_GLOBAL_PART_ID; } + bool is_specify_partition_gather() const; + + bool is_specify_column_gather() const; + uint64_t tenant_id_; ObString db_name_; @@ -490,7 +486,6 @@ struct ObTableStatParam { ObGlobalStatParam global_stat_param_; ObPartitionStatParam part_stat_param_; ObPartitionStatParam subpart_stat_param_; - int64_t total_part_cnt_; ObString part_name_; ObSEArray part_infos_; @@ -512,9 +507,6 @@ struct ObTableStatParam { bool no_invalidate_; bool force_; - - ObSEArray part_ids_; - ObSEArray subpart_ids_; ObSEArray no_regather_partition_ids_; bool is_subpart_name_; @@ -561,8 +553,6 @@ struct ObTableStatParam { K(column_params_), K(no_invalidate_), K(force_), - K(part_ids_), - K(subpart_ids_), K(no_regather_partition_ids_), K(is_subpart_name_), K(stat_category_), @@ -585,6 +575,77 @@ struct ObTableStatParam { K(column_group_params_)); }; +struct ObOptStatGatherParam { + ObOptStatGatherParam () : + tenant_id_(0), + db_name_(), + tab_name_(), + table_id_(OB_INVALID_ID), + stat_level_(INVALID_LEVEL), + need_histogram_(false), + sample_info_(), + degree_(1), + partition_infos_(), + column_params_(), + column_group_params_(), + allocator_(NULL), + partition_id_block_map_(NULL), + gather_start_time_(0), + stattype_(StatTypeLocked::NULL_TYPE), + is_split_gather_(false), + max_duration_time_(0), + need_approx_ndv_(true), + data_table_name_(), + global_part_id_(-1), + gather_vectorize_(DEFAULT_STAT_GATHER_VECTOR_BATCH_SIZE), + sepcify_scn_(0) + {} + int assign(const ObOptStatGatherParam &other); + uint64_t tenant_id_; + ObString db_name_; + ObString tab_name_; + uint64_t table_id_; + StatLevel stat_level_; + bool need_histogram_; + ObAnalyzeSampleInfo sample_info_; + int64_t degree_; + ObSEArray partition_infos_; + ObSEArray column_params_; + ObArray column_group_params_; + common::ObIAllocator *allocator_; + const PartitionIdBlockMap *partition_id_block_map_; + int64_t gather_start_time_; + StatTypeLocked stattype_; + bool is_split_gather_; + int64_t max_duration_time_; + bool need_approx_ndv_; + ObString data_table_name_; + int64_t global_part_id_; + int64_t gather_vectorize_; + uint64_t sepcify_scn_; + + TO_STRING_KV(K(tenant_id_), + K(db_name_), + K(tab_name_), + K(table_id_), + K(stat_level_), + K(need_histogram_), + K(sample_info_), + K(degree_), + K(partition_infos_), + K(column_params_), + K(column_group_params_), + K(gather_start_time_), + K(stattype_), + K(is_split_gather_), + K(max_duration_time_), + K(need_approx_ndv_), + K(data_table_name_), + K(global_part_id_), + K(gather_vectorize_), + K(sepcify_scn_)); +}; + struct ObOptStat { ObOptStat() : table_stat_(NULL), column_stats_() { @@ -595,7 +656,7 @@ struct ObOptStat // turn the column stat into pointer ObArray column_stats_; TO_STRING_KV(K(table_stat_), - K(column_stats_.count())); + K(column_stats_)); }; struct ObHistogramParam @@ -690,22 +751,31 @@ struct ObPartitionStatInfo ObPartitionStatInfo(): partition_id_(-1), row_cnt_(0), - is_stat_locked_(false) + is_stat_locked_(false), + is_no_dml_modified_(false), + is_no_stale_(false) {} - ObPartitionStatInfo(int64_t partition_id, int64_t row_cnt, bool is_locked): + ObPartitionStatInfo(int64_t partition_id, int64_t row_cnt, bool is_locked, bool is_no_dml_modified): partition_id_(partition_id), row_cnt_(row_cnt), - is_stat_locked_(is_locked) + is_stat_locked_(is_locked), + is_no_dml_modified_(is_no_dml_modified), + is_no_stale_(false) {} int64_t partition_id_; int64_t row_cnt_; bool is_stat_locked_; + bool is_no_dml_modified_; + bool is_no_stale_; + bool is_regather() const { return !is_stat_locked_ && !is_no_dml_modified_ && !is_no_stale_; } TO_STRING_KV(K(partition_id_), K(row_cnt_), - K(is_stat_locked_)); + K(is_stat_locked_), + K(is_no_dml_modified_), + K(is_no_stale_)); }; enum ObOptDmlStatType { diff --git a/src/share/stat/ob_stat_item.cpp b/src/share/stat/ob_stat_item.cpp index 8c7f5ff9b5..756f2596ba 100644 --- a/src/share/stat/ob_stat_item.cpp +++ b/src/share/stat/ob_stat_item.cpp @@ -16,6 +16,7 @@ #include "pl/sys_package/ob_dbms_stats.h" #include "share/stat/ob_opt_table_stat.h" #include "share/stat/ob_hybrid_hist_estimator.h" +#include "share/stat/ob_topk_hist_estimator.h" #include "share/stat/ob_dbms_stats_utils.h" #include "sql/engine/expr/ob_expr_lob_utils.h" namespace oceanbase @@ -117,7 +118,7 @@ int ObStatMaxValue::decode(ObObj &obj, ObIAllocator &allocator) if (OB_ISNULL(col_stat_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("col stat is not given", K(ret), K(col_stat_)); - } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj, allocator))) { + } else if (OB_FAIL(ObDbmsStatsUtils::truncate_string_for_opt_stats(obj, allocator))) { LOG_WARN("fail to truncate string", K(ret)); } else { col_stat_->set_max_value(obj); @@ -147,7 +148,7 @@ int ObStatMinValue::decode(ObObj &obj, ObIAllocator &allocator) if (OB_ISNULL(col_stat_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("col stat is not given", K(ret), K(col_stat_)); - } else if (OB_FAIL(ObDbmsStatsUtils::shadow_truncate_string_for_opt_stats(obj, allocator))) { + } else if (OB_FAIL(ObDbmsStatsUtils::truncate_string_for_opt_stats(obj, allocator))) { LOG_WARN("fail to truncate string", K(ret)); } else { col_stat_->set_min_value(obj); @@ -227,7 +228,6 @@ bool ObStatTopKHist::is_needed() const { return NULL != col_param_ && col_param_->need_basic_stat() && - !col_param_->is_no_need_histogram() && col_param_->bucket_num_ > 1; } @@ -250,12 +250,13 @@ int ObStatTopKHist::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) double err_rate = 1.0 / (1000 * (bkt_num / MIN_BUCKET_SIZE)); if (OB_SUCC(ret)) { if (OB_FAIL(databuff_printf(buf, buf_len, pos, - lib::is_oracle_mode() ? " TOP_K_FRE_HIST(%lf, \"%.*s\", %ld)" : - " TOP_K_FRE_HIST(%lf, `%.*s`, %ld)", + lib::is_oracle_mode() ? " TOP_K_FRE_HIST(%lf, \"%.*s\", %ld, %ld)" : + " TOP_K_FRE_HIST(%lf, `%.*s`, %ld, %ld)", err_rate, col_param_->column_name_.length(), col_param_->column_name_.ptr(), - col_param_->bucket_num_))) { + col_param_->bucket_num_, + max_disuse_cnt_))) { LOG_WARN("failed to print buf topk hist expr", K(ret)); } } @@ -701,10 +702,6 @@ int ObStatHybridHist::gen_expr(char *buf, const int64_t buf_len, int64_t &pos) if (OB_ISNULL(col_param_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column param is null", K(ret), K(col_param_)); - } else if (is_null_item_) { - if (OB_FAIL(databuff_printf(buf, buf_len, pos, " NULL"))) { - LOG_WARN("failed to print buf", K(ret)); - } else {/*do nothing*/} } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, lib::is_oracle_mode() ? " HYBRID_HIST(\"%.*s\", %ld)" : " HYBRID_HIST(`%.*s`, %ld)", @@ -744,6 +741,11 @@ int ObStatHybridHist::decode(ObObj &obj, ObIAllocator &allocator) col_stat_->get_histogram().set_sample_size(hybrid_hist.get_total_count()); col_stat_->get_histogram().set_pop_frequency(hybrid_hist.get_pop_freq()); col_stat_->get_histogram().set_pop_count(hybrid_hist.get_pop_count()); + col_stat_->get_histogram().calc_density(ObHistType::HYBIRD, + hybrid_hist.get_total_count(), + hybrid_hist.get_pop_freq(), + col_stat_->get_num_distinct(), + hybrid_hist.get_pop_count()); LOG_TRACE("succeed to build hybrid hist", K(hybrid_hist), K(col_stat_->get_histogram())); } } diff --git a/src/share/stat/ob_stat_item.h b/src/share/stat/ob_stat_item.h index 036e5fa9bc..c19681c242 100644 --- a/src/share/stat/ob_stat_item.h +++ b/src/share/stat/ob_stat_item.h @@ -14,12 +14,12 @@ #define OB_STAT_ITEM_H #include "share/stat/ob_stat_define.h" #include "share/stat/ob_opt_column_stat.h" -#include "share/stat/ob_topk_hist_estimator.h" namespace oceanbase { namespace common { class ObOptTableStat; +class ObTopkItem; /** * @brief The ObStatItem class @@ -62,14 +62,12 @@ public: class ObStatTabItem : public ObStatItem { public: - ObStatTabItem() : tab_param_(NULL), tab_stat_(NULL) {} - ObStatTabItem(const ObTableStatParam *param, - ObOptTableStat *stat) : - tab_param_(param), tab_stat_(stat) + ObStatTabItem() : tab_stat_(NULL) {} + ObStatTabItem(ObOptTableStat *stat) : + tab_stat_(stat) {} protected: - const ObTableStatParam *tab_param_; ObOptTableStat *tab_stat_; }; @@ -77,9 +75,8 @@ class ObStatRowCount : public ObStatTabItem { public: ObStatRowCount() {} - ObStatRowCount(const ObTableStatParam *param, - ObOptTableStat *stat) : - ObStatTabItem(param, stat) + ObStatRowCount(ObOptTableStat *stat) : + ObStatTabItem(stat) {} virtual int gen_expr(char *buf, const int64_t buf_len, int64_t &pos) override; virtual int decode(ObObj &obj) override; @@ -89,10 +86,9 @@ class ObStatAvgRowLen : public ObStatTabItem { public: ObStatAvgRowLen() : col_stats_(NULL) {} - ObStatAvgRowLen(const ObTableStatParam *param, - ObOptTableStat *stat, + ObStatAvgRowLen(ObOptTableStat *stat, ObIArray &col_stats) : - ObStatTabItem(param, stat), + ObStatTabItem(stat), col_stats_(&col_stats) {} virtual int gen_expr(char *buf, const int64_t buf_len, int64_t &pos) override; @@ -229,12 +225,14 @@ public: class ObStatTopKHist : public ObStatColItem { public: - ObStatTopKHist() : tab_stat_(NULL) {} + ObStatTopKHist() : ObStatColItem(), tab_stat_(NULL), max_disuse_cnt_(0) {} ObStatTopKHist(const ObColumnStatParam *param, ObOptTableStat *tab_stat, - ObOptColumnStat *stat) : + ObOptColumnStat *stat, + int64_t max_disuse_cnt) : ObStatColItem(param, stat), - tab_stat_(tab_stat) + tab_stat_(tab_stat), + max_disuse_cnt_(max_disuse_cnt) {} static int build_histogram_from_topk_items(ObIAllocator &allocator, @@ -264,17 +262,17 @@ public: virtual int decode(ObObj &obj, ObIAllocator &allocator) override; protected: ObOptTableStat *tab_stat_; + int64_t max_disuse_cnt_; }; class ObPartitionId : public ObStatTabItem { public: ObPartitionId() : calc_partition_id_str_(), partition_id_(common::OB_INVALID_ID) {} - ObPartitionId(const ObTableStatParam *param, - ObOptTableStat *stat, + ObPartitionId(ObOptTableStat *stat, ObString &calc_part_id_str, int64_t partition_id) : - ObStatTabItem(param, stat), + ObStatTabItem(stat), calc_partition_id_str_(calc_part_id_str), partition_id_(partition_id) {} @@ -288,17 +286,14 @@ class ObPartitionId : public ObStatTabItem class ObStatHybridHist : public ObStatColItem { public: - ObStatHybridHist() : is_null_item_(false) {} + ObStatHybridHist() {} ObStatHybridHist(const ObColumnStatParam *param, - ObOptColumnStat *stat, - bool is_null_item = false) : - ObStatColItem(param, stat), is_null_item_(is_null_item) + ObOptColumnStat *stat) : + ObStatColItem(param, stat) {} virtual int gen_expr(char *buf, const int64_t buf_len, int64_t &pos) override; virtual int decode(ObObj &obj, ObIAllocator &allocator) override; -private: - bool is_null_item_; }; class ObGlobalTableStat @@ -365,7 +360,7 @@ class ObGlobalNdvEval { const int64_t NUM_LLC_BUCKET = ObOptColumnStat::NUM_LLC_BUCKET; public: - ObGlobalNdvEval() : global_ndv_(-1), part_cnt_(0) { + ObGlobalNdvEval() : global_ndv_(0), part_cnt_(0) { MEMSET(global_llc_bitmap_, 0, ObOptColumnStat::NUM_LLC_BUCKET); } void add(int64_t ndv, const char *llc_bitmap); diff --git a/src/share/stat/ob_stats_estimator.cpp b/src/share/stat/ob_stats_estimator.cpp index ddd93fba1e..29b850ce1c 100644 --- a/src/share/stat/ob_stats_estimator.cpp +++ b/src/share/stat/ob_stats_estimator.cpp @@ -63,8 +63,8 @@ int ObStatsEstimator::pack(ObSqlString &raw_sql_str) if (OB_FAIL(gen_select_filed())) { LOG_WARN("failed to generate select filed", K(ret)); } else if (OB_FAIL(raw_sql_str.append_fmt(is_oracle_mode() ? - "SELECT %.*s %.*s FROM \"%.*s\".\"%.*s\" %.*s %.*s %.*s %.*s" : - "SELECT %.*s %.*s FROM `%.*s`.`%.*s` %.*s %.*s %.*s %.*s" , + "SELECT %.*s %.*s FROM \"%.*s\".\"%.*s\" %.*s %.*s %.*s %.*s %.*s" : + "SELECT %.*s %.*s FROM `%.*s`.`%.*s` %.*s %.*s %.*s %.*s %.*s" , other_hints_.length(), other_hints_.ptr(), static_cast(select_fields_.length()), @@ -77,6 +77,8 @@ int ObStatsEstimator::pack(ObSqlString &raw_sql_str) partition_string_.ptr(), sample_hint_.length(), sample_hint_.ptr(), + current_scn_string_.length(), + current_scn_string_.ptr(), where_string_.length(), where_string_.ptr(), group_by_string_.length(), @@ -90,8 +92,7 @@ int ObStatsEstimator::pack(ObSqlString &raw_sql_str) int ObStatsEstimator::fill_sample_info(common::ObIAllocator &alloc, double est_percent, - bool block_sample, - ObString &sample_hint) + bool block_sample) { int ret = OB_SUCCESS; if (est_percent>= 0.000001 && est_percent <= 100.0) { @@ -111,7 +112,7 @@ int ObStatsEstimator::fill_sample_info(common::ObIAllocator &alloc, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(real_len)); } else { - sample_hint.assign_ptr(buf, real_len); + sample_hint_.assign_ptr(buf, real_len); LOG_TRACE("succeed to add sample string", K(buf), K(real_len)); } } @@ -131,8 +132,7 @@ int ObStatsEstimator::fill_sample_info(common::ObIAllocator &alloc, //do nothing } else if (OB_FAIL(fill_sample_info(alloc, sample_info.sample_value_, - sample_info.is_block_sample_, - sample_hint_))) { + sample_info.is_block_sample_))) { LOG_WARN("failed to fill sample info", K(ret)); } else { sample_value_ = sample_info.sample_value_; @@ -140,6 +140,32 @@ int ObStatsEstimator::fill_sample_info(common::ObIAllocator &alloc, return ret; } +int ObStatsEstimator::fill_specify_scn_info(common::ObIAllocator &alloc, + uint64_t sepcify_scn) +{ + int ret = OB_SUCCESS; + if (sepcify_scn > 0) { + const char* fmt_str = lib::is_oracle_mode() ? "as of scn %lu" : "as of snapshot %lu"; + char *buf = NULL; + int64_t buf_len = strlen(fmt_str) + 30;//uint64_t:0 ~ 18446744073709551615,length no more than 20 + int64_t real_len = -1; + if (OB_ISNULL(buf = static_cast(alloc.alloc(buf_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), K(buf), K(buf_len)); + } else { + real_len = sprintf(buf, fmt_str, sepcify_scn); + if (OB_UNLIKELY(real_len <= 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(real_len)); + } else { + current_scn_string_.assign_ptr(buf, real_len); + LOG_TRACE("succeed to fill specify scn info", K(current_scn_string_)); + } + } + } + return ret; +} + int ObStatsEstimator::fill_parallel_info(common::ObIAllocator &alloc, int64_t degree) { @@ -238,20 +264,49 @@ int ObStatsEstimator::add_hint(const ObString &hint_str, } int ObStatsEstimator::fill_partition_info(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra) + const ObIArray &partition_infos) { int ret = OB_SUCCESS; - PartInfo part_info; - if (extra.type_ == TABLE_LEVEL) { - // do nothing - } else if (OB_FAIL(ObDbmsStatsUtils::get_part_info(param, extra, part_info))) { - LOG_WARN("failed to get part info", K(ret)); - } else if (OB_UNLIKELY(part_info.part_name_.empty())) { + ObSqlString tmp_part_str; + if (OB_UNLIKELY(partition_infos.count() <= 1)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("partition name is empty", K(ret), K(part_info)); + LOG_WARN("get unexpected error", K(ret), K(partition_infos)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < partition_infos.count(); ++i) { + if (OB_UNLIKELY(partition_infos.at(i).part_name_.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("partition name is empty", K(ret), K(partition_infos), K(i)); + } else if (OB_FAIL(tmp_part_str.append_fmt(lib::is_oracle_mode() ? "%s\"%.*s\"%s" : "%s`%.*s`%s", + i == 0 ? "PARTITION(" : " ", + partition_infos.at(i).part_name_.length(), + partition_infos.at(i).part_name_.ptr(), + i == partition_infos.count() - 1 ? ")" : ", "))) { + LOG_WARN("failed to append", K(ret)); + } else {/*do nothing*/} + } + if (OB_SUCC(ret)) { + char *buf = NULL; + int64_t buf_len = tmp_part_str.length(); + if (OB_ISNULL(buf = static_cast(allocator.alloc(tmp_part_str.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret), K(tmp_part_str.length())); + } else { + MEMCPY(buf, tmp_part_str.ptr(), tmp_part_str.length()); + partition_string_.assign(buf, tmp_part_str.length()); + } + } + } + return ret; +} + +int ObStatsEstimator::fill_partition_info(ObIAllocator &allocator, + const ObString &part_name) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(part_name.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("partition name is empty", K(ret), K(part_name)); } else { - const ObString &part_name = part_info.part_name_; const char *fmt_str = lib::is_oracle_mode() ? "PARTITION (\"%.*s\")" : "PARTITION (`%.*s`)"; char *buf = NULL; const int64_t len = strlen(fmt_str) + part_name.length(); @@ -274,8 +329,7 @@ int ObStatsEstimator::fill_partition_info(ObIAllocator &allocator, //Specify the partition name or non-partition table don't use group by. int ObStatsEstimator::fill_group_by_info(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra, + const ObOptStatGatherParam ¶m, ObString &calc_part_id_str) { int ret = OB_SUCCESS; @@ -283,27 +337,22 @@ int ObStatsEstimator::fill_group_by_info(ObIAllocator &allocator, : "GROUP BY CALC_PARTITION_ID(`%.*s`, %.*s)"; char *buf = NULL; ObString type_str; - if (extra.type_ == PARTITION_LEVEL) { + if (param.stat_level_ == PARTITION_LEVEL) { type_str = ObString(4, "PART"); - } else if (extra.type_ == SUBPARTITION_LEVEL) { + } else if (param.stat_level_ == SUBPARTITION_LEVEL) { type_str = ObString(7, "SUBPART"); } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected type", K(extra.type_), K(ret)); + LOG_WARN("get unexpected type", K(param.stat_level_), K(ret)); } if (OB_SUCC(ret)) { - const int64_t len = strlen(fmt_str) + - (param.is_index_stat_ ? param.data_table_name_.length() : param.tab_name_.length()) + - type_str.length(); + const int64_t len = strlen(fmt_str) + param.tab_name_.length() + type_str.length(); int32_t real_len = -1; if (OB_ISNULL(buf = static_cast(allocator.alloc(len)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory", K(ret), K(len)); } else { - real_len = sprintf(buf, fmt_str, param.is_index_stat_ ? param.data_table_name_.length() : param.tab_name_.length(), - param.is_index_stat_ ? param.data_table_name_.ptr() : param.tab_name_.ptr(), - type_str.length(), - type_str.ptr()); + real_len = sprintf(buf, fmt_str, param.tab_name_.length(), param.tab_name_.ptr(), type_str.length(), type_str.ptr()); if (OB_UNLIKELY(real_len < 0 || real_len >= len)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to print partition hint", K(ret), K(real_len), K(len)); @@ -322,7 +371,7 @@ int ObStatsEstimator::fill_group_by_info(ObIAllocator &allocator, int ObStatsEstimator::do_estimate(uint64_t tenant_id, const ObString &raw_sql, - CopyStatType copy_type, + bool need_copy_basic_stat, ObOptStat &src_opt_stat, ObIArray &dst_opt_stats) { @@ -386,11 +435,8 @@ int ObStatsEstimator::do_estimate(uint64_t tenant_id, if (OB_SUCC(ret)) { if (OB_FAIL(decode(allocator_))) { LOG_WARN("failed to decode results", K(ret)); - } else if (copy_type == COPY_ALL_STAT && - OB_FAIL(copy_opt_stat(src_opt_stat, dst_opt_stats))) { - LOG_WARN("failed to copy stat to target opt stat", K(ret)); - } else if (copy_type == COPY_HYBRID_HIST_STAT && - OB_FAIL(copy_hybrid_hist_stat(src_opt_stat, dst_opt_stats))) { + } else if (need_copy_basic_stat && + OB_FAIL(copy_basic_opt_stat(src_opt_stat, dst_opt_stats))) { LOG_WARN("failed to copy stat to target opt stat", K(ret)); } else { results_.reset(); @@ -433,8 +479,8 @@ int ObStatsEstimator::decode(ObIAllocator &allocator) return ret; } -int ObStatsEstimator::copy_opt_stat(ObOptStat &src_opt_stat, - ObIArray &dst_opt_stats) +int ObStatsEstimator::copy_basic_opt_stat(ObOptStat &src_opt_stat, + ObIArray &dst_opt_stats) { int ret = OB_SUCCESS; ObOptTableStat *tmp_tab_stat = src_opt_stat.table_stat_; @@ -458,23 +504,22 @@ int ObStatsEstimator::copy_opt_stat(ObOptStat &src_opt_stat, dst_opt_stats.at(i).table_stat_->set_row_count(row_cnt); dst_opt_stats.at(i).table_stat_->set_avg_row_size(tmp_tab_stat->get_avg_row_size()); dst_opt_stats.at(i).table_stat_->set_sample_size(tmp_tab_stat->get_row_count()); - if (OB_FAIL(copy_col_stats(tmp_tab_stat->get_row_count(), row_cnt, tmp_col_stats, dst_opt_stats.at(i).column_stats_))) { + if (OB_FAIL(copy_basic_col_stats(tmp_tab_stat->get_row_count(), row_cnt, tmp_col_stats, dst_opt_stats.at(i).column_stats_))) { LOG_WARN("failed to copy col stat", K(ret)); } else {/*do nothing*/} } else {/*do nothing*/} } if (OB_SUCC(ret) && !find_it) { - LOG_TRACE("this partition id isn't in needed partition ids, no need gather stats", - K(partition_id)); + LOG_TRACE("this partition id isn't in needed partition ids, no need gather stats",K(partition_id)); } } return ret; } -int ObStatsEstimator::copy_col_stats(const int64_t cur_row_cnt, - const int64_t total_row_cnt, - ObIArray &src_col_stats, - ObIArray &dst_col_stats) +int ObStatsEstimator::copy_basic_col_stats(const int64_t cur_row_cnt, + const int64_t total_row_cnt, + ObIArray &src_col_stats, + ObIArray &dst_col_stats) { int ret = OB_SUCCESS; if (OB_UNLIKELY(src_col_stats.count() != dst_col_stats.count())) { @@ -497,6 +542,7 @@ int ObStatsEstimator::copy_col_stats(const int64_t cur_row_cnt, dst_col_stats.at(i)->set_max_value(src_col_stats.at(i)->get_max_value()); dst_col_stats.at(i)->set_min_value(src_col_stats.at(i)->get_min_value()); dst_col_stats.at(i)->set_num_not_null(num_not_null); + dst_col_stats.at(i)->get_histogram().set_sample_size(src_col_stats.at(i)->get_num_not_null()); dst_col_stats.at(i)->set_num_null(num_null); dst_col_stats.at(i)->set_num_distinct(num_distinct); dst_col_stats.at(i)->set_avg_len(src_col_stats.at(i)->get_avg_len()); @@ -514,16 +560,7 @@ int ObStatsEstimator::copy_col_stats(const int64_t cur_row_cnt, src_col_stats.at(i)->get_llc_bitmap(), src_col_stats.at(i)->get_llc_bitmap_size()); dst_col_stats.at(i)->set_llc_bitmap_size(src_col_stats.at(i)->get_llc_bitmap_size()); - ObHistogram &src_hist = src_col_stats.at(i)->get_histogram(); - dst_col_stats.at(i)->get_histogram().set_type(src_hist.get_type()); - dst_col_stats.at(i)->get_histogram().set_sample_size(src_col_stats.at(i)->get_num_not_null()); - dst_col_stats.at(i)->get_histogram().set_density(src_hist.get_density()); - dst_col_stats.at(i)->get_histogram().set_bucket_cnt(src_hist.get_bucket_cnt()); - if (OB_FAIL(dst_col_stats.at(i)->get_histogram().get_buckets().assign(src_hist.get_buckets()))) { - LOG_WARN("failed to assign buckets", K(ret)); - } else { - LOG_TRACE("Succeed to copy col stat", K(*dst_col_stats.at(i)), K(*src_col_stats.at(i))); - } + LOG_TRACE("Succeed to copy basic col stats", K(*dst_col_stats.at(i)), K(*src_col_stats.at(i))); } } } @@ -531,73 +568,5 @@ int ObStatsEstimator::copy_col_stats(const int64_t cur_row_cnt, return ret; } -int ObStatsEstimator::copy_hybrid_hist_stat(ObOptStat &src_opt_stat, - ObIArray &dst_opt_stats) -{ - int ret = OB_SUCCESS; - bool find_it = false; - if (OB_ISNULL(src_opt_stat.table_stat_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(src_opt_stat.table_stat_)); - } - for (int64_t i = 0; OB_SUCC(ret) && !find_it && i < dst_opt_stats.count(); ++i) { - if (OB_ISNULL(dst_opt_stats.at(i).table_stat_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(dst_opt_stats.at(i).table_stat_)); - } else if (dst_opt_stats.at(i).table_stat_->get_partition_id() == - src_opt_stat.table_stat_->get_partition_id()) { - find_it = true; - if (OB_UNLIKELY(dst_opt_stats.at(i).column_stats_.count() != - src_opt_stat.column_stats_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(dst_opt_stats.at(i).column_stats_.count()), - K(src_opt_stat.column_stats_.count())); - } else { - for (int64_t j = 0; OB_SUCC(ret) && j < src_opt_stat.column_stats_.count(); ++j) { - ObOptColumnStat *src_col_stat = NULL; - ObOptColumnStat *dst_col_stat = NULL; - bool is_skewed = false; - if (OB_ISNULL(src_col_stat = src_opt_stat.column_stats_.at(j)) || - OB_ISNULL(dst_col_stat = dst_opt_stats.at(i).column_stats_.at(j))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(src_col_stat), K(dst_col_stat), K(j)); - } else if (!dst_col_stat->get_histogram().is_hybrid() || - dst_col_stat->get_histogram().is_valid() || - !src_col_stat->get_histogram().is_valid()) { - LOG_TRACE("no need copy histogram", K(src_col_stat->get_histogram()), - K(dst_col_stat->get_histogram()), K(i), K(j)); - if (!src_col_stat->get_histogram().is_valid() && - !dst_col_stat->get_histogram().is_valid()) { - dst_col_stat->get_histogram().reset(); - dst_col_stat->get_histogram().set_sample_size(dst_col_stat->get_num_not_null()); - } - } else { - ObHistogram &src_hist = src_col_stat->get_histogram(); - dst_col_stat->get_histogram().set_type(src_hist.get_type()); - dst_col_stat->get_histogram().set_sample_size(src_hist.get_sample_size()); - dst_col_stat->get_histogram().set_bucket_cnt(src_hist.get_bucket_cnt()); - dst_col_stat->get_histogram().calc_density(ObHistType::HYBIRD, - src_hist.get_sample_size(), - src_hist.get_pop_frequency(), - dst_col_stat->get_num_distinct(), - src_hist.get_pop_count()); - if (OB_FAIL(dst_col_stat->get_histogram().get_buckets().assign(src_hist.get_buckets()))) { - LOG_WARN("failed to assign buckets", K(ret)); - } else { - LOG_TRACE("Succeed to copy histogram", K(*dst_col_stat), K(i), K(j)); - } - } - } - } - } else {/*do nothing*/} - } - if (OB_SUCC(ret) && !find_it) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error, can't find specify partition id", K(ret), K(find_it), - K(*src_opt_stat.table_stat_)); - } - return ret; -} - } // end of common } // end of ocenabase diff --git a/src/share/stat/ob_stats_estimator.h b/src/share/stat/ob_stats_estimator.h index 1666764a2a..00918fd96e 100644 --- a/src/share/stat/ob_stats_estimator.h +++ b/src/share/stat/ob_stats_estimator.h @@ -26,12 +26,6 @@ using namespace sql; namespace common { -enum CopyStatType -{ - COPY_ALL_STAT, - COPY_HYBRID_HIST_STAT -}; - class ObStatsEstimator { public: @@ -49,7 +43,7 @@ protected: int do_estimate(uint64_t tenant_id, const ObString &raw_sql, - CopyStatType copy_type, + bool need_copy_basic_stat, ObOptStat &src_opt_stat, ObIArray &dst_opt_stats); @@ -67,8 +61,7 @@ protected: int fill_sample_info(common::ObIAllocator &alloc, double est_percent, - bool block_sample, - ObString &sample_hint); + bool block_sample); int fill_sample_info(common::ObIAllocator &alloc, const ObAnalyzeSampleInfo &sample_info); @@ -80,30 +73,32 @@ protected: const int64_t duration_timeout); int fill_partition_info(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra); + const ObString &part_nam); + + int fill_partition_info(ObIAllocator &allocator, + const ObIArray &partition_infos); int add_hint(const ObString &hint_str, common::ObIAllocator &alloc); int fill_group_by_info(ObIAllocator &allocator, - const ObTableStatParam ¶m, - const ObExtraParam &extra, + const ObOptStatGatherParam ¶m, ObString &calc_part_id_str); void reset_select_items() { stat_items_.reset(); select_fields_.reset(); } + void reset_sample_hint() { sample_hint_.reset(); } + void reset_other_hint() { other_hints_.reset(); } + + int fill_specify_scn_info(common::ObIAllocator &alloc, uint64_t sepcify_scn); private: - int copy_opt_stat(ObOptStat &src_opt_stat, - ObIArray &dst_opt_stats); + int copy_basic_opt_stat(ObOptStat &src_opt_stat, + ObIArray &dst_opt_stats); - int copy_hybrid_hist_stat(ObOptStat &src_opt_stat, - ObIArray &dst_opt_stats); - - int copy_col_stats(const int64_t cur_row_cnt, - const int64_t total_row_cnt, - ObIArray &src_col_stats, - ObIArray &dst_col_stats); + int copy_basic_col_stats(const int64_t cur_row_cnt, + const int64_t total_row_cnt, + ObIArray &src_col_stats, + ObIArray &dst_col_stats); protected: @@ -123,6 +118,7 @@ protected: ObArray stat_items_; ObArray results_; double sample_value_; + ObString current_scn_string_; }; diff --git a/src/share/stat/ob_topk_hist_estimator.cpp b/src/share/stat/ob_topk_hist_estimator.cpp index 154a909c98..555942d79b 100644 --- a/src/share/stat/ob_topk_hist_estimator.cpp +++ b/src/share/stat/ob_topk_hist_estimator.cpp @@ -12,6 +12,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "ob_topk_hist_estimator.h" +#include "share/stat/ob_dbms_stats_utils.h" #include namespace oceanbase @@ -183,7 +184,7 @@ int ObTopKFrequencyHistograms::merge_distribute_top_k_fre_items(const ObObj &obj bucket_num_ = N_ / window_size_; if (bucket_num_ > bucket_num_old) { if (OB_FAIL(shrink_topk_items())) { - LOG_WARN("failed to try shrinking topk fre item"); + LOG_WARN("failed to try shrinking topk fre item", K(ret)); } } } @@ -232,7 +233,7 @@ int ObTopKFrequencyHistograms::add_top_k_frequency_item(uint64_t datum_hash, con return ret; } -int ObTopKFrequencyHistograms::add_top_k_frequency_item(const ObObj &obj) +int ObTopKFrequencyHistograms::add_top_k_frequency_item(const ObObj &obj, int64_t repeat_cnt) { int ret = OB_SUCCESS; if (OB_UNLIKELY(window_size_ <= 0 || @@ -251,7 +252,6 @@ int ObTopKFrequencyHistograms::add_top_k_frequency_item(const ObObj &obj) } } if (OB_SUCC(ret)) { - ++ N_; if (!by_pass_) { ObTopkBaseItem *item = NULL; uint64_t obj_hash = 0; @@ -259,13 +259,18 @@ int ObTopKFrequencyHistograms::add_top_k_frequency_item(const ObObj &obj) if (OB_FAIL(obj.hash_murmur(obj_hash, seed))) { LOG_WARN("fail to do hash", K(ret)); } else if (NULL != (item = get_entry(obj_hash))) { - item->fre_times_ ++; - } else if (OB_FAIL(add_entry(obj_hash, obj, 1L, bucket_num_ - 1))) { + item->fre_times_ += repeat_cnt; + } else if (OB_FAIL(add_entry(obj_hash, obj, repeat_cnt, bucket_num_ - 1))) { LOG_WARN("failed to add new entry", K(ret)); } - if (OB_SUCC(ret) && N_ % window_size_ == 0) { - if (OB_FAIL(shrink_topk_items())) { - LOG_WARN("failed to try shrinking topk fre item"); + if (OB_SUCC(ret)) { + int64_t bucket_num_old = bucket_num_; + N_ = N_ + repeat_cnt; + bucket_num_ = N_ > window_size_ ? N_ / window_size_ : bucket_num_; + if (bucket_num_ > bucket_num_old) { + if (OB_FAIL(shrink_topk_items())) { + LOG_WARN("failed to try shrinking topk fre item", K(ret)); + } } } } @@ -435,7 +440,9 @@ int ObTopKFrequencyHistograms::shrink_topk_items() bool ObTopKFrequencyHistograms::is_satisfied_by_pass() const { bool is_satisfied = false; - if (N_ % (window_size_ * 100) == 0) { + if (max_disuse_cnt_ > 0) { + is_satisfied = disuse_cnt_ >= max_disuse_cnt_; + } else if (N_ % (window_size_ * 100) == 0) { if (1.0 * disuse_cnt_ / N_ > get_current_min_topk_ratio()) { is_satisfied = true; LOG_INFO("topk hist is statisfied by pass", K(disuse_cnt_), K(N_), K(get_current_min_topk_ratio())); @@ -621,5 +628,88 @@ OB_DEF_DESERIALIZE(ObTopKFrequencyHistograms) OB_UNIS_DECODE(disuse_cnt_); return ret; } + +ObTopkHistEstimator::ObTopkHistEstimator(ObExecContext &ctx, ObIAllocator &allocator) + : ObBasicStatsEstimator(ctx, allocator) +{} + +int ObTopkHistEstimator::estimate(const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat) +{ + int ret = OB_SUCCESS; + ObArenaAllocator allocator("ObTopkHistEst", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); + ObSqlString raw_sql; + int64_t duration_time = -1; + ObSEArray tmp_opt_stats; + if (OB_FAIL(add_topk_hist_stat_items(param.column_params_, opt_stat))) { + LOG_WARN("failed to add topk hist stat items", K(ret)); + } else if (get_item_size() <= 0) { + //no need topk histogram item. + } else if (OB_FAIL(fill_hints(allocator, param.tab_name_, param.gather_vectorize_))) { + LOG_WARN("failed to fill hints", K(ret)); + } else if (OB_FAIL(add_from_table(param.db_name_, param.tab_name_))) { + LOG_WARN("failed to add from table", K(ret)); + } else if (OB_UNLIKELY(param.partition_infos_.count() > 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(param)); + } else if (!param.partition_infos_.empty() && + OB_FAIL(fill_partition_info(allocator, param.partition_infos_.at(0).part_name_))) { + LOG_WARN("failed to add partition info", K(ret)); + } else if (OB_FAIL(fill_parallel_info(allocator, param.degree_))) { + LOG_WARN("failed to add query sql parallel info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsUtils::get_valid_duration_time(param.gather_start_time_, + param.max_duration_time_, + duration_time))) { + LOG_WARN("failed to get valid duration time", K(ret)); + } else if (OB_FAIL(fill_query_timeout_info(allocator, duration_time))) { + LOG_WARN("failed to fill query timeout info", K(ret)); + } else if (OB_FAIL(fill_sample_info(allocator, param.sample_info_))) { + LOG_WARN("failed to fill sample info", K(ret)); + } else if (OB_FAIL(fill_specify_scn_info(allocator, param.sepcify_scn_))) { + LOG_WARN("failed to fill specify scn info", K(ret)); + } else if (OB_FAIL(pack(raw_sql))) { + LOG_WARN("failed to pack raw sql", K(ret)); + } else if (OB_FAIL(tmp_opt_stats.push_back(opt_stat))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(do_estimate(param.tenant_id_, raw_sql.string(), false, + opt_stat, tmp_opt_stats))) { + LOG_WARN("failed to evaluate basic stats", K(ret)); + } else { + LOG_TRACE("succeed to gather topk histogram", K(opt_stat.column_stats_)); + } + return ret; +} + +int ObTopkHistEstimator::add_topk_hist_stat_items(const ObIArray &column_params, + ObOptStat &opt_stat) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(opt_stat.table_stat_) || + OB_UNLIKELY(opt_stat.column_stats_.count() != column_params.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(opt_stat), K(column_params)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < column_params.count(); ++i) { + const ObColumnStatParam *col_param = &column_params.at(i); + if (OB_ISNULL(opt_stat.column_stats_.at(i)) || + OB_UNLIKELY(col_param->column_id_ != opt_stat.column_stats_.at(i)->get_column_id())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), KPC(opt_stat.column_stats_.at(i)), KPC(col_param)); + } else if (opt_stat.column_stats_.at(i)->get_histogram().get_type() != ObHistType::TOP_FREQUENCY) { + //do nothing + } else { + int64_t max_disuse_cnt = std::ceil(opt_stat.column_stats_.at(i)->get_num_not_null() * 1.0 / col_param->bucket_num_); + if (OB_FAIL(add_stat_item(ObStatTopKHist(col_param, + opt_stat.table_stat_, + opt_stat.column_stats_.at(i), + max_disuse_cnt)))) { + LOG_WARN("failed to add statistic item", K(ret)); + } else {/*do noting*/} + } + } + } + return ret; +} + } } diff --git a/src/share/stat/ob_topk_hist_estimator.h b/src/share/stat/ob_topk_hist_estimator.h index 9cdcb19976..b37e645ba8 100644 --- a/src/share/stat/ob_topk_hist_estimator.h +++ b/src/share/stat/ob_topk_hist_estimator.h @@ -15,6 +15,7 @@ #include "share/stat/ob_stat_define.h" #include "share/rc/ob_tenant_base.h" +#include "share/stat/ob_basic_stats_estimator.h" namespace oceanbase { @@ -91,7 +92,8 @@ class ObTopKFrequencyHistograms obj_buf2_("OptTopObjbuf2"), by_pass_(false), disuse_cnt_(0), - obj_memory_limit_(0) + obj_memory_limit_(0), + max_disuse_cnt_(0) { topk_buf_.set_tenant_id(MTL_ID()); obj_buf1_.set_tenant_id(MTL_ID()); @@ -104,7 +106,7 @@ class ObTopKFrequencyHistograms int create_topk_fre_items(const ObObjMeta *obj_meta = NULL); int sort_topk_fre_items(ObIArray &items); int shrink_topk_items(); - int add_top_k_frequency_item(const ObObj &obj); + int add_top_k_frequency_item(const ObObj &obj, int64_t repeat_cnt); int add_top_k_frequency_item(uint64_t datum_hash, const ObDatum &datum); int merge_distribute_top_k_fre_items(const ObObj &obj); void set_window_size(int64_t window_size) { window_size_ = window_size; } @@ -126,6 +128,8 @@ class ObTopKFrequencyHistograms double get_current_min_topk_ratio() const; void set_the_obj_memory_use_limit(); int64_t get_max_reserved_item_size() const { return item_size_ * 1.5; } + void set_max_disuse_cnt(int64_t max_disuse_cnt) { max_disuse_cnt_ = max_disuse_cnt; } + //int64_t get_max_disuse_cnt() ObTopkItem *alloc_topk_item(); ObTopkDatumItem *alloc_topk_datum_item(); ObIAllocator &get_allocator() @@ -179,6 +183,19 @@ class ObTopKFrequencyHistograms bool by_pass_; int64_t disuse_cnt_; int64_t obj_memory_limit_; + int64_t max_disuse_cnt_; +}; + +class ObTopkHistEstimator : public ObBasicStatsEstimator +{ +public: + explicit ObTopkHistEstimator(ObExecContext &ctx, ObIAllocator &allocator); + + int estimate(const ObOptStatGatherParam ¶m, + ObOptStat &opt_stat); +private: + int add_topk_hist_stat_items(const ObIArray &column_params, + ObOptStat &opt_stat); }; } // end of namespace common diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index 3ee2dfec00..fd84d8e51d 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -6414,32 +6414,39 @@ int ObStaticEngineCG::generate_top_fre_hist_expr_operator(ObAggFunRawExpr &raw_e { int ret = OB_SUCCESS; if (OB_UNLIKELY(T_FUN_TOP_FRE_HIST != raw_expr.get_expr_type() || - raw_expr.get_param_count() != 3)) { + raw_expr.get_param_count() != 4)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get invalid argument", K(raw_expr), K(ret)); } else { ObRawExpr *win_raw_expr = raw_expr.get_param_expr(0); ObRawExpr *param_raw_expr = raw_expr.get_param_expr(1); ObRawExpr *item_raw_expr = raw_expr.get_param_expr(2); + ObRawExpr *max_disuse_raw_expr = raw_expr.get_param_expr(3); ObExpr *win_expr = NULL; ObExpr *item_expr = NULL; + ObExpr *max_disuse_expr = NULL; raw_expr.get_real_param_exprs_for_update().reset(); - if (OB_ISNULL(win_raw_expr) || OB_ISNULL(param_raw_expr) || OB_ISNULL(item_raw_expr)) { + if (OB_ISNULL(win_raw_expr) || OB_ISNULL(param_raw_expr) || + OB_ISNULL(item_raw_expr) || OB_ISNULL(max_disuse_raw_expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(win_raw_expr), K(param_raw_expr), K(item_raw_expr), K(ret)); + LOG_WARN("get unexpected null", K(win_raw_expr), K(param_raw_expr), + K(item_raw_expr), K(max_disuse_raw_expr), K(ret)); } else if (OB_FAIL(generate_rt_expr(*win_raw_expr, win_expr)) || - OB_FAIL(generate_rt_expr(*item_raw_expr, item_expr))) { - LOG_WARN("failed to generate_rt_expr", K(ret), K(*win_raw_expr), K(*item_raw_expr)); - } else if (OB_ISNULL(win_expr) || OB_ISNULL(item_expr)) { + OB_FAIL(generate_rt_expr(*item_raw_expr, item_expr)) || + OB_FAIL(generate_rt_expr(*max_disuse_raw_expr, max_disuse_expr))) { + LOG_WARN("failed to generate_rt_expr", K(ret), K(*win_raw_expr), K(*item_raw_expr), K(*max_disuse_raw_expr)); + } else if (OB_ISNULL(win_expr) || OB_ISNULL(item_expr) || OB_ISNULL(max_disuse_expr)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("expr is null ", K(ret), K(win_expr), K(item_expr)); + LOG_WARN("expr is null ", K(ret), K(win_expr), K(item_expr), K(max_disuse_expr)); } else if (OB_UNLIKELY(!win_expr->obj_meta_.is_numeric_type() || - !item_expr->obj_meta_.is_numeric_type())) { + !item_expr->obj_meta_.is_numeric_type() || + !max_disuse_expr->obj_meta_.is_numeric_type())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("expr node is null", K(ret), K(win_expr->obj_meta_), K(item_expr->obj_meta_)); + LOG_WARN("expr node is null", K(ret), K(win_expr->obj_meta_), K(item_expr->obj_meta_), K(max_disuse_expr->obj_meta_)); } else { aggr_info.window_size_param_expr_ = win_expr; aggr_info.item_size_param_expr_ = item_expr; + aggr_info.max_disuse_param_expr_ = max_disuse_expr; aggr_info.is_need_deserialize_row_ = raw_expr.is_need_deserialize_row(); if (OB_FAIL(raw_expr.add_real_param_expr(param_raw_expr))) { LOG_WARN("fail to add param expr to agg expr", K(ret)); diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index 4deb95fd50..82f337db94 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -2639,16 +2639,28 @@ int ObAggregateProcessor::prepare_aggr_result(const ObChunkDatumStore::StoredRow } else if (OB_FAIL(extra->topk_fre_hist_.merge_distribute_top_k_fre_items(obj))) { LOG_WARN("failed to process row", K(ret)); } - } else if (OB_FAIL(shadow_truncate_string_for_hist(aggr_info.param_exprs_.at(0)->obj_meta_, - const_cast(stored_row.cells()[0])))) { - LOG_WARN("failed to shadow truncate string for hist", K(ret)); } else { - ObExprHashFuncType hash_func = aggr_info.param_exprs_.at(0)->basic_funcs_->murmur_hash_; - uint64_t datum_value = 0; - if (OB_FAIL(hash_func(stored_row.cells()[0], datum_value, datum_value))) { - LOG_WARN("fail to do hash", K(ret)); - } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(datum_value, stored_row.cells()[0]))) { - LOG_WARN("failed to process row", K(ret)); + common::ObArenaAllocator tmp_alloctor("CalcTopkHist", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + const ObObjMeta obj_meta = aggr_info.param_exprs_.at(0)->obj_meta_; + ObDatum new_prev_datum; + if (!obj_meta.is_lob_storage() && + OB_FAIL(shadow_truncate_string_for_hist(obj_meta, const_cast(stored_row.cells()[0])))) { + LOG_WARN("failed to shadow truncate string for hist", K(ret)); + } else if (obj_meta.is_lob_storage() && + OB_FAIL(ObHybridHistograms::build_prefix_str_datum_for_lob(tmp_alloctor, + obj_meta, + stored_row.cells()[0], + new_prev_datum))) { + LOG_WARN("failed to build prefix str datum for lob", K(ret)); + } else { + const ObDatum &datum = obj_meta.is_lob_storage() ? new_prev_datum : stored_row.cells()[0]; + ObExprHashFuncType hash_func = aggr_info.param_exprs_.at(0)->basic_funcs_->murmur_hash_; + uint64_t datum_value = 0; + if (OB_FAIL(hash_func(datum, datum_value, datum_value))) { + LOG_WARN("fail to do hash", K(ret)); + } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(datum_value, datum))) { + LOG_WARN("failed to process row", K(ret)); + } } } break; @@ -3098,16 +3110,28 @@ int ObAggregateProcessor::process_aggr_result(const ObChunkDatumStore::StoredRow } else if (OB_FAIL(extra->topk_fre_hist_.merge_distribute_top_k_fre_items(obj))) { LOG_WARN("failed to process row", K(ret)); } - } else if (OB_FAIL(shadow_truncate_string_for_hist(aggr_info.param_exprs_.at(0)->obj_meta_, - const_cast(stored_row.cells()[0])))) { - LOG_WARN("failed to shadow truncate string for hist", K(ret)); } else { - ObExprHashFuncType hash_func = aggr_info.param_exprs_.at(0)->basic_funcs_->murmur_hash_; - uint64_t datum_value = 0; - if (OB_FAIL(hash_func(stored_row.cells()[0], datum_value, datum_value))) { - LOG_WARN("fail to do hash", K(ret)); - } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(datum_value, stored_row.cells()[0]))) { - LOG_WARN("failed to process row", K(ret)); + common::ObArenaAllocator tmp_alloctor("CalcTopkHist", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + const ObObjMeta obj_meta = aggr_info.param_exprs_.at(0)->obj_meta_; + ObDatum new_prev_datum; + if (!obj_meta.is_lob_storage() && + OB_FAIL(shadow_truncate_string_for_hist(obj_meta, const_cast(stored_row.cells()[0])))) { + LOG_WARN("failed to shadow truncate string for hist", K(ret)); + } else if (obj_meta.is_lob_storage() && + OB_FAIL(ObHybridHistograms::build_prefix_str_datum_for_lob(tmp_alloctor, + obj_meta, + stored_row.cells()[0], + new_prev_datum))) { + LOG_WARN("failed to build prefix str datum for lob", K(ret)); + } else { + const ObDatum &datum = obj_meta.is_lob_storage() ? new_prev_datum : stored_row.cells()[0]; + ObExprHashFuncType hash_func = aggr_info.param_exprs_.at(0)->basic_funcs_->murmur_hash_; + uint64_t datum_value = 0; + if (OB_FAIL(hash_func(datum, datum_value, datum_value))) { + LOG_WARN("fail to do hash", K(ret)); + } else if (OB_FAIL(extra->topk_fre_hist_.add_top_k_frequency_item(datum_value, datum))) { + LOG_WARN("failed to process row", K(ret)); + } } } break; @@ -4877,6 +4901,8 @@ int ObAggregateProcessor::top_fre_hist_calc_batch( uint64_t nth_row = selector.get_batch_index(it); ObDatum *datum = arg_datums.at(nth_row); int32_t origin_str_len = 0; + common::ObArenaAllocator tmp_alloctor("BatchTopkHist", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObDatum new_prev_datum; if (datum->is_null()) { continue; } @@ -4887,9 +4913,17 @@ int ObAggregateProcessor::top_fre_hist_calc_batch( } else if (OB_FAIL(extra_info->topk_fre_hist_.merge_distribute_top_k_fre_items(obj))) { LOG_WARN("failed to process row", K(ret)); } - } else if (OB_FAIL(shadow_truncate_string_for_hist(obj_meta, *datum, &origin_str_len))) { + } else if (!obj_meta.is_lob_storage() && + OB_FAIL(shadow_truncate_string_for_hist(obj_meta, *datum, &origin_str_len))) { LOG_WARN("failed to shadow truncate string for hist", K(ret)); + } else if (obj_meta.is_lob_storage() && + OB_FAIL(ObHybridHistograms::build_prefix_str_datum_for_lob(tmp_alloctor, + obj_meta, + *datum, + new_prev_datum))) { + LOG_WARN("failed to build prefix str datum for lob", K(ret)); } else { + datum = obj_meta.is_lob_storage() ? &new_prev_datum : datum; ObExprHashFuncType hash_func = aggr_info.param_exprs_.at(0)->basic_funcs_->murmur_hash_; uint64_t datum_value = 0; if (OB_FAIL(hash_func(*datum, datum_value, datum_value))) { @@ -4982,7 +5016,7 @@ int ObAggregateProcessor::approx_count_calc_batch( has_null_cell = true; } OB_ASSERT(NULL != expr->basic_funcs_); - ObExprHashFuncType hash_func = expr->basic_funcs_->default_hash_; + ObExprHashFuncType hash_func = expr->basic_funcs_->murmur_hash_; if (OB_FAIL(hash_func(*arg_datums.at(nth_row), hash_value, hash_value))) { LOG_WARN("fail to do hash", K(ret)); } @@ -5556,7 +5590,7 @@ int ObAggregateProcessor::llc_calc_hash_value(const ObChunkDatumStore::StoredRow has_null_cell = true; } else { OB_ASSERT(NULL != expr.basic_funcs_); - ObExprHashFuncType hash_func = expr.basic_funcs_->default_hash_; + ObExprHashFuncType hash_func = expr.basic_funcs_->murmur_hash_; if (OB_FAIL(hash_func(datum, hash_value, hash_value))) { LOG_WARN("failed to do hash", K(ret)); } @@ -5978,17 +6012,25 @@ int ObAggregateProcessor::init_topk_fre_histogram_item( } else { ObDatum *window_size_result = NULL; ObDatum *item_size_result = NULL; + ObDatum *max_disuse_cnt_result = NULL; int64_t window_size = 0; int64_t item_size = 0; + int64_t max_disuse_cnt = 0; if (OB_UNLIKELY(!aggr_info.window_size_param_expr_->obj_meta_.is_numeric_type() || - !aggr_info.item_size_param_expr_->obj_meta_.is_numeric_type())) { + !aggr_info.item_size_param_expr_->obj_meta_.is_numeric_type() || + (aggr_info.max_disuse_param_expr_ != NULL && + !aggr_info.max_disuse_param_expr_->obj_meta_.is_numeric_type()))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("expr node is null", K(ret), KPC(aggr_info.window_size_param_expr_), - KPC(aggr_info.item_size_param_expr_)); + KPC(aggr_info.item_size_param_expr_), + KPC(aggr_info.max_disuse_param_expr_)); } else if (OB_FAIL(aggr_info.window_size_param_expr_->eval(eval_ctx_, window_size_result)) || - OB_FAIL(aggr_info.item_size_param_expr_->eval(eval_ctx_, item_size_result))) { + OB_FAIL(aggr_info.item_size_param_expr_->eval(eval_ctx_, item_size_result)) || + (aggr_info.max_disuse_param_expr_ != NULL && + OB_FAIL(aggr_info.max_disuse_param_expr_->eval(eval_ctx_, max_disuse_cnt_result)))) { LOG_WARN("eval failed", K(ret)); - } else if (OB_ISNULL(window_size_result) || OB_ISNULL(item_size_result)) { + } else if (OB_ISNULL(window_size_result) || + OB_ISNULL(item_size_result)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(window_size_result), K(item_size_result)); } else if (OB_FAIL(ObExprUtil::get_int_param_val( @@ -5996,13 +6038,20 @@ int ObAggregateProcessor::init_topk_fre_histogram_item( window_size)) || OB_FAIL(ObExprUtil::get_int_param_val( item_size_result, aggr_info.item_size_param_expr_->obj_meta_.is_decimal_int(), - item_size))) { + item_size)) + || (aggr_info.max_disuse_param_expr_ != NULL && OB_FAIL(ObExprUtil::get_int_param_val( + max_disuse_cnt_result, aggr_info.max_disuse_param_expr_->obj_meta_.is_decimal_int(), + max_disuse_cnt)))) { LOG_WARN("failed to get int param val", K(*window_size_result), K(window_size), - K(*item_size_result), K(item_size), K(ret)); + K(*item_size_result), K(item_size), + KPC(max_disuse_cnt_result), K(max_disuse_cnt), K(ret)); } else { topk_fre_hist->set_window_size(window_size); topk_fre_hist->set_item_size(item_size); topk_fre_hist->set_is_topk_hist_need_des_row(aggr_info.is_need_deserialize_row_); + topk_fre_hist->set_max_disuse_cnt(max_disuse_cnt); + LOG_TRACE("succeed to init topk fre histogram item", K(window_size), K(item_size), + K(aggr_info.is_need_deserialize_row_), K(max_disuse_cnt)); } } return ret; @@ -6228,6 +6277,7 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info int64_t repeat_count = 0; const ObChunkDatumStore::StoredRow *stored_row = NULL; const ObChunkDatumStore::StoredRow *mat_stored_row = NULL; + const ObObjMeta &obj_meta = aggr_info.param_exprs_.at(0)->obj_meta_; // get null count while (OB_SUCC(ret) && OB_SUCC(extra->get_next_row_from_sort(stored_row))) { if (OB_ISNULL(stored_row) || OB_UNLIKELY(stored_row->cnt_ != 1)) { @@ -6235,8 +6285,7 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info LOG_WARN("get unexpected null", K(ret), K(stored_row)); } else if (stored_row->cells()[0].is_null()) { ++ null_count; - } else if (OB_FAIL(shadow_truncate_string_for_hist(aggr_info.param_exprs_.at(0)->obj_meta_, - const_cast(stored_row->cells()[0])))) { + } else if (OB_FAIL(shadow_truncate_string_for_hist(obj_meta, const_cast(stored_row->cells()[0])))) { LOG_WARN("failed to shadow truncate string for hist", K(ret)); } else if (OB_FAIL(prev_row.save_store_row(*stored_row))) { LOG_WARN("failed to deep copy limit last rows", K(ret)); @@ -6254,11 +6303,14 @@ int ObAggregateProcessor::compute_hybrid_hist_result(const ObAggrInfo &aggr_info if (OB_ISNULL(stored_row) || OB_UNLIKELY(stored_row->cnt_ != 1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(stored_row)); - } else if (OB_FAIL(shadow_truncate_string_for_hist(aggr_info.param_exprs_.at(0)->obj_meta_, - const_cast(stored_row->cells()[0])))) { + } else if (OB_FAIL(shadow_truncate_string_for_hist(obj_meta, const_cast(stored_row->cells()[0])))) { LOG_WARN("failed to shadow truncate string for hist", K(ret)); - } else if (OB_FAIL(check_rows_equal(prev_row, *stored_row, aggr_info, is_equal))) { + } else if (!obj_meta.is_lob_storage() && + OB_FAIL(check_rows_equal(prev_row, *stored_row, aggr_info, is_equal))) { LOG_WARN("failed to is order by item equal with prev row", K(ret)); + } else if (obj_meta.is_lob_storage() && + OB_FAIL(check_rows_prefix_str_equal_for_hybrid_hist(prev_row, *stored_row, aggr_info, obj_meta, is_equal))) { + LOG_WARN("failed to check rows prefix str equal for hybrid hist", K(ret)); } else if (is_equal) { ++ repeat_count; } else if (OB_ISNULL(prev_row.store_row_)) { @@ -6332,17 +6384,19 @@ int ObAggregateProcessor::shadow_truncate_string_for_hist(const ObObjMeta obj_me { int ret = OB_SUCCESS; if (ObColumnStatParam::is_valid_opt_col_type(obj_meta.get_type()) && obj_meta.is_string_type() && !datum.is_null()) { - const ObString &str = datum.get_string(); - int64_t truncated_str_len = ObDbmsStatsUtils::get_truncated_str_len(str, obj_meta.get_collation_type()); - if (OB_UNLIKELY(truncated_str_len < 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(obj_meta), K(datum), K(truncated_str_len)); - } else { - datum.set_string(str.ptr(), static_cast(truncated_str_len)); - if (origin_str_len != NULL && static_cast(truncated_str_len) < str.length()) { - *origin_str_len = str.length(); + if (!obj_meta.is_lob_storage()) { + const ObString &str = datum.get_string(); + int64_t truncated_str_len = ObDbmsStatsUtils::get_truncated_str_len(str, obj_meta.get_collation_type()); + if (OB_UNLIKELY(truncated_str_len < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(obj_meta), K(datum), K(truncated_str_len)); + } else { + datum.set_string(str.ptr(), static_cast(truncated_str_len)); + if (origin_str_len != NULL && static_cast(truncated_str_len) < str.length()) { + *origin_str_len = str.length(); + } + LOG_TRACE("Succeed to shadow truncate string for hist", K(datum)); } - LOG_TRACE("Succeed to shadow truncate string for hist", K(datum)); } } return ret; @@ -7456,6 +7510,55 @@ int ObAggregateProcessor::fast_single_row_agg_batch(ObEvalCtx &eval_ctx, const i return ret; } +int ObAggregateProcessor::check_rows_prefix_str_equal_for_hybrid_hist(const ObChunkDatumStore::LastStoredRow &prev_row, + const ObChunkDatumStore::StoredRow &cur_row, + const ObAggrInfo &aggr_info, + const ObObjMeta &obj_meta, + bool &is_equal) +{ + int ret = OB_SUCCESS; + is_equal = false; + if (OB_ISNULL(prev_row.store_row_) || + OB_UNLIKELY(!obj_meta.is_lob_storage() || + prev_row.store_row_->cnt_ != cur_row.cnt_ || + aggr_info.sort_collations_.count() != cur_row.cnt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(prev_row.store_row_), K(obj_meta), + K(cur_row.cnt_), K(aggr_info.sort_collations_.count())); + } else { + is_equal = true; + for (int64_t i = 0; OB_SUCC(ret) && is_equal && i < aggr_info.sort_collations_.count(); ++i) { + uint32_t index = aggr_info.sort_collations_.at(i).field_idx_; + common::ObArenaAllocator tmp_alloctor("CalcHybridHist", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObDatum new_prev_datum; + ObDatum new_cur_datum; + if (OB_UNLIKELY(index >= prev_row.store_row_->cnt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get invalid argument", K(ret), K(index), K(prev_row.store_row_->cnt_)); + } else if (OB_FAIL(ObHybridHistograms::build_prefix_str_datum_for_lob(tmp_alloctor, + obj_meta, + prev_row.store_row_->cells()[index], + new_prev_datum)) || + OB_FAIL(ObHybridHistograms::build_prefix_str_datum_for_lob(tmp_alloctor, + obj_meta, + cur_row.cells()[index], + new_cur_datum))) { + LOG_WARN("failed to build prefix str datum for lob", K(ret)); + } else { + int cmp_ret = 0; + if (OB_FAIL(aggr_info.sort_cmp_funcs_.at(i).cmp_func_(new_prev_datum, + new_cur_datum, + cmp_ret))) { + LOG_WARN("failed to cmp", K(ret), K(index)); + } else { + is_equal = 0 == cmp_ret; + } + } + } + } + return ret; +} + ObAggregateProcessor::ObDecIntAggOpFunc ObAggregateProcessor::DEC_INT_OP_FUNCS[DECIMAL_INT_MAX][3][3]; ObAggregateProcessor::ObDecIntAggOpFunc ObAggregateProcessor::DEC_INT_MERGE_FUNCS[DECIMAL_INT_MAX][3]; ObAggregateProcessor::ObDecIntAggOpBatchFunc ObAggregateProcessor::DEC_INT_ADD_BATCH_FUNCS[DECIMAL_INT_MAX][3][2][2]; diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.h b/src/sql/engine/aggregate/ob_aggregate_processor.h index add6dad95e..257238f258 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.h +++ b/src/sql/engine/aggregate/ob_aggregate_processor.h @@ -989,6 +989,12 @@ private: ObDatum &datum, int32_t *origin_str_len = NULL); + int check_rows_prefix_str_equal_for_hybrid_hist(const ObChunkDatumStore::LastStoredRow &prev_row, + const ObChunkDatumStore::StoredRow &cur_row, + const ObAggrInfo &aggr_info, + const ObObjMeta &obj_meta, + bool &is_equal); + OB_INLINE void clear_op_evaluated_flag() { if (OB_NOT_NULL(op_eval_infos_)) { diff --git a/src/sql/engine/cmd/ob_analyze_executor.cpp b/src/sql/engine/cmd/ob_analyze_executor.cpp index 39777131af..1f2e03b040 100644 --- a/src/sql/engine/cmd/ob_analyze_executor.cpp +++ b/src/sql/engine/cmd/ob_analyze_executor.cpp @@ -44,8 +44,7 @@ namespace sql int ObAnalyzeExecutor::execute(ObExecContext &ctx, ObAnalyzeStmt &stmt) { int ret = OB_SUCCESS; - ObTableStatParam param; - param.allocator_ = &ctx.get_allocator(); + ObSEArray params; share::schema::ObSchemaGetterGuard *schema_guard = ctx.get_virtual_table_ctx().schema_guard_; ObSQLSessionInfo *session = ctx.get_my_session(); bool in_restore = false; @@ -59,48 +58,71 @@ int ObAnalyzeExecutor::execute(ObExecContext &ctx, ObAnalyzeStmt &stmt) GCTX.is_standby_cluster()) { ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "analyze table during restore or standby cluster"); - } else if (OB_FAIL(stmt.fill_table_stat_param(ctx, param))) { + } else if (OB_FAIL(stmt.fill_table_stat_params(ctx, params))) { LOG_WARN("failed to fill table stat param", K(ret)); - } else if (OB_FAIL(pl::ObDbmsStats::process_not_size_manual_column(ctx, param))) { - LOG_WARN("failed to process not size_manual column", K(ret)); - } else if (!stmt.is_delete_histogram()) { - int64_t task_cnt = 1; - int64_t seq_id = 1; - int64_t start_time = ObTimeUtility::current_time(); - ObOptStatTaskInfo task_info; - if (OB_FAIL(pl::ObDbmsStats::init_gather_task_info(ctx, ObOptStatGatherType::MANUAL_GATHER, - start_time, task_cnt, task_info))) { - LOG_WARN("failed to init gather task info", K(ret)); - } else { - ObOptStatGatherStat gather_stat(task_info); - ObOptStatGatherStatList::instance().push(gather_stat); - ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), start_time, param.allocator_->used(), gather_stat); - if (OB_FAIL(running_monitor.add_table_info(param))) { - LOG_WARN("failed to add table info", K(ret)); - } else if (OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, param))) { - LOG_WARN("failed check stat locked", K(ret)); - } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx, false, true))) { - LOG_WARN("failed to do flush database monitoring info", K(ret)); - } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, param))) { - LOG_WARN("failed to gather table stats", K(ret)); - } else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(session->get_rpc_tenant_id(), param))) { - LOG_WARN("failed to update stat cache", K(ret)); - } else { - LOG_TRACE("succeed to gather table stats", K(param)); - } - running_monitor.set_monitor_result(ret, ObTimeUtility::current_time(), param.allocator_->used()); - ObOptStatGatherStatList::instance().remove(gather_stat); - task_info.task_end_time_ = ObTimeUtility::current_time(); - task_info.ret_code_ = ret; - task_info.failed_count_ = ret == OB_SUCCESS ? 0 : 1; - ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); - ObOptStatManager::get_instance().update_opt_stat_gather_stat(gather_stat); - } } else { - if (OB_FAIL(ObDbmsStatsExecutor::delete_table_stats(ctx, param, true))) { - LOG_WARN("failed to drop table stats", K(ret)); - } else { - LOG_TRACE("succeed to drop table stats", K(param)); + for (int64_t i = 0; OB_SUCC(ret) && i < params.count(); ++i) { + if (OB_FAIL(pl::ObDbmsStats::process_not_size_manual_column(ctx, params.at(i)))) { + LOG_WARN("failed to process not size_manual column", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (stmt.is_delete_histogram()) { + //must be only one param + if (OB_FAIL(ObDbmsStatsExecutor::delete_table_stats(ctx, params.at(0), true))) { + LOG_WARN("failed to drop table stats", K(ret)); + } else { + LOG_TRACE("succeed to drop table stats", K(params)); + } + } else { + int64_t task_cnt = params.count(); + int64_t start_time = ObTimeUtility::current_time(); + ObOptStatTaskInfo task_info; + if (OB_FAIL(pl::ObDbmsStats::init_gather_task_info(ctx, ObOptStatGatherType::MANUAL_GATHER, + start_time, task_cnt, task_info))) { + LOG_WARN("failed to init gather task info", K(ret)); + } else { + int64_t i = 0; + for (; OB_SUCC(ret) && i < params.count(); ++i) { + ObTableStatParam ¶m = params.at(i); + ObArenaAllocator tmp_alloc("OptStatGather", OB_MALLOC_NORMAL_BLOCK_SIZE, param.tenant_id_); + param.allocator_ = &tmp_alloc;//use the temp allocator to free memory after gather stats. + start_time = ObTimeUtility::current_time(); + ObOptStatGatherStat gather_stat(task_info); + ObOptStatGatherStatList::instance().push(gather_stat); + ObOptStatRunningMonitor running_monitor(ctx.get_allocator(), start_time, param.allocator_->used(), gather_stat); + if (OB_FAIL(running_monitor.add_monitor_info(ObOptStatRunningPhase::GATHER_PREPARE))) { + LOG_WARN("failed to add add monitor info", K(ret)); + } else if (OB_FAIL(running_monitor.add_table_info(param))) { + LOG_WARN("failed to add table info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsLockUnlock::check_stat_locked(ctx, param))) { + LOG_WARN("failed check stat locked", K(ret)); + } else if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx, false, true))) { + LOG_WARN("failed to do flush database monitoring info", K(ret)); + } else if (OB_FAIL(ObDbmsStatsExecutor::gather_table_stats(ctx, param, running_monitor))) { + LOG_WARN("failed to gather table stats", K(ret)); + } else if (OB_FAIL(pl::ObDbmsStats::update_stat_cache(session->get_rpc_tenant_id(), param))) { + LOG_WARN("failed to update stat cache", K(ret)); + } else { + LOG_TRACE("succeed to gather table stats", K(param)); + } + running_monitor.set_monitor_result(ret, ObTimeUtility::current_time(), param.allocator_->used()); + sql::ObSQLSessionInfo *origin_session = THIS_WORKER.get_session(); + THIS_WORKER.set_session(NULL); + ObOptStatManager::get_instance().update_opt_stat_gather_stat(gather_stat); + THIS_WORKER.set_session(origin_session); + ObOptStatGatherStatList::instance().remove(gather_stat); + task_info.completed_table_count_ ++; + } + task_info.task_end_time_ = ObTimeUtility::current_time(); + task_info.ret_code_ = ret; + task_info.failed_count_ = ret == OB_SUCCESS ? 0 : params.count() - i + 1; + sql::ObSQLSessionInfo *origin_session = THIS_WORKER.get_session(); + THIS_WORKER.set_session(NULL); + ObOptStatManager::get_instance().update_opt_stat_task_stat(task_info); + THIS_WORKER.set_session(origin_session); + } + } } } return ret; diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index b5b980a037..3ffe9c8fa5 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -1989,12 +1989,9 @@ int ObLoadDataDirectImpl::init_execute_param() } // need_sort_ if (OB_SUCC(ret)) { - int64_t append = 0; int64_t enable_direct = 0; int64_t hint_need_sort = 0; - if (OB_FAIL(hint.get_value(ObLoadDataHint::APPEND, append))) { - LOG_WARN("fail to get value of APPEND", K(ret)); - } else if (OB_FAIL(hint.get_value(ObLoadDataHint::ENABLE_DIRECT, enable_direct))) { + if (OB_FAIL(hint.get_value(ObLoadDataHint::ENABLE_DIRECT, enable_direct))) { LOG_WARN("fail to get value of ENABLE_DIRECT", K(ret)); } else if (OB_FAIL(hint.get_value(ObLoadDataHint::NEED_SORT, hint_need_sort))) { LOG_WARN("fail to get value of NEED_SORT", KR(ret), K(hint)); @@ -2019,8 +2016,7 @@ int ObLoadDataDirectImpl::init_execute_param() } // online_opt_stat_gather_ if (OB_SUCC(ret)) { - int64_t append = 0; - int64_t gather_optimizer_statistics = 0 ; + int64_t no_gather_optimizer_statistics = 0 ; ObSQLSessionInfo *session = nullptr; ObObj obj; if (OB_ISNULL(session = ctx_->get_my_session())) { @@ -2028,11 +2024,9 @@ int ObLoadDataDirectImpl::init_execute_param() LOG_WARN("session is null", KR(ret)); } else if (OB_FAIL(session->get_sys_variable(SYS_VAR__OPTIMIZER_GATHER_STATS_ON_LOAD, obj))) { LOG_WARN("fail to get sys variable", K(ret)); - } else if (OB_FAIL(hint.get_value(ObLoadDataHint::APPEND, append))) { + } else if (OB_FAIL(hint.get_value(ObLoadDataHint::NO_GATHER_OPTIMIZER_STATISTICS, no_gather_optimizer_statistics))) { LOG_WARN("fail to get value of APPEND", K(ret)); - } else if (OB_FAIL(hint.get_value(ObLoadDataHint::GATHER_OPTIMIZER_STATISTICS, gather_optimizer_statistics))) { - LOG_WARN("fail to get value of APPEND", K(ret)); - } else if (((append != 0) || (gather_optimizer_statistics != 0)) && obj.get_bool()) { + } else if (no_gather_optimizer_statistics == 0 && obj.get_bool()) { execute_param_.online_opt_stat_gather_ = true; } else { execute_param_.online_opt_stat_gather_ = false; diff --git a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp index 9717013596..fd8094c10e 100644 --- a/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp +++ b/src/sql/engine/cmd/ob_table_direct_insert_ctx.cpp @@ -72,7 +72,7 @@ int ObTableDirectInsertCtx::init(ObExecContext *exec_ctx, param.parallel_ = parallel; param.session_count_ = MIN(parallel, (int64_t)tenant->unit_max_cpu() * 2); param.px_mode_ = true; - param.online_opt_stat_gather_ = false; + param.online_opt_stat_gather_ = true; param.need_sort_ = true; param.max_error_row_count_ = 0; param.dup_action_ = sql::ObLoadDupActionType::LOAD_STOP_ON_DUP; diff --git a/src/sql/engine/ob_exec_context.cpp b/src/sql/engine/ob_exec_context.cpp index 94ada8ac26..d8a83e6838 100644 --- a/src/sql/engine/ob_exec_context.cpp +++ b/src/sql/engine/ob_exec_context.cpp @@ -130,7 +130,8 @@ ObExecContext::ObExecContext(ObIAllocator &allocator) table_direct_insert_ctx_(), errcode_(OB_SUCCESS), dblink_snapshot_map_(), - cur_row_num_(-1) + cur_row_num_(-1), + is_online_stats_gathering_(false) { } diff --git a/src/sql/engine/ob_exec_context.h b/src/sql/engine/ob_exec_context.h index 927a1edbe7..82133b78de 100644 --- a/src/sql/engine/ob_exec_context.h +++ b/src/sql/engine/ob_exec_context.h @@ -477,6 +477,8 @@ public: } return ret; } + void set_is_online_stats_gathering(bool v) { is_online_stats_gathering_ = v; } + bool is_online_stats_gathering() const { return is_online_stats_gathering_; } int get_local_var_array(int64_t local_var_array_id, const ObLocalSessionVar *&var_array); private: int build_temp_expr_ctx(const ObTempExpr &temp_expr, ObTempExprCtx *&temp_expr_ctx); @@ -657,6 +659,8 @@ protected: ObExecFeedbackInfo fb_info_; // for dml report user warning/error at specific row int64_t cur_row_num_; + // for online stats gathering + bool is_online_stats_gathering_; //--------------- private: DISALLOW_COPY_AND_ASSIGN(ObExecContext); diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 70e6170f80..747b3d78b5 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -4683,6 +4683,18 @@ bool ObSQLUtils::is_fk_nested_sql(ObExecContext *cur_ctx) return bret; } +//this sql is triggered by online stat gathering +bool ObSQLUtils::is_online_stat_gathering_nested_sql(ObExecContext *cur_ctx) +{ + bool bret = false; + if (cur_ctx != nullptr && + cur_ctx->get_parent_ctx() != nullptr && + cur_ctx->get_parent_ctx()->is_online_stats_gathering()) { + bret = true; + } + return bret; +} + bool ObSQLUtils::is_iter_uncommitted_row(ObExecContext *cur_ctx) { bool bret = false; @@ -4698,7 +4710,7 @@ bool ObSQLUtils::is_iter_uncommitted_row(ObExecContext *cur_ctx) //then it is not nested sql, nor is it restricted by the constraints of nested sql bool ObSQLUtils::is_nested_sql(ObExecContext *cur_ctx) { - return is_pl_nested_sql(cur_ctx) || is_fk_nested_sql(cur_ctx); + return is_pl_nested_sql(cur_ctx) || is_fk_nested_sql(cur_ctx) || is_online_stat_gathering_nested_sql(cur_ctx); } bool ObSQLUtils::is_in_autonomous_block(ObExecContext *cur_ctx) diff --git a/src/sql/ob_sql_utils.h b/src/sql/ob_sql_utils.h index 470cea91bd..0b33193580 100644 --- a/src/sql/ob_sql_utils.h +++ b/src/sql/ob_sql_utils.h @@ -602,6 +602,7 @@ public: static bool is_support_batch_exec(ObItemType type); static bool is_pl_nested_sql(ObExecContext *cur_ctx); static bool is_fk_nested_sql(ObExecContext *cur_ctx); + static bool is_online_stat_gathering_nested_sql(ObExecContext *cur_ctx); static bool is_iter_uncommitted_row(ObExecContext *cur_ctx); static bool is_nested_sql(ObExecContext *cur_ctx); static bool is_in_autonomous_block(ObExecContext *cur_ctx); diff --git a/src/sql/optimizer/ob_del_upd_log_plan.cpp b/src/sql/optimizer/ob_del_upd_log_plan.cpp index d1a85afbab..cbc2b18824 100644 --- a/src/sql/optimizer/ob_del_upd_log_plan.cpp +++ b/src/sql/optimizer/ob_del_upd_log_plan.cpp @@ -1199,7 +1199,7 @@ int ObDelUpdLogPlan::allocate_optimizer_stats_gathering_as_top(ObLogicalOperator LOG_WARN("failed to allocate sequence operator", K(ret)); } else { OSG_TYPE type = old_top->need_osg_merge() ? OSG_TYPE::MERGE_OSG : - old_top->is_distributed() ? OSG_TYPE::GATHER_OSG : OSG_TYPE::NORMAL_OSG; + old_top->is_sharding() ? OSG_TYPE::GATHER_OSG : OSG_TYPE::NORMAL_OSG; osg->set_child(ObLogicalOperator::first_child, old_top); osg->set_osg_type(type); osg->set_table_id(info.table_id_); diff --git a/src/sql/optimizer/ob_insert_log_plan.cpp b/src/sql/optimizer/ob_insert_log_plan.cpp index d5a48e59a8..131f8b68bb 100644 --- a/src/sql/optimizer/ob_insert_log_plan.cpp +++ b/src/sql/optimizer/ob_insert_log_plan.cpp @@ -95,7 +95,7 @@ int ObInsertLogPlan::generate_normal_raw_plan() LOG_WARN("failed to compute dml parallel", K(ret)); } else if (use_pdml() && OB_FAIL(set_is_direct_insert())) { LOG_WARN("failed to set is direct insert", K(ret)); - } else if (OB_FAIL(check_need_online_stats_gather(need_osg))) { + } else if (!is_direct_insert() && OB_FAIL(check_need_online_stats_gather(need_osg))) { LOG_WARN("fail to check wether we need optimizer stats gathering operator", K(ret)); } else if (need_osg && OB_FAIL(generate_osg_share_info(osg_info))) { LOG_WARN("failed to generate osg share info"); diff --git a/src/sql/optimizer/ob_optimizer_util.cpp b/src/sql/optimizer/ob_optimizer_util.cpp index 2b6ba86401..57896f1484 100644 --- a/src/sql/optimizer/ob_optimizer_util.cpp +++ b/src/sql/optimizer/ob_optimizer_util.cpp @@ -8068,7 +8068,7 @@ int ObOptimizerUtil::generate_pullup_aggr_expr(ObRawExprFactory &expr_factory, LOG_WARN("failed to pullup grouping aggr expr", K(ret)); } } else if (T_FUN_TOP_FRE_HIST == origin_expr->get_expr_type()) { - if (OB_UNLIKELY(3 != origin_expr->get_real_param_count())) { + if (OB_UNLIKELY(4 != origin_expr->get_real_param_count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("real param count is invalid", K(ret)); } else if (OB_FAIL(expr_factory.create_raw_expr(T_FUN_TOP_FRE_HIST, pullup_aggr))) { @@ -8082,6 +8082,9 @@ int ObOptimizerUtil::generate_pullup_aggr_expr(ObRawExprFactory &expr_factory, } else if (OB_FAIL(pullup_aggr->add_real_param_expr( origin_expr->get_real_param_exprs_for_update().at(2)))) { LOG_WARN("failed to add real param expr", K(ret)); + } else if (OB_FAIL(pullup_aggr->add_real_param_expr( + origin_expr->get_real_param_exprs_for_update().at(3)))) { + LOG_WARN("failed to add real param expr", K(ret)); } else if (FALSE_IT(pullup_aggr->set_is_need_deserialize_row(true))) { // do nothing } else if (OB_FAIL(pullup_aggr->formalize(session_info))) { diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 105d08f92d..84d8780008 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -1986,10 +1986,10 @@ COUNT '(' opt_all '*' ')' OVER new_generalized_window_clause malloc_non_terminal_node($$, result->malloc_pool_, T_WIN_FUN_NTH_VALUE, 4, $3, $5, $7, $8); malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $10); } -| TOP_K_FRE_HIST '(' DECIMAL_VAL ',' bit_expr ',' INTNUM ')' OVER new_generalized_window_clause +| TOP_K_FRE_HIST '(' DECIMAL_VAL ',' bit_expr ',' INTNUM ',' expr_const ')' OVER new_generalized_window_clause { - malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_TOP_FRE_HIST, 3, $3, $5, $7); - malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $10); + malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_TOP_FRE_HIST, 4, $3, $5, $7, $9); + malloc_non_terminal_node($$, result->malloc_pool_, T_WINDOW_FUNCTION, 2, $$, $12); } | HYBRID_HIST '(' bit_expr ',' INTNUM ')' OVER new_generalized_window_clause { @@ -2404,9 +2404,9 @@ MOD '(' expr ',' expr ')' merge_nodes(group_concat_exprs, result, T_EXPR_LIST, $4); malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_GROUP_CONCAT, 4, $3, group_concat_exprs, $5, $6); } -| TOP_K_FRE_HIST '(' DECIMAL_VAL ',' bit_expr ',' INTNUM ')' +| TOP_K_FRE_HIST '(' DECIMAL_VAL ',' bit_expr ',' INTNUM ',' expr_const ')' { - malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_TOP_FRE_HIST, 3, $3, $5, $7); + malloc_non_terminal_node($$, result->malloc_pool_, T_FUN_TOP_FRE_HIST, 4, $3, $5, $7, $9); } | HYBRID_HIST '(' bit_expr ',' INTNUM ')' { @@ -11772,6 +11772,12 @@ ANALYZE TABLE relation_factor UPDATE HISTOGRAM ON column_name_list WITH INTNUM B } malloc_non_terminal_node($$, result->malloc_pool_, T_MYSQL_UPDATE_HISTOGRAM, 3, $3, column_name_list, $9); } +| ANALYZE TABLE relation_factor UPDATE HISTOGRAM ON column_name_list +{ + ParseNode *column_name_list = NULL; + merge_nodes(column_name_list, result, T_ANALYZE_MYSQL_COLUMN_LIST, $7); + malloc_non_terminal_node($$, result->malloc_pool_, T_MYSQL_UPDATE_HISTOGRAM, 3, $3, column_name_list, NULL); +} | ANALYZE TABLE relation_factor DROP HISTOGRAM ON column_name_list { ParseNode *column_name_list = NULL; @@ -11786,6 +11792,12 @@ ANALYZE TABLE relation_factor UPDATE HISTOGRAM ON column_name_list WITH INTNUM B { malloc_non_terminal_node($$, result->malloc_pool_, T_ANALYZE, 3, $3, $4, $5); } +| ANALYZE TABLE table_list +{ + ParseNode *compute_statictic = NULL; + malloc_non_terminal_node(compute_statictic, result->malloc_pool_, T_ANALYZE_STATISTICS, 2, NULL, NULL); + malloc_non_terminal_node($$, result->malloc_pool_, T_MYSQL_ANALYZE, 3, $3, NULL, compute_statictic); +} ; /***************************************************************************** diff --git a/src/sql/resolver/ddl/ob_analyze_stmt.cpp b/src/sql/resolver/ddl/ob_analyze_stmt.cpp index 753dafdad2..d5a34be9cf 100644 --- a/src/sql/resolver/ddl/ob_analyze_stmt.cpp +++ b/src/sql/resolver/ddl/ob_analyze_stmt.cpp @@ -19,28 +19,60 @@ namespace oceanbase namespace sql { -ObAnalyzeStmt::ObAnalyzeStmt() - : ObStmt(NULL, stmt::T_ANALYZE), - tenant_id_(common::OB_INVALID_ID), - database_name_(), +ObAnalyzeTableInfo::ObAnalyzeTableInfo() + : database_name_(), database_id_(common::OB_INVALID_ID), table_name_(), table_id_(common::OB_INVALID_ID), part_level_(share::schema::PARTITION_LEVEL_ZERO), - total_part_cnt_(0), partition_name_(), partition_infos_(), subpartition_infos_(), column_params_(), column_group_params_(), + ref_table_type_(share::schema::ObTableType::MAX_TABLE_TYPE), + gather_subpart_hist_(false), + is_sepcify_subpart_(false) +{ +} + +ObAnalyzeTableInfo::~ObAnalyzeTableInfo() +{ +} + +int ObAnalyzeTableInfo::assign(const ObAnalyzeTableInfo &other) +{ + int ret = OB_SUCCESS; + database_name_ = other.database_name_; + database_id_ = other.database_id_; + table_name_ = other.table_name_; + table_id_ = other.table_id_; + part_level_ = other.part_level_; + partition_name_ = other.partition_name_; + ref_table_type_ = other.ref_table_type_; + gather_subpart_hist_ = other.gather_subpart_hist_; + is_sepcify_subpart_ = other.is_sepcify_subpart_; + if (OB_FAIL(partition_infos_.assign(other.partition_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(subpartition_infos_.assign(other.subpartition_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(column_params_.assign(other.column_params_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(all_partition_infos_.assign(other.all_partition_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } else if (OB_FAIL(all_subpartition_infos_.assign(other.all_subpartition_infos_))) { + LOG_WARN("failed to assign", K(ret)); + } else {/*do nothing*/} + return ret; +} + +ObAnalyzeStmt::ObAnalyzeStmt() + : ObStmt(NULL, stmt::T_ANALYZE), + tenant_id_(common::OB_INVALID_ID), statistic_type_(InvalidStatistics), sample_info_(), parallel_degree_(1), - is_drop_(false), - part_ids_(), - subpart_ids_(), - ref_table_type_(share::schema::ObTableType::MAX_TABLE_TYPE), - gather_subpart_hist_(false) + is_drop_(false) { } @@ -48,7 +80,7 @@ ObAnalyzeStmt::~ObAnalyzeStmt() { } -int ObAnalyzeStmt::set_column_params(const ObIArray &col_params) +int ObAnalyzeTableInfo::set_column_params(const ObIArray &col_params) { int ret = OB_SUCCESS; if (OB_FAIL(column_params_.assign(col_params))) { @@ -57,7 +89,7 @@ int ObAnalyzeStmt::set_column_params(const ObIArray &col_para return ret; } -ObColumnStatParam *ObAnalyzeStmt::get_column_param(const uint64_t column_id) +ObColumnStatParam *ObAnalyzeTableInfo::get_column_param(const uint64_t column_id) { ObColumnStatParam *ret = NULL; for (int64_t i = 0; NULL == ret && i < column_params_.count(); ++i) { @@ -68,10 +100,26 @@ ObColumnStatParam *ObAnalyzeStmt::get_column_param(const uint64_t column_id) return ret; } -int ObAnalyzeStmt::fill_table_stat_param(ObExecContext &ctx, common::ObTableStatParam ¶m) +int ObAnalyzeStmt::fill_table_stat_params(ObExecContext &ctx, ObIArray ¶ms) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(params.prepare_allocate(tables_.count()))) { + LOG_WARN("prepare allocate failed", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < tables_.count(); ++i) { + params.at(i).tenant_id_ = tenant_id_; + params.at(i).sample_info_ = sample_info_; + params.at(i).degree_ = parallel_degree_; + if (OB_FAIL(tables_.at(i).fill_table_stat_param(ctx, params.at(i)))) { + LOG_WARN("fill table stat param failed", K(ret)); + } + } + return ret; +} + +int ObAnalyzeTableInfo::fill_table_stat_param(ObExecContext &ctx, common::ObTableStatParam ¶m) { int ret = OB_SUCCESS; - param.tenant_id_ = tenant_id_; param.db_name_ = database_name_; param.db_id_ = database_id_; @@ -80,7 +128,6 @@ int ObAnalyzeStmt::fill_table_stat_param(ObExecContext &ctx, common::ObTableStat param.table_id_ = table_id_; param.ref_table_type_ = ref_table_type_; param.part_level_ = part_level_; - param.total_part_cnt_ = total_part_cnt_; param.part_name_ = partition_name_; @@ -88,55 +135,80 @@ int ObAnalyzeStmt::fill_table_stat_param(ObExecContext &ctx, common::ObTableStat OZ (param.subpart_infos_.assign(subpartition_infos_)); OZ (param.column_params_.assign(column_params_)); OZ (param.column_group_params_.assign(column_group_params_)); - OZ (param.part_ids_.assign(part_ids_)); - OZ (param.subpart_ids_.assign(subpart_ids_)); OZ (pl::ObDbmsStats::set_param_global_part_id(ctx, param)); OZ (param.all_part_infos_.assign(all_partition_infos_)); OZ (param.all_subpart_infos_.assign(all_subpartition_infos_)); if (OB_SUCC(ret)) { - param.sample_info_ = sample_info_; - param.degree_ = parallel_degree_; //analyze stmt default use granularity is based partition type(oracle 12c),maybe refine it later param.global_stat_param_.need_modify_ = partition_name_.empty(); - param.part_stat_param_.need_modify_ = !partition_infos_.empty(); - param.subpart_stat_param_.need_modify_ = !subpartition_infos_.empty(); + param.part_stat_param_.need_modify_ = !partition_infos_.empty() && !is_sepcify_subpart_; + param.subpart_stat_param_.need_modify_ = !subpartition_infos_.empty() && (partition_name_.empty() || is_sepcify_subpart_); param.subpart_stat_param_.gather_histogram_ = gather_subpart_hist_; + if (param.global_stat_param_.need_modify_ && param.part_stat_param_.need_modify_) { + param.global_stat_param_.gather_approx_ = true; + } + if (param.part_stat_param_.need_modify_ && param.subpart_stat_param_.need_modify_) { + param.part_stat_param_.can_use_approx_ = true; + } } LOG_TRACE("link bug", K(param)); return ret; } -int ObAnalyzeStmt::set_part_infos(ObIArray &part_infos) +int ObAnalyzeTableInfo::set_part_infos(ObIArray &part_infos) { return partition_infos_.assign(part_infos); } -int ObAnalyzeStmt::set_subpart_infos(ObIArray &subpart_infos) +int ObAnalyzeTableInfo::set_subpart_infos(ObIArray &subpart_infos) { return subpartition_infos_.assign(subpart_infos); } -int ObAnalyzeStmt::set_part_ids(ObIArray &part_ids) -{ - return part_ids_.assign(part_ids); -} - -int ObAnalyzeStmt::set_subpart_ids(ObIArray &subpart_ids) -{ - return subpart_ids_.assign(subpart_ids); -} - -int ObAnalyzeStmt::set_all_part_infos(ObIArray &all_part_infos) +int ObAnalyzeTableInfo::set_all_part_infos(ObIArray &all_part_infos) { return all_partition_infos_.assign(all_part_infos); } -int ObAnalyzeStmt::set_all_subpart_infos(ObIArray &all_subpart_infos) +int ObAnalyzeTableInfo::set_all_subpart_infos(ObIArray &all_subpart_infos) { return all_subpartition_infos_.assign(all_subpart_infos); } +int ObAnalyzeStmt::add_table(const ObString database_name, + const uint64_t database_id, + const ObString table_name, + const uint64_t table_id, + const share::schema::ObTableType table_type) +{ + int ret = OB_SUCCESS; + ObAnalyzeTableInfo table; + if (is_mysql_mode()) { + for (int64_t i = 0; OB_SUCC(ret) && i < tables_.count(); ++i) { + if (tables_.at(i).get_table_id() == table_id + && tables_.at(i).get_database_id() == database_id) { + ret = OB_ERR_NONUNIQ_TABLE; + LOG_USER_ERROR(OB_ERR_NONUNIQ_TABLE, table_name.length(), table_name.ptr()); + } + } + } + if (OB_SUCC(ret)) { + ObAnalyzeTableInfo *table = NULL; + if (OB_ISNULL(table = tables_.alloc_place_holder())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("Allocate table info from array error", K(ret)); + } else { + table->set_database_name(database_name); + table->set_database_id(database_id); + table->set_table_name(table_name); + table->set_table_id(table_id); + table->set_ref_table_type(table_type); + } + } + return ret; +} + } /* namespace sql */ } /* namespace oceanbase */ diff --git a/src/sql/resolver/ddl/ob_analyze_stmt.h b/src/sql/resolver/ddl/ob_analyze_stmt.h index 61c1978476..03628d10ed 100644 --- a/src/sql/resolver/ddl/ob_analyze_stmt.h +++ b/src/sql/resolver/ddl/ob_analyze_stmt.h @@ -33,41 +33,21 @@ enum StatisticType InvalidStatistics }; -class ObAnalyzeStmt: public ObStmt, public ObICmd +class ObAnalyzeTableInfo { public: - ObAnalyzeStmt(); - virtual ~ObAnalyzeStmt(); - virtual int get_cmd_type() const { return get_stmt_type(); } + ObAnalyzeTableInfo(); + virtual ~ObAnalyzeTableInfo(); + void set_database_name(const ObString &str) { database_name_ = str; } + void set_database_id(const uint64_t id) { database_id_ = id; } + void set_table_name(const ObString &str) { table_name_ = str; } + void set_table_id(const uint64_t id) { table_id_ = id; } + void set_ref_table_type(const share::schema::ObTableType type) { ref_table_type_ = type; } - void set_tenant_id(const uint64_t tenant_id) { tenant_id_ = tenant_id; } - void set_table(const ObString database_name, - const uint64_t database_id, - const ObString table_name, - const uint64_t table_id, - const share::schema::ObTableType table_type) - { - database_name_ = database_name; - database_id_ = database_id; - table_name_ = table_name; - table_id_ = table_id; - ref_table_type_ = table_type; - } - - void set_sample_info(const ObAnalyzeSampleInfo &sample_info) - { sample_info_ = sample_info; } - void set_statistic_scope(const StatisticType statistic_type) - { statistic_type_ = statistic_type; } - void set_is_drop() { is_drop_ = true; } - uint64_t get_tenant_id() { return tenant_id_; } + uint64_t get_database_id() { return database_id_; } uint64_t get_table_id() { return table_id_; } share::schema::ObTableType get_table_type() { return ref_table_type_; } - StatisticType get_statistic_type() { return statistic_type_; } - const ObAnalyzeSampleInfo &get_sample_info() const { return sample_info_; } - ObAnalyzeSampleInfo &get_sample_info() { return sample_info_; } - bool is_delete_histogram() { return is_drop_; } ObString &get_partition_name() { return partition_name_; } - int set_column_params(const ObIArray &col_params); ObColumnStatParam *get_column_param(const uint64_t column_id); @@ -82,17 +62,6 @@ public: part_level_ = part_level; } - void set_part_cnt(int64_t total_part_cnt) - { - total_part_cnt_ = total_part_cnt; - } - - void set_degree(uint64_t degree) - { - if (degree > 1) { - parallel_degree_ = degree; - } - } int fill_table_stat_param(ObExecContext &ctx, common::ObTableStatParam ¶m); int set_part_infos(ObIArray &part_infos); @@ -102,28 +71,21 @@ public: int set_all_part_infos(ObIArray &all_part_infos); int set_all_subpart_infos(ObIArray &all_subpart_infos); - int set_part_ids(ObIArray &part_ids); - int set_subpart_ids(ObIArray &subpart_ids); - void set_gather_subpart_hist(bool gather_subpart_hist) { gather_subpart_hist_ = gather_subpart_hist; } + void set_is_sepcify_subpart(bool is_sepcify_subpart) { is_sepcify_subpart_ = is_sepcify_subpart; } + int assign(const ObAnalyzeTableInfo &other); - - TO_STRING_KV(K_(tenant_id), - K_(table_id), - K_(statistic_type), - K_(sample_info), - K_(parallel_degree), + TO_STRING_KV(K_(table_id), K_(partition_infos), K_(subpartition_infos), - K_(part_ids), - K_(subpart_ids), + K_(column_params), K_(ref_table_type), K_(gather_subpart_hist), + K_(is_sepcify_subpart), K_(column_params), K_(column_group_params)); private: - uint64_t tenant_id_; ObString database_name_; uint64_t database_id_; @@ -131,7 +93,6 @@ private: uint64_t table_id_; share::schema::ObPartitionLevel part_level_; - int64_t total_part_cnt_; ObString partition_name_; common::ObSEArray partition_infos_; @@ -139,6 +100,62 @@ private: common::ObSEArray column_params_; common::ObSEArray column_group_params_; + + common::ObSEArray all_partition_infos_; + common::ObSEArray all_subpartition_infos_; + share::schema::ObTableType ref_table_type_; + bool gather_subpart_hist_; + bool is_sepcify_subpart_; +}; + +class ObAnalyzeStmt: public ObStmt, public ObICmd +{ +public: + ObAnalyzeStmt(); + virtual ~ObAnalyzeStmt(); + virtual int get_cmd_type() const { return get_stmt_type(); } + + void set_tenant_id(const uint64_t tenant_id) { tenant_id_ = tenant_id; } + int add_table(const ObString database_name, + const uint64_t database_id, + const ObString table_name, + const uint64_t table_id, + const share::schema::ObTableType table_type); + ObIArray & get_tables() { return tables_; } + + void set_sample_info(const ObAnalyzeSampleInfo &sample_info) + { sample_info_ = sample_info; } + void set_statistic_scope(const StatisticType statistic_type) + { statistic_type_ = statistic_type; } + void set_is_drop() { is_drop_ = true; } + uint64_t get_tenant_id() { return tenant_id_; } + StatisticType get_statistic_type() { return statistic_type_; } + const ObAnalyzeSampleInfo &get_sample_info() const { return sample_info_; } + ObAnalyzeSampleInfo &get_sample_info() { return sample_info_; } + bool is_delete_histogram() { return is_drop_; } + + void set_degree(uint64_t degree) + { + if (degree > 1) { + parallel_degree_ = degree; + } + } + int fill_table_stat_params(ObExecContext &ctx, ObIArray ¶ms); + + + TO_STRING_KV(K_(tenant_id), + K_(statistic_type), + K_(sample_info), + K_(parallel_degree), + K_(tables)); + +private: + uint64_t tenant_id_; + //ANALYZE TABLE without histogram clause support multi tables.For example: + // ANALYZE TABLE db1.t1, db2.t1; + //And in other cases, size of tables_ is always 1. + ObSEArray tables_; + StatisticType statistic_type_; ObAnalyzeSampleInfo sample_info_; @@ -146,12 +163,6 @@ private: bool is_drop_; - common::ObSEArray part_ids_; - common::ObSEArray subpart_ids_; - common::ObSEArray all_partition_infos_; - common::ObSEArray all_subpartition_infos_; - share::schema::ObTableType ref_table_type_; - bool gather_subpart_hist_; DISALLOW_COPY_AND_ASSIGN(ObAnalyzeStmt); }; diff --git a/src/sql/resolver/ddl/ob_analyze_stmt_resolver.cpp b/src/sql/resolver/ddl/ob_analyze_stmt_resolver.cpp index 70fc9bdfa5..c7b59c0ccd 100644 --- a/src/sql/resolver/ddl/ob_analyze_stmt_resolver.cpp +++ b/src/sql/resolver/ddl/ob_analyze_stmt_resolver.cpp @@ -58,6 +58,10 @@ int ObAnalyzeStmtResolver::resolve(const ParseNode &parse_tree) } else if (OB_FAIL(resolve_oracle_analyze(parse_tree, *analyze_stmt))) { LOG_WARN("failed to resolve oracle analyze stmt", K(ret)); } else { /*do nothing*/ } + } else if (T_MYSQL_ANALYZE == parse_tree.type_) { + if (OB_FAIL(resolve_oracle_analyze(parse_tree, *analyze_stmt))) { + LOG_WARN("failed to resolve oracle analyze stmt", K(ret)); + } else { /*do nothing*/ } } else if (T_MYSQL_UPDATE_HISTOGRAM == parse_tree.type_) { if (OB_FAIL(resolve_mysql_update_histogram(parse_tree, *analyze_stmt))) { LOG_WARN("failed to resolve mysql update histogram info", K(ret)); @@ -75,6 +79,23 @@ int ObAnalyzeStmtResolver::resolve(const ParseNode &parse_tree) if (OB_SUCC(ret)) { analyze_stmt->set_degree(parallel_degree); stmt_ = analyze_stmt; + ObSEArray new_column_params; + for (int64_t j = 0; OB_SUCC(ret) && j < analyze_stmt->get_tables().count(); ++j) { + ObAnalyzeTableInfo &table = analyze_stmt->get_tables().at(j); + new_column_params.reuse(); + for (int64_t i = 0 ; OB_SUCC(ret) && i < table.get_column_params().count(); ++i) { + if (table.get_column_params().at(i).need_col_stat()) { + if (OB_FAIL(new_column_params.push_back(table.get_column_params().at(i)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(table.get_column_params().assign(new_column_params))) { + LOG_WARN("failed to assign", K(ret)); + } + } + } LOG_DEBUG("analyze statement", K(*analyze_stmt)); } return ret; @@ -91,15 +112,14 @@ int ObAnalyzeStmtResolver::resolve_oracle_analyze(const ParseNode &parse_node, ParseNode *table_node = parse_node.children_[0]; ParseNode *part_node = parse_node.children_[1]; ParseNode *statistic_node = parse_node.children_[2]; - bool is_hist_subpart = false; if (OB_ISNULL(table_node) || OB_ISNULL(statistic_node)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null parse node", K(table_node), K(statistic_node), K(ret)); - } else if (OB_FAIL(resolve_table_info(table_node, analyze_stmt))) { + } else if (OB_FAIL(recursive_resolve_table_info(table_node, analyze_stmt))) { LOG_WARN("failed to resolve table info", K(ret)); - } else if (OB_FAIL(resolve_partition_info(part_node, analyze_stmt, is_hist_subpart))) { + } else if (OB_FAIL(resolve_partition_info(part_node, analyze_stmt))) { LOG_WARN("failed to resolve partition info", K(ret)); - } else if (OB_FAIL(resolve_statistic_info(statistic_node, is_hist_subpart, analyze_stmt))) { + } else if (OB_FAIL(resolve_statistic_info(statistic_node, analyze_stmt))) { LOG_WARN("failed to resolve statistic info", K(ret)); } else { /*do nothing*/ } } @@ -117,23 +137,28 @@ int ObAnalyzeStmtResolver::resolve_mysql_update_histogram(const ParseNode &parse ParseNode *table_node = parse_node.children_[0]; ParseNode *column_node = parse_node.children_[1]; ParseNode *bucket_node = parse_node.children_[2]; - bool dumy_bool = false; - if (OB_ISNULL(table_node) || OB_ISNULL(column_node) || OB_ISNULL(bucket_node)) { + int64_t bkt_num = 1; + if (OB_ISNULL(bucket_node)) { + bkt_num = ObColumnStatParam::DEFAULT_HISTOGRAM_BUCKET_NUM; + } else { + bkt_num = bucket_node->value_; + } + if (OB_ISNULL(table_node) || OB_ISNULL(column_node)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("null parse node", K(table_node), K(column_node), K(bucket_node), K(ret)); + LOG_WARN("null parse node", K(table_node), K(column_node), K(ret)); } else if (OB_FAIL(resolve_table_info(table_node, analyze_stmt))) { LOG_WARN("failed to resolve table info", K(ret)); - } else if (OB_FAIL(resolve_partition_info(NULL, analyze_stmt, dumy_bool))) { + } else if (OB_FAIL(resolve_partition_info(NULL, analyze_stmt))) { LOG_WARN("failed to resolve partition info", K(ret)); } else if (OB_FAIL(resolve_mysql_column_bucket_info(column_node, - bucket_node->value_, + bkt_num, analyze_stmt))) { LOG_WARN("failed to resolve column bucket info", K(ret)); } else { ObAnalyzeSampleInfo sample_info; sample_info.is_sample_ = true; sample_info.sample_type_ = SampleType::RowSample; - sample_info.sample_value_ = bucket_node->value_ * DEFAULT_SAMPLE_ROWCOUNT_PER_BUCKET; + sample_info.sample_value_ = bkt_num * DEFAULT_SAMPLE_ROWCOUNT_PER_BUCKET; analyze_stmt.set_sample_info(sample_info); } } @@ -156,7 +181,7 @@ int ObAnalyzeStmtResolver::resolve_mysql_delete_histogram(const ParseNode &parse LOG_WARN("null parse node", K(ret)); } else if (OB_FAIL(resolve_table_info(table_node, analyze_stmt))) { LOG_WARN("failed to resolve table info", K(ret)); - } else if (OB_FAIL(resolve_partition_info(NULL, analyze_stmt, dumy_bool))) { + } else if (OB_FAIL(resolve_partition_info(NULL, analyze_stmt))) { LOG_WARN("failed to resolve partition info", K(ret)); } else if (OB_FAIL(resolve_mysql_column_bucket_info(column_node, 0, analyze_stmt))) { @@ -172,12 +197,13 @@ int ObAnalyzeStmtResolver::resolve_mysql_column_bucket_info(const ParseNode *col { int ret = OB_SUCCESS; const ObTableSchema *table_schema = NULL; + ObAnalyzeTableInfo &table_info = analyze_stmt.get_tables().at(0); if (OB_ISNULL(column_node) || OB_ISNULL(schema_checker_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid params", K(ret), K(column_node), K(schema_checker_)); - } else if (OB_FAIL(schema_checker_->get_table_schema(analyze_stmt.get_tenant_id(), analyze_stmt.get_table_id(), + } else if (OB_FAIL(schema_checker_->get_table_schema(analyze_stmt.get_tenant_id(), table_info.get_table_id(), table_schema))) { - LOG_WARN("failed to get table schema", K(analyze_stmt.get_tenant_id()), K(analyze_stmt.get_table_id()), K(ret)); + LOG_WARN("failed to get table schema", K(analyze_stmt.get_tenant_id()), K(table_info.get_table_id()), K(ret)); } else if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null table schema", K(ret)); @@ -195,7 +221,7 @@ int ObAnalyzeStmtResolver::resolve_mysql_column_bucket_info(const ParseNode *col if (OB_ISNULL(column_schema = table_schema->get_column_schema(column_name))) { ret = OB_ERR_COLUMN_NOT_FOUND; LOG_WARN("failed to get column schema", K(column_name), K(ret)); - } else if (OB_ISNULL(col_param = analyze_stmt.get_column_param(column_schema->get_column_id()))) { + } else if (OB_ISNULL(col_param = table_info.get_column_param(column_schema->get_column_id()))) { // do nothing } else if (col_param->is_valid_opt_col()) { col_param->set_need_basic_stat(); @@ -207,6 +233,25 @@ int ObAnalyzeStmtResolver::resolve_mysql_column_bucket_info(const ParseNode *col return ret; } +int ObAnalyzeStmtResolver::recursive_resolve_table_info(const ParseNode *table_list_node, + ObAnalyzeStmt &analyze_stmt) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_list_node)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("null point", K(ret), KP(table_list_node)); + } else if (T_LINK_NODE == table_list_node->type_) { + if (OB_FAIL(SMART_CALL(recursive_resolve_table_info(table_list_node->children_[0], analyze_stmt)))) { + LOG_WARN("recursive resolve table list node failed", K(ret)); + } else if (OB_FAIL(SMART_CALL(recursive_resolve_table_info(table_list_node->children_[1], analyze_stmt)))) { + LOG_WARN("recursive resolve table list node failed", K(ret)); + } + } else if (OB_FAIL(resolve_table_info(table_list_node, analyze_stmt))) { + LOG_WARN("resolve table info failed", K(ret)); + } + return ret; +} + int ObAnalyzeStmtResolver::resolve_table_info(const ParseNode *table_node, ObAnalyzeStmt &analyze_stmt) { @@ -229,41 +274,58 @@ int ObAnalyzeStmtResolver::resolve_table_info(const ParseNode *table_node, } else if (OB_ISNULL(table_schema)) { ret = OB_TABLE_NOT_EXIST; LOG_WARN("null table schema", K(ret)); - } else if (OB_FAIL(pl::ObDbmsStats::init_column_stat_params(*allocator_, - *schema_checker_->get_schema_guard(), - *table_schema, - analyze_stmt.get_column_params()))) { - LOG_WARN("failed to init column stat param", K(ret)); - } else if (OB_FAIL(pl::ObDbmsStats::init_column_group_stat_param(*table_schema, - analyze_stmt.get_column_group_params()))) { - LOG_WARN("failed to init column stat param", K(ret)); + } else if (OB_FAIL(analyze_stmt.add_table(database_name, database_id, table_name, table_schema->get_table_id(), + table_schema->get_table_type()))) { + LOG_WARN("add table failed", K(ret)); } else { - analyze_stmt.set_table(database_name, database_id, table_name, table_schema->get_table_id(), - table_schema->get_table_type()); + ObAnalyzeTableInfo &last_table = analyze_stmt.get_tables().at(analyze_stmt.get_tables().count() - 1); + if (OB_FAIL(pl::ObDbmsStats::init_column_stat_params(*allocator_, + *schema_checker_->get_schema_guard(), + *table_schema, + last_table.get_column_params()))) { + LOG_WARN("failed to init column stat param", K(ret)); + } else if (OB_FAIL(pl::ObDbmsStats::init_column_group_stat_param(*table_schema, + last_table.get_column_group_params()))) { + LOG_WARN("failed to init column stat param", K(ret)); + } else { + LOG_TRACE("succeed to resolve table info", K(last_table)); + } } return ret; } int ObAnalyzeStmtResolver::resolve_partition_info(const ParseNode *part_node, - ObAnalyzeStmt &analyze_stmt, - bool &is_hist_subpart) + ObAnalyzeStmt &analyze_stmt) +{ + int ret = OB_SUCCESS; + ObIArray &tables = analyze_stmt.get_tables(); + for (int64_t i = 0; OB_SUCC(ret) && i < tables.count(); ++i) { + if (OB_FAIL(inner_resolve_partition_info(part_node, analyze_stmt.get_tenant_id(), tables.at(i)))) { + LOG_WARN("resolve table partition info failed", K(ret)); + } + } + return ret; +} + +int ObAnalyzeStmtResolver::inner_resolve_partition_info(const ParseNode *part_node, + uint64_t tenant_id, + ObAnalyzeTableInfo &table_info) { int ret = OB_SUCCESS; const ObTableSchema *table_schema = NULL; - int64_t table_id = analyze_stmt.get_table_id(); + int64_t table_id = table_info.get_table_id(); ObSEArray part_infos; ObSEArray subpart_infos; ObSEArray part_ids; ObSEArray subpart_ids; bool is_subpart_name = false; - is_hist_subpart = false; - ObString &partition_name = analyze_stmt.get_partition_name(); + ObString &partition_name = table_info.get_partition_name(); if (OB_ISNULL(schema_checker_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null schema checker", K(schema_checker_), K(ret)); - } else if (OB_FAIL(schema_checker_->get_table_schema(analyze_stmt.get_tenant_id(), table_id, table_schema))) { - LOG_WARN("failed to get table schema", K(analyze_stmt.get_tenant_id()), K(table_id), K(ret)); + } else if (OB_FAIL(schema_checker_->get_table_schema(tenant_id, table_id, table_schema))) { + LOG_WARN("failed to get table schema", K(tenant_id), K(table_id), K(ret)); } else if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null table schema", K(ret)); @@ -273,20 +335,15 @@ int ObAnalyzeStmtResolver::resolve_partition_info(const ParseNode *part_node, part_ids, subpart_ids))) { LOG_WARN("failed to get part infos", K(ret)); - } else if (OB_FAIL(analyze_stmt.set_part_ids(part_ids))) { + } else if (OB_FAIL(table_info.set_all_part_infos(part_infos))) { LOG_WARN("failed to set part infos", K(ret)); - } else if (OB_FAIL(analyze_stmt.set_subpart_ids(subpart_ids))) { - LOG_WARN("failed to set subpart infos", K(ret)); - } else if (OB_FAIL(analyze_stmt.set_all_part_infos(part_infos))) { - LOG_WARN("failed to set part infos", K(ret)); - } else if (OB_FAIL(analyze_stmt.set_all_subpart_infos(subpart_infos))) { + } else if (OB_FAIL(table_info.set_all_subpart_infos(subpart_infos))) { LOG_WARN("failed to set subpart infos", K(ret)); } else if (NULL == part_node) { - analyze_stmt.set_part_level(table_schema->get_part_level()); - analyze_stmt.set_part_cnt(0); - if (OB_FAIL(analyze_stmt.set_part_infos(part_infos))) { + table_info.set_part_level(table_schema->get_part_level()); + if (OB_FAIL(table_info.set_part_infos(part_infos))) { LOG_WARN("failed to set part infos", K(ret)); - } else if (OB_FAIL(analyze_stmt.set_subpart_infos(subpart_infos))) { + } else if (OB_FAIL(table_info.set_subpart_infos(subpart_infos))) { LOG_WARN("failed to set subpart infos", K(ret)); } } else if (is_virtual_table(table_id) && @@ -300,8 +357,7 @@ int ObAnalyzeStmtResolver::resolve_partition_info(const ParseNode *part_node, K(part_node->num_child_), K(part_node->children_[0]), K(ret)); } else { const ParseNode *name_list = part_node->children_[0]; - analyze_stmt.set_part_level(table_schema->get_part_level()); - analyze_stmt.set_part_cnt(0); + table_info.set_part_level(table_schema->get_part_level()); for (int64_t i = 0; OB_SUCC(ret) && i < name_list->num_child_; i++) { partition_name.assign_ptr(name_list->children_[i]->str_value_, static_cast(name_list->children_[i]->str_len_)); @@ -312,22 +368,19 @@ int ObAnalyzeStmtResolver::resolve_partition_info(const ParseNode *part_node, part_infos, subpart_infos, false, - analyze_stmt.get_part_infos(), - analyze_stmt.get_subpart_infos(), + table_info.get_part_infos(), + table_info.get_subpart_infos(), is_subpart_name))) { LOG_WARN("failed to find selected part infos", K(ret)); + } else { + table_info.set_is_sepcify_subpart(is_subpart_name); } } } - if (OB_SUCC(ret) && share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO == table_schema->get_part_level()) { - is_hist_subpart = (is_range_part(table_schema->get_sub_part_option().get_part_func_type()) || - is_list_part(table_schema->get_sub_part_option().get_part_func_type())); - } return ret; } int ObAnalyzeStmtResolver::resolve_statistic_info(const ParseNode *statistic_node, - const bool is_hist_subpart, ObAnalyzeStmt &analyze_stmt) { int ret = OB_SUCCESS; @@ -342,7 +395,7 @@ int ObAnalyzeStmtResolver::resolve_statistic_info(const ParseNode *statistic_nod } else { const ParseNode *for_clause_node = statistic_node->children_[0]; const ParseNode *sample_clause_node = statistic_node->children_[1]; - if (OB_FAIL(resolve_for_clause_info(for_clause_node, is_hist_subpart, analyze_stmt))) { + if (OB_FAIL(resolve_for_clause_info(for_clause_node, analyze_stmt))) { LOG_WARN("failed to resolve for clause info", K(ret)); } else if (OB_FAIL(resolve_sample_clause_info(sample_clause_node, analyze_stmt))) { LOG_WARN("failed to resolve sample clause info", K(ret)); @@ -352,26 +405,43 @@ int ObAnalyzeStmtResolver::resolve_statistic_info(const ParseNode *statistic_nod } int ObAnalyzeStmtResolver::resolve_for_clause_info(const ParseNode *for_clause_node, - const bool is_hist_subpart, ObAnalyzeStmt &analyze_stmt) { int ret = OB_SUCCESS; const ObTableSchema *table_schema = NULL; + ObIArray &tables = analyze_stmt.get_tables(); + bool is_hist_subpart = false; if (OB_ISNULL(schema_checker_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null schema checker", K(schema_checker_), K(ret)); - } else if (OB_FAIL(schema_checker_->get_table_schema(analyze_stmt.get_tenant_id(), analyze_stmt.get_table_id(), table_schema))) { - LOG_WARN("failed to get table schema", K(analyze_stmt.get_tenant_id()), K(analyze_stmt.get_table_id()), K(ret)); + } else if (NULL == for_clause_node) { + //there can be multi tables in mysql mode without for clause + for (int64_t i = 0; OB_SUCC(ret) && i < tables.count(); ++i) { + if (OB_FAIL(schema_checker_->get_table_schema(analyze_stmt.get_tenant_id(), tables.at(i).get_table_id(), table_schema))) { + LOG_WARN("failed to get table schema", K(analyze_stmt.get_tenant_id()), K(tables.at(i).get_table_id()), K(ret)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null table schema", K(ret)); + } else if (OB_FAIL(pl::ObDbmsStats::set_default_column_params(tables.at(i).get_column_params()))) { + LOG_WARN("failed to set default column params", K(ret)); + } else { + if (share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO == table_schema->get_part_level()) { + is_hist_subpart = (is_range_part(table_schema->get_sub_part_option().get_part_func_type()) || + is_list_part(table_schema->get_sub_part_option().get_part_func_type())); + } + tables.at(i).set_gather_subpart_hist(is_hist_subpart); + } + } + } else if (OB_FAIL(schema_checker_->get_table_schema(analyze_stmt.get_tenant_id(), tables.at(0).get_table_id(), table_schema))) { + LOG_WARN("failed to get table schema", K(analyze_stmt.get_tenant_id()), K(tables.at(0).get_table_id()), K(ret)); } else if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null table schema", K(ret)); - } else if (NULL == for_clause_node) { - if (OB_FAIL(pl::ObDbmsStats::set_default_column_params(analyze_stmt.get_column_params()))) { - LOG_WARN("failed to set default column params", K(ret)); - } else { - analyze_stmt.set_gather_subpart_hist(is_hist_subpart); - } } else { + if (share::schema::ObPartitionLevel::PARTITION_LEVEL_TWO == table_schema->get_part_level()) { + is_hist_subpart = (is_range_part(table_schema->get_sub_part_option().get_part_func_type()) || + is_list_part(table_schema->get_sub_part_option().get_part_func_type())); + } for (int64_t i = 0; OB_SUCC(ret) && i < for_clause_node->num_child_; i++) { if (OB_FAIL(resolve_for_clause_element(for_clause_node->children_[i], is_hist_subpart, @@ -389,6 +459,7 @@ int ObAnalyzeStmtResolver::resolve_for_clause_element(const ParseNode *for_claus { int ret = OB_SUCCESS; ObSEArray all_for_col; + ObAnalyzeTableInfo &table_info = analyze_stmt.get_tables().at(0); if (OB_ISNULL(for_clause_node)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null parse node", K(ret)); @@ -400,19 +471,19 @@ int ObAnalyzeStmtResolver::resolve_for_clause_element(const ParseNode *for_claus } else if (T_FOR_ALL == for_clause_node->type_) { bool use_size_auto = false; if (OB_FAIL(pl::ObDbmsStats::parser_for_all_clause(for_clause_node, - analyze_stmt.get_column_params(), + table_info.get_column_params(), use_size_auto))) { LOG_WARN("failed to resolve for all clause", K(ret)); } else { - analyze_stmt.set_gather_subpart_hist(!use_size_auto || (use_size_auto && is_hist_subpart)); + table_info.set_gather_subpart_hist(!use_size_auto || (use_size_auto && is_hist_subpart)); } } else if (T_FOR_COLUMNS == for_clause_node->type_) { if (OB_FAIL(pl::ObDbmsStats::parser_for_columns_clause(for_clause_node, - analyze_stmt.get_column_params(), + table_info.get_column_params(), all_for_col))) { LOG_WARN("failed to parser for columns clause", K(ret)); } else { - analyze_stmt.set_gather_subpart_hist(true); + table_info.set_gather_subpart_hist(true); } } else { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/resolver/ddl/ob_analyze_stmt_resolver.h b/src/sql/resolver/ddl/ob_analyze_stmt_resolver.h index 61a6b4b77c..01b627dff1 100644 --- a/src/sql/resolver/ddl/ob_analyze_stmt_resolver.h +++ b/src/sql/resolver/ddl/ob_analyze_stmt_resolver.h @@ -20,6 +20,7 @@ namespace oceanbase namespace sql { class ObAnalyzeStmt; +class ObAnalyzeTableInfo; class ObAnalyzeStmtResolver: public ObDDLResolver { @@ -40,14 +41,16 @@ private: ObAnalyzeStmt &analyze_stmt); int resolve_table_info(const ParseNode *table_node, ObAnalyzeStmt &analyze_stmt); + int recursive_resolve_table_info(const ParseNode *table_list_node, + ObAnalyzeStmt &analyze_stmt); int resolve_partition_info(const ParseNode *part_node, - ObAnalyzeStmt &analyze_stmt, - bool &is_hist_subpart); + ObAnalyzeStmt &analyze_stmt); + int inner_resolve_partition_info(const ParseNode *part_node, + const uint64_t tenant_id, + ObAnalyzeTableInfo &table_info); int resolve_statistic_info(const ParseNode *statistic_node, - const bool is_hist_subpart, ObAnalyzeStmt &analyze_stmt); int resolve_for_clause_info(const ParseNode *for_clause_node, - const bool is_hist_subpart, ObAnalyzeStmt &analyze_stmt); int resolve_for_clause_element(const ParseNode *for_clause_node, const bool is_hist_subpart, diff --git a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp index 71cf2929e0..d1b1166ff3 100644 --- a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp @@ -1849,7 +1849,7 @@ int ObRawExprDeduceType::visit(ObAggFunRawExpr &expr) if (OB_ISNULL(param_expr = expr.get_param_expr(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(param_expr), K(expr.get_param_count())); - } else if (i == 0 || i == 2) { + } else if (i == 0 || i == 2 || i == 3) { if (lib::is_oracle_mode()) { const_cast(param_expr->get_result_type()).set_calc_type(ObNumberType); } else { diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index ced82f7162..74ca1dafa8 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -4146,10 +4146,11 @@ int ObRawExprResolverImpl::process_agg_node(const ParseNode *node, ObRawExpr *&e ObSysFunRawExpr *window_size_expr = NULL; ObRawExpr *div_expr = NULL; ObConstRawExpr *one_expr = NULL; + ObRawExpr *max_disuse_expr = NULL; if (OB_ISNULL(ctx_.session_info_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(ctx_.session_info_)); - } else if (OB_UNLIKELY(3 != node->num_child_)) { + } else if (OB_UNLIKELY(4 != node->num_child_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(node->num_child_)); } else if (OB_FAIL(SMART_CALL(recursive_resolve(node->children_[0], error_expr)))) { @@ -4193,6 +4194,14 @@ int ObRawExprResolverImpl::process_agg_node(const ParseNode *node, ObRawExpr *&e LOG_WARN("get invalid item_size_expr", K(ret), K(item_size_expr)); } else if (OB_FAIL(agg_expr->add_real_param_expr(item_size_expr))) { LOG_WARN("fail to add param expr to agg expr", K(ret)); + } else if (OB_FAIL(SMART_CALL(recursive_resolve(node->children_[3], max_disuse_expr)))) { + LOG_WARN("fail to recursive resolve expr item", K(ret)); + } else if (OB_ISNULL(max_disuse_expr) || + OB_UNLIKELY(ObRawExpr::EXPR_CONST != max_disuse_expr->get_expr_class())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid item_size_expr", K(ret), K(max_disuse_expr)); + } else if (OB_FAIL(agg_expr->add_real_param_expr(max_disuse_expr))) { + LOG_WARN("fail to add param expr to agg expr", K(ret)); } else {/*do nothing*/} } else if (T_FUN_HYBRID_HIST == node->type_ || T_FUN_JSON_OBJECTAGG == node->type_) { if (OB_UNLIKELY(2 != node->num_child_)) { diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 07c19e6650..961ef11aa7 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -896,7 +896,8 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS } case T_ANALYZE: case T_MYSQL_UPDATE_HISTOGRAM: - case T_MYSQL_DROP_HISTOGRAM: { + case T_MYSQL_DROP_HISTOGRAM: + case T_MYSQL_ANALYZE: { REGISTER_STMT_RESOLVER(AnalyzeStmt); break; } diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index 13f54cb4a7..e6021dc6ab 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -4779,6 +4779,7 @@ COMPLETED_TABLE_COUNT bigint(20) NO NULL RUNNING_TABLE_OWNER varchar(128) NO NULL RUNNING_TABLE_NAME varchar(256) NO NULL RUNNING_TABLE_DURATION_TIME bigint(20) NO NULL +RUNNING_TABLE_PROGRESS varchar(256) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.GV$OB_OPT_STAT_GATHER_MONITOR limit 1); cnt 1 @@ -4798,6 +4799,7 @@ COMPLETED_TABLE_COUNT bigint(20) NO RUNNING_TABLE_OWNER varchar(128) NO RUNNING_TABLE_NAME varchar(256) NO RUNNING_TABLE_DURATION_TIME bigint(20) NO +RUNNING_TABLE_PROGRESS varchar(256) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_OPT_STAT_GATHER_MONITOR limit 1); cnt 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index b5f153ac0f..13e4c22988 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -6676,6 +6676,7 @@ COMPLETED_TABLE_COUNT bigint(20) NO NULL RUNNING_TABLE_OWNER varchar(128) NO NULL RUNNING_TABLE_NAME varchar(256) NO NULL RUNNING_TABLE_DURATION_TIME bigint(20) NO NULL +RUNNING_TABLE_PROGRESS varchar(256) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.GV$OB_OPT_STAT_GATHER_MONITOR limit 1); cnt 1 @@ -6695,6 +6696,7 @@ COMPLETED_TABLE_COUNT bigint(20) NO RUNNING_TABLE_OWNER varchar(128) NO RUNNING_TABLE_NAME varchar(256) NO RUNNING_TABLE_DURATION_TIME bigint(20) NO +RUNNING_TABLE_PROGRESS varchar(256) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_OPT_STAT_GATHER_MONITOR limit 1); cnt 1