diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp index 5359f2e7c0..dad0708001 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp @@ -2432,7 +2432,7 @@ int ObSyncTabletAutoincSeqCtx::sync() } } } - if (OB_LS_NOT_EXIST == ret || is_location_service_renew_error(ret)) { + if (share::ObIDDLTask::in_ddl_retry_white_list(ret)) { need_renew_location_ = true; } return ret; diff --git a/src/rootserver/ddl_task/ob_ddl_task.cpp b/src/rootserver/ddl_task/ob_ddl_task.cpp index f840142d23..c34e3226e8 100644 --- a/src/rootserver/ddl_task/ob_ddl_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_task.cpp @@ -979,11 +979,10 @@ int ObDDLTask::switch_status(const ObDDLTaskStatus new_status, const bool enable if (OB_ISNULL(root_service = GCTX.root_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("error unexpected, root service must not be nullptr", K(ret)); - } else if (OB_FAIL(ObDDLUtil::check_tenant_status_normal(&root_service->get_sql_proxy(), dst_tenant_id_)) - || OB_FAIL(ObDDLUtil::check_tenant_status_normal(&root_service->get_sql_proxy(), tenant_id_))) { + } else if (OB_FAIL(ObDDLUtil::check_tenant_status_normal(&root_service->get_sql_proxy(), dst_tenant_id_))) { if (OB_TENANT_HAS_BEEN_DROPPED == ret || OB_STANDBY_READ_ONLY == ret) { need_retry_ = false; - LOG_INFO("tenant status is abnormal, exit anyway", K(ret), K(task_id_), K(parent_task_id_), K(tenant_id_), K(dst_tenant_id_)); + LOG_INFO("tenant status is abnormal, exit anyway", K(ret), K_(task_id), K_(parent_task_id), K_(dst_tenant_id)); } } else if (OB_FAIL(trans.start(&root_service->get_sql_proxy(), dst_tenant_id_))) { LOG_WARN("start transaction failed", K(ret)); @@ -1059,33 +1058,17 @@ int ObDDLTask::refresh_status() int ObDDLTask::refresh_schema_version() { int ret = OB_SUCCESS; - ObMultiVersionSchemaService &schema_service = ObMultiVersionSchemaService::get_instance(); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObDDLTask has not been inited", K(ret)); - } - - if (OB_SUCC(ret) && (schema_version_ > 0 && schema_version_ != UINT64_MAX)) { - int64_t refreshed_schema_version = 0; - if (OB_FAIL(schema_service.get_tenant_refreshed_schema_version(tenant_id_, refreshed_schema_version))) { - LOG_WARN("get refreshed schema version failed", K(ret), K(tenant_id_)); - } else if (!ObSchemaService::is_formal_version(refreshed_schema_version) || refreshed_schema_version < schema_version_) { - ret = OB_SCHEMA_EAGAIN; - if (REACH_TIME_INTERVAL(1000L * 1000L)) { - LOG_INFO("tenant schema not refreshed to the target version", K(ret), K(tenant_id_), K(schema_version_), K(refreshed_schema_version)); - } + } else if (OB_FAIL(ObDDLUtil::check_schema_version_refreshed(tenant_id_, schema_version_))) { + if (OB_SCHEMA_EAGAIN != ret) { + LOG_WARN("check schema version refreshed failed", K(ret), K_(tenant_id), K_(schema_version)); } - } - - if (OB_SUCC(ret) && (tenant_id_ != dst_tenant_id_) && (dst_schema_version_ > 0 && dst_schema_version_ != UINT64_MAX)) { - int64_t refreshed_schema_version = 0; - if (OB_FAIL(schema_service.get_tenant_refreshed_schema_version(dst_tenant_id_, refreshed_schema_version))) { - LOG_WARN("get refreshed schema version failed", K(ret), K(dst_tenant_id_)); - } else if (!ObSchemaService::is_formal_version(refreshed_schema_version) || refreshed_schema_version < dst_schema_version_) { - ret = OB_SCHEMA_EAGAIN; - if (REACH_TIME_INTERVAL(1000L * 1000L)) { - LOG_INFO("tenant schema not refreshed to the target version", K(ret), K(dst_tenant_id_), K(dst_schema_version_), K(refreshed_schema_version)); - } + } else if (tenant_id_ == dst_tenant_id_) { + } else if (OB_FAIL(ObDDLUtil::check_schema_version_refreshed(dst_tenant_id_, dst_schema_version_))) { + if (OB_SCHEMA_EAGAIN != ret) { + LOG_WARN("check schema version refreshed failed", K(ret), K_(dst_tenant_id), K_(dst_schema_version)); } } return ret; @@ -1119,7 +1102,7 @@ int ObDDLTask::report_error_code(const ObString &forward_user_message, const int if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObIndexBuildTask has not been inited", K(ret)); - } else if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_table_id(tenant_id_, object_id_, is_oracle_mode))) { + } else if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_table_id(dst_tenant_id_, object_id_, is_oracle_mode))) { LOG_WARN("check if oracle mode failed", K(ret), K(object_id_)); } else { ObDDLErrorMessageTableOperator::ObBuildDDLErrorMessage error_message; diff --git a/src/rootserver/ddl_task/ob_recover_restore_table_task.cpp b/src/rootserver/ddl_task/ob_recover_restore_table_task.cpp index 982183db1f..07f8b8c11b 100755 --- a/src/rootserver/ddl_task/ob_recover_restore_table_task.cpp +++ b/src/rootserver/ddl_task/ob_recover_restore_table_task.cpp @@ -161,7 +161,7 @@ int ObRecoverRestoreTableTask::success() int ObRecoverRestoreTableTask::fail() { int ret = OB_SUCCESS; - ObArenaAllocator tmp_arena; + ObArenaAllocator tmp_arena("RestoreDDLClean"); int64_t rpc_timeout = 0; int64_t all_orig_index_tablet_count = 0; const ObDatabaseSchema *db_schema = nullptr; @@ -171,8 +171,8 @@ int ObRecoverRestoreTableTask::fail() obrpc::ObTableItem table_item; obrpc::ObDropTableArg drop_table_arg; obrpc::ObDDLRes drop_table_res; + bool need_cleanup = true; { - ObSchemaGetterGuard src_tenant_schema_guard; ObSchemaGetterGuard dst_tenant_schema_guard; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -180,16 +180,13 @@ int ObRecoverRestoreTableTask::fail() } else if (OB_ISNULL(root_service)) { ret = OB_ERR_SYS; LOG_WARN("error sys, root service must not be nullptr", K(ret)); - } else if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(tenant_id_, src_tenant_schema_guard))) { - LOG_WARN("get schema guard failed", K(ret), K(tenant_id_)); - } else if (OB_FAIL(get_orig_all_index_tablet_count(src_tenant_schema_guard, all_orig_index_tablet_count))) { - LOG_WARN("get orig all tablet count failed", K(ret)); } else if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard(dst_tenant_id_, dst_tenant_schema_guard))) { LOG_WARN("get schema guard failed", K(ret), K(dst_tenant_id_)); } else if (OB_FAIL(dst_tenant_schema_guard.get_table_schema(dst_tenant_id_, target_object_id_, table_schema))) { LOG_WARN("get table schema failed", K(ret), K(dst_tenant_id_), K(target_object_id_)); } else if (OB_ISNULL(table_schema)) { // already dropped. + need_cleanup = false; LOG_INFO("already dropped", K(ret), K(dst_tenant_id_), K(target_object_id_)); } else if (OB_FAIL(dst_tenant_schema_guard.get_database_schema(dst_tenant_id_, table_schema->get_database_id(), db_schema))) { LOG_WARN("get db schema failed", K(ret), K(dst_tenant_id_), KPC(table_schema)); @@ -218,11 +215,11 @@ int ObRecoverRestoreTableTask::fail() drop_table_arg.compat_mode_ = is_oracle_mode ? lib::Worker::CompatMode::ORACLE : lib::Worker::CompatMode::MYSQL; } } - if (OB_SUCC(ret)) { - obrpc::ObCommonRpcProxy common_rpc_proxy = root_service->get_common_rpc_proxy().to(GCTX.self_addr()).timeout(rpc_timeout); + if (OB_SUCC(ret) && need_cleanup) { if (OB_FAIL(drop_table_arg.tables_.push_back(table_item))) { LOG_WARN("push back failed", K(ret), K(drop_table_arg)); - } else if (OB_FAIL(common_rpc_proxy.drop_table(drop_table_arg, drop_table_res))) { + } else if (OB_FAIL(root_service->get_common_rpc_proxy().to(GCTX.self_addr()) + .timeout(rpc_timeout).drop_table(drop_table_arg, drop_table_res))) { LOG_WARN("drop table failed", K(ret), K(rpc_timeout), K(drop_table_arg)); } } @@ -234,6 +231,43 @@ int ObRecoverRestoreTableTask::fail() return ret; } +int ObRecoverRestoreTableTask::check_health() +{ + int ret = OB_SUCCESS; + ObRootService *root_service = GCTX.root_service_; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_ISNULL(root_service)) { + ret = OB_ERR_SYS; + LOG_WARN("error sys", K(ret)); + } else if (!root_service->in_service()) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("root service not in service, do not need retry", K(ret), K(object_id_), K(target_object_id_)); + need_retry_ = false; + } else if (OB_FAIL(ObDDLUtil::check_tenant_status_normal(&root_service->get_sql_proxy(), tenant_id_))) { + // switch to build failed if the source tenant is been dropped, + // in order to remove the destination tenant's persistent task record. + if (OB_TENANT_HAS_BEEN_DROPPED == ret) { + const ObDDLTaskStatus old_status = static_cast(task_status_); + const ObDDLTaskStatus new_status = ObDDLTaskStatus::FAIL; + int tmp_ret = switch_status(new_status, false, ret); + LOG_INFO("switch status to build_failed", K(ret), K(tmp_ret), K_(task_status), K(old_status), K(new_status)); + ret = OB_SUCCESS; + } else if (OB_STANDBY_READ_ONLY == ret) { + // do not care about the role of the source tenant is expected. + if (OB_FAIL(ObDDLRedefinitionTask::check_health())) { + LOG_WARN("check health failed", K(ret), K_(task_status)); + } + } else { + LOG_WARN("check tenant status normal failed", K(ret), K_(tenant_id)); + } + } else if (OB_FAIL(ObDDLRedefinitionTask::check_health())) { + LOG_WARN("check health failed", K(ret), K_(task_status)); + } + return ret; +} + int ObRecoverRestoreTableTask::process() { int ret = OB_SUCCESS; diff --git a/src/rootserver/ddl_task/ob_recover_restore_table_task.h b/src/rootserver/ddl_task/ob_recover_restore_table_task.h index ac0aca1f2c..a5125f2e57 100644 --- a/src/rootserver/ddl_task/ob_recover_restore_table_task.h +++ b/src/rootserver/ddl_task/ob_recover_restore_table_task.h @@ -57,6 +57,8 @@ protected: virtual int obtain_snapshot(const share::ObDDLTaskStatus next_task_status) override; virtual int fail() override; virtual int success() override; +private: + int check_health(); private: static const int64_t OB_RECOVER_RESTORE_TABLE_TASK_VERSION = 1L; }; diff --git a/src/share/ob_ddl_common.cpp b/src/share/ob_ddl_common.cpp index a0fb5b14f3..d4dd2082f4 100644 --- a/src/share/ob_ddl_common.cpp +++ b/src/share/ob_ddl_common.cpp @@ -1383,6 +1383,27 @@ int ObDDLUtil::check_tenant_status_normal( return ret; } +int ObDDLUtil::check_schema_version_refreshed( + const uint64_t tenant_id, + const int64_t target_schema_version) +{ + int ret = OB_SUCCESS; + int64_t refreshed_schema_version = 0; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || target_schema_version <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret), K(tenant_id), K(target_schema_version)); + } else if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_refreshed_schema_version( + tenant_id, refreshed_schema_version))) { + LOG_WARN("get refreshed schema version failed", K(ret), K(tenant_id), K(refreshed_schema_version)); + } else if (!ObSchemaService::is_formal_version(refreshed_schema_version) || refreshed_schema_version < target_schema_version) { + ret = OB_SCHEMA_EAGAIN; + if (REACH_TIME_INTERVAL(1000L * 1000L)) { + LOG_INFO("tenant schema not refreshed to the target version", K(ret), K(tenant_id), K(target_schema_version), K(refreshed_schema_version)); + } + } + return ret; +} + /****************** ObCheckTabletDataComplementOp *************/ int ObCheckTabletDataComplementOp::check_task_inner_sql_session_status( diff --git a/src/share/ob_ddl_common.h b/src/share/ob_ddl_common.h index bb72c76d15..f081ff9ad8 100644 --- a/src/share/ob_ddl_common.h +++ b/src/share/ob_ddl_common.h @@ -499,6 +499,9 @@ public: static int check_tenant_status_normal( ObISQLClient *proxy, const uint64_t check_tenant_id); + static int check_schema_version_refreshed( + const uint64_t tenant_id, + const int64_t target_schema_version); private: static int generate_order_by_str( const ObIArray &select_column_ids, diff --git a/src/storage/ddl/ob_complement_data_task.cpp b/src/storage/ddl/ob_complement_data_task.cpp index 65dbe4dfa8..0ab83ac973 100644 --- a/src/storage/ddl/ob_complement_data_task.cpp +++ b/src/storage/ddl/ob_complement_data_task.cpp @@ -80,7 +80,16 @@ int ObComplementDataParam::init(const ObDDLBuildSingleReplicaRequestArg &arg) LOG_WARN("invalid arg", K(ret), K(arg)); } else { MTL_SWITCH (OB_SYS_TENANT_ID) { - if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard( + if (OB_FAIL(ObDDLUtil::check_schema_version_refreshed(orig_tenant_id, orig_schema_version))) { + if (OB_SCHEMA_EAGAIN != ret) { + LOG_WARN("check schema version refreshed failed", K(ret), K(orig_tenant_id), K(orig_schema_version)); + } + } else if (orig_tenant_id != dest_tenant_id + && OB_FAIL(ObDDLUtil::check_schema_version_refreshed(dest_tenant_id, dest_schema_version))) { + if (OB_SCHEMA_EAGAIN != ret) { + LOG_WARN("check schema version refreshed failed", K(ret), K(dest_tenant_id), K(dest_schema_version)); + } + } else if (OB_FAIL(ObMultiVersionSchemaService::get_instance().get_tenant_schema_guard( orig_tenant_id, src_tenant_schema_guard, orig_schema_version))) { LOG_WARN("fail to get tenant schema guard", K(ret), K(orig_tenant_id), K(orig_schema_version)); } else if (OB_FAIL(src_tenant_schema_guard.get_tenant_info(orig_tenant_id, tenant_schema))) {