fix cancel drop index hang due to check standby tenant failed.

This commit is contained in:
obdev
2023-03-17 03:13:50 +00:00
committed by ob-robot
parent 1abd119e08
commit 105bf1c2f4
2 changed files with 99 additions and 74 deletions

View File

@ -73,20 +73,26 @@ int ObDDLExecutorUtil::wait_ddl_finish(
FORWARD_USER_ERROR(ret, error_message.user_message_); FORWARD_USER_ERROR(ret, error_message.user_message_);
} }
break; break;
} else {
if (OB_FAIL(ret)) {
} else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped( } else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped(
tenant_id, is_tenant_dropped))) { tenant_id, is_tenant_dropped))) {
LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id)); LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id));
} else if (is_tenant_dropped) { } else if (is_tenant_dropped) {
ret = OB_TENANT_HAS_BEEN_DROPPED; ret = OB_TENANT_HAS_BEEN_DROPPED;
LOG_WARN("tenant has been dropped", K(ret), K(tenant_id)); LOG_WARN("tenant has been dropped", K(ret), K(tenant_id));
break; }
if (OB_FAIL(ret)) {
} else if (OB_TMP_FAIL(ObAllTenantInfoProxy::is_standby_tenant(GCTX.sql_proxy_, tenant_id, is_tenant_standby))) { } else if (OB_TMP_FAIL(ObAllTenantInfoProxy::is_standby_tenant(GCTX.sql_proxy_, tenant_id, is_tenant_standby))) {
LOG_WARN("check is standby tenant failed", K(tmp_ret), K(tenant_id)); LOG_WARN("check is standby tenant failed", K(tmp_ret), K(tenant_id));
} else if (is_tenant_standby) { } else if (is_tenant_standby) {
ret = OB_STANDBY_READ_ONLY; ret = OB_STANDBY_READ_ONLY;
FORWARD_USER_ERROR(ret, "DDL not finish, need check"); FORWARD_USER_ERROR(ret, "DDL not finish, need check");
LOG_WARN("tenant is standby now, stop wait", K(ret), K(tenant_id)); LOG_WARN("tenant is standby now, stop wait", K(ret), K(tenant_id));
break; }
if (OB_FAIL(ret)) {
} else if (OB_FAIL(handle_session_exception(session))) { } else if (OB_FAIL(handle_session_exception(session))) {
LOG_WARN("session exeception happened", K(ret), K(is_support_cancel)); LOG_WARN("session exeception happened", K(ret), K(is_support_cancel));
if (is_support_cancel && OB_TMP_FAIL(cancel_ddl_task(tenant_id, common_rpc_proxy))) { if (is_support_cancel && OB_TMP_FAIL(cancel_ddl_task(tenant_id, common_rpc_proxy))) {
@ -100,6 +106,7 @@ int ObDDLExecutorUtil::wait_ddl_finish(
} }
} }
} }
}
return ret; return ret;
} }
@ -125,12 +132,17 @@ int ObDDLExecutorUtil::wait_build_index_finish(const uint64_t tenant_id, const i
FORWARD_USER_ERROR(ret, error_message.user_message_); FORWARD_USER_ERROR(ret, error_message.user_message_);
} }
is_finish = true; is_finish = true;
} else {
if (OB_FAIL(ret)) {
} else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped( } else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped(
tenant_id, is_tenant_dropped))) { tenant_id, is_tenant_dropped))) {
LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id)); LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id));
} else if (is_tenant_dropped) { } else if (is_tenant_dropped) {
ret = OB_TENANT_HAS_BEEN_DROPPED; ret = OB_TENANT_HAS_BEEN_DROPPED;
LOG_WARN("tenant has been dropped", K(ret), K(tenant_id)); LOG_WARN("tenant has been dropped", K(ret), K(tenant_id));
}
if (OB_FAIL(ret)) {
} else if (OB_TMP_FAIL(ObAllTenantInfoProxy::is_standby_tenant(GCTX.sql_proxy_, tenant_id, is_tenant_standby))) { } else if (OB_TMP_FAIL(ObAllTenantInfoProxy::is_standby_tenant(GCTX.sql_proxy_, tenant_id, is_tenant_standby))) {
LOG_WARN("check is standby tenant failed", K(tmp_ret), K(tenant_id)); LOG_WARN("check is standby tenant failed", K(tmp_ret), K(tenant_id));
} else if (is_tenant_standby) { } else if (is_tenant_standby) {
@ -138,6 +150,7 @@ int ObDDLExecutorUtil::wait_build_index_finish(const uint64_t tenant_id, const i
FORWARD_USER_ERROR(ret, "DDL not finish, need check"); FORWARD_USER_ERROR(ret, "DDL not finish, need check");
LOG_WARN("tenant is standby now, stop wait", K(ret), K(tenant_id)); LOG_WARN("tenant is standby now, stop wait", K(ret), K(tenant_id));
} }
}
return ret; return ret;
} }
@ -198,13 +211,17 @@ int ObDDLExecutorUtil::wait_ddl_retry_task_finish(
} }
} }
break; break;
} else {
if (OB_FAIL(ret)) {
} else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped( } else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped(
tenant_id, is_tenant_dropped))) { tenant_id, is_tenant_dropped))) {
LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id)); LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id));
} else if (is_tenant_dropped) { } else if (is_tenant_dropped) {
ret = OB_TENANT_HAS_BEEN_DROPPED; ret = OB_TENANT_HAS_BEEN_DROPPED;
LOG_WARN("tenant has been dropped", K(ret), K(tenant_id)); LOG_WARN("tenant has been dropped", K(ret), K(tenant_id));
break; }
if (OB_FAIL(ret)) {
} else if (OB_TMP_FAIL(ObAllTenantInfoProxy::is_standby_tenant(GCTX.sql_proxy_, tenant_id, is_tenant_standby))) { } else if (OB_TMP_FAIL(ObAllTenantInfoProxy::is_standby_tenant(GCTX.sql_proxy_, tenant_id, is_tenant_standby))) {
LOG_WARN("check is standby tenant failed", K(tmp_ret), K(tenant_id)); LOG_WARN("check is standby tenant failed", K(tmp_ret), K(tenant_id));
} else if (is_tenant_standby) { } else if (is_tenant_standby) {
@ -212,6 +229,8 @@ int ObDDLExecutorUtil::wait_ddl_retry_task_finish(
FORWARD_USER_ERROR(ret, "DDL not finish, need check"); FORWARD_USER_ERROR(ret, "DDL not finish, need check");
LOG_WARN("tenant is standby now, stop wait", K(ret), K(tenant_id)); LOG_WARN("tenant is standby now, stop wait", K(ret), K(tenant_id));
break; break;
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(handle_session_exception(session))) { } else if (OB_FAIL(handle_session_exception(session))) {
LOG_WARN("session exeception happened", K(ret)); LOG_WARN("session exeception happened", K(ret));
if (OB_TMP_FAIL(cancel_ddl_task(tenant_id, common_rpc_proxy))) { if (OB_TMP_FAIL(cancel_ddl_task(tenant_id, common_rpc_proxy))) {
@ -224,6 +243,7 @@ int ObDDLExecutorUtil::wait_ddl_retry_task_finish(
ob_usleep(retry_interval); ob_usleep(retry_interval);
} }
} }
}
affected_rows = error_message.affected_rows_; affected_rows = error_message.affected_rows_;
} }
return ret; return ret;

View File

@ -301,20 +301,24 @@ int ObDropIndexExecutor::wait_drop_index_finish(
FORWARD_USER_ERROR(ret, error_message.user_message_); FORWARD_USER_ERROR(ret, error_message.user_message_);
} }
break; break;
} else {
if (OB_FAIL(ret)) {
} else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped( } else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped(
tenant_id, is_tenant_dropped))) { tenant_id, is_tenant_dropped))) {
LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id)); LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id));
} else if (is_tenant_dropped) { } else if (is_tenant_dropped) {
ret = OB_TENANT_HAS_BEEN_DROPPED; ret = OB_TENANT_HAS_BEEN_DROPPED;
LOG_WARN("tenant has been dropped", K(ret), K(tenant_id)); LOG_WARN("tenant has been dropped", K(ret), K(tenant_id));
break; }
if (OB_FAIL(ret)) {
} else if (OB_TMP_FAIL(ObAllTenantInfoProxy::is_standby_tenant(GCTX.sql_proxy_, tenant_id, is_tenant_standby))) { } else if (OB_TMP_FAIL(ObAllTenantInfoProxy::is_standby_tenant(GCTX.sql_proxy_, tenant_id, is_tenant_standby))) {
LOG_WARN("check is standby tenant failed", K(tmp_ret), K(tenant_id)); LOG_WARN("check is standby tenant failed", K(tmp_ret), K(tenant_id));
} else if (is_tenant_standby) { } else if (is_tenant_standby) {
ret = OB_STANDBY_READ_ONLY; ret = OB_STANDBY_READ_ONLY;
FORWARD_USER_ERROR(ret, "DDL not finish, need check"); FORWARD_USER_ERROR(ret, "DDL not finish, need check");
LOG_WARN("tenant is standby now, stop wait", K(ret), K(tenant_id)); LOG_WARN("tenant is standby now, stop wait", K(ret), K(tenant_id));
break; }
if (OB_FAIL(ret)) {
} else if (OB_FAIL(session.check_session_status())) { } else if (OB_FAIL(session.check_session_status())) {
LOG_WARN("session exeception happened", K(ret)); LOG_WARN("session exeception happened", K(ret));
} else { } else {
@ -322,6 +326,7 @@ int ObDropIndexExecutor::wait_drop_index_finish(
} }
} }
} }
}
return ret; return ret;
} }