diff --git a/deps/oblib/src/common/ob_timeout_ctx.cpp b/deps/oblib/src/common/ob_timeout_ctx.cpp index 324aefbf0e..7fdb336271 100644 --- a/deps/oblib/src/common/ob_timeout_ctx.cpp +++ b/deps/oblib/src/common/ob_timeout_ctx.cpp @@ -148,5 +148,13 @@ int64_t ObTimeoutCtx::get_trx_timeout_us() const } return timeout; } + +// Attention!! This function will ignore the result of previous ObTimeouCtx. +void ObTimeoutCtx::reset_timeout_us() +{ + abs_timeout_us_ = -1; + trx_timeout_us_ = -1; +} + } // end namespace common } // end namespace oceanbase diff --git a/deps/oblib/src/common/ob_timeout_ctx.h b/deps/oblib/src/common/ob_timeout_ctx.h index 5cb47af886..7ecbb431a4 100644 --- a/deps/oblib/src/common/ob_timeout_ctx.h +++ b/deps/oblib/src/common/ob_timeout_ctx.h @@ -50,6 +50,8 @@ public: // return previous timeout context or an initialized one if no previous context. static const ObTimeoutCtx &get_ctx(); + void reset_timeout_us(); + TO_STRING_KV(K_(abs_timeout_us)); private: diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 9c0e55aeb0..49f06427d5 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -26676,13 +26676,10 @@ int ObDDLService::refresh_schema(uint64_t tenant_id, int64_t *publish_schema_ver LOG_WARN("fail to push back tenant_id", KR(ret), K(tenant_id)); } while (!stopped_) { + // reset ctx to retry to die common::ObTimeoutCtx ctx; - if (OB_FAIL(schema_service_->set_timeout_ctx(ctx))) { - LOG_ERROR("fail to set timeout_ctx, refresh schema failed", KR(ret), K(tenant_id)); - break; - } else { - ret = schema_service_->refresh_and_add_schema(tenant_ids); - } + ctx.reset_timeout_us(); + ret = schema_service_->refresh_and_add_schema(tenant_ids); if (OB_SUCC(ret)) { break; @@ -26704,7 +26701,7 @@ int ObDDLService::refresh_schema(uint64_t tenant_id, int64_t *publish_schema_ver } ob_usleep(REFRESH_SCHEMA_INTERVAL_US); } - } + } // end while if (OB_SUCC(ret) && !stopped_) { int64_t schema_version = OB_INVALID_VERSION; if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version( diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index f4fc7fdd29..1b3c2f0a26 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -561,6 +561,7 @@ class ObString; ACT(BEFORE_CHECK_LS_TRANSFER_SCN_FOR_STANDBY,)\ ACT(BEFORE_GET_CONFIG_VERSION_AND_TRANSFER_SCN,)\ ACT(LS_GC_BEFORE_OFFLINE,)\ + ACT(BEFORE_FETCH_SIMPLE_TABLES,)\ ACT(BEFORE_DROP_TENANT,)\ ACT(MAX_DEBUG_SYNC_POINT,) diff --git a/src/share/schema/ob_schema_service_sql_impl.cpp b/src/share/schema/ob_schema_service_sql_impl.cpp index cb07284cea..4ad48e7600 100644 --- a/src/share/schema/ob_schema_service_sql_impl.cpp +++ b/src/share/schema/ob_schema_service_sql_impl.cpp @@ -4826,6 +4826,7 @@ int ObSchemaServiceSQLImpl::fetch_tables( const uint64_t exec_tenant_id = fill_exec_tenant_id(schema_status); const char *table_name = NULL; int64_t start_time = ObTimeUtility::current_time(); + DEBUG_SYNC(BEFORE_FETCH_SIMPLE_TABLES); if (!check_inner_stat()) { ret = OB_NOT_INIT; LOG_WARN("check inner stat fail", K(ret)); diff --git a/src/share/schema/ob_server_schema_service.cpp b/src/share/schema/ob_server_schema_service.cpp index a274ec971a..b9a613cce7 100644 --- a/src/share/schema/ob_server_schema_service.cpp +++ b/src/share/schema/ob_server_schema_service.cpp @@ -5593,8 +5593,6 @@ int ObServerSchemaService::refresh_increment_schema( const ObRefreshSchemaStatus &schema_status) { int ret = OB_SUCCESS; - ObTimeoutCtx ctx; - observer::ObUseWeakGuard use_weak_guard; const uint64_t tenant_id = schema_status.tenant_id_; ObSchemaMgr *schema_mgr_for_cache = NULL; if (!check_inner_stat()) { @@ -5613,16 +5611,27 @@ int ObServerSchemaService::refresh_increment_schema( int64_t core_schema_version = 0; int64_t schema_version = OB_INVALID_VERSION; int64_t retry_count = 0; + const int64_t start_ts = ObTimeUtility::current_time(); + int64_t abs_timeout = OB_INVALID_TIMESTAMP; + if (OB_FAIL(ObShareUtil::get_abs_timeout(MAX_FETCH_SCHEMA_TIMEOUT_US, abs_timeout))) { + LOG_WARN("fail to get abs timeout", KR(ret)); + } while (OB_SUCC(ret)) { if (OB_FAIL(check_stop())) { LOG_WARN("observer is stopping", KR(ret), K(schema_status)); break; } else if (retry_count > 0) { LOG_WARN("refresh_increment_schema failed", K(retry_count), K(schema_status)); - if (OB_FAIL(set_timeout_ctx(ctx))) { - LOG_WARN("fail to set timeout ctx", KR(ret), K(schema_status)); + + const int64_t current_ts = ObTimeUtility::current_time(); + if (current_ts >= abs_timeout) { + // ret will be overwrite when core/system table schemas were changed in the meantime. + // In such situations, try use timeout remain to retry locally. + ret = OB_TIMEOUT; + LOG_WARN("already timeout", KR(ret), K(start_ts), K(abs_timeout), K(current_ts), K(abs_timeout)); break; } + } ObISQLClient &sql_client = *sql_proxy_; if (OB_SUCC(ret) && core_schema_change) { @@ -5774,37 +5783,6 @@ int ObServerSchemaService::refresh_increment_schema( return ret; } -int ObServerSchemaService::set_timeout_ctx(ObTimeoutCtx &ctx) -{ - int ret = OB_SUCCESS; - int64_t abs_timeout_us = ctx.get_abs_timeout(); - int64_t worker_timeout_us = THIS_WORKER.get_timeout_ts(); - - if (abs_timeout_us < 0) { - abs_timeout_us = ObTimeUtility::current_time() + DEFAULT_FETCH_SCHEMA_TIMEOUT_US; - } - - if (INT64_MAX == worker_timeout_us) { - // The background schema refresh task triggered by the heartbeat, the system tenant schemea - // needs to retry until it succeeds - abs_timeout_us = ObTimeUtility::current_time() + MAX_FETCH_SCHEMA_TIMEOUT_US; - } else if (worker_timeout_us > 0 && worker_timeout_us < abs_timeout_us) { - abs_timeout_us = worker_timeout_us; - } - - if (OB_FAIL(ctx.set_abs_timeout(abs_timeout_us))) { - LOG_WARN("set timeout failed", K(ret), K(abs_timeout_us)); - } else if (ctx.is_timeouted()) { - ret = OB_TIMEOUT; - LOG_WARN("is timeout", - K(ret), - "abs_timeout", ctx.get_abs_timeout(), - "this worker timeout ts", THIS_WORKER.get_timeout_ts()); - } - - return ret; -} - int ObServerSchemaService::try_fetch_publish_core_schemas( const ObRefreshSchemaStatus &schema_status, const int64_t core_schema_version, diff --git a/src/share/schema/ob_server_schema_service.h b/src/share/schema/ob_server_schema_service.h index df722c3a54..36a8699604 100644 --- a/src/share/schema/ob_server_schema_service.h +++ b/src/share/schema/ob_server_schema_service.h @@ -878,8 +878,6 @@ public: int get_refresh_schema_info(ObRefreshSchemaInfo &schema_info); - int set_timeout_ctx(common::ObTimeoutCtx &ctx); - // Fetch increments schemas in DDL trans. This interface won't return the following increment schemas: // 1. schema which is dropped in DDL trans. // 2. changed inner tables.