diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 4c5c6764c4..52575eee2f 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -2915,15 +2915,15 @@ int ObServer::clean_up_invalid_tables_by_tenant( int tmp_ret = OB_SUCCESS; ObSchemaGetterGuard schema_guard; const ObDatabaseSchema *database_schema = NULL; - ObSEArray table_schemas; const int64_t CONNECT_TIMEOUT_VALUE = 50L * 60L * 60L * 1000L * 1000L; //default value is 50hrs + ObArray table_ids; obrpc::ObDropTableArg drop_table_arg; obrpc::ObTableItem table_item; obrpc::ObCommonRpcProxy *common_rpc_proxy = NULL; char create_host_str[OB_MAX_HOST_NAME_LENGTH]; if (OB_FAIL(schema_service_.get_tenant_schema_guard(tenant_id, schema_guard))) { LOG_WARN("fail to get schema guard", K(ret)); - } else if (OB_FAIL(schema_guard.get_table_schemas_in_tenant(tenant_id, table_schemas))) { + } else if (OB_FAIL(schema_guard.get_table_ids_in_tenant(tenant_id, table_ids))) { LOG_WARN("fail to get table schema", K(ret), K(tenant_id)); } else { ObCTASCleanUp ctas_cleanup(this, true); @@ -2931,11 +2931,19 @@ int ObServer::clean_up_invalid_tables_by_tenant( drop_table_arg.to_recyclebin_ = false; common_rpc_proxy = GCTX.rs_rpc_proxy_; MYADDR.ip_port_to_string(create_host_str, OB_MAX_HOST_NAME_LENGTH); - for (int64_t i = 0; i < table_schemas.count() && OB_SUCC(tmp_ret); i++) { + // only OB_ISNULL(GCTX.session_mgr_) will exit the loop + for (int64_t i = 0; i < table_ids.count() && OB_SUCC(tmp_ret); i++) { bool is_oracle_mode = false; - const ObTableSchema * table_schema = table_schemas.at(i); - if (OB_ISNULL(table_schema)) { - ret = OB_ERR_UNEXPECTED; + const ObTableSchema *table_schema = NULL; + const uint64_t table_id = table_ids.at(i); + // schema guard cannot be used repeatedly in iterative logic, + // otherwise it will cause a memory hike in schema cache + if (OB_FAIL(schema_service_.get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("get schema guard failed", K(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { + LOG_WARN("get table schema failed", K(ret), KT(table_id)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; LOG_WARN("got invalid schema", KR(ret), K(i)); } else if (OB_FAIL(table_schema->check_if_oracle_compat_mode(is_oracle_mode))) { LOG_WARN("fail to check table if oracle compat mode", KR(ret)); diff --git a/src/rootserver/ob_root_inspection.cpp b/src/rootserver/ob_root_inspection.cpp index 7b05647b33..19e6fbc8bb 100644 --- a/src/rootserver/ob_root_inspection.cpp +++ b/src/rootserver/ob_root_inspection.cpp @@ -227,6 +227,7 @@ ObTableGroupChecker::ObTableGroupChecker(share::schema::ObMultiVersionSchemaServ : schema_service_(schema_service), check_part_option_map_(), part_option_not_match_set_(), + allocator_(ObModIds::OB_SCHEMA), is_inited_(false) { } @@ -283,6 +284,7 @@ int ObTableGroupChecker::inspect_( { int ret = OB_SUCCESS; ObSchemaGetterGuard schema_guard; + allocator_.reset(); check_part_option_map_.reuse(); part_option_not_match_set_.reuse(); ObArray table_ids; @@ -297,11 +299,14 @@ int ObTableGroupChecker::inspect_( for (int64_t i = 0; OB_SUCC(ret) && i < table_ids.count(); i++) { const uint64_t table_id = table_ids.at(i); const ObTableSchema *table = NULL; - if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table))) { + // schema guard cannot be used repeatedly in iterative logic, + // otherwise it will cause a memory hike in schema cache + if (OB_FAIL(schema_service_.get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("get schema guard failed", K(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table))) { LOG_WARN("get table schema failed", K(ret), KT(table_id)); } else if (OB_ISNULL(table)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table not exist", K(ret), K(table_id)); + //ignore table not exist } else if (is_sys_table(table->get_table_id()) || !table->has_partition()) { // skip, check the partitioned user table } else if (OB_FAIL(check_part_option(*table, schema_guard))) { @@ -352,8 +357,11 @@ int ObTableGroupChecker::check_part_option(const ObTableSchema &table, ObSchemaG if (OB_FAIL(check_part_option_map_.get_refactored(tablegroup_id, table_in_map))) { //set into map while not in check_part_option_map_ if (OB_HASH_NOT_EXIST == ret) { - if (OB_FAIL(check_part_option_map_.set_refactored(tablegroup_id, &table))) { - LOG_WARN("set table_schema in hashmap fail", K(ret), K(tablegroup_id), K(table)); + ObTableSchema *new_table_schema = NULL; + if (OB_FAIL(ObSchemaUtils::alloc_schema(allocator_, table, new_table_schema))) { + LOG_WARN("alloc schema failed", K(ret), K(table)); + } else if (OB_FAIL(check_part_option_map_.set_refactored(tablegroup_id, new_table_schema))) { + LOG_WARN("set table_schema in hashmap fail", K(ret), K(tablegroup_id), K(new_table_schema)); } } else { LOG_WARN("check tablegroup_id in hashmap fail", K(ret), K(tablegroup_id)); diff --git a/src/rootserver/ob_root_inspection.h b/src/rootserver/ob_root_inspection.h index 9350402a97..50def8fb61 100644 --- a/src/rootserver/ob_root_inspection.h +++ b/src/rootserver/ob_root_inspection.h @@ -107,6 +107,7 @@ private: share::schema::ObMultiVersionSchemaService &schema_service_; ObTableGroupCheckInfoMap check_part_option_map_; common::hash::ObHashSet part_option_not_match_set_; + common::ObArenaAllocator allocator_; bool is_inited_; }; diff --git a/src/share/schema/ob_schema_getter_guard.cpp b/src/share/schema/ob_schema_getter_guard.cpp index 4ff5d847b2..751a084a17 100644 --- a/src/share/schema/ob_schema_getter_guard.cpp +++ b/src/share/schema/ob_schema_getter_guard.cpp @@ -97,7 +97,8 @@ ObSchemaGetterGuard::ObSchemaGetterGuard() schema_guard_type_(INVALID_SCHEMA_GUARD_TYPE), is_standby_cluster_(false), restore_tenant_exist_(false), - is_inited_(false) + is_inited_(false), + pin_cache_size_(0) { } @@ -112,13 +113,18 @@ ObSchemaGetterGuard::ObSchemaGetterGuard(const ObSchemaMgrItem::Mod mod) schema_guard_type_(INVALID_SCHEMA_GUARD_TYPE), is_standby_cluster_(false), restore_tenant_exist_(false), - is_inited_(false) + is_inited_(false), + pin_cache_size_(0) { } ObSchemaGetterGuard::~ObSchemaGetterGuard() { // Destruct handles_ will reduce reference count automatically. + if (pin_cache_size_ >= FULL_SCHEMA_MEM_THREHOLD) { + int ret = OB_SUCCESS; + FLOG_WARN("hold too much full schema memory", K(tenant_id_), K(pin_cache_size_), K(lbt())); + } } int ObSchemaGetterGuard::init( @@ -130,6 +136,7 @@ int ObSchemaGetterGuard::init( LOG_WARN("init twice", KR(ret)); } else { is_standby_cluster_ = is_standby_cluster; + pin_cache_size_ = 0; is_inited_ = true; } return ret; @@ -143,8 +150,10 @@ int ObSchemaGetterGuard::reset() is_standby_cluster_ = false; restore_tenant_exist_ = false; - - + if (pin_cache_size_ >= FULL_SCHEMA_MEM_THREHOLD) { + FLOG_WARN("hold too much full schema memory", K(tenant_id_), K(pin_cache_size_), K(lbt())); + } + pin_cache_size_ = 0; tenant_id_ = OB_INVALID_TENANT_ID; for (int64_t i = 0; i < schema_mgr_infos_.count(); i++) { @@ -4422,6 +4431,14 @@ int ObSchemaGetterGuard::put_to_local_cache( schema_obj.schema_id_ = schema_id; schema_obj.schema_ = const_cast(schema); schema_obj.handle_.move_from(handle); + if (schema_obj.handle_.is_valid() + && OB_NOT_NULL(schema) + && pin_cache_size_ < FULL_SCHEMA_MEM_THREHOLD) { + pin_cache_size_ += schema->get_convert_size(); + if (pin_cache_size_ >= FULL_SCHEMA_MEM_THREHOLD) { + FLOG_WARN("hold too much full schema memory", K(tenant_id), K(pin_cache_size_), K(lbt())); + } + } } return ret; } diff --git a/src/share/schema/ob_schema_getter_guard.h b/src/share/schema/ob_schema_getter_guard.h index aea90d7243..c58aebe87c 100644 --- a/src/share/schema/ob_schema_getter_guard.h +++ b/src/share/schema/ob_schema_getter_guard.h @@ -1159,7 +1159,7 @@ private: // it's use to control if table is visable in some sessions static const int MAX_ID_SCHEMAS = 32; - + const static int64_t FULL_SCHEMA_MEM_THREHOLD = 100 * 1024 * 1024L;//100M // tenant_id_ is valid means it's tenant schema guard uint64_t tenant_id_; SchemaMgrInfos schema_mgr_infos_; @@ -1171,6 +1171,7 @@ private: bool is_standby_cluster_; bool restore_tenant_exist_; bool is_inited_; + int64_t pin_cache_size_; private: DISALLOW_COPY_AND_ASSIGN(ObSchemaGetterGuard); }; diff --git a/src/sql/engine/cmd/ob_table_executor.cpp b/src/sql/engine/cmd/ob_table_executor.cpp index 654ffb9cc0..ff3f09c90f 100644 --- a/src/sql/engine/cmd/ob_table_executor.cpp +++ b/src/sql/engine/cmd/ob_table_executor.cpp @@ -2037,7 +2037,7 @@ int ObOptimizeTenantExecutor::optimize_tenant(const obrpc::ObOptimizeTenantArg & ret = OB_OP_NOT_ALLOW; LOG_WARN("tenant id mismatch", K(tenant_id), K(effective_tenant_id)); } else { - ObArray table_schemas; + ObSEArray table_schemas; if (OB_FAIL(schema_service.get_tenant_schema_guard(tenant_id, schema_guard))) { LOG_WARN("fail to get tenant schema guard", K(ret)); } else if (OB_FAIL(schema_guard.get_table_schemas_in_tenant(tenant_id, table_schemas))) { @@ -2045,7 +2045,7 @@ int ObOptimizeTenantExecutor::optimize_tenant(const obrpc::ObOptimizeTenantArg & } else { LOG_INFO("optimize tenant, table schema count", K(table_schemas.count())); for (int64_t i = 0; OB_SUCC(ret) && i < table_schemas.count(); ++i) { - const ObTableSchema *table_schema = table_schemas.at(i); + const ObSimpleTableSchemaV2 *table_schema = table_schemas.at(i); const ObDatabaseSchema *database_schema = nullptr; obrpc::ObOptimizeTableArg optimize_table_arg; if (OB_ISNULL(table_schema)) {