From a589195fdbce1faa551e0120bbea12d70248eb05 Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 27 Oct 2022 12:06:05 +0000 Subject: [PATCH] Enable system table has multi indexes. --- .../src/lib/mysqlclient/ob_isql_client.h | 1 + .../src/lib/mysqlclient/ob_mysql_proxy.h | 3 +- .../lib/mysqlclient/ob_mysql_transaction.cpp | 3 +- .../ob_single_connection_proxy.cpp | 1 - .../mysqlclient/ob_single_connection_proxy.h | 2 +- deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h | 1 + src/observer/ob_sql_client_decorator.cpp | 144 ++++------------- src/observer/ob_sql_client_decorator.h | 23 ++- src/rootserver/ob_bootstrap.cpp | 42 ++--- src/rootserver/ob_ddl_service.cpp | 54 +++---- src/rootserver/ob_ddl_service.h | 4 - src/rootserver/ob_root_inspection.cpp | 20 +-- .../generate_inner_table_schema.py | 108 +++++++++++-- .../ob_inner_table_schema.28151_28200.cpp | 8 +- src/share/inner_table/ob_inner_table_schema.h | 10 ++ .../ob_inner_table_schema_misc.ipp | 110 +++++++++++++ src/share/ob_core_table_proxy.cpp | 3 + .../ob_multi_version_schema_service.cpp | 52 ++---- src/share/schema/ob_schema_struct.cpp | 153 +++++++++++------- src/share/schema/ob_schema_struct.h | 9 +- src/share/schema/ob_schema_utils.cpp | 25 +-- 21 files changed, 428 insertions(+), 348 deletions(-) diff --git a/deps/oblib/src/lib/mysqlclient/ob_isql_client.h b/deps/oblib/src/lib/mysqlclient/ob_isql_client.h index 837c53a27..a45559763 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_isql_client.h +++ b/deps/oblib/src/lib/mysqlclient/ob_isql_client.h @@ -66,6 +66,7 @@ public: } virtual sqlclient::ObISQLConnectionPool *get_pool() = 0; + virtual sqlclient::ObISQLConnection *get_connection() = 0; virtual bool is_oracle_mode() const = 0; diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h index c57e42e4a..5d533b91b 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h @@ -114,7 +114,8 @@ public: using ObISQLClient::write; bool is_inited() const { return NULL != pool_; } - sqlclient::ObISQLConnectionPool *get_pool() override { return pool_; } + virtual sqlclient::ObISQLConnectionPool *get_pool() override { return pool_; } + virtual sqlclient::ObISQLConnection *get_connection() override { return NULL; } // can only use assign() to copy to prevent passing ObCommonSqlProxy by value unintentionally. void assign(const ObCommonSqlProxy &proxy) { *this = proxy; } diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.cpp index 68feee28e..455591185 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_transaction.cpp @@ -24,7 +24,8 @@ namespace common using namespace sqlclient; ObMySQLTransaction::ObMySQLTransaction(bool enable_query_stash) - :start_time_(0), + :ObSingleConnectionProxy(), + start_time_(0), in_trans_(false), enable_query_stash_(enable_query_stash) { diff --git a/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.cpp b/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.cpp index 908b61d21..d1d006063 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.cpp @@ -71,7 +71,6 @@ int ObSingleConnectionProxy::connect(const uint64_t tenant_id, ObISQLClient *sql return ret; } - int ObSingleConnectionProxy::read(ReadResult &res, const uint64_t tenant_id, const char *sql) { diff --git a/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.h index 3040656c1..74c4930e1 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_single_connection_proxy.h @@ -44,8 +44,8 @@ public: using ObISQLClient::write; int connect(const uint64_t tenant_id, ObISQLClient *sql_client); - sqlclient::ObISQLConnection *get_connection() { return conn_; } virtual sqlclient::ObISQLConnectionPool *get_pool() override { return pool_; } + virtual sqlclient::ObISQLConnection *get_connection() override { return conn_; } virtual bool is_oracle_mode() const override { return oracle_mode_; } // in some situation, it allows continuation of SQL execution after failure in transaction, diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index f5571d50f..18351d7ad 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -256,6 +256,7 @@ PCODE_DEF(OB_BATCH_SWITCH_RS_LEADER, 0x2A0) PCODE_DEF(OB_RUN_UPGRADE_JOB, 0x2A1) PCODE_DEF(OB_ADMIN_WASH_MEMORY_FRAGMENTATION, 0x2A2) PCODE_DEF(OB_CHECKPOINT_SLOG, 0x2A3) +PCODE_DEF(OB_UPGRADE_TABLE_SCHEMA, 0x2A4) PCODE_DEF(OB_CREATE_OUTLINE, 0x350) PCODE_DEF(OB_DROP_OUTLINE, 0x351) diff --git a/src/observer/ob_sql_client_decorator.cpp b/src/observer/ob_sql_client_decorator.cpp index 03b22a58f..d44a2e18e 100644 --- a/src/observer/ob_sql_client_decorator.cpp +++ b/src/observer/ob_sql_client_decorator.cpp @@ -76,6 +76,15 @@ sqlclient::ObISQLConnectionPool *ObSQLClientRetry::get_pool() return pool; } +sqlclient::ObISQLConnection *ObSQLClientRetry::get_connection() +{ + sqlclient::ObISQLConnection *conn = NULL; + if (NULL != sql_client_) { + conn = sql_client_->get_connection(); + } + return conn; +} + //////////////////////////////////////////////////////////////// int ObSQLClientRetryWeak::escape(const char *from, const int64_t from_size, char *to, const int64_t to_size, int64_t &out_size) @@ -89,94 +98,22 @@ int ObSQLClientRetryWeak::escape(const char *from, const int64_t from_size, return ret; } -// @修铭 In the case of weakly consistent reads, the READ-COMMITTED isolation level is always used -// Under the SERIALIZABLE isolation level, weakly consistent reading is not supported, nor is it supported to read the specified snapshot version -int ObSQLClientRetryWeak::weak_read(ReadResult &res, const uint64_t tenant_id, const char *sql) -{ - int ret = OB_SUCCESS; - ObSingleConnectionProxy single_conn_proxy; - sqlclient::ObISQLConnection *conn = NULL; - int64_t old_read_consistency = 0; - ObString ob_read_consistency = ObString::make_string("ob_read_consistency"); - ObString read_snapshot_version_name= ObString::make_string("ob_read_snapshot_version"); - ObString tx_isolation = ObString::make_string("tx_isolation"); - int64_t old_tx_isolation = 0; - int64_t read_committed = 1; // ObTransIsolation::READ_COMMITTED - if (OB_FAIL(single_conn_proxy.connect(tenant_id, sql_client_))) { - LOG_WARN("failed to get mysql connect", K(ret)); - } else if (OB_ISNULL(conn = single_conn_proxy.get_connection())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null pointer", K(ret)); - } else if (OB_FAIL(conn->get_session_variable(ob_read_consistency, old_read_consistency))) { - LOG_WARN("failed to get session variable ob_read_consistency", K(ret)); - } else if (OB_FAIL(conn->set_session_variable(ob_read_consistency, static_cast(WEAK)))) { - LOG_WARN("failed to set session variable ob_read_consistency", K(ret)); - } else if (OB_FAIL(conn->get_session_variable(tx_isolation, old_tx_isolation))) { - LOG_WARN("failed to get session variable tx_isolation", K(ret)); - } - // Mandatory set to READ-COMMITTED isolation level - else if (OB_FAIL(conn->set_session_variable(tx_isolation, read_committed))) { - LOG_WARN("failed to set session variable ob_read_consistency", K(ret)); - } else if (snapshot_timestamp_ > 0 && - OB_FAIL(conn->set_session_variable(read_snapshot_version_name, snapshot_timestamp_))) { - LOG_WARN("failed to set session variable ob_read_snapshot_version", K(ret), K_(snapshot_timestamp)); - } else { - - if (check_sys_variable_) { - ret = single_conn_proxy.read(res, tenant_id, sql); - } else { - ret = read_without_check_sys_variable(single_conn_proxy, res, tenant_id, sql); - } - if (OB_FAIL(ret)) { - LOG_WARN("failed to weak read", K(ret), K(sql), K(tenant_id), - K_(snapshot_timestamp), K_(check_sys_variable), K_(snapshot_timestamp)); - } else { - LOG_INFO("weak read succeeded!", K(ret), K(sql), K(tenant_id), - K_(snapshot_timestamp), K_(check_sys_variable), K_(snapshot_timestamp), - "old_read_consistency", old_read_consistency, K(old_tx_isolation)); - } - - // restore the old value - int tmp_ret = conn->set_session_variable(ob_read_consistency, old_read_consistency); - if (OB_SUCCESS != tmp_ret) { - LOG_WARN("failed to set session variable ob_read_consistency", K(ret), K(old_read_consistency)); - } - ret = (OB_SUCCESS == ret) ? tmp_ret : ret; - - tmp_ret = conn->set_session_variable(tx_isolation, old_tx_isolation); - if (OB_SUCCESS != tmp_ret) { - LOG_WARN("failed to set session variable tx_isolation", K(ret), K(old_tx_isolation)); - } - ret = (OB_SUCCESS == ret) ? tmp_ret : ret; - - // reset ob_read_snapshot_version - tmp_ret = conn->set_session_variable(read_snapshot_version_name, OB_INVALID_VERSION); - if (OB_SUCCESS != tmp_ret) { - LOG_WARN("failed to set session variable ob_read_snapshot_version", K(ret)); - } - ret = (OB_SUCCESS == ret) ? tmp_ret : ret; - - } - return ret; -} - int ObSQLClientRetryWeak::read_without_check_sys_variable( - ObSingleConnectionProxy &single_conn_proxy, + sqlclient::ObISQLConnection *conn, ReadResult &res, const uint64_t tenant_id, const char *sql) { int ret = OB_SUCCESS; ObString check_sys_variable_name = ObString::make_string("ob_check_sys_variable"); - sqlclient::ObISQLConnection *conn = NULL; - if (OB_ISNULL(conn = single_conn_proxy.get_connection())) { + if (OB_ISNULL(conn)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null pointer", K(ret)); } else if (OB_FAIL(conn->set_session_variable(check_sys_variable_name, static_cast(check_sys_variable_)))) { LOG_WARN("failed to set session variable ob_check_sys_variable", K(ret)); } else { - ret = single_conn_proxy.read(res, tenant_id, sql); + ret = conn->execute_read(tenant_id, sql, res); if (OB_FAIL(ret)) { LOG_WARN("failed to read without check sys variable", K(ret), K(sql), K(tenant_id), K_(check_sys_variable), K_(snapshot_timestamp)); @@ -207,18 +144,22 @@ int ObSQLClientRetryWeak::read(ReadResult &res, const uint64_t tenant_id, const int ret = OB_SUCCESS; if (OB_ISNULL(sql_client_)) { ret = OB_INNER_STAT_ERROR; - } else if (OB_FAIL(update_weak_read_snapshot_timestamp(tenant_id))) { - LOG_WARN("fail to update weak read snapshot timestamp", K(ret), K(tenant_id)); } else { // normal read if (check_sys_variable_) { ret = sql_client_->read(res, tenant_id, sql); } else { + sqlclient::ObISQLConnection *conn = sql_client_->get_connection(); ObSingleConnectionProxy single_conn_proxy; - if (OB_FAIL(single_conn_proxy.connect(tenant_id, sql_client_))) { - LOG_WARN("failed to get mysql connect", K(ret)); + if (OB_NOT_NULL(conn)) { + // for transaction + } else if (OB_FAIL(single_conn_proxy.connect(tenant_id, sql_client_))) { + LOG_WARN("failed to get mysql connect", KR(ret), K(tenant_id)); } else { - ret = read_without_check_sys_variable(single_conn_proxy, res, tenant_id, sql); + conn = single_conn_proxy.get_connection(); + } + if (OB_SUCC(ret) && OB_NOT_NULL(conn)) { + ret = read_without_check_sys_variable(conn, res, tenant_id, sql); } } } @@ -236,40 +177,6 @@ int ObSQLClientRetryWeak::write(const uint64_t tenant_id, const char *sql, int64 return ret; } -int ObSQLClientRetryWeak::update_weak_read_snapshot_timestamp(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - if (!is_auto_mode()) { - // skip - } else if (OB_INVALID_TENANT_ID == tenant_id_ - || is_virtual_tenant_id(tenant_id) - || OB_INVALID_ID == table_id_ - || tenant_id != tenant_id_) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(tenant_id_), K(table_id_), K(tenant_id)); - } else if (GCTX.is_standby_cluster() && is_user_tenant(tenant_id)) { - ObRefreshSchemaStatus schema_status; - if (OB_ISNULL(GCTX.schema_status_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("schema_status_proxy is null", K(ret)); - } else if (OB_FAIL(GCTX.schema_status_proxy_->get_refresh_schema_status(tenant_id_, schema_status))) { - LOG_WARN("fail to get schema status", K(ret), K(tenant_id_)); - } else if (0 == schema_status.snapshot_timestamp_) { - // The weakly read version number is 0, the transaction will negotiate a readable version number, which does not match expectations, here is a defense - ret = OB_REPLICA_NOT_READABLE; - LOG_WARN("snapshot_timestamp is invalid", K(ret), K(schema_status)); - } else if (OB_INVALID_TIMESTAMP == schema_status.snapshot_timestamp_ - && OB_INVALID_VERSION == schema_status.readable_schema_version_) { - snapshot_timestamp_ = OB_INVALID_TIMESTAMP; - } else { - snapshot_timestamp_ = schema_status.snapshot_timestamp_; - } - } else { - snapshot_timestamp_ = OB_INVALID_TIMESTAMP; - } - return ret; -} - sqlclient::ObISQLConnectionPool *ObSQLClientRetryWeak::get_pool() { sqlclient::ObISQLConnectionPool *pool = NULL; @@ -279,3 +186,12 @@ sqlclient::ObISQLConnectionPool *ObSQLClientRetryWeak::get_pool() return pool; } +sqlclient::ObISQLConnection *ObSQLClientRetryWeak::get_connection() +{ + sqlclient::ObISQLConnection *conn = NULL; + if (NULL != sql_client_) { + conn = sql_client_->get_connection(); + } + return conn; +} + diff --git a/src/observer/ob_sql_client_decorator.h b/src/observer/ob_sql_client_decorator.h index 089f1d573..0c1a2cab2 100644 --- a/src/observer/ob_sql_client_decorator.h +++ b/src/observer/ob_sql_client_decorator.h @@ -34,6 +34,7 @@ public: virtual int read(ReadResult &res, const int64_t cluster_id, const uint64_t tenant_id, const char *sql) override; virtual int write(const uint64_t tenant_id, const char *sql, int64_t &affected_rows) override; virtual sqlclient::ObISQLConnectionPool *get_pool() override; + virtual sqlclient::ObISQLConnection *get_connection() override; using ObISQLClient::read; using ObISQLClient::write; @@ -52,10 +53,10 @@ private: }; class ObMySQLProxy; -// use READ_CONSISTENCY(WEAK) when did_retry_weak is true class ObSQLClientRetryWeak: public ObISQLClient { public: + // only check_sys_variable is still useful ObSQLClientRetryWeak(ObISQLClient *sql_client, bool did_use_retry = false, int64_t snapshot_timestamp = OB_INVALID_TIMESTAMP, @@ -68,9 +69,7 @@ public: { UNUSED(did_use_retry); } - // tenant_id is the name space where SQL is actually executed - // The default semantics means strong consistent read - // For ordinary tenant system tables that require physical synchronization for the standby database, use the weakly consistent read of the specified snapshot version + // not useful, it just use sql_client directly ObSQLClientRetryWeak(ObISQLClient *sql_client, bool did_use_retry, const uint64_t tenant_id, @@ -94,31 +93,27 @@ public: using ObISQLClient::write; virtual sqlclient::ObISQLConnectionPool *get_pool() override; + virtual sqlclient::ObISQLConnection *get_connection() override; + bool is_oracle_mode() const override { return NULL == sql_client_ ? false : sql_client_->is_oracle_mode(); } -private: - bool is_auto_mode() { return OB_INVALID_TENANT_ID != tenant_id_ || OB_INVALID_ID != table_id_; } - int update_weak_read_snapshot_timestamp(const uint64_t tenant_id); private: // disallow copy DISALLOW_COPY_AND_ASSIGN(ObSQLClientRetryWeak); // functions - int weak_read(ReadResult &res, const uint64_t tenant_id, const char *sql); - int get_session_read_consistency(ObConsistencyLevel &level); - int set_session_read_consistency(ObConsistencyLevel &level); int read_without_check_sys_variable( - ObSingleConnectionProxy &single_conn_proxy, + sqlclient::ObISQLConnection *conn, ReadResult &res, const uint64_t tenant_id, const char *sql); private: ObISQLClient *sql_client_; - int64_t snapshot_timestamp_; + int64_t snapshot_timestamp_; // deprecated bool check_sys_variable_; - uint64_t tenant_id_; - uint64_t table_id_; + uint64_t tenant_id_; // deprecated + uint64_t table_id_; // deprecated }; } // end namespace common diff --git a/src/rootserver/ob_bootstrap.cpp b/src/rootserver/ob_bootstrap.cpp index 441bf3372..f00298396 100644 --- a/src/rootserver/ob_bootstrap.cpp +++ b/src/rootserver/ob_bootstrap.cpp @@ -579,22 +579,13 @@ int ObBootstrap::generate_table_schema_array_for_create_partition( const uint64_t table_id = tschema.get_table_id(); int64_t tschema_idx = table_schema_array.count(); - HEAP_VAR(ObTableSchema, index_schema) { - if (OB_FAIL(table_schema_array.push_back(tschema))) { - LOG_WARN("fail to push back", KR(ret)); - } else if (ObSysTableChecker::is_sys_table_has_index(table_id)) { - if (OB_FAIL(ObSysTableChecker::get_sys_table_index_schema( - table_id, index_schema))) { - LOG_WARN("fail to get sys table's index schema", KR(ret), K(table_id)); - } else if (OB_FAIL(table_schema_array.push_back(index_schema))) { - LOG_WARN("fail to push back index schema", KR(ret), K(index_schema)); - } - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(add_sys_table_lob_aux_table(table_id, table_schema_array))) { - LOG_WARN("fail to add lob table to sys table", KR(ret), K(table_id)); - } + if (OB_FAIL(table_schema_array.push_back(tschema))) { + LOG_WARN("fail to push back", KR(ret)); + } else if (OB_FAIL(ObSysTableChecker::append_sys_table_index_schemas( + OB_SYS_TENANT_ID, table_id, table_schema_array))) { + LOG_WARN("fail to append sys table index schemas", KR(ret), K(table_id)); + } else if (OB_FAIL(add_sys_table_lob_aux_table(table_id, table_schema_array))) { + LOG_WARN("fail to add lob table to sys table", KR(ret), K(table_id)); } return ret; } @@ -797,7 +788,7 @@ int ObBootstrap::construct_all_schema(ObIArray &table_schemas) } else if (OB_FAIL(table_schemas.reserve(OB_SYS_TABLE_COUNT))) { LOG_WARN("reserve failed", "capacity", OB_SYS_TABLE_COUNT, KR(ret)); } else { - HEAP_VARS_2((ObTableSchema, index_schema), (ObTableSchema, data_schema)) { + HEAP_VAR(ObTableSchema, data_schema) { for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(creator_ptr_arrays); ++i) { for (const schema_create_func *creator_ptr = creator_ptr_arrays[i]; OB_SUCCESS == ret && NULL != *creator_ptr; ++creator_ptr) { @@ -812,19 +803,12 @@ int ObBootstrap::construct_all_schema(ObIArray &table_schemas) } else if (!exist) { // skip } else if (ObSysTableChecker::is_sys_table_has_index(table_schema.get_table_id())) { - index_schema.reset(); const int64_t data_table_id = table_schema.get_table_id(); - if (OB_FAIL(ObSysTableChecker::get_sys_table_index_schema(data_table_id, index_schema))) { - LOG_WARN("fail to get sys table's index schema", KR(ret), K(data_table_id)); - } else if (OB_FAIL(table_schema.add_simple_index_info(ObAuxTableMetaInfo( - index_schema.get_table_id(), - index_schema.get_table_type(), - index_schema.get_index_type())))) { - LOG_WARN("fail to add simple_index_info", KR(ret), K(table_schema)); - } else if (OB_FAIL(table_schemas.push_back(index_schema))) { - LOG_WARN("push_back failed", KR(ret), K(index_schema)); - } else { - LOG_INFO("create sys index schema", KR(ret), K(table_schema), K(index_schema)); + if (OB_FAIL(ObSysTableChecker::fill_sys_index_infos(table_schema))) { + LOG_WARN("fail to fill sys index infos", KR(ret), K(data_table_id)); + } else if (OB_FAIL(ObSysTableChecker::append_sys_table_index_schemas( + OB_SYS_TENANT_ID, data_table_id, table_schemas))) { + LOG_WARN("fail to append sys table index schemas", KR(ret), K(data_table_id)); } } diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 603c169fd..98541580d 100644 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -1796,7 +1796,8 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist, if (OB_SUCC(ret) && ObSysTableChecker::is_sys_table_has_index(first_table.get_table_id())) { ObArray schemas; - if (OB_FAIL(add_sys_table_index(tenant_id, first_table.get_table_id(), schemas))) { + if (OB_FAIL(ObSysTableChecker::append_sys_table_index_schemas( + tenant_id, first_table.get_table_id(), schemas))) { LOG_WARN("fail to add sys table index", K(ret), K(tenant_id), "table_id", first_table.get_table_id()); } else if (OB_FAIL(ddl_operator.create_table(schemas.at(0), @@ -19239,6 +19240,7 @@ int ObDDLService::create_tenant_sys_tablets( sql_proxy_); common::ObArray ls_id_array; ObArray table_schemas; + ObArray index_tids; if (OB_FAIL(trans.start(sql_proxy_, tenant_id))) { LOG_WARN("fail to start trans", KR(ret), K(tenant_id)); } else if (OB_FAIL(table_creator.init())) { @@ -19254,17 +19256,25 @@ int ObDDLService::create_tenant_sys_tablets( if (OB_FAIL(table_schemas.push_back(&data_table))) { LOG_WARN("fail to push back data table ptr", KR(ret), K(data_table_id)); } else if (ObSysTableChecker::is_sys_table_has_index(data_table_id)) { - // sys table's index should be next to its data table. - int64_t index_id = OB_INVALID_ID; - if (i + 1 >= tables.count()) { + if (OB_FAIL(ObSysTableChecker::get_sys_table_index_tids(data_table_id, index_tids))) { + LOG_WARN("fail to get sys index tids", KR(ret), K(data_table_id)); + } else if (i + index_tids.count() >= tables.count() + || index_tids.count() <= 0) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("sys table's index should be next to its data table", KR(ret), K(i)); - } else if (FALSE_IT(index_id = tables.at(i + 1).get_table_id())) { - } else if (index_id != ObSysTableChecker::get_sys_table_index_tid(data_table_id)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sys table's index not matched", KR(ret), K(index_id), K(data_table_id)); - } else if (OB_FAIL(table_schemas.push_back(&tables.at(i + 1)))) { - LOG_WARN("fail to push back index table ptr", KR(ret), K(index_id), K(data_table_id)); + LOG_WARN("sys table's index should be next to its data table", + KR(ret), K(i), "index_cnt", index_tids.count()); + } else { + for (int64_t j = 0; OB_SUCC(ret) && j < index_tids.count(); j++) { + const ObTableSchema &index_schema = tables.at(i + j + 1); + const int64_t index_id = index_schema.get_table_id(); + if (index_id != index_tids.at(j) + || data_table_id != index_schema.get_data_table_id()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sys index schema order is not match", KR(ret), K(data_table_id), K(j), K(index_schema)); + } else if (OB_FAIL(table_schemas.push_back(&index_schema))) { + LOG_WARN("fail to push back index schema", KR(ret), K(index_id), K(data_table_id)); + } + } // end for } } @@ -29518,28 +29528,6 @@ int ObDDLService::force_set_locality( return ret; } -int ObDDLService::add_sys_table_index( - const uint64_t tenant_id, - const uint64_t table_id, - common::ObIArray &schemas) -{ - int ret = OB_SUCCESS; - if (ObSysTableChecker::is_sys_table_has_index(table_id)) { - ObTableSchema index_schema; - const int64_t data_table_id = table_id; - if (OB_FAIL(ObSysTableChecker::get_sys_table_index_schema( - data_table_id, index_schema))) { - LOG_WARN("fail to get sys table's index schema", KR(ret), K(data_table_id)); - } else if (OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table( - tenant_id, index_schema))) { - LOG_WARN("fail to construct tenant space table", KR(ret), K(tenant_id)); - } else if (OB_FAIL(schemas.push_back(index_schema))) { - LOG_WARN("fail to push back index schema", KR(ret), K(index_schema)); - } - } - return ret; -} - int ObDDLService::add_sys_table_lob_aux( const int64_t tenant_id, const uint64_t table_id, diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index a9914a4e1..b123bf368 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -2115,10 +2115,6 @@ private: const int64_t tenant_id, const share::schema::AlterTableSchema &alter_table_schema, const share::schema::ObTableSchema &orig_table_schema); - int add_sys_table_index( - const uint64_t tenant_id, - const uint64_t table_id, - common::ObIArray &schemas); int add_sys_table_lob_aux(const int64_t tenant_id, const uint64_t table_id, ObTableSchema &meta_schema, ObTableSchema &data_schema); int check_has_multi_autoinc(share::schema::ObTableSchema &table_schema); diff --git a/src/rootserver/ob_root_inspection.cpp b/src/rootserver/ob_root_inspection.cpp index d99b903b0..9573cc35b 100644 --- a/src/rootserver/ob_root_inspection.cpp +++ b/src/rootserver/ob_root_inspection.cpp @@ -1242,7 +1242,11 @@ int ObRootInspection::check_sys_table_schemas_( share::core_table_schema_creators, share::sys_table_schema_creators, share::virtual_table_schema_creators, - share::sys_view_schema_creators, NULL }; + share::sys_view_schema_creators, + // TODO: will check sys index when compat case changed. + //share::core_index_table_schema_creators, + //share::sys_index_table_schema_creators, + NULL }; int back_ret = OB_SUCCESS; ObTableSchema table_schema; @@ -1269,6 +1273,8 @@ int ObRootInspection::check_sys_table_schemas_( } else if (OB_FAIL(check_table_schema(tenant_id, table_schema))) { // don't print table_schema, otherwise log will be too much LOG_WARN("check_table_schema failed", KR(ret), K(tenant_id)); + } else if (OB_FAIL(check_sys_view_(tenant_id, table_schema))) { + LOG_WARN("fail to check sys view", KR(ret), K(tenant_id)); } back_ret = OB_SUCCESS == back_ret ? ret : back_ret; ret = OB_SUCCESS; @@ -1320,8 +1326,6 @@ int ObRootInspection::check_table_schema( can_retry_ = true; } else if (OB_FAIL(check_table_schema(hard_code_table, *table))) { LOG_WARN("fail to check table schema", KR(ret), K(tenant_id), K(hard_code_table), KPC(table)); - } else if (OB_FAIL(check_sys_view_(tenant_id, hard_code_table))) { - LOG_WARN("fail to check sys view", KR(ret), K(tenant_id), K(hard_code_table)); } return ret; } @@ -1642,15 +1646,7 @@ int ObRootInspection::check_column_schema(const ObString &table_name, } if (!ignore_column_id) { - uint64_t tid = hard_code_column.get_table_id(); - uint64_t cid = hard_code_column.get_column_id(); - if ((OB_ALL_TABLE_HISTORY_TID == tid && cid <= 75) - || (OB_ALL_TABLE_TID == tid && cid <= 74) - || (OB_ALL_TENANT_TID == tid && cid <= 36) - || (OB_ALL_TENANT_HISTORY_TID == tid && cid <= 38)) { - } else { - CMP_COLUMN_ATTR(column_id); - } + CMP_COLUMN_ATTR(column_id); } CMP_COLUMN_ATTR(tenant_id); CMP_COLUMN_ATTR(table_id); diff --git a/src/share/inner_table/generate_inner_table_schema.py b/src/share/inner_table/generate_inner_table_schema.py index 1ba6aac20..8fd14cef8 100755 --- a/src/share/inner_table/generate_inner_table_schema.py +++ b/src/share/inner_table/generate_inner_table_schema.py @@ -25,6 +25,7 @@ base_lob_meta_table_id = int(50000) base_lob_piece_table_id = int(60000) max_lob_table_id = int(70000) min_sys_index_id = int(100000) +max_core_index_id = int(101000) max_sys_index_id = int(200000) min_shadow_column_id = int(32767) @@ -57,6 +58,10 @@ def is_lob_table(table_id): table_id = int(table_id) return (table_id > base_lob_meta_table_id) and (table_id < max_lob_table_id) +def is_core_index_table(table_id): + table_id = int(table_id) + return (table_id > min_sys_index_id) and (table_id < max_core_index_id) + def is_sys_index_table(table_id): table_id = int(table_id) return (table_id > min_sys_index_id) and (table_id < max_sys_index_id) @@ -90,6 +95,7 @@ column_collation = 'CS_TYPE_INVALID' # virtual tables only accessible by sys tenant or sys views. restrict_access_virtual_tables = [] is_oracle_sys_table = False +sys_index_tables = [] copyright = """/** * Copyright (c) 2021 OceanBase * OceanBase CE is licensed under Mulan PubL v2. @@ -744,26 +750,31 @@ def find_column_def(keywords, column_name, is_shadow_pk_column): def add_index_column(keywords, rowkey_id, index_id, column): (idx, column_def) = find_column_def(keywords, column, False) add_column(column_def, rowkey_id, index_id, 0, idx) + return idx # For shadow pk column, rowkey_position != 0 and index_position = 0. def add_shadow_pk_column(keywords, rowkey_id, column): (idx, column_def) = find_column_def(keywords, column, True) add_column(column_def, rowkey_id, 0, 0, idx, is_hidden='true') + return idx def add_index_columns(columns, **keywords): rowkey_id = 1 is_unique_index = keywords['index_type'] == 'INDEX_TYPE_UNIQUE_LOCAL' + max_used_column_idx = 1 if is_unique_index: # specified index columns. for column in columns: - add_index_column(keywords, rowkey_id, rowkey_id, column) + column_idx = add_index_column(keywords, rowkey_id, rowkey_id, column) + max_used_column_idx = max(max_used_column_idx, column_idx) rowkey_id += 1 - + # generate shadow pk column whose number equals to rowkeys' number. shadow_pk_col_idx = 0 for col in keywords['rowkey_columns']: shadow_pk_col_name = 'shadow_pk_' + str(shadow_pk_col_idx) - add_shadow_pk_column(keywords, rowkey_id, shadow_pk_col_name) + column_idx = add_shadow_pk_column(keywords, rowkey_id, shadow_pk_col_name) + max_used_column_idx = max(max_used_column_idx, column_idx) rowkey_id += 1 shadow_pk_col_idx += 1 @@ -773,15 +784,19 @@ def add_index_columns(columns, **keywords): if col[0] not in columns: (idx, column_def) = find_column_def(keywords, col[0], False) add_column(column_def, 0, 0, 0, idx) + max_used_column_idx = max(max_used_column_idx, idx) else: for column in columns: - add_index_column(keywords, rowkey_id, rowkey_id, column) + column_idx = add_index_column(keywords, rowkey_id, rowkey_id, column) + max_used_column_idx = max(max_used_column_idx, column_idx) rowkey_id += 1 for col in keywords['rowkey_columns']: if col[0] not in columns: - add_index_column(keywords, rowkey_id, 0, col[0]) + column_idx = add_index_column(keywords, rowkey_id, 0, col[0]) + max_used_column_idx = max(max_used_column_idx, column_idx) rowkey_id += 1 - return rowkey_id - 1 + return max_used_column_idx + def add_rowkey_columns(columns, *args): rowkey_id = 1 index_id = 0 @@ -1154,6 +1169,60 @@ def generate_cluster_private_table(f): cluster_private_switch += 'case ' + table_name2tid(kw['table_name'] + kw['name_postfix']) + ':\n' f.write('\n\n#ifdef CLUSTER_PRIVATE_TABLE_SWITCH\n' + cluster_private_switch + '\n#endif\n') +def generate_sys_index_table_misc_data(f): + global sys_index_tables + + data_table_dict = {} + for kw in sys_index_tables: + if not data_table_dict.has_key(kw['table_name']): + data_table_dict[kw['table_name']] = [] + data_table_dict[kw['table_name']].append(kw) + + sys_index_table_id_switch = '\n' + for kw in sys_index_tables: + sys_index_table_id_switch += 'case ' + table_name2index_tid(kw['table_name'], kw['index_name']) + ':\n' + f.write('\n\n#ifdef SYS_INDEX_TABLE_ID_SWITCH\n' + sys_index_table_id_switch + '\n#endif\n') + + sys_index_data_table_id_switch = '\n' + for kw in sys_index_tables: + sys_index_data_table_id_switch += 'case ' + table_name2tid(kw['table_name']) + ':\n' + f.write('\n\n#ifdef SYS_INDEX_DATA_TABLE_ID_SWITCH\n' + sys_index_data_table_id_switch + '\n#endif\n') + + sys_index_data_table_id_to_index_ids_switch = '\n' + for data_table_name, sys_indexs in data_table_dict.items(): + sys_index_data_table_id_to_index_ids_switch += 'case ' + table_name2tid(data_table_name) + ': {\n' + for kw in sys_indexs: + sys_index_data_table_id_to_index_ids_switch += ' if (FAILEDx(index_tids.push_back(' + table_name2index_tid(kw['table_name'], kw['index_name']) + '))) {\n' + sys_index_data_table_id_to_index_ids_switch += ' LOG_WARN(\"fail to push back index tid\", KR(ret));\n' + sys_index_data_table_id_to_index_ids_switch += ' }\n' + sys_index_data_table_id_to_index_ids_switch += ' break;\n' + sys_index_data_table_id_to_index_ids_switch += '}\n' + f.write('\n\n#ifdef SYS_INDEX_DATA_TABLE_ID_TO_INDEX_IDS_SWITCH\n' + sys_index_data_table_id_to_index_ids_switch + '\n#endif\n') + + sys_index_data_table_id_to_index_schema_switch = '\n' + for data_table_name, sys_indexs in data_table_dict.items(): + sys_index_data_table_id_to_index_schema_switch += 'case ' + table_name2tid(data_table_name) + ': {\n' + for kw in sys_indexs: + method_name = kw['table_name'].replace('$', '_').strip('_').lower() + '_' + kw['index_name'].lower() + '_schema' + sys_index_data_table_id_to_index_schema_switch += ' if (FAILEDx(ObInnerTableSchema::' + method_name +'(index_schema))) {\n' + sys_index_data_table_id_to_index_schema_switch += ' LOG_WARN(\"fail to create index schema\", KR(ret), K(tenant_id), K(data_table_id));\n' + sys_index_data_table_id_to_index_schema_switch += ' } else if (!is_sys_tenant(tenant_id) && OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table(tenant_id, index_schema))) {\n' + sys_index_data_table_id_to_index_schema_switch += ' LOG_WARN(\"fail to construct full table\", KR(ret), K(tenant_id), K(data_table_id));\n' + sys_index_data_table_id_to_index_schema_switch += ' } else if (OB_FAIL(tables.push_back(index_schema))) {\n' + sys_index_data_table_id_to_index_schema_switch += ' LOG_WARN(\"fail to push back index\", KR(ret), K(tenant_id), K(data_table_id));\n' + sys_index_data_table_id_to_index_schema_switch += ' }\n' + sys_index_data_table_id_to_index_schema_switch += ' break;\n' + sys_index_data_table_id_to_index_schema_switch += '}\n' + + f.write('\n\n#ifdef SYS_INDEX_DATA_TABLE_ID_TO_INDEX_SCHEMAS_SWITCH\n' + sys_index_data_table_id_to_index_schema_switch + '\n#endif\n') + + add_sys_index_id = '\n' + for kw in sys_index_tables: + index_id = table_name2index_tid(kw['table_name'], kw['index_name']) + add_sys_index_id += ' } else if (OB_FAIL(table_ids.push_back(' + index_id +'))) {\n' + add_sys_index_id += ' LOG_WARN(\"add index id failed\", KR(ret), K(tenant_id));\n' + f.write('\n\n#ifdef ADD_SYS_INDEX_ID\n' + add_sys_index_id + '\n#endif\n') + def generate_virtual_agent_misc_data(f): global all_agent_virtual_tables all_agent = [x for x in all_agent_virtual_tables] @@ -1206,12 +1275,17 @@ def def_sys_index_table(index_name, index_table_id, index_columns, index_using_t global cpp_f global cpp_f_tmp global StringIO + global sys_index_tables kw = copy.deepcopy(keywords) if kw.has_key('index'): raise Exception("should not have index", kw['table_name']) - if False == is_sys_table(kw['table_id']): + if not is_sys_table(kw['table_id']): raise Exception("only support sys table", kw['table_name']) + if not is_sys_index_table(index_table_id): + raise Exception("index table id is invalid", index_table_id) + if is_core_table(kw['table_id']) and not is_core_index_table(index_table_id): + raise Exception("index table id for core table should be less than 101000", index_table_id, kw['table_id']) index_def = '' cpp_f_tmp = cpp_f @@ -1228,6 +1302,7 @@ def def_sys_index_table(index_name, index_table_id, index_columns, index_using_t kw['partition_columns'] = [] kw['partition_expr'] = [] kw['storing_columns'] =[] + sys_index_tables.append(kw) def_table_schema(**kw) index_def = cpp_f.getvalue() cpp_f = cpp_f_tmp @@ -1725,7 +1800,7 @@ def def_table_schema(**keywords): cpp_f = cpp_f_tmp elif field == 'index_columns': # only index generation will enter here - index_num_rowkeys = add_index_columns(value, **keywords) + max_used_column_idx = add_index_columns(value, **keywords) elif field == 'storing_columns': # only virtual table index generation will enter here add_storing_columns(value,**keywords) @@ -1755,7 +1830,7 @@ def def_table_schema(**keywords): add_field('aux_lob_piece_tid', ptid) if keywords.has_key("index_name") and not type(keywords['index']) == dict: - add_index_method_end(index_num_rowkeys) + add_index_method_end(max_used_column_idx) else: add_method_end() @@ -1956,7 +2031,7 @@ private: h_f.write("const schema_create_func virtual_table_schema_creators [] = {\n") for (table_name, table_id) in table_name_postfix_ids: - if is_virtual_table(table_id): + if is_virtual_table(table_id): h_f.write(method_name.format(table_name.replace('$', '_').lower().strip('_'), table_name)) virtual_table_count = virtual_table_count + 1 for index_l in index_name_ids: @@ -1972,6 +2047,18 @@ private: sys_view_count = sys_view_count + 1 h_f.write(" NULL,};\n\n") + h_f.write("const schema_create_func core_index_table_schema_creators [] = {\n") + for index_l in index_name_ids: + if is_core_index_table(index_l[1]): + h_f.write(method_name.format(index_l[2].replace('$', '_').strip('_').lower()+'_'+index_l[0].lower(), index_l[2])) + h_f.write(" NULL,};\n\n") + + h_f.write("const schema_create_func sys_index_table_schema_creators [] = {\n") + for index_l in index_name_ids: + if not is_core_index_table(index_l[1]) and is_sys_index_table(index_l[1]): + h_f.write(method_name.format(index_l[2].replace('$', '_').strip('_').lower()+'_'+index_l[0].lower(), index_l[2])) + h_f.write(" NULL,};\n\n") + # just to make test happy h_f.write("const schema_create_func information_schema_table_schema_creators[] = {\n") h_f.write(" NULL,};\n\n") @@ -2411,6 +2498,7 @@ if __name__ == "__main__": generate_iterate_private_virtual_table_misc_data(f) generate_iterate_virtual_table_misc_data(f) generate_cluster_private_table(f) + generate_sys_index_table_misc_data(f) f.close() diff --git a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp index e6c603bc5..0deb814c2 100644 --- a/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28151_28200.cpp @@ -285,7 +285,7 @@ int ObInnerTableSchema::all_table_history_idx_data_table_id_schema(ObTableSchema table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); table_schema.set_data_table_id(OB_ALL_TABLE_HISTORY_TID); - table_schema.set_max_used_column_id(column_id + 4); + table_schema.set_max_used_column_id(column_id + 23); return ret; } @@ -419,7 +419,7 @@ int ObInnerTableSchema::all_log_archive_piece_files_idx_status_schema(ObTableSch table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); table_schema.set_data_table_id(OB_ALL_LOG_ARCHIVE_PIECE_FILES_TID); - table_schema.set_max_used_column_id(column_id + 5); + table_schema.set_max_used_column_id(column_id + 18); return ret; } @@ -534,7 +534,7 @@ int ObInnerTableSchema::all_backup_set_files_idx_status_schema(ObTableSchema &ta table_schema.set_index_type(INDEX_TYPE_NORMAL_LOCAL); table_schema.set_data_table_id(OB_ALL_BACKUP_SET_FILES_TID); - table_schema.set_max_used_column_id(column_id + 4); + table_schema.set_max_used_column_id(column_id + 11); return ret; } @@ -666,7 +666,7 @@ int ObInnerTableSchema::all_ddl_task_status_idx_task_key_schema(ObTableSchema &t table_schema.set_index_type(INDEX_TYPE_UNIQUE_LOCAL); table_schema.set_data_table_id(OB_ALL_DDL_TASK_STATUS_TID); - table_schema.set_max_used_column_id(column_id + 4); + table_schema.set_max_used_column_id(column_id + 32768); return ret; } diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index 90344d705..c6015fdef 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -3473,6 +3473,16 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::dba_ob_table_locations_ora_schema, NULL,}; +const schema_create_func core_index_table_schema_creators [] = { + NULL,}; + +const schema_create_func sys_index_table_schema_creators [] = { + ObInnerTableSchema::all_table_history_idx_data_table_id_schema, + ObInnerTableSchema::all_log_archive_piece_files_idx_status_schema, + ObInnerTableSchema::all_backup_set_files_idx_status_schema, + ObInnerTableSchema::all_ddl_task_status_idx_task_key_schema, + NULL,}; + const schema_create_func information_schema_table_schema_creators[] = { NULL,}; diff --git a/src/share/inner_table/ob_inner_table_schema_misc.ipp b/src/share/inner_table/ob_inner_table_schema_misc.ipp index c90a8aa4d..06c89e48a 100644 --- a/src/share/inner_table/ob_inner_table_schema_misc.ipp +++ b/src/share/inner_table/ob_inner_table_schema_misc.ipp @@ -3412,3 +3412,113 @@ case OB_TENANT_PARAMETER_AUX_LOB_META_TID: case OB_TENANT_PARAMETER_AUX_LOB_PIECE_TID: #endif + + +#ifdef SYS_INDEX_TABLE_ID_SWITCH + +case OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID: +case OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID: +case OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID: +case OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID: + +#endif + + +#ifdef SYS_INDEX_DATA_TABLE_ID_SWITCH + +case OB_ALL_TABLE_HISTORY_TID: +case OB_ALL_LOG_ARCHIVE_PIECE_FILES_TID: +case OB_ALL_BACKUP_SET_FILES_TID: +case OB_ALL_DDL_TASK_STATUS_TID: + +#endif + + +#ifdef SYS_INDEX_DATA_TABLE_ID_TO_INDEX_IDS_SWITCH + +case OB_ALL_TABLE_HISTORY_TID: { + if (FAILEDx(index_tids.push_back(OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID))) { + LOG_WARN("fail to push back index tid", KR(ret)); + } + break; +} +case OB_ALL_LOG_ARCHIVE_PIECE_FILES_TID: { + if (FAILEDx(index_tids.push_back(OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID))) { + LOG_WARN("fail to push back index tid", KR(ret)); + } + break; +} +case OB_ALL_DDL_TASK_STATUS_TID: { + if (FAILEDx(index_tids.push_back(OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID))) { + LOG_WARN("fail to push back index tid", KR(ret)); + } + break; +} +case OB_ALL_BACKUP_SET_FILES_TID: { + if (FAILEDx(index_tids.push_back(OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID))) { + LOG_WARN("fail to push back index tid", KR(ret)); + } + break; +} + +#endif + + +#ifdef SYS_INDEX_DATA_TABLE_ID_TO_INDEX_SCHEMAS_SWITCH + +case OB_ALL_TABLE_HISTORY_TID: { + if (FAILEDx(ObInnerTableSchema::all_table_history_idx_data_table_id_schema(index_schema))) { + LOG_WARN("fail to create index schema", KR(ret), K(tenant_id), K(data_table_id)); + } else if (!is_sys_tenant(tenant_id) && OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table(tenant_id, index_schema))) { + LOG_WARN("fail to construct full table", KR(ret), K(tenant_id), K(data_table_id)); + } else if (OB_FAIL(tables.push_back(index_schema))) { + LOG_WARN("fail to push back index", KR(ret), K(tenant_id), K(data_table_id)); + } + break; +} +case OB_ALL_LOG_ARCHIVE_PIECE_FILES_TID: { + if (FAILEDx(ObInnerTableSchema::all_log_archive_piece_files_idx_status_schema(index_schema))) { + LOG_WARN("fail to create index schema", KR(ret), K(tenant_id), K(data_table_id)); + } else if (!is_sys_tenant(tenant_id) && OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table(tenant_id, index_schema))) { + LOG_WARN("fail to construct full table", KR(ret), K(tenant_id), K(data_table_id)); + } else if (OB_FAIL(tables.push_back(index_schema))) { + LOG_WARN("fail to push back index", KR(ret), K(tenant_id), K(data_table_id)); + } + break; +} +case OB_ALL_DDL_TASK_STATUS_TID: { + if (FAILEDx(ObInnerTableSchema::all_ddl_task_status_idx_task_key_schema(index_schema))) { + LOG_WARN("fail to create index schema", KR(ret), K(tenant_id), K(data_table_id)); + } else if (!is_sys_tenant(tenant_id) && OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table(tenant_id, index_schema))) { + LOG_WARN("fail to construct full table", KR(ret), K(tenant_id), K(data_table_id)); + } else if (OB_FAIL(tables.push_back(index_schema))) { + LOG_WARN("fail to push back index", KR(ret), K(tenant_id), K(data_table_id)); + } + break; +} +case OB_ALL_BACKUP_SET_FILES_TID: { + if (FAILEDx(ObInnerTableSchema::all_backup_set_files_idx_status_schema(index_schema))) { + LOG_WARN("fail to create index schema", KR(ret), K(tenant_id), K(data_table_id)); + } else if (!is_sys_tenant(tenant_id) && OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table(tenant_id, index_schema))) { + LOG_WARN("fail to construct full table", KR(ret), K(tenant_id), K(data_table_id)); + } else if (OB_FAIL(tables.push_back(index_schema))) { + LOG_WARN("fail to push back index", KR(ret), K(tenant_id), K(data_table_id)); + } + break; +} + +#endif + + +#ifdef ADD_SYS_INDEX_ID + + } else if (OB_FAIL(table_ids.push_back(OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID))) { + LOG_WARN("add index id failed", KR(ret), K(tenant_id)); + } else if (OB_FAIL(table_ids.push_back(OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID))) { + LOG_WARN("add index id failed", KR(ret), K(tenant_id)); + } else if (OB_FAIL(table_ids.push_back(OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID))) { + LOG_WARN("add index id failed", KR(ret), K(tenant_id)); + } else if (OB_FAIL(table_ids.push_back(OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID))) { + LOG_WARN("add index id failed", KR(ret), K(tenant_id)); + +#endif diff --git a/src/share/ob_core_table_proxy.cpp b/src/share/ob_core_table_proxy.cpp index 02451ae35..3d9a46164 100644 --- a/src/share/ob_core_table_proxy.cpp +++ b/src/share/ob_core_table_proxy.cpp @@ -1113,6 +1113,9 @@ int ObCoreTableProxy::execute_update_sql(const Row &row, const ObIArrayvalue_.ptr()) && OB_NOT_NULL(uc->cell_.value_.ptr())) + || (OB_NOT_NULL(c->value_.ptr()) && OB_ISNULL(uc->cell_.value_.ptr()))) { + // NULL == ObString.ptr() means NULL, which is different with empty string(data_length is 0, but ptr is not null) } else if (c->value_ == uc->cell_.value_) { LOG_INFO("value is same, just continue", KPC(c), KPC(uc)); continue; diff --git a/src/share/schema/ob_multi_version_schema_service.cpp b/src/share/schema/ob_multi_version_schema_service.cpp index 3ef94e9c3..29145ed86 100644 --- a/src/share/schema/ob_multi_version_schema_service.cpp +++ b/src/share/schema/ob_multi_version_schema_service.cpp @@ -231,32 +231,17 @@ int ObMultiVersionSchemaService::update_schema_cache( LOG_TRACE("update schema cache", K(lbt())); for (int64_t i = 0; OB_SUCC(ret) && i < schema_array.count(); ++i) { ObTableSchema *table = schema_array.at(i); - int64_t table_id = OB_INVALID_ID; if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("schema is null", K(ret)); - } else if (FALSE_IT(table_id = table->get_table_id())) { - } else if (ObSysTableChecker::is_sys_table_has_index(table_id) - && table->get_index_tid_count() <= 0) { - // Prevent resetting index_tid_array in the second stage of refreshing the schema - const int64_t index_id = ObSysTableChecker::get_sys_table_index_tid(table_id); - if (OB_INVALID_ID == index_id) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid sys table's index_id", KR(ret), K(table_id)); - } else if (OB_FAIL(table->add_simple_index_info(ObAuxTableMetaInfo( - index_id, - USER_INDEX, - INDEX_TYPE_NORMAL_LOCAL)))) { - LOG_WARN("fail to add simple_index_info", K(ret), K(table_id), K(index_id)); - } - } - if (OB_FAIL(ret)) { + LOG_WARN("schema is null", KR(ret)); + } else if (OB_FAIL(ObSysTableChecker::fill_sys_index_infos(*table))) { + LOG_WARN("fail to fill sys indexes", KR(ret), "table_id", table->get_table_id()); } else if (OB_FAIL(schema_cache_.put_schema(TABLE_SCHEMA, table->get_tenant_id(), table->get_table_id(), table->get_schema_version(), *table))) { - LOG_WARN("put schema failed", K(ret)); + LOG_WARN("put schema failed", KR(ret), "table_id", table->get_table_id()); } else { LOG_INFO("put schema succeed", K(*table)); } @@ -271,28 +256,14 @@ int ObMultiVersionSchemaService::update_schema_cache( LOG_TRACE("update schema cache", K(lbt())); for (int64_t i = 0; OB_SUCC(ret) && i < schema_array.count(); ++i) { ObTableSchema &table = schema_array.at(i); - const int64_t table_id = table.get_table_id(); - if (ObSysTableChecker::is_sys_table_has_index(table_id) - && table.get_index_tid_count() <= 0) { - // Prevent resetting index_tid_array in the second stage of refreshing the schema - const int64_t index_id = ObSysTableChecker::get_sys_table_index_tid(table_id); - if (OB_INVALID_ID == index_id) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid sys table's index_id", KR(ret), K(table_id)); - } else if (OB_FAIL(table.add_simple_index_info(ObAuxTableMetaInfo( - index_id, - USER_INDEX, - INDEX_TYPE_NORMAL_LOCAL)))) { - LOG_WARN("fail to add simple_index_info", K(ret), K(index_id)); - } - } - if (OB_FAIL(ret)) { + if (OB_FAIL(ObSysTableChecker::fill_sys_index_infos(table))) { + LOG_WARN("fail to fill sys indexes", KR(ret), "table_id", table.get_table_id()); } else if (OB_FAIL(schema_cache_.put_schema(TABLE_SCHEMA, table.get_tenant_id(), table.get_table_id(), table.get_schema_version(), table))) { - LOG_WARN("put schema failed", K(ret), "table_id", table.get_table_id()); + LOG_WARN("put schema failed", KR(ret), "table_id", table.get_table_id()); } else { LOG_INFO("put schema succeed", K(table)); } @@ -576,13 +547,8 @@ int ObMultiVersionSchemaService::get_schema(const ObSchemaMgr *mgr, || table_schema->is_aux_lob_table()) { // do-nothing } else if (ObSysTableChecker::is_sys_table_has_index(schema_id)) { - const int64_t index_id = ObSysTableChecker::get_sys_table_index_tid(schema_id); - if (OB_INVALID_ID == index_id) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sys table's index_id is invalid", KR(ret), K(schema_id)); - } else if (OB_FAIL(table_schema->add_simple_index_info( - ObAuxTableMetaInfo(index_id, USER_INDEX, INDEX_TYPE_NORMAL_LOCAL)))) { - LOG_WARN("fail to add simple_index_info", K(ret), K(tenant_id), K(schema_id), K(index_id)); + if (OB_FAIL(ObSysTableChecker::fill_sys_index_infos(*table_schema))) { + LOG_WARN("fail to fill sys indexes", KR(ret), K(tenant_id), "table_id", schema_id); } } else if (is_lazy) { ObSEArray aux_table_metas; diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index 34f3f5263..2e46daaae 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -395,69 +395,107 @@ int ObSysTableChecker::check_inner_table_exist( /* -- hard code info for sys table indexes -- */ bool ObSysTableChecker::is_sys_table_index_tid(const int64_t index_id) { - const int64_t pure_index_id = index_id; - return (pure_index_id == OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID - || pure_index_id == OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID - || pure_index_id == OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID - || pure_index_id == OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID); + bool bret = false; + switch (index_id) { +#define SYS_INDEX_TABLE_ID_SWITCH +#include "share/inner_table/ob_inner_table_schema_misc.ipp" +#undef SYS_INDEX_TABLE_ID_SWITCH + { + bret = true; + break; + } + default : { + bret = false; + break; + } + } + return bret; } bool ObSysTableChecker::is_sys_table_has_index(const int64_t table_id) { - bool has_index = false; - const int64_t pure_table_id = table_id; - if (OB_ALL_TABLE_HISTORY_TID == pure_table_id - || OB_ALL_BACKUP_SET_FILES_TID == pure_table_id - || OB_ALL_LOG_ARCHIVE_PIECE_FILES_TID == pure_table_id - || OB_ALL_DDL_TASK_STATUS_TID == pure_table_id) { - has_index = true; + bool bret = false; + switch (table_id) { +#define SYS_INDEX_DATA_TABLE_ID_SWITCH +#include "share/inner_table/ob_inner_table_schema_misc.ipp" +#undef SYS_INDEX_DATA_TABLE_ID_SWITCH + { + bret = true; + break; + } + default : { + bret = false; + break; + } } - return has_index; + return bret; } -int64_t ObSysTableChecker::get_sys_table_index_tid( - const int64_t table_id) -{ - int64_t index_id = OB_INVALID_ID; - if (OB_ALL_TABLE_HISTORY_TID == table_id) { - index_id = OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID; - } else if (OB_ALL_BACKUP_SET_FILES_TID == table_id) { - index_id = OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID; - } else if (OB_ALL_LOG_ARCHIVE_PIECE_FILES_TID == table_id) { - index_id = OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID; - } else if (OB_ALL_DDL_TASK_STATUS_TID == table_id) { - index_id = OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID; - } else { - index_id = OB_INVALID_ID; - } - return index_id; -} - -int ObSysTableChecker::get_sys_table_index_schema( - const int64_t data_table_id, - ObTableSchema &index_schema) +int ObSysTableChecker::fill_sys_index_infos(ObTableSchema &table) { int ret = OB_SUCCESS; - const int64_t pure_data_table_id = data_table_id; - if (OB_ALL_TABLE_HISTORY_TID == pure_data_table_id) { - if (OB_FAIL(ObInnerTableSchema::all_table_history_idx_data_table_id_schema(index_schema))) { - LOG_WARN("fail to create index schema", KR(ret), K(pure_data_table_id)); + const int64_t table_id = table.get_table_id(); + if (ObSysTableChecker::is_sys_table_has_index(table_id) + && table.get_index_tid_count() <= 0) { + ObArray index_tids; + if (OB_FAIL(get_sys_table_index_tids(table_id, index_tids))) { + LOG_WARN("fail to get index tids", KR(ret), K(table_id)); } - } else if (OB_ALL_BACKUP_SET_FILES_TID == pure_data_table_id) { - if (OB_FAIL(ObInnerTableSchema::all_backup_set_files_idx_status_schema(index_schema))) { - LOG_WARN("fail to create index schema", KR(ret), K(pure_data_table_id)); + for (int64_t i = 0; OB_SUCC(ret) && i < index_tids.count(); i++) { + const int64_t index_id = index_tids.at(i); + if (OB_INVALID_ID == index_id) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid sys table's index_id", KR(ret), K(table_id)); + } else if (OB_FAIL(table.add_simple_index_info(ObAuxTableMetaInfo( + index_id, + USER_INDEX, + INDEX_TYPE_NORMAL_LOCAL)))) { + LOG_WARN("fail to add simple_index_info", KR(ret), K(table_id), K(index_id)); + } + } // end for + } + return ret; +} + + +int ObSysTableChecker::get_sys_table_index_tids( + const int64_t table_id, + ObIArray &index_tids) +{ + int ret = OB_SUCCESS; + index_tids.reset(); + switch (table_id) { +#define SYS_INDEX_DATA_TABLE_ID_TO_INDEX_IDS_SWITCH +#include "share/inner_table/ob_inner_table_schema_misc.ipp" +#undef SYS_INDEX_DATA_TABLE_ID_TO_INDEX_IDS_SWITCH + default : { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid data table id", KR(ret), K(table_id)); + break; } - } else if (OB_ALL_LOG_ARCHIVE_PIECE_FILES_TID == pure_data_table_id) { - if (OB_FAIL(ObInnerTableSchema::all_log_archive_piece_files_idx_status_schema(index_schema))) { - LOG_WARN("fail to create index schema", KR(ret), K(pure_data_table_id)); - } - } else if (OB_ALL_DDL_TASK_STATUS_TID == pure_data_table_id) { - if (OB_FAIL(ObInnerTableSchema::all_ddl_task_status_idx_task_key_schema(index_schema))) { - LOG_WARN("fail to create index schema", KR(ret), K(pure_data_table_id)); - } - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("data table is invalid", KR(ret), K(pure_data_table_id)); + } + return ret; +} + +int ObSysTableChecker::append_sys_table_index_schemas( + const uint64_t tenant_id, + const uint64_t data_table_id, + ObIArray &tables) +{ + int ret = OB_SUCCESS; + if (ObSysTableChecker::is_sys_table_has_index(data_table_id)) { + SMART_VAR(ObTableSchema, index_schema) { + switch (data_table_id) { +#define SYS_INDEX_DATA_TABLE_ID_TO_INDEX_SCHEMAS_SWITCH +#include "share/inner_table/ob_inner_table_schema_misc.ipp" +#undef SYS_INDEX_DATA_TABLE_ID_TO_INDEX_SCHEMAS_SWITCH + default : { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("data table is invalid", KR(ret), K(data_table_id)); + break; + } + } + } // end SMART_VAR } return ret; } @@ -470,14 +508,9 @@ int ObSysTableChecker::add_sys_table_index_ids( if (OB_INVALID_TENANT_ID == tenant_id) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid tenant id", KR(ret), K(tenant_id)); - } else if (OB_FAIL(table_ids.push_back(OB_ALL_TABLE_HISTORY_IDX_DATA_TABLE_ID_TID))) { - LOG_WARN("add index table id failed", KR(ret), K(tenant_id)); - } else if (OB_FAIL(table_ids.push_back(OB_ALL_BACKUP_SET_FILES_IDX_STATUS_TID))) { - LOG_WARN("add index table id failed", KR(ret), K(tenant_id)); - } else if (OB_FAIL(table_ids.push_back(OB_ALL_LOG_ARCHIVE_PIECE_FILES_IDX_STATUS_TID))) { - LOG_WARN("add index table id failed", KR(ret), K(tenant_id)); - } else if (OB_FAIL(table_ids.push_back(OB_ALL_DDL_TASK_STATUS_IDX_TASK_KEY_TID))) { - LOG_WARN("add index table id failed", KR(ret), K(tenant_id)); +#define ADD_SYS_INDEX_ID +#include "share/inner_table/ob_inner_table_schema_misc.ipp" +#undef ADD_SYS_INDEX_ID } return ret; } diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index 8dd4009ff..8b50bacf0 100644 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -5683,9 +5683,12 @@ public: static bool is_sys_table_index_tid(const int64_t index_id); static bool is_sys_table_has_index(const int64_t table_id); - static int64_t get_sys_table_index_tid(const int64_t table_id); - static int get_sys_table_index_schema(const int64_t data_table_id, - share::schema::ObTableSchema &index_schema); + static int fill_sys_index_infos(share::schema::ObTableSchema &table); + static int get_sys_table_index_tids(const int64_t table_id, common::ObIArray &index_tids); + static int append_sys_table_index_schemas( + const uint64_t tenant_id, + const uint64_t data_table_id, + common::ObIArray &tables); static int add_sys_table_index_ids( const uint64_t tenant_id, common::ObIArray &table_ids); diff --git a/src/share/schema/ob_schema_utils.cpp b/src/share/schema/ob_schema_utils.cpp index fdedb05a3..f0354d04c 100644 --- a/src/share/schema/ob_schema_utils.cpp +++ b/src/share/schema/ob_schema_utils.cpp @@ -369,10 +369,8 @@ int ObSchemaUtils::construct_tenant_space_full_table( } else { // index const int64_t table_id = table.get_table_id(); - const int64_t index_id = ObSysTableChecker::get_sys_table_index_tid(table_id); - if (OB_INVALID_ID != index_id && OB_FAIL(table.add_simple_index_info( - ObAuxTableMetaInfo(index_id, USER_INDEX, INDEX_TYPE_NORMAL_LOCAL)))) { - LOG_WARN("fail to add simple index info", KR(ret), K(index_id), K(table_id)); + if (OB_FAIL(ObSysTableChecker::fill_sys_index_infos(table))) { + LOG_WARN("fail to fill sys indexes", KR(ret), K(tenant_id), K(table_id)); } // lob aux if (OB_SUCC(ret) && is_system_table(table_id)) { @@ -452,7 +450,7 @@ int ObSchemaUtils::construct_inner_table_schemas( virtual_table_schema_creators, sys_view_schema_creators }; - HEAP_VARS_3((ObTableSchema, table_schema), (ObTableSchema, index_schema), (ObTableSchema, data_schema)) { + HEAP_VARS_2((ObTableSchema, table_schema), (ObTableSchema, data_schema)) { for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(creator_ptr_arrays); ++i) { for (const schema_create_func *creator_ptr = creator_ptr_arrays[i]; OB_SUCC(ret) && OB_NOT_NULL(*creator_ptr); ++creator_ptr) { @@ -471,20 +469,11 @@ int ObSchemaUtils::construct_inner_table_schemas( // skip } else if (OB_FAIL(tables.push_back(table_schema))) { LOG_WARN("fail to push back table schema", KR(ret), K(table_schema)); - } else if (ObSysTableChecker::is_sys_table_has_index(table_schema.get_table_id())) { - index_schema.reset(); - const int64_t data_table_id = table_schema.get_table_id(); - if (OB_FAIL(ObSysTableChecker::get_sys_table_index_schema( - data_table_id, index_schema))) { - LOG_WARN("fail to get sys table's index schema", KR(ret), K(data_table_id)); - } else if (OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table( - tenant_id, index_schema))) { - LOG_WARN("fail to construct tenant space table", KR(ret), K(tenant_id)); - } else if (OB_FAIL(tables.push_back(index_schema))) { - LOG_WARN("fail to push back table schema", KR(ret), K(index_schema)); - } + } else if (OB_FAIL(ObSysTableChecker::append_sys_table_index_schemas( + tenant_id, table_schema.get_table_id(), tables))) { + LOG_WARN("fail to append sys table index schemas", + KR(ret), K(tenant_id), "table_id", table_schema.get_table_id()); } - const int64_t data_table_id = table_schema.get_table_id(); if (OB_SUCC(ret) && exist) { if (OB_FAIL(add_sys_table_lob_aux_table(tenant_id, data_table_id, tables))) {