From da5b2fb1e46f532c6fcc12c33a30b1398d95a060 Mon Sep 17 00:00:00 2001 From: tino247 Date: Fri, 2 Dec 2022 06:35:46 +0000 Subject: [PATCH] Report sys tenant's ls info to __all_ls_meta_table --- src/observer/ob_lease_state_mgr.cpp | 26 ++-- src/observer/report/ob_ls_table_updater.cpp | 34 +++++- src/observer/report/ob_ls_table_updater.h | 8 ++ .../report/ob_server_meta_table_checker.cpp | 2 - .../report/ob_tenant_meta_checker.cpp | 43 +++++-- src/observer/report/ob_tenant_meta_checker.h | 6 +- .../backup/ob_backup_data_set_task_mgr.cpp | 3 +- .../backup/ob_backup_schedule_task.cpp | 6 +- .../ob_major_merge_progress_checker.cpp | 3 +- src/rootserver/ob_ddl_service.cpp | 5 +- src/rootserver/ob_disaster_recovery_task.cpp | 3 + .../ob_disaster_recovery_worker.cpp | 3 + src/rootserver/ob_empty_server_checker.cpp | 2 +- src/rootserver/ob_lost_replica_checker.cpp | 33 +++-- .../ob_migrate_unit_finish_checker.cpp | 1 + src/rootserver/ob_primary_ls_service.cpp | 3 +- src/rootserver/ob_root_service.cpp | 14 ++- src/rootserver/ob_system_admin_util.cpp | 3 +- src/rootserver/ob_update_rs_list_task.cpp | 3 +- .../restore/ob_restore_scheduler.cpp | 1 - .../virtual_table/ob_core_meta_table.cpp | 2 +- .../ob_inner_table_schema.21151_21200.cpp | 4 +- .../inner_table/ob_inner_table_schema_def.py | 3 +- .../location_cache/ob_ls_location_service.cpp | 6 +- src/share/ls/ob_inmemory_ls_table.cpp | 28 +++-- src/share/ls/ob_inmemory_ls_table.h | 9 +- src/share/ls/ob_ls_info.cpp | 37 ++++++ src/share/ls/ob_ls_info.h | 2 + src/share/ls/ob_ls_table.h | 17 ++- src/share/ls/ob_ls_table_iterator.cpp | 20 +++- src/share/ls/ob_ls_table_iterator.h | 13 +- src/share/ls/ob_ls_table_operator.cpp | 113 +++++++++++++----- src/share/ls/ob_ls_table_operator.h | 41 ++++--- src/share/ls/ob_persistent_ls_table.cpp | 11 +- src/share/ls/ob_persistent_ls_table.h | 10 +- src/share/ls/ob_rpc_ls_table.cpp | 28 +++-- src/share/ls/ob_rpc_ls_table.h | 10 +- src/share/ob_debug_sync_point.h | 1 + src/share/ob_leader_election_waiter.cpp | 3 +- src/share/ob_rs_mgr.cpp | 1 + .../ob_ls_migration_handler.cpp | 3 +- 41 files changed, 419 insertions(+), 145 deletions(-) diff --git a/src/observer/ob_lease_state_mgr.cpp b/src/observer/ob_lease_state_mgr.cpp index d1d93c42f8..dc55fa7714 100644 --- a/src/observer/ob_lease_state_mgr.cpp +++ b/src/observer/ob_lease_state_mgr.cpp @@ -199,29 +199,33 @@ int ObLeaseStateMgr::try_report_sys_ls() ret = OB_SERVER_IS_STOPPING; LOG_WARN("lease manager is stopped", KR(ret)); } else { - MTL_SWITCH(OB_SYS_TENANT_ID) { + const uint64_t tenant_id = OB_SYS_TENANT_ID; + const ObLSID ls_id = SYS_LS; + MTL_SWITCH(tenant_id) { bool ls_exist = false; - ObLSService *ls_svr = nullptr; - if (OB_UNLIKELY(nullptr == (ls_svr = MTL(ObLSService*)))) { + ObLSService *ls_svr = NULL; + if (OB_ISNULL(ls_svr = MTL(ObLSService*))) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant storage ptr is null", KR(ret), "tenant_id", OB_SYS_TENANT_ID); - } else if (OB_FAIL(ls_svr->check_ls_exist(SYS_LS, ls_exist))) { - LOG_WARN("fail to check log stream exist", KR(ret), K(SYS_LS)); + LOG_WARN("tenant storage ptr is null", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ls_svr->check_ls_exist(ls_id, ls_exist))) { + LOG_WARN("fail to check log stream exist", KR(ret), K(ls_id)); } else if (!ls_exist) { // core log stream not exist } else { share::ObLSTableOperator *lst_operator = GCTX.lst_operator_; share::ObLSReplica ls_replica; - if (OB_UNLIKELY(nullptr == ob_service_ || nullptr == lst_operator)) { + if (OB_ISNULL(ob_service_) || OB_ISNULL(lst_operator)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ob_service or lst_operator ptr is null", KR(ret), KP(ob_service_), KP(lst_operator)); } else if (OB_FAIL(ob_service_->fill_ls_replica( - OB_SYS_TENANT_ID, SYS_LS, ls_replica))) { + tenant_id, ls_id, ls_replica))) { LOG_WARN("fail to fill log stream replica", KR(ret), - "tenant_id", OB_SYS_TENANT_ID, K(ls_replica)); - } else if (OB_FAIL(lst_operator->update(ls_replica))) { + K(tenant_id), K(ls_replica)); + } else if (OB_FAIL(lst_operator->update(ls_replica, false/*inner_table_only*/))) { LOG_WARN("fail to report sys log stream", KR(ret), K(ls_replica)); + } else if (OB_FAIL(ob_service_->submit_ls_update_task(tenant_id, ls_id))) { + LOG_WARN("fail to add async update task", KR(ret), K(tenant_id), K(ls_id)); } else { LOG_INFO("try report sys log stream succeed"); } @@ -230,7 +234,7 @@ int ObLeaseStateMgr::try_report_sys_ls() if (OB_TENANT_NOT_IN_SERVER == ret) { ret = OB_SUCCESS; } else { - LOG_WARN("fail to switch tenant", KR(ret), "tenant_id", OB_SYS_TENANT_ID); + LOG_WARN("fail to switch tenant", KR(ret), K(tenant_id)); } } } diff --git a/src/observer/report/ob_ls_table_updater.cpp b/src/observer/report/ob_ls_table_updater.cpp index 2fcf1db7dc..a82c3d6d14 100644 --- a/src/observer/report/ob_ls_table_updater.cpp +++ b/src/observer/report/ob_ls_table_updater.cpp @@ -29,6 +29,7 @@ namespace observer int ObLSTableUpdateTask::init( const uint64_t tenant_id, const ObLSID &ls_id, + const bool inner_table_only, const int64_t add_timestamp) { int ret = OB_SUCCESS; @@ -39,6 +40,7 @@ int ObLSTableUpdateTask::init( } else { tenant_id_ = tenant_id; ls_id_ = ls_id; + inner_table_only_ = inner_table_only; add_timestamp_ = add_timestamp; } return ret; @@ -48,6 +50,7 @@ void ObLSTableUpdateTask::reset() { tenant_id_ = OB_INVALID_TENANT_ID; ls_id_.reset(); + inner_table_only_ = false; add_timestamp_ = OB_INVALID_TIMESTAMP; } @@ -64,6 +67,7 @@ int ObLSTableUpdateTask::assign(const ObLSTableUpdateTask &other) if (this != &other) { tenant_id_ = other.tenant_id_; ls_id_ = other.ls_id_; + inner_table_only_ = other.inner_table_only_; add_timestamp_ = other.add_timestamp_; } return ret; @@ -86,7 +90,10 @@ bool ObLSTableUpdateTask::operator ==(const ObLSTableUpdateTask &other) const equal = true; } else { equal = (tenant_id_ == other.tenant_id_ - && ls_id_ == other.ls_id_); + && ls_id_ == other.ls_id_); + if (equal && is_sys_tenant(tenant_id_)) { + equal = (inner_table_only_ == other.inner_table_only_); + } } return equal; } @@ -178,13 +185,28 @@ int ObLSTableUpdateQueueSet::add_task(const ObLSTableUpdateTask &task) } else { const uint64_t &tenant_id = task.get_tenant_id(); if (is_sys_tenant(tenant_id)) { - if (OB_FAIL(sys_tenant_queue_.add(task))) { + if (task.is_inner_table_only()) { + // only report sys tenant's ls to inner table + } else if (OB_FAIL(sys_tenant_queue_.add(task))) { if (OB_EAGAIN != ret) { LOG_WARN("sys_tenant_queue add_task failed", KR(ret), K(task)); } else { ret = OB_SUCCESS; } } + if (OB_SUCC(ret)) { + ObLSTableUpdateTask new_task(task.get_tenant_id(), + task.get_ls_id(), + true/*inner_table_only*/, + task.get_add_timestamp()); + if (OB_FAIL(meta_tenant_queue_.add(new_task))) { + if (OB_EAGAIN != ret) { + LOG_WARN("meta_tenant_queue add_task failed", KR(ret), K(new_task)); + } else { + ret = OB_SUCCESS; + } + } + } } else if (is_meta_tenant(tenant_id)) { if (OB_FAIL(meta_tenant_queue_.add(task))) { if (OB_EAGAIN != ret) { @@ -252,7 +274,8 @@ int ObLSTableUpdater::async_update( { int ret = OB_SUCCESS; int64_t now = ObTimeUtility::current_time(); - ObLSTableUpdateTask task(tenant_id, ls_id, now); + bool inner_table_only = false; + ObLSTableUpdateTask task(tenant_id, ls_id, inner_table_only, now); if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("init twice", KR(ret)); @@ -284,6 +307,7 @@ int ObLSTableUpdater::batch_process_tasks( const ObLSTableUpdateTask &task = tasks.at(0); const uint64_t tenant_id = task.get_tenant_id(); const ObLSID &ls_id = task.get_ls_id(); + const bool inner_table_only = task.is_inner_table_only(); bool tenant_dropped = false; bool schema_not_ready = false; uint64_t superior_tenant_id = OB_INVALID_TENANT_ID; @@ -315,7 +339,7 @@ int ObLSTableUpdater::batch_process_tasks( ls_id, replica))) { if (OB_LS_NOT_EXIST == ret || OB_TENANT_NOT_IN_SERVER == ret) { // remove from table if not exist - if (OB_FAIL(GCTX.lst_operator_->remove(tenant_id, ls_id, server))) { + if (OB_FAIL(GCTX.lst_operator_->remove(tenant_id, ls_id, server, inner_table_only))) { LOG_WARN("fail to remove replica", KR(ret), K(tenant_id), K(ls_id), "self_addr", server); } else { @@ -325,7 +349,7 @@ int ObLSTableUpdater::batch_process_tasks( } else { LOG_WARN("fail to fill log stream replica", KR(ret), K(tenant_id), K(ls_id)); } - } else if (OB_FAIL(GCTX.lst_operator_->update(replica))) { + } else if (OB_FAIL(GCTX.lst_operator_->update(replica, inner_table_only))) { LOG_WARN("fail to update replica", KR(ret), K(tenant_id), K(ls_id), K(replica)); } else { ObTaskController::get().allow_next_syslog(); diff --git a/src/observer/report/ob_ls_table_updater.h b/src/observer/report/ob_ls_table_updater.h index 0ed318fec8..97d43617ee 100644 --- a/src/observer/report/ob_ls_table_updater.h +++ b/src/observer/report/ob_ls_table_updater.h @@ -26,18 +26,22 @@ public: ObLSTableUpdateTask() : tenant_id_(OB_INVALID_TENANT_ID), ls_id_(), + inner_table_only_(false), add_timestamp_(OB_INVALID_TIMESTAMP) {} explicit ObLSTableUpdateTask( const uint64_t tenant_id, const share::ObLSID &ls_id, + const bool inner_table_only, const int64_t add_timestamp) : tenant_id_(tenant_id), ls_id_(ls_id), + inner_table_only_(inner_table_only), add_timestamp_(add_timestamp) {} virtual ~ObLSTableUpdateTask() {} int init( const uint64_t tenant_id, const share::ObLSID &ls_id, + const bool inner_table_only, const int64_t add_timestamp); int assign(const ObLSTableUpdateTask &other); virtual void reset(); @@ -54,10 +58,14 @@ public: inline int64_t get_tenant_id() const { return tenant_id_; } inline share::ObLSID get_ls_id() const { return ls_id_; } inline int64_t get_add_timestamp() const { return add_timestamp_; } + inline bool is_inner_table_only() const { return inner_table_only_; } TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(add_timestamp)); private: uint64_t tenant_id_; share::ObLSID ls_id_; + // 1. For sys tenant, when inner_table_only_ = true, task will be add to meta_tenant_queue_. Otherwise, task will be add to sys_tenant_queue_. + // 2. For other tenants, inner_table_only_ is useless. + bool inner_table_only_; int64_t add_timestamp_; }; diff --git a/src/observer/report/ob_server_meta_table_checker.cpp b/src/observer/report/ob_server_meta_table_checker.cpp index a7e715cdb3..4bffbab4b7 100644 --- a/src/observer/report/ob_server_meta_table_checker.cpp +++ b/src/observer/report/ob_server_meta_table_checker.cpp @@ -16,9 +16,7 @@ #include "observer/ob_server_struct.h" // GCTX #include "share/ob_thread_define.h" // ServerMetaChecker #include "share/ls/ob_ls_table_operator.h" // ObLSTableOperator -#include "share/ls/ob_ls_table_iterator.h" // ObLSTableIterator #include "share/tablet/ob_tablet_table_operator.h" // ObTabletTableOperator -#include "share/tablet/ob_tablet_table_iterator.h" // ObTenantTabletTableIterator #include "share/schema/ob_multi_version_schema_service.h" // ObMultiVersionSchemaService #include "observer/omt/ob_multi_tenant.h" // ObMultiTenant #include "share/tablet/ob_tablet_info.h" // ObTabletInfo diff --git a/src/observer/report/ob_tenant_meta_checker.cpp b/src/observer/report/ob_tenant_meta_checker.cpp index 55e9796bff..70db0e873b 100644 --- a/src/observer/report/ob_tenant_meta_checker.cpp +++ b/src/observer/report/ob_tenant_meta_checker.cpp @@ -177,23 +177,46 @@ void ObTenantMetaChecker::destroy() int ObTenantMetaChecker::check_ls_table() { int ret = OB_SUCCESS; - int64_t dangling_count = 0; // replica only in ls meta table - int64_t report_count = 0; // replica not in/match ls meta table ObCurTraceId::init(GCONF.self_addr_); + share::ObLSTable::Mode mode = share::ObLSTable::DEFAULT_MODE; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else { + if (OB_FAIL(check_ls_table_(mode))) { + LOG_WARN("check ls table failed", KR(ret), K(mode)); + } + // Additionally, check sys tenant's inner table + if (is_sys_tenant(tenant_id_)) { // overwrite ret + mode = share::ObLSTable::INNER_TABLE_ONLY_MODE; + if (OB_FAIL(check_ls_table_(mode))) { + LOG_WARN("check ls table failed", KR(ret), K(mode)); + } + } + } + return ret; +} + +int ObTenantMetaChecker::check_ls_table_( + const share::ObLSTable::Mode mode) +{ + int ret = OB_SUCCESS; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); } else { const int64_t start_time = ObTimeUtility::current_time(); ObLSReplicaMap replica_map; - if (OB_FAIL(build_replica_map_(replica_map))) { - LOG_WARN("build replica map from ls table failed", KR(ret)); + int64_t dangling_count = 0; // replica only in ls meta table + int64_t report_count = 0; // replica not in/match ls meta table + if (OB_FAIL(build_replica_map_(replica_map, mode))) { + LOG_WARN("build replica map from ls table failed", KR(ret), K(mode)); } else if (OB_FAIL(check_dangling_replicas_(replica_map, dangling_count))) { LOG_WARN("check replicas exist in ls table but not in local failed", KR(ret)); } else if (OB_FAIL(check_report_replicas_(replica_map, report_count))) { LOG_WARN("check replicas not in/match ls table failed", KR(ret)); } else if (dangling_count != 0 || report_count != 0) { - LOG_INFO("checker found and corrected dangling or to report replicas for ls meta table", + LOG_INFO("checker found and corrected dangling or to report replicas for ls meta table", KR(ret), K_(tenant_id), K(dangling_count), K(report_count), K_(ls_checker_tg_id)); } LOG_TRACE("finish checking ls table", KR(ret), K_(tenant_id), @@ -221,7 +244,7 @@ int ObTenantMetaChecker::check_tablet_table() } else if (OB_FAIL(check_report_replicas_(replica_map, report_count))) { LOG_WARN("check replicas not in/match tablet table failed", KR(ret)); } else if (dangling_count != 0 || report_count != 0) { - LOG_INFO("checker found and corrected dangling or to report replicas for tablet meta table", + LOG_INFO("checker found and corrected dangling or to report replicas for tablet meta table", KR(ret), K_(tenant_id), K(dangling_count), K(report_count), K_(tablet_checker_tg_id)); } LOG_TRACE("finish checking tablet table", KR(ret), K_(tenant_id), @@ -276,7 +299,9 @@ int ObTenantMetaChecker::schedule_tablet_meta_check_task() return ret; } -int ObTenantMetaChecker::build_replica_map_(ObLSReplicaMap &replica_map) +int ObTenantMetaChecker::build_replica_map_( + ObLSReplicaMap &replica_map, + const share::ObLSTable::Mode mode) { int ret = OB_SUCCESS; ObLSTableIterator lst_iter; @@ -290,8 +315,8 @@ int ObTenantMetaChecker::build_replica_map_(ObLSReplicaMap &replica_map) hash::cal_next_prime(LS_REPLICA_MAP_BUCKET_NUM), "LSCheckMap"))) { LOG_WARN("fail to create replica_map", KR(ret)); - } else if (OB_FAIL(lst_iter.init(*lst_operator_, tenant_id_))) { - LOG_WARN("fail to init ls meta table iter", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(lst_iter.init(*lst_operator_, tenant_id_, mode))) { + LOG_WARN("fail to init ls meta table iter", KR(ret), K_(tenant_id), K(mode)); } else if (OB_FAIL(lst_iter.get_filters().set_reserved_server(GCONF.self_addr_))) { LOG_WARN("fail to set server for filter", KR(ret), "server", GCONF.self_addr_); } else { diff --git a/src/observer/report/ob_tenant_meta_checker.h b/src/observer/report/ob_tenant_meta_checker.h index f8c116889c..14a80ac0dd 100644 --- a/src/observer/report/ob_tenant_meta_checker.h +++ b/src/observer/report/ob_tenant_meta_checker.h @@ -16,6 +16,7 @@ #include "lib/task/ob_timer.h" // ObTimerTask #include "share/ls/ob_ls_info.h" // ObLSReplica #include "share/tablet/ob_tablet_info.h" // ObTabletReplica +#include "share/ls/ob_ls_table.h" // ObLSTable::Mode namespace oceanbase { @@ -71,14 +72,15 @@ public: int check_tablet_table(); int schedule_ls_meta_check_task(); int schedule_tablet_meta_check_task(); - +private: + int check_ls_table_(const share::ObLSTable::Mode mode); private: static const int64_t LS_REPLICA_MAP_BUCKET_NUM = 10; static const int64_t TABLET_REPLICA_MAP_BUCKET_NUM = 64 * 1024; typedef common::hash::ObHashMap ObLSReplicaMap; typedef common::hash::ObHashMap ObTabletReplicaMap; - int build_replica_map_(ObLSReplicaMap &replica_map); + int build_replica_map_(ObLSReplicaMap &replica_map, const share::ObLSTable::Mode mode); int build_replica_map_(ObTabletReplicaMap &replica_map); int check_dangling_replicas_(ObLSReplicaMap &replica_map, int64_t &dangling_count); int check_dangling_replicas_(ObTabletReplicaMap &replica_map, int64_t &dangling_count); diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp index 52b12e3540..48f10c3ebf 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp @@ -724,7 +724,8 @@ int ObBackupSetTaskMgr::get_dst_server_(const ObLSID &ls_id, ObAddr &dst) } else if (OB_ISNULL(lst_operator)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[DATA_BACKUP]lst_operator ptr is null", K(ret)); - } else if (OB_FAIL(lst_operator->get(cluster_id, tenant_id, ls_id, ls_info))) { + } else if (OB_FAIL(lst_operator->get(cluster_id, tenant_id, + ls_id, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("[DATA_BACKUP]failed to get log stream info", K(ret), K(cluster_id), K(tenant_id), K(ls_id)); } else { const ObLSInfo::ReplicaArray &replica_array = ls_info.get_replicas(); diff --git a/src/rootserver/backup/ob_backup_schedule_task.cpp b/src/rootserver/backup/ob_backup_schedule_task.cpp index 22dfa88c41..c334d2f66d 100644 --- a/src/rootserver/backup/ob_backup_schedule_task.cpp +++ b/src/rootserver/backup/ob_backup_schedule_task.cpp @@ -461,7 +461,8 @@ int ObBackupDataLSTask::set_optional_servers_(const ObIArray &bl if (nullptr == lst_operator) { ret = OB_ERR_UNEXPECTED; LOG_WARN("lst_operator ptr is null", K(ret)); - } else if (OB_FAIL(lst_operator->get(cluster_id, tenant_id, ls_id_, ls_info))) { + } else if (OB_FAIL(lst_operator->get(cluster_id, tenant_id, + ls_id_, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("failed to get log stream info", K(ret), K(cluster_id), K(tenant_id), K(ls_id_)); } else { const ObLSInfo::ReplicaArray &replica_array = ls_info.get_replicas(); @@ -816,7 +817,8 @@ int ObBackupBuildIndexTask::set_optional_servers_(const ObIArray if (nullptr == lst_operator) { ret = OB_ERR_UNEXPECTED; LOG_WARN("lst_operator ptr is null", K(ret)); - } else if (OB_FAIL(lst_operator->get(cluster_id, tenant_id, ls_id, ls_info))) { + } else if (OB_FAIL(lst_operator->get(cluster_id, tenant_id, + ls_id, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("failed to get log stream info", K(ret), K(cluster_id), K(tenant_id), K(ls_id)); } else { const ObLSInfo::ReplicaArray &replica_array = ls_info.get_replicas(); diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp index 09f190ea41..e0ffae7440 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp @@ -195,7 +195,8 @@ int ObMajorMergeProgressChecker::check_tablet( ObLSInfo ls_info; int64_t cluster_id = GCONF.cluster_id; const ObLSID &ls_id = tablet.get_ls_id(); - if (OB_FAIL(lst_operator_->get(cluster_id, tenant_id_, ls_id, ls_info))) { + if (OB_FAIL(lst_operator_->get(cluster_id, tenant_id_, + ls_id, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("fail to get ls info", KR(ret), K_(tenant_id), K(ls_id)); } else if (OB_FAIL(check_majority_integrated(schema_guard, tablet, ls_info))) { LOG_WARN("fail to check majority integrated", KR(ret)); diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index cc4cf45231..df8ee5d4e7 100644 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -19490,8 +19490,8 @@ int ObDDLService::broadcast_sys_table_schemas( ObArray addrs; const ObLSReplica *leader = NULL; ObLSReplica::MemberList member_list; - if (OB_FAIL(lst_operator_->get( - GCONF.cluster_id, tenant_id, SYS_LS, ls_info))) { + if (OB_FAIL(lst_operator_->get(GCONF.cluster_id, tenant_id, + SYS_LS, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("fail to get sys ls info", KR(ret), K(tenant_id)); } else if (OB_FAIL(ls_info.find_leader(leader))) { LOG_WARN("fail to get leader", KR(ret), K(tenant_id)); @@ -19777,6 +19777,7 @@ int ObDDLService::init_tenant_schema( GCONF.cluster_id, tenant_id, SYS_LS, + share::ObLSTable::DEFAULT_MODE, sys_ls_info))) { LOG_WARN("fail to get sys ls info by operator", KR(ret), K(tenant_id)); } else if (OB_FAIL(sys_ls_info.get_paxos_member_addrs(addrs))) { diff --git a/src/rootserver/ob_disaster_recovery_task.cpp b/src/rootserver/ob_disaster_recovery_task.cpp index 12fcfb5a81..5538fefca7 100644 --- a/src/rootserver/ob_disaster_recovery_task.cpp +++ b/src/rootserver/ob_disaster_recovery_task.cpp @@ -570,6 +570,7 @@ int ObMigrateLSReplicaTask::check_before_execute( GCONF.cluster_id, get_tenant_id(), get_ls_id(), + share::ObLSTable::COMPOSITE_MODE, ls_info))) { LOG_WARN("fail to get log stream info", KR(ret), "tenant_id", get_tenant_id(), @@ -1022,6 +1023,7 @@ int ObAddLSReplicaTask::check_before_execute( GCONF.cluster_id, get_tenant_id(), get_ls_id(), + share::ObLSTable::COMPOSITE_MODE, ls_info))) { LOG_WARN("fail to get log stream info", KR(ret), "tenant_id", get_tenant_id(), @@ -1494,6 +1496,7 @@ int ObLSTypeTransformTask::check_before_execute( GCONF.cluster_id, get_tenant_id(), get_ls_id(), + share::ObLSTable::COMPOSITE_MODE, ls_info))) { LOG_WARN("fail to get log stream info", KR(ret), "tenant_id", get_tenant_id(), diff --git a/src/rootserver/ob_disaster_recovery_worker.cpp b/src/rootserver/ob_disaster_recovery_worker.cpp index 0036c76aa5..7daefaef0b 100644 --- a/src/rootserver/ob_disaster_recovery_worker.cpp +++ b/src/rootserver/ob_disaster_recovery_worker.cpp @@ -1825,6 +1825,7 @@ int ObDRWorker::check_tenant_locality_match( GCONF.cluster_id, ls_status_info.tenant_id_, ls_status_info.ls_id_, + share::ObLSTable::COMPOSITE_MODE, ls_info))) { LOG_WARN("fail to get log stream info", KR(ret)); } else if (OB_FAIL(dr_ls_info.init())) { @@ -1886,6 +1887,7 @@ int ObDRWorker::check_ls_locality_match_( int ObDRWorker::try_disaster_recovery() { int ret = OB_SUCCESS; + DEBUG_SYNC(BEFORE_TRY_DISASTER_RECOVERY); ObCurTraceId::init(GCONF.self_addr_); ObArray tenant_id_array; if (OB_UNLIKELY(!inited_)) { @@ -1968,6 +1970,7 @@ int ObDRWorker::try_tenant_disaster_recovery( GCONF.cluster_id, ls_status_info.tenant_id_, ls_status_info.ls_id_, + share::ObLSTable::COMPOSITE_MODE, ls_info))) { LOG_WARN("fail to get log stream info", KR(tmp_ret)); } else if (OB_SUCCESS != (tmp_ret = dr_ls_info.init())) { diff --git a/src/rootserver/ob_empty_server_checker.cpp b/src/rootserver/ob_empty_server_checker.cpp index 1afdb4378d..b2951a4d0d 100644 --- a/src/rootserver/ob_empty_server_checker.cpp +++ b/src/rootserver/ob_empty_server_checker.cpp @@ -22,7 +22,7 @@ #include "share/schema/ob_multi_version_schema_service.h" #include "share/config/ob_server_config.h" #include "share/ob_rpc_struct.h"//GetLSReportCnt -#include "share/ls/ob_ls_table_iterator.h"//ObLSTableIterator +#include "share/ls/ob_ls_table_iterator.h"//ObAllLSTableIterator #include "share/ls/ob_ls_info.h"//ObLSInfo #include "observer/ob_server_struct.h" diff --git a/src/rootserver/ob_lost_replica_checker.cpp b/src/rootserver/ob_lost_replica_checker.cpp index 153f50b3c0..f178427738 100644 --- a/src/rootserver/ob_lost_replica_checker.cpp +++ b/src/rootserver/ob_lost_replica_checker.cpp @@ -21,7 +21,7 @@ #include "share/config/ob_server_config.h" #include "share/schema/ob_multi_version_schema_service.h" #include "share/schema/ob_schema_getter_guard.h" -#include "share/ls/ob_ls_table_iterator.h"//ObLSTableIterator +#include "share/ls/ob_ls_table_iterator.h"//ObTenantLSTableIterator #include "share/ls/ob_ls_info.h"//ObLSInfo #include "rootserver/ob_server_manager.h" #include "observer/ob_server_struct.h" @@ -162,35 +162,46 @@ int ObLostReplicaChecker::check_lost_replicas() int ObLostReplicaChecker::check_lost_replica_by_ls_(const share::ObLSInfo &ls_info) { int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; bool is_lost_replica = false; int64_t lost_count = 0; LOG_DEBUG("start checking lost replicas by ls", K(ls_info)); if (!inited_) { ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); + LOG_WARN("not init", KR(ret)); } else if (OB_UNLIKELY(!ls_info.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls info invalid", K(ret), K(ls_info)); + LOG_WARN("ls info invalid", KR(ret), K(ls_info)); } else if (OB_ISNULL(lst_operator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls operator is null", KR(ret), KP(lst_operator_)); } else { const share::ObLSInfo::ReplicaArray &replicas = ls_info.get_replicas(); - FOREACH_CNT_X(replica, replicas, OB_SUCCESS == ret) { + FOREACH_CNT_X(replica, replicas, OB_SUCC(ret)) { is_lost_replica = false; if (OB_FAIL(check_lost_replica_(ls_info, *replica, is_lost_replica))) { - LOG_WARN("check_lost_replica failed", K(ls_info), "replica", - *replica, KR(ret)); + LOG_WARN("check_lost_replica failed", KR(ret), K(ls_info), KPC(replica)); } else if (is_lost_replica) { lost_count++; if (OB_FAIL(lst_operator_->remove(replica->get_tenant_id(), replica->get_ls_id(), - replica->get_server()))) { - LOG_WARN("lst_operator remove replica failed", KR(ret), "replica", *replica); + replica->get_server(), + false/*inner_table_only*/))) { + LOG_WARN("lst_operator remove replica failed", KR(ret), KPC(replica)); } else { - LOG_INFO("lost replica checker remove lost replica finish", "replica", - *replica, KR(ret), K(tmp_ret)); + LOG_INFO("lost replica checker remove lost replica finish", KR(ret), KPC(replica)); + } + + if (OB_SUCC(ret) && is_sys_tenant(replica->get_tenant_id())) { + if (OB_FAIL(lst_operator_->remove(replica->get_tenant_id(), + replica->get_ls_id(), + replica->get_server(), + true/*inner_table_only*/))) { + LOG_WARN("lst_operator remove replica from inner table failed", + KR(ret), KPC(replica)); + } else { + LOG_INFO("lost replica checker remove lost replica from inner table finish", + KR(ret), KPC(replica)); + } } } else { // do nothing diff --git a/src/rootserver/ob_migrate_unit_finish_checker.cpp b/src/rootserver/ob_migrate_unit_finish_checker.cpp index eae662b767..4f2346d387 100644 --- a/src/rootserver/ob_migrate_unit_finish_checker.cpp +++ b/src/rootserver/ob_migrate_unit_finish_checker.cpp @@ -229,6 +229,7 @@ int ObMigrateUnitFinishChecker::try_check_migrate_unit_finish_by_tenant( GCONF.cluster_id, ls_status_info.tenant_id_, ls_status_info.ls_id_, + share::ObLSTable::COMPOSITE_MODE, ls_info))) { LOG_WARN("fail to get log stream info", KR(ret)); } else if (OB_FAIL(dr_ls_info.build_disaster_ls_info( diff --git a/src/rootserver/ob_primary_ls_service.cpp b/src/rootserver/ob_primary_ls_service.cpp index 509f885fc2..aacb3b222f 100755 --- a/src/rootserver/ob_primary_ls_service.cpp +++ b/src/rootserver/ob_primary_ls_service.cpp @@ -1603,7 +1603,8 @@ int ObTenantLSInfo::check_ls_can_offline_by_rpc_(const share::ObLSStatusInfo &in ret = OB_ERR_UNEXPECTED; LOG_WARN("lst operator or proxy is null", KR(ret), KP(lst_operator_), KP(rpc_proxy_)); - } else if (OB_FAIL(lst_operator_->get(GCONF.cluster_id, info.tenant_id_, info.ls_id_, ls_info))) { + } else if (OB_FAIL(lst_operator_->get(GCONF.cluster_id, info.tenant_id_, + info.ls_id_, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("failed to get ls info", KR(ret), K(info)); } else if (OB_FAIL(ls_info.find_leader(replica))) { LOG_WARN("failed to find leader", KR(ret), K(ls_info)); diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index ec638dd0e0..cff649984e 100644 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -285,6 +285,7 @@ int ObRootService::ObOfflineServerTask::process() cluster_id, tenant_id, SYS_LS, + share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("fail to get", KR(ret)); } else { @@ -2242,6 +2243,7 @@ int ObRootService::report_sys_ls(const share::ObLSReplica &replica) int ret = OB_SUCCESS; ObInMemoryLSTable *inmemory_ls = NULL; ObRole role = FOLLOWER; + bool inner_table_only = false; LOG_INFO("receive request to report sys ls", K(replica)); if (OB_UNLIKELY(!inited_) || OB_ISNULL(lst_operator_)) { ret = OB_NOT_INIT; @@ -2257,7 +2259,7 @@ int ObRootService::report_sys_ls(const share::ObLSReplica &replica) } else if (OB_ISNULL(inmemory_ls = lst_operator_->get_inmemory_ls())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get inmemory ls", KR(ret), K(replica)); - } else if (OB_FAIL(inmemory_ls->update(replica))) { + } else if (OB_FAIL(inmemory_ls->update(replica, inner_table_only))) { LOG_WARN("update sys ls failed", KR(ret), K(replica)); } else { LOG_INFO("update sys ls on rs success", K(replica)); @@ -2270,6 +2272,7 @@ int ObRootService::remove_sys_ls(const obrpc::ObRemoveSysLsArg &arg) int ret = OB_SUCCESS; ObInMemoryLSTable *inmemory_ls = NULL; ObRole role = FOLLOWER; + bool inner_table_only = false; LOG_INFO("receive request to remove sys ls", K(arg)); if (OB_UNLIKELY(!inited_) || OB_ISNULL(lst_operator_)) { ret = OB_NOT_INIT; @@ -2288,7 +2291,8 @@ int ObRootService::remove_sys_ls(const obrpc::ObRemoveSysLsArg &arg) } else if (OB_FAIL(inmemory_ls->remove( OB_SYS_TENANT_ID, SYS_LS, - arg.server_))) { + arg.server_, + inner_table_only))) { LOG_WARN("remove sys ls failed", KR(ret), K(arg)); } else { LOG_INFO("remove sys ls on rs success", K(arg)); @@ -5144,6 +5148,7 @@ int ObRootService::fetch_sys_tenant_ls_info() int ret = OB_SUCCESS; ObLSReplica replica; ObRole role = FOLLOWER; + bool inner_table_only = false; ObMemberList member_list; // TODO: automatically decide to use rpc or inmemory ObLSTable* inmemory_ls; @@ -5176,7 +5181,7 @@ int ObRootService::fetch_sys_tenant_ls_info() SYS_LS, replica))) { LOG_WARN("fail to fill log stream replica", KR(ret), K(replica)); - } else if (OB_FAIL(inmemory_ls->update(replica))) { + } else if (OB_FAIL(inmemory_ls->update(replica, inner_table_only))) { LOG_WARN("fail to update ls replica", KR(ret), K(replica)); } } @@ -5196,7 +5201,7 @@ int ObRootService::fetch_sys_tenant_ls_info() } else if (replica.is_strong_leader()) { ret = OB_ENTRY_EXIST; LOG_WARN("role should be follower", KR(ret), K(replica)); - } else if (OB_FAIL(inmemory_ls->update(replica))) { + } else if (OB_FAIL(inmemory_ls->update(replica, inner_table_only))) { LOG_WARN("update sys_ls info failed", KR(ret), K(replica)); } else { LOG_INFO("update sys_tenant ls replica succeed", K(replica), "server", addr); @@ -6667,6 +6672,7 @@ int ObRootService::construct_rs_list_arg( GCONF.cluster_id, tenant_id, SYS_LS, + share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("fail to get", KR(ret)); } else { diff --git a/src/rootserver/ob_system_admin_util.cpp b/src/rootserver/ob_system_admin_util.cpp index 101f2c794b..262ca2ad19 100644 --- a/src/rootserver/ob_system_admin_util.cpp +++ b/src/rootserver/ob_system_admin_util.cpp @@ -191,7 +191,8 @@ int ObAdminSwitchReplicaRole::execute(const ObAdminSwitchReplicaRoleArg &arg) } else if (OB_ISNULL(GCTX.lst_operator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("GCTX.lst_operator_ is NULL", K(arg), KR(ret), K(tenant_id)); - } else if (OB_FAIL(GCTX.lst_operator_->get(GCONF.cluster_id, tenant_id, ls_id, ls_info))) { + } else if (OB_FAIL(GCTX.lst_operator_->get(GCONF.cluster_id, tenant_id, + ls_id, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("get ls info from GCTX.lst_operator_ failed", K(arg), KR(ret), K(tenant_id)); } else if (OB_FAIL(update_ls_election_reference_info_table(arg, tenant_id, ls_info))) { LOG_WARN("fail to update ls election reference info", K(arg), KR(ret), K(tenant_id)); diff --git a/src/rootserver/ob_update_rs_list_task.cpp b/src/rootserver/ob_update_rs_list_task.cpp index cef73ddfc0..91cc857f05 100644 --- a/src/rootserver/ob_update_rs_list_task.cpp +++ b/src/rootserver/ob_update_rs_list_task.cpp @@ -233,7 +233,8 @@ int ObUpdateRsListTask::get_rs_list( int ret = OB_SUCCESS; ObLSInfo ls_info; const int64_t tenant_id = OB_SYS_TENANT_ID; - if (OB_FAIL(lst.get(GCONF.cluster_id, tenant_id, SYS_LS, ls_info))) { + if (OB_FAIL(lst.get(GCONF.cluster_id, tenant_id, + SYS_LS, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("lst_operator get failed", KR(ret), K(tenant_id), K(SYS_LS), K(ls_info)); } else { const ObLSReplica *leader_replica = NULL; diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index cf82072d93..0a03ca827b 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -25,7 +25,6 @@ #include "share/ls/ob_ls_status_operator.h" //ObLSStatusOperator #include "share/ls/ob_ls_operator.h"//ObLSAttr #include "share/backup/ob_backup_data_store.h"//ObBackupDataLSAttrDesc -#include "share/ls/ob_ls_table_iterator.h"//ObLSTableIterator #include "share/restore/ob_physical_restore_info.h"//ObPhysicalRestoreInfo #include "share/restore/ob_physical_restore_table_operator.h"//ObPhysicalRestoreTableOperator #include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo diff --git a/src/rootserver/virtual_table/ob_core_meta_table.cpp b/src/rootserver/virtual_table/ob_core_meta_table.cpp index 512dba57c8..0ba1f254fb 100644 --- a/src/rootserver/virtual_table/ob_core_meta_table.cpp +++ b/src/rootserver/virtual_table/ob_core_meta_table.cpp @@ -73,7 +73,7 @@ int ObCoreMetaTable::inner_get_next_row(ObNewRow *&row) ret = OB_TABLE_NOT_EXIST; LOG_WARN("get_table_schema failed", K(table_id), KR(ret)); } else if (OB_FAIL(lst_operator_->get(GCONF.cluster_id, OB_SYS_TENANT_ID, - SYS_LS, ls_info))) { + SYS_LS, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("lst_operator get failed", KR(ret), K(ls_info)); } else { ObArray columns; diff --git a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp index 323f830cb9..d74b000563 100644 --- a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp @@ -909,7 +909,7 @@ int ObInnerTableSchema::dba_ob_ls_locations_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT NOW(6) AS CREATE_TIME, NOW(6) AS MODIFY_TIME, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_CORE_META_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) UNION ALL ( SELECT GMT_CREATE AS CREATE_TIME, GMT_MODIFIED AS MODIFY_TIME, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT NOW(6) AS CREATE_TIME, NOW(6) AS MODIFY_TIME, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_CORE_META_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) UNION ALL ( SELECT GMT_CREATE AS CREATE_TIME, GMT_MODIFIED AS MODIFY_TIME, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE WHERE TENANT_ID = EFFECTIVE_TENANT_ID() AND TENANT_ID != 1 ) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -959,7 +959,7 @@ int ObInnerTableSchema::cdb_ob_ls_locations_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT NOW(6) AS CREATE_TIME, NOW(6) AS MODIFY_TIME, TENANT_ID, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_CORE_META_TABLE ) UNION ALL ( SELECT GMT_CREATE AS CREATE_TIME, GMT_MODIFIED AS MODIFY_TIME, TENANT_ID, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE ) )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( ( SELECT NOW(6) AS CREATE_TIME, NOW(6) AS MODIFY_TIME, TENANT_ID, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_CORE_META_TABLE ) UNION ALL ( SELECT GMT_CREATE AS CREATE_TIME, GMT_MODIFIED AS MODIFY_TIME, TENANT_ID, LS_ID, SVR_IP, SVR_PORT, SQL_PORT, ZONE, (CASE ROLE WHEN 1 THEN "LEADER" ELSE "FOLLOWER" END) AS ROLE, (CASE ROLE WHEN 1 THEN MEMBER_LIST ELSE NULL END) AS MEMBER_LIST, (CASE ROLE WHEN 1 THEN PAXOS_REPLICA_NUMBER ELSE NULL END) AS PAXOS_REPLICA_NUMBER, (CASE REPLICA_TYPE WHEN 0 THEN "FULL" WHEN 5 THEN "LOGONLY" WHEN 16 THEN "READONLY" WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE WHERE TENANT_ID != 1 ) )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 4207e7805e..d71ad5fbb7 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -14965,7 +14965,7 @@ def_table_schema( ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE WHERE - TENANT_ID = EFFECTIVE_TENANT_ID() + TENANT_ID = EFFECTIVE_TENANT_ID() AND TENANT_ID != 1 ) """.replace("\n", " "), ) @@ -15020,6 +15020,7 @@ def_table_schema( WHEN 261 THEN "ENCRYPTION LOGONLY" ELSE NULL END) AS REPLICA_TYPE FROM OCEANBASE.__ALL_VIRTUAL_LS_META_TABLE + WHERE TENANT_ID != 1 ) """.replace("\n", " "), ) diff --git a/src/share/location_cache/ob_ls_location_service.cpp b/src/share/location_cache/ob_ls_location_service.cpp index e989ae7067..454f52fe93 100644 --- a/src/share/location_cache/ob_ls_location_service.cpp +++ b/src/share/location_cache/ob_ls_location_service.cpp @@ -656,6 +656,7 @@ int ObLSLocationService::renew_all_ls_locations() ObArray tenant_ids; const bool can_erase = true; const int64_t renew_all_ls_loc_timeout = GCONF.location_cache_refresh_sql_timeout; + const bool inner_table_only = false; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("service not init", KR(ret)); @@ -678,7 +679,7 @@ int ObLSLocationService::renew_all_ls_locations() ctx, renew_all_ls_loc_timeout))) { LOG_WARN("fail to set default_timeout_ctx", KR(ret)); - } else if (OB_FAIL(lst_->get_by_tenant(tenant_id, ls_infos))) { + } else if (OB_FAIL(lst_->get_by_tenant(tenant_id, inner_table_only, ls_infos))) { LOG_WARN("fail to get all ls info", KR(ret), K(tenant_id), K(ls_infos)); } else { ARRAY_FOREACH_N(ls_infos, i, cnt) { @@ -798,7 +799,8 @@ int ObLSLocationService::renew_location( if (FAILEDx(ObShareUtil::set_default_timeout_ctx(ctx, default_timeout))) { LOG_WARN("fail to set default_timeout_ctx", KR(ret)); - } else if (OB_FAIL(lst_->get(cluster_id, tenant_id, ls_id, ls_info))) { + } else if (OB_FAIL(lst_->get(cluster_id, tenant_id, + ls_id, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("fail to get log stream info by operator", KR(ret), K(cluster_id), K(tenant_id), K(ls_id)); if (ObLocationServiceUtility::treat_sql_as_timeout(ret)) { diff --git a/src/share/ls/ob_inmemory_ls_table.cpp b/src/share/ls/ob_inmemory_ls_table.cpp index 162f7af252..2df7efaa83 100644 --- a/src/share/ls/ob_inmemory_ls_table.cpp +++ b/src/share/ls/ob_inmemory_ls_table.cpp @@ -59,6 +59,7 @@ int ObInMemoryLSTable::get( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const ObLSTable::Mode mode, ObLSInfo &ls_info) { int ret = OB_SUCCESS; @@ -67,11 +68,12 @@ int ObInMemoryLSTable::get( if (OB_UNLIKELY(!is_inited())) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (INVALID_CLUSTER_ID == cluster_id - || local_cluster_id != cluster_id) { + } else if (OB_UNLIKELY(INVALID_CLUSTER_ID == cluster_id + || local_cluster_id != cluster_id + || ObLSTable::INNER_TABLE_ONLY_MODE == mode)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get through inmemory with invalid cluster_id or not local cluster_id", - KR(ret), K(cluster_id), K(local_cluster_id)); + LOG_WARN("get through inmemory with invalid cluster_id or not local cluster_id or invalid mode", + KR(ret), K(cluster_id), K(local_cluster_id), K(mode)); } else if (!is_sys_tenant(tenant_id) || !ls_id.is_sys_ls()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument for sys tenant's sys ls", KR(ret), KT(tenant_id), K(ls_id)); @@ -106,7 +108,9 @@ int ObInMemoryLSTable::inner_get_( return ret; } -int ObInMemoryLSTable::update(const ObLSReplica &replica) +int ObInMemoryLSTable::update( + const ObLSReplica &replica, + const bool inner_table_only) { int ret = OB_SUCCESS; WLockGuard lock_guard(lock_); @@ -115,10 +119,11 @@ int ObInMemoryLSTable::update(const ObLSReplica &replica) LOG_WARN("not init", KR(ret)); } else if (!replica.is_valid() || !is_sys_tenant(replica.get_tenant_id()) - || !replica.get_ls_id().is_sys_ls()) { + || !replica.get_ls_id().is_sys_ls() + || inner_table_only) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(replica), "tenant_id", replica.get_tenant_id(), - "ls_id", replica.get_ls_id().id()); + "ls_id", replica.get_ls_id().id(), K(inner_table_only)); } else if (replica.is_strong_leader()) { // TODO: try to short the logic of update_leader/follower_replica if (OB_FAIL(update_leader_replica_(replica))) { @@ -233,7 +238,8 @@ int ObInMemoryLSTable::update_leader_replica_(const ObLSReplica &replica) int ObInMemoryLSTable::remove( const uint64_t tenant_id, const ObLSID &ls_id, - const ObAddr &server) + const ObAddr &server, + const bool inner_table_only) { int ret = OB_SUCCESS; WLockGuard lock_guard(lock_); @@ -242,9 +248,11 @@ int ObInMemoryLSTable::remove( LOG_WARN("not init", KR(ret)); } else if (OB_UNLIKELY(!is_sys_tenant(tenant_id) || !ls_id.is_sys_ls() - || !server.is_valid())) { + || !server.is_valid()) + || inner_table_only) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("remove with invalid argument", KR(ret), K(tenant_id), K(ls_id), K(server)); + LOG_WARN("remove with invalid argument", KR(ret), + K(tenant_id), K(ls_id), K(server), K(inner_table_only)); } else if (OB_FAIL(ls_info_.remove(server))) { LOG_WARN("remove server replica failed", KR(ret), K(server), K_(ls_info)); } diff --git a/src/share/ls/ob_inmemory_ls_table.h b/src/share/ls/ob_inmemory_ls_table.h index 10a6759d73..64d4b1a774 100644 --- a/src/share/ls/ob_inmemory_ls_table.h +++ b/src/share/ls/ob_inmemory_ls_table.h @@ -47,26 +47,31 @@ public: // @param [in] cluster_id, belong to which cluste // @parma [in] tenant_id, get whose ls info // @param [in] ls_id, get which ls info + // @param [in] mode, should not be ObLSTable::INNER_TABLE_ONLY_MODE // @param [out] ls_info, informations about a certain ls // TODO: enable cluster_id virtual int get( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const ObLSTable::Mode mode, ObLSInfo &ls_info) override; // update a certain log stream's replica info to meta table // @param [in] replica, the new replica infos to update - virtual int update(const ObLSReplica &replica) override; + // @param [in] inner_table_only, should be false. + virtual int update(const ObLSReplica &replica, const bool inner_table_only) override; // remove ls replica // @param [in] tenant_id, which tenant's log stream // @param [in] ls_id, identifier for log stream // @param [in] server, where is this replica + // @param [in] inner_table_only, should be false. virtual int remove( const uint64_t tenant_id, const ObLSID &ls_id, - const ObAddr &server) override; + const ObAddr &server, + const bool inner_table_only) override; private: // the real action to read LSinfo directly from memory diff --git a/src/share/ls/ob_ls_info.cpp b/src/share/ls/ob_ls_info.cpp index f447a8c318..ccdebbd96c 100644 --- a/src/share/ls/ob_ls_info.cpp +++ b/src/share/ls/ob_ls_info.cpp @@ -608,6 +608,7 @@ int ObLSInfo::assign(const ObLSInfo &other) { int ret = OB_SUCCESS; if (this != &other) { + reset(); tenant_id_ = other.get_tenant_id(); ls_id_ = other.get_ls_id(); if (OB_FAIL(copy_assign(replicas_, other.replicas_))) { @@ -617,6 +618,42 @@ int ObLSInfo::assign(const ObLSInfo &other) return ret; } +int ObLSInfo::composite_with(const ObLSInfo &other) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY( + tenant_id_ != other.get_tenant_id() + || ls_id_ != other.get_ls_id())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_id or ls_id not matched", KR(ret), KPC(this), K(other)); + } else { + ObLSReplica tmp_replica; + int64_t idx = OB_INVALID_INDEX; // not used + int tmp_ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < other.get_replicas().count(); i++) { + const ObLSReplica &ls_replica = other.get_replicas().at(i); + tmp_ret = find_idx_(ls_replica, idx); + if (OB_ENTRY_NOT_EXIST != tmp_ret) { + // ls replica exist or warn, do nothing + ret = tmp_ret; + } else { + tmp_replica.reset(); + if (OB_FAIL(tmp_replica.assign(ls_replica))) { + LOG_WARN("fail to assign replica", KR(ret), K(ls_replica)); + } else if (FALSE_IT(tmp_replica.update_to_follower_role())) { + } else if (OB_FAIL(add_replica(tmp_replica))) { + LOG_WARN("fail to add replica", KR(ret), K(tmp_replica)); + } + } + } // end for + + if (FAILEDx(update_replica_status())) { + LOG_WARN("fail to update replica status", KR(ret), KPC(this)); + } + } + return ret; +} + // TODO: make sure the actions of this function int ObLSInfo::update_replica_status() { diff --git a/src/share/ls/ob_ls_info.h b/src/share/ls/ob_ls_info.h index cf1c13027d..28c05112fe 100644 --- a/src/share/ls/ob_ls_info.h +++ b/src/share/ls/ob_ls_info.h @@ -217,6 +217,8 @@ public: TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(replicas)); // operator-related functions int assign(const ObLSInfo &other); + // composite with other ls to add missing follower replica + int composite_with(const ObLSInfo &other); // other functions virtual int add_replica(const ObLSReplica &replica); int update_replica_status(); // TODO: have to make sure actions in this function diff --git a/src/share/ls/ob_ls_table.h b/src/share/ls/ob_ls_table.h index bd31247caf..f531f0ec11 100644 --- a/src/share/ls/ob_ls_table.h +++ b/src/share/ls/ob_ls_table.h @@ -33,6 +33,14 @@ class ObLSInfo; // [class_attention] None class ObLSTable { +public: + // Mode is used to control data source of sys tenant's ls info. + // Other tenant's ls infos always come from inner table. + enum Mode { + DEFAULT_MODE = 0, // sys tenant's ls info come from rs + INNER_TABLE_ONLY_MODE = 1, // sys tenant's ls info come from inner table + COMPOSITE_MODE = 2 // sys tenant's ls info come from both rs and inner table + }; public: explicit ObLSTable(); virtual ~ObLSTable(); @@ -65,26 +73,31 @@ public: // @param [in] cluster_id, belong to which cluster // @parma [in] tenant_id, get whose ls info // @param [in] ls_id, get which ls info + // @param [in] mode, determine data source of sys tenant's ls info // @param [out] ls_info, informations about a certain ls // TODO: enable cluster_id virtual int get( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const ObLSTable::Mode mode, ObLSInfo &ls_info) = 0; // base function to report a new replica // @param [in] replica, new replica informations to update - virtual int update(const ObLSReplica &replica) = 0; + // @param [in] inner_table_only, determine whether the sys tenant's ls info is recorded in inner table or in memory. + virtual int update(const ObLSReplica &replica, const bool inner_table_only) = 0; // remove ls replica from __all_ls_meta_table // // @param [in] tenant_id, the tenant which the ls belongs to // @param [in] ls_id, the ls which you want to remove // @param [in] server, address of the ls replica + // @param [in] inner_table_only, determine whether the sys tenant's ls info is recorded in inner table or in memory. virtual int remove( const uint64_t tenant_id, const ObLSID &ls_id, - const ObAddr &server) = 0; + const ObAddr &server, + const bool inner_table_only) = 0; // check whether have to update a replica // @param [in] lhs, the old log stream replica diff --git a/src/share/ls/ob_ls_table_iterator.cpp b/src/share/ls/ob_ls_table_iterator.cpp index 9909027ac2..eda1bae3ec 100644 --- a/src/share/ls/ob_ls_table_iterator.cpp +++ b/src/share/ls/ob_ls_table_iterator.cpp @@ -26,11 +26,15 @@ ObLSTableIterator::ObLSTableIterator() inner_idx_(0), lst_operator_(NULL), inner_ls_infos_(), - filters_() + filters_(), + inner_table_only_(false) { } -int ObLSTableIterator::init(ObLSTableOperator &lst_operator, const uint64_t tenant_id) +int ObLSTableIterator::init( + ObLSTableOperator &lst_operator, + const uint64_t tenant_id, + const share::ObLSTable::Mode mode) { int ret = OB_SUCCESS; if (OB_UNLIKELY(inited_)) { @@ -39,9 +43,15 @@ int ObLSTableIterator::init(ObLSTableOperator &lst_operator, const uint64_t tena } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY( + share::ObLSTable::DEFAULT_MODE != mode + && share::ObLSTable::INNER_TABLE_ONLY_MODE != mode)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid mode", KR(ret), K(mode)); } else { lst_operator_ = &lst_operator; tenant_id_ = tenant_id; + inner_table_only_ = (share::ObLSTable::INNER_TABLE_ONLY_MODE == mode); inited_ = true; } return ret; @@ -88,8 +98,10 @@ int ObLSTableIterator::inner_open_() ret = OB_ERR_UNEXPECTED; LOG_WARN("inner_idx_ should be equal to the count of inner array", KR(ret), K_(inner_idx), "inner_ls_infos count", inner_ls_infos_.count()); - } else if (OB_FAIL(lst_operator_->get_by_tenant(tenant_id_, inner_ls_infos_))) { - LOG_WARN("fail to get ls infos by tenant", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(lst_operator_->get_by_tenant( + tenant_id_, inner_table_only_, inner_ls_infos_))) { + LOG_WARN("fail to get ls infos by tenant", + KR(ret), K_(tenant_id), K_(inner_table_only)); } return ret; } diff --git a/src/share/ls/ob_ls_table_iterator.h b/src/share/ls/ob_ls_table_iterator.h index 9c1137f103..8ed493d28b 100644 --- a/src/share/ls/ob_ls_table_iterator.h +++ b/src/share/ls/ob_ls_table_iterator.h @@ -16,6 +16,7 @@ #include "lib/container/ob_array.h" // ObArray #include "share/ls/ob_ls_replica_filter.h" // ObLSReplicaFilterHolder #include "share/schema/ob_table_iter.h"//ObTenantIterator +#include "share/ls/ob_ls_table.h" // ObLSTable::Mode namespace oceanbase { @@ -29,12 +30,15 @@ class ObLSTableOperator; class ObLSInfo; // This class is used to iterate __all_ls_meta_table for target tenant. +// Used by ObTenantMetaChecker class ObLSTableIterator { public: ObLSTableIterator(); virtual ~ObLSTableIterator() {} - int init(ObLSTableOperator &lst_operator, const uint64_t tenant_id); + int init(ObLSTableOperator &lst_operator, + const uint64_t tenant_id, + const share::ObLSTable::Mode mode); int next(ObLSInfo &ls_info); ObLSReplicaFilterHolder &get_filters() { return filters_; } private: @@ -46,9 +50,11 @@ private: ObLSTableOperator *lst_operator_; common::ObArray inner_ls_infos_; ObLSReplicaFilterHolder filters_; + bool inner_table_only_; }; -//get all ls of each tenant's __all_ls_meta_table +// get all ls of each tenant's __all_ls_meta_table +// used by ObEmptyServerChecker class ObAllLSTableIterator { public: @@ -72,7 +78,8 @@ private: DISALLOW_COPY_AND_ASSIGN(ObAllLSTableIterator); }; -//get all ls of tenant's __all_ls_meta_table +// get all ls of tenant's __all_ls_meta_table +// used by ObLostReplicaChecker class ObTenantLSTableIterator { public: diff --git a/src/share/ls/ob_ls_table_operator.cpp b/src/share/ls/ob_ls_table_operator.cpp index 578039cd6e..7c787cf607 100644 --- a/src/share/ls/ob_ls_table_operator.cpp +++ b/src/share/ls/ob_ls_table_operator.cpp @@ -118,7 +118,9 @@ int ObLSTableOperator::set_callback_for_obs( return ret; } -int ObLSTableOperator::update(const ObLSReplica &replica) +int ObLSTableOperator::update( + const ObLSReplica &replica, + const bool inner_table_only) { int ret = OB_SUCCESS; const int64_t begin = ObTimeUtility::fast_current_time(); @@ -129,16 +131,18 @@ int ObLSTableOperator::update(const ObLSReplica &replica) } else if (!replica.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(replica)); - } else if (OB_FAIL(get_ls_table_(GCONF.cluster_id, replica.get_tenant_id(), replica.get_ls_id(), lst))) { - LOG_WARN("get ls table failed", KR(ret), K(replica)); + } else if (OB_FAIL(get_ls_table_(GCONF.cluster_id, replica.get_tenant_id(), + replica.get_ls_id(), inner_table_only, lst))) { + LOG_WARN("get ls table failed", KR(ret), K(replica), K(inner_table_only)); } else if (OB_ISNULL(lst)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL ls table", KR(ret)); - } else if (OB_FAIL(lst->update(replica))) { - LOG_WARN("update replica failed", KR(ret), K(replica)); + } else if (OB_FAIL(lst->update(replica, inner_table_only))) { + LOG_WARN("update replica failed", KR(ret), K(replica), K(inner_table_only)); } - LOG_INFO("update ls replica", - KR(ret), "time_used", ObTimeUtility::fast_current_time() - begin, K(replica)); + LOG_INFO("update ls replica", KR(ret), + "time_used", ObTimeUtility::fast_current_time() - begin, + K(replica), K(inner_table_only)); return ret; } @@ -146,19 +150,22 @@ int ObLSTableOperator::get( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const ObLSTable::Mode mode, ObLSInfo &ls_info) { int ret = OB_SUCCESS; int64_t start_time = ObTimeUtility::fast_current_time(); ObLSTable *lst = NULL; + // For sys tenant in COMPOSITE_MODE, it will fetch ls from memory first. + const bool inner_table_only = (ObLSTable::INNER_TABLE_ONLY_MODE == mode); if (OB_UNLIKELY(!is_inited())) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); } else if (!is_valid_key(tenant_id, ls_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), KT(tenant_id), K(ls_id)); - } else if (OB_FAIL(get_ls_table_(cluster_id, tenant_id, ls_id, lst))) { - LOG_WARN("get ls table failed", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(get_ls_table_(cluster_id, tenant_id, ls_id, inner_table_only, lst))) { + LOG_WARN("get ls table failed", KR(ret), K(tenant_id), K(ls_id), K(mode)); } else if (OB_ISNULL(lst)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL ls table", KR(ret)); @@ -167,8 +174,21 @@ int ObLSTableOperator::get( ObTimeoutCtx ctx; if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); - } else if (OB_FAIL(lst->get(cluster_id, tenant_id, ls_id, ls_info))) { - LOG_WARN("get log stream info failed", KR(ret), KT(tenant_id), K(ls_id)); + } else if (OB_FAIL(lst->get(cluster_id, tenant_id, ls_id, mode, ls_info))) { + LOG_WARN("get log stream info failed", KR(ret), KT(tenant_id), K(ls_id), K(mode)); + } + } + if (OB_SUCC(ret) + && is_sys_tenant(tenant_id) + && ObLSTable::COMPOSITE_MODE == mode) { + ObLSInfo tmp_ls_info; + ObTimeoutCtx ctx; + if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { + LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); + } else if (OB_FAIL(persistent_ls_.get(cluster_id, tenant_id, ls_id, mode, tmp_ls_info))) { + LOG_WARN("get log stream info failed", KR(ret), KT(tenant_id), K(ls_id), K(mode)); + } else if (OB_FAIL(ls_info.composite_with(tmp_ls_info))) { + LOG_WARN("fail to composit with ls", KR(ret), K(tenant_id), K(ls_id), K(ls_info), K(tmp_ls_info)); } } if (OB_SUCC(ret)) { @@ -183,6 +203,7 @@ int ObLSTableOperator::get_ls_table_( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const bool inner_table_only, ObLSTable *&ls_table) { int ret = OB_SUCCESS; @@ -192,7 +213,7 @@ int ObLSTableOperator::get_ls_table_( } else if (!is_valid_key(tenant_id, ls_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id)); - } else if (is_sys_tenant(tenant_id)) { + } else if (!inner_table_only && is_sys_tenant(tenant_id)) { if (GCONF.cluster_id == cluster_id) { ls_table = root_ls_; } else { @@ -208,6 +229,7 @@ int ObLSTableOperator::get_ls_table_( int ObLSTableOperator::get_by_tenant( const uint64_t tenant_id, + const bool inner_table_only, ObIArray &ls_infos) { int ret = OB_SUCCESS; @@ -215,14 +237,17 @@ int ObLSTableOperator::get_by_tenant( if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (is_sys_tenant(tenant_id)) { + } else if (!inner_table_only && is_sys_tenant(tenant_id)) { ObLSInfo ls_info; - if (OB_FAIL(root_ls_->get(GCONF.cluster_id, OB_SYS_TENANT_ID, SYS_LS, ls_info))) { + if (OB_FAIL(root_ls_->get(GCONF.cluster_id, OB_SYS_TENANT_ID, + SYS_LS, share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("fail to get sys_tenant ls", KR(ret), K(tenant_id)); } else if (OB_FAIL(ls_infos.push_back(ls_info))) { LOG_WARN("fail to assign", KR(ret), K(ls_info)); } - } else if (is_meta_tenant(tenant_id) || is_user_tenant(tenant_id)) { + } else if (is_sys_tenant(tenant_id) + || is_meta_tenant(tenant_id) + || is_user_tenant(tenant_id)) { if (OB_FAIL(persistent_ls_.get_by_tenant(tenant_id, ls_infos))) { LOG_WARN("get all ls info by persistent_ls_ failed", KR(ret), K(tenant_id)); } @@ -245,19 +270,40 @@ int ObLSTableOperator::load_all_ls_in_tenant( } else if (OB_UNLIKELY(is_user_tenant(exec_tenant_id))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("can not be user tenant", KR(ret), K(exec_tenant_id)); - } - - if (FAILEDx(persistent_ls_.load_all_ls_in_tenant(exec_tenant_id, ls_infos))) { + } else if (OB_FAIL(persistent_ls_.load_all_ls_in_tenant(exec_tenant_id, ls_infos))) { LOG_WARN("load tenant ls info failed", KR(ret), K(exec_tenant_id)); } else if (is_sys_tenant(exec_tenant_id)) { - ObLSInfo ls_info; - if (OB_ISNULL(root_ls_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("root ls is null", KR(ret)); - } else if (OB_FAIL(root_ls_->get(GCONF.cluster_id, OB_SYS_TENANT_ID, SYS_LS, ls_info))) { - LOG_WARN("fail to get sys_tenant ls", KR(ret), K(exec_tenant_id)); - } else if (OB_FAIL(ls_infos.push_back(ls_info))) { - LOG_WARN("fail to assign", KR(ret), K(ls_info)); + ObLSInfo sys_ls_info; + int64_t idx = OB_INVALID_INDEX; // sys ls position + for (int64_t i = 0; OB_INVALID_INDEX == idx && OB_SUCC(ret) && i < ls_infos.count(); i++) { + const ObLSInfo &ls = ls_infos.at(i); + if (is_sys_tenant(ls.get_tenant_id()) && SYS_LS == ls.get_ls_id()) { + if (OB_FAIL(sys_ls_info.assign(ls))) { + LOG_WARN("fail to assign sys ls", KR(ret), K(ls)); + } else { + idx = i; + } + } + } // end for + + if (OB_SUCC(ret)) { + ObLSInfo ls_info; + if (OB_ISNULL(root_ls_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("root ls is null", KR(ret)); + } else if (OB_FAIL(root_ls_->get(GCONF.cluster_id, OB_SYS_TENANT_ID, + SYS_LS, share::ObLSTable::DEFAULT_MODE, ls_info))) { + LOG_WARN("fail to get sys_tenant ls", KR(ret), K(exec_tenant_id)); + } else if (OB_INVALID_INDEX != idx) { + if (OB_FAIL(ls_infos.remove(idx))) { + LOG_WARN("fail to remove ls from array", KR(ret), K(idx)); + } else if (OB_FAIL(ls_info.composite_with(sys_ls_info))) { + LOG_WARN("composite ls failed", KR(ret), K(ls_info), K(sys_ls_info)); + } + } + if (FAILEDx(ls_infos.push_back(ls_info))) { + LOG_WARN("fail to assign", KR(ret), K(ls_info)); + } } } return ret; @@ -266,7 +312,8 @@ int ObLSTableOperator::load_all_ls_in_tenant( int ObLSTableOperator::remove( const uint64_t tenant_id, const ObLSID &ls_id, - const ObAddr &server) + const ObAddr &server, + const bool inner_table_only) { int ret = OB_SUCCESS; ObLSTable *lst = NULL; @@ -276,8 +323,9 @@ int ObLSTableOperator::remove( } else if (!ls_id.is_valid_with_tenant(tenant_id) || !server.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(server)); - } else if (OB_FAIL(get_ls_table_(GCONF.cluster_id, tenant_id, ls_id, lst))) { - LOG_WARN("get ls table failed", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_FAIL(get_ls_table_(GCONF.cluster_id, tenant_id, + ls_id, inner_table_only, lst))) { + LOG_WARN("get ls table failed", KR(ret), K(tenant_id), K(ls_id), K(inner_table_only)); } else if (OB_ISNULL(lst)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL ls table", KR(ret), K(tenant_id), K(ls_id), K(server)); @@ -285,11 +333,12 @@ int ObLSTableOperator::remove( ObTimeoutCtx ctx; if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); - } else if (OB_FAIL(lst->remove(tenant_id, ls_id, server))) { - LOG_WARN("remove ls failed", KR(ret), K(tenant_id), K(ls_id), K(server)); + } else if (OB_FAIL(lst->remove(tenant_id, ls_id, server, inner_table_only))) { + LOG_WARN("remove ls failed", KR(ret), K(tenant_id), + K(ls_id), K(server), K(inner_table_only)); } else { LOG_INFO("success to remove ls by operator", - KR(ret), K(tenant_id), K(ls_id), K(server)); + KR(ret), K(tenant_id), K(ls_id), K(server), K(inner_table_only)); } } return ret; diff --git a/src/share/ls/ob_ls_table_operator.h b/src/share/ls/ob_ls_table_operator.h index 66cd2a42d0..a140e5c194 100644 --- a/src/share/ls/ob_ls_table_operator.h +++ b/src/share/ls/ob_ls_table_operator.h @@ -66,40 +66,51 @@ public: // @param [in] cluster_id, belong to which cluster // @parma [in] tenant_id, get whose ls info // @param [in] ls_id, get which ls info + // @param [in] mode, determine data source of sys tenant's ls info // @param [out] ls_info, informations about a certain ls // TODO: make sure how to enable cluster_id virtual int get( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const ObLSTable::Mode mode, ObLSInfo &ls_info) override; // operator for report - update a certain log stream's info // @param [in] replica, new replica informations to update - virtual int update(const ObLSReplica &replica) override; - // get all ls info in __all_ls_meta_table according to tenant_id - // - // @param [in] tenant_id, target tenant - // @param [out] ls_infos, all ls infos in __all_ls_meta_table - int get_by_tenant(const uint64_t tenant_id, ObIArray &ls_infos); - - // load all ls info in __all_ls_meta_table of the tenant_id - // - // @param [in] tenant_id, target tenant, only sys and meta tenant - // @param [out] ls_infos, all ls infos in __all_ls_meta_table or __all_virtual_core_meta_table - int load_all_ls_in_tenant(const uint64_t exec_tenant_id, ObIArray &ls_infos); + // @param [in] inner_table_only, determine whether the sys tenant's ls info is recorded in inner table or in memory. + virtual int update(const ObLSReplica &replica, const bool inner_table_only) override; // remove ls replica from __all_ls_meta_table // // @param [in] tenant_id, the tenant which the ls belongs to // @param [in] ls_id, the ls which you want to remove // @param [in] server, address of the ls replica + // @param [in] inner_table_only, determine whether the sys tenant's ls info is recorded in inner table or in memory. virtual int remove( const uint64_t tenant_id, const ObLSID &ls_id, - const ObAddr &server) override; - // remove residual ls in __all_ls_meta_table for ObServerMetaTableChecker + const ObAddr &server, + const bool inner_table_only) override; + + // get all ls info in __all_ls_meta_table according to tenant_id + // used for ObLSTableIterator in ObTenantMetaChecker + // @param [in] tenant_id, target tenant + // @param [out] ls_infos, all ls infos in __all_ls_meta_table + // @param [in] inner_table_only, determine whether the sys tenant's ls info is recorded in inner table or in memory. + int get_by_tenant(const uint64_t tenant_id, const bool inner_table_only, ObIArray &ls_infos); + + // load all ls info in __all_ls_meta_table of the tenant_id // + // used by ObAllLSTableIterator/ObTenantLSTableIterator in ObEmptyServerChecker/ObLostReplicaChecker + // will use share::ObLSTable::COMPOSITE_MODE to fetch sys tenant's ls + // + // @param [in] tenant_id, target tenant, only sys and meta tenant + // @param [out] ls_infos, all ls infos in __all_ls_meta_table or __all_virtual_core_meta_table + int load_all_ls_in_tenant(const uint64_t exec_tenant_id, ObIArray &ls_infos); + + // remove residual ls in __all_ls_meta_table for ObServerMetaTableChecker + // won't deal with sys tenant // @param [in] tenant_id, tenant for query // @param [in] server, target ObAddr // @param [out] residual_count, count of residual ls in table @@ -125,11 +136,13 @@ private: // decide which ls_table to use according to ls_id: inmemory/rpc/persistent // @param [in] tenant_id, get whose ls info // @param [in] ls_id, get which ls info + // @param [in] inner_table_only, determine whether the sys tenant's ls info is recorded in inner table or in memory. // @param [out] ls_table, decide inmemory/rpc/persistent ls_table to use int get_ls_table_( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const bool inner_table_only, ObLSTable *&ls_table); private: diff --git a/src/share/ls/ob_persistent_ls_table.cpp b/src/share/ls/ob_persistent_ls_table.cpp index a34c0955ed..b8dacac880 100644 --- a/src/share/ls/ob_persistent_ls_table.cpp +++ b/src/share/ls/ob_persistent_ls_table.cpp @@ -95,11 +95,13 @@ int ObPersistentLSTable::get( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const ObLSTable::Mode mode, ObLSInfo &ls_info) { int ret = OB_SUCCESS; const bool filter_flag_replica = true; const bool lock_replica = false; + UNUSED(mode); if (OB_UNLIKELY(!is_inited())) { ret = OB_NOT_INIT; LOG_WARN("ObPersistentLSTable not init", KR(ret)); @@ -315,9 +317,12 @@ int ObPersistentLSTable::construct_ls_info( return ret; } -int ObPersistentLSTable::update(const ObLSReplica &replica) +int ObPersistentLSTable::update( + const ObLSReplica &replica, + const bool inner_table_only) { int ret = OB_SUCCESS; + UNUSED(inner_table_only); DEBUG_SYNC(BEFORE_UPDATE_LS_META_TABLE); const bool lock_replica = replica.is_strong_leader(); AutoTransProxy proxy(*sql_proxy_); @@ -715,9 +720,11 @@ int ObPersistentLSTable::construct_ls_infos_( int ObPersistentLSTable::remove( const uint64_t tenant_id, const ObLSID &ls_id, - const ObAddr &server) + const ObAddr &server, + const bool inner_table_only) { int ret = OB_SUCCESS; + UNUSED(inner_table_only); char ip[OB_MAX_SERVER_ADDR_SIZE] = ""; ObSqlString sql; const uint64_t sql_tenant_id = get_private_table_exec_tenant_id(tenant_id); diff --git a/src/share/ls/ob_persistent_ls_table.h b/src/share/ls/ob_persistent_ls_table.h index da48c4144e..debf7a186e 100644 --- a/src/share/ls/ob_persistent_ls_table.h +++ b/src/share/ls/ob_persistent_ls_table.h @@ -72,17 +72,20 @@ public: // @param [in] cluster_id, belong to which cluster // @parma [in] tenant_id, get whose ls info // @param [in] ls_id, get which ls info + // @param [in] mode, useless // @param [out] ls_info, informations about this ls // TODO: enable cluster_id virtual int get( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const ObLSTable::Mode mode, ObLSInfo &ls_info) override; // update new replica infos to meta table // @param [in] replica, new replica infos to update - virtual int update(const ObLSReplica &replica) override; + // @param [in] inner_table_only, useless + virtual int update(const ObLSReplica &replica, const bool inner_table_only) override; // get all ls info from __all_ls_meta_table according to tenant_id // @@ -95,10 +98,13 @@ public: // @param [out] ls_infos, all ls infos in __all_ls_meta_table int load_all_ls_in_tenant(const uint64_t exec_tenant_id, ObIArray &ls_infos); + // @param [in] inner_table_only, useless virtual int remove( const uint64_t tenant_id, const ObLSID &ls_id, - const ObAddr &server) override; + const ObAddr &server, + const bool inner_table_only) override; + // remove residual ls in __all_tablet_meta_table for ObServerMetaTableChecker // // @param [in] tenant_id, tenant for query diff --git a/src/share/ls/ob_rpc_ls_table.cpp b/src/share/ls/ob_rpc_ls_table.cpp index acb7b0e651..9f06f4c5d9 100644 --- a/src/share/ls/ob_rpc_ls_table.cpp +++ b/src/share/ls/ob_rpc_ls_table.cpp @@ -82,6 +82,7 @@ int ObRpcLSTable::get( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const ObLSTable::Mode mode, ObLSInfo &ls_info) { int ret = OB_SUCCESS; @@ -90,9 +91,12 @@ int ObRpcLSTable::get( LOG_WARN("fail to init ls info", KR(ret), K(tenant_id), K(ls_id)); } else if (OB_FAIL(check_inner_stat_())) { LOG_WARN("fail to check inner stat", KR(ret)); - } else if (!is_sys_tenant(tenant_id) || !ls_id.is_sys_ls()) { + } else if (OB_UNLIKELY(!is_sys_tenant(tenant_id) + || !ls_id.is_sys_ls() + || ObLSTable::INNER_TABLE_ONLY_MODE == mode)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument for sys tenant's sys ls", KR(ret), KT(tenant_id), K(ls_id)); + LOG_WARN("invalid argument for sys tenant's sys ls", + KR(ret), KT(tenant_id), K(ls_id), K(mode)); } else if (local_cluster_id == cluster_id) { if (OB_FAIL(get_ls_info_(ls_info))) { LOG_WARN("failed to get ls info", KR(ret)); @@ -160,7 +164,9 @@ int ObRpcLSTable::get_ls_info_across_cluster_(const int64_t cluster_id, ObLSInfo return ret; } -int ObRpcLSTable::update(const ObLSReplica &replica) +int ObRpcLSTable::update( + const ObLSReplica &replica, + const bool inner_table_only) { int ret = OB_SUCCESS; ObAddr rs_addr; @@ -168,9 +174,10 @@ int ObRpcLSTable::update(const ObLSReplica &replica) LOG_WARN("fail to check inner stat", KR(ret)); } else if (!replica.is_valid() || !is_sys_tenant(replica.get_tenant_id()) - || !replica.get_ls_id().is_sys_ls()) { + || !replica.get_ls_id().is_sys_ls() + || inner_table_only) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(replica)); + LOG_WARN("invalid argument", KR(ret), K(replica), K(inner_table_only)); } else if(OB_FAIL(rs_mgr_->get_master_root_server(rs_addr))) { LOG_WARN("get master root service failed", KR(ret)); } else { @@ -367,7 +374,8 @@ int ObRpcLSTable::deal_with_result_ls_( int ObRpcLSTable::remove( const uint64_t tenant_id, const ObLSID &ls_id, - const ObAddr &server) + const ObAddr &server, + const bool inner_table_only) { int ret = OB_SUCCESS; ObAddr rs_addr; @@ -375,14 +383,16 @@ int ObRpcLSTable::remove( LOG_WARN("fail to check inner stat", KR(ret)); } else if (OB_UNLIKELY(!is_sys_tenant(tenant_id) || !ls_id.is_sys_ls() - || !server.is_valid())) { + || !server.is_valid()) + || inner_table_only) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(server)); + LOG_WARN("invalid argument", KR(ret), K(tenant_id), + K(ls_id), K(server), K(inner_table_only)); } else if(OB_FAIL(rs_mgr_->get_master_root_server(rs_addr))) { LOG_WARN("get master root service failed", KR(ret)); } else { int64_t timeout_us = 0; - obrpc::ObRemoveSysLsArg arg(server); + obrpc::ObRemoveSysLsArg arg(server); if (OB_FAIL(get_timeout_(timeout_us))) { LOG_WARN("get timeout failed", KR(ret)); } else if (timeout_us <= 0) { diff --git a/src/share/ls/ob_rpc_ls_table.h b/src/share/ls/ob_rpc_ls_table.h index ef2a3ddf23..0ec32a1406 100644 --- a/src/share/ls/ob_rpc_ls_table.h +++ b/src/share/ls/ob_rpc_ls_table.h @@ -53,21 +53,27 @@ public: // @param [in] cluster_id, belong to which cluster // @parma [in] tenant_id, get whose ls info // @param [in] ls_id, get which ls info + // @param [in] mode, should not be ObLSTable::INNER_TABLE_ONLY_MODE // @param [out] ls_info, informations about a certain ls // TODO: enable cluster_id virtual int get( const int64_t cluster_id, const uint64_t tenant_id, const ObLSID &ls_id, + const ObLSTable::Mode mode, ObLSInfo &ls_info) override; // update new log stream replica to meta table // @param [in] replica, new informations to update - virtual int update(const ObLSReplica &replica) override; + // @param [in] inner_table_only, shoud be false + virtual int update(const ObLSReplica &replica, const bool inner_table_only) override; + + // @param [in] inner_table_only, shoud be false virtual int remove( const uint64_t tenant_id, const ObLSID &ls_id, - const ObAddr &server) override; + const ObAddr &server, + const bool inner_table_only) override; private: int check_inner_stat_() const; diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index bad843a456..03dac37666 100644 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -412,6 +412,7 @@ class ObString; ACT(BEFORE_DELETE_DRTASK_FROM_INNER_TABLE,)\ ACT(BEFORE_FINISH_LOCALITY,)\ ACT(BEFORE_TRY_MIGRATE_UNIT,)\ + ACT(BEFORE_TRY_DISASTER_RECOVERY,)\ ACT(BEFORE_TRY_LOCALITY_ALIGNMENT,)\ ACT(BEFORE_TABLET_MIGRATION_GENERATE_NEXT_DAG,)\ ACT(BEFORE_TABLET_GROUP_MIGRATION_GENERATE_NEXT_DAG,)\ diff --git a/src/share/ob_leader_election_waiter.cpp b/src/share/ob_leader_election_waiter.cpp index 6c0f3a2c65..15a6f9bdd9 100644 --- a/src/share/ob_leader_election_waiter.cpp +++ b/src/share/ob_leader_election_waiter.cpp @@ -172,7 +172,8 @@ int ObLSLeaderElectionWaiter::wait_elect_leader( int64_t sleep_interval = std::max(1l, check_interval / 100); while (!stop_) { const int64_t cluster_id = GCONF.cluster_id; - if (OB_FAIL(lst_operator_.get(cluster_id, tenant_id, ls_id, ls_info))) { + if (OB_FAIL(lst_operator_.get(cluster_id, tenant_id, + ls_id, share::ObLSTable::DEFAULT_MODE,ls_info))) { LOG_WARN("get partition info failed", K(tenant_id), K(ls_id), KR(ret)); } else if (OB_FAIL(ls_info.find_leader(leader_replica))) { // failure is normal, since leader may have not taked over diff --git a/src/share/ob_rs_mgr.cpp b/src/share/ob_rs_mgr.cpp index cbbcb6cdaf..2f2ecfccb6 100644 --- a/src/share/ob_rs_mgr.cpp +++ b/src/share/ob_rs_mgr.cpp @@ -334,6 +334,7 @@ int ObRsMgr::renew_master_rootserver(const int64_t cluster_id) } else if (OB_FAIL(GCTX.lst_operator_->get(cluster_id, OB_SYS_TENANT_ID, SYS_LS, + share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("get root log stream failed", KR(ret), K(cluster_id), diff --git a/src/storage/high_availability/ob_ls_migration_handler.cpp b/src/storage/high_availability/ob_ls_migration_handler.cpp index bc96fd9429..76a61a5762 100644 --- a/src/storage/high_availability/ob_ls_migration_handler.cpp +++ b/src/storage/high_availability/ob_ls_migration_handler.cpp @@ -1147,7 +1147,8 @@ int ObLSMigrationHandler::get_ls_info_( } else if (nullptr == lst_operator) { ret = OB_ERR_UNEXPECTED; LOG_WARN("lst_operator ptr is null", K(ret)); - } else if (OB_FAIL(lst_operator->get(cluster_id, tenant_id, ls_->get_ls_id(), ls_info))) { + } else if (OB_FAIL(lst_operator->get(cluster_id, tenant_id, + ls_->get_ls_id(), share::ObLSTable::DEFAULT_MODE, ls_info))) { LOG_WARN("failed to get log stream info", K(ret), K(cluster_id), K(tenant_id), "ls id", ls_->get_ls_id()); } return ret;