From 9d1758967869a5c43e2a56eb831c8bd83125d6bd Mon Sep 17 00:00:00 2001 From: obdev Date: Mon, 9 Jan 2023 13:11:46 +0000 Subject: [PATCH] suspend merge when tenant in restore --- deps/oblib/src/lib/utility/ob_tracepoint.h | 3 +- .../test_multi_version_sstable_merge.cpp | 1 + .../compaction/ob_medium_compaction_func.cpp | 15 ++++++++++ .../compaction/ob_partition_merge_policy.cpp | 10 +++++++ .../compaction/ob_tenant_freeze_info_mgr.cpp | 30 ++++++++++++++----- .../compaction/ob_tenant_freeze_info_mgr.h | 1 + .../compaction/ob_tenant_tablet_scheduler.cpp | 4 +-- 7 files changed, 54 insertions(+), 10 deletions(-) diff --git a/deps/oblib/src/lib/utility/ob_tracepoint.h b/deps/oblib/src/lib/utility/ob_tracepoint.h index 3f74b1c4d5..eaf3414ca1 100644 --- a/deps/oblib/src/lib/utility/ob_tracepoint.h +++ b/deps/oblib/src/lib/utility/ob_tracepoint.h @@ -597,6 +597,7 @@ class EventTable EN_MEDIUM_CREATE_DAG = 706, EN_MEDIUM_VERIFY_GROUP_SKIP_SET_VERIFY = 707, EN_MEDIUM_VERIFY_GROUP_SKIP_COLUMN_CHECKSUM = 708, + EN_SCHEDULE_MEDIUM_COMPACTION = 709, // please add new trace point after 750 EN_SESSION_LEAK_COUNT_THRESHOLD = 751, @@ -647,7 +648,7 @@ class EventTable EN_BACKUP_READ_MACRO_BLOCK_FAILED = 1111, EN_FETCH_TABLE_INFO_RPC = 1112, // END OF STORAGE HA - 1101 - 2000 - + EVENT_TABLE_MAX = SIZE_OF_EVENT_TABLE }; diff --git a/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp b/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp index 3654f2a317..027a732039 100644 --- a/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp +++ b/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp @@ -126,6 +126,7 @@ void TestMultiVersionMerge::SetUpTestCase() ObLSHandle ls_handle; ObLSService *ls_svr = MTL(ObLSService*); ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD)); + MTL(ObTenantTabletScheduler*)->resume_major_merge(); // create tablet obrpc::ObBatchCreateTabletArg create_tablet_arg; diff --git a/src/storage/compaction/ob_medium_compaction_func.cpp b/src/storage/compaction/ob_medium_compaction_func.cpp index 70ace3fc3f..314f485a0a 100644 --- a/src/storage/compaction/ob_medium_compaction_func.cpp +++ b/src/storage/compaction/ob_medium_compaction_func.cpp @@ -173,6 +173,14 @@ int ObMediumCompactionScheduleFunc::schedule_next_medium_primary_cluster( } LOG_DEBUG("schedule next medium in primary cluster", K(ret), KPC(this), K(schedule_medium_flag), K(schedule_major_snapshot), K(adaptive_merge_reason), KPC(last_major), K(medium_list), K(max_sync_medium_scn)); +#ifdef ERRSIM + ret = OB_E(EventTable::EN_SCHEDULE_MEDIUM_COMPACTION) ret; + if (OB_FAIL(ret) && tablet_.get_tablet_meta().tablet_id_.id() > ObTabletID::MIN_USER_TABLET_ID) { + FLOG_INFO("set schedule medium with errsim", KPC(this)); + ret = OB_SUCCESS; + schedule_medium_flag = true; + } +#endif if (OB_FAIL(ret) || !schedule_medium_flag) { } else if (ObMediumCompactionInfo::MAJOR_COMPACTION == medium_list.get_last_compaction_type()) { @@ -275,6 +283,13 @@ int ObMediumCompactionScheduleFunc::decide_medium_snapshot( } } } +#ifdef ERRSIM + ret = OB_E(EventTable::EN_SCHEDULE_MEDIUM_COMPACTION) ret; + if (OB_FAIL(ret) && tablet_.get_tablet_meta().tablet_id_.id() > ObTabletID::MIN_USER_TABLET_ID) { + FLOG_INFO("set schedule medium with errsim", KPC(this)); + ret = OB_SUCCESS; + } +#endif if (FAILEDx(prepare_medium_info(result, medium_info))) { if (OB_TABLE_IS_DELETED == ret) { ret = OB_SUCCESS; diff --git a/src/storage/compaction/ob_partition_merge_policy.cpp b/src/storage/compaction/ob_partition_merge_policy.cpp index 44de24625b..dad6df1641 100644 --- a/src/storage/compaction/ob_partition_merge_policy.cpp +++ b/src/storage/compaction/ob_partition_merge_policy.cpp @@ -1315,6 +1315,7 @@ int ObAdaptiveMergePolicy::find_meta_major_tables( LOG_WARN("failed to add minor table to meta merge result", K(ret)); } } // end of for + if (OB_FAIL(ret)) { } else if (tx_determ_table_cnt < 2) { ret = OB_NO_NEED_MERGE; @@ -1327,6 +1328,15 @@ int ObAdaptiveMergePolicy::find_meta_major_tables( ret = OB_NO_NEED_MERGE; LOG_INFO("chosen snapshot is abandoned", K(ret), K(result), K(tablet.get_multi_version_start())); } + +#ifdef ERRSIM + ret = OB_E(EventTable::EN_SCHEDULE_MEDIUM_COMPACTION) ret; + if (OB_FAIL(ret) && tablet.get_tablet_meta().tablet_id_.id() > ObTabletID::MIN_USER_TABLET_ID) { + FLOG_INFO("set schedule medium with errsim", "tablet_id", tablet.get_tablet_meta().tablet_id_); + ret = OB_SUCCESS; + } +#endif + } return ret; } diff --git a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp index ff670641e8..a3cdd2a05f 100644 --- a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp +++ b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp @@ -744,6 +744,7 @@ int64_t ObTenantFreezeInfoMgr::get_snapshot_gc_ts() ObTenantFreezeInfoMgr::ReloadTask::ReloadTask(ObTenantFreezeInfoMgr &mgr) : inited_(false), + check_tenant_status_(true), mgr_(mgr), sql_proxy_(NULL), snapshot_proxy_(), @@ -862,13 +863,28 @@ int ObTenantFreezeInfoMgr::ReloadTask::refresh_merge_info() scheduler->stop_major_merge(); LOG_INFO("schedule zone to stop major merge", K(tenant_id), K(zone_merge_info), K(global_merge_info)); } else { - scheduler->resume_major_merge(); - const int64_t scheduler_frozen_version = scheduler->get_frozen_version(); - if (zone_merge_info.broadcast_scn_.get_scn().get_val_for_tx() > scheduler_frozen_version) { - FLOG_INFO("try to schedule merge", K(tenant_id), "zone", zone_merge_info.zone_, "broadcast_scn", - zone_merge_info.broadcast_scn_, K(scheduler_frozen_version)); - if (OB_FAIL(scheduler->schedule_merge(zone_merge_info.broadcast_scn_.get_scn().get_val_for_tx()))) { - LOG_WARN("fail to schedule merge", K(ret), K(zone_merge_info)); + if (check_tenant_status_) { + bool is_restore = false; + if (OB_FAIL(ObMultiVersionSchemaService::get_instance().check_tenant_is_restore(nullptr, tenant_id, is_restore))) { + LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (is_restore) { + if (REACH_TENANT_TIME_INTERVAL(10L * 1000L * 1000L)) { + LOG_INFO("skip restoring tenant to schedule major merge", K(tenant_id), K(is_restore)); + } + } else { + check_tenant_status_ = false; + LOG_INFO("finish check tenant restore", K(tenant_id), K(is_restore)); + } + } + if (!check_tenant_status_) { + scheduler->resume_major_merge(); + const int64_t scheduler_frozen_version = scheduler->get_frozen_version(); + if (zone_merge_info.broadcast_scn_.get_scn().get_val_for_tx() > scheduler_frozen_version) { + FLOG_INFO("try to schedule merge", K(tenant_id), "zone", zone_merge_info.zone_, "broadcast_scn", + zone_merge_info.broadcast_scn_, K(scheduler_frozen_version)); + if (OB_FAIL(scheduler->schedule_merge(zone_merge_info.broadcast_scn_.get_scn().get_val_for_tx()))) { + LOG_WARN("fail to schedule merge", K(ret), K(zone_merge_info)); + } } } } diff --git a/src/storage/compaction/ob_tenant_freeze_info_mgr.h b/src/storage/compaction/ob_tenant_freeze_info_mgr.h index 3cbb2d1961..816fe6b079 100644 --- a/src/storage/compaction/ob_tenant_freeze_info_mgr.h +++ b/src/storage/compaction/ob_tenant_freeze_info_mgr.h @@ -173,6 +173,7 @@ private: common::ObIArray &freeze_info); bool inited_; + bool check_tenant_status_; ObTenantFreezeInfoMgr &mgr_; common::ObISQLClient *sql_proxy_; share::ObSnapshotTableProxy snapshot_proxy_; diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index b36b517c00..c655229977 100755 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -150,7 +150,7 @@ constexpr ObMergeType ObTenantTabletScheduler::MERGE_TYPES[]; ObTenantTabletScheduler::ObTenantTabletScheduler() : is_inited_(false), - major_merge_status_(true), + major_merge_status_(false), is_stop_(true), merge_loop_tg_id_(0), medium_loop_tg_id_(0), @@ -979,7 +979,7 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge( schedule_stats_.schedule_cnt_++; } - if (OB_TMP_FAIL(ObMediumCompactionScheduleFunc::schedule_tablet_medium_merge( + if (could_major_merge && OB_TMP_FAIL(ObMediumCompactionScheduleFunc::schedule_tablet_medium_merge( ls, *tablet, major_frozen_scn))) {