From a39c058fbc821d7ef407e7e883c9beef99a7eaf2 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 23 Aug 2024 17:18:39 +0000 Subject: [PATCH] Fix copy major of rebuild when restore --- .../high_availability/ob_ls_migration.cpp | 2 +- .../high_availability/ob_storage_ha_dag.cpp | 9 +++- .../high_availability/ob_storage_ha_dag.h | 2 + .../ob_storage_ha_tablet_builder.cpp | 29 ++++++++++++- .../ob_storage_ha_tablet_builder.h | 5 +++ .../ob_tablet_group_restore.cpp | 1 + src/storage/ls/ob_ls_tablet_service.cpp | 4 +- src/storage/tablet/ob_tablet.cpp | 43 ++++++++++++++++++- src/storage/tablet/ob_tablet.h | 1 + src/storage/tablet/ob_tablet_table_store.cpp | 7 +++ 10 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index 5bac1cb7e..32f39e176 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -3084,7 +3084,7 @@ int ObTabletMigrationTask::check_need_copy_sstable_( LOG_WARN("check need copy sstable get invlaid argument", K(ret), K(table_key)); } else if (OB_FAIL(ctx_->ha_table_info_mgr_.get_table_info(copy_tablet_ctx_->tablet_id_, table_key, copy_table_info))) { LOG_WARN("failed to get table info", K(ret), KPC(copy_tablet_ctx_), K(table_key)); - } else if (OB_FAIL(ObStorageHATaskUtils::check_need_copy_sstable(*copy_table_info, copy_tablet_ctx_->tablet_handle_, need_copy))) { + } else if (OB_FAIL(ObStorageHATaskUtils::check_need_copy_sstable(*copy_table_info, false /* is_restore */, copy_tablet_ctx_->tablet_handle_, need_copy))) { LOG_WARN("failed to check need copy sstable", K(ret), KPC(copy_tablet_ctx_), K(table_key)); } return ret; diff --git a/src/storage/high_availability/ob_storage_ha_dag.cpp b/src/storage/high_availability/ob_storage_ha_dag.cpp index 9a86eb1b0..cf5f4ce98 100644 --- a/src/storage/high_availability/ob_storage_ha_dag.cpp +++ b/src/storage/high_availability/ob_storage_ha_dag.cpp @@ -733,6 +733,7 @@ int ObHATabletGroupMgr::get_tablet_group_ctx( /******************ObStorageHATaskUtils*********************/ int ObStorageHATaskUtils::check_need_copy_sstable( const ObMigrationSSTableParam ¶m, + const bool &is_restore, ObTabletHandle &tablet_handle, bool &need_copy) { @@ -742,7 +743,7 @@ int ObStorageHATaskUtils::check_need_copy_sstable( ret = OB_INVALID_ARGUMENT; LOG_WARN("check need copy sstable get invalid argument", K(ret), K(param)); } else if (param.table_key_.is_major_sstable()) { - if (OB_FAIL(check_major_sstable_need_copy_(param, tablet_handle, need_copy))) { + if (OB_FAIL(check_major_sstable_need_copy_(param, is_restore, tablet_handle, need_copy))) { LOG_WARN("failed to check major sstable need copy", K(ret), K(param), K(tablet_handle)); } } else if (param.table_key_.is_minor_sstable()) { @@ -764,6 +765,7 @@ int ObStorageHATaskUtils::check_need_copy_sstable( int ObStorageHATaskUtils::check_major_sstable_need_copy_( const ObMigrationSSTableParam ¶m, + const bool &is_restore, ObTabletHandle &tablet_handle, bool &need_copy) { @@ -793,6 +795,11 @@ int ObStorageHATaskUtils::check_major_sstable_need_copy_( LOG_WARN("failed to get sstable meta handle", K(ret)); } else if (OB_FAIL(ObSSTableMetaChecker::check_sstable_meta(param, sst_meta_hdl.get_sstable_meta()))) { LOG_WARN("failed to check sstable meta", K(ret), K(param), K(sstable_wrapper)); + } else if (!is_restore && sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup() + && param.basic_meta_.table_backup_flag_.has_no_backup()) { + // when migration or rebuild, if dst sstable has backup macro and the corresponding src sstable has no backup macro, need copy + need_copy = true; + FLOG_INFO("dst sstable has backup macro and src sstable has no backup macro, need copy", K(need_copy), K(sst_meta_hdl), K(param.basic_meta_)); } else { need_copy = false; } diff --git a/src/storage/high_availability/ob_storage_ha_dag.h b/src/storage/high_availability/ob_storage_ha_dag.h index 5af011892..f2564c2a3 100644 --- a/src/storage/high_availability/ob_storage_ha_dag.h +++ b/src/storage/high_availability/ob_storage_ha_dag.h @@ -193,12 +193,14 @@ class ObStorageHATaskUtils public: static int check_need_copy_sstable( const ObMigrationSSTableParam ¶m, + const bool &is_restore, ObTabletHandle &tablet_handle, bool &need_copy); private: static int check_major_sstable_need_copy_( const ObMigrationSSTableParam ¶m, + const bool &is_restore, ObTabletHandle &tablet_handle, bool &need_copy); diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp index 578e84222..a4d7b95af 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp @@ -937,8 +937,17 @@ int ObStorageHATabletsBuilder::get_major_sstable_max_snapshot_( } else if (major_sstable_array.count() > 0 && OB_FAIL(major_sstable_array.get_all_table_wrappers(sstables))) { LOG_WARN("failed to get all tables", K(ret), K(param_)); } else { + ObSSTableMetaHandle sst_meta_hdl; + MajorSSTableSnapshotVersionCmp cmp; + + // step 1: sort major sstables by snapshot version + lib::ob_sort(sstables.begin(), sstables.end(), cmp); + + // step 2: get the major that has maximun snapshot version + // and all major snapshot version lower than it has no backup for (int64_t i = 0; OB_SUCC(ret) && i < sstables.count(); ++i) { - const ObITable *table = sstables.at(i).get_sstable(); + ObITable *table = sstables.at(i).get_sstable(); + sst_meta_hdl.reset(); if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; @@ -946,6 +955,11 @@ int ObStorageHATabletsBuilder::get_major_sstable_max_snapshot_( } else if (!table->is_major_sstable()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("sstable type is unexpected", K(ret), KP(table), K(param_)); + } else if (OB_FAIL(static_cast(table)->get_meta(sst_meta_hdl))) { + LOG_WARN("failed to get sstable meta handle", K(ret), K(table)); + } else if (sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { + // stop at first major sstable that has backup + break; } else { max_snapshot_version = std::max(max_snapshot_version, table->get_key().get_snapshot_version()); } @@ -1313,6 +1327,19 @@ int ObStorageHATabletsBuilder::hold_local_tablet_( return ret; } +bool ObStorageHATabletsBuilder::MajorSSTableSnapshotVersionCmp::operator()(const ObSSTableWrapper &lhs, const ObSSTableWrapper &rhs) const +{ + int ret = OB_SUCCESS; + int result = false; + if (!lhs.is_valid() || !rhs.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid sstable wrapper", K(ret), K(lhs), K(rhs)); + } else { + result = lhs.get_sstable()->get_snapshot_version() < rhs.get_sstable()->get_snapshot_version(); + } + return result; +} + /******************ObStorageHATabletTableInfoMgr*********************/ ObStorageHATableInfoMgr::ObStorageHATabletTableInfoMgr::ObStorageHATabletTableInfoMgr() : is_inited_(false), diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.h b/src/storage/high_availability/ob_storage_ha_tablet_builder.h index 74b03739c..3569221cc 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.h +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.h @@ -154,6 +154,11 @@ private: int hold_local_tablet_( common::ObIArray &tablet_handle_array); private: + struct MajorSSTableSnapshotVersionCmp + { + bool operator()(const ObSSTableWrapper &lhs, const ObSSTableWrapper &rhs) const; + }; + bool is_inited_; ObStorageHATabletsBuilderParam param_; DISALLOW_COPY_AND_ASSIGN(ObStorageHATabletsBuilder); diff --git a/src/storage/high_availability/ob_tablet_group_restore.cpp b/src/storage/high_availability/ob_tablet_group_restore.cpp index dbc8d40dd..4af4116ae 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.cpp +++ b/src/storage/high_availability/ob_tablet_group_restore.cpp @@ -2643,6 +2643,7 @@ int ObTabletRestoreTask::check_need_copy_sstable_( LOG_WARN("failed to get table info", K(ret), KPC(tablet_restore_ctx_), K(table_key)); } else if (!ObTabletRestoreAction::is_restore_replace_remote_sstable(restore_action)) { if (OB_FAIL(ObStorageHATaskUtils::check_need_copy_sstable(*copy_table_info, + tablet_restore_ctx_->is_leader_ /* is_restore */, tablet_restore_ctx_->tablet_handle_, need_copy))) { LOG_WARN("failed to check need copy sstable", K(ret), KPC(tablet_restore_ctx_), K(table_key)); diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 25f0fffb1..ec7da2599 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -1682,7 +1682,9 @@ int ObLSTabletService::update_tablet_ha_data_status( } else if (OB_FAIL(tablet->tablet_meta_.ha_status_.set_data_status(data_status))) { LOG_WARN("failed to set data status", K(ret), KPC(tablet), K(data_status)); } else { - if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(*tablet, new_tablet_handle))) { + if (OB_FAIL(tablet->check_valid())) { + LOG_WARN("failed to check tablet valid", K(ret), K(data_status), KPC(tablet)); + } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(*tablet, new_tablet_handle))) { LOG_WARN("fail to persist and transform tablet", K(ret), KPC(tablet), K(new_tablet_handle)); } else if (FALSE_IT(time_guard.click("Persist"))) { } else if (FALSE_IT(disk_addr = new_tablet_handle.get_obj()->tablet_addr_)) { diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index b8b3d9686..e7d885ce0 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -1652,7 +1652,7 @@ int ObTablet::check_sstable_column_checksum() const ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, table is nullptr", K(ret), KPC(table)); } else { - ObSSTable *cur = reinterpret_cast(table); + ObSSTable *cur = static_cast(table); ObSSTableMetaHandle meta_handle; if (OB_FAIL(cur->get_meta(meta_handle))) { LOG_WARN("fail to get sstable meta", K(ret), KPC(cur), KPC(this)); @@ -1670,6 +1670,45 @@ int ObTablet::check_sstable_column_checksum() const return ret; } +int ObTablet::check_no_backup_data() const +{ + int ret = OB_SUCCESS; + ObTableStoreIterator iter; + + if (!tablet_meta_.ha_status_.is_none()) { + // skip + } else if (OB_FAIL(inner_get_all_sstables(iter, true /* need_unpack */))) { + LOG_WARN("fail to get all sstables", K(ret)); + } else { + ObITable *table = nullptr; + while (OB_SUCC(ret)) { + if (OB_FAIL(iter.get_next(table))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("failed to get next table", K(ret), KPC(this)); + } + } else if (OB_ISNULL(table) || OB_UNLIKELY(!table->is_sstable())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, table is nullptr", K(ret), KPC(table)); + } else { + ObSSTable *cur = static_cast(table); + ObSSTableMetaHandle meta_handle; + if (OB_FAIL(cur->get_meta(meta_handle))) { + LOG_WARN("fail to get sstable meta", K(ret), KPC(cur), KPC(this)); + } else if (OB_UNLIKELY(meta_handle.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("The sstable has backup data, but restore status is full.", + K(ret), KPC(cur), KPC(this), K(tablet_meta_.ha_status_)); + } + } + } + } + + return ret; +} + int ObTablet::serialize(char *buf, const int64_t len, int64_t &pos, const ObSArray &meta_arr) const { int ret = OB_SUCCESS; @@ -7680,6 +7719,8 @@ int ObTablet::inner_check_valid(const bool ignore_ha_status) const LOG_WARN("failed to check medium list", K(ret), KPC(this)); } else if (OB_FAIL(check_sstable_column_checksum())) { LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); + } else if (OB_FAIL(check_no_backup_data())) { + LOG_WARN("failed to check no backup data", K(ret), KPC(this)); } return ret; } diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index b15a72e95..2418aa34c 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -714,6 +714,7 @@ private: int check_medium_list() const; int check_sstable_column_checksum() const; + int check_no_backup_data() const; int get_finish_medium_scn(int64_t &finish_medium_scn) const; int inner_get_mds_sstables(ObTableStoreIterator &table_store_iter) const; diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index ef56e9395..abcafde98 100644 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -1402,6 +1402,13 @@ int ObTabletTableStore::check_and_build_new_major_tables( new_sst_meta_hdl.get_sstable_meta(), old_sst_meta_hdl.get_sstable_meta()))) { LOG_WARN("failed to check sstable meta", K(ret), KPC(new_sstable), KPC(old_sstable)); } else { + if (new_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_no_backup() + && old_sst_meta_hdl.get_sstable_meta().get_basic_meta().table_backup_flag_.has_backup()) { + // if new sstable has no backup macro block and old sstable has backup macro block + // replace the old sstable with the new one + FLOG_INFO("replace major sstable with new one", KPC(new_sstable), KPC(old_sstable)); + major_tables.at(j) = new_table; + } need_add = false; } }