From 6892b58c900aab2f3d860be39df89c93dea95005 Mon Sep 17 00:00:00 2001 From: godyangfight Date: Mon, 20 Mar 2023 20:45:18 +0800 Subject: [PATCH] Fix ls migration online succeed but inner retry return 4016 bug --- .../blocksstable/ob_macro_block_checker.cpp | 2 ++ .../blocksstable/ob_macro_block_checker.h | 5 ++-- .../high_availability/ob_ls_migration.cpp | 28 +++++++++++++++++-- .../high_availability/ob_ls_migration.h | 1 + .../ob_storage_ha_macro_block_writer.cpp | 14 ++++++---- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/storage/blocksstable/ob_macro_block_checker.cpp b/src/storage/blocksstable/ob_macro_block_checker.cpp index a847f54fa8..4f2065a2f1 100644 --- a/src/storage/blocksstable/ob_macro_block_checker.cpp +++ b/src/storage/blocksstable/ob_macro_block_checker.cpp @@ -38,6 +38,8 @@ int ObSSTableMacroBlockChecker::check( ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "Invalid argument", K(ret), KP(macro_block_buf), K(macro_block_buf_size), K(check_level)); + } else if (ObMacroBlockCheckLevel::CHECK_LEVEL_NONE == check_level) { + //do nothing } else if (OB_FAIL(common_header.deserialize(macro_block_buf, macro_block_buf_size, pos))) { STORAGE_LOG(ERROR, "fail to deserialize common header", K(ret), KP(macro_block_buf), K(macro_block_buf_size), K(pos), K(common_header)); diff --git a/src/storage/blocksstable/ob_macro_block_checker.h b/src/storage/blocksstable/ob_macro_block_checker.h index a70f8d2639..cf82766c65 100644 --- a/src/storage/blocksstable/ob_macro_block_checker.h +++ b/src/storage/blocksstable/ob_macro_block_checker.h @@ -26,8 +26,9 @@ namespace blocksstable { enum ObMacroBlockCheckLevel { - CHECK_LEVEL_PHYSICAL = 0, // verify data checksum - CHECK_LEVEL_LOGICAL = 1, // verify column checksum + CHECK_LEVEL_NONE = 0, // no check + CHECK_LEVEL_PHYSICAL = 1, // verify data checksum + CHECK_LEVEL_LOGICAL = 2, // verify column checksum CHECK_LEVEL_MAX, }; diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index e0b38202f8..20d77d5941 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -2915,7 +2915,6 @@ int ObDataTabletsMigrationTask::process() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - bool is_ls_online_success = false; LOG_INFO("start do data tablets migration task", K(ret), KPC(ctx_)); #ifdef ERRSIM SERVER_EVENT_SYNC_ADD("storage_ha", "before_data_tablets_migration_task", @@ -2935,7 +2934,6 @@ int ObDataTabletsMigrationTask::process() LOG_WARN("failed to try remove unneeded tablets", K(ret), KPC(ctx_)); } else if (OB_FAIL(ls_online_())) { LOG_WARN("failed to start realy log", K(ret), K(*ctx_)); - } else if (FALSE_IT(is_ls_online_success = true)) { } else if (OB_FAIL(build_tablet_group_info_())) { LOG_WARN("failed to build tablet group info", K(ret), KPC(ctx_)); } else { @@ -2966,7 +2964,12 @@ int ObDataTabletsMigrationTask::process() if (OB_FAIL(ret)) { int tmp_ret = OB_SUCCESS; - const bool allow_retry = !is_ls_online_success; + bool allow_retry = true; + if (OB_SUCCESS != (tmp_ret = try_offline_ls_())) { + LOG_WARN("failed to try offline ls", K(tmp_ret)); + } else if (FALSE_IT(allow_retry = OB_SUCCESS == tmp_ret)) { + } + if (OB_SUCCESS != (tmp_ret = ObStorageHADagUtils::deal_with_fo(ret, this->get_dag(), allow_retry))) { LOG_WARN("failed to deal with fo", K(ret), K(tmp_ret), K(*ctx_)); } @@ -3303,6 +3306,25 @@ int ObDataTabletsMigrationTask::try_remove_unneeded_tablets_() return ret; } +int ObDataTabletsMigrationTask::try_offline_ls_() +{ + int ret = OB_SUCCESS; + ObLSHandle ls_handle; + ObLS *ls = nullptr; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("start migration task do not init", K(ret)); + } else if (OB_FAIL(ObStorageHADagUtils::get_ls(ctx_->arg_.ls_id_, ls_handle))) { + LOG_WARN("failed to get ls", K(ret), KPC(ctx_)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls should not be NULL", K(ret), KPC(ctx_)); + } else if (OB_FAIL(ls->offline())) { + LOG_WARN("failed to offline ls", K(ret), KPC(ctx_)); + } + return ret; +} int ObDataTabletsMigrationTask::record_server_event_() { diff --git a/src/storage/high_availability/ob_ls_migration.h b/src/storage/high_availability/ob_ls_migration.h index 5a3de2ccb8..69f14eb36a 100644 --- a/src/storage/high_availability/ob_ls_migration.h +++ b/src/storage/high_availability/ob_ls_migration.h @@ -418,6 +418,7 @@ private: int build_tablet_group_info_(); int generate_tablet_group_dag_(); int try_remove_unneeded_tablets_(); + int try_offline_ls_(); int record_server_event_(); private: diff --git a/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp b/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp index d8c96efb4f..26f0fc6758 100644 --- a/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp +++ b/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp @@ -69,10 +69,15 @@ int ObStorageHAMacroBlockWriter::check_macro_block_( ObMacroBlockCheckLevel check_level = ObMacroBlockCheckLevel::CHECK_LEVEL_MAX; switch (migrate_verify_level) { case 0: - check_level = ObMacroBlockCheckLevel::CHECK_LEVEL_PHYSICAL; + check_level = ObMacroBlockCheckLevel::CHECK_LEVEL_NONE; break; case 1: - check_level = ObMacroBlockCheckLevel::CHECK_LEVEL_LOGICAL; + check_level = ObMacroBlockCheckLevel::CHECK_LEVEL_PHYSICAL; + break; + case 2: + //check_level = ObMacroBlockCheckLevel::CHECK_LEVEL_LOGICAL; + //Here using logical has a bug. + check_level = ObMacroBlockCheckLevel::CHECK_LEVEL_PHYSICAL; break; default: check_level = ObMacroBlockCheckLevel::CHECK_LEVEL_MAX; @@ -80,10 +85,9 @@ int ObStorageHAMacroBlockWriter::check_macro_block_( STORAGE_LOG(WARN, "invalid check level", K(ret), K(migrate_verify_level)); break; } - //Here using logical has a bug. - check_level = ObMacroBlockCheckLevel::CHECK_LEVEL_PHYSICAL; - if (OB_FAIL(macro_checker_.check(data.data(), data.length(), check_level))) { + if (OB_FAIL(ret)) { + } else if (OB_FAIL(macro_checker_.check(data.data(), data.length(), check_level))) { STORAGE_LOG(ERROR, "failed to check macro block", K(ret), K(data), K(check_level)); } }