From 5640c64a58d445ac6d8ceaf48a74e262d9bf4455 Mon Sep 17 00:00:00 2001 From: HaHaJeff Date: Thu, 13 Apr 2023 09:36:47 +0000 Subject: [PATCH] fixed read block return -4016 in flashback --- ...test_ob_simple_log_single_replica_func.cpp | 48 +++++++++++++++++++ src/logservice/palf/log_iterator_impl.h | 3 +- src/logservice/palf/log_storage.cpp | 35 +++++++------- src/logservice/palf/log_storage.h | 10 +++- 4 files changed, 75 insertions(+), 21 deletions(-) diff --git a/mittest/logservice/test_ob_simple_log_single_replica_func.cpp b/mittest/logservice/test_ob_simple_log_single_replica_func.cpp index 23560eadf..4d93ca097 100644 --- a/mittest/logservice/test_ob_simple_log_single_replica_func.cpp +++ b/mittest/logservice/test_ob_simple_log_single_replica_func.cpp @@ -1671,6 +1671,54 @@ TEST_F(TestObSimpleLogClusterSingleReplica, test_iterator_with_flashback) } +TEST_F(TestObSimpleLogClusterSingleReplica, read_block_in_flashback) +{ + SET_CASE_LOG_FILE(TEST_NAME, "read_block_in_flashback"); + OB_LOGGER.set_log_level("TRACE"); + const int64_t id = ATOMIC_AAF(&palf_id_, 1); + int64_t leader_idx = 0; + PalfHandleImplGuard leader; + PalfEnv *palf_env = NULL; + EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); + + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 2 * 32 + 2, id, MAX_LOG_BODY_SIZE)); + EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.get_palf_handle_impl()->get_max_lsn())); + + block_id_t min_block_id, max_block_id; + LogStorage *log_storage = &leader.get_palf_handle_impl()->log_engine_.log_storage_; + EXPECT_EQ(OB_SUCCESS, log_storage->get_block_id_range(min_block_id, max_block_id)); + EXPECT_EQ(2, max_block_id); + SCN scn; + char block_name_tmp[OB_MAX_FILE_NAME_LENGTH]; + EXPECT_EQ(OB_SUCCESS, block_id_to_tmp_string(max_block_id, block_name_tmp, OB_MAX_FILE_NAME_LENGTH)); + char block_name[OB_MAX_FILE_NAME_LENGTH]; + EXPECT_EQ(OB_SUCCESS, block_id_to_string(max_block_id, block_name, OB_MAX_FILE_NAME_LENGTH)); + ::renameat(log_storage->block_mgr_.dir_fd_, block_name, log_storage->block_mgr_.dir_fd_, block_name_tmp); + EXPECT_EQ(-1, ::openat(log_storage->block_mgr_.dir_fd_, block_name, LOG_READ_FLAG)); + EXPECT_EQ(OB_NEED_RETRY, read_log(leader)); + EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn)); + + // 测试边界场景，read_log_tail_为文件中间，最后一个文件完全被flashback掉, 此时log_tail_是最后一个文件头 + log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE); + EXPECT_EQ(OB_NEED_RETRY, read_log(leader)); + EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn)); + + // 测试边界场景，read_log_tail_最后一个文件头，最后一个文件完全被flashback掉 + log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE); + log_storage->readable_log_tail_ = LSN(2*PALF_BLOCK_SIZE); + EXPECT_EQ(OB_ITER_END, read_log(leader)); + EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn)); + + // 测试边界场景，readable_log_tail_还没改变前检验是否可读通过，直接读文件时报错文件不存在。 + log_storage->log_tail_ = LSN(3*PALF_BLOCK_SIZE); + log_storage->readable_log_tail_ = LSN(3*PALF_BLOCK_SIZE); + // 设置max_block_id_为1是为了构造check_read_out_of_bound返回OB_ERR_OUT_OF_UPPER_BOUND的场景 + log_storage->block_mgr_.max_block_id_ = 1; + // log_storage返回OB_ERR_OUT_OF_UPPER_BOUND, iterator将其转换为OB_ITER_END + EXPECT_EQ(OB_ITER_END, read_log(leader)); + EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn)); +} + } // namespace unittest } // namespace oceanbase diff --git a/src/logservice/palf/log_iterator_impl.h b/src/logservice/palf/log_iterator_impl.h index 6d3f74290..70b94acca 100644 --- a/src/logservice/palf/log_iterator_impl.h +++ b/src/logservice/palf/log_iterator_impl.h @@ -106,7 +106,8 @@ public: // - if the end_lsn get from get_file_end_lsn is smaller than 'log_tail_' of LogStorage, and it's // not the exact boundary of LogGroupEntry(for PalgGroupeBufferIterator, or LogEntry for PalfBufferIterator), // OB_NEED_RETRY may be return. - // + // - if read_data_from_storage_ is concurrent with the last step of flashback, opening last block on disk may be failed + // due to rename, return OB_NEED_RETRY in this case.(TODO by runlin: retry by myself) // OB_ERR_OUT_LOWER_BOUND // - block has been recycled // OB_CHECKSUM_ERROR diff --git a/src/logservice/palf/log_storage.cpp b/src/logservice/palf/log_storage.cpp index 8847b481f..eb49ab428 100644 --- a/src/logservice/palf/log_storage.cpp +++ b/src/logservice/palf/log_storage.cpp @@ -603,20 +603,29 @@ int LogStorage::do_init_(const char *base_dir, return ret; } -bool LogStorage::check_read_out_of_lower_bound_(const block_id_t &block_id) const +int LogStorage::check_read_out_of_bound_(const block_id_t &block_id) const { - bool bool_ret = false; int ret = OB_SUCCESS; block_id_t min_block_id = LOG_INVALID_BLOCK_ID; block_id_t max_block_id = LOG_INVALID_BLOCK_ID; if (OB_FAIL(get_block_id_range(min_block_id, max_block_id)) && OB_ENTRY_NOT_EXIST != ret) { PALF_LOG(WARN, "get_block_id_range failed", K(ret), K(min_block_id), K(max_block_id)); + } else if (max_block_id == block_id) { + ret = OB_NEED_RETRY; + PALF_LOG(WARN, "the block to be read is in flashback, need read retry", K(min_block_id), K(max_block_id), K(block_id)); + } else if (max_block_id < block_id) { + ret = OB_ERR_OUT_OF_UPPER_BOUND; + PALF_LOG(WARN, "read something out of upper bound, the blocks may be deleted by flashback", + K(min_block_id), K(max_block_id), K(block_id)); + } else if (min_block_id > block_id) { + ret = OB_ERR_OUT_OF_LOWER_BOUND; + PALF_LOG(TRACE, "read something out of lower bound", K(min_block_id), K(max_block_id), K(block_id)); } else { - bool_ret = (min_block_id > block_id || max_block_id < block_id || OB_ENTRY_NOT_EXIST == ret); - PALF_LOG(TRACE, "curr block id range", K(min_block_id), K(max_block_id)); + ret = OB_ERR_UNEXPECTED; + PALF_LOG(ERROR, "unexpected error, the block may be deleted by human", K(min_block_id), K(max_block_id), K(block_id)); } - return bool_ret; + return ret; } int LogStorage::inner_switch_block_() @@ -763,13 +772,7 @@ int LogStorage::read_block_header_(const block_id_t block_id, } if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { - if (true == check_read_out_of_lower_bound_(block_id)) { - ret = OB_ERR_OUT_OF_LOWER_BOUND; - PALF_LOG(WARN, "this block has been deleted", K(ret), K(block_id)); - } else { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(WARN, "unexpected error, maybe deleted by human or flashabck!!!", K(ret), K(block_id), KPC(this)); - } + ret = check_read_out_of_bound_(block_id); } return ret; } @@ -834,13 +837,7 @@ int LogStorage::inner_pread_(const LSN &read_lsn, } if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { - if (true == check_read_out_of_lower_bound_(lsn_2_block(read_lsn, logical_block_size_))) { - ret = OB_ERR_OUT_OF_LOWER_BOUND; - PALF_LOG(WARN, "this block has been deleted", K(ret), K(read_lsn)); - } else { - ret = OB_ERR_UNEXPECTED; - PALF_LOG(ERROR, "unexpected error, maybe deleted by human!!!", K(ret), K(read_lsn)); - } + ret = check_read_out_of_bound_(lsn_2_block(read_lsn, logical_block_size_)); } return ret; } diff --git a/src/logservice/palf/log_storage.h b/src/logservice/palf/log_storage.h index 5c4a9ace1..6ba8a4202 100644 --- a/src/logservice/palf/log_storage.h +++ b/src/logservice/palf/log_storage.h @@ -127,7 +127,15 @@ private: const int64_t align_buf_size, const UpdateManifestCallback &update_manifest_cb, ILogBlockPool *log_block_pool); - bool check_read_out_of_lower_bound_(const block_id_t &block_id) const; + // @ret val: + // OB_SUCCESS + // OB_ERR_OUT_OF_LOWER_BOUND + // OB_ERR_OUT_OF_UPPER_BOUND + // in flashback, (flashback_block_id, max_block_id] may be deleted, however, fetch log may read + // some blocks in range of (flashback_block_id, max_block_id]. + // OB_NEED_RETRY + // OB_ERR_UNEXPECTED + int check_read_out_of_bound_(const block_id_t &block_id) const; int inner_switch_block_(); int append_block_header_used_for_meta_storage_(); int append_block_header_(const LSN &block_min_lsn, const share::SCN &block_min_scn);