fixed read block return -4016 in flashback

mittest/logservice/test_ob_simple_log_single_replica_func.cpp

@@ -1671,6 +1671,54 @@ TEST_F(TestObSimpleLogClusterSingleReplica, test_iterator_with_flashback)
 
 }
 
+TEST_F(TestObSimpleLogClusterSingleReplica, read_block_in_flashback)
+{
+  SET_CASE_LOG_FILE(TEST_NAME, "read_block_in_flashback");
+  OB_LOGGER.set_log_level("TRACE");
+  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
+  int64_t leader_idx = 0;
+  PalfHandleImplGuard leader;
+  PalfEnv *palf_env = NULL;
+  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
+
+  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 2 * 32 + 2, id, MAX_LOG_BODY_SIZE));
+  EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.get_palf_handle_impl()->get_max_lsn()));
+
+  block_id_t min_block_id, max_block_id;
+  LogStorage *log_storage = &leader.get_palf_handle_impl()->log_engine_.log_storage_;
+  EXPECT_EQ(OB_SUCCESS, log_storage->get_block_id_range(min_block_id, max_block_id));
+  EXPECT_EQ(2, max_block_id);
+  SCN scn;
+  char block_name_tmp[OB_MAX_FILE_NAME_LENGTH];
+  EXPECT_EQ(OB_SUCCESS, block_id_to_tmp_string(max_block_id, block_name_tmp, OB_MAX_FILE_NAME_LENGTH));
+  char block_name[OB_MAX_FILE_NAME_LENGTH];
+  EXPECT_EQ(OB_SUCCESS, block_id_to_string(max_block_id, block_name, OB_MAX_FILE_NAME_LENGTH));
+  ::renameat(log_storage->block_mgr_.dir_fd_, block_name, log_storage->block_mgr_.dir_fd_, block_name_tmp);
+  EXPECT_EQ(-1, ::openat(log_storage->block_mgr_.dir_fd_, block_name, LOG_READ_FLAG));
+  EXPECT_EQ(OB_NEED_RETRY, read_log(leader));
+  EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn));
+
+  // 测试边界场景，read_log_tail_为文件中间，最后一个文件完全被flashback掉, 此时log_tail_是最后一个文件头
+  log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE);
+  EXPECT_EQ(OB_NEED_RETRY, read_log(leader));
+  EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn));
+
+  // 测试边界场景，read_log_tail_最后一个文件头，最后一个文件完全被flashback掉
+  log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE);
+  log_storage->readable_log_tail_ = LSN(2*PALF_BLOCK_SIZE);
+  EXPECT_EQ(OB_ITER_END, read_log(leader));
+  EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn));
+
+  // 测试边界场景，readable_log_tail_还没改变前检验是否可读通过，直接读文件时报错文件不存在。
+  log_storage->log_tail_ = LSN(3*PALF_BLOCK_SIZE);
+  log_storage->readable_log_tail_ = LSN(3*PALF_BLOCK_SIZE);
+  // 设置max_block_id_为1是为了构造check_read_out_of_bound返回OB_ERR_OUT_OF_UPPER_BOUND的场景
+  log_storage->block_mgr_.max_block_id_ = 1;
+  // log_storage返回OB_ERR_OUT_OF_UPPER_BOUND, iterator将其转换为OB_ITER_END
+  EXPECT_EQ(OB_ITER_END, read_log(leader));
+  EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn));
+}
+
 
 } // namespace unittest
 } // namespace oceanbase

src/logservice/palf/log_iterator_impl.h

@@ -106,7 +106,8 @@ public:
   //      - if the end_lsn get from get_file_end_lsn is smaller than 'log_tail_' of LogStorage, and it's
   //        not the exact boundary of LogGroupEntry(for PalgGroupeBufferIterator, or LogEntry for PalfBufferIterator),
   //        OB_NEED_RETRY may be return.
-  //
+  //      - if read_data_from_storage_ is concurrent with the last step of flashback, opening last block on disk may be failed
+  //        due to rename, return OB_NEED_RETRY in this case.(TODO by runlin: retry by myself)
   //   OB_ERR_OUT_LOWER_BOUND
   //      - block has been recycled
   //   OB_CHECKSUM_ERROR

src/logservice/palf/log_storage.cpp

@@ -603,20 +603,29 @@ int LogStorage::do_init_(const char *base_dir,
   return ret;
 }
 
-bool LogStorage::check_read_out_of_lower_bound_(const block_id_t &block_id) const
+int LogStorage::check_read_out_of_bound_(const block_id_t &block_id) const
 {
-  bool bool_ret = false;
   int ret = OB_SUCCESS;
   block_id_t min_block_id = LOG_INVALID_BLOCK_ID;
   block_id_t max_block_id = LOG_INVALID_BLOCK_ID;
   if (OB_FAIL(get_block_id_range(min_block_id, max_block_id))
       && OB_ENTRY_NOT_EXIST != ret) {
     PALF_LOG(WARN, "get_block_id_range failed", K(ret), K(min_block_id), K(max_block_id));
+  } else if (max_block_id == block_id) {
+    ret = OB_NEED_RETRY;
+    PALF_LOG(WARN, "the block to be read is in flashback, need read retry", K(min_block_id), K(max_block_id), K(block_id));
+  } else if (max_block_id < block_id) {
+    ret = OB_ERR_OUT_OF_UPPER_BOUND;
+    PALF_LOG(WARN, "read something out of upper bound, the blocks may be deleted by flashback",
+        K(min_block_id), K(max_block_id), K(block_id));
+  } else if (min_block_id > block_id) {
+    ret = OB_ERR_OUT_OF_LOWER_BOUND;
+    PALF_LOG(TRACE, "read something out of lower bound", K(min_block_id), K(max_block_id), K(block_id));
   } else {
-    bool_ret = (min_block_id > block_id || max_block_id < block_id || OB_ENTRY_NOT_EXIST == ret);
-    PALF_LOG(TRACE, "curr block id range", K(min_block_id), K(max_block_id));
+    ret = OB_ERR_UNEXPECTED;
+    PALF_LOG(ERROR, "unexpected error, the block may be deleted by human", K(min_block_id), K(max_block_id), K(block_id));
   }
-  return bool_ret;
+  return ret;
 }
 
 int LogStorage::inner_switch_block_()
@@ -763,13 +772,7 @@ int LogStorage::read_block_header_(const block_id_t block_id,
   }
 
   if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) {
-    if (true == check_read_out_of_lower_bound_(block_id)) {
-      ret = OB_ERR_OUT_OF_LOWER_BOUND;
-      PALF_LOG(WARN, "this block has been deleted", K(ret), K(block_id));
-    } else {
-      ret = OB_ERR_UNEXPECTED;
-      PALF_LOG(WARN, "unexpected error, maybe deleted by human or flashabck!!!", K(ret), K(block_id), KPC(this));
-    }
+    ret = check_read_out_of_bound_(block_id);
   }
   return ret;
 }
@@ -834,13 +837,7 @@ int LogStorage::inner_pread_(const LSN &read_lsn,
   }
 
   if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) {
-    if (true == check_read_out_of_lower_bound_(lsn_2_block(read_lsn, logical_block_size_))) {
-      ret = OB_ERR_OUT_OF_LOWER_BOUND;
-      PALF_LOG(WARN, "this block has been deleted", K(ret), K(read_lsn));
-    } else {
-      ret = OB_ERR_UNEXPECTED;
-      PALF_LOG(ERROR, "unexpected error, maybe deleted by human!!!", K(ret), K(read_lsn));
-    }
+    ret = check_read_out_of_bound_(lsn_2_block(read_lsn, logical_block_size_));
   }
   return ret;
 }

src/logservice/palf/log_storage.h

@@ -127,7 +127,15 @@ private:
                const int64_t align_buf_size,
                const UpdateManifestCallback &update_manifest_cb,
                ILogBlockPool *log_block_pool);
-  bool check_read_out_of_lower_bound_(const block_id_t &block_id) const;
+  // @ret val:
+  //   OB_SUCCESS
+  //   OB_ERR_OUT_OF_LOWER_BOUND
+  //   OB_ERR_OUT_OF_UPPER_BOUND
+  //      in flashback, (flashback_block_id, max_block_id] may be deleted, however, fetch log may read
+  //      some blocks in range of (flashback_block_id, max_block_id].
+  //   OB_NEED_RETRY
+  //   OB_ERR_UNEXPECTED
+  int check_read_out_of_bound_(const block_id_t &block_id) const;
   int inner_switch_block_();
   int append_block_header_used_for_meta_storage_();
   int append_block_header_(const LSN &block_min_lsn, const share::SCN &block_min_scn);