optimize log print when restart observer failed because of check log pool integrity failed.

mittest/logservice/test_ob_simple_log_restart.cpp

@@ -541,6 +541,63 @@ TEST_F(TestObSimpleLogClusterRestart, restart_and_clear_tmp_files)
   PALF_LOG(INFO, "end test restart", K(id), K(guard));
 }
 
+TEST_F(TestObSimpleLogClusterRestart, test_with_tmp_dir)
+{
+  SET_CASE_LOG_FILE(TEST_NAME, "test_with_tmp_dir");
+  OB_LOGGER.set_log_level("INFO");
+  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
+  PALF_LOG(INFO, "start advance_base_lsn", K(id));
+  int64_t leader_idx = 0;
+  int64_t log_ts = 1;
+  ObServerLogBlockMgr *log_pool = nullptr;
+  std::string logserver_dir;
+  {
+    PalfHandleImplGuard leader;
+    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
+    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
+    logserver_dir = leader.palf_env_impl_->log_dir_;
+    sleep(2);
+    LSN log_tail =
+        leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_;
+    int count = (LSN(PALF_META_BLOCK_SIZE) - log_tail)/4096;
+    for (int64_t i = 0; i < count; i++) {
+      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->enable_vote());
+    }
+    while (LSN(PALF_META_BLOCK_SIZE) !=
+        leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_)
+    {
+      sleep(1);
+    }
+    sleep(1);
+    EXPECT_EQ(LSN(PALF_META_BLOCK_SIZE), leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_);
+    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->log_engine_.log_meta_storage_.block_mgr_.switch_next_block(1));
+    ObISimpleLogServer *i_server = get_cluster()[leader_idx];
+    ObSimpleLogServer *server = dynamic_cast<ObSimpleLogServer*>(i_server);
+    log_pool = &server->log_block_pool_;
+  }
+  const std::string log_pool_dir = log_pool->log_pool_path_;
+  std::size_t pos = log_pool_dir.find_last_of('/');
+  const std::string log_base_dir = log_pool_dir.substr(0, pos);
+  const std::string mkdir_unexpected_dir = "mkdir -p " + log_base_dir + "/unexpected_dir";
+  CLOG_LOG(INFO, "runlin trace print mkdir", K(mkdir_unexpected_dir.c_str()));
+  system(mkdir_unexpected_dir.c_str());
+  EXPECT_EQ(OB_ERR_UNEXPECTED, restart_paxos_groups());
+  const std::string rm_unexpected_dir = "rm -rf " + log_base_dir + "/unexpected_dir";
+  system(rm_unexpected_dir.c_str());
+  const std::string tmp_log_dir = logserver_dir + "/tmp_dir";
+  const std::string mkdir_tmp_log_dir = "mkdir -p " + tmp_log_dir;
+  system(mkdir_tmp_log_dir.c_str());
+  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
+  {
+    PalfHandleImplGuard leader;
+    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
+    EXPECT_LT(LSN(PALF_META_BLOCK_SIZE), leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_);
+    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->set_base_lsn(LSN(0)));
+    sleep(1);
+    EXPECT_LT(LSN(PALF_META_BLOCK_SIZE) + 4096, leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_);
+  }
+}
+
 } // namespace unittest
 } // namespace oceanbase
 

src/logservice/ob_server_log_block_mgr.cpp

@@ -1222,8 +1222,8 @@ int ObServerLogBlockMgr::get_has_allocated_blocks_cnt_in_(
   int ret = OB_SUCCESS;
   DIR *dir = NULL;
   struct dirent *entry = NULL;
-  std::regex pattern_log(".*/tenant_[1-9]\\d*/[1-9]\\d*/log");
+  std::regex pattern_tenant(".*/tenant_[1-9]\\d*");
-  std::regex pattern_meta(".*/tenant_[1-9]\\d*/[1-9]\\d*/meta");
+  std::regex pattern_log_pool(".*/log_pool/*");
   if (NULL == (dir = opendir(log_disk_path))) {
     ret = OB_ERR_SYS;
     CLOG_LOG(WARN, "opendir failed", K(log_disk_path));
@@ -1242,23 +1242,19 @@ int ObServerLogBlockMgr::get_has_allocated_blocks_cnt_in_(
       } else if (OB_FAIL(FileDirectoryUtils::is_directory(current_file_path, is_dir))) {
         CLOG_LOG(WARN, "is_directory failed", K(ret), K(entry->d_name));
       } else if (false == is_dir) {
-      } else if (true == std::regex_match(current_file_path, pattern_log)
+        ret = OB_ERR_UNEXPECTED;
-                 || true == std::regex_match(current_file_path, pattern_meta)) {
+        LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
-        GetBlockCountFunctor functor(current_file_path);
+                      "OceanBase. Please confirm whether manual deletion is required",
-        if (OB_FAIL(palf::scan_dir(current_file_path, functor))) {
+                      ", unexpected file path", current_file_path);
-          LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
+      } else if (true == std::regex_match(current_file_path, pattern_tenant)) {
-                        "OceanBase. Please confirm whether manual deletion is required",
+        ret = scan_tenant_dir_(current_file_path, has_allocated_block_cnt);
-                        "log directory", current_file_path);
+      } else if (true == std::regex_match(current_file_path, pattern_log_pool)) {
-        } else {
+        CLOG_LOG(INFO, "ignore log_pool path", K(current_file_path), KPC(this));
-          has_allocated_block_cnt += functor.get_block_count();
-          CLOG_LOG(INFO, "get_has_allocated_blocks_cnt_in_ success", K(ret),
-                   K(current_file_path), "block_cnt", functor.get_block_count());
-        }
-      } else if (OB_FAIL(get_has_allocated_blocks_cnt_in_(current_file_path,
-                                                          has_allocated_block_cnt))) {
-        CLOG_LOG(WARN, "get_has_allocated_blocks_cnt_in_ failed", K(ret),
-                 K(current_file_path), K(has_allocated_block_cnt));
       } else {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
+                      "OceanBase. Please confirm whether manual deletion is required",
+                      ", unexpected directory", current_file_path);
       }
     }
   }
@@ -1586,5 +1582,111 @@ int ObServerLogBlockMgr::force_update_tenant_log_disk(const uint64_t tenant_id,
   }
   return ret;
 }
+
+// the prefix is tenant_xxx
+int ObServerLogBlockMgr::scan_tenant_dir_(const char *tenant_dir,
+                                          int64_t &has_allocated_block_cnt)
+{
+  int ret = OB_SUCCESS;
+  DIR *dir = NULL;
+  std::regex pattern_log_stream(".*/tenant_[1-9]\\d*/[1-9]\\d*");
+  std::regex pattern_tmp_dir(".*/tenant_[1-9]\\d*/tmp_dir");
+  struct dirent *entry = NULL;
+  if (NULL == (dir = opendir(tenant_dir))) {
+    ret = OB_ERR_SYS;
+    CLOG_LOG(WARN, "opendir failed", K(tenant_dir));
+  } else {
+    char current_file_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'};
+    while ((entry = readdir(dir)) != NULL && OB_SUCC(ret)) {
+      bool is_dir = false;
+      MEMSET(current_file_path, '\0', OB_MAX_FILE_NAME_LENGTH);
+      if (0 == strcmp(entry->d_name, ".") || 0 == strcmp(entry->d_name, "..")) {
+        // do nothing
+      } else if (0 >= snprintf(current_file_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s",
+                               tenant_dir, entry->d_name)) {
+        ret = OB_ERR_UNEXPECTED;
+        CLOG_LOG(WARN, "snprintf failed", K(ret), K(current_file_path), K(tenant_dir),
+                K(entry->d_name));
+      } else if (OB_FAIL(FileDirectoryUtils::is_directory(current_file_path, is_dir))) {
+        CLOG_LOG(WARN, "is_directory failed", K(ret), K(entry->d_name));
+      } else if (false == is_dir) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
+                      "OceanBase. Please confirm whether manual deletion is required",
+                      ", unexpected file path", current_file_path);
+      } else if (true == std::regex_match(current_file_path, pattern_log_stream)) {
+        ret = scan_ls_dir_(current_file_path, has_allocated_block_cnt);
+      } else if (true == std::regex_match(current_file_path, pattern_tmp_dir)) {
+        CLOG_LOG(INFO, "ignore tmp_dir", K(current_file_path), K(has_allocated_block_cnt), KPC(this));
+      } else {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
+                      "OceanBase. Please confirm whether manual deletion is required",
+                      ", unexpected directory", current_file_path);
+      }
+    }
+  }
+  if (NULL != dir) {
+    closedir(dir);
+  }
+  return ret;
+}
+
+// the prefix of ls_dir tenant_xxx/xxxx
+int ObServerLogBlockMgr::scan_ls_dir_(const char *ls_dir,
+                                      int64_t &has_allocated_block_cnt)
+{
+  int ret = OB_SUCCESS;
+  DIR *dir = NULL;
+  std::regex pattern_log(".*/tenant_[1-9]\\d*/[1-9]\\d*/log");
+  std::regex pattern_meta(".*/tenant_[1-9]\\d*/[1-9]\\d*/meta");
+  struct dirent *entry = NULL;
+  if (NULL == (dir = opendir(ls_dir))) {
+    ret = OB_ERR_SYS;
+    CLOG_LOG(WARN, "opendir failed", K(ls_dir));
+  } else {
+    char current_file_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'};
+    while ((entry = readdir(dir)) != NULL && OB_SUCC(ret)) {
+      bool is_dir = false;
+      MEMSET(current_file_path, '\0', OB_MAX_FILE_NAME_LENGTH);
+      if (0 == strcmp(entry->d_name, ".") || 0 == strcmp(entry->d_name, "..")) {
+        // do nothing
+      } else if (0 >= snprintf(current_file_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s",
+                               ls_dir, entry->d_name)) {
+        ret = OB_ERR_UNEXPECTED;
+        CLOG_LOG(WARN, "snprintf failed", K(ret), K(current_file_path), K(ls_dir),
+                K(entry->d_name));
+      } else if (OB_FAIL(FileDirectoryUtils::is_directory(current_file_path, is_dir))) {
+        CLOG_LOG(WARN, "is_directory failed", K(ret), K(entry->d_name));
+      } else if (false == is_dir) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
+                      "OceanBase. Please confirm whether manual deletion is required",
+                      ", unexpected file path", current_file_path);
+      } else if (true == std::regex_match(current_file_path, pattern_log)
+                 || true == std::regex_match(current_file_path, pattern_meta)) {
+        GetBlockCountFunctor functor(current_file_path);
+        if (OB_FAIL(palf::scan_dir(current_file_path, functor))) {
+          LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
+                        "OceanBase. Please confirm whether manual deletion is required",
+                        ", log directory", current_file_path);
+        } else {
+          has_allocated_block_cnt += functor.get_block_count();
+          CLOG_LOG(INFO, "get_has_allocated_blocks_cnt_in_ success", K(ret),
+                   K(current_file_path), "block_cnt", functor.get_block_count());
+        }
+      } else {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
+                      "OceanBase. Please confirm whether manual deletion is required",
+                      ", unexpected directory", current_file_path);
+      }
+    }
+  }
+  if (NULL != dir) {
+    closedir(dir);
+  }
+  return ret;
+}
 } // namespace logservice
 } // namespace oceanbase

src/logservice/ob_server_log_block_mgr.h

@@ -294,6 +294,8 @@ private:
                            const int64_t src_buf_len, const int64_t offset);
   int read_unitl_success_(const palf::FileDesc &src_fd, char *dest_buf,
                           const int64_t dest_buf_len, const int64_t offset);
+  int scan_tenant_dir_(const char *tenant_dir, int64_t &has_allocated_block_cnt);
+  int scan_ls_dir_(const char *tenant_dir, int64_t &has_allocated_block_cnt);
 private:
   typedef common::ObFunction<int(int64_t&)> GetTenantsLogDiskSize;
   mutable ObSpinLock log_pool_meta_lock_;

unittest/logservice/test_server_log_block_mgr.cpp

@@ -390,9 +390,11 @@ TEST_F(TestServerLogBlockMgr, dirty_ls_dir_and_log_pool_file)
   system("mkdir clog_disk/clog/log_pool/1.tmp");
   log_block_mgr_.destroy();
   bool result = false;
-  EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_pool_base_path_));
+  EXPECT_EQ(OB_ERR_UNEXPECTED, log_block_mgr_.init(log_pool_base_path_));
   EXPECT_EQ(OB_SUCCESS, FileDirectoryUtils::is_exists("clog_disk/clog/tenant_1/1/meta/10000.tmp",result));
   EXPECT_EQ(false, result);
+  system("rm -rf clog_disk/clog/tenant_0111");
+  EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_pool_base_path_));
 }
 
 TEST_F(TestServerLogBlockMgr, resize_failed_and_restar)