print error log when rename failed and make thread into dead loop

src/logservice/palf/log_io_utils.cpp

@@ -11,6 +11,9 @@
  */
 #include "log_io_utils.h"
 #include <linux/falloc.h> // FALLOC_FL_ZERO_RANGE for linux kernel 3.15
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
 #include "log_block_pool_interface.h"
 #include "share/ob_errno.h"
 #include "logservice/ob_server_log_block_mgr.h"
@@ -59,6 +62,86 @@ int close_with_ret(const int fd)
   return ret;
 }
 
+int check_file_exist(const char *file_name,
+                     bool &exist)
+{
+  int ret = OB_SUCCESS;
+  exist = false;
+  struct stat64 file_info;
+  if (OB_ISNULL(file_name) || OB_UNLIKELY(strlen(file_name) == 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    PALF_LOG(WARN, "invalid arguments.", KCSTRING(file_name), K(ret));
+  } else {
+    exist = (0 == ::stat64(file_name, &file_info));
+  }
+  return ret;
+}
+
+int check_file_exist(const int dir_fd,
+                     const char *file_name,
+                     bool &exist)
+{
+  int ret = OB_SUCCESS;
+  exist = false;
+  struct stat64 file_info;
+  const int64_t flag = 0;
+  if (OB_ISNULL(file_name) || OB_UNLIKELY(strlen(file_name) == 0)) {
+    ret = OB_INVALID_ARGUMENT;
+    PALF_LOG(WARN, "invalid arguments.", KCSTRING(file_name), K(ret));
+  } else {
+    exist = (0 == ::fstatat64(dir_fd, file_name, &file_info, flag));
+  }
+  return ret;
+}
+
+bool check_rename_success(const char *src_name,
+                          const char *dest_name)
+{
+  bool bool_ret = false;
+  bool src_exist = false;
+  bool dest_exist = false;
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(check_file_exist(src_name, src_exist))) {
+    PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name));
+  } else if (!src_exist && OB_FAIL(check_file_exist(dest_name, dest_exist))) {
+    PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name));
+  } else if (!src_exist && dest_exist) {
+    bool_ret = true;
+    PALF_LOG(INFO, "check_rename_success return true",
+             KR(ret), K(src_name), K(dest_name), K(src_exist), K(dest_exist));
+  } else {
+    bool_ret = false;
+    LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "rename file failed, unexpected error",
+                  KR(ret), K(errno), K(src_name), K(dest_name), K(src_exist), K(dest_exist));
+  }
+  return bool_ret;
+}
+
+bool check_renameat_success(const int src_dir_fd,
+                            const char *src_name,
+                            const int dest_dir_fd,
+                            const char *dest_name)
+{
+  bool bool_ret = false;
+  bool src_exist = false;
+  bool dest_exist = false;
+  int ret = OB_SUCCESS;
+  if (OB_FAIL(check_file_exist(src_dir_fd, src_name, src_exist))) {
+    PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name));
+  } else if (!src_exist && OB_FAIL(check_file_exist(dest_dir_fd, dest_name, dest_exist))) {
+    PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name));
+  } else if (!src_exist && dest_exist) {
+    bool_ret = true;
+    PALF_LOG(INFO, "check_renameat_success return true",
+             KR(ret), K(src_name), K(dest_name), K(src_dir_fd), K(dest_dir_fd), K(src_exist), K(dest_exist));
+  } else {
+    bool_ret = false;
+    LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "renameat file failed, unexpected error",
+                  KR(ret), K(errno), K(src_name), K(dest_name), K(src_exist), K(dest_exist));
+  }
+  return bool_ret;
+}
+
 int rename_with_retry(const char *src_name,
                       const char *dest_name)
 {
@@ -70,15 +153,16 @@ int rename_with_retry(const char *src_name,
     do {
       if (-1 == ::rename(src_name, dest_name)) {
         ret  = convert_sys_errno();
-        PALF_LOG(WARN, "rename file failed", KR(ret), K(src_name), K(dest_name));
+        LOG_DBA_WARN(OB_IO_ERROR, "msg", "rename file failed",
+                     KR(ret), K(errno), K(src_name), K(dest_name));
         // for xfs, source file not exist and dest file exist after rename return ENOSPC, therefore, next rename will return
-        // OB_NO_SUCH_FILE_OR_DIRECTORY.
+        // OB_NO_SUCH_FILE_OR_DIRECTORY, however, for some reason, we can not return OB_SUCCESS when rename return OB_NO_SUCH_FILE_OR_DIRECTORY.
-        if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) {
+        // consider that, if file names with 'src_name' has been delted by human and file names with 'dest_name' not exist.
+        if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret && check_rename_success(src_name, dest_name)) {
           ret = OB_SUCCESS;
-          PALF_LOG(WARN, "rename file failed, source file not exist, return OB_SUCCESS.", K(src_name), K(dest_name));
+          break;
-        } else {
-          ob_usleep(RETRY_INTERVAL);
         }
+        usleep(RETRY_INTERVAL);
       }
     } while(OB_FAIL(ret));
   }
@@ -99,15 +183,16 @@ int renameat_with_retry(const int src_dir_fd,
     do {
       if (-1 == ::renameat(src_dir_fd, src_name, dest_dir_fd, dest_name)) {
         ret  = convert_sys_errno();
-        PALF_LOG(WARN, "rename file failed", KR(ret), K(src_name), K(dest_name));
+        LOG_DBA_WARN(OB_IO_ERROR, "msg", "renameat file failed",
-        // for xfs, source file not exist and dest file exist after rename return ENOSPC, therefore, next rename will return
+                     KR(ret), K(errno), K(src_name), K(dest_name), K(src_dir_fd), K(dest_dir_fd));
-        // OB_NO_SUCH_FILE_OR_DIRECTORY.
+        // for xfs, source file not exist and dest file exist after renameat return ENOSPC, therefore, next renameat will return
-        if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) {
+        // OB_NO_SUCH_FILE_OR_DIRECTORY, however, for some reason, we can not return OB_SUCCESS when renameat return OB_NO_SUCH_FILE_OR_DIRECTORY.
+        // consider that, if file names with 'src_name' has been delted by human and file names with 'dest_name' not exist.
+        if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret && check_renameat_success(src_dir_fd, src_name, dest_dir_fd, dest_name)) {
           ret = OB_SUCCESS;
-          PALF_LOG(WARN, "rename file failed, source file not exist, return OB_SUCCESS.", K(src_name), K(dest_name));
+          break;
-        } else {
-          ob_usleep(RETRY_INTERVAL);
         }
+        ob_usleep(RETRY_INTERVAL);
       }
     } while(OB_FAIL(ret));
   }

src/logservice/palf/log_io_utils.h

@@ -86,6 +86,17 @@ private:
 };
 
 int reuse_block_at(const int fd, const char *block_path);
+bool check_rename_success(const char *src_name,
+                          const char *dest_name);
+bool check_renameat_success(const int src_dir_fd,
+                            const char *src_name,
+                            const int dest_dir_fd,
+                            const char *dest_name);
+int check_file_exist(const char *file_name,
+                     bool &exist);
+int check_file_exist(const int dir_fd,
+                     const char *file_name,
+                     bool &exist);
 
 } // end namespace palf
 } // end namespace oceanbase

unittest/logservice/CMakeLists.txt

@@ -40,6 +40,7 @@ ob_unittest(test_palf_throttling)
 ob_unittest(test_net_standby_restore_source)
 ob_unittest(test_log_external_storage_handler)
 ob_unittest(test_log_external_storage_io_task)
+ob_unittest(test_log_io_utils)
 if(OB_BUILD_CLOSE_MODULES)
   ob_unittest(test_arb_gc_utils)
   ob_unittest(test_ob_arbitration_service)

unittest/logservice/test_log_io_utils.cpp

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include <gtest/gtest.h>
+#include "lib/ob_define.h"
+#include "logservice/palf/log_io_utils.h"
+
+namespace oceanbase
+{
+namespace palf
+{
+using namespace common;
+using namespace palf;
+
+TEST(TestLogIOUtils, test_rename)
+{
+  EXPECT_EQ(OB_INVALID_ARGUMENT, renameat_with_retry(-1, NULL, -1, NULL));
+  EXPECT_EQ(OB_INVALID_ARGUMENT, rename_with_retry(NULL, NULL));
+
+  const char *src_name = "src_file";
+  const char *dest_name = "src_file.tmp";
+  const char *curr_dir_name  = get_current_dir_name();
+  int dir_fd = 0;
+  system("rm -rf src_file*");
+  EXPECT_LE(0, dir_fd = ::open(curr_dir_name, O_DIRECTORY | O_RDONLY));
+  EXPECT_EQ(false, check_rename_success(src_name, dest_name));
+  EXPECT_EQ(false, check_renameat_success(dir_fd, src_name, dir_fd, dest_name));
+  const char *touch_dest_cmd = "touch src_file.tmp";
+  const char *touch_src_cmd = "touch src_file";
+  system(touch_dest_cmd);
+  EXPECT_EQ(true, check_rename_success(src_name, dest_name));
+  EXPECT_EQ(true, check_renameat_success(dir_fd, src_name, dir_fd, dest_name));
+  system(touch_src_cmd);
+  EXPECT_EQ(false, check_rename_success(src_name, dest_name));
+  EXPECT_EQ(false, check_renameat_success(dir_fd, src_name, dir_fd, dest_name));
+  EXPECT_EQ(OB_SUCCESS, rename_with_retry(src_name, dest_name));
+  system(touch_src_cmd);
+  EXPECT_EQ(OB_SUCCESS, rename_with_retry(src_name, dest_name));
+}
+
+} // end namespace unittest
+} // end namespace oceanbase
+
+int main(int argc, char **argv)
+{
+  OB_LOGGER.set_file_name("test_log_io_utils.log", true);
+  OB_LOGGER.set_log_level("TRACE");
+  PALF_LOG(INFO, "begin unittest::test_log_io_utils");
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}