fixed remove directory failed because of physical disk full.

src/logservice/palf/log_block_handler.cpp

@@ -21,7 +21,7 @@
 #include "share/ob_errno.h"                             // errno
 #include "share/rc/ob_tenant_base.h"                    // mtl_malloc
 #include "log_writer_utils.h"                           // LogWriteBuf
-#include "log_io_uitls.h"                               // close_with_ret
+#include "log_io_utils.h"                               // close_with_ret
 namespace oceanbase
 {
 using namespace common;

src/logservice/palf/log_block_mgr.cpp

@@ -21,7 +21,7 @@
 #include "share/ob_errno.h"                             // OB_NO_SUCH_FILE_OR_DIRECTORY
 #include "log_writer_utils.h"                           // LogWriteBuf
 #include "lsn.h"                                        // LSN
-#include "log_io_uitls.h"                               // openat_with_retry
+#include "log_io_utils.h"                               // openat_with_retry
 
 namespace oceanbase
 {

src/logservice/palf/log_block_pool_interface.cpp

@@ -69,7 +69,7 @@ int remove_directory_rec(const char *path, ILogBlockPool *log_block_pool)
   DIR *dir = NULL;
   struct dirent *entry = NULL;
   if (NULL == (dir = opendir(path))) {
-    ret = OB_ERR_SYS;
+    ret = convert_sys_errno();
     PALF_LOG(WARN, "opendir failed", K(path));
   } else {
     char current_file_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'};

src/logservice/palf/log_io_utils.cpp

@@ -9,7 +9,7 @@
  * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
  * See the Mulan PubL v2 for more details.
  */
-#include "log_io_uitls.h"
+#include "log_io_utils.h"
 #include "log_define.h"
 namespace oceanbase
 {
@@ -54,5 +54,24 @@ int close_with_ret(const int fd)
   }
   return ret;
 }
+
+int rename_with_retry(const char *src_name,
+                      const char *dest_name)
+{
+  int ret = OB_SUCCESS;
+  if (OB_ISNULL(src_name) || OB_ISNULL(dest_name)) {
+    ret = OB_INVALID_ARGUMENT;
+    PALF_LOG(WARN, "invalid argument", KP(src_name), KP(dest_name));
+  } else {
+    do {
+      if (-1 == ::rename(src_name, dest_name)) {
+        ret  = convert_sys_errno();
+        PALF_LOG(WARN, "rename file failed", KR(ret), K(src_name), K(dest_name));
+        ob_usleep(RETRY_INTERVAL);
+      }
+    } while(OB_ALLOCATE_DISK_SPACE_FAILED == ret);
+  }
+  return ret;
+}
 } // end namespace palf
 } // end namespace oceanbase

src/logservice/palf/log_io_utils.h

@@ -24,6 +24,8 @@ int openat_with_retry(const int dir_fd,
                       int &fd);
 int close_with_ret(const int fd);
 
+int rename_with_retry(const char *src_name, const char *dest_name);
+
 } // end namespace palf
 } // end namespace oceanbase
 #endif

src/logservice/palf/palf_env_impl.cpp

@@ -29,6 +29,7 @@
 #include "log_loop_thread.h"
 #include "log_rpc.h"
 #include "log_block_pool_interface.h"
+#include "log_io_utils.h"
 
 namespace oceanbase
 {
@@ -419,9 +420,8 @@ int PalfEnvImpl::create_palf_handle_impl_(const int64_t palf_id,
     // NB: always insert value into hash map finally.
   } else if (OB_FAIL(palf_handle_impl_map_.insert_and_get(hash_map_key, palf_handle_impl))) {
     PALF_LOG(WARN, "palf_handle_impl_map_ insert_and_get failed", K(ret), K(palf_id));
-  } else if (OB_FAIL(palf_handle_impl->set_monitor_cb(monitor_))) {
-    PALF_LOG(WARN, "set_monitor_cb failed", K(ret), K(palf_id), KP_(monitor));
   } else {
+    (void) palf_handle_impl->set_monitor_cb(monitor_);
     palf_handle_impl->set_scan_disk_log_finished();
     ipalf_handle_impl = palf_handle_impl;
   }
@@ -548,8 +548,7 @@ int PalfEnvImpl::create_directory(const char *base_dir)
   } else if (-1 == (::mkdir(meta_dir, mode))) {
     ret = convert_sys_errno();
     PALF_LOG(WARN, "mkdir failed", K(ret), K(errno), K(tmp_base_dir), K(base_dir));
-  } else if (-1 == (::rename(tmp_base_dir, base_dir))) {
+  } else if (OB_FAIL(rename_with_retry(tmp_base_dir, base_dir))) {
-    ret = OB_ERR_UNEXPECTED;
     PALF_LOG(ERROR, "rename tmp dir to normal dir failed", K(ret), K(errno), K(tmp_base_dir), K(base_dir));
   } else if (OB_FAIL(FileDirectoryUtils::fsync_dir(log_dir_))) {
     PALF_LOG(ERROR, "fsync_dir failed", K(ret), K(errno), K(tmp_base_dir), K(base_dir));
@@ -557,8 +556,8 @@ int PalfEnvImpl::create_directory(const char *base_dir)
     PALF_LOG(INFO, "prepare_directory_for_creating_ls success", K(ret), K(base_dir));
   }
   if (OB_FAIL(ret)) {
-    remove_directory_rec(tmp_base_dir, log_block_pool_);
+    remove_directory(tmp_base_dir);
-    remove_directory_rec(base_dir, log_block_pool_);
+    remove_directory(base_dir);
   }
   return ret;
 }
@@ -566,8 +565,8 @@ int PalfEnvImpl::create_directory(const char *base_dir)
 // step:
 // 1. rename log directory to tmp directory.
 // 2. delete tmp directory.
-// NB: '%s.tmp' is invalid block or invalid directory, before the restart phash of PalfEnvImpl,
+// NB: '%s.tmp' is invalid block or invalid directory, before the restart phase of PalfEnvImpl,
-//     need delete thses tmp block or directory.
+//     need delete these tmp block or directory.
 int PalfEnvImpl::remove_directory(const char *log_dir)
 {
   int ret = OB_SUCCESS;
@@ -576,16 +575,28 @@ int PalfEnvImpl::remove_directory(const char *log_dir)
   if (0 > (pret = snprintf(tmp_log_dir, MAX_PATH_SIZE, "%s%s", log_dir, TMP_SUFFIX))) {
     ret = OB_ERR_UNEXPECTED;
     PALF_LOG(ERROR, "snprintf failed", K(ret), K(pret), K(log_dir), K(tmp_log_dir));
-  } else if (-1 == ::rename(log_dir, tmp_log_dir)) {
+  } else if (OB_FAIL(rename_with_retry(log_dir, tmp_log_dir))) {
-    ret = convert_sys_errno();
+    PALF_LOG(WARN, "rename log dir to tmp dir failed", K(ret), K(errno), K(tmp_log_dir), K(log_dir));
-    PALF_LOG(ERROR, "rename log dir to tmp dir failed", K(ret), K(errno), K(tmp_log_dir), K(log_dir));
+  } else {
-  } else if (OB_FAIL(remove_directory_rec(tmp_log_dir, log_block_pool_))) {
+    bool result = true;
-    PALF_LOG(WARN, "remove_directory_rec failed", K(ret), K(log_dir), K(errno));
+    do {
-  } else if (OB_FAIL(FileDirectoryUtils::fsync_dir(log_dir_))) {
+      if (OB_FAIL(FileDirectoryUtils::is_exists(tmp_log_dir, result))) {
-    PALF_LOG(ERROR, "fsync_dir failed", K(ret), KPC(this), K(log_dir));
+        CLOG_LOG(WARN, "check directory exists failed", KPC(this), K(log_dir));
+      } else if (!result) {
+        CLOG_LOG(WARN, "directory not exists", KPC(this), K(log_dir));
+        break;
+      } else if (OB_FAIL(remove_directory_rec(tmp_log_dir, log_block_pool_))) {
+        PALF_LOG(WARN, "remove_directory_rec failed", K(tmp_log_dir), KP(log_block_pool_));
       } else {
-    PALF_LOG(WARN, "remove success", K(ret), K(log_dir));
       }
+      if (OB_FAIL(ret) && true == result) {
+        PALF_LOG(WARN, "remove directory failed, may be physical disk full", K(ret), KPC(this));
+        usleep(100*1000);
+      }
+    } while (OB_FAIL(ret));
+  }
+  (void)FileDirectoryUtils::fsync_dir(log_dir_);
+  PALF_LOG(WARN, "remove_directory finished", KR(ret), K(log_dir), KP(this));
   return ret;
 }
 
@@ -1218,9 +1229,8 @@ int PalfEnvImpl::move_incomplete_palf_into_tmp_dir_(const int64_t palf_id)
   } else if (0 > (pret = snprintf(dest_log_dir, MAX_PATH_SIZE, "%s/%ld_%ld", tmp_log_dir_, palf_id, timestamp))) {
     ret = OB_ERR_UNEXPECTED;
     PALF_LOG(ERROR, "snprintf failed, unexpected error", K(ret));
-  } else if (-1 == ::rename(src_log_dir, dest_log_dir)) {
+  } else if (OB_FAIL(rename_with_retry(src_log_dir, dest_log_dir))) {
-    ret = convert_sys_errno();
+    PALF_LOG(ERROR, "rename failed", K(ret), KPC(this), K(src_log_dir), K(dest_log_dir));
-    PALF_LOG(ERROR, "::rename failed", K(ret), KPC(this), K(src_log_dir), K(dest_log_dir));
   } else if (OB_FAIL(FileDirectoryUtils::fsync_dir(log_dir_))) {
     PALF_LOG(ERROR, "fsync_dir failed", K(ret), KPC(this), K(src_log_dir), K(dest_log_dir));
   } else {