hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fixed remove directory failed because of physical disk full.

Browse Source
This commit is contained in:
HaHaJeff
2023-09-13 05:48:59 +00:00
committed by ob-robot
parent f461e41881
commit 03f4d9d194
6 changed files with 56 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/logservice/palf/log_block_handler.cpp
View File

@ -21,7 +21,7 @@
#include "share/ob_errno.h" // errno
#include "share/rc/ob_tenant_base.h" // mtl_malloc
#include "log_writer_utils.h" // LogWriteBuf
#include "log_io_uitls.h" // close_with_ret
#include "log_io_utils.h" // close_with_ret
namespace oceanbase
{
using namespace common;

2
src/logservice/palf/log_block_mgr.cpp
View File

@ -21,7 +21,7 @@
#include "share/ob_errno.h" // OB_NO_SUCH_FILE_OR_DIRECTORY
#include "log_writer_utils.h" // LogWriteBuf
#include "lsn.h" // LSN
#include "log_io_uitls.h" // openat_with_retry
#include "log_io_utils.h" // openat_with_retry
namespace oceanbase
{

2
src/logservice/palf/log_block_pool_interface.cpp
View File

@ -69,7 +69,7 @@ int remove_directory_rec(const char *path, ILogBlockPool *log_block_pool)
DIR *dir = NULL;
struct dirent *entry = NULL;
if (NULL == (dir = opendir(path))) {
ret = OB_ERR_SYS;
ret = convert_sys_errno();
PALF_LOG(WARN, "opendir failed", K(path));
} else {
char current_file_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'};

21
src/logservice/palf/log_io_utils.cpp
View File

@ -9,7 +9,7 @@
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "log_io_uitls.h"
#include "log_io_utils.h"
#include "log_define.h"
namespace oceanbase
{
@ -54,5 +54,24 @@ int close_with_ret(const int fd)
}
return ret;
}
int rename_with_retry(const char *src_name,
const char *dest_name)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(src_name) || OB_ISNULL(dest_name)) {
ret = OB_INVALID_ARGUMENT;
PALF_LOG(WARN, "invalid argument", KP(src_name), KP(dest_name));
} else {
do {
if (-1 == ::rename(src_name, dest_name)) {
ret = convert_sys_errno();
PALF_LOG(WARN, "rename file failed", KR(ret), K(src_name), K(dest_name));
ob_usleep(RETRY_INTERVAL);
}
} while(OB_ALLOCATE_DISK_SPACE_FAILED == ret);
}
return ret;
}
} // end namespace palf
} // end namespace oceanbase

2
src/logservice/palf/log_io_uitls.h → src/logservice/palf/log_io_utils.h
View File

@ -24,6 +24,8 @@ int openat_with_retry(const int dir_fd,
int &fd);
int close_with_ret(const int fd);
int rename_with_retry(const char *src_name, const char *dest_name);
} // end namespace palf
} // end namespace oceanbase
#endif

48
src/logservice/palf/palf_env_impl.cpp
View File

@ -29,6 +29,7 @@
#include "log_loop_thread.h"
#include "log_rpc.h"
#include "log_block_pool_interface.h"
#include "log_io_utils.h"
namespace oceanbase
{
@ -419,9 +420,8 @@ int PalfEnvImpl::create_palf_handle_impl_(const int64_t palf_id,
// NB: always insert value into hash map finally.
} else if (OB_FAIL(palf_handle_impl_map_.insert_and_get(hash_map_key, palf_handle_impl))) {
PALF_LOG(WARN, "palf_handle_impl_map_ insert_and_get failed", K(ret), K(palf_id));
} else if (OB_FAIL(palf_handle_impl->set_monitor_cb(monitor_))) {
PALF_LOG(WARN, "set_monitor_cb failed", K(ret), K(palf_id), KP_(monitor));
} else {
(void) palf_handle_impl->set_monitor_cb(monitor_);
palf_handle_impl->set_scan_disk_log_finished();
ipalf_handle_impl = palf_handle_impl;
}
@ -548,8 +548,7 @@ int PalfEnvImpl::create_directory(const char *base_dir)
} else if (-1 == (::mkdir(meta_dir, mode))) {
ret = convert_sys_errno();
PALF_LOG(WARN, "mkdir failed", K(ret), K(errno), K(tmp_base_dir), K(base_dir));
} else if (-1 == (::rename(tmp_base_dir, base_dir))) {
ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(rename_with_retry(tmp_base_dir, base_dir))) {
PALF_LOG(ERROR, "rename tmp dir to normal dir failed", K(ret), K(errno), K(tmp_base_dir), K(base_dir));
} else if (OB_FAIL(FileDirectoryUtils::fsync_dir(log_dir_))) {
PALF_LOG(ERROR, "fsync_dir failed", K(ret), K(errno), K(tmp_base_dir), K(base_dir));
@ -557,8 +556,8 @@ int PalfEnvImpl::create_directory(const char *base_dir)
PALF_LOG(INFO, "prepare_directory_for_creating_ls success", K(ret), K(base_dir));
}
if (OB_FAIL(ret)) {
remove_directory_rec(tmp_base_dir, log_block_pool_);
remove_directory_rec(base_dir, log_block_pool_);
remove_directory(tmp_base_dir);
remove_directory(base_dir);
}
return ret;
}
@ -566,8 +565,8 @@ int PalfEnvImpl::create_directory(const char *base_dir)
// step:
// 1. rename log directory to tmp directory.
// 2. delete tmp directory.
// NB: '%s.tmp' is invalid block or invalid directory, before the restart phash of PalfEnvImpl,
// need delete thses tmp block or directory.
// NB: '%s.tmp' is invalid block or invalid directory, before the restart phase of PalfEnvImpl,
// need delete these tmp block or directory.
int PalfEnvImpl::remove_directory(const char *log_dir)
{
int ret = OB_SUCCESS;
@ -576,16 +575,28 @@ int PalfEnvImpl::remove_directory(const char *log_dir)
if (0 > (pret = snprintf(tmp_log_dir, MAX_PATH_SIZE, "%s%s", log_dir, TMP_SUFFIX))) {
ret = OB_ERR_UNEXPECTED;
PALF_LOG(ERROR, "snprintf failed", K(ret), K(pret), K(log_dir), K(tmp_log_dir));
} else if (-1 == ::rename(log_dir, tmp_log_dir)) {
ret = convert_sys_errno();
PALF_LOG(ERROR, "rename log dir to tmp dir failed", K(ret), K(errno), K(tmp_log_dir), K(log_dir));
} else if (OB_FAIL(remove_directory_rec(tmp_log_dir, log_block_pool_))) {
PALF_LOG(WARN, "remove_directory_rec failed", K(ret), K(log_dir), K(errno));
} else if (OB_FAIL(FileDirectoryUtils::fsync_dir(log_dir_))) {
PALF_LOG(ERROR, "fsync_dir failed", K(ret), KPC(this), K(log_dir));
} else if (OB_FAIL(rename_with_retry(log_dir, tmp_log_dir))) {
PALF_LOG(WARN, "rename log dir to tmp dir failed", K(ret), K(errno), K(tmp_log_dir), K(log_dir));
} else {
PALF_LOG(WARN, "remove success", K(ret), K(log_dir));
bool result = true;
do {
if (OB_FAIL(FileDirectoryUtils::is_exists(tmp_log_dir, result))) {
CLOG_LOG(WARN, "check directory exists failed", KPC(this), K(log_dir));
} else if (!result) {
CLOG_LOG(WARN, "directory not exists", KPC(this), K(log_dir));
break;
} else if (OB_FAIL(remove_directory_rec(tmp_log_dir, log_block_pool_))) {
PALF_LOG(WARN, "remove_directory_rec failed", K(tmp_log_dir), KP(log_block_pool_));
} else {
}
if (OB_FAIL(ret) && true == result) {
PALF_LOG(WARN, "remove directory failed, may be physical disk full", K(ret), KPC(this));
usleep(100*1000);
}
} while (OB_FAIL(ret));
}
(void)FileDirectoryUtils::fsync_dir(log_dir_);
PALF_LOG(WARN, "remove_directory finished", KR(ret), K(log_dir), KP(this));
return ret;
}
@ -1218,9 +1229,8 @@ int PalfEnvImpl::move_incomplete_palf_into_tmp_dir_(const int64_t palf_id)
} else if (0 > (pret = snprintf(dest_log_dir, MAX_PATH_SIZE, "%s/%ld_%ld", tmp_log_dir_, palf_id, timestamp))) {
ret = OB_ERR_UNEXPECTED;
PALF_LOG(ERROR, "snprintf failed, unexpected error", K(ret));
} else if (-1 == ::rename(src_log_dir, dest_log_dir)) {
ret = convert_sys_errno();
PALF_LOG(ERROR, "::rename failed", K(ret), KPC(this), K(src_log_dir), K(dest_log_dir));
} else if (OB_FAIL(rename_with_retry(src_log_dir, dest_log_dir))) {
PALF_LOG(ERROR, "rename failed", K(ret), KPC(this), K(src_log_dir), K(dest_log_dir));
} else if (OB_FAIL(FileDirectoryUtils::fsync_dir(log_dir_))) {
PALF_LOG(ERROR, "fsync_dir failed", K(ret), KPC(this), K(src_log_dir), K(dest_log_dir));
} else {