hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

print error log when rename failed and make thread into dead loop

Browse Source
This commit is contained in:
HaHaJeff 2024-02-07 23:39:10 +00:00 committed by ob-robot
parent cbaed137f1
commit 4cc88e0b6e
4 changed files with 170 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
src/logservice/palf/log_io_utils.cpp
View File

@ -11,6 +11,9 @@
*/
#include "log_io_utils.h"
#include <linux/falloc.h> // FALLOC_FL_ZERO_RANGE for linux kernel 3.15
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "log_block_pool_interface.h"
#include "share/ob_errno.h"
#include "logservice/ob_server_log_block_mgr.h"
@ -59,6 +62,86 @@ int close_with_ret(const int fd)
return ret;
}
int check_file_exist(const char *file_name,
bool &exist)
{
int ret = OB_SUCCESS;
exist = false;
struct stat64 file_info;
if (OB_ISNULL(file_name) || OB_UNLIKELY(strlen(file_name) == 0)) {
ret = OB_INVALID_ARGUMENT;
PALF_LOG(WARN, "invalid arguments.", KCSTRING(file_name), K(ret));
} else {
exist = (0 == ::stat64(file_name, &file_info));
}
return ret;
}
int check_file_exist(const int dir_fd,
const char *file_name,
bool &exist)
{
int ret = OB_SUCCESS;
exist = false;
struct stat64 file_info;
const int64_t flag = 0;
if (OB_ISNULL(file_name) || OB_UNLIKELY(strlen(file_name) == 0)) {
ret = OB_INVALID_ARGUMENT;
PALF_LOG(WARN, "invalid arguments.", KCSTRING(file_name), K(ret));
} else {
exist = (0 == ::fstatat64(dir_fd, file_name, &file_info, flag));
}
return ret;
}
bool check_rename_success(const char *src_name,
const char *dest_name)
{
bool bool_ret = false;
bool src_exist = false;
bool dest_exist = false;
int ret = OB_SUCCESS;
if (OB_FAIL(check_file_exist(src_name, src_exist))) {
PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name));
} else if (!src_exist && OB_FAIL(check_file_exist(dest_name, dest_exist))) {
PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name));
} else if (!src_exist && dest_exist) {
bool_ret = true;
PALF_LOG(INFO, "check_rename_success return true",
KR(ret), K(src_name), K(dest_name), K(src_exist), K(dest_exist));
} else {
bool_ret = false;
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "rename file failed, unexpected error",
KR(ret), K(errno), K(src_name), K(dest_name), K(src_exist), K(dest_exist));
}
return bool_ret;
}
bool check_renameat_success(const int src_dir_fd,
const char *src_name,
const int dest_dir_fd,
const char *dest_name)
{
bool bool_ret = false;
bool src_exist = false;
bool dest_exist = false;
int ret = OB_SUCCESS;
if (OB_FAIL(check_file_exist(src_dir_fd, src_name, src_exist))) {
PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name));
} else if (!src_exist && OB_FAIL(check_file_exist(dest_dir_fd, dest_name, dest_exist))) {
PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name));
} else if (!src_exist && dest_exist) {
bool_ret = true;
PALF_LOG(INFO, "check_renameat_success return true",
KR(ret), K(src_name), K(dest_name), K(src_dir_fd), K(dest_dir_fd), K(src_exist), K(dest_exist));
} else {
bool_ret = false;
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "renameat file failed, unexpected error",
KR(ret), K(errno), K(src_name), K(dest_name), K(src_exist), K(dest_exist));
}
return bool_ret;
}
int rename_with_retry(const char *src_name,
const char *dest_name)
{
@ -70,15 +153,16 @@ int rename_with_retry(const char *src_name,
do {
if (-1 == ::rename(src_name, dest_name)) {
ret = convert_sys_errno();
PALF_LOG(WARN, "rename file failed", KR(ret), K(src_name), K(dest_name));
LOG_DBA_WARN(OB_IO_ERROR, "msg", "rename file failed",
KR(ret), K(errno), K(src_name), K(dest_name));
// for xfs, source file not exist and dest file exist after rename return ENOSPC, therefore, next rename will return
// OB_NO_SUCH_FILE_OR_DIRECTORY.
if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) {
// OB_NO_SUCH_FILE_OR_DIRECTORY, however, for some reason, we can not return OB_SUCCESS when rename return OB_NO_SUCH_FILE_OR_DIRECTORY.
// consider that, if file names with 'src_name' has been delted by human and file names with 'dest_name' not exist.
if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret && check_rename_success(src_name, dest_name)) {
ret = OB_SUCCESS;
PALF_LOG(WARN, "rename file failed, source file not exist, return OB_SUCCESS.", K(src_name), K(dest_name));
} else {
ob_usleep(RETRY_INTERVAL);
break;
}
usleep(RETRY_INTERVAL);
}
} while(OB_FAIL(ret));
}
@ -99,15 +183,16 @@ int renameat_with_retry(const int src_dir_fd,
do {
if (-1 == ::renameat(src_dir_fd, src_name, dest_dir_fd, dest_name)) {
ret = convert_sys_errno();
PALF_LOG(WARN, "rename file failed", KR(ret), K(src_name), K(dest_name));
// for xfs, source file not exist and dest file exist after rename return ENOSPC, therefore, next rename will return
// OB_NO_SUCH_FILE_OR_DIRECTORY.
if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) {
LOG_DBA_WARN(OB_IO_ERROR, "msg", "renameat file failed",
KR(ret), K(errno), K(src_name), K(dest_name), K(src_dir_fd), K(dest_dir_fd));
// for xfs, source file not exist and dest file exist after renameat return ENOSPC, therefore, next renameat will return
// OB_NO_SUCH_FILE_OR_DIRECTORY, however, for some reason, we can not return OB_SUCCESS when renameat return OB_NO_SUCH_FILE_OR_DIRECTORY.
// consider that, if file names with 'src_name' has been delted by human and file names with 'dest_name' not exist.
if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret && check_renameat_success(src_dir_fd, src_name, dest_dir_fd, dest_name)) {
ret = OB_SUCCESS;
PALF_LOG(WARN, "rename file failed, source file not exist, return OB_SUCCESS.", K(src_name), K(dest_name));
} else {
ob_usleep(RETRY_INTERVAL);
break;
}
ob_usleep(RETRY_INTERVAL);
}
} while(OB_FAIL(ret));
}

11
src/logservice/palf/log_io_utils.h
View File

@ -86,6 +86,17 @@ private:
};
int reuse_block_at(const int fd, const char *block_path);
bool check_rename_success(const char *src_name,
const char *dest_name);
bool check_renameat_success(const int src_dir_fd,
const char *src_name,
const int dest_dir_fd,
const char *dest_name);
int check_file_exist(const char *file_name,
bool &exist);
int check_file_exist(const int dir_fd,
const char *file_name,
bool &exist);
} // end namespace palf
} // end namespace oceanbase

1
unittest/logservice/CMakeLists.txt
View File

@ -40,6 +40,7 @@ ob_unittest(test_palf_throttling)
ob_unittest(test_net_standby_restore_source)
ob_unittest(test_log_external_storage_handler)
ob_unittest(test_log_external_storage_io_task)
ob_unittest(test_log_io_utils)
if(OB_BUILD_CLOSE_MODULES)
ob_unittest(test_arb_gc_utils)
ob_unittest(test_ob_arbitration_service)

60
unittest/logservice/test_log_io_utils.cpp Normal file
View File

@ -0,0 +1,60 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include "lib/ob_define.h"
#include "logservice/palf/log_io_utils.h"
namespace oceanbase
{
namespace palf
{
using namespace common;
using namespace palf;
TEST(TestLogIOUtils, test_rename)
{
EXPECT_EQ(OB_INVALID_ARGUMENT, renameat_with_retry(-1, NULL, -1, NULL));
EXPECT_EQ(OB_INVALID_ARGUMENT, rename_with_retry(NULL, NULL));
const char *src_name = "src_file";
const char *dest_name = "src_file.tmp";
const char *curr_dir_name = get_current_dir_name();
int dir_fd = 0;
system("rm -rf src_file*");
EXPECT_LE(0, dir_fd = ::open(curr_dir_name, O_DIRECTORY | O_RDONLY));
EXPECT_EQ(false, check_rename_success(src_name, dest_name));
EXPECT_EQ(false, check_renameat_success(dir_fd, src_name, dir_fd, dest_name));
const char *touch_dest_cmd = "touch src_file.tmp";
const char *touch_src_cmd = "touch src_file";
system(touch_dest_cmd);
EXPECT_EQ(true, check_rename_success(src_name, dest_name));
EXPECT_EQ(true, check_renameat_success(dir_fd, src_name, dir_fd, dest_name));
system(touch_src_cmd);
EXPECT_EQ(false, check_rename_success(src_name, dest_name));
EXPECT_EQ(false, check_renameat_success(dir_fd, src_name, dir_fd, dest_name));
EXPECT_EQ(OB_SUCCESS, rename_with_retry(src_name, dest_name));
system(touch_src_cmd);
EXPECT_EQ(OB_SUCCESS, rename_with_retry(src_name, dest_name));
}
} // end namespace unittest
} // end namespace oceanbase
int main(int argc, char **argv)
{
OB_LOGGER.set_file_name("test_log_io_utils.log", true);
OB_LOGGER.set_log_level("TRACE");
PALF_LOG(INFO, "begin unittest::test_log_io_utils");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}