From 4cc88e0b6e582dc406fdb9eb3110cdd55b89f424 Mon Sep 17 00:00:00 2001 From: HaHaJeff Date: Wed, 7 Feb 2024 23:39:10 +0000 Subject: [PATCH] print error log when rename failed and make thread into dead loop --- src/logservice/palf/log_io_utils.cpp | 111 +++++++++++++++++++--- src/logservice/palf/log_io_utils.h | 11 +++ unittest/logservice/CMakeLists.txt | 1 + unittest/logservice/test_log_io_utils.cpp | 60 ++++++++++++ 4 files changed, 170 insertions(+), 13 deletions(-) create mode 100644 unittest/logservice/test_log_io_utils.cpp diff --git a/src/logservice/palf/log_io_utils.cpp b/src/logservice/palf/log_io_utils.cpp index 6e3bbebf1..f01ffc655 100644 --- a/src/logservice/palf/log_io_utils.cpp +++ b/src/logservice/palf/log_io_utils.cpp @@ -11,6 +11,9 @@ */ #include "log_io_utils.h" #include // FALLOC_FL_ZERO_RANGE for linux kernel 3.15 +#include +#include +#include #include "log_block_pool_interface.h" #include "share/ob_errno.h" #include "logservice/ob_server_log_block_mgr.h" @@ -59,6 +62,86 @@ int close_with_ret(const int fd) return ret; } +int check_file_exist(const char *file_name, + bool &exist) +{ + int ret = OB_SUCCESS; + exist = false; + struct stat64 file_info; + if (OB_ISNULL(file_name) || OB_UNLIKELY(strlen(file_name) == 0)) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid arguments.", KCSTRING(file_name), K(ret)); + } else { + exist = (0 == ::stat64(file_name, &file_info)); + } + return ret; +} + +int check_file_exist(const int dir_fd, + const char *file_name, + bool &exist) +{ + int ret = OB_SUCCESS; + exist = false; + struct stat64 file_info; + const int64_t flag = 0; + if (OB_ISNULL(file_name) || OB_UNLIKELY(strlen(file_name) == 0)) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid arguments.", KCSTRING(file_name), K(ret)); + } else { + exist = (0 == ::fstatat64(dir_fd, file_name, &file_info, flag)); + } + return ret; +} + +bool check_rename_success(const char *src_name, + const char *dest_name) +{ + bool bool_ret = false; + bool src_exist = false; + bool dest_exist = false; + int ret = OB_SUCCESS; + if (OB_FAIL(check_file_exist(src_name, src_exist))) { + PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name)); + } else if (!src_exist && OB_FAIL(check_file_exist(dest_name, dest_exist))) { + PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name)); + } else if (!src_exist && dest_exist) { + bool_ret = true; + PALF_LOG(INFO, "check_rename_success return true", + KR(ret), K(src_name), K(dest_name), K(src_exist), K(dest_exist)); + } else { + bool_ret = false; + LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "rename file failed, unexpected error", + KR(ret), K(errno), K(src_name), K(dest_name), K(src_exist), K(dest_exist)); + } + return bool_ret; +} + +bool check_renameat_success(const int src_dir_fd, + const char *src_name, + const int dest_dir_fd, + const char *dest_name) +{ + bool bool_ret = false; + bool src_exist = false; + bool dest_exist = false; + int ret = OB_SUCCESS; + if (OB_FAIL(check_file_exist(src_dir_fd, src_name, src_exist))) { + PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name)); + } else if (!src_exist && OB_FAIL(check_file_exist(dest_dir_fd, dest_name, dest_exist))) { + PALF_LOG(WARN, "check_file_exist failed", KR(ret), K(src_name), K(dest_name)); + } else if (!src_exist && dest_exist) { + bool_ret = true; + PALF_LOG(INFO, "check_renameat_success return true", + KR(ret), K(src_name), K(dest_name), K(src_dir_fd), K(dest_dir_fd), K(src_exist), K(dest_exist)); + } else { + bool_ret = false; + LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "renameat file failed, unexpected error", + KR(ret), K(errno), K(src_name), K(dest_name), K(src_exist), K(dest_exist)); + } + return bool_ret; +} + int rename_with_retry(const char *src_name, const char *dest_name) { @@ -70,15 +153,16 @@ int rename_with_retry(const char *src_name, do { if (-1 == ::rename(src_name, dest_name)) { ret = convert_sys_errno(); - PALF_LOG(WARN, "rename file failed", KR(ret), K(src_name), K(dest_name)); + LOG_DBA_WARN(OB_IO_ERROR, "msg", "rename file failed", + KR(ret), K(errno), K(src_name), K(dest_name)); // for xfs, source file not exist and dest file exist after rename return ENOSPC, therefore, next rename will return - // OB_NO_SUCH_FILE_OR_DIRECTORY. - if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { + // OB_NO_SUCH_FILE_OR_DIRECTORY, however, for some reason, we can not return OB_SUCCESS when rename return OB_NO_SUCH_FILE_OR_DIRECTORY. + // consider that, if file names with 'src_name' has been delted by human and file names with 'dest_name' not exist. + if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret && check_rename_success(src_name, dest_name)) { ret = OB_SUCCESS; - PALF_LOG(WARN, "rename file failed, source file not exist, return OB_SUCCESS.", K(src_name), K(dest_name)); - } else { - ob_usleep(RETRY_INTERVAL); + break; } + usleep(RETRY_INTERVAL); } } while(OB_FAIL(ret)); } @@ -99,15 +183,16 @@ int renameat_with_retry(const int src_dir_fd, do { if (-1 == ::renameat(src_dir_fd, src_name, dest_dir_fd, dest_name)) { ret = convert_sys_errno(); - PALF_LOG(WARN, "rename file failed", KR(ret), K(src_name), K(dest_name)); - // for xfs, source file not exist and dest file exist after rename return ENOSPC, therefore, next rename will return - // OB_NO_SUCH_FILE_OR_DIRECTORY. - if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret) { + LOG_DBA_WARN(OB_IO_ERROR, "msg", "renameat file failed", + KR(ret), K(errno), K(src_name), K(dest_name), K(src_dir_fd), K(dest_dir_fd)); + // for xfs, source file not exist and dest file exist after renameat return ENOSPC, therefore, next renameat will return + // OB_NO_SUCH_FILE_OR_DIRECTORY, however, for some reason, we can not return OB_SUCCESS when renameat return OB_NO_SUCH_FILE_OR_DIRECTORY. + // consider that, if file names with 'src_name' has been delted by human and file names with 'dest_name' not exist. + if (OB_NO_SUCH_FILE_OR_DIRECTORY == ret && check_renameat_success(src_dir_fd, src_name, dest_dir_fd, dest_name)) { ret = OB_SUCCESS; - PALF_LOG(WARN, "rename file failed, source file not exist, return OB_SUCCESS.", K(src_name), K(dest_name)); - } else { - ob_usleep(RETRY_INTERVAL); + break; } + ob_usleep(RETRY_INTERVAL); } } while(OB_FAIL(ret)); } diff --git a/src/logservice/palf/log_io_utils.h b/src/logservice/palf/log_io_utils.h index bdb306885..8fbb43951 100644 --- a/src/logservice/palf/log_io_utils.h +++ b/src/logservice/palf/log_io_utils.h @@ -86,6 +86,17 @@ private: }; int reuse_block_at(const int fd, const char *block_path); +bool check_rename_success(const char *src_name, + const char *dest_name); +bool check_renameat_success(const int src_dir_fd, + const char *src_name, + const int dest_dir_fd, + const char *dest_name); +int check_file_exist(const char *file_name, + bool &exist); +int check_file_exist(const int dir_fd, + const char *file_name, + bool &exist); } // end namespace palf } // end namespace oceanbase diff --git a/unittest/logservice/CMakeLists.txt b/unittest/logservice/CMakeLists.txt index e77f08dd8..d0074e17a 100644 --- a/unittest/logservice/CMakeLists.txt +++ b/unittest/logservice/CMakeLists.txt @@ -40,6 +40,7 @@ ob_unittest(test_palf_throttling) ob_unittest(test_net_standby_restore_source) ob_unittest(test_log_external_storage_handler) ob_unittest(test_log_external_storage_io_task) +ob_unittest(test_log_io_utils) if(OB_BUILD_CLOSE_MODULES) ob_unittest(test_arb_gc_utils) ob_unittest(test_ob_arbitration_service) diff --git a/unittest/logservice/test_log_io_utils.cpp b/unittest/logservice/test_log_io_utils.cpp new file mode 100644 index 000000000..e11f46bc2 --- /dev/null +++ b/unittest/logservice/test_log_io_utils.cpp @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "lib/ob_define.h" +#include "logservice/palf/log_io_utils.h" + +namespace oceanbase +{ +namespace palf +{ +using namespace common; +using namespace palf; + +TEST(TestLogIOUtils, test_rename) +{ + EXPECT_EQ(OB_INVALID_ARGUMENT, renameat_with_retry(-1, NULL, -1, NULL)); + EXPECT_EQ(OB_INVALID_ARGUMENT, rename_with_retry(NULL, NULL)); + + const char *src_name = "src_file"; + const char *dest_name = "src_file.tmp"; + const char *curr_dir_name = get_current_dir_name(); + int dir_fd = 0; + system("rm -rf src_file*"); + EXPECT_LE(0, dir_fd = ::open(curr_dir_name, O_DIRECTORY | O_RDONLY)); + EXPECT_EQ(false, check_rename_success(src_name, dest_name)); + EXPECT_EQ(false, check_renameat_success(dir_fd, src_name, dir_fd, dest_name)); + const char *touch_dest_cmd = "touch src_file.tmp"; + const char *touch_src_cmd = "touch src_file"; + system(touch_dest_cmd); + EXPECT_EQ(true, check_rename_success(src_name, dest_name)); + EXPECT_EQ(true, check_renameat_success(dir_fd, src_name, dir_fd, dest_name)); + system(touch_src_cmd); + EXPECT_EQ(false, check_rename_success(src_name, dest_name)); + EXPECT_EQ(false, check_renameat_success(dir_fd, src_name, dir_fd, dest_name)); + EXPECT_EQ(OB_SUCCESS, rename_with_retry(src_name, dest_name)); + system(touch_src_cmd); + EXPECT_EQ(OB_SUCCESS, rename_with_retry(src_name, dest_name)); +} + +} // end namespace unittest +} // end namespace oceanbase + +int main(int argc, char **argv) +{ + OB_LOGGER.set_file_name("test_log_io_utils.log", true); + OB_LOGGER.set_log_level("TRACE"); + PALF_LOG(INFO, "begin unittest::test_log_io_utils"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}