hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

implement list_files_recursion for calibrate disk space in shared-storage mode

Browse Source
This commit is contained in:
Revendell
2024-10-15 14:44:16 +00:00
committed by ob-robot
parent 01e0e0ead9
commit a4f4c1b5f2
3 changed files with 129 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
deps/oblib/src/lib/file/file_directory_utils.cpp vendored
View File

@ -15,6 +15,8 @@
#include "lib/ob_define.h"
#include "lib/allocator/ob_malloc.h"
#include "lib/ob_errno.h"
#include "common/ob_smart_call.h"
#include "lib/utility/ob_hang_fatal_error.h"
#include <stdlib.h>
#include <stdio.h>
@ -415,7 +417,7 @@ int FileDirectoryUtils::delete_directory_rec(const char *path)
{
int ret = OB_SUCCESS;
DIR *dir = NULL;
struct dirent *entry;
struct dirent *entry = nullptr;
if (NULL == (dir = opendir(path))) {
if (ENOENT != errno) {
ret = OB_FILE_NOT_OPENED;
@ -437,7 +439,7 @@ int FileDirectoryUtils::delete_directory_rec(const char *path)
} else if (OB_FAIL(FileDirectoryUtils::is_directory(current_file_path, is_dir))) {
LIB_LOG(WARN, "is_directory failed", K(ret), K(entry->d_name));
// delecte directory recursive
} else if (true == is_dir && OB_FAIL(delete_directory_rec(current_file_path))) {
} else if (true == is_dir && OB_FAIL(SMART_CALL(delete_directory_rec(current_file_path)))) {
LIB_LOG(WARN, "delete directory failed", K(ret), K(entry->d_name), K(path));
// delete normal file
} else if (false == is_dir && OB_FAIL(FileDirectoryUtils::delete_file(current_file_path))) {
@ -452,6 +454,7 @@ int FileDirectoryUtils::delete_directory_rec(const char *path)
}
if (NULL != dir) {
closedir(dir);
dir = nullptr;
}
return ret;
}
@ -460,7 +463,7 @@ int FileDirectoryUtils::delete_tmp_file_or_directory_at(const char *path)
{
int ret = OB_SUCCESS;
DIR *dir = NULL;
struct dirent *entry;
struct dirent *entry = nullptr;
if (NULL == (dir = opendir(path))) {
ret = OB_ERR_SYS;
LIB_LOG(WARN, "opendir failed", K(path));

109
src/share/ob_io_device_helper.cpp
View File

@ -23,6 +23,7 @@
#include "share/ob_force_print_log.h"
#include "share/ob_io_device_helper.h"
#include "observer/ob_server_struct.h"
#include "common/ob_smart_call.h"
using namespace oceanbase::common;
@ -105,6 +106,27 @@ int ObGetFileSizeFunctor::func(const dirent *entry)
return ret;
}
/**
* --------------------------------ObScanDirOp------------------------------------
*/
int ObScanDirOp::func(const dirent *entry)
{
int ret = OB_SUCCESS;
return ret;
}
int ObScanDirOp::set_dir(const char *dir)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(dir == nullptr || strlen(dir) >= common::MAX_PATH_SIZE)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(dir));
} else {
dir_ = dir;
}
return ret;
}
/**
* --------------------------------ObSNIODeviceWrapper------------------------------------
*/
@ -336,7 +358,7 @@ int ObIODeviceLocalFileOp::scan_dir(const char *dir_name, int (*func)(const dire
int ret = OB_SUCCESS;
DIR *open_dir = nullptr;
struct dirent entry;
struct dirent *result;
struct dirent *result = nullptr;
if (OB_ISNULL(dir_name)) {
ret = OB_INVALID_ARGUMENT;
@ -367,6 +389,7 @@ int ObIODeviceLocalFileOp::scan_dir(const char *dir_name, int (*func)(const dire
//close dir
if (NULL != open_dir) {
::closedir(open_dir);
open_dir = nullptr;
}
}
return ret;
@ -384,7 +407,7 @@ int ObIODeviceLocalFileOp::scan_dir(const char *dir_name, common::ObBaseDirEntry
int ret = OB_SUCCESS;
DIR *open_dir = nullptr;
struct dirent entry;
struct dirent *result;
struct dirent *result = nullptr;
if (OB_ISNULL(dir_name)) {
ret = OB_INVALID_ARGUMENT;
@ -415,11 +438,93 @@ int ObIODeviceLocalFileOp::scan_dir(const char *dir_name, common::ObBaseDirEntry
//close dir
if (NULL != open_dir) {
::closedir(open_dir);
open_dir = nullptr;
}
}
return ret;
}
/*
* scan_dir_rec is scan directory recursion,
* you need to provide file_op and dir_op, dir_op is used to operate directory, file_op is used to operate other files, for example reg file, link file.
*/
int ObIODeviceLocalFileOp::scan_dir_rec(const char *dir_name,
ObScanDirOp &file_op,
ObScanDirOp &dir_op)
{
int ret = OB_SUCCESS;
DIR *open_dir = nullptr;
struct dirent entry;
struct dirent *result = nullptr;
if (OB_ISNULL(dir_name)) {
ret = OB_INVALID_ARGUMENT;
SHARE_LOG(WARN, "invalid argument", K(ret), K(dir_name));
} else if (OB_ISNULL(open_dir = ::opendir(dir_name))) {
if (ENOENT != errno) {
ret = OB_FILE_NOT_OPENED;
SHARE_LOG(WARN, "fail to open dir", K(ret), K(dir_name));
} else {
ret = OB_NO_SUCH_FILE_OR_DIRECTORY;
SHARE_LOG(WARN, "dir does not exist", K(ret), K(dir_name));
}
} else if (OB_FAIL(file_op.set_dir(dir_name))) {
SHARE_LOG(WARN, "fail to set dir", K(ret), K(dir_name));
} else if (OB_FAIL(dir_op.set_dir(dir_name))) {
SHARE_LOG(WARN, "fail to set dir", K(ret), K(dir_name));
} else {
char current_file_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'};
while (OB_SUCC(ret) && nullptr != open_dir) {
if (0 != ::readdir_r(open_dir, &entry, &result)) {
ret = convert_sys_errno();
SHARE_LOG(WARN, "read dir error", K(ret), KERRMSG);
} else if (nullptr != result
&& 0 != STRCMP(entry.d_name, ".")
&& 0 != STRCMP(entry.d_name, "..")) {
bool is_dir = false;
MEMSET(current_file_path, '\0', OB_MAX_FILE_NAME_LENGTH);
int pret = snprintf(current_file_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s", dir_name, entry.d_name);
if (pret <= 0 || pret >= OB_MAX_FILE_NAME_LENGTH) {
ret = OB_BUF_NOT_ENOUGH;
SHARE_LOG(WARN, "snprintf failed", K(ret), K(current_file_path), K(dir_name), K(entry.d_name));
} else if (DT_DIR == entry.d_type) {
is_dir = true;
} else if (DT_UNKNOWN == entry.d_type) {
ObIODFileStat statbuf;
if (OB_FAIL(ObIODeviceLocalFileOp::stat(current_file_path, statbuf))) {
SHARE_LOG(WARN, "fail to stat file", K(ret), K(current_file_path));
} else if (S_ISDIR(statbuf.mode_)) {
is_dir = true;
}
}
if (OB_FAIL(ret)) {
} else if (false == is_dir) {
if (OB_FAIL(file_op.func(&entry))) {
SHARE_LOG(WARN, "fail to operate file entry", K(ret), K(dir_name), K(entry.d_name));
}
} else if (true == is_dir) {
if (OB_FAIL(SMART_CALL(scan_dir_rec(current_file_path, file_op, dir_op)))) {
SHARE_LOG(WARN, "scan directory failed", K(ret), K(current_file_path));
} else if (OB_FAIL(file_op.set_dir(dir_name))) {
SHARE_LOG(WARN, "fail to set dir", K(ret), K(dir_name));
} else if (OB_FAIL(dir_op.set_dir(dir_name))) {
SHARE_LOG(WARN, "fail to set dir", K(ret), K(dir_name));
} else if (OB_FAIL(dir_op.func(&entry))) {
SHARE_LOG(WARN, "fail to operate dir entry", K(ret), K(dir_name), K(entry.d_name));
}
}
} else if (NULL == result) {
break; //end file
}
}
}
//close dir
if (nullptr != open_dir) {
::closedir(open_dir);
open_dir = nullptr;
}
return ret;
}
int ObIODeviceLocalFileOp::fsync(const ObIOFd &fd)
{
int ret = OB_SUCCESS;

16
src/share/ob_io_device_helper.h
View File

@ -65,6 +65,21 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObGetFileSizeFunctor);
};
class ObScanDirOp : public common::ObBaseDirEntryOperator
{
public:
ObScanDirOp(const char *dir = nullptr)
: dir_(dir) {}
virtual ~ObScanDirOp() = default;
virtual int func(const dirent *entry) override;
int set_dir(const char *dir);
const char *get_dir() const { return dir_; }
TO_STRING_KV(K_(dir));
private:
const char *dir_;
DISALLOW_COPY_AND_ASSIGN(ObScanDirOp);
};
/*this class seems like the adapter for local/ofs device*/
class ObSNIODeviceWrapper final
{
@ -147,6 +162,7 @@ public:
static int seal_file(const common::ObIOFd &fd);
static int scan_dir(const char *dir_name, int (*func)(const dirent *entry));
static int scan_dir(const char *dir_name, common::ObBaseDirEntryOperator &op);
static int scan_dir_rec(const char *dir_name, ObScanDirOp &reg_op, ObScanDirOp &dir_op);
static int is_tagging(const char *pathname, bool &is_tagging);
static int fsync(const common::ObIOFd &fd);
static int fdatasync(const common::ObIOFd &fd);