hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Check slog disk during restart

Browse Source
This commit is contained in:
JiahuaChen 2024-04-16 08:16:15 +00:00 committed by ob-robot
parent eabb1ecfb0
commit 67d7a0f8de
11 changed files with 60 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mittest/mtlenv/mock_tenant_module_env.h
View File

@ -598,7 +598,7 @@ int MockTenantModuleEnv::init_slogger_mgr(const char *log_dir)
log_file_spec.retry_write_policy_ = "normal";
log_file_spec.log_create_policy_ = "normal";
log_file_spec.log_write_policy_ = "truncate";
if (OB_FAIL(SLOGGERMGR.init(log_dir, MAX_FILE_SIZE, log_file_spec))) {
if (OB_FAIL(SLOGGERMGR.init(log_dir, log_dir, MAX_FILE_SIZE, log_file_spec))) {
STORAGE_LOG(WARN, "fail to init SLOGGERMGR", K(ret));
}

4
src/observer/ob_server.cpp
View File

@ -423,8 +423,8 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg)
} else if (OB_FAIL(ObExternalTableFileManager::get_instance().init())) {
LOG_ERROR("init external table file manager failed", KR(ret));
} else if (OB_FAIL(SLOGGERMGR.init(storage_env_.log_spec_.log_dir_,
storage_env_.log_spec_.max_log_file_size_, storage_env_.slog_file_spec_,
true/*need_reserved*/))) {
storage_env_.sstable_dir_, storage_env_.log_spec_.max_log_file_size_,
storage_env_.slog_file_spec_))) {
LOG_ERROR("init ObStorageLoggerManager failed", KR(ret));
} else if (OB_FAIL(ObVirtualTenantManager::get_instance().init())) {
LOG_ERROR("init tenant manager failed", KR(ret));

38
src/storage/slog/ob_storage_logger_manager.cpp
View File

@ -11,6 +11,7 @@
*/
#define USING_LOG_PREFIX STORAGE_REDO
#include <sys/statvfs.h>
#include "lib/ob_define.h"
#include "lib/ob_running_mode.h"
#include "ob_storage_logger_manager.h"
@ -47,28 +48,29 @@ ObStorageLoggerManager &ObStorageLoggerManager::get_instance()
int ObStorageLoggerManager::init(
const char *log_dir,
const char *data_dir,
const int64_t max_log_file_size,
const blocksstable::ObLogFileSpec &log_file_spec,
const bool need_reserved)
const blocksstable::ObLogFileSpec &log_file_spec)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
STORAGE_REDO_LOG(WARN, "The ObStorageLoggerManager has been inited.", K(ret));
} else if (OB_UNLIKELY(nullptr == log_dir || max_log_file_size <= 0)) {
} else if (OB_UNLIKELY(nullptr == log_dir || nullptr == data_dir || max_log_file_size <= 0)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_REDO_LOG(WARN, "invalid arguments", K(ret), KP(log_dir), K(max_log_file_size));
STORAGE_REDO_LOG(WARN, "invalid arguments", K(ret), KP(log_dir), KP(data_dir), K(max_log_file_size));
} else if (OB_FAIL(prepare_log_buffers(MAX_CONCURRENT_ITEM_CNT, NORMAL_LOG_ITEM_SIZE))) {
STORAGE_REDO_LOG(WARN, "fail to prepare log buffers", K(ret),
LITERAL_K(MAX_CONCURRENT_ITEM_CNT), LITERAL_K(NORMAL_LOG_ITEM_SIZE));
} else if (OB_FAIL(prepare_log_items(MAX_CONCURRENT_ITEM_CNT))) {
STORAGE_REDO_LOG(WARN, "fail to prepare log items", K(ret), LITERAL_K(MAX_CONCURRENT_ITEM_CNT));
} else if (OB_FAIL(check_log_disk(data_dir, log_dir))) {
STORAGE_REDO_LOG(WARN, "fail to set need reserved", K(ret));
} else {
log_dir_ = log_dir;
max_log_file_size_ = max_log_file_size;
log_file_spec_ = log_file_spec;
need_reserved_ = need_reserved;
if (OB_FAIL(server_slogger_.init(*this, OB_SERVER_TENANT_ID))) {
STORAGE_REDO_LOG(WARN, "fail to init server slogger", K(ret));
} else if (OB_FAIL(server_slogger_.start())) {
@ -300,6 +302,32 @@ int ObStorageLoggerManager::get_tenant_slog_dir(
return ret;
}
int ObStorageLoggerManager::check_log_disk(
const char *data_dir,
const char *log_dir)
{
int ret = OB_SUCCESS;
struct statvfs data_svfs;
struct statvfs log_svfs;
need_reserved_ = false;
if (OB_ISNULL(data_dir) || OB_ISNULL(log_dir)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("Invalid argument", K(ret), KP(data_dir), KP(log_dir));
} else if (OB_UNLIKELY(0 != statvfs(data_dir, &data_svfs))) {
ret = OB_IO_ERROR;
LOG_WARN("fail to get sstable directory vfs", K(ret), K(data_dir));
} else if (OB_UNLIKELY(0 != statvfs(log_dir, &log_svfs))) {
ret = OB_IO_ERROR;
LOG_WARN("fail to get slog directory vfs", K(ret), K(log_dir));
} else if (OB_UNLIKELY(0 >= log_svfs.f_bavail)) {
ret = OB_DISK_ERROR;
LOG_ERROR("slog disk is full, please check", K(ret), K(log_dir), K(log_svfs.f_bavail));
} else {
// if slog and data are on the same disk, need reserved space when resize file
need_reserved_ = (data_svfs.f_fsid == log_svfs.f_fsid);
}
return ret;
}
int ObStorageLoggerManager::get_using_disk_space(int64_t &using_space) const
{

6
src/storage/slog/ob_storage_logger_manager.h
View File

@ -33,9 +33,9 @@ public:
static ObStorageLoggerManager &get_instance();
int init(
const char *log_dir,
const char *data_dir,
const int64_t max_log_file_size,
const blocksstable::ObLogFileSpec &log_file_spec,
const bool need_reserved = false);
const blocksstable::ObLogFileSpec &log_file_spec);
void destroy();
// allocate item and its buffer
@ -69,6 +69,8 @@ private:
int get_using_disk_space(int64_t &using_space) const;
int check_log_disk(const char *data_dir, const char *log_dir);
private:
static constexpr int64_t NORMAL_LOG_ITEM_SIZE = 8 * 1024; //8KB
static constexpr int64_t MAX_CONCURRENT_ITEM_CNT = 1024;

2
tools/ob_admin/ob_admin_executor.cpp
View File

@ -156,7 +156,7 @@ int ObAdminExecutor::init_slogger_mgr()
if (OB_FAIL(databuff_printf(slog_dir_, OB_MAX_FILE_NAME_LENGTH, "%s/slog/", data_dir_))) {
LOG_WARN("failed to gen slog dir", K(ret));
} else if (OB_FAIL(SLOGGERMGR.init(slog_dir_, MAX_FILE_SIZE, storage_env_.slog_file_spec_))) {
} else if (OB_FAIL(SLOGGERMGR.init(slog_dir_, sstable_dir_, MAX_FILE_SIZE, storage_env_.slog_file_spec_))) {
STORAGE_LOG(WARN, "fail to init SLOGGERMGR", K(ret));
}

18
unittest/storage/blocksstable/ob_data_file_prepare.h
View File

@ -228,20 +228,22 @@ int TestDataFilePrepareUtil::open()
} else if (0 != system(cmd)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "failed to exec cmd", K(ret), K(cmd), K(errno), KERRMSG);
} else if (OB_FAIL(SLOGGERMGR.init(storage_env_.log_spec_.log_dir_,
storage_env_.log_spec_.max_log_file_size_, storage_env_.slog_file_spec_))) {
STORAGE_LOG(WARN, "ObStorageLoggerManager init failed", KR(ret));
} else if (OB_FAIL(ObKVGlobalCache::get_instance().init(getter_,
bucket_num,
max_cache_size,
block_size))) {
STORAGE_LOG(WARN, "failed to init kv cache", K(ret));
} else if (OB_FAIL(FileDirectoryUtils::create_full_path(clog_dir_))) {
STORAGE_LOG(WARN, "failed to create clog dir", K(ret), K(clog_dir_));
} else if (OB_FAIL(FileDirectoryUtils::create_full_path(slog_dir_))) {
STORAGE_LOG(WARN, "failed to create slog dir", K(ret), K(slog_dir_));
} else if (OB_FAIL(FileDirectoryUtils::create_full_path(file_dir_))) {
STORAGE_LOG(WARN, "failed to create file dir", K(ret), K(file_dir_));
} else if (OB_FAIL(SLOGGERMGR.init(storage_env_.log_spec_.log_dir_,
storage_env_.log_spec_.log_dir_,
storage_env_.log_spec_.max_log_file_size_, storage_env_.slog_file_spec_))) {
STORAGE_LOG(WARN, "ObStorageLoggerManager init failed", KR(ret));
} else if (FALSE_IT(SLOGGERMGR.need_reserved_ = false)) {
} else if (OB_FAIL(ObKVGlobalCache::get_instance().init(getter_,
bucket_num,
max_cache_size,
block_size))) {
STORAGE_LOG(WARN, "failed to init kv cache", K(ret));
} else if (disk_num_ > 0) {
for (int64_t i = 0; OB_SUCC(ret) && i < disk_num_; ++i) {
if (OB_FAIL(databuff_printf(dirname, sizeof(dirname), "%s/%ld", data_dir_, i))) {

2
unittest/storage/mock_ob_tenant_storage.h
View File

@ -48,7 +48,7 @@ public:
log_file_spec_.retry_write_policy_ = "normal";
log_file_spec_.log_create_policy_ = "normal";
log_file_spec_.log_write_policy_ = "truncate";
if (OB_FAIL(SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_))) {
if (OB_FAIL(SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_))) {
}

2
unittest/storage/slog/test_storage_log_read_write.cpp
View File

@ -97,7 +97,7 @@ void TestStorageLogRW::SetUp()
TestDataFilePrepare::SetUp();
FileDirectoryUtils::create_full_path("./test_storage_log_read_write");
SLOGGERMGR.destroy();
SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_);
SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_);
ObStorageLogger *tmp_slogger = OB_NEW(ObStorageLogger, ObModIds::TEST);
ASSERT_EQ(OB_SUCCESS, tmp_slogger->init(SLOGGERMGR, OB_SERVER_TENANT_ID));

2
unittest/storage/slog/test_storage_log_replay.cpp
View File

@ -72,7 +72,7 @@ void TestStorageLogReplay::SetUp()
TestDataFilePrepare::TearDown();
TestDataFilePrepare::SetUp();
FileDirectoryUtils::create_full_path("./test_storage_log_replay");
SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_);
SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_);
}
void TestStorageLogReplay::TearDown()

9
unittest/storage/slog/test_storage_logger_manager.cpp
View File

@ -86,11 +86,12 @@ TEST_F(TestStorageLoggerManager, test_manager_basic)
int ret = OB_SUCCESS;
// test invalid init
ret = SLOGGERMGR.init(nullptr, MAX_FILE_SIZE, log_file_spec_);
ret = SLOGGERMGR.init(nullptr, nullptr, MAX_FILE_SIZE, log_file_spec_);
ASSERT_NE(OB_SUCCESS, ret);
// test normal init
ret = SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_);
ret = SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(SLOGGERMGR.need_reserved_);
ASSERT_EQ(MAX_CONCURRENT_ITEM_CNT, SLOGGERMGR.log_buffers_.capacity());
ASSERT_EQ(MAX_CONCURRENT_ITEM_CNT, SLOGGERMGR.slog_items_.capacity());
@ -131,7 +132,7 @@ TEST_F(TestStorageLoggerManager, test_slogger_basic)
cursor.log_id_ = 5;
cursor.offset_ = 500;
SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_);
SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_);
ObStorageLogger *tmp_slogger = OB_NEW(ObStorageLogger, ObModIds::TEST);
ASSERT_EQ(OB_SUCCESS, tmp_slogger->init(SLOGGERMGR, 500));
@ -184,7 +185,7 @@ TEST_F (TestStorageLoggerManager, test_build_item)
ObTenantBase tenant_base(5);
ObTenantEnv::set_tenant(&tenant_base);
ASSERT_EQ(OB_SUCCESS, tenant_base.init());
SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_);
SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_);
ObStorageLogger *tmp_slogger = OB_NEW(ObStorageLogger, ObModIds::TEST);
ASSERT_EQ(OB_SUCCESS, tmp_slogger->init(SLOGGERMGR, 500));

2
unittest/storage/test_create_tablet_memtable.cpp
View File

@ -102,7 +102,7 @@ void TestTabletCreateMemtable::SetUp()
ASSERT_EQ(OB_SUCCESS, ret);
ObStorageLogger *slogger = nullptr;
SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_);
SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_);
SLOGGERMGR.register_tenant(1);
ret = SLOGGERMGR.get_tenant_storage_logger(1, slogger);
ASSERT_EQ(OB_SUCCESS, ret);