From 67d7a0f8de84cfbb2f80278fc780e6dc43cd659d Mon Sep 17 00:00:00 2001 From: JiahuaChen Date: Tue, 16 Apr 2024 08:16:15 +0000 Subject: [PATCH] Check slog disk during restart --- mittest/mtlenv/mock_tenant_module_env.h | 2 +- src/observer/ob_server.cpp | 4 +- .../slog/ob_storage_logger_manager.cpp | 38 ++++++++++++++++--- src/storage/slog/ob_storage_logger_manager.h | 6 ++- tools/ob_admin/ob_admin_executor.cpp | 2 +- .../blocksstable/ob_data_file_prepare.h | 18 +++++---- unittest/storage/mock_ob_tenant_storage.h | 2 +- .../slog/test_storage_log_read_write.cpp | 2 +- .../storage/slog/test_storage_log_replay.cpp | 2 +- .../slog/test_storage_logger_manager.cpp | 9 +++-- .../storage/test_create_tablet_memtable.cpp | 2 +- 11 files changed, 60 insertions(+), 27 deletions(-) diff --git a/mittest/mtlenv/mock_tenant_module_env.h b/mittest/mtlenv/mock_tenant_module_env.h index f7b3951c1..de639de35 100644 --- a/mittest/mtlenv/mock_tenant_module_env.h +++ b/mittest/mtlenv/mock_tenant_module_env.h @@ -598,7 +598,7 @@ int MockTenantModuleEnv::init_slogger_mgr(const char *log_dir) log_file_spec.retry_write_policy_ = "normal"; log_file_spec.log_create_policy_ = "normal"; log_file_spec.log_write_policy_ = "truncate"; - if (OB_FAIL(SLOGGERMGR.init(log_dir, MAX_FILE_SIZE, log_file_spec))) { + if (OB_FAIL(SLOGGERMGR.init(log_dir, log_dir, MAX_FILE_SIZE, log_file_spec))) { STORAGE_LOG(WARN, "fail to init SLOGGERMGR", K(ret)); } diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 578f33eb9..58037b2aa 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -423,8 +423,8 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) } else if (OB_FAIL(ObExternalTableFileManager::get_instance().init())) { LOG_ERROR("init external table file manager failed", KR(ret)); } else if (OB_FAIL(SLOGGERMGR.init(storage_env_.log_spec_.log_dir_, - storage_env_.log_spec_.max_log_file_size_, storage_env_.slog_file_spec_, - true/*need_reserved*/))) { + storage_env_.sstable_dir_, storage_env_.log_spec_.max_log_file_size_, + storage_env_.slog_file_spec_))) { LOG_ERROR("init ObStorageLoggerManager failed", KR(ret)); } else if (OB_FAIL(ObVirtualTenantManager::get_instance().init())) { LOG_ERROR("init tenant manager failed", KR(ret)); diff --git a/src/storage/slog/ob_storage_logger_manager.cpp b/src/storage/slog/ob_storage_logger_manager.cpp index b05d00fc3..e31614212 100644 --- a/src/storage/slog/ob_storage_logger_manager.cpp +++ b/src/storage/slog/ob_storage_logger_manager.cpp @@ -11,6 +11,7 @@ */ #define USING_LOG_PREFIX STORAGE_REDO +#include #include "lib/ob_define.h" #include "lib/ob_running_mode.h" #include "ob_storage_logger_manager.h" @@ -47,28 +48,29 @@ ObStorageLoggerManager &ObStorageLoggerManager::get_instance() int ObStorageLoggerManager::init( const char *log_dir, + const char *data_dir, const int64_t max_log_file_size, - const blocksstable::ObLogFileSpec &log_file_spec, - const bool need_reserved) + const blocksstable::ObLogFileSpec &log_file_spec) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; STORAGE_REDO_LOG(WARN, "The ObStorageLoggerManager has been inited.", K(ret)); - } else if (OB_UNLIKELY(nullptr == log_dir || max_log_file_size <= 0)) { + } else if (OB_UNLIKELY(nullptr == log_dir || nullptr == data_dir || max_log_file_size <= 0)) { ret = OB_INVALID_ARGUMENT; - STORAGE_REDO_LOG(WARN, "invalid arguments", K(ret), KP(log_dir), K(max_log_file_size)); + STORAGE_REDO_LOG(WARN, "invalid arguments", K(ret), KP(log_dir), KP(data_dir), K(max_log_file_size)); } else if (OB_FAIL(prepare_log_buffers(MAX_CONCURRENT_ITEM_CNT, NORMAL_LOG_ITEM_SIZE))) { STORAGE_REDO_LOG(WARN, "fail to prepare log buffers", K(ret), LITERAL_K(MAX_CONCURRENT_ITEM_CNT), LITERAL_K(NORMAL_LOG_ITEM_SIZE)); } else if (OB_FAIL(prepare_log_items(MAX_CONCURRENT_ITEM_CNT))) { STORAGE_REDO_LOG(WARN, "fail to prepare log items", K(ret), LITERAL_K(MAX_CONCURRENT_ITEM_CNT)); + } else if (OB_FAIL(check_log_disk(data_dir, log_dir))) { + STORAGE_REDO_LOG(WARN, "fail to set need reserved", K(ret)); } else { log_dir_ = log_dir; max_log_file_size_ = max_log_file_size; log_file_spec_ = log_file_spec; - need_reserved_ = need_reserved; if (OB_FAIL(server_slogger_.init(*this, OB_SERVER_TENANT_ID))) { STORAGE_REDO_LOG(WARN, "fail to init server slogger", K(ret)); } else if (OB_FAIL(server_slogger_.start())) { @@ -300,6 +302,32 @@ int ObStorageLoggerManager::get_tenant_slog_dir( return ret; } +int ObStorageLoggerManager::check_log_disk( + const char *data_dir, + const char *log_dir) +{ + int ret = OB_SUCCESS; + struct statvfs data_svfs; + struct statvfs log_svfs; + need_reserved_ = false; + if (OB_ISNULL(data_dir) || OB_ISNULL(log_dir)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), KP(data_dir), KP(log_dir)); + } else if (OB_UNLIKELY(0 != statvfs(data_dir, &data_svfs))) { + ret = OB_IO_ERROR; + LOG_WARN("fail to get sstable directory vfs", K(ret), K(data_dir)); + } else if (OB_UNLIKELY(0 != statvfs(log_dir, &log_svfs))) { + ret = OB_IO_ERROR; + LOG_WARN("fail to get slog directory vfs", K(ret), K(log_dir)); + } else if (OB_UNLIKELY(0 >= log_svfs.f_bavail)) { + ret = OB_DISK_ERROR; + LOG_ERROR("slog disk is full, please check", K(ret), K(log_dir), K(log_svfs.f_bavail)); + } else { + // if slog and data are on the same disk, need reserved space when resize file + need_reserved_ = (data_svfs.f_fsid == log_svfs.f_fsid); + } + return ret; +} int ObStorageLoggerManager::get_using_disk_space(int64_t &using_space) const { diff --git a/src/storage/slog/ob_storage_logger_manager.h b/src/storage/slog/ob_storage_logger_manager.h index f2a96f6c8..024329ed1 100644 --- a/src/storage/slog/ob_storage_logger_manager.h +++ b/src/storage/slog/ob_storage_logger_manager.h @@ -33,9 +33,9 @@ public: static ObStorageLoggerManager &get_instance(); int init( const char *log_dir, + const char *data_dir, const int64_t max_log_file_size, - const blocksstable::ObLogFileSpec &log_file_spec, - const bool need_reserved = false); + const blocksstable::ObLogFileSpec &log_file_spec); void destroy(); // allocate item and its buffer @@ -69,6 +69,8 @@ private: int get_using_disk_space(int64_t &using_space) const; + int check_log_disk(const char *data_dir, const char *log_dir); + private: static constexpr int64_t NORMAL_LOG_ITEM_SIZE = 8 * 1024; //8KB static constexpr int64_t MAX_CONCURRENT_ITEM_CNT = 1024; diff --git a/tools/ob_admin/ob_admin_executor.cpp b/tools/ob_admin/ob_admin_executor.cpp index 25736b690..87f809b6d 100644 --- a/tools/ob_admin/ob_admin_executor.cpp +++ b/tools/ob_admin/ob_admin_executor.cpp @@ -156,7 +156,7 @@ int ObAdminExecutor::init_slogger_mgr() if (OB_FAIL(databuff_printf(slog_dir_, OB_MAX_FILE_NAME_LENGTH, "%s/slog/", data_dir_))) { LOG_WARN("failed to gen slog dir", K(ret)); - } else if (OB_FAIL(SLOGGERMGR.init(slog_dir_, MAX_FILE_SIZE, storage_env_.slog_file_spec_))) { + } else if (OB_FAIL(SLOGGERMGR.init(slog_dir_, sstable_dir_, MAX_FILE_SIZE, storage_env_.slog_file_spec_))) { STORAGE_LOG(WARN, "fail to init SLOGGERMGR", K(ret)); } diff --git a/unittest/storage/blocksstable/ob_data_file_prepare.h b/unittest/storage/blocksstable/ob_data_file_prepare.h index 3db6d258a..ff4d45582 100644 --- a/unittest/storage/blocksstable/ob_data_file_prepare.h +++ b/unittest/storage/blocksstable/ob_data_file_prepare.h @@ -228,20 +228,22 @@ int TestDataFilePrepareUtil::open() } else if (0 != system(cmd)) { ret = OB_ERR_SYS; STORAGE_LOG(ERROR, "failed to exec cmd", K(ret), K(cmd), K(errno), KERRMSG); - } else if (OB_FAIL(SLOGGERMGR.init(storage_env_.log_spec_.log_dir_, - storage_env_.log_spec_.max_log_file_size_, storage_env_.slog_file_spec_))) { - STORAGE_LOG(WARN, "ObStorageLoggerManager init failed", KR(ret)); - } else if (OB_FAIL(ObKVGlobalCache::get_instance().init(getter_, - bucket_num, - max_cache_size, - block_size))) { - STORAGE_LOG(WARN, "failed to init kv cache", K(ret)); } else if (OB_FAIL(FileDirectoryUtils::create_full_path(clog_dir_))) { STORAGE_LOG(WARN, "failed to create clog dir", K(ret), K(clog_dir_)); } else if (OB_FAIL(FileDirectoryUtils::create_full_path(slog_dir_))) { STORAGE_LOG(WARN, "failed to create slog dir", K(ret), K(slog_dir_)); } else if (OB_FAIL(FileDirectoryUtils::create_full_path(file_dir_))) { STORAGE_LOG(WARN, "failed to create file dir", K(ret), K(file_dir_)); + } else if (OB_FAIL(SLOGGERMGR.init(storage_env_.log_spec_.log_dir_, + storage_env_.log_spec_.log_dir_, + storage_env_.log_spec_.max_log_file_size_, storage_env_.slog_file_spec_))) { + STORAGE_LOG(WARN, "ObStorageLoggerManager init failed", KR(ret)); + } else if (FALSE_IT(SLOGGERMGR.need_reserved_ = false)) { + } else if (OB_FAIL(ObKVGlobalCache::get_instance().init(getter_, + bucket_num, + max_cache_size, + block_size))) { + STORAGE_LOG(WARN, "failed to init kv cache", K(ret)); } else if (disk_num_ > 0) { for (int64_t i = 0; OB_SUCC(ret) && i < disk_num_; ++i) { if (OB_FAIL(databuff_printf(dirname, sizeof(dirname), "%s/%ld", data_dir_, i))) { diff --git a/unittest/storage/mock_ob_tenant_storage.h b/unittest/storage/mock_ob_tenant_storage.h index edf1e8203..2c26d216c 100644 --- a/unittest/storage/mock_ob_tenant_storage.h +++ b/unittest/storage/mock_ob_tenant_storage.h @@ -48,7 +48,7 @@ public: log_file_spec_.retry_write_policy_ = "normal"; log_file_spec_.log_create_policy_ = "normal"; log_file_spec_.log_write_policy_ = "truncate"; - if (OB_FAIL(SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_))) { + if (OB_FAIL(SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_))) { } diff --git a/unittest/storage/slog/test_storage_log_read_write.cpp b/unittest/storage/slog/test_storage_log_read_write.cpp index 716e7cb5d..a1f392179 100644 --- a/unittest/storage/slog/test_storage_log_read_write.cpp +++ b/unittest/storage/slog/test_storage_log_read_write.cpp @@ -97,7 +97,7 @@ void TestStorageLogRW::SetUp() TestDataFilePrepare::SetUp(); FileDirectoryUtils::create_full_path("./test_storage_log_read_write"); SLOGGERMGR.destroy(); - SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_); + SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_); ObStorageLogger *tmp_slogger = OB_NEW(ObStorageLogger, ObModIds::TEST); ASSERT_EQ(OB_SUCCESS, tmp_slogger->init(SLOGGERMGR, OB_SERVER_TENANT_ID)); diff --git a/unittest/storage/slog/test_storage_log_replay.cpp b/unittest/storage/slog/test_storage_log_replay.cpp index 907319686..84aeb2b43 100644 --- a/unittest/storage/slog/test_storage_log_replay.cpp +++ b/unittest/storage/slog/test_storage_log_replay.cpp @@ -72,7 +72,7 @@ void TestStorageLogReplay::SetUp() TestDataFilePrepare::TearDown(); TestDataFilePrepare::SetUp(); FileDirectoryUtils::create_full_path("./test_storage_log_replay"); - SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_); + SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_); } void TestStorageLogReplay::TearDown() diff --git a/unittest/storage/slog/test_storage_logger_manager.cpp b/unittest/storage/slog/test_storage_logger_manager.cpp index b426a651c..16efaebd1 100644 --- a/unittest/storage/slog/test_storage_logger_manager.cpp +++ b/unittest/storage/slog/test_storage_logger_manager.cpp @@ -86,11 +86,12 @@ TEST_F(TestStorageLoggerManager, test_manager_basic) int ret = OB_SUCCESS; // test invalid init - ret = SLOGGERMGR.init(nullptr, MAX_FILE_SIZE, log_file_spec_); + ret = SLOGGERMGR.init(nullptr, nullptr, MAX_FILE_SIZE, log_file_spec_); ASSERT_NE(OB_SUCCESS, ret); // test normal init - ret = SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_); + ret = SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_); ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_TRUE(SLOGGERMGR.need_reserved_); ASSERT_EQ(MAX_CONCURRENT_ITEM_CNT, SLOGGERMGR.log_buffers_.capacity()); ASSERT_EQ(MAX_CONCURRENT_ITEM_CNT, SLOGGERMGR.slog_items_.capacity()); @@ -131,7 +132,7 @@ TEST_F(TestStorageLoggerManager, test_slogger_basic) cursor.log_id_ = 5; cursor.offset_ = 500; - SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_); + SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_); ObStorageLogger *tmp_slogger = OB_NEW(ObStorageLogger, ObModIds::TEST); ASSERT_EQ(OB_SUCCESS, tmp_slogger->init(SLOGGERMGR, 500)); @@ -184,7 +185,7 @@ TEST_F (TestStorageLoggerManager, test_build_item) ObTenantBase tenant_base(5); ObTenantEnv::set_tenant(&tenant_base); ASSERT_EQ(OB_SUCCESS, tenant_base.init()); - SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_); + SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_); ObStorageLogger *tmp_slogger = OB_NEW(ObStorageLogger, ObModIds::TEST); ASSERT_EQ(OB_SUCCESS, tmp_slogger->init(SLOGGERMGR, 500)); diff --git a/unittest/storage/test_create_tablet_memtable.cpp b/unittest/storage/test_create_tablet_memtable.cpp index 12035f504..7f9247066 100644 --- a/unittest/storage/test_create_tablet_memtable.cpp +++ b/unittest/storage/test_create_tablet_memtable.cpp @@ -102,7 +102,7 @@ void TestTabletCreateMemtable::SetUp() ASSERT_EQ(OB_SUCCESS, ret); ObStorageLogger *slogger = nullptr; - SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_); + SLOGGERMGR.init(dir_, dir_, MAX_FILE_SIZE, log_file_spec_); SLOGGERMGR.register_tenant(1); ret = SLOGGERMGR.get_tenant_storage_logger(1, slogger); ASSERT_EQ(OB_SUCCESS, ret);