hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

optimize log print when restart observer failed because of check log pool integrity failed.

Browse Source
This commit is contained in:
HaHaJeff
2024-04-30 12:56:29 +00:00
committed by ob-robot
parent 51ded7db45
commit 82b9d623ef
4 changed files with 182 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
mittest/logservice/test_ob_simple_log_restart.cpp
View File

@ -541,6 +541,63 @@ TEST_F(TestObSimpleLogClusterRestart, restart_and_clear_tmp_files)
PALF_LOG(INFO, "end test restart", K(id), K(guard));
}
TEST_F(TestObSimpleLogClusterRestart, test_with_tmp_dir)
{
SET_CASE_LOG_FILE(TEST_NAME, "test_with_tmp_dir");
OB_LOGGER.set_log_level("INFO");
const int64_t id = ATOMIC_AAF(&palf_id_, 1);
PALF_LOG(INFO, "start advance_base_lsn", K(id));
int64_t leader_idx = 0;
int64_t log_ts = 1;
ObServerLogBlockMgr *log_pool = nullptr;
std::string logserver_dir;
{
PalfHandleImplGuard leader;
EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
logserver_dir = leader.palf_env_impl_->log_dir_;
sleep(2);
LSN log_tail =
leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_;
int count = (LSN(PALF_META_BLOCK_SIZE) - log_tail)/4096;
for (int64_t i = 0; i < count; i++) {
EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->enable_vote());
}
while (LSN(PALF_META_BLOCK_SIZE) !=
leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_)
{
sleep(1);
}
sleep(1);
EXPECT_EQ(LSN(PALF_META_BLOCK_SIZE), leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_);
EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->log_engine_.log_meta_storage_.block_mgr_.switch_next_block(1));
ObISimpleLogServer *i_server = get_cluster()[leader_idx];
ObSimpleLogServer *server = dynamic_cast<ObSimpleLogServer*>(i_server);
log_pool = &server->log_block_pool_;
}
const std::string log_pool_dir = log_pool->log_pool_path_;
std::size_t pos = log_pool_dir.find_last_of('/');
const std::string log_base_dir = log_pool_dir.substr(0, pos);
const std::string mkdir_unexpected_dir = "mkdir -p " + log_base_dir + "/unexpected_dir";
CLOG_LOG(INFO, "runlin trace print mkdir", K(mkdir_unexpected_dir.c_str()));
system(mkdir_unexpected_dir.c_str());
EXPECT_EQ(OB_ERR_UNEXPECTED, restart_paxos_groups());
const std::string rm_unexpected_dir = "rm -rf " + log_base_dir + "/unexpected_dir";
system(rm_unexpected_dir.c_str());
const std::string tmp_log_dir = logserver_dir + "/tmp_dir";
const std::string mkdir_tmp_log_dir = "mkdir -p " + tmp_log_dir;
system(mkdir_tmp_log_dir.c_str());
EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
{
PalfHandleImplGuard leader;
EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
EXPECT_LT(LSN(PALF_META_BLOCK_SIZE), leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_);
EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->set_base_lsn(LSN(0)));
sleep(1);
EXPECT_LT(LSN(PALF_META_BLOCK_SIZE) + 4096, leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_);
}
}
} // namespace unittest
} // namespace oceanbase

134
src/logservice/ob_server_log_block_mgr.cpp
View File

@ -1222,8 +1222,8 @@ int ObServerLogBlockMgr::get_has_allocated_blocks_cnt_in_(
int ret = OB_SUCCESS;
DIR *dir = NULL;
struct dirent *entry = NULL;
std::regex pattern_log(".*/tenant_[1-9]\\d*/[1-9]\\d*/log");
std::regex pattern_meta(".*/tenant_[1-9]\\d*/[1-9]\\d*/meta");
std::regex pattern_tenant(".*/tenant_[1-9]\\d*");
std::regex pattern_log_pool(".*/log_pool/*");
if (NULL == (dir = opendir(log_disk_path))) {
ret = OB_ERR_SYS;
CLOG_LOG(WARN, "opendir failed", K(log_disk_path));
@ -1242,23 +1242,19 @@ int ObServerLogBlockMgr::get_has_allocated_blocks_cnt_in_(
} else if (OB_FAIL(FileDirectoryUtils::is_directory(current_file_path, is_dir))) {
CLOG_LOG(WARN, "is_directory failed", K(ret), K(entry->d_name));
} else if (false == is_dir) {
} else if (true == std::regex_match(current_file_path, pattern_log)
|| true == std::regex_match(current_file_path, pattern_meta)) {
GetBlockCountFunctor functor(current_file_path);
if (OB_FAIL(palf::scan_dir(current_file_path, functor))) {
ret = OB_ERR_UNEXPECTED;
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
"OceanBase. Please confirm whether manual deletion is required",
"log directory", current_file_path);
} else {
has_allocated_block_cnt += functor.get_block_count();
CLOG_LOG(INFO, "get_has_allocated_blocks_cnt_in_ success", K(ret),
K(current_file_path), "block_cnt", functor.get_block_count());
}
} else if (OB_FAIL(get_has_allocated_blocks_cnt_in_(current_file_path,
has_allocated_block_cnt))) {
CLOG_LOG(WARN, "get_has_allocated_blocks_cnt_in_ failed", K(ret),
K(current_file_path), K(has_allocated_block_cnt));
", unexpected file path", current_file_path);
} else if (true == std::regex_match(current_file_path, pattern_tenant)) {
ret = scan_tenant_dir_(current_file_path, has_allocated_block_cnt);
} else if (true == std::regex_match(current_file_path, pattern_log_pool)) {
CLOG_LOG(INFO, "ignore log_pool path", K(current_file_path), KPC(this));
} else {
ret = OB_ERR_UNEXPECTED;
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
"OceanBase. Please confirm whether manual deletion is required",
", unexpected directory", current_file_path);
}
}
}
@ -1586,5 +1582,111 @@ int ObServerLogBlockMgr::force_update_tenant_log_disk(const uint64_t tenant_id,
}
return ret;
}
// the prefix is tenant_xxx
int ObServerLogBlockMgr::scan_tenant_dir_(const char *tenant_dir,
int64_t &has_allocated_block_cnt)
{
int ret = OB_SUCCESS;
DIR *dir = NULL;
std::regex pattern_log_stream(".*/tenant_[1-9]\\d*/[1-9]\\d*");
std::regex pattern_tmp_dir(".*/tenant_[1-9]\\d*/tmp_dir");
struct dirent *entry = NULL;
if (NULL == (dir = opendir(tenant_dir))) {
ret = OB_ERR_SYS;
CLOG_LOG(WARN, "opendir failed", K(tenant_dir));
} else {
char current_file_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'};
while ((entry = readdir(dir)) != NULL && OB_SUCC(ret)) {
bool is_dir = false;
MEMSET(current_file_path, '\0', OB_MAX_FILE_NAME_LENGTH);
if (0 == strcmp(entry->d_name, ".") || 0 == strcmp(entry->d_name, "..")) {
// do nothing
} else if (0 >= snprintf(current_file_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s",
tenant_dir, entry->d_name)) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(WARN, "snprintf failed", K(ret), K(current_file_path), K(tenant_dir),
K(entry->d_name));
} else if (OB_FAIL(FileDirectoryUtils::is_directory(current_file_path, is_dir))) {
CLOG_LOG(WARN, "is_directory failed", K(ret), K(entry->d_name));
} else if (false == is_dir) {
ret = OB_ERR_UNEXPECTED;
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
"OceanBase. Please confirm whether manual deletion is required",
", unexpected file path", current_file_path);
} else if (true == std::regex_match(current_file_path, pattern_log_stream)) {
ret = scan_ls_dir_(current_file_path, has_allocated_block_cnt);
} else if (true == std::regex_match(current_file_path, pattern_tmp_dir)) {
CLOG_LOG(INFO, "ignore tmp_dir", K(current_file_path), K(has_allocated_block_cnt), KPC(this));
} else {
ret = OB_ERR_UNEXPECTED;
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
"OceanBase. Please confirm whether manual deletion is required",
", unexpected directory", current_file_path);
}
}
}
if (NULL != dir) {
closedir(dir);
}
return ret;
}
// the prefix of ls_dir tenant_xxx/xxxx
int ObServerLogBlockMgr::scan_ls_dir_(const char *ls_dir,
int64_t &has_allocated_block_cnt)
{
int ret = OB_SUCCESS;
DIR *dir = NULL;
std::regex pattern_log(".*/tenant_[1-9]\\d*/[1-9]\\d*/log");
std::regex pattern_meta(".*/tenant_[1-9]\\d*/[1-9]\\d*/meta");
struct dirent *entry = NULL;
if (NULL == (dir = opendir(ls_dir))) {
ret = OB_ERR_SYS;
CLOG_LOG(WARN, "opendir failed", K(ls_dir));
} else {
char current_file_path[OB_MAX_FILE_NAME_LENGTH] = {'\0'};
while ((entry = readdir(dir)) != NULL && OB_SUCC(ret)) {
bool is_dir = false;
MEMSET(current_file_path, '\0', OB_MAX_FILE_NAME_LENGTH);
if (0 == strcmp(entry->d_name, ".") || 0 == strcmp(entry->d_name, "..")) {
// do nothing
} else if (0 >= snprintf(current_file_path, OB_MAX_FILE_NAME_LENGTH, "%s/%s",
ls_dir, entry->d_name)) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(WARN, "snprintf failed", K(ret), K(current_file_path), K(ls_dir),
K(entry->d_name));
} else if (OB_FAIL(FileDirectoryUtils::is_directory(current_file_path, is_dir))) {
CLOG_LOG(WARN, "is_directory failed", K(ret), K(entry->d_name));
} else if (false == is_dir) {
ret = OB_ERR_UNEXPECTED;
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
"OceanBase. Please confirm whether manual deletion is required",
", unexpected file path", current_file_path);
} else if (true == std::regex_match(current_file_path, pattern_log)
|| true == std::regex_match(current_file_path, pattern_meta)) {
GetBlockCountFunctor functor(current_file_path);
if (OB_FAIL(palf::scan_dir(current_file_path, functor))) {
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
"OceanBase. Please confirm whether manual deletion is required",
", log directory", current_file_path);
} else {
has_allocated_block_cnt += functor.get_block_count();
CLOG_LOG(INFO, "get_has_allocated_blocks_cnt_in_ success", K(ret),
K(current_file_path), "block_cnt", functor.get_block_count());
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "Attention!!!", "There are several files in the log directory that are not generated by "
"OceanBase. Please confirm whether manual deletion is required",
", unexpected directory", current_file_path);
}
}
}
if (NULL != dir) {
closedir(dir);
}
return ret;
}
} // namespace logservice
} // namespace oceanbase

2
src/logservice/ob_server_log_block_mgr.h
View File

@ -294,6 +294,8 @@ private:
const int64_t src_buf_len, const int64_t offset);
int read_unitl_success_(const palf::FileDesc &src_fd, char *dest_buf,
const int64_t dest_buf_len, const int64_t offset);
int scan_tenant_dir_(const char *tenant_dir, int64_t &has_allocated_block_cnt);
int scan_ls_dir_(const char *tenant_dir, int64_t &has_allocated_block_cnt);
private:
typedef common::ObFunction<int(int64_t&)> GetTenantsLogDiskSize;
mutable ObSpinLock log_pool_meta_lock_;

4
unittest/logservice/test_server_log_block_mgr.cpp
View File

@ -390,9 +390,11 @@ TEST_F(TestServerLogBlockMgr, dirty_ls_dir_and_log_pool_file)
system("mkdir clog_disk/clog/log_pool/1.tmp");
log_block_mgr_.destroy();
bool result = false;
EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_pool_base_path_));
EXPECT_EQ(OB_ERR_UNEXPECTED, log_block_mgr_.init(log_pool_base_path_));
EXPECT_EQ(OB_SUCCESS, FileDirectoryUtils::is_exists("clog_disk/clog/tenant_1/1/meta/10000.tmp",result));
EXPECT_EQ(false, result);
system("rm -rf clog_disk/clog/tenant_0111");
EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_pool_base_path_));
}
TEST_F(TestServerLogBlockMgr, resize_failed_and_restar)