hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] merge transfer

Browse Source 
Co-authored-by: wxhwang <wxhwang@126.com>
Co-authored-by: godyangfight <godyangfight@gmail.com>
Co-authored-by: Tyshawn <tuyunshan@gmail.com>
This commit is contained in:
xuhuleon
2023-06-21 11:42:26 +00:00
committed by ob-robot
parent d06678002e
commit 9dae112952
1280 changed files with 149724 additions and 48813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
unittest/storage/slog/simple_ob_storage_redo_module.h
View File

@ -20,6 +20,14 @@
#include "lib/random/ob_random.h"
#include "lib/allocator/ob_mod_define.h"
#include "storage/slog/ob_storage_log_struct.h"
#include "src/storage/meta_mem/ob_tablet_map_key.h"
#include "src/storage/ob_super_block_struct.h"
#include "src/storage/slog/ob_storage_log_reader.h"
#include "src/storage/slog/ob_storage_logger.h"
#include "src/storage/ls/ob_ls_tablet_service.h"
#include "src/storage/tx_storage/ob_ls_handle.h"
#include "src/storage/tx_storage/ob_ls_service.h"
#include "src/storage/tablet/ob_tablet.h"
namespace oceanbase
{
@ -132,6 +140,16 @@ public:
const char *buf,
const int64_t len,
FILE *stream) override;
int inner_replay_empty_shell_tablet(const ObRedoModuleReplayParam &param);
int read_from_disk(
const ObMetaDiskAddr &addr,
common::ObArenaAllocator &allocator,
char *&buf,
int64_t &buf_len);
int read_from_disk_addr(const ObMetaDiskAddr &phy_addr, char *buf, const int64_t buf_len, char *&r_buf, int64_t &r_len);
int read_from_share_blk(const ObMetaDiskAddr &addr, common::ObArenaAllocator &allocator, char *&buf, int64_t &buf_len);
int read_from_slog(const ObMetaDiskAddr &phy_addr, char *buf, const int64_t buf_len, int64_t &pos);
int get_tablet_svr(const share::ObLSID &ls_id, ObLSTabletService *&ls_tablet_svr, ObLSHandle &ls_handle);
bool operator ==(SimpleObStorageModule &redo_module);
void reset();
@ -182,6 +200,10 @@ int SimpleObStorageModule::replay(const ObRedoModuleReplayParam &param)
}
} else if (ObRedoLogSubType::OB_REDO_LOG_DELETE_LS == sub_type) {
// do nothing
} else if (ObRedoLogSubType::OB_REDO_LOG_EMPTY_SHELL_TABLET == sub_type) {
if (OB_FAIL(inner_replay_empty_shell_tablet(param))) {
STORAGE_REDO_LOG(WARN, "Fail to inner replay empty shell tablet", K(sub_type), K(param));
}
}
if (OB_SUCC(ret)) {
@ -192,6 +214,151 @@ int SimpleObStorageModule::replay(const ObRedoModuleReplayParam &param)
return ret;
}
int SimpleObStorageModule::get_tablet_svr(
const ObLSID &ls_id,
ObLSTabletService *&ls_tablet_svr,
ObLSHandle &ls_handle)
{
int ret = OB_SUCCESS;
ObLS *ls = nullptr;
if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) {
STORAGE_LOG(WARN, "fail to get ls handle", K(ret), K(ls_id));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "ls is null", K(ret), K(ls_id));
} else if (OB_ISNULL(ls_tablet_svr = ls->get_tablet_svr())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "tablet service is null", K(ret), K(ls_id));
}
return ret;
}
int SimpleObStorageModule::inner_replay_empty_shell_tablet(const ObRedoModuleReplayParam &param)
{
int ret = OB_SUCCESS;
int64_t pos = 0;
ObArenaAllocator allocator;
ObLSTabletService *ls_tablet_svr = nullptr;
ObLSHandle ls_handle;
char *buf = nullptr;
int64_t buf_len = 0;
ObEmptyShellTabletLog slog;
ObTabletTransferInfo tablet_transfer_info;
if (OB_FAIL(slog.deserialize_id(param.buf_, param.disk_addr_.size(), pos))) {
STORAGE_LOG(WARN, "failed to serialize tablet_id_", K(ret), K(param.disk_addr_.size()), K(pos));
} else if (OB_FAIL(read_from_disk(param.disk_addr_, allocator, buf, buf_len))) {
STORAGE_LOG(WARN, "read from disk failed", K(ret), K(param.disk_addr_), K(buf_len));
} else if (OB_FAIL(get_tablet_svr(slog.ls_id_, ls_tablet_svr, ls_handle))) {
STORAGE_LOG(WARN, "get tablet svr failed", K(ret), K(slog.ls_id_));
} else if (OB_FAIL(ls_tablet_svr->replay_create_tablet(param.disk_addr_, buf, buf_len, slog.tablet_id_, tablet_transfer_info))) {
STORAGE_LOG(WARN, "replay empty shell tablet failed", K(ret), K(param.disk_addr_), K(slog.tablet_id_));
}
return ret;
}
int SimpleObStorageModule::read_from_disk(
const ObMetaDiskAddr &addr,
common::ObArenaAllocator &allocator,
char *&buf,
int64_t &buf_len)
{
int ret = OB_SUCCESS;
char *read_buf = nullptr;
const int64_t read_buf_len = addr.size();
if (ObMetaDiskAddr::DiskType::FILE == addr.type()) {
ObEmptyShellTabletLog slog;
int64_t pos = 0;
if (OB_ISNULL(buf)) {
if (OB_ISNULL(buf = static_cast<char*>(allocator.alloc(addr.size())))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
} else {
buf_len = addr.size();
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(read_from_slog(addr, buf, buf_len, pos))) {
STORAGE_LOG(WARN, "fail to read from slog", K(ret), K(addr), KP(buf), K(buf_len), K(pos));
} else if (OB_FAIL(slog.deserialize_id(buf, buf_len, pos))) {
STORAGE_LOG(WARN, "fail to deserialize id", K(ret), K(addr), KP(buf), K(buf_len), K(pos));
} else {
buf += pos;
buf_len -= pos;
}
}
} else if (OB_ISNULL(read_buf = static_cast<char*>(allocator.alloc(read_buf_len)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "fail to allocate buffer", K(ret), K(read_buf_len), KP(read_buf));
} else if (OB_FAIL(read_from_disk_addr(addr, read_buf, read_buf_len, buf, buf_len))) {
STORAGE_LOG(WARN, "fail to read tablet from addr", K(ret), K(addr), KP(read_buf), K(read_buf_len));
}
return ret;
}
int SimpleObStorageModule::read_from_disk_addr(const ObMetaDiskAddr &addr,
char *buf, const int64_t buf_len, char *&r_buf, int64_t &r_len)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!addr.is_valid() || buf_len < addr.size()) || OB_ISNULL(buf)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(addr), KP(buf), K(buf_len));
} else {
switch (addr.type()) {
case ObMetaDiskAddr::DiskType::FILE: {
int64_t pos = 0;
if (OB_FAIL(read_from_slog(addr, buf, buf_len, pos))) {
STORAGE_LOG(WARN, "fail to read from slog", K(ret), K(addr), KP(buf), K(buf_len));
} else {
r_buf = buf + pos;
r_len = addr.size() - pos;
}
break;
}
default: {
ret = OB_NOT_SUPPORTED;
STORAGE_LOG(WARN, "unknown meta disk address type", K(ret), K(addr), KP(buf), K(buf_len));
break;
}
}
}
return ret;
}
int SimpleObStorageModule::read_from_slog(const ObMetaDiskAddr &addr,
char *buf, const int64_t buf_len, int64_t &pos)
{
int ret = OB_SUCCESS;
ObStorageLogger *logger = MTL(ObStorageLogger*);
if (OB_UNLIKELY(!addr.is_valid()
|| !addr.is_file()
|| buf_len < addr.size())
|| OB_ISNULL(buf)
|| OB_ISNULL(logger)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(addr), KP(buf), K(buf_len), KP(logger));
} else {
// The reason for retrying, here, is that the current SLOG didn't handle the read and write
// concurrency for the latest item, and an -4103 error will be returned. At present, the
// optimized changes for SLOG are relatively large, and we will bypass it in the short term.
int64_t retry_count = 2;
do {
int64_t tmp_pos = pos;
if (OB_FAIL(ObStorageLogReader::read_log(logger->get_dir(), addr, buf_len, buf, tmp_pos, MTL_ID()))) {
STORAGE_LOG(WARN, "fail to read slog", K(ret), "logger directory", logger->get_dir(), K(addr),
K(buf_len), KP(buf));
if (retry_count > 1) {
sleep(1); // sleep 1s
}
} else {
pos = tmp_pos;
}
} while (OB_FAIL(ret) && --retry_count > 0);
}
return ret;
}
bool SimpleObStorageModule::operator ==(SimpleObStorageModule &redo_module)
{
bool ret = true;

2
unittest/storage/slog/test_storage_logger_manager.cpp
View File

@ -135,7 +135,6 @@ TEST_F(TestStorageLoggerManager, test_slogger_basic)
SLOGGERMGR.init(dir_, MAX_FILE_SIZE, log_file_spec_);
ObStorageLogger *tmp_slogger = OB_NEW(ObStorageLogger, ObModIds::TEST);
ASSERT_EQ(0, tmp_slogger->get_pwrite_ts());
ASSERT_EQ(OB_SUCCESS, tmp_slogger->init(SLOGGERMGR, 500));
ASSERT_EQ(OB_SUCCESS, tmp_slogger->start());
@ -149,7 +148,6 @@ TEST_F(TestStorageLoggerManager, test_slogger_basic)
guard.switch_to(5);
ObStorageLogger *slogger = MTL(ObStorageLogger*);
slogger->start_log(cursor);
ASSERT_EQ(0, slogger->get_pwrite_ts());
// test get_active_cursor
ObLogCursor tmp_cursor;