[FEAT MERGE] merge transfer

Co-authored-by: wxhwang <wxhwang@126.com>
Co-authored-by: godyangfight <godyangfight@gmail.com>
Co-authored-by: Tyshawn <tuyunshan@gmail.com>
This commit is contained in:
xuhuleon
2023-06-21 11:42:26 +00:00
committed by ob-robot
parent d06678002e
commit 9dae112952
1280 changed files with 149724 additions and 48813 deletions

View File

@ -14,6 +14,7 @@
#include "storage/ob_i_table.h"
#include "share/ob_force_print_log.h"
#include "storage/blocksstable/ob_sstable.h"
#include "storage/blocksstable/ob_storage_cache_suite.h"
#include "storage/memtable/ob_memtable.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
#include "storage/tablelock/ob_lock_memtable.h"
@ -128,7 +129,7 @@ int ObITable::safe_to_destroy(bool &is_safe)
int ObITable::exist(
ObStoreCtx &ctx,
const uint64_t table_id,
const storage::ObTableReadInfo &read_info,
const storage::ObITableReadInfo &read_info,
const blocksstable::ObDatumRowkey &rowkey,
bool &is_exist,
bool &has_found)
@ -142,6 +143,21 @@ int ObITable::exist(
return common::OB_NOT_SUPPORTED;
}
int ObITable::exist(
const ObTableIterParam &param,
ObTableAccessContext &context,
const blocksstable::ObDatumRowkey &rowkey,
bool &is_exist,
bool &has_found)
{
UNUSED(param);
UNUSED(context);
UNUSED(rowkey);
is_exist = false;
has_found = false;
return common::OB_NOT_SUPPORTED;
}
int ObITable::exist(
ObRowsInfo &rows_info,
bool &is_exist,
@ -167,16 +183,11 @@ int64_t ObITable::to_string(char *buf, const int64_t buf_len) const
}
ObTableHandleV2::ObTableHandleV2()
: table_(nullptr), t3m_(nullptr), allocator_(nullptr), table_type_(ObITable::TableType::MAX_TABLE_TYPE)
: table_(nullptr), t3m_(nullptr), allocator_(nullptr),
meta_handle_(), table_type_(ObITable::TableType::MAX_TABLE_TYPE)
{
}
ObTableHandleV2::ObTableHandleV2(ObITable *table, ObTenantMetaMemMgr *t3m, ObITable::TableType type)
: table_(nullptr), t3m_(nullptr), allocator_(nullptr), table_type_(type)
{
abort_unless(OB_SUCCESS == set_table(table, t3m, table_type_));
}
ObTableHandleV2::~ObTableHandleV2()
{
reset();
@ -184,8 +195,17 @@ ObTableHandleV2::~ObTableHandleV2()
bool ObTableHandleV2::is_valid() const
{
return nullptr != table_
&& ((nullptr != t3m_ && nullptr == allocator_) || (nullptr == t3m_ && nullptr != allocator_));
bool bret = false;
if (nullptr == table_) {
} else if (ObITable::is_memtable(table_type_)) {
bret = (nullptr != t3m_) ^ (nullptr != allocator_);
} else if (ObITable::is_ddl_mem_sstable(table_type_)) {
bret = nullptr != t3m_;
} else {
// all other sstables
bret = (meta_handle_.is_valid() ^ (nullptr != allocator_)) || lifetime_guaranteed_by_tablet_;
}
return bret;
}
void ObTableHandleV2::reset()
@ -199,18 +219,23 @@ void ObTableHandleV2::reset()
if (0 == ref_cnt) {
if (nullptr != t3m_) {
t3m_->push_table_into_gc_queue(table_, table_type_);
} else {
} else if (nullptr != allocator_) {
table_->~ObITable();
allocator_->free(table_);
} else if (OB_UNLIKELY(!ObITable::is_sstable(table_type_))) {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "possible table leak!!!", K(ref_cnt), KPC(table_), K(table_type_));
}
} else if (OB_UNLIKELY(ref_cnt < 0)) {
} else if (OB_UNLIKELY(ref_cnt < 0 && !ObITable::is_sstable(table_type_))) {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "table ref cnt may be leaked", K(ref_cnt), KP(table_), K(table_type_));
}
table_ = nullptr;
t3m_ = nullptr;
allocator_ = nullptr;
}
}
table_ = nullptr;
t3m_ = nullptr;
allocator_ = nullptr;
table_type_ = ObITable::TableType::MAX_TABLE_TYPE;
meta_handle_.reset();
lifetime_guaranteed_by_tablet_ = false;
}
int ObTableHandleV2::get_sstable(blocksstable::ObSSTable *&sstable)
@ -418,7 +443,12 @@ int ObTableHandleV2::get_lock_memtable(const ObLockMemtable *&memtable) const
}
ObTableHandleV2::ObTableHandleV2(const ObTableHandleV2 &other)
: table_(nullptr), t3m_(nullptr)
: table_(nullptr),
t3m_(nullptr),
allocator_(nullptr),
meta_handle_(),
table_type_(ObITable::TableType::MAX_TABLE_TYPE),
lifetime_guaranteed_by_tablet_(false)
{
*this = other;
}
@ -436,8 +466,10 @@ ObTableHandleV2 &ObTableHandleV2::operator= (const ObTableHandleV2 &other)
other.table_->inc_ref();
t3m_ = other.t3m_;
allocator_ = other.allocator_;
meta_handle_ = other.meta_handle_;
table_type_ = other.table_type_;
if (OB_UNLIKELY(other.table_->get_ref() < 2)) {
lifetime_guaranteed_by_tablet_ = other.lifetime_guaranteed_by_tablet_;
if (!ObITable::is_sstable(table_type_) && OB_UNLIKELY(other.table_->get_ref() < 2)) {
STORAGE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "The reference count of the table is unexpectedly decreased,"
" the possible reason is that the table handle has concurrency", K(other));
}
@ -459,6 +491,9 @@ int ObTableHandleV2::set_table(
OB_UNLIKELY(!ObITable::is_table_type_valid(table_type))) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), KP(table), KP(t3m), K(table_type));
} else if (OB_UNLIKELY(ObITable::is_sstable(table_type) && !ObITable::is_ddl_mem_sstable(table_type))) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(ERROR, "sstable should not use this interface", K(ret), KP(table), K(table_type));
} else {
table_ = table;
table_->inc_ref();
@ -469,7 +504,7 @@ int ObTableHandleV2::set_table(
return ret;
}
int ObTableHandleV2::set_table(
int ObTableHandleV2::set_sstable(
ObITable *table,
common::ObIAllocator *allocator)
{
@ -483,16 +518,51 @@ int ObTableHandleV2::set_table(
table_->inc_ref();
t3m_ = nullptr;
allocator_ = allocator;
table_type_ = ObITable::TableType::MAX_TABLE_TYPE;
table_type_ = table->get_key().table_type_;
}
return ret;
}
int ObTableHandleV2::set_sstable_with_tablet(ObITable *table)
{
int ret = OB_SUCCESS;
reset();
if (OB_ISNULL(table)
|| OB_UNLIKELY(!ObITable::is_sstable(table->get_key().table_type_))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KPC(table));
} else {
table_ = table;
table_->inc_ref();
lifetime_guaranteed_by_tablet_ = true;
table_type_ = table->get_key().table_type_;
}
return ret;
}
int ObTableHandleV2::set_sstable(ObITable *table, const ObStorageMetaHandle &meta_handle)
{
int ret = OB_SUCCESS;
reset();
if (OB_ISNULL(table)
|| OB_UNLIKELY(!ObITable::is_sstable(table->get_key().table_type_))
|| OB_UNLIKELY(!meta_handle.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KPC(table), K(meta_handle));
} else {
table_ = table;
table_->inc_ref();
t3m_ = nullptr;
allocator_ = nullptr;
meta_handle_ = meta_handle;
table_type_ = table->get_key().table_type_;
}
return ret;
}
ObTablesHandleArray::ObTablesHandleArray()
: meta_mem_mgr_(nullptr),
allocator_(nullptr),
tablet_id_(),
tables_()
: tablet_id_(),
handles_array_()
{
}
@ -503,106 +573,79 @@ ObTablesHandleArray::~ObTablesHandleArray()
void ObTablesHandleArray::reset()
{
if (tables_.count() > 0) {
for (int64_t i = 0; i < tables_.count(); ++i) {
ObITable *table = tables_.at(i);
if (OB_NOT_NULL(table)) {
const int64_t ref_cnt = table->dec_ref();
if (0 == ref_cnt) {
if (OB_ISNULL(meta_mem_mgr_) && OB_ISNULL(allocator_)) {
STORAGE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "[MEMORY LEAK] meta_mem_mgr is unexpected null!!!", KPC(table), K(tablet_id_), KP(meta_mem_mgr_), KP(allocator_));
} else if (nullptr != meta_mem_mgr_) {
meta_mem_mgr_->push_table_into_gc_queue(table, table->get_key().table_type_);
} else {
table->~ObITable();
allocator_->free(table);
}
} else if (OB_UNLIKELY(ref_cnt < 0)) {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "table ref cnt may be leaked", K(ref_cnt), KP(table), "table type", table->get_key().table_type_);
}
}
tables_.at(i) = nullptr;
}
tables_.reset();
tablet_id_.reset();
meta_mem_mgr_ = nullptr;
allocator_ = nullptr;
}
tablet_id_.reset();
handles_array_.reset();
}
int ObTablesHandleArray::add_table(ObITable *table, common::ObIAllocator *allocator)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(table) || OB_ISNULL(allocator)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid arguments", K(ret), KP(table), KP(allocator));
} else if (OB_NOT_NULL(meta_mem_mgr_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "tables either use an allocator or a t3m, not a mix", K(ret), KP(allocator_),
KP(meta_mem_mgr_), KP(MTL(ObTenantMetaMemMgr *)));
} else if (0 == tables_.count()) { // first add table, set allocator_ && tablet_id
allocator_ = allocator;
tablet_id_ = table->get_key().tablet_id_;
} else if (OB_UNLIKELY(allocator_ != allocator)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "tables must own the same allocator!!!", K(ret), KP(allocator_), KP(allocator));
} else if (OB_UNLIKELY(tablet_id_ != table->get_key().tablet_id_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "tables must belong to the same tablet!!!", K(ret), K(tablet_id_), K(table->get_key()));
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(tables_.push_back(table))) {
STORAGE_LOG(WARN, "failed to add table", K(ret));
} else {
table->inc_ref();
}
return ret;
}
int ObTablesHandleArray::add_table(ObITable *table)
int ObTablesHandleArray::add_memtable(ObITable *table)
{
int ret = OB_SUCCESS;
ObTableHandleV2 handle;
if (OB_ISNULL(table)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "get invalid arguments", K(ret), KP(table));
} else if (0 == tables_.count()) { // first add table, set meta_mem_mgr && tablet_id
if (OB_ISNULL(meta_mem_mgr_ = MTL(ObTenantMetaMemMgr *))) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "failed to get TenantMetaMemMgr from MTL", K(ret));
} else {
allocator_ = nullptr;
tablet_id_ = table->get_key().tablet_id_;
}
} else if (OB_NOT_NULL(allocator_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "tables either use an allocator or a t3m, not a mix", K(ret), KP(allocator_),
KP(meta_mem_mgr_), KP(MTL(ObTenantMetaMemMgr *)));
} else if (OB_UNLIKELY(meta_mem_mgr_ != MTL(ObTenantMetaMemMgr *))) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "tables must own the same t3m!!!", K(ret), KP(meta_mem_mgr_), KP(MTL(ObTenantMetaMemMgr *)));
} else if (OB_UNLIKELY(tablet_id_ != table->get_key().tablet_id_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "tables must belong to the same tablet!!!", K(ret), K(tablet_id_), K(table->get_key()));
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(tables_.push_back(table))) {
STORAGE_LOG(WARN, "failed to add table", K(ret));
} else {
table->inc_ref();
LOG_WARN("get invalid arguments", K(ret), KP(table));
} else if (OB_FAIL(tablet_id_check(table->get_key().get_tablet_id()))) {
LOG_WARN("failed to check tablet id", K(ret), KPC(table));
} else if (OB_FAIL(handle.set_table(table, MTL(ObTenantMetaMemMgr *), table->get_key().table_type_))) {
LOG_WARN("failed to set table to handle", K(ret));
} else if (OB_FAIL(handles_array_.push_back(handle))) {
LOG_WARN("failed to add table handle", K(ret), K(handle));
}
return ret;
}
int ObTablesHandleArray::add_table(ObTableHandleV2 &handle)
int ObTablesHandleArray::add_table(const ObTableHandleV2 &handle)
{
int ret = OB_SUCCESS;
if (OB_NOT_NULL(handle.get_allocator())) {
if (OB_FAIL(add_table(handle.get_table(), handle.get_allocator()))) {
STORAGE_LOG(WARN, "fail to add table", K(ret), K(handle));
if (OB_FAIL(tablet_id_check(handle.get_table()->get_key().get_tablet_id()))) {
LOG_WARN("failed to add table handle to array", K(ret));
} else if (OB_FAIL(handles_array_.push_back(handle))) {
STORAGE_LOG(WARN, "failed to add sstable", K(ret), K(handle));
}
return ret;
}
int ObTablesHandleArray::add_sstable(ObITable *table, const ObStorageMetaHandle &meta_handle)
{
// invalid meta_handle means table lifetime is guaranteed by tablet handle
int ret = OB_SUCCESS;
ObTableHandleV2 table_handle;
if (OB_UNLIKELY(!table->is_sstable())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid table type", K(ret), KPC(table));
} else if (OB_FAIL(tablet_id_check(table->get_key().get_tablet_id()))) {
LOG_WARN("failed to check tablet id", K(ret), KPC(table));
} else if (static_cast<ObSSTable *>(table)->is_loaded()) {
if (!meta_handle.is_valid()) {
if (OB_FAIL(table_handle.set_sstable_with_tablet(table))) {
LOG_WARN("fail to set sstable with tablet", K(ret), KPC(table));
}
} else if (OB_FAIL(table_handle.set_sstable(table, meta_handle))) {
LOG_WARN("fail to set table handle", K(ret), KPC(table));
}
} else if (OB_FAIL(add_table(handle.get_table()))) {
STORAGE_LOG(WARN, "fail to add table", K(ret), K(handle));
} else {
const ObMetaDiskAddr addr = static_cast<ObSSTable *>(table)->get_addr();
if (OB_UNLIKELY(!addr.is_valid() || addr.is_none())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(addr));
} else {
// FIXME: this reload logic looks weird here, should remove after resolve dependencies
ObStorageMetaCache &meta_cache = OB_STORE_CACHE.get_storage_meta_cache();
ObStorageMetaKey meta_key(MTL_ID(), addr);
ObStorageMetaHandle handle;
ObSSTable *sstable = nullptr;
if (OB_FAIL(meta_cache.get_meta(ObStorageMetaValue::MetaType::SSTABLE, meta_key, handle, nullptr))) {
LOG_WARN("fail to get sstable from meta cache", K(ret), K(addr));
} else if (OB_FAIL(handle.get_sstable(sstable))) {
LOG_WARN("fail to get sstable", K(ret), K(handle));
} else if (OB_FAIL(table_handle.set_sstable(sstable, handle))) {
LOG_WARN("fail to set table handle", K(ret), KPC(table), KPC(sstable));
}
}
}
if (FAILEDx(handles_array_.push_back(table_handle))) {
STORAGE_LOG(WARN, "failed to push back table", K(ret), KPC(table), K(table_handle));
}
return ret;
}
@ -611,32 +654,64 @@ int ObTablesHandleArray::assign(const ObTablesHandleArray &other)
{
int ret = OB_SUCCESS;
reset();
for (int64_t i = 0; OB_SUCC(ret) && i < other.get_count(); ++i) {
ObITable *table = other.get_table(i);
if (OB_ISNULL(table)) {
ret = OB_ERR_SYS;
STORAGE_LOG(WARN, "table must not null", K(ret), K(other));
} else if (OB_NOT_NULL(other.allocator_)) {
if (OB_FAIL(add_table(table, other.allocator_))) {
STORAGE_LOG(WARN, "fail to add table", K(ret));
}
} else if (OB_FAIL(add_table(table))) {
STORAGE_LOG(WARN, "failed to add table", K(ret));
if (OB_FAIL(add_table(other.handles_array_.at(i)))) {
LOG_WARN("fail to add table", K(ret), K(i), K(other));
}
}
return ret;
}
int ObTablesHandleArray::get_table(const int64_t idx, ObTableHandleV2 &table_handle) const
{
int ret = OB_SUCCESS;
ObITable *table = nullptr;
if (OB_UNLIKELY(idx >= handles_array_.count() || idx < 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(idx), K(handles_array_.count()));
} else {
const ObTableHandleV2 &handle = handles_array_.at(idx);
if (OB_UNLIKELY(!handle.is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected invalid table handle", K(ret), K(idx), K(handle));
} else {
table_handle = handle;
}
}
return ret;
}
int ObTablesHandleArray::get_table(const ObITable::TableKey &table_key, ObTableHandleV2 &table_handle) const
{
int ret = OB_SUCCESS;
bool found = false;
for (int64_t i = 0; OB_SUCC(ret) && !found && i < handles_array_.count(); ++i) {
const ObITable *table = nullptr;
if (OB_ISNULL(table = handles_array_.at(i).get_table())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "unexpected null table pointer");
} else if (table->get_key() == table_key) {
found = true;
if (OB_FAIL(get_table(i, table_handle))) {
STORAGE_LOG(WARN, "failed to get table by index", K(ret));
}
}
}
if (OB_SUCC(ret) && !found) {
ret = OB_ENTRY_NOT_EXIST;
}
return ret;
}
int ObTablesHandleArray::get_tables(common::ObIArray<ObITable *> &tables) const
{
int ret = OB_SUCCESS;
tables.reset();
for (int64_t i = 0; OB_SUCC(ret) && i < tables_.count(); ++i) {
if (OB_ISNULL(tables_.at(i))) {
for (int64_t i = 0; OB_SUCC(ret) && i < handles_array_.count(); ++i) {
if (OB_UNLIKELY(!handles_array_.at(i).is_valid())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get unexpected null table", K(ret), K(i), K(tables_));
} else if (OB_FAIL(tables.push_back(tables_.at(i)))) {
STORAGE_LOG(WARN, "invalid table handle", K(ret), K(i), K_(handles_array));
} else if (OB_FAIL(tables.push_back(handles_array_.at(i).table_))) {
STORAGE_LOG(WARN, "failed to add table", K(ret));
}
}
@ -647,9 +722,9 @@ int ObTablesHandleArray::get_first_memtable(memtable::ObIMemtable *&memtable) co
{
int ret = OB_SUCCESS;
memtable = nullptr;
for (int64_t i = 0; OB_SUCC(ret) && i < tables_.count(); ++i) {
if (tables_.at(i)->is_memtable()) {
memtable = static_cast<memtable::ObIMemtable*>(tables_.at(i));
for (int64_t i = 0; OB_SUCC(ret) && i < handles_array_.count(); ++i) {
if (handles_array_.at(i).get_table()->is_memtable()) {
memtable = static_cast<memtable::ObIMemtable*>(handles_array_.at(i).table_);
break;
}
}
@ -664,8 +739,8 @@ int ObTablesHandleArray::get_all_minor_sstables(common::ObIArray<ObITable *> &ta
int ret = OB_SUCCESS;
tables.reset();
for (int64_t i = 0; OB_SUCC(ret) && i < tables_.count(); ++i) {
ObITable *table = tables_.at(i);
for (int64_t i = 0; OB_SUCC(ret) && i < handles_array_.count(); ++i) {
ObITable *table = handles_array_.at(i).table_;
if (table->is_minor_sstable() && OB_FAIL(tables.push_back(table))) {
STORAGE_LOG(WARN, "failed to add minor sstable", K(ret), K(i));
}
@ -677,14 +752,14 @@ int ObTablesHandleArray::check_continues(const share::ObScnRange *scn_range) con
{
int ret = OB_SUCCESS;
if (!tables_.empty()) {
if (!handles_array_.empty()) {
// 1:check major sstable
// there can only be one major or meta merge
const ObITable *last_table = nullptr;
const ObITable *table = nullptr;
SCN base_end_scn = SCN::min_scn();
int64_t i = 0;
if (OB_ISNULL(table = tables_.at(i))) {
if (OB_ISNULL(table = handles_array_.at(i).get_table())) {
ret = OB_ERR_SYS;
LOG_WARN("table is NULL", KPC(table));
} else if (table->is_major_sstable() || table->is_meta_major_sstable()) {
@ -692,8 +767,8 @@ int ObTablesHandleArray::check_continues(const share::ObScnRange *scn_range) con
i++;
}
// 2:check minor sstable
for ( ; OB_SUCC(ret) && i < tables_.count(); ++i) {
table = tables_.at(i);
for ( ; OB_SUCC(ret) && i < handles_array_.count(); ++i) {
table = handles_array_.at(i).get_table();
if (OB_ISNULL(table)) {
ret = OB_ERR_SYS;
LOG_WARN("table is NULL", KPC(table));
@ -720,21 +795,31 @@ int ObTablesHandleArray::check_continues(const share::ObScnRange *scn_range) con
return ret;
}
int ObTablesHandleArray::tablet_id_check(const common::ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
if (0 == handles_array_.count()) {
tablet_id_ = tablet_id;
} else if (OB_UNLIKELY(tablet_id != tablet_id_)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("tablet id check in handle array failed", K(ret), K(tablet_id), K_(tablet_id));
}
return ret;
}
int64_t ObTablesHandleArray::to_string(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
if (OB_ISNULL(buf) || buf_len <= 0) {
} else {
J_OBJ_START();
J_KV(KP(meta_mem_mgr_), KP(allocator_));
J_COMMA();
J_KV("tablet_id", tablet_id_);
J_COMMA();
J_KV("table_count", tables_.count());
J_KV("table_count", handles_array_.count());
J_COMMA();
J_ARRAY_START();
for (int64_t i = 0; i < tables_.count(); ++i) {
const ObITable *table = tables_.at(i);
for (int64_t i = 0; i < handles_array_.count(); ++i) {
const ObITable *table = handles_array_.at(i).get_table();
if (NULL != table) {
J_OBJ_START();
J_KV(K(i), "table_key", table->get_key(), "ref", table->get_ref());