hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] tablet_transfer_seq(ss) + flying_tablet_pointer_map + macro_block_verification

Browse Source
This commit is contained in:
gaishun 2024-09-25 04:58:10 +00:00 committed by ob-robot
parent 02dbf9bcbd
commit 8b869c9fdd
89 changed files with 1104 additions and 288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h
View File

@ -616,7 +616,7 @@ void TestIndexBlockDataPrepare::prepare_data(const int64_t micro_block_size)
scn.convert_for_tx(SNAPSHOT_VERSION);
ObWholeDataStoreDesc desc;
ASSERT_EQ(OB_SUCCESS, desc.init(false/*is_ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION,
DATA_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), scn));
DATA_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), 0 /*transfer_seq*/, scn));
desc.get_desc().static_desc_->schema_version_ = 10;
void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder(false /* not need writer buffer*/);
@ -684,7 +684,7 @@ void TestIndexBlockDataPrepare::prepare_cg_data()
ObWholeDataStoreDesc desc;
share::SCN scn;
scn.convert_for_tx(SNAPSHOT_VERSION);
ASSERT_EQ(OB_SUCCESS, desc.init(false/*is_ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, DATA_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), scn));
ASSERT_EQ(OB_SUCCESS, desc.init(false/*is_ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, DATA_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), 0 /*transfer_seq*/, scn));
ObIArray<ObColDesc> &col_descs = desc.get_desc().col_desc_->col_desc_array_;
for (int64_t i = 0; i < col_descs.count(); ++i) {
if (col_descs.at(i).col_type_.type_ == ObIntType) {
@ -706,6 +706,7 @@ void TestIndexBlockDataPrepare::prepare_cg_data()
OK(data_desc.init(false/*is_ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_),
merge_type_, SNAPSHOT_VERSION, DATA_CURRENT_VERSION,
table_schema_.get_micro_index_clustered(),
0 /*transfer_seq*/,
scn, &cg_schema, 0));
data_desc.get_desc().static_desc_->schema_version_ = 10;
void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
@ -885,7 +886,7 @@ void TestIndexBlockDataPrepare::prepare_partial_ddl_data()
ObWholeDataStoreDesc desc;
share::SCN end_scn;
end_scn.convert_from_ts(ObTimeUtility::current_time());
ASSERT_EQ(OB_SUCCESS, desc.init(true/*is ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), end_scn));
ASSERT_EQ(OB_SUCCESS, desc.init(true/*is ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), 0 /*transfer_seq*/, end_scn));
void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
merge_root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder(false /* not need writer buffer */);
ASSERT_NE(nullptr, merge_root_index_builder_);
@ -963,7 +964,7 @@ void TestIndexBlockDataPrepare::prepare_partial_cg_data()
ObWholeDataStoreDesc desc;
share::SCN end_scn;
end_scn.convert_from_ts(ObTimeUtility::current_time());
ASSERT_EQ(OB_SUCCESS, desc.init(true/*is ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), end_scn));
ASSERT_EQ(OB_SUCCESS, desc.init(true/*is ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), 0 /*transfer_seq*/, end_scn));
void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
merge_root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder(false /* not need index buffer */);
ASSERT_NE(nullptr, merge_root_index_builder_);
@ -1193,7 +1194,7 @@ void TestIndexBlockDataPrepare::prepare_contrastive_sstable()
ObWholeDataStoreDesc desc;
share::SCN end_scn;
end_scn.convert_from_ts(ObTimeUtility::current_time());
ASSERT_EQ(OB_SUCCESS, desc.init(false/*is_ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), end_scn));
ASSERT_EQ(OB_SUCCESS, desc.init(false/*is_ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, table_schema_.get_micro_index_clustered(), 0 /*transfer_seq*/, end_scn));
void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder(false /* not need writer buffer*/);
ASSERT_NE(nullptr, root_index_builder_);

3
mittest/mtlenv/storage/blocksstable/test_clustered_index_writer.cpp
View File

@ -88,7 +88,8 @@ void TestClusteredIndexWriter::prepare_data_store_desc(
ret = data_desc.init(false/*is_ddl*/, table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), MAJOR_MERGE,
ObTimeUtility::fast_current_time() /*snapshot_version*/,
DATA_CURRENT_VERSION,
table_schema_.get_micro_index_clustered());
table_schema_.get_micro_index_clustered(),
0 /*transfer_seq*/);
data_desc.get_desc().sstable_index_builder_ = sstable_index_builder;
ASSERT_EQ(OB_SUCCESS, ret);
}

1
mittest/mtlenv/storage/blocksstable/test_index_block_row_struct.cpp
View File

@ -64,6 +64,7 @@ void TestIndexBlockRowStruct::SetUp()
static_desc.ls_id_.id_ = 1;
static_desc.tablet_id_.id_ = 1;
static_desc.compressor_type_ = ObCompressorType::NONE_COMPRESSOR;
static_desc.tablet_transfer_seq_ = 0;
data_desc.micro_block_size_ = 8 * 1024;
static_desc.micro_block_size_limit_ = 8 * 1024;
col_desc.row_column_count_ = rowkey_column_count + 1;

10
mittest/mtlenv/storage/blocksstable/test_index_tree.cpp
View File

@ -432,7 +432,7 @@ void TestIndexTree::prepare_data()
ret = data_desc.init(false/*is_ddl*/, table_schema_, ObLSID(1), ObTabletID(1), MAJOR_MERGE,
ObTimeUtility::fast_current_time()/*snapshot_version*/, DATA_CURRENT_VERSION,
table_schema_.get_micro_index_clustered());
table_schema_.get_micro_index_clustered(), 0/*transfer_seq*/);
ASSERT_EQ(OB_SUCCESS, ret);
ObMacroSeqParam seq_param;
seq_param.seq_type_ = ObMacroSeqParam::SEQ_TYPE_INC;
@ -604,7 +604,7 @@ void TestIndexTree::prepare_data_desc(ObWholeDataStoreDesc &data_desc,
int ret = OB_SUCCESS;
ret = data_desc.init(false/*is_ddl*/, table_schema_, ObLSID(1), ObTabletID(1), MAJOR_MERGE,
ObTimeUtility::fast_current_time()/*snapshot_version*/, DATA_CURRENT_VERSION,
table_schema_.get_micro_index_clustered());
table_schema_.get_micro_index_clustered(), 0/*transfer_seq*/);
data_desc.get_desc().sstable_index_builder_ = sstable_builder;
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -618,7 +618,7 @@ void TestIndexTree::prepare_cg_data_desc(ObWholeDataStoreDesc &data_desc,
scn.convert_for_tx(SNAPSHOT_VERSION);
const bool is_ddl = false;
ASSERT_EQ(OB_SUCCESS, desc.init(is_ddl, table_schema_, ObLSID(1), ObTabletID(1),
MAJOR_MERGE, SNAPSHOT_VERSION, DATA_CURRENT_VERSION, false/*micro_index_clustered*/));
MAJOR_MERGE, SNAPSHOT_VERSION, DATA_CURRENT_VERSION, false/*micro_index_clustered*/, 0/*transfer_seq*/));
ObIArray<ObColDesc> &col_descs = desc.get_desc().col_desc_->col_desc_array_;
for (int64_t i = 0; i < col_descs.count(); ++i) {
if (col_descs.at(i).col_type_.type_ == ObIntType) {
@ -636,7 +636,7 @@ void TestIndexTree::prepare_cg_data_desc(ObWholeDataStoreDesc &data_desc,
cg_schema.column_idxs_ = cg_cols;
ASSERT_EQ(OB_SUCCESS, data_desc.init(is_ddl, table_schema_, ObLSID(1), ObTabletID(1),
MAJOR_MERGE, SNAPSHOT_VERSION, DATA_CURRENT_VERSION, false/*micro_index_clustered*/,
MAJOR_MERGE, SNAPSHOT_VERSION, DATA_CURRENT_VERSION, false/*micro_index_clustered*/, 0/*transfer_seq*/,
scn, &cg_schema, 0));
data_desc.get_desc().static_desc_->schema_version_ = 10;
data_desc.get_desc().sstable_index_builder_ = sstable_builder;
@ -2337,7 +2337,7 @@ TEST_F(TestIndexTree, test_close_with_old_schema)
ObWholeDataStoreDesc index_desc;
OK(index_desc.init(false/*is_ddl*/, table_schema_, ObLSID(1), ObTabletID(1), MAJOR_MERGE,
ObTimeUtility::fast_current_time()/*snapshot*/, 0/*cluster_version*/,
table_schema_.get_micro_index_clustered()));
table_schema_.get_micro_index_clustered(), 0/*transfer_seq*/));
index_desc.static_desc_.major_working_cluster_version_ = DATA_VERSION_4_0_0_0;
--index_desc.get_desc().col_desc_->full_stored_col_cnt_;
index_desc.get_desc().col_desc_->col_default_checksum_array_.pop_back();

1
mittest/mtlenv/storage/blocksstable/test_sstable_macro_block_header.cpp
View File

@ -61,6 +61,7 @@ void TestSSTableMacroBlockHeader::SetUp()
col_desc.allocator_.set_tenant_id(500);
static_desc.ls_id_.id_ = 1;
static_desc.tablet_id_ = tablet_id;
static_desc.tablet_transfer_seq_ = 0;
data_desc.micro_block_size_ = 8 * 1024;
static_desc.micro_block_size_limit_ = 8 * 1024;
col_desc.row_column_count_ = COLUMN_COUNT;

1
mittest/mtlenv/storage/test_co_merge.cpp
View File

@ -396,6 +396,7 @@ void TestCOMerge::prepare_co_sstable(
snapshot_version,
DATA_CURRENT_VERSION,
table_schema.get_micro_index_clustered(),
0 /*tablet_transfer_seq*/,
share::SCN::invalid_scn(),
&cg_schema,
i));

54
mittest/mtlenv/storage/test_ls_tablet_service.cpp
View File

@ -178,7 +178,7 @@ void TestLSTabletService::construct_and_get_tablet_list(
ASSERT_EQ(OB_SUCCESS, ret);
ret = ls_tablet_service_->get_tablet(node_tablet_id, tmp_tablet_handle_tail);
ASSERT_EQ(OB_SUCCESS, ret);
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tmp_tablet_handle_head.get_obj()->get_transfer_seq());
ret = ObTabletPersister::persist_and_transform_tablet(persist_param, *tmp_tablet_handle_head.get_obj(), tablet_handle_head);
ASSERT_EQ(OB_SUCCESS, ret);
@ -270,6 +270,7 @@ TEST_F(TestLSTabletService, test_create_tablet_without_index)
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_serialize_tablet)
@ -294,7 +295,7 @@ TEST_F(TestLSTabletService, test_serialize_tablet)
const ObTablet *orig_tablet = orig_tablet_handle.get_obj();
ObTabletHandle tiny_tablet_handle;
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id, orig_tablet->get_transfer_seq());
ret = ObTabletPersister::persist_and_transform_tablet(persist_param, *orig_tablet, tiny_tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
@ -305,6 +306,8 @@ TEST_F(TestLSTabletService, test_serialize_tablet)
const ObMetaDiskAddr &tablet_addr = tiny_tablet->tablet_addr_;
ret = tiny_tablet->serialize(buf, tablet_length, pos);
ASSERT_EQ(OB_SUCCESS, ret);
orig_tablet_handle.reset();
tiny_tablet_handle.reset();
const ObTabletMapKey key(ls_id_, tablet_id);
@ -319,6 +322,7 @@ TEST_F(TestLSTabletService, test_serialize_tablet)
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(tablet_id, new_4k_tablet->tablet_meta_.tablet_id_);
ASSERT_EQ(OB_SUCCESS, new_4k_tablet->inc_macro_ref_cnt());
new_4k_tablet_handle.reset();
ObTabletHandle new_tmp_tablet_handle;
ret = ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator_, new_tmp_tablet_handle);
@ -330,11 +334,12 @@ TEST_F(TestLSTabletService, test_serialize_tablet)
ret = new_tmp_tablet->deserialize(allocator_, buf, tablet_length, de_pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(tablet_id, new_tmp_tablet->tablet_meta_.tablet_id_);
new_tmp_tablet_handle.reset();
ob_free(buf);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
/**
@ -569,6 +574,7 @@ TEST_F(TestLSTabletService, test_create_tablet_with_index)
ASSERT_EQ(OB_SUCCESS, ret);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, index_tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_create_index_tablet)
@ -616,13 +622,14 @@ TEST_F(TestLSTabletService, test_create_index_tablet)
ASSERT_EQ(OB_SUCCESS, ret);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, index_tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_get_ls_min_end_scn)
{
// create_tablet_without_index
int ret = OB_SUCCESS;
ObTabletID tablet_id(10000009);
ObTabletID tablet_id(10000019);
share::schema::ObTableSchema schema;
TestSchemaUtils::prepare_data_schema(schema);
@ -639,6 +646,7 @@ TEST_F(TestLSTabletService, test_get_ls_min_end_scn)
ObTabletHandle tablet_handle;
ret = t3m->get_tablet(WashTabletPriority::WTP_HIGH, key, tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
tablet_handle.reset();
share::SCN expect_scn;
expect_scn.val_ = 0;
@ -650,6 +658,7 @@ TEST_F(TestLSTabletService, test_get_ls_min_end_scn)
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_replay_empty_shell)
@ -659,6 +668,7 @@ TEST_F(TestLSTabletService, test_replay_empty_shell)
ObTabletID tablet_id(10000009);
share::schema::ObTableSchema schema;
TestSchemaUtils::prepare_data_schema(schema);
LOG_INFO("FEIDU test_replay_empty_shell", K(ls_id_), K(tablet_id));
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
@ -706,8 +716,10 @@ TEST_F(TestLSTabletService, test_replay_empty_shell)
test_tablet_handle.get_obj()->fetch_table_store(wrapper);
ASSERT_EQ(nullptr, wrapper.get_member()->get_major_sstables().get_boundary_table(true));
ASSERT_EQ(tablet_addr, test_tablet_handle.get_obj()->tablet_addr_);
test_tablet_handle.reset();
LOG_INFO("FEIDU first remove", K(ls_id_), K(tablet_id));
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
LOG_INFO("FEIDU first success", K(ls_id_), K(tablet_id));
ASSERT_EQ(OB_SUCCESS, ret);
ObLogCursor replay_start_cursor_;
@ -734,9 +746,13 @@ TEST_F(TestLSTabletService, test_replay_empty_shell)
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *empty_shell_tablet = tablet_handle.get_obj();
ASSERT_EQ(true, empty_shell_tablet->is_empty_shell());
tablet_handle.reset();
LOG_INFO("FEIDU seconde remove", K(ls_id_), K(tablet_id));
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
LOG_INFO("FEIDU seconde success", K(ls_id_), K(tablet_id));
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_cover_empty_shell)
@ -763,6 +779,7 @@ TEST_F(TestLSTabletService, test_cover_empty_shell)
ObMigrationTabletParam param;
ret = old_tablet_handle.get_obj()->build_migration_tablet_param(param);
ASSERT_EQ(OB_SUCCESS, ret);
old_tablet_handle.reset();
ret = ls_tablet_service_->update_tablet_to_empty_shell(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
@ -772,6 +789,7 @@ TEST_F(TestLSTabletService, test_cover_empty_shell)
ObTabletCreateDeleteMdsUserData user_data;
ASSERT_EQ(OB_SUCCESS, test_tablet_handle.get_obj()->get_tablet_status(share::SCN::max_scn(), user_data));
ASSERT_EQ(ObTabletStatus::DELETED, user_data.tablet_status_);
test_tablet_handle.reset();
ObTabletHandle tablet_handle;
ret = ls_tablet_service_->create_transfer_in_tablet(ls_id_, param, tablet_handle);
@ -780,6 +798,8 @@ TEST_F(TestLSTabletService, test_cover_empty_shell)
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
tablet_handle.reset();
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_migrate_empty_shell)
@ -810,6 +830,7 @@ TEST_F(TestLSTabletService, test_migrate_empty_shell)
tablet_meta.reset();
ret = old_tablet_handle.get_obj()->build_migration_tablet_param(tablet_meta);
ASSERT_EQ(OB_SUCCESS, ret);
old_tablet_handle.reset();
char buf[4096] = {0};
int64_t pos = 0;
@ -834,9 +855,11 @@ TEST_F(TestLSTabletService, test_migrate_empty_shell)
ObTablet *empty_shell_tablet = tablet_handle.get_obj();
ASSERT_EQ(true, empty_shell_tablet->is_empty_shell());
tablet_handle.reset();
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_serialize_sstable_full_and_shell)
@ -872,13 +895,14 @@ TEST_F(TestLSTabletService, test_serialize_sstable_full_and_shell)
pos = 0;
ret = sstable.deserialize(allocator_, full_buf, size, pos);
ASSERT_EQ(common::OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_migrate_param)
{
// create_tablet_without_index
int ret = OB_SUCCESS;
ObTabletID tablet_id(10000011);
ObTabletID tablet_id(10000012);
share::schema::ObTableSchema schema;
TestSchemaUtils::prepare_data_schema(schema);
common::ObArenaAllocator allocator;
@ -899,6 +923,7 @@ TEST_F(TestLSTabletService, test_migrate_param)
tablet_meta.reset();
ret = old_tablet_handle.get_obj()->build_migration_tablet_param(tablet_meta);
ASSERT_EQ(OB_SUCCESS, ret);
old_tablet_handle.reset();
int64_t serialize_size = tablet_meta.get_serialize_size();
char *buf = static_cast<char*>(allocator.alloc(serialize_size));
@ -928,13 +953,14 @@ TEST_F(TestLSTabletService, test_migrate_param)
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_migrate_param_empty_shell)
{
// create_tablet_without_index
int ret = OB_SUCCESS;
ObTabletID tablet_id(10000011);
ObTabletID tablet_id(10000013);
share::schema::ObTableSchema schema;
TestSchemaUtils::prepare_data_schema(schema);
common::ObArenaAllocator allocator;
@ -958,6 +984,7 @@ TEST_F(TestLSTabletService, test_migrate_param_empty_shell)
tablet_meta.reset();
ret = old_tablet_handle.get_obj()->build_migration_tablet_param(tablet_meta);
ASSERT_EQ(OB_SUCCESS, ret);
old_tablet_handle.reset();
int64_t serialize_size = tablet_meta.get_serialize_size();
char *buf = static_cast<char*>(allocator.alloc(serialize_size));
@ -985,13 +1012,14 @@ TEST_F(TestLSTabletService, test_migrate_param_empty_shell)
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, test_update_empty_shell)
{
// create_tablet_without_index
int ret = OB_SUCCESS;
ObTabletID tablet_id(10000009);
ObTabletID tablet_id(10000014);
share::schema::ObTableSchema schema;
TestSchemaUtils::prepare_data_schema(schema);
ObLSHandle ls_handle;
@ -1007,9 +1035,11 @@ TEST_F(TestLSTabletService, test_update_empty_shell)
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *empty_shell_tablet = tablet_handle.get_obj();
ASSERT_TRUE(empty_shell_tablet->is_empty_shell());
tablet_handle.reset();
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
valid_tablet_num(3);
}
TEST_F(TestLSTabletService, update_tablet_release_memtable_for_offline)
@ -1039,10 +1069,15 @@ TEST_F(TestLSTabletService, update_tablet_release_memtable_for_offline)
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->create_memtable(data_tablet_id, 100, false, false));
ASSERT_EQ(1, tablet_handle.get_obj()->memtable_count_);
LOG_WARN("FEIDU: release memtable 1");
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->update_tablet_release_memtable_for_offline(data_tablet_id, SCN::max_scn()));
LOG_WARN("FEIDU: release memtable 2");
ASSERT_EQ(OB_TABLET_NOT_EXIST, ls_handle.get_ls()->get_tablet_svr()->update_tablet_release_memtable_for_offline(data_tablet_id, SCN::max_scn()));
LOG_WARN("FEIDU: release memtable done");
tablet_handle.reset();
ret = ls_tablet_service_->do_remove_tablet(ls_id_, data_tablet_id);
LOG_WARN("FEIDU: release memtable remove tablet");
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -1079,7 +1114,7 @@ TEST_F(TestLSTabletService, update_tablet_ddl_commit_scn)
ObTabletHandle new_tablet_hdl;
ObUpdateTabletPointerParam param;
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), data_tablet_id);
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), data_tablet_id, tablet_handle.get_obj()->get_transfer_seq());
ASSERT_EQ(OB_SUCCESS, ObTabletPersister::persist_and_transform_tablet(persist_param, *tablet_handle.get_obj(), new_tablet_hdl));
ret = new_tablet_hdl.get_obj()->get_updating_tablet_pointer_param(param);
ASSERT_EQ(OB_SUCCESS, ret);
@ -1089,6 +1124,7 @@ TEST_F(TestLSTabletService, update_tablet_ddl_commit_scn)
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->get_tablet(data_tablet_id, tablet_handle));
ASSERT_EQ(ddl_commit_scn, tablet_handle.get_obj()->tablet_meta_.ddl_commit_scn_);
tablet_handle.reset();
ret = ls_tablet_service_->do_remove_tablet(ls_id_, data_tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}

2
mittest/mtlenv/storage/test_meta_snapshot.cpp
View File

@ -94,7 +94,7 @@ int TestMetaSnapshot::persist_tablet(ObTabletHandle &new_tablet_handle)
} else if (OB_FAIL(meta_service->get_shared_object_reader_writer().switch_object(object_handle, default_opt))) {
LOG_WARN("fail to switch shared meta block", K(ret));
} else {
ObTabletPersisterParam persister_param(ls_id, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
ObTabletPersisterParam persister_param(ls_id, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet_handle.get_obj()->get_transfer_seq());
if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(persister_param, *(tablet_handle.get_obj()), new_tablet_handle))) {
LOG_WARN("fail to persist and transform tablet", K(ret), K(tablet_handle));
} else if (OB_FAIL(new_tablet_handle.get_obj()->get_updating_tablet_pointer_param(param))) {

2
mittest/mtlenv/storage/test_tablet_aggregated_info.cpp
View File

@ -68,7 +68,7 @@ TEST_F(TestTabletAggregatedInfo, test_space_usage)
// check tablet's space_usage with empty major sstable
ObTabletHandle new_tablet_handle;
const ObTabletPersisterParam param(ls_id, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
ASSERT_EQ(OB_SUCCESS, ObTabletPersister::persist_and_transform_tablet(param, *tablet, new_tablet_handle));
ObTabletSpaceUsage space_usage = new_tablet_handle.get_obj()->tablet_meta_.space_usage_;
ASSERT_EQ(0, space_usage.all_sstable_data_occupy_size_);

4
mittest/mtlenv/storage/test_tablet_block_id_list.cpp
View File

@ -224,7 +224,7 @@ TEST_F(TestBlockIdList, test_meta_macro_ref_cnt)
blocksstable::ObStorageObjectOpt default_opt;
// persist 4k tablet
const ObTabletPersisterParam param(ls_id, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
ObTenantStorageMetaService *meta_service = MTL(ObTenantStorageMetaService*);
ASSERT_EQ(OB_SUCCESS, meta_service->get_shared_object_reader_writer().switch_object(object_handle, default_opt));
ASSERT_EQ(OB_SUCCESS, ObTabletPersister::persist_and_transform_tablet(param, *tablet, new_tablet_handle));
@ -417,7 +417,7 @@ TEST_F(TestBlockIdList, test_empty_shell_macro_ref_cnt)
ObBlockManager::BlockInfo block_info;
int64_t ref_cnt = 0;
ObTabletHandle new_tablet_handle;
const ObTabletPersisterParam param(ls_id, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
ASSERT_EQ(OB_SUCCESS, ObTabletPersister::persist_and_transform_tablet(param, *tablet, new_tablet_handle));
ObTablet *new_tablet = new_tablet_handle.get_obj();
ASSERT_EQ(OB_SUCCESS, new_tablet->tablet_addr_.get_block_addr(macro_id, offset, size));

2
mittest/mtlenv/storage/test_tablet_member_load_and_free.cpp
View File

@ -171,7 +171,7 @@ TEST_F(TestTabletMemberLoadAndFree, storage_schema)
ObLSHandle ls_handle;
ASSERT_EQ(OB_SUCCESS, MTL(ObLSService*)->get_ls(LS_ID, ls_handle, ObLSGetMod::STORAGE_MOD));
ObTabletPersisterParam persister_param(
ls_handle.get_ls()->get_ls_id(), ls_handle.get_ls()->get_ls_epoch(), tablet_id);
ls_handle.get_ls()->get_ls_id(), ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
ret = ObTabletPersister::persist_and_transform_tablet(persister_param, *tablet, new_tablet_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *new_tablet = new_tablet_handle.get_obj();

24
mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp
View File

@ -20,6 +20,7 @@
#include "lib/alloc/memory_dump.h"
#include "storage/tablet/ob_tablet_persister.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
#include "storage/meta_mem/ob_tablet_map_key.h"
#include "storage/meta_mem/ob_tablet_leak_checker.h"
#include "storage/ls/ob_ls.h"
#include "storage/schema_utils.h"
@ -336,6 +337,7 @@ void TestConcurrentT3M::run1()
SpinWLockGuard guard(lock_);
int ret = t3m_.create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle);
handle.t3m_ = &t3m_; // temporary code, use local t3m
handle.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_TRUE(handle.is_valid());
}
@ -387,6 +389,7 @@ void TestConcurrentT3M::run1()
if (0 == count % N) {
ret = t3m_.del_tablet(key);
ASSERT_EQ(common::OB_SUCCESS, ret);
tmp_handle.reset();
} else {
ret = t3m_.tablet_map_.erase(key, tmp_handle);
ASSERT_EQ(common::OB_SUCCESS, ret);
@ -577,6 +580,7 @@ TEST_F(TestTenantMetaMemMgr, test_tablet)
ret = t3m_.create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle);
handle.t3m_ = &t3m_; // temporary code, use local t3m
handle.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(0, t3m_.tablet_buffer_pool_.inner_used_num_);
ASSERT_EQ(1, t3m_.tablet_map_.map_.size());
@ -688,6 +692,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_tablet)
ret = t3m_.create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle);
handle.t3m_ = &t3m_; // temporary code, use local t3m
handle.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(0, t3m_.tablet_buffer_pool_.inner_used_num_);
ASSERT_EQ(1, t3m_.tablet_map_.map_.size());
@ -727,7 +732,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_tablet)
false/*micro_index_clustered*/, false/*need_generate_cs_replica_cg_array*/, false/*has_cs_replica*/, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
ObTabletPersister persister(persist_param, ObCtxIds::DEFAULT_CTX_ID);
ObSArray<MacroBlockId> shared_meta_id_arr;
@ -787,6 +792,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_inner_tablet)
ret = t3m_.create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle);
handle.t3m_ = &t3m_; // temporary code, use local t3m
handle.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(0, t3m_.tablet_buffer_pool_.inner_used_num_);
ASSERT_EQ(1, t3m_.tablet_map_.map_.size());
@ -830,7 +836,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_inner_tablet)
ASSERT_EQ(1, tablet->get_ref());
ObTabletHandle new_handle;
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
ObTabletPersister persister(persist_param, ObCtxIds::DEFAULT_CTX_ID);
ObSArray<MacroBlockId> shared_meta_id_arr;
@ -902,6 +908,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_no_sstable_tablet)
ret = t3m_.create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle);
handle.t3m_ = &t3m_; // temporary code, use local t3m
handle.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(0, t3m_.tablet_buffer_pool_.inner_used_num_);
ASSERT_EQ(1, t3m_.tablet_map_.map_.size());
@ -943,7 +950,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_no_sstable_tablet)
ASSERT_EQ(1, tablet->get_ref());
ObTabletHandle new_handle;
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
ObTabletPersister persister(persist_param, ObCtxIds::DEFAULT_CTX_ID);
ObSArray<MacroBlockId> shared_meta_id_arr;
ASSERT_EQ(common::OB_SUCCESS, t3m_.acquire_tablet_from_pool(ObTabletPoolType::TP_NORMAL, WashTabletPriority::WTP_HIGH, key, new_handle));
@ -993,6 +1000,7 @@ TEST_F(TestTenantMetaMemMgr, test_get_tablet_with_allocator)
ret = t3m_.create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle);
handle.t3m_ = &t3m_; // temporary code, use local t3m
handle.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(0, t3m_.tablet_buffer_pool_.inner_used_num_);
ASSERT_EQ(1, t3m_.tablet_map_.map_.size());
@ -1042,7 +1050,7 @@ TEST_F(TestTenantMetaMemMgr, test_get_tablet_with_allocator)
ASSERT_EQ(1, tablet->get_ref());
ObTabletHandle new_handle;
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id_, ls_handle.get_ls()->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
ObTabletPersister persister(persist_param, ObCtxIds::DEFAULT_CTX_ID);
ObSArray<MacroBlockId> shared_meta_id_arr;
ASSERT_EQ(common::OB_SUCCESS, t3m_.acquire_tablet_from_pool(ObTabletPoolType::TP_NORMAL, WashTabletPriority::WTP_HIGH, key, new_handle));
@ -1098,6 +1106,7 @@ TEST_F(TestTenantMetaMemMgr, test_wash_mem_tablet)
ret = t3m_.create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle);
handle.t3m_ = &t3m_; // temporary code, use local t3m
handle.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(0, t3m_.tablet_buffer_pool_.inner_used_num_);
ASSERT_EQ(1, t3m_.tablet_map_.map_.size());
@ -1225,6 +1234,7 @@ TEST_F(TestTenantMetaMemMgr, test_replace_tablet)
ret = t3m_.create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle);
handle.t3m_ = &t3m_; // temporary code, use local t3m
handle.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(common::OB_SUCCESS, ret);
tablet = handle.get_obj();
ASSERT_TRUE(nullptr != tablet);
@ -1241,6 +1251,7 @@ TEST_F(TestTenantMetaMemMgr, test_replace_tablet)
ret = t3m_.create_msd_tablet(WashTabletPriority::WTP_HIGH, key, ls_handle, handle);
handle.t3m_ = &t3m_; // temporary code, use local t3m
handle.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(0, t3m_.tablet_buffer_pool_.inner_used_num_);
ASSERT_EQ(1, t3m_.tablet_map_.map_.size());
@ -1556,6 +1567,7 @@ TEST_F(TestTenantMetaMemMgr, test_leak_checker)
ObTabletMapKey(ls_id_, tablet_id_1),
ls_handle,
tb_handle_1);
tb_handle_1.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(MTL(ObTenantMetaMemMgr*)->leak_checker_.tb_ref_bucket_[index_1], 1);
ASSERT_EQ(functor(), initial_size + 1);
LOG_INFO("test_leak_checker, should be log pinned tablet info here (maybe n lines)");
@ -1577,6 +1589,7 @@ TEST_F(TestTenantMetaMemMgr, test_leak_checker)
ObTabletMapKey(ls_id_, tablet_id_2),
ls_handle,
tb_handle_2);
tb_handle_2.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(MTL(ObTenantMetaMemMgr*)->leak_checker_.tb_ref_bucket_[index_2], 1);
ASSERT_EQ(functor(), initial_size + 1);
LOG_INFO("test_leak_checker, should be log pinned tablet info here (maybe n lines)");
@ -1620,6 +1633,7 @@ TEST_F(TestTenantMetaMemMgr, test_leak_checker_copy)
MTL(ObTenantMetaMemMgr *)->create_msd_tablet(WashTabletPriority::WTP_HIGH,
ObTabletMapKey(ls_id_, tablet_id_1),
ls_handle, tb_handle_1));
tb_handle_1.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(MTL(ObTenantMetaMemMgr*)->leak_checker_.tb_ref_bucket_[index_1], 1);
ObTabletHandle tb_handle_2;
tb_handle_2 = tb_handle_1;
@ -1651,6 +1665,7 @@ TEST_F(TestTenantMetaMemMgr, test_leak_checker_switch)
MTL(ObTenantMetaMemMgr *)->create_msd_tablet(WashTabletPriority::WTP_HIGH,
ObTabletMapKey(ls_id_, tablet_id_1),
ls_handle, tb_handle_1));
tb_handle_1.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(MTL(ObTenantMetaMemMgr*)->leak_checker_.tb_ref_bucket_[index_1], 1);
// new handle_2 with leak_checker disabled
@ -1664,6 +1679,7 @@ TEST_F(TestTenantMetaMemMgr, test_leak_checker_switch)
MTL(ObTenantMetaMemMgr *)->create_msd_tablet(WashTabletPriority::WTP_HIGH,
ObTabletMapKey(ls_id_, tablet_id_2),
ls_handle, tb_handle_2));
tb_handle_2.get_obj()->tablet_meta_.transfer_info_.transfer_seq_ = 0;
ASSERT_EQ(MTL(ObTenantMetaMemMgr*)->leak_checker_.tb_ref_bucket_[index_2], 0);
// destroy handle_1 with leak_checker disabled, dec ref failed (switch-off)

2
mittest/mtlenv/storage/test_write_tablet_slog.cpp
View File

@ -128,7 +128,7 @@ TEST_F(TestWriteTabletSlog, basic)
// persist and transform tablet
const ObTabletMapKey key(ls_id, tablet_id);
ObTabletHandle new_tablet_hdl;
const ObTabletPersisterParam param(ls_id, ls->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls->get_ls_epoch(), tablet_id, 0 /*transfer_seq*/);
ASSERT_EQ(OB_SUCCESS, ObTabletPersister::persist_and_transform_tablet(param, *tablet, new_tablet_hdl));
// write create tablet slog

4
src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.cpp
View File

@ -234,7 +234,7 @@ int ObAllVirtualTabletSSTableMacroInfo::get_macro_info(
} else if (macro_id.is_shared_data_or_meta()) {
info.macro_block_index_ = macro_id.third_id();
} else if (macro_id.is_private_data_or_meta()) {
info.macro_block_index_ = macro_id.fourth_id();
info.macro_block_index_ = macro_id.tenant_seq();
}
info.macro_block_id_ = macro_id;
info.row_count_ = macro_header.fixed_header_.row_count_;
@ -294,7 +294,7 @@ int ObAllVirtualTabletSSTableMacroInfo::get_macro_info(
} else if (macro_desc.macro_block_id_.is_shared_data_or_meta()) {
info.macro_block_index_ = macro_desc.macro_block_id_.third_id();
} else if (macro_desc.macro_block_id_.is_private_data_or_meta()) {
info.macro_block_index_ = macro_desc.macro_block_id_.fourth_id();
info.macro_block_index_ = macro_desc.macro_block_id_.tenant_seq();
}
info.macro_block_id_ = macro_desc.macro_block_id_;
info.row_count_ = macro_desc.row_count_;

7
src/storage/CMakeLists.txt
View File

@ -672,21 +672,22 @@ ob_set_subtarget(ob_storage common
)
ob_set_subtarget(ob_storage common_mixed
checkpoint/ob_checkpoint_diagnose.cpp
checkpoint/ob_checkpoint_executor.cpp
checkpoint/ob_data_checkpoint.cpp
checkpoint/ob_freeze_checkpoint.cpp
meta_mem/ob_tablet_leak_checker.cpp
checkpoint/ob_checkpoint_diagnose.cpp
meta_mem/ob_external_tablet_cnt_map.cpp
meta_mem/ob_flying_tablet_pointer_map.cpp
meta_mem/ob_meta_obj_struct.cpp
meta_mem/ob_storage_meta_cache.cpp
meta_mem/ob_tablet_handle.cpp
meta_mem/ob_tablet_leak_checker.cpp
meta_mem/ob_tablet_map_key.cpp
meta_mem/ob_tablet_pointer_map.cpp
meta_mem/ob_tablet_pointer_handle.cpp
meta_mem/ob_tablet_pointer.cpp
meta_mem/ob_tenant_meta_mem_mgr.cpp
meta_mem/ob_tenant_meta_obj_pool.cpp
meta_mem/ob_storage_meta_cache.cpp
utl_file/ob_utl_file_handler.cpp
blockstore/ob_shared_object_reader_writer.cpp
concurrency_control/ob_multi_version_garbage_collector.cpp

2
src/storage/backup/ob_backup_index_block_builder_mgr.cpp
View File

@ -831,6 +831,7 @@ int ObBackupTabletSSTableIndexBuilderMgr::prepare_data_store_desc_(const share::
tablet->get_snapshot_version(),
0/*cluster_version*/,
false/*micro_index_clustered*/,
tablet->get_transfer_seq(),
table_key.get_end_scn()))) {
LOG_WARN("failed to init static desc", K(ret), KPC(storage_schema));
}
@ -861,6 +862,7 @@ int ObBackupTabletSSTableIndexBuilderMgr::prepare_data_store_desc_(const share::
tablet->get_snapshot_version(),
0/*cluster_version*/,
false/*micro_index_clustered*/,
tablet->get_transfer_seq(),
table_key.get_end_scn(),
cg_schema,
cg_idx))) {

6
src/storage/blocksstable/index_block/ob_sstable_meta_info.cpp
View File

@ -814,6 +814,7 @@ int ObSSTableMacroInfo::serialize_(char *buf, const int64_t buf_len, int64_t &po
int ObSSTableMacroInfo::persist_block_ids(
const ObTabletID &tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
common::ObArenaAllocator &allocator,
storage::ObSSTableLinkBlockWriteInfo * const link_write_info,
@ -821,7 +822,7 @@ int ObSSTableMacroInfo::persist_block_ids(
{
int ret = OB_SUCCESS;
ObLinkedMacroBlockItemWriter block_writer;
if (OB_FAIL(write_block_ids(tablet_id, snapshot_version, block_writer, entry_id_, link_write_info))) {
if (OB_FAIL(write_block_ids(tablet_id, tablet_transfer_seq, snapshot_version, block_writer, entry_id_, link_write_info))) {
LOG_WARN("fail to write other block ids", K(ret), KPC(link_write_info));
} else if (OB_FAIL(save_linked_block_list(block_writer.get_meta_block_list(), allocator))) {
LOG_WARN("fail to save linked block ids", K(ret));
@ -1150,6 +1151,7 @@ DEF_TO_STRING(ObSSTableMacroInfo)
int ObSSTableMacroInfo::write_block_ids(
const ObTabletID &tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
storage::ObLinkedMacroBlockItemWriter &writer,
MacroBlockId &entry_id,
@ -1164,7 +1166,7 @@ int ObSSTableMacroInfo::write_block_ids(
ret = OB_ERR_UNEXPECTED;
LOG_WARN("data_block_count_ and other_block_count_ shouldn't be both 0", K(ret), K(data_block_count_),
K(other_block_count_));
} else if (OB_FAIL(writer.init_for_object(tablet_id.id(), snapshot_version, link_write_info->start_macro_seq_, link_write_info->get_ddl_redo_callback()))) {
} else if (OB_FAIL(writer.init_for_object(tablet_id.id(), tablet_transfer_seq, snapshot_version, link_write_info->start_macro_seq_, link_write_info->get_ddl_redo_callback()))) {
LOG_WARN("fail to initialize item writer", K(ret), KPC(link_write_info));
} else if (OB_NOT_NULL(data_block_ids_) && OB_FAIL(flush_ids(data_block_ids_, data_block_count_, writer))) {
LOG_WARN("fail to flush data block ids", K(ret), K(data_block_count_));

2
src/storage/blocksstable/index_block/ob_sstable_meta_info.h
View File

@ -235,12 +235,14 @@ private:
int64_t &other_block_count);
int persist_block_ids(
const ObTabletID &tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
common::ObArenaAllocator &allocator,
storage::ObSSTableLinkBlockWriteInfo * const link_write_info,
ObSharedObjectsWriteCtx &linked_block_write_ctx);
int write_block_ids(
const ObTabletID &tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
storage::ObLinkedMacroBlockItemWriter &writer,
MacroBlockId &entry_id,

14
src/storage/blocksstable/ob_data_store_desc.cpp
View File

@ -28,16 +28,22 @@ ObStaticDataStoreDesc::ObStaticDataStoreDesc()
bool ObStaticDataStoreDesc::is_valid() const
{
return ls_id_.is_valid()
bool is_valid =
ls_id_.is_valid()
&& tablet_id_.is_valid()
&& compressor_type_ > ObCompressorType::INVALID_COMPRESSOR
&& snapshot_version_ > 0
&& schema_version_ >= 0;
if (GCTX.is_shared_storage_mode()) {
is_valid &= (tablet_transfer_seq_ != ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ);
}
return is_valid;
}
void ObStaticDataStoreDesc::reset()
{
MEMSET(this, 0, sizeof(*this));
tablet_transfer_seq_ = ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ;
need_submit_io_ = true;
}
@ -49,6 +55,7 @@ int ObStaticDataStoreDesc::assign(const ObStaticDataStoreDesc &desc)
compressor_type_ = desc.compressor_type_;
ls_id_ = desc.ls_id_;
tablet_id_ = desc.tablet_id_;
tablet_transfer_seq_ = desc.tablet_transfer_seq_;
macro_block_size_ = desc.macro_block_size_;
macro_store_size_ = desc.macro_store_size_;
micro_block_size_limit_ = desc.micro_block_size_limit_;
@ -107,6 +114,7 @@ int ObStaticDataStoreDesc::init(
const ObMergeSchema &merge_schema,
const share::ObLSID &ls_id,
const common::ObTabletID tablet_id,
const int64_t tablet_transfer_seq,
const compaction::ObMergeType merge_type,
const int64_t snapshot_version,
const share::SCN &end_scn,
@ -129,6 +137,7 @@ int ObStaticDataStoreDesc::init(
merge_type_ = merge_type;
ls_id_ = ls_id;
tablet_id_ = tablet_id;
tablet_transfer_seq_ = tablet_transfer_seq;
exec_mode_ = exec_mode;
encoding_granularity_ = encoding_granularity;
@ -891,6 +900,7 @@ int ObWholeDataStoreDesc::init(
const int64_t snapshot_version,
const int64_t cluster_version,
const bool micro_index_clustered,
const int64_t tablet_transfer_seq,
const share::SCN &end_scn,
const storage::ObStorageColumnGroupSchema *cg_schema,
const uint16_t table_cg_idx,
@ -909,7 +919,7 @@ int ObWholeDataStoreDesc::init(
}
}
if (OB_FAIL(static_desc_.init(is_ddl, merge_schema, ls_id, tablet_id, merge_type,
if (OB_FAIL(static_desc_.init(is_ddl, merge_schema, ls_id, tablet_id, tablet_transfer_seq, merge_type,
snapshot_version, end_scn, cluster_version,
exec_mode, micro_index_clustered, need_submit_io, encoding_granularity))) {
STORAGE_LOG(WARN, "failed to init static desc", KR(ret));

5
src/storage/blocksstable/ob_data_store_desc.h
View File

@ -54,6 +54,7 @@ public:
const share::schema::ObMergeSchema &merge_schema,
const share::ObLSID &ls_id,
const common::ObTabletID tablet_id,
const int64_t tablet_transfer_seq,
const compaction::ObMergeType merge_type,
const int64_t snapshot_version,
const share::SCN &end_scn,
@ -68,6 +69,7 @@ public:
TO_STRING_KV(
K_(ls_id),
K_(tablet_id),
K_(tablet_transfer_seq),
"merge_type", merge_type_to_str(merge_type_),
K_(snapshot_version),
K_(end_scn),
@ -99,6 +101,7 @@ public:
ObCompressorType compressor_type_;
share::ObLSID ls_id_;
ObTabletID tablet_id_;
int64_t tablet_transfer_seq_;
int64_t macro_block_size_;
int64_t macro_store_size_; //macro_block_size_ * reserved_percent
int64_t micro_block_size_limit_;
@ -270,6 +273,7 @@ public:
STATIC_DESC_FUNC(compaction::ObMergeType, merge_type);
STATIC_DESC_FUNC(const share::ObLSID&, ls_id);
STATIC_DESC_FUNC(const ObTabletID&, tablet_id);
STATIC_DESC_FUNC(int64_t, tablet_transfer_seq);
STATIC_DESC_FUNC(int64_t, progressive_merge_round);
STATIC_DESC_FUNC(int64_t, schema_version);
STATIC_DESC_FUNC(int64_t, snapshot_version);
@ -368,6 +372,7 @@ struct ObWholeDataStoreDesc
const int64_t snapshot_version,
const int64_t cluster_version,
const bool micro_index_clustered,
const int64_t tablet_transfer_seq,
const share::SCN &end_scn = share::SCN::invalid_scn(),
const storage::ObStorageColumnGroupSchema *cg_schema = nullptr,
const uint16_t table_cg_idx = 0,

23
src/storage/blocksstable/ob_macro_block_id.cpp
View File

@ -13,6 +13,7 @@
#define USING_LOG_PREFIX STORAGE
#include "lib/utility/ob_print_utils.h"
#include "storage/blocksstable/ob_macro_block_id.h"
#include "storage/blocksstable/ob_object_manager.h"
#include "storage/backup/ob_backup_data_struct.h"
namespace oceanbase
@ -89,11 +90,20 @@ bool MacroBlockId::is_valid() const
bool is_valid = true;
if (id_mode_ == (uint64_t)ObMacroBlockIdMode::ID_MODE_LOCAL) {
is_valid = second_id_ >= AUTONOMIC_BLOCK_INDEX && second_id_ < INT64_MAX && third_id_ >= 0;
is_valid &= MACRO_BLOCK_ID_VERSION_V2 == version_ && id_mode_ < (uint64_t)ObMacroBlockIdMode::ID_MODE_MAX;
is_valid &= second_id_ >= AUTONOMIC_BLOCK_INDEX && second_id_ < INT64_MAX && third_id_ >= 0;
} else if (id_mode_ == (uint64_t)ObMacroBlockIdMode::ID_MODE_BACKUP) {
is_valid = backup::ObBackupDeviceMacroBlockId::check_valid(first_id_, second_id_, third_id_);
} else {
is_valid = MACRO_BLOCK_ID_VERSION_V2 == version_ && id_mode_ < (uint64_t)ObMacroBlockIdMode::ID_MODE_MAX;
// BACKUP_MODE use BACKUP_MACRO_BLOCK_ID_VERSION
is_valid &= backup::ObBackupDeviceMacroBlockId::check_valid(first_id_, second_id_, third_id_);
} else if (is_valid && id_mode_ == (uint64_t)ObMacroBlockIdMode::ID_MODE_SHARE) {
is_valid &= MACRO_BLOCK_ID_VERSION_V2 == version_ && id_mode_ < (uint64_t)ObMacroBlockIdMode::ID_MODE_MAX;
if (is_private_data_or_meta()) {
is_valid &= meta_transfer_seq() != -1 && meta_version_id() != ObStorageObjectOpt::INVALID_TABLET_VERSION;
// -1 : INVLAID_TABLET_TRANSFER_SEQ;
// ObStorageObjectOpt::INVALID_TABLET_VERSION : macro_seq / tablet_meta_version
} else if (is_shared_data_or_meta()) {
is_valid &= third_id_ != -1; // macro_seq != -1
}
}
return is_valid;
}
@ -115,10 +125,11 @@ int64_t MacroBlockId::to_string(char *buf, const int64_t buf_len) const
databuff_printf(buf, buf_len, pos,
"[2nd=%lu]"
"[3rd=%lu]"
"[4th=%lu]}",
"[trans_seq=%lu, sec_part=%lu]}",
(uint64_t) second_id_,
(uint64_t) third_id_,
(uint64_t) fourth_id_);
(int64_t) macro_transfer_seq_,
(int64_t) tenant_seq_);
break;
default:
databuff_printf(buf, buf_len, pos,

18
src/storage/blocksstable/ob_macro_block_id.h
View File

@ -200,13 +200,13 @@ public:
void set_incarnation_id(const uint64_t incarnation_id) { incarnation_id_ = incarnation_id; }
int64_t column_group_id() const { return column_group_id_; }
void set_column_group_id(const uint64_t column_group_id) { column_group_id_ = column_group_id; }
int64_t macro_transfer_seq() const { return 0; } // TODO 这里应该return macro_transfer_seq_; 需要费渡修改 @gaishun.gs by xiaotao
void set_macro_transfer_seq(const uint64_t macro_transfer_seq) { macro_transfer_seq_ = macro_transfer_seq; }
int64_t tenant_seq() const { return fourth_id_; } // TODO 这里应该return tenant_seq_; 需要费渡修改 @gaishun.gs by xiaotao
int64_t macro_transfer_seq() const { return macro_transfer_seq_; }
void set_macro_transfer_seq(const int64_t macro_transfer_seq) { macro_transfer_seq_ = macro_transfer_seq; }
uint64_t tenant_seq() const { return tenant_seq_; }
void set_tenant_seq(const uint64_t tenant_seq) { tenant_seq_ = tenant_seq; }
int64_t meta_transfer_seq() const { return 0; } // TODO 这里应该return meta_transfer_seq_; 需要费渡修改 @gaishun.gs by xiaotao
void set_meta_transfer_seq(const uint64_t meta_transfer_seq) { meta_transfer_seq_ = meta_transfer_seq; }
int64_t meta_version_id() const { return fourth_id_; } // TODO 这里应该return meta_version_id_; 需要费渡修改 @gaishun.gs by xiaotao
int64_t meta_transfer_seq() const { return meta_transfer_seq_; }
void set_meta_transfer_seq(const int64_t meta_transfer_seq) { meta_transfer_seq_ = meta_transfer_seq; }
uint64_t meta_version_id() const { return meta_version_id_; }
void set_meta_version_id(const uint64_t meta_version_id) { meta_version_id_ = meta_version_id; }
// Deivce mode
@ -251,7 +251,7 @@ public:
static const uint64_t SF_BIT_VERSION = 4;
static const uint64_t SF_BIT_TRANSFER_SEQ = 20;
static const uint64_t SF_BIT_TENANT_SEQ = 44;
static const uint64_t SF_BIT_META_VERSION_ID = 44;
static constexpr uint64_t SF_BIT_META_VERSION_ID = 44;
static const uint64_t MAX_WRITE_SEQ = (0x1UL << MacroBlockId::SF_BIT_WRITE_SEQ) - 1;
@ -292,12 +292,12 @@ private:
int64_t fourth_id_;
// for PRIVATE_DATA_MACRO and PRIVATE_META_MACRO
struct {
uint64_t macro_transfer_seq_ : SF_BIT_TRANSFER_SEQ;
int64_t macro_transfer_seq_ : SF_BIT_TRANSFER_SEQ;
uint64_t tenant_seq_ : SF_BIT_TENANT_SEQ;
};
// for PRIVATE_TABLET_META and PRIVATE_TABLET_CURRENT_VERSION
struct {
uint64_t meta_transfer_seq_ : SF_BIT_TRANSFER_SEQ;
int64_t meta_transfer_seq_ : SF_BIT_TRANSFER_SEQ;
uint64_t meta_version_id_ : SF_BIT_META_VERSION_ID;
};
};

6
src/storage/blocksstable/ob_macro_block_writer.cpp
View File

@ -1695,9 +1695,11 @@ int ObMacroBlockWriter::alloc_block()
}
} else {
if (data_store_desc_->is_for_index_or_meta()) {
storage_opt.set_private_meta_macro_object_opt(data_store_desc_->get_tablet_id().id());
storage_opt.set_private_meta_macro_object_opt(data_store_desc_->get_tablet_id().id(),
data_store_desc_->get_tablet_transfer_seq());
} else {
storage_opt.set_private_object_opt(data_store_desc_->get_tablet_id().id());
storage_opt.set_private_object_opt(data_store_desc_->get_tablet_id().id(),
data_store_desc_->get_tablet_transfer_seq());
}
}
if (macro_blocks_[current_index_].is_dirty()) { // has been allocated

33
src/storage/blocksstable/ob_object_manager.cpp
View File

@ -33,10 +33,10 @@ int64_t ObStorageObjectOpt::to_string(char *buf, const int64_t buf_len) const
switch (object_type_) {
case ObStorageObjectType::PRIVATE_DATA_MACRO:
case ObStorageObjectType::PRIVATE_META_MACRO: {
if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type:%s (tablet_id=%lu)",
get_storage_objet_type_str(object_type_), private_opt_.tablet_id_))) {
if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type:%s (tablet_id=%lu, transfer_seq=%lu)",
get_storage_objet_type_str(object_type_), private_opt_.tablet_id_, private_opt_.tablet_trasfer_seq_))) {
LOG_WARN("failed to print data into buf", K(ret), K(buf_len), K(pos), K(get_storage_objet_type_str(object_type_)),
K(private_opt_.tablet_id_));
K(private_opt_.tablet_id_), K(private_opt_.tablet_trasfer_seq_));
}
break;
}
@ -88,11 +88,13 @@ int64_t ObStorageObjectOpt::to_string(char *buf, const int64_t buf_len) const
break;
}
case ObStorageObjectType::PRIVATE_TABLET_META: {
if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type=%s (ls_id=%lu,tablet_id=%lu,version=%lu)",
if(OB_FAIL(databuff_printf(buf, buf_len, pos, "object_type=%s (ls_id=%lu,tablet_id=%lu,version=%lu,transfer_seq=%lu)",
get_storage_objet_type_str(object_type_),
ss_private_tablet_opt_.ls_id_, ss_private_tablet_opt_.tablet_id_, ss_private_tablet_opt_.version_))) {
ss_private_tablet_opt_.ls_id_, ss_private_tablet_opt_.tablet_id_,
ss_private_tablet_opt_.version_, ss_private_tablet_opt_.tablet_transfer_seq_))) {
LOG_WARN("failed to print data into buf", K(ret), K(buf_len), K(pos), K(get_storage_objet_type_str(object_type_)),
K(ss_private_tablet_opt_.ls_id_), K(ss_private_tablet_opt_.tablet_id_), K(ss_private_tablet_opt_.version_));
K(ss_private_tablet_opt_.ls_id_), K(ss_private_tablet_opt_.tablet_id_),
K(ss_private_tablet_opt_.version_), K(ss_private_tablet_opt_.tablet_transfer_seq_));
}
break;
}
@ -198,19 +200,6 @@ int64_t ObStorageObjectOpt::to_string(char *buf, const int64_t buf_len) const
return pos;
}
int ObStorageObjectOpt::get_tablet_version(const MacroBlockId &tablet_object_id, int64_t &version)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!tablet_object_id.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("object id is invalid", K(ret), K(tablet_object_id));
} else if (OB_UNLIKELY(0 > (version = tablet_object_id.fourth_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid tablet version", K(ret), K(version));
}
return ret;
}
//================================ ObObjectManager =====================================//
ObObjectManager &ObObjectManager::ObObjectManager::get_instance()
@ -624,7 +613,8 @@ int ObObjectManager::ss_get_object_id(const ObStorageObjectOpt &opt, MacroBlockI
if (OB_FAIL(TENANT_SEQ_GENERATOR.get_private_object_seq(seq))) {
LOG_WARN("fail to get private object seq", K(ret), K(opt));
} else {
object_id.set_fourth_id(seq);
object_id.set_tenant_seq(seq);
object_id.set_macro_transfer_seq(opt.private_opt_.tablet_trasfer_seq_);
}
break;
}
@ -670,7 +660,8 @@ int ObObjectManager::ss_get_object_id(const ObStorageObjectOpt &opt, MacroBlockI
set_ss_object_first_id_(obj_type, default_incarnation_id, default_cg_id, object_id);
object_id.set_second_id(opt.ss_private_tablet_opt_.ls_id_);
object_id.set_third_id(opt.ss_private_tablet_opt_.tablet_id_);
object_id.set_fourth_id(opt.ss_private_tablet_opt_.version_);
object_id.set_meta_version_id(opt.ss_private_tablet_opt_.version_);
object_id.set_meta_transfer_seq(opt.ss_private_tablet_opt_.tablet_transfer_seq_);
break;
}
case ObStorageObjectType::PRIVATE_TABLET_CURRENT_VERSION: {

17
src/storage/blocksstable/ob_object_manager.h
View File

@ -32,16 +32,17 @@ public:
: object_type_(ObStorageObjectType::PRIVATE_DATA_MACRO) {}
~ObStorageObjectOpt() {}
static int get_tablet_version(const MacroBlockId &tablet_object_id, int64_t &version);
void set_private_object_opt(const int64_t tablet_id = -1)
void set_private_object_opt(const int64_t tablet_id = -1, const int64_t tablet_transfer_seq = -1)
{
object_type_ = ObStorageObjectType::PRIVATE_DATA_MACRO;
private_opt_.tablet_id_ = tablet_id;
private_opt_.tablet_trasfer_seq_ = tablet_transfer_seq;
}
void set_private_meta_macro_object_opt(const int64_t tablet_id = -1)
void set_private_meta_macro_object_opt(const int64_t tablet_id = -1, const int64_t tablet_transfer_seq = -1)
{
object_type_ = ObStorageObjectType::PRIVATE_META_MACRO;
private_opt_.tablet_id_ = tablet_id;
private_opt_.tablet_trasfer_seq_ = tablet_transfer_seq;
}
void set_ss_share_data_macro_object_opt(
const int64_t tablet_id,
@ -88,12 +89,13 @@ public:
};
void set_ss_private_tablet_meta_object_opt(
const int64_t ls_id, const uint64_t tablet_id, const int64_t version)
const int64_t ls_id, const uint64_t tablet_id, const uint64_t version, const int64_t tablet_trasfer_seq)
{
object_type_ = ObStorageObjectType::PRIVATE_TABLET_META;
ss_private_tablet_opt_.ls_id_ = ls_id;
ss_private_tablet_opt_.tablet_id_ = tablet_id;
ss_private_tablet_opt_.version_ = version;
ss_private_tablet_opt_.tablet_transfer_seq_ = tablet_trasfer_seq;
}
void set_ss_private_tablet_meta_current_verison_object_opt(
const int64_t ls_id, const uint64_t tablet_id)
@ -193,16 +195,18 @@ public:
int64_t to_string(char *buf, const int64_t buf_len) const;
public:
static const int64_t INVALID_TABLET_VERSION = -1;
static constexpr uint64_t INVALID_TABLET_VERSION = (1llu << ((MacroBlockId::SF_BIT_META_VERSION_ID) + 1)) - 1;
static const int64_t INVALID_TABLET_TRANSFER_SEQ = -1;
static bool is_inaccurate_tablet_addr(const storage::ObMetaDiskAddr &tablet_meta_addr)
{
return tablet_meta_addr.is_block() && tablet_meta_addr.fifth_id() == INVALID_TABLET_VERSION;
return tablet_meta_addr.is_block() && tablet_meta_addr.block_id().meta_version_id() == INVALID_TABLET_VERSION;
}
private:
struct PrivateObjectOpt
{
uint64_t tablet_id_;
uint64_t tablet_trasfer_seq_;
};
struct SSShareObjectOpt
{
@ -236,6 +240,7 @@ private:
uint64_t ls_id_;
uint64_t tablet_id_;
int64_t version_;
int64_t tablet_transfer_seq_;
};
struct SSPrivateTabletCurrentVersionObjectOpt
{

2
src/storage/blocksstable/ob_shared_macro_block_manager.cpp
View File

@ -794,6 +794,7 @@ int ObSharedMacroBlockMgr::prepare_data_desc(
snapshot_version,
cluster_version,
tablet.get_tablet_meta().micro_index_clustered_,
tablet.get_transfer_seq(),
end_scn))) {
LOG_WARN("failed to init static desc", K(ret), KPC(storage_schema),
K(tablet), "merge_type", merge_type_to_str(merge_type), K(snapshot_version), K(cluster_version));
@ -825,6 +826,7 @@ int ObSharedMacroBlockMgr::prepare_data_desc(
snapshot_version,
cluster_version,
tablet.get_tablet_meta().micro_index_clustered_,
tablet.get_transfer_seq(),
end_scn,
cg_schema,
cg_idx))) {

2
src/storage/blocksstable/ob_sstable.cpp
View File

@ -1974,6 +1974,7 @@ int ObSSTable::get_cs_range(
int ObSSTable::persist_linked_block_if_need(
ObArenaAllocator &allocator,
const ObTabletID &tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
blocksstable::ObIMacroBlockFlushCallback *ddl_redo_cb,
int64_t &macro_start_seq,
@ -1992,6 +1993,7 @@ int ObSSTable::persist_linked_block_if_need(
} else if (OB_FAIL(link_write_info.init(ddl_redo_cb))) {
LOG_WARN("fail to init link_write_info", K(ret), KP(ddl_redo_cb));
} else if (OB_FAIL(meta_->macro_info_.persist_block_ids(tablet_id,
tablet_transfer_seq,
snapshot_version,
allocator,
&link_write_info,

1
src/storage/blocksstable/ob_sstable.h
View File

@ -288,6 +288,7 @@ public:
int persist_linked_block_if_need(
ObArenaAllocator &allocator,
const ObTabletID &tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
blocksstable::ObIMacroBlockFlushCallback *ddl_redo_cb,
int64_t &macro_start_seq,

12
src/storage/column_store/ob_co_merge_dag.cpp
View File

@ -250,8 +250,9 @@ int ObCOMergePrepareTask::schedule_minor_exec_dag(
result.version_range_.multi_version_start_ = ctx.get_tablet()->get_multi_version_start();
result.version_range_.base_version_ = 0;
result.version_range_.snapshot_version_ = ctx.get_tablet()->get_snapshot_version();
result.transfer_seq_ = ctx.get_tablet()->get_transfer_seq();
ObTabletMergeDagParam dag_param(MINOR_MERGE, ctx.get_ls_id(),
ctx.get_tablet_id(), ctx.get_transfer_seq());
ctx.get_tablet_id(), ctx.get_schedule_transfer_seq());
if (OB_FAIL(MTL(share::ObTenantDagScheduler *)->alloc_dag(minor_exe_dag))) {
LOG_WARN("failed to alloc dag", K(ret));
} else if (OB_FAIL(minor_exe_dag->prepare_init(
@ -1395,10 +1396,10 @@ int ObCOMergeDagNet::swap_tablet_after_minor()
ObTabletCommon::DEFAULT_GET_TABLET_NO_WAIT,
storage::ObMDSGetTabletMode::READ_ALL_COMMITED))) {
LOG_WARN("failed to get tablet", K(ret));
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*tmp_tablet_handle.get_obj(), co_merge_ctx_->get_transfer_seq()))) {
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*tmp_tablet_handle.get_obj(), co_merge_ctx_->get_schedule_transfer_seq()))) {
LOG_WARN("tmp tablet transfer seq not eq with old transfer seq", K(ret),
"tmp_tablet_meta", tmp_tablet_handle.get_obj()->get_tablet_meta(),
"old_transfer_seq", co_merge_ctx_->get_transfer_seq());
"old_transfer_seq", co_merge_ctx_->get_schedule_transfer_seq());
} else if (OB_FAIL(ObPartitionMergePolicy::get_result_by_snapshot(
*tmp_tablet_handle.get_obj(),
co_merge_ctx_->get_merge_version(),
@ -1408,6 +1409,7 @@ int ObCOMergeDagNet::swap_tablet_after_minor()
LOG_WARN("failed to assign tables handle", K(ret), K(tmp_result));
} else {
co_merge_ctx_->tablet_handle_ = tmp_tablet_handle;
co_merge_ctx_->static_param_.tablet_transfer_seq_ = tmp_tablet_handle.get_obj()->get_transfer_seq();
co_merge_ctx_->static_param_.rowkey_read_info_ =
static_cast<const ObRowkeyReadInfo *>(&(co_merge_ctx_->get_tablet()->get_rowkey_read_info()));
LOG_INFO("success to swap tablet after minor", K(ret), K(tmp_result),
@ -1574,10 +1576,10 @@ int ObCOMergeDagNet::get_compat_mode()
0/*timeout_us*/,
storage::ObMDSGetTabletMode::READ_ALL_COMMITED))) {
LOG_WARN("failed to get tablet", K(ret), K(ls_id_), K(tablet_id_));
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*tmp_tablet_handle.get_obj(), basic_param_.transfer_seq_))) {
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*tmp_tablet_handle.get_obj(), basic_param_.schedule_transfer_seq_))) {
LOG_WARN("tmp tablet transfer seq not eq with old transfer seq", K(ret),
"tmp_tablet_meta", tmp_tablet_handle.get_obj()->get_tablet_meta(),
"old_transfer_seq", basic_param_.transfer_seq_);
"old_transfer_seq", basic_param_.schedule_transfer_seq_);
} else {
basic_param_.dag_net_id_ = get_dag_id();
basic_param_.skip_get_tablet_ = true;

32
src/storage/compaction/ob_basic_tablet_merge_ctx.cpp
View File

@ -64,7 +64,8 @@ ObStaticMergeParam::ObStaticMergeParam(ObTabletMergeDagParam &dag_param)
tx_id_(0),
multi_version_column_descs_(),
pre_warm_param_(),
tablet_schema_guard_()
tablet_schema_guard_(),
tablet_transfer_seq_(ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ)
{
merge_scn_.set_max();
}
@ -85,6 +86,7 @@ void ObStaticMergeParam::reset()
tx_id_ = 0;
tablet_schema_guard_.reset();
encoding_granularity_ = 0;
tablet_transfer_seq_ = ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ;
}
bool ObStaticMergeParam::is_valid() const
@ -106,6 +108,9 @@ bool ObStaticMergeParam::is_valid() const
bret = false;
LOG_WARN_RET(OB_ERR_UNEXPECTED, "column desc is empty or create snapshot is invalid", K_(multi_version_column_descs),
K_(create_snapshot_version));
} else if (GCTX.is_shared_storage_mode() && ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ == tablet_transfer_seq_) {
bret = false;
LOG_WARN_RET(OB_ERR_UNEXPECTED, "tablet_transfer_seq in ss mode should not be invalid", K(tablet_transfer_seq_));
} else {
bret = true;
}
@ -197,6 +202,17 @@ int ObStaticMergeParam::get_basic_info_from_result(
} else {
create_snapshot_version_ = 0;
}
if (ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ == tablet_transfer_seq_) {
// If not set tranfser_seq specifically, set it.
// The tablet_transfer_seq_ can be set to write macro_block to the specific transfer_seq_directory
// by tasks in ob_tablet_backfill_tx.cpp.
tablet_transfer_seq_ = get_merge_table_result.transfer_seq_;
if (GCTX.is_shared_storage_mode() && ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ == tablet_transfer_seq_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet_transfer_seq in ss mode should not be invalid", K(ret), KPC(this), K(get_merge_table_result), K(lbt()));
}
}
}
return ret;
}
@ -383,10 +399,10 @@ int ObBasicTabletMergeCtx::build_ctx(bool &finish_flag)
if (OB_TABLET_NOT_EXIST != ret) {
LOG_PRINT_WRAPPER("failed to get ls_handle/tablet_handle/rebuild_seq");
}
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*get_tablet(), get_transfer_seq()))) {
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*get_tablet(), get_schedule_transfer_seq()))) {
LOG_WARN("new tablet transfer seq not eq with old transfer seq", K(ret),
"new_tablet_meta", get_tablet()->get_tablet_meta(),
"old_transfer_seq", get_transfer_seq());
"old_transfer_seq", get_schedule_transfer_seq());
} else if (OB_FAIL(get_merge_tables(get_merge_table_result))) {
if (OB_NO_NEED_MERGE != ret) {
LOG_PRINT_WRAPPER("failed to get merge tables");
@ -798,6 +814,7 @@ int ObBasicTabletMergeCtx::init_static_param_and_desc()
if (OB_FAIL(static_param_.init_static_info(get_concurrent_cnt(), tablet_handle_))) {
LOG_WARN("failed to init basic info", KR(ret));
} else if (OB_FAIL(static_desc_.init(false/*is_ddl*/, *get_schema(), get_ls_id(), get_tablet_id(),
static_param_.tablet_transfer_seq_,
get_merge_type(), get_snapshot(),
static_param_.scn_range_.end_scn_,
static_param_.data_version_,
@ -1218,10 +1235,15 @@ int ObBasicTabletMergeCtx::swap_tablet(ObGetMergeTablesResult &get_merge_table_r
tables_handle.reset(); // clear tables array
if (OB_FAIL(swap_tablet())) {
LOG_WARN("failed to get alloc tablet handle", KR(ret));
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*get_tablet(), get_transfer_seq()))) {
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*get_tablet(), get_schedule_transfer_seq()))) {
LOG_WARN("new tablet transfer seq not eq with old transfer seq", K(ret),
"new_tablet_meta", get_tablet()->get_tablet_meta(),
"old_transfer_seq", get_transfer_seq());
"old_transfer_seq", get_schedule_transfer_seq());
} else if (GCTX.is_shared_storage_mode() &&
OB_FAIL(ObTablet::check_transfer_seq_equal(*get_tablet(), get_merge_table_result.transfer_seq_))) {
LOG_WARN("new tablet transfer seq not eq with old transfer seq in ss", K(ret),
"new_tablet_meta", get_tablet()->get_tablet_meta(),
"old_transfer_seq", get_merge_table_result.transfer_seq_, K(lbt()));
} else if (OB_FAIL(get_merge_tables(get_merge_table_result))) {
if (OB_NO_NEED_MERGE != ret) {
LOG_WARN("failed to get merge tables", KR(ret), KPC(this));

5
src/storage/compaction/ob_basic_tablet_merge_ctx.h
View File

@ -64,7 +64,7 @@ public:
K_(sstable_logic_seq), K_(tables_handle), K_(is_rebuild_column_store), K_(is_schema_changed), K_(is_tenant_major_merge),
K_(is_cs_replica), K_(read_base_version), K_(merge_scn), K_(need_parallel_minor_merge),
KP_(schema), "multi_version_column_descs_cnt", multi_version_column_descs_.count(),
K_(ls_handle), K_(snapshot_info), K_(is_backfill), K_(tablet_schema_guard));
K_(ls_handle), K_(snapshot_info), K_(is_backfill), K_(tablet_schema_guard), K_(tablet_transfer_seq));
ObTabletMergeDagParam &dag_param_;
bool is_full_merge_; // full merge or increment merge
@ -98,6 +98,7 @@ public:
common::ObSEArray<share::schema::ObColDesc, 2 * OB_ROW_DEFAULT_COLUMNS_COUNT> multi_version_column_descs_;
share::ObPreWarmerParam pre_warm_param_;
storage::ObCSReplicaStorageSchemaGuard tablet_schema_guard_; // original storage schema on tablet, used only in cs replcia
int64_t tablet_transfer_seq_; // only used in shared_storage mode, used to init statis_desc;
DISALLOW_COPY_AND_ASSIGN(ObStaticMergeParam);
};
@ -217,7 +218,7 @@ public:
DAG_PARAM_FUNC(const ObLSID &, ls_id);
DAG_PARAM_FUNC(const ObTabletID &, tablet_id);
DAG_PARAM_FUNC(int64_t, merge_version);
DAG_PARAM_FUNC(int64_t, transfer_seq);
DAG_PARAM_FUNC(int64_t, schedule_transfer_seq);
DAG_PARAM_FUNC(ObExecMode, exec_mode);
STATIC_PARAM_FUNC(bool, is_tenant_major_merge);
STATIC_PARAM_FUNC(bool, is_full_merge);

11
src/storage/compaction/ob_partition_merge_policy.cpp
View File

@ -124,6 +124,8 @@ int ObPartitionMergePolicy::get_medium_merge_tables(
if (OB_SUCC(ret) && OB_FAIL(result.handle_.check_continues(nullptr))) {
LOG_WARN("failed to check continues for major merge", K(ret));
SET_DIAGNOSE_LOCATION(result.error_location_);
} else {
result.transfer_seq_ = tablet.get_transfer_seq();
}
}
@ -199,6 +201,7 @@ int ObPartitionMergePolicy::get_mds_merge_tables(
result.handle_.reset();
} else {
result.version_range_.snapshot_version_ = tablet.get_snapshot_version();
result.transfer_seq_ = tablet.get_transfer_seq();
}
return ret;
}
@ -342,6 +345,8 @@ int ObPartitionMergePolicy::get_mini_merge_tables(
// do nothing
} else if (OB_FAIL(deal_with_minor_result(merge_type, ls, tablet, result))) {
LOG_WARN("failed to deal with minor merge result", K(ret));
} else {
result.transfer_seq_ = tablet.get_transfer_seq();
}
return ret;
@ -524,6 +529,8 @@ int ObPartitionMergePolicy::get_minor_merge_tables(
if (OB_NO_NEED_MERGE != ret) {
LOG_WARN("failed to get minor merge tables", K(ret), K(max_snapshot_version));
}
} else {
result.transfer_seq_ = tablet.get_transfer_seq();
}
return ret;
@ -778,6 +785,8 @@ int ObPartitionMergePolicy::get_hist_minor_merge_tables(
if (OB_NO_NEED_MERGE != ret) {
LOG_WARN("failed to get minor tables for hist minor merge", K(ret));
}
} else {
result.transfer_seq_ = tablet.get_transfer_seq();
}
return ret;
}
@ -1538,6 +1547,8 @@ int ObAdaptiveMergePolicy::get_meta_merge_tables(
if (OB_FAIL(ObPartitionMergePolicy::get_multi_version_start(
merge_type, ls, tablet, result.version_range_, result.snapshot_info_))) {
LOG_WARN("failed to get multi version_start", K(ret));
} else {
result.transfer_seq_ = tablet.get_transfer_seq();
}
}

10
src/storage/compaction/ob_tablet_merge_task.cpp
View File

@ -238,7 +238,7 @@ ObTabletMergeDagParam::ObTabletMergeDagParam()
exec_mode_(ObExecMode::EXEC_MODE_LOCAL),
merge_type_(INVALID_MERGE_TYPE),
merge_version_(0),
transfer_seq_(-1),
schedule_transfer_seq_(-1),
ls_id_(),
tablet_id_()
{
@ -248,14 +248,14 @@ ObTabletMergeDagParam::ObTabletMergeDagParam(
const compaction::ObMergeType merge_type,
const share::ObLSID &ls_id,
const ObTabletID &tablet_id,
const int64_t transfer_seq)
const int64_t schedule_transfer_seq)
: skip_get_tablet_(false),
need_swap_tablet_flag_(false),
is_reserve_mode_(false),
exec_mode_(ObExecMode::EXEC_MODE_LOCAL),
merge_type_(merge_type),
merge_version_(0),
transfer_seq_(transfer_seq),
schedule_transfer_seq_(schedule_transfer_seq),
ls_id_(ls_id),
tablet_id_(tablet_id)
{
@ -311,10 +311,10 @@ int ObTabletMergeDag::get_tablet_and_compat_mode()
if (OB_NO_NEED_MERGE != ret) {
LOG_WARN("failed to check need merge", K(ret));
}
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*tmp_tablet_handle.get_obj(), param_.transfer_seq_))) {
} else if (OB_FAIL(ObTablet::check_transfer_seq_equal(*tmp_tablet_handle.get_obj(), param_.schedule_transfer_seq_))) {
LOG_WARN("tmp tablet transfer seq not eq with old transfer seq", K(ret),
"tmp_tablet_meta", tmp_tablet_handle.get_obj()->get_tablet_meta(),
"old_transfer_seq", param_.transfer_seq_);
"old_transfer_seq", param_.schedule_transfer_seq_);
} else if (FALSE_IT(compat_mode_ = tmp_tablet_handle.get_obj()->get_tablet_meta().compat_mode_)) {
} else if (is_mini_merge(merge_type_)) {
int64_t inc_sstable_cnt = 0;

6
src/storage/compaction/ob_tablet_merge_task.h
View File

@ -108,11 +108,11 @@ struct ObTabletMergeDagParam : public share::ObIDagInitParam
const compaction::ObMergeType merge_type,
const share::ObLSID &ls_id,
const ObTabletID &tablet_id,
const int64_t transfer_seq);
const int64_t schedule_transfer_seq);
virtual bool is_valid() const override;
VIRTUAL_TO_STRING_KV(K_(skip_get_tablet), "merge_type", merge_type_to_str(merge_type_), K_(merge_version),
K_(ls_id), K_(tablet_id), "exec_mode", exec_mode_to_str(exec_mode_),
K_(need_swap_tablet_flag), K_(is_reserve_mode), K_(transfer_seq));
K_(need_swap_tablet_flag), K_(is_reserve_mode), K_(schedule_transfer_seq));
bool skip_get_tablet_;
bool need_swap_tablet_flag_;
@ -120,7 +120,7 @@ struct ObTabletMergeDagParam : public share::ObIDagInitParam
ObExecMode exec_mode_;
compaction::ObMergeType merge_type_;
int64_t merge_version_;
int64_t transfer_seq_; // only affect minor and major now
int64_t schedule_transfer_seq_; // only affect minor and major now
share::ObLSID ls_id_;
ObTabletID tablet_id_;
ObCompactionParam compaction_param_; // used for adaptive compaction dag scheduling

9
src/storage/compaction/ob_tenant_tablet_scheduler.cpp
View File

@ -1000,7 +1000,7 @@ int ObTenantTabletScheduler::schedule_merge_dag(
param.merge_type_ = merge_type;
param.merge_version_ = merge_snapshot_version;
param.compat_mode_ = tablet.get_tablet_meta().compat_mode_;
param.transfer_seq_ = tablet.get_tablet_meta().transfer_info_.transfer_seq_;
param.schedule_transfer_seq_ = tablet.get_tablet_meta().transfer_info_.transfer_seq_;
param.exec_mode_ = exec_mode;
if (OB_UNLIKELY(nullptr != dag_net_id)) {
param.dag_net_id_ = *dag_net_id;
@ -1017,7 +1017,7 @@ int ObTenantTabletScheduler::schedule_merge_dag(
param.tablet_id_ = tablet.get_tablet_meta().tablet_id_;
param.merge_type_ = merge_type;
param.merge_version_ = merge_snapshot_version;
param.transfer_seq_ = tablet.get_tablet_meta().transfer_info_.transfer_seq_;
param.schedule_transfer_seq_ = tablet.get_tablet_meta().transfer_info_.transfer_seq_;
param.exec_mode_ = exec_mode;
if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_tablet_merge_dag(param))) {
if (OB_EAGAIN != ret && OB_SIZE_OVERFLOW != ret) {
@ -1114,7 +1114,7 @@ int ObTenantTabletScheduler::schedule_tablet_meta_merge(
dag_param.tablet_id_ = tablet_id;
dag_param.merge_type_ = META_MAJOR_MERGE;
dag_param.merge_version_ = result.merge_version_;
dag_param.transfer_seq_ = tablet->get_tablet_meta().transfer_info_.transfer_seq_;
dag_param.schedule_transfer_seq_ = tablet->get_tablet_meta().transfer_info_.transfer_seq_;
dag_param.exec_mode_ = EXEC_MODE_LOCAL;
if (OB_FAIL(schedule_merge_execute_dag<ObTabletMergeExecuteDag>(dag_param, ls_handle, tablet_handle, result))) {
@ -1131,7 +1131,7 @@ int ObTenantTabletScheduler::schedule_tablet_meta_merge(
dag_param.merge_type_ = META_MAJOR_MERGE;
dag_param.merge_version_ = result.merge_version_;
dag_param.compat_mode_ = tablet->get_tablet_meta().compat_mode_;
dag_param.transfer_seq_ = tablet->get_tablet_meta().transfer_info_.transfer_seq_;
dag_param.schedule_transfer_seq_ = tablet->get_tablet_meta().transfer_info_.transfer_seq_;
dag_param.exec_mode_ = EXEC_MODE_LOCAL;
if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_tablet_co_merge_dag_net(dag_param))) {
if (OB_EAGAIN != ret && OB_SIZE_OVERFLOW != ret) {
@ -1288,6 +1288,7 @@ int ObTenantTabletScheduler::schedule_tablet_minor_merge(
LOG_WARN("failed to check need merge", K(ret), K(merge_type), K(tablet_id), K(tablet_handle));
}
} else {
result.transfer_seq_ = tablet_handle.get_obj()->get_transfer_seq();
int64_t minor_compact_trigger = ObPartitionMergePolicy::DEFAULT_MINOR_COMPACT_TRIGGER;
{
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));

2
src/storage/ddl/ob_ddl_merge_task.cpp
View File

@ -790,6 +790,7 @@ int ObTabletDDLUtil::prepare_index_data_desc(ObTablet &tablet,
compaction::ObMergeType::MAJOR_MERGE,
snapshot_version, data_format_version,
tablet.get_tablet_meta().micro_index_clustered_,
tablet.get_transfer_seq(),
end_scn, &cur_cg_schema, cg_idx))) {
LOG_WARN("init data desc for cg failed", K(ret));
} else {
@ -807,6 +808,7 @@ int ObTabletDDLUtil::prepare_index_data_desc(ObTablet &tablet,
snapshot_version,
data_format_version,
tablet.get_tablet_meta().micro_index_clustered_,
tablet.get_transfer_seq(),
end_scn))) {
// use storage schema to init ObDataStoreDesc
// all cols' default checksum will assigned to 0

6
src/storage/ddl/ob_ddl_replay_executor.cpp
View File

@ -405,7 +405,8 @@ int ObDDLRedoReplayExecutor::do_inc_replay_(
// do nothing
} else {
ObStorageObjectOpt opt;
opt.set_private_object_opt(tablet_handle.get_obj()->get_tablet_id().id());
opt.set_private_object_opt(tablet_handle.get_obj()->get_tablet_id().id(),
tablet_handle.get_obj()->get_transfer_seq());
ObStorageObjectWriteInfo object_write_info;
object_write_info.buffer_ = write_info.buffer_;
object_write_info.size_= write_info.size_;
@ -471,7 +472,8 @@ int ObDDLRedoReplayExecutor::do_full_replay_(
} else {
const ObDDLMacroBlockRedoInfo &redo_info = log_->get_redo_info();
ObStorageObjectOpt opt;
opt.set_private_object_opt(tablet_handle.get_obj()->get_tablet_id().id());
opt.set_private_object_opt(tablet_handle.get_obj()->get_tablet_id().id(),
tablet_handle.get_obj()->get_transfer_seq());
ObStorageObjectHandle macro_handle;
ObStorageObjectWriteInfo write_info;
write_info.buffer_ = redo_info.data_buffer_.ptr();

8
src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp
View File

@ -1228,7 +1228,7 @@ ObTabletDirectLoadMgr::ObTabletDirectLoadMgr()
lock_(), ref_cnt_(0), direct_load_type_(ObDirectLoadType::DIRECT_LOAD_INVALID),
need_process_cs_replica_(false), need_fill_column_group_(false), sqc_build_ctx_(),
column_items_(), lob_column_idxs_(), lob_col_types_(), schema_item_(), dir_id_(0), task_cnt_(0), cg_cnt_(0),
micro_index_clustered_(false)
micro_index_clustered_(false), tablet_transfer_seq_(ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ)
{
column_items_.set_attr(ObMemAttr(MTL_ID(), "DL_schema"));
lob_column_idxs_.set_attr(ObMemAttr(MTL_ID(), "DL_schema"));
@ -1254,6 +1254,7 @@ ObTabletDirectLoadMgr::~ObTabletDirectLoadMgr()
schema_item_.reset();
is_schema_item_ready_ = false;
micro_index_clustered_ = false;
tablet_transfer_seq_ = ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ;
}
bool ObTabletDirectLoadMgr::is_valid()
@ -1358,6 +1359,7 @@ int ObTabletDirectLoadMgr::update(
direct_load_type_ = build_param.common_param_.direct_load_type_;
data_format_version_ = build_param.common_param_.data_format_version_;
micro_index_clustered_ = tablet_handle.get_obj()->get_tablet_meta().micro_index_clustered_;
tablet_transfer_seq_ = tablet_handle.get_obj()->get_transfer_seq();
is_inited_ = true;
}
}
@ -2280,7 +2282,7 @@ int ObTabletDirectLoadMgr::prepare_index_builder_if_need(const ObTableSchema &ta
} else if (OB_FAIL(index_block_desc.init(true/*is ddl*/, table_schema, ls_id_, tablet_id_,
is_full_direct_load(direct_load_type_) ? compaction::ObMergeType::MAJOR_MERGE : compaction::ObMergeType::MINOR_MERGE,
is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : 1L,
data_format_version_, get_micro_index_clustered(),
data_format_version_, get_micro_index_clustered(), get_tablet_transfer_seq(),
is_full_direct_load(direct_load_type_) ? SCN::invalid_scn() : table_key_.get_end_scn()))) {
LOG_WARN("fail to init data desc", K(ret));
} else if (FALSE_IT(index_block_desc.get_static_desc().schema_version_ = sqc_build_ctx_.build_param_.runtime_only_param_.schema_version_)) {
@ -2304,7 +2306,7 @@ int ObTabletDirectLoadMgr::prepare_index_builder_if_need(const ObTableSchema &ta
} else if (OB_FAIL(sqc_build_ctx_.data_block_desc_.init(true/*is ddl*/, table_schema, ls_id_, tablet_id_,
is_full_direct_load(direct_load_type_) ? compaction::ObMergeType::MAJOR_MERGE : compaction::ObMergeType::MINOR_MERGE,
is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : 1L,
data_format_version_, get_micro_index_clustered(),
data_format_version_, get_micro_index_clustered(), get_tablet_transfer_seq(),
is_full_direct_load(direct_load_type_) ? SCN::invalid_scn() : table_key_.get_end_scn()))) {
LOG_WARN("fail to init data block desc", K(ret));
} else {

4
src/storage/ddl/ob_direct_insert_sstable_ctx_new.h
View File

@ -395,6 +395,7 @@ public:
const ObIArray<ObColumnSchemaItem> &get_column_info() const { return column_items_; };
bool is_schema_item_ready() { return is_schema_item_ready_; }
bool get_micro_index_clustered() { return micro_index_clustered_; }
bool get_tablet_transfer_seq() { return tablet_transfer_seq_; }
int prepare_storage_schema(ObTabletHandle &tablet_handle);
int64_t get_task_cnt() { return task_cnt_; }
int64_t get_cg_cnt() {return cg_cnt_; }
@ -414,7 +415,7 @@ public:
VIRTUAL_TO_STRING_KV(K_(is_inited), K_(is_schema_item_ready), K_(ls_id), K_(tablet_id), K_(table_key), K_(data_format_version), K_(ref_cnt),
K_(direct_load_type), K_(need_process_cs_replica), K_(need_fill_column_group),K_(sqc_build_ctx), KPC(lob_mgr_handle_.get_obj()), K_(schema_item), K_(column_items), K_(lob_column_idxs),
K_(task_cnt), K_(cg_cnt), K_(micro_index_clustered));
K_(task_cnt), K_(cg_cnt), K_(micro_index_clustered), K_(tablet_transfer_seq));
protected:
int prepare_schema_item_on_demand(const uint64_t table_id,
@ -470,6 +471,7 @@ protected:
int64_t task_cnt_;
int64_t cg_cnt_;
bool micro_index_clustered_;
int64_t tablet_transfer_seq_;
};
class ObTabletFullDirectLoadMgr final : public ObTabletDirectLoadMgr

1
src/storage/ddl/ob_direct_load_struct.cpp
View File

@ -1886,6 +1886,7 @@ int ObCOSliceWriter::init(const ObStorageSchema *storage_schema, const int64_t c
table_key.get_snapshot_version(),
data_format_version,
tablet_direct_load_mgr->get_micro_index_clustered(),
tablet_direct_load_mgr->get_tablet_transfer_seq(),
SCN::min_scn(),
&cg_schema,
cg_idx,

1
src/storage/high_availability/ob_sstable_copy_finish_task.cpp
View File

@ -819,6 +819,7 @@ int ObSSTableCopyFinishTask::prepare_data_store_desc_(
tablet->get_snapshot_version(),
0/*cluster_version*/,
tablet_handle.get_obj()->get_tablet_meta().micro_index_clustered_,
tablet->get_transfer_seq(),
sstable_param->table_key_.get_end_scn(),
cg_schema,
cg_idx,

3
src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp
View File

@ -305,7 +305,6 @@ int ObStorageHALocalMacroBlockWriter::set_macro_write_info_(
blocksstable::ObStorageObjectOpt &opt)
{
int ret = OB_SUCCESS;
UNUSED(macro_block_id);
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
@ -317,7 +316,7 @@ int ObStorageHALocalMacroBlockWriter::set_macro_write_info_(
write_info.io_timeout_ms_ = GCONF._data_storage_io_timeout / 1000L;
write_info.mtl_tenant_id_ = MTL_ID();
write_info.offset_ = 0;
opt.set_private_object_opt(tablet_id_.id());
opt.set_private_object_opt(tablet_id_.id(), macro_block_id.macro_transfer_seq());
}
return ret;
}

73
src/storage/high_availability/ob_tablet_backfill_tx.cpp
View File

@ -587,6 +587,7 @@ int ObTabletBackfillTXTask::generate_backfill_tx_task_()
ObArray<ObTableHandleV2> table_array;
ObTablesHandleArray sstable_handles;
ObITask *child_task = nullptr;
int64_t dest_transfer_seq = -1;
if (!is_inited_) {
ret = OB_NOT_INIT;
@ -605,9 +606,11 @@ int ObTabletBackfillTXTask::generate_backfill_tx_task_()
LOG_WARN("failed to get all sstable handles", K(ret), KPC(tablet));
} else if (OB_FAIL(get_all_backfill_tx_tables_(sstable_handles, tablet, table_array))) {
LOG_WARN("get all backfill tx tabels", K(ret), KPC(tablet));
} else if (OB_FAIL(generate_mds_table_backfill_task_(transfer_replace_task, child_task))) {
} else if (OB_FAIL(get_dest_transfer_seq_(tablet_info_.tablet_id_, dest_transfer_seq))) {
LOG_WARN("failed to get dest_transfser_seq", K(ret), K(tablet_info_));
} else if (OB_FAIL(generate_mds_table_backfill_task_(dest_transfer_seq, transfer_replace_task, child_task))) {
LOG_WARN("failed to generate mds table backfill task", K(ret), KPC(tablet));
} else if (OB_FAIL(generate_table_backfill_tx_task_(transfer_replace_task, table_array, child_task))) {
} else if (OB_FAIL(generate_table_backfill_tx_task_(dest_transfer_seq, transfer_replace_task, table_array, child_task))) {
LOG_WARN("failed to generate minor sstables backfill tx task", K(ret), K(ls_id_), K(tablet_info_));
} else if (OB_FAIL(dag_->add_task(*transfer_replace_task))) {
LOG_WARN("failed to add transfer replace task to dag", K(ret));
@ -738,6 +741,7 @@ int ObTabletBackfillTXTask::get_backfill_tx_memtables_(
}
int ObTabletBackfillTXTask::generate_table_backfill_tx_task_(
const int64_t dest_transfer_seq,
share::ObITask *replace_task,
common::ObIArray<ObTableHandleV2> &table_array,
share::ObITask *child)
@ -770,7 +774,7 @@ int ObTabletBackfillTXTask::generate_table_backfill_tx_task_(
LOG_WARN("table should not be NULL or table type is unexpected", K(ret), KPC(table));
} else if (OB_FAIL(tablet_backfill_tx_dag->alloc_task(table_backfill_tx_task))) {
LOG_WARN("failed to alloc table backfill tx task", K(ret), KPC(ha_dag_net_ctx_), K(ls_id_), K(tablet_info_));
} else if (OB_FAIL(table_backfill_tx_task->init(ls_id_, tablet_info_, tablet_handle, table_array.at(i), replace_task))) {
} else if (OB_FAIL(table_backfill_tx_task->init(ls_id_, tablet_info_, dest_transfer_seq, tablet_handle, table_array.at(i), replace_task))) {
LOG_WARN("failed to init table backfill tx task", K(ret), K(ls_id_), K(tablet_info_));
} else if (OB_FAIL(pre_task->add_child(*table_backfill_tx_task))) {
LOG_WARN("failed to add table backfill tx task as child", K(ret), K(ls_id_), K(tablet_info_), KPC(table), KPC(pre_task));
@ -1014,6 +1018,41 @@ int ObTabletBackfillTXTask::get_diagnose_support_info_(share::ObLSID &dest_ls_id
return ret;
}
int ObTabletBackfillTXTask::get_dest_transfer_seq_(const common::ObTabletID &tablet_id, int64_t &transfer_seq) const
{
int ret = OB_SUCCESS;
transfer_seq = -1;
ObLSID dest_ls_id;
share::SCN unused_scn;
ObTabletHandle handle;
ObLSService *ls_service;
ObLSHandle ls_handle;
ObLS *ls;
if (!tablet_id.is_valid()) {
LOG_WARN("invalid argument", K(ret));
} else if (OB_FAIL(get_diagnose_support_info_(dest_ls_id, unused_scn))) {
LOG_WARN("failed to get ls_id", K(ret), K(tablet_id), K(dest_ls_id));
} else if (OB_ISNULL(ls_service = MTL(ObLSService*))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get ls_service", K(ret), KP(ls_service));
} else if (OB_FAIL(ls_service->get_ls(dest_ls_id, ls_handle, ObLSGetMod::HA_MOD))) {
LOG_WARN("failed to get ls", K(ret), K(dest_ls_id));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get ls", K(ret), K(dest_ls_id), KP(ls));
} else if (OB_FAIL(ls->get_tablet_svr()->get_tablet(tablet_id, handle, 0/*timeout*/, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) {
LOG_WARN("failed to get tablet", K(ret), K(dest_ls_id), K(tablet_id));
} else if (!handle.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid tablet_handle", K(ret), K(dest_ls_id), K(tablet_id), K(handle));
} else {
transfer_seq = handle.get_obj()->get_transfer_seq();
}
return ret;
}
void ObTabletBackfillTXTask::process_transfer_perf_diagnose_(
const int64_t timestamp,
const int64_t start_ts,
@ -1061,6 +1100,7 @@ int ObTabletBackfillTXTask::add_sstable_into_handles_(
}
int ObTabletBackfillTXTask::generate_mds_table_backfill_task_(
const int64_t dest_tranfser_seq,
share::ObITask *finish_task,
share::ObITask *&child)
{
@ -1081,7 +1121,7 @@ int ObTabletBackfillTXTask::generate_mds_table_backfill_task_(
LOG_WARN("failed to get tablet handler", K(ret), KPC(ha_dag_net_ctx_), K(ls_id_), K(tablet_info_));
} else if (OB_FAIL(tablet_backfill_tx_dag->alloc_task(mds_table_backfill_tx_task))) {
LOG_WARN("failed to alloc table backfill tx task", K(ret), KPC(ha_dag_net_ctx_), K(ls_id_), K(tablet_info_));
} else if (OB_FAIL(mds_table_backfill_tx_task->init(ls_id_, tablet_info_, tablet_handle))) {
} else if (OB_FAIL(mds_table_backfill_tx_task->init(ls_id_, tablet_info_, dest_tranfser_seq, tablet_handle))) {
LOG_WARN("failed to init mds table backfill tx task", K(ret), K(ls_id_), K(tablet_info_));
} else if (OB_FAIL(this->add_child(*mds_table_backfill_tx_task))) {
LOG_WARN("failed to add table backfill tx task as child", K(ret), K(ls_id_), K(tablet_info_));
@ -1233,6 +1273,7 @@ ObTabletTableBackfillTXTask::ObTabletTableBackfillTXTask()
ha_dag_net_ctx_(nullptr),
ls_id_(),
tablet_info_(),
dest_transfer_seq_(ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ),
tablet_handle_(),
table_handle_(),
tablets_table_mgr_(nullptr),
@ -1247,6 +1288,7 @@ ObTabletTableBackfillTXTask::~ObTabletTableBackfillTXTask()
int ObTabletTableBackfillTXTask::init(
const share::ObLSID &ls_id,
const ObTabletBackfillInfo &tablet_info,
const int64_t dest_transfer_seq,
ObTabletHandle &tablet_handle,
ObTableHandleV2 &table_handle,
share::ObITask *child)
@ -1257,7 +1299,7 @@ int ObTabletTableBackfillTXTask::init(
if (is_inited_) {
ret = OB_INIT_TWICE;
LOG_WARN("tablet table back fill tx task do not init", K(ret));
} else if (!ls_id.is_valid() || !tablet_info.is_valid() || !tablet_handle.is_valid() || !table_handle.is_valid() || OB_ISNULL(child)) {
} else if (!ls_id.is_valid() || !tablet_info.is_valid() || !tablet_handle.is_valid() || !table_handle.is_valid() || OB_ISNULL(child) || dest_transfer_seq < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("init tablet table backfill task get invalid argument", K(ret), K(ls_id),
K(tablet_info), K(tablet_handle), K(table_handle));
@ -1267,6 +1309,7 @@ int ObTabletTableBackfillTXTask::init(
backfill_tx_ctx_ = tablet_backfill_tx_dag->get_backfill_tx_ctx();
ls_id_ = ls_id;
tablet_info_ = tablet_info;
dest_transfer_seq_ = dest_transfer_seq;
tablet_handle_ = tablet_handle;
table_handle_ = table_handle;
tablets_table_mgr_ = tablet_backfill_tx_dag->get_backfill_tablets_table_mgr();
@ -1361,7 +1404,7 @@ int ObTabletTableBackfillTXTask::generate_merge_task_()
LOG_WARN("tablet table backfill tx task do not init", K(ret));
} else if (OB_FAIL(dag_->alloc_task(finish_backfill_task))) {
LOG_WARN("failed to alloc tablet finish backfill tx task", K(ret), KPC(ha_dag_net_ctx_), K(ls_id_), K(tablet_info_));
} else if (OB_FAIL(finish_backfill_task->init(ls_id_, tablet_info_, tablet_handle_, table_handle_, child_))) {
} else if (OB_FAIL(finish_backfill_task->init(ls_id_, tablet_info_, dest_transfer_seq_, tablet_handle_, table_handle_, child_))) {
LOG_WARN("failed to init table finish backfill tx task", K(ret), K(ls_id_), K(tablet_info_), K(table_handle_));
} else if (OB_FAIL(finish_backfill_task->add_child(*child_))) {
LOG_WARN("failed to add child task", K(ret), K(ls_id_), K(tablet_info_));
@ -1442,6 +1485,7 @@ ObTabletTableFinishBackfillTXTask::~ObTabletTableFinishBackfillTXTask()
int ObTabletTableFinishBackfillTXTask::init(
const share::ObLSID &ls_id,
const ObTabletBackfillInfo &tablet_info,
const int64_t dest_transfer_seq,
ObTabletHandle &tablet_handle,
ObTableHandleV2 &table_handle,
share::ObITask *child)
@ -1467,7 +1511,7 @@ int ObTabletTableFinishBackfillTXTask::init(
tablets_table_mgr_ = tablet_backfill_tx_dag->get_backfill_tablets_table_mgr();
child_ = child;
if (OB_FAIL(prepare_merge_ctx_())) {
if (OB_FAIL(prepare_merge_ctx_(dest_transfer_seq))) {
LOG_WARN("failed to prepare merge ctx", K(ret), K(ls_id_), K(tablet_info_), K(table_handle_));
} else {
is_inited_ = true;
@ -1476,7 +1520,7 @@ int ObTabletTableFinishBackfillTXTask::init(
return ret;
}
int ObTabletTableFinishBackfillTXTask::prepare_merge_ctx_()
int ObTabletTableFinishBackfillTXTask::prepare_merge_ctx_(const int64_t dest_transfer_seq)
{
int ret = OB_SUCCESS;
param_.merge_version_ = ObVersion::MIN_VERSION; //only major or meta major need this, mini or minor do not use it
@ -1488,7 +1532,7 @@ int ObTabletTableFinishBackfillTXTask::prepare_merge_ctx_()
int64_t local_rebuild_seq = 0;
if (OB_FAIL(tablets_table_mgr_->get_local_rebuild_seq(local_rebuild_seq))) {
LOG_WARN("failed to get local rebuild seq", K(ret), K(ls_id_), K(tablet_info_), K(table_handle_));
} else if (OB_FAIL(tablet_merge_ctx_.init(backfill_tx_ctx_->backfill_scn_, local_rebuild_seq, tablet_handle_, table_handle_))) {
} else if (OB_FAIL(tablet_merge_ctx_.init(backfill_tx_ctx_->backfill_scn_, local_rebuild_seq, dest_transfer_seq, tablet_handle_, table_handle_))) {
LOG_WARN("failed to init tablet merge ctx", K(ret), K(ls_id_), K(tablet_info_), K(table_handle_));
} else if (OB_FAIL(tablet_merge_ctx_.build_ctx(unused_finish_flag))) {
LOG_WARN("failed to build ctx", K(ret), K(ls_id_), K(tablet_info_));
@ -1759,6 +1803,7 @@ ObTabletMdsTableBackfillTXTask::ObTabletMdsTableBackfillTXTask()
ha_dag_net_ctx_(nullptr),
ls_id_(),
tablet_info_(),
dest_transfer_seq_(ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ),
tablet_handle_(),
allocator_("MdsBackfillTX"),
merger_arena_("TblBkfilMerger", OB_MALLOC_NORMAL_BLOCK_SIZE),
@ -1773,6 +1818,7 @@ ObTabletMdsTableBackfillTXTask::~ObTabletMdsTableBackfillTXTask()
int ObTabletMdsTableBackfillTXTask::init(
const share::ObLSID &ls_id,
const ObTabletBackfillInfo &tablet_info,
const int64_t dest_transfer_seq,
ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
@ -1794,6 +1840,7 @@ int ObTabletMdsTableBackfillTXTask::init(
backfill_tx_ctx_ = tablet_backfill_tx_dag->get_backfill_tx_ctx();
ls_id_ = ls_id;
tablet_info_ = tablet_info;
dest_transfer_seq_ = dest_transfer_seq;
tablet_handle_ = tablet_handle;
tablets_table_mgr_ = tablet_backfill_tx_dag->get_backfill_tablets_table_mgr();
is_inited_ = true;
@ -1904,6 +1951,9 @@ int ObTabletMdsTableBackfillTXTask::prepare_mds_table_merge_ctx_(
static_param.merge_scn_ = backfill_tx_ctx_->backfill_scn_;
static_param.create_snapshot_version_ = 0;
static_param.need_parallel_minor_merge_ = false;
// double set insurance
static_param.tablet_transfer_seq_ = dest_transfer_seq_;
tablet_merge_ctx.static_desc_.tablet_transfer_seq_ = dest_transfer_seq_;
if (OB_FAIL(tablet_merge_ctx.init_tablet_merge_info())) {
LOG_WARN("failed to init tablet merge info", K(ret), K(ls_id_), K(tablet_info_), K(tablet_merge_ctx));
@ -2044,6 +2094,7 @@ int ObTabletMdsTableBackfillTXTask::prepare_mds_sstable_merge_ctx_(
static_param.merge_scn_ = backfill_tx_ctx_->backfill_scn_;
static_param.create_snapshot_version_ = 0;
static_param.need_parallel_minor_merge_ = false;
static_param.tablet_transfer_seq_ = dest_transfer_seq_;
if (OB_FAIL(tablet_merge_ctx.prepare_merge_tables(mds_sstable_array))) {
LOG_WARN("failed to prepare merge tables", K(ret), K(mds_sstable_array));
} else if (OB_FAIL(tablet_merge_ctx.prepare_schema())) {
@ -2168,6 +2219,7 @@ ObTabletBackfillMergeCtx::~ObTabletBackfillMergeCtx()
int ObTabletBackfillMergeCtx::init(
const SCN &backfill_scn,
const int64_t ls_rebuild_seq,
const int64_t dest_transfer_seq,
ObTabletHandle &tablet_handle,
storage::ObTableHandleV2 &backfill_table_handle)
{
@ -2183,6 +2235,7 @@ int ObTabletBackfillMergeCtx::init(
tablet_handle_ = tablet_handle;
backfill_table_handle_ = backfill_table_handle;
ls_rebuild_seq_ = ls_rebuild_seq;
static_param_.tablet_transfer_seq_ = dest_transfer_seq;
is_inited_ = true;
}
return ret;
@ -2253,6 +2306,8 @@ int ObTabletBackfillMergeCtx::get_merge_tables(ObGetMergeTablesResult &get_merge
//snapshot_info is only for calculate multi version start, backfill will not change mulit version start
get_merge_table_result.snapshot_info_.snapshot_type_ = ObStorageSnapshotInfo::SNAPSHOT_MULTI_VERSION_START_ON_TABLET;
get_merge_table_result.snapshot_info_.snapshot_ = get_tablet()->get_multi_version_start();
//src_tablet_transfer_seq, should not be used to write block;
get_merge_table_result.transfer_seq_ = get_tablet()->get_transfer_seq();
}
}
return ret;

11
src/storage/high_availability/ob_tablet_backfill_tx.h
View File

@ -77,6 +77,7 @@ public:
int init(
const SCN &backfill_scn,
const int64_t ls_rebuild_seq,
const int64_t dest_transfer_seq,
ObTabletHandle &tablet_handle,
storage::ObTableHandleV2 &backfill_table_handle);
protected:
@ -144,6 +145,7 @@ public:
private:
int generate_backfill_tx_task_();
int generate_table_backfill_tx_task_(
const int64_t dest_tranfser_seq,
share::ObITask *replace_task,
common::ObIArray<ObTableHandleV2> &table_array,
share::ObITask *child);
@ -175,12 +177,14 @@ private:
ObTableStoreIterator &sstable_iter,
ObTablesHandleArray &sstable_handles);
int generate_mds_table_backfill_task_(
const int64_t dest_tranfser_seq,
share::ObITask *finish_task,
share::ObITask *&child);
int wait_memtable_frozen_();
int init_tablet_handle_();
int get_diagnose_support_info_(share::ObLSID &dest_ls_id, share::SCN &backfill_scn) const;
int get_dest_transfer_seq_(const common::ObTabletID &tablet_id, int64_t &transfer_seq) const;
void process_transfer_perf_diagnose_(
const int64_t timestamp,
const int64_t start_ts,
@ -205,6 +209,7 @@ public:
int init(
const share::ObLSID &ls_id,
const ObTabletBackfillInfo &tablet_info,
const int64_t dest_transfer_seq,
ObTabletHandle &tablet_handle,
ObTableHandleV2 &table_handle,
share::ObITask *child);
@ -228,6 +233,7 @@ private:
ObIHADagNetCtx *ha_dag_net_ctx_;
share::ObLSID ls_id_;
ObTabletBackfillInfo tablet_info_;
int64_t dest_transfer_seq_;
ObTabletHandle tablet_handle_;
ObTableHandleV2 table_handle_;
ObBackfillTabletsTableMgr *tablets_table_mgr_;
@ -243,6 +249,7 @@ public:
int init(
const share::ObLSID &ls_id,
const ObTabletBackfillInfo &tablet_info,
const int64_t dest_tranfser_seq,
ObTabletHandle &tablet_handle,
ObTableHandleV2 &table_handle,
share::ObITask *child);
@ -251,7 +258,7 @@ public:
VIRTUAL_TO_STRING_KV(K("ObTabletTableFinishBackfillTXTask"), KP(this), KPC(ha_dag_net_ctx_));
private:
int prepare_merge_ctx_();
int prepare_merge_ctx_(const int64_t dest_transfer_seq);
int update_merge_sstable_();
private:
@ -322,6 +329,7 @@ public:
int init(
const share::ObLSID &ls_id,
const ObTabletBackfillInfo &tablet_info,
const int64_t dest_transfer_seq_,
ObTabletHandle &tablet_handle);
virtual int process() override;
VIRTUAL_TO_STRING_KV(K("ObTabletMdsTableBackfillTXTask"), KP(this), KPC(ha_dag_net_ctx_));
@ -353,6 +361,7 @@ private:
ObIHADagNetCtx *ha_dag_net_ctx_;
share::ObLSID ls_id_;
ObTabletBackfillInfo tablet_info_;
int64_t dest_transfer_seq_;
ObTabletHandle tablet_handle_;
common::ObArenaAllocator allocator_;
compaction::ObLocalArena merger_arena_;

2
src/storage/high_availability/ob_tablet_transfer_info.cpp
View File

@ -31,7 +31,7 @@ int ObTabletTransferInfo::init()
int ret = OB_SUCCESS;
ls_id_ = TRANSFER_INIT_LS_ID;
transfer_start_scn_.set_min();
transfer_seq_ = 0;
transfer_seq_ = TRANSFER_INIT_SEQ;
has_transfer_table_ = false;
return ret;
}

1
src/storage/high_availability/ob_tablet_transfer_info.h
View File

@ -44,6 +44,7 @@ public:
share::SCN transfer_start_scn_;
int64_t transfer_seq_;
bool has_transfer_table_;
static const int64_t TRANSFER_INIT_SEQ = 0;
private:
static const int64_t TRANSFER_INIT_LS_ID = 0;
};

45
src/storage/ls/ob_ls_tablet_service.cpp
View File

@ -754,7 +754,7 @@ int ObLSTabletService::trim_old_tablets(const ObTabletID &tablet_id)
tablet_head->trim_tablet_list();
ObMetaDiskAddr disk_addr;
const ObTabletMapKey key(ls_->get_ls_id(), tablet_id);
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id, tablet_head->get_transfer_seq());
if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tablet_head, new_tablet_handle))) {
LOG_WARN("fail to persist and transfor tablet", K(ret), KPC(tablet_head), K(new_tablet_handle));
@ -794,7 +794,7 @@ int ObLSTabletService::rollback_rebuild_tablet(const ObTabletID &tablet_id)
const ObTabletMapKey key(ls_->get_ls_id(), tablet_id);
ObMetaDiskAddr disk_addr;
ObTabletHandle new_tablet_handle;
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id, next_tablet_handle.get_obj()->get_transfer_seq());
if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *next_tablet_handle.get_obj(), new_tablet_handle))) {
LOG_WARN("fail to persist and transform tablet", K(ret), K(next_tablet_handle), K(new_tablet_handle));
@ -828,7 +828,7 @@ int ObLSTabletService::rebuild_tablet_with_old(
const common::ObTabletID &tablet_id = mig_tablet_param.tablet_id_;
const share::ObLSID &ls_id = mig_tablet_param.ls_id_;
const ObTabletMapKey key(ls_id, tablet_id);
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id, mig_tablet_param.transfer_info_.transfer_seq_);
if (OB_FAIL(direct_get_tablet(tablet_id, old_tablet_hdl))) {
LOG_WARN("failed to get tablet", K(ret), K(key));
@ -872,7 +872,7 @@ int ObLSTabletService::migrate_update_tablet(
ObTablet *new_tablet = nullptr;
ObMetaDiskAddr disk_addr;
ObFreezer *freezer = ls_->get_freezer();
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id, mig_tablet_param.transfer_info_.transfer_seq_);
const bool is_transfer = false;
if (OB_FAIL(direct_get_tablet(tablet_id, old_tablet_hdl))) {
@ -913,7 +913,7 @@ int ObLSTabletService::migrate_create_tablet(
ObTabletHandle tmp_tablet_hdl;
ObTabletHandle tablet_handle;
ObMetaDiskAddr disk_addr;
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id, mig_tablet_param.transfer_info_.transfer_seq_);
ObTransService *tx_svr = MTL(ObTransService*);
const bool is_transfer = false;
@ -1072,7 +1072,7 @@ int ObLSTabletService::update_tablet_table_store(
const share::ObLSID &ls_id = ls_->get_ls_id();
const ObTabletMapKey key(ls_id, tablet_id);
ObMetaDiskAddr disk_addr;
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id, old_tablet->get_transfer_seq());
if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_hdl))) {
LOG_WARN("fail to acquire temporary tablet", K(ret), K(key));
@ -1139,7 +1139,7 @@ int ObLSTabletService::update_tablet_table_store(
time_guard.click("GetTablet");
ObTablet *old_tablet = old_tablet_hdl.get_obj();
ObMetaDiskAddr disk_addr;
const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id, old_tablet->get_transfer_seq());
share::SCN not_used_scn;
if (!is_mds_merge(param.compaction_info_.merge_type_) && OB_FAIL(tmp_tablet->init_for_merge(allocator, param, *old_tablet))) {
LOG_WARN("failed to init tablet", K(ret), K(param), KPC(old_tablet));
@ -1193,7 +1193,7 @@ int ObLSTabletService::update_tablet_to_empty_shell(const common::ObTabletID &ta
ObTablet *tmp_tablet = tmp_tablet_handle.get_obj();
ObTablet *new_tablet = nullptr;
ObMetaDiskAddr disk_addr;
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id, old_tablet->get_transfer_seq());
if (OB_FAIL(tmp_tablet->init_empty_shell(*tmp_tablet_handle.get_allocator(), *old_tablet))) {
LOG_WARN("failed to init tablet", K(ret), KPC(old_tablet));
} else if (FALSE_IT(time_guard.click("InitNew"))) {
@ -1251,7 +1251,7 @@ int ObLSTabletService::update_medium_compaction_info(
const share::ObLSID &ls_id = ls_->get_ls_id();
const ObTabletMapKey key(ls_id, tablet_id);
ObMetaDiskAddr disk_addr;
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id, old_tablet->get_transfer_seq());
if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_hdl))) {
if (OB_ENTRY_NOT_EXIST == ret) {
@ -1319,7 +1319,6 @@ int ObLSTabletService::build_new_tablet_from_mds_table(
} else if (OB_FAIL(mds_mini_sstable_handle.get_sstable(mds_sstable))) {
LOG_WARN("fail to get sstable from mds mini handle", K(ret), K(mds_mini_sstable_handle));
} else {
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id);
ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash());
time_guard.click("Lock");
if (OB_FAIL(direct_get_tablet(tablet_id, old_tablet_hdl))) {
@ -1336,6 +1335,7 @@ int ObLSTabletService::build_new_tablet_from_mds_table(
ObTablet *old_tablet = old_tablet_hdl.get_obj();
ObTablet *tmp_tablet = tmp_tablet_hdl.get_obj();
ObMetaDiskAddr disk_addr;
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id, old_tablet->get_transfer_seq());
ObUpdateTableStoreParam mds_param(ctx.static_param_.version_range_.snapshot_version_,
1/*multi_version_start*/,
ObMdsSchemaHelper::get_instance().get_storage_schema(),
@ -1401,7 +1401,7 @@ int ObLSTabletService::update_tablet_release_memtable_for_offline(
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
ObTabletHandle new_tablet_handle;
ObUpdateTabletPointerParam param;
const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
if (OB_FAIL(ObTabletPersister::copy_from_old_tablet(persist_param, *tablet, new_tablet_handle))) {
LOG_WARN("fail to copy from old tablet", K(ret), KPC(tablet));
} else if (FALSE_IT(time_guard.click("CpTablet"))) {
@ -1481,7 +1481,7 @@ int ObLSTabletService::update_tablet_ddl_commit_scn(
ObUpdateTabletPointerParam param;
ObTabletHandle new_handle;
const ObTablet &old_tablet = *old_handle.get_obj();
const ObTabletPersisterParam persist_param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id, old_tablet.get_transfer_seq());
if (OB_FAIL(ObTabletPersister::persist_and_transform_only_tablet_meta(persist_param, old_tablet, modifier, new_handle))) {
LOG_WARN("fail to persist and transform only tablet meta", K(ret), K(old_tablet), K(ddl_commit_scn));
@ -1535,7 +1535,7 @@ int ObLSTabletService::update_tablet_report_status(
}
if (need_report) {
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tablet, new_tablet_handle))) {
LOG_WARN("fail to persist and transform tablet", K(ret), KPC(tablet), K(new_tablet_handle));
} else if (FALSE_IT(time_guard.click("Persist"))) {
@ -1579,7 +1579,7 @@ int ObLSTabletService::update_tablet_snapshot_version(
const share::ObLSID &ls_id = ls_->get_ls_id();
const ObTabletMapKey key(ls_id, tablet_id);
ObMetaDiskAddr disk_addr;
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id, old_tablet->get_transfer_seq());
ObTabletDataStatus::STATUS current_status = ObTabletDataStatus::DATA_STATUS_MAX;
if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_hdl))) {
@ -1648,7 +1648,7 @@ int ObLSTabletService::update_tablet_restore_status(
&& OB_FALSE_IT((void)tablet->tablet_meta_.reset_transfer_table())) {
} else {
// TODO(jiahua.cjh) move check valid to tablet init after generate new version tablet.
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
if (OB_FAIL(tablet->check_valid())) {
LOG_WARN("failed to check tablet valid", K(ret), K(restore_status), KPC(tablet));
} else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(param, *tablet, new_tablet_handle))) {
@ -1708,7 +1708,7 @@ int ObLSTabletService::update_tablet_ha_data_status(
common::ObArenaAllocator allocator("UpdateSchema", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID(), ObCtxIds::DEFAULT_CTX_ID);
ObTabletHandle new_tablet_handle;
ObTabletHandle tmp_tablet_handle;
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id, old_tablet->get_transfer_seq());
bool is_row_store_with_co_major = false;
if (OB_FAIL(old_tablet->tablet_meta_.ha_status_.get_data_status(current_status))) {
@ -1781,7 +1781,7 @@ int ObLSTabletService::update_tablet_ha_expected_status(
const ObTabletMapKey key(ls_->get_ls_id(), tablet_id);
ObTablet *tablet = tablet_handle.get_obj();
ObTabletHandle new_tablet_handle;
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_->get_ls_id(), ls_->get_ls_epoch(), tablet_id, tablet->get_transfer_seq());
if (OB_FAIL(tablet->tablet_meta_.ha_status_.get_expected_status(current_status))) {
LOG_WARN("failed to get data status", K(ret), KPC(tablet));
@ -1861,6 +1861,7 @@ int ObLSTabletService::upload_major_compaction_tablet_meta(
ObTabletHandle old_tablet_hdl;
ObTabletHandle new_tablet_hdl;
ObTablet tmp_tablet;
int64_t old_tablet_transfer_seq = ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ;
ObTimeGuard time_guard("ObLSTabletService::UploadTabletMeta", 100_ms);
if (IS_NOT_INIT) {
@ -1881,6 +1882,7 @@ int ObLSTabletService::upload_major_compaction_tablet_meta(
} else if (old_tablet_hdl.get_obj()->is_empty_shell()) {
LOG_INFO("old tablet is empty shell tablet, should skip this operation", K(key));
} else {
old_tablet_transfer_seq = old_tablet_hdl.get_obj()->get_transfer_seq();
const ObTabletPoolType pool_type = old_tablet_hdl.get_obj()->get_try_cache_size() > ObTenantMetaMemMgr::NORMAL_TABLET_POOL_SIZE
? ObTabletPoolType::TP_LARGE : ObTabletPoolType::TP_NORMAL;
if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tablet_from_pool(pool_type, key, new_tablet_hdl))) {
@ -1891,7 +1893,8 @@ int ObLSTabletService::upload_major_compaction_tablet_meta(
}
}
const ObTabletPersisterParam persist_param(tablet_id, param.snapshot_version_, curr_macro_seq,
const ObTabletPersisterParam persist_param(tablet_id, old_tablet_transfer_seq,
param.snapshot_version_, curr_macro_seq,
param.ddl_info_.ddl_redo_callback_, param.ddl_info_.ddl_finish_callback_);
if (OB_FAIL(ret)) {
} else if (old_tablet_hdl.get_obj()->is_empty_shell()) {
@ -2455,7 +2458,7 @@ int ObLSTabletService::create_inner_tablet(
ObFreezer *freezer = ls_->get_freezer();
ObTabletHandle tmp_tablet_hdl;
ObMetaDiskAddr disk_addr;
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_->get_ls_epoch(), tablet_id, ObTabletTransferInfo::TRANSFER_INIT_SEQ);
ObTimeGuard time_guard("ObLSTabletService::create_inner_tablet", 10_ms);
ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash());
@ -2618,7 +2621,7 @@ int ObLSTabletService::create_empty_shell_tablet(
ObTabletHandle tablet_handle;
ObTablet *new_tablet = nullptr;
ObMetaDiskAddr disk_addr;
const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id, param.transfer_info_.transfer_seq_);
if (OB_FAIL(tmp_tablet->init_with_migrate_param(allocator, param, false/*is_update*/, freezer, is_transfer))) {
LOG_WARN("failed to init tablet", K(ret), K(param));
} else if (FALSE_IT(time_guard.click("InitTablet"))) {
@ -3932,7 +3935,7 @@ int ObLSTabletService::build_ha_tablet_new_table_store(
ObTablet *tmp_tablet = nullptr;
const share::ObLSID &ls_id = ls_->get_ls_id();
const ObTabletMapKey key(ls_id, tablet_id);
const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam persist_param(ls_id, ls_->get_ls_epoch(), tablet_id, old_tablet->get_transfer_seq());
if (OB_FAIL(ObTabletCreateDeleteHelper::acquire_tmp_tablet(key, allocator, tmp_tablet_handle))) {
LOG_WARN("failed to acquire tablet", K(ret), K(key));

1
src/storage/meta_mem/ob_external_tablet_cnt_map.h
View File

@ -30,6 +30,7 @@ public:
int check_exist(const ObDieingTabletMapKey &key, bool &exist);
int reg_tablet(const ObDieingTabletMapKey &key);
int unreg_tablet(const ObDieingTabletMapKey &key);
int64_t count() const { return ex_tablet_map_.size(); }
void destroy();
private:
bool is_inited_;

157
src/storage/meta_mem/ob_flying_tablet_pointer_map.cpp Normal file
View File

@ -0,0 +1,157 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include "storage/meta_mem/ob_flying_tablet_pointer_map.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/meta_store/ob_storage_meta_io_util.h"
#include "storage/blocksstable/ob_object_manager.h"
#include "storage/ls/ob_ls.h"
#include "storage/meta_store/ob_tenant_storage_meta_service.h"
namespace oceanbase
{
using namespace blocksstable;
namespace storage
{
ObFlyingTabletPointerMap::ObFlyingTabletPointerMap(const int64_t capacity)
: is_inited_(false),
capacity_(capacity),
bucket_lock_(),
map_()
{
}
int ObFlyingTabletPointerMap::init(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
int64_t bucket_num = 999;
if (is_inited_) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", K(ret));
} else if (OB_INVALID_TENANT_ID == tenant_id) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(tenant_id));
} else if (OB_FAIL(map_.create(bucket_num, "FlyTabletPtrMap", "FlyTabletPtrMap", tenant_id))) {
LOG_WARN("fail to initialize external tablet cnt map");
} else if (OB_FAIL(bucket_lock_.init(bucket_num, ObLatchIds::DEFAULT_BUCKET_LOCK, ObMemAttr(tenant_id, "FlyTabletMapLk")))) {
LOG_WARN("fail to init bucket lock", K(ret), K(bucket_num));
} else {
is_inited_ = true;
}
return ret;
}
int ObFlyingTabletPointerMap::set(const ObDieingTabletMapKey &key, ObTabletPointerHandle &handle)
{
int ret = OB_SUCCESS;
int64_t ls_id = handle.get_resource_ptr()->get_ls()->get_ls_id().id();
if (OB_UNLIKELY(!is_inited_)) {
ret = common::OB_NOT_INIT;
LOG_WARN("ObResourceMap has not been inited", K(ret));
} else if (OB_FAIL(map_.set_refactored(key, handle))) {
LOG_WARN("fail to set into ResourceMap", K(ret), K(key));
} else {
FLOG_INFO("success to push tablet_pointer to flying_map", K(ret), K(ls_id), K(key), KP(handle.get_resource_ptr()), KPC(handle.get_resource_ptr()), K(count()));
}
return ret;
}
int ObFlyingTabletPointerMap::check_exist(
const ObDieingTabletMapKey &key,
bool &is_exist)
{
int ret = common::OB_SUCCESS;
ObInnerTPHandlePtr handle_ptr;
ObTabletPointer *t_ptr = nullptr;
is_exist = false;
uint64_t hash_val = 0;
if (OB_UNLIKELY(!is_inited_)) {
ret = common::OB_NOT_INIT;
LOG_WARN("ObResourceMap has not been inited", K(ret));
} else if (OB_UNLIKELY(!key.is_valid())) {
ret = common::OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(key));
} else {
common::ObBucketHashRLockGuard lock_guard(bucket_lock_, key.hash());
if (OB_ISNULL(handle_ptr = map_.get(key))) {
is_exist = false;
LOG_INFO("tablet handle not exist", K(ret), K(key));
} else if (OB_ISNULL(t_ptr = handle_ptr->get_resource_ptr())) {
ret = common::OB_ERR_UNEXPECTED;
LOG_WARN("fail to get tablet pointer", K(ret), KP(t_ptr));
} else {
is_exist = true;
}
}
return ret;
}
void ObFlyingTabletPointerMap::destroy()
{
is_inited_ = false;
bucket_lock_.destroy();
for (common::hash::ObHashMap<ObDieingTabletMapKey, ObTabletPointerHandle>::iterator iter = map_.begin();
iter != map_.end();
++iter) {
iter->second.reset();
}
map_.destroy();
}
int ObFlyingTabletPointerMap::erase(const ObDieingTabletMapKey &key)
{
int ret = common::OB_SUCCESS;
bool is_exist = false;
if (OB_UNLIKELY(!is_inited_)) {
ret = common::OB_NOT_INIT;
LOG_WARN("ObResourceMap has not been inited", K(ret));
} else if (OB_FAIL(check_exist(key, is_exist))) {
LOG_WARN("failed to check exist", K(ret), K(key));
} else if (!is_exist) {
LOG_WARN("this key is not exist, do not erase", K(ret), K(key));
} else if (OB_FAIL(inner_erase_(key))) {
LOG_WARN("fail to erase meta pointer", K(ret), K(key));
}
FLOG_INFO("success to remove tablet_pointer to flying_map", K(ret), K(key), K(count()));
return ret;
}
int ObFlyingTabletPointerMap::inner_erase_(const ObDieingTabletMapKey &key)
{
int ret = common::OB_SUCCESS;
ObInnerTPHandlePtr handle_ptr;
uint64_t hash_val = 0;
if (OB_UNLIKELY(!key.is_valid())) {
ret = common::OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(key));
} else {
common::ObBucketHashWLockGuard lock_guard(bucket_lock_, key.hash());
if (OB_ISNULL(handle_ptr = map_.get(key))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get from map", K(ret), K(key));
} else if (!handle_ptr->get_resource_ptr()->need_remove_from_flying_()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet_pointer should not be erased when tablet has been referred",
K(ret), KP(handle_ptr->get_resource_ptr()), KPC(handle_ptr->get_resource_ptr()) );
} else if (OB_FAIL(map_.erase_refactored(key))) {
LOG_WARN("fail to erase from map", K(ret));
}
}
return ret;
}
} // end namespace storage
} // end namespace oceanbase

51
src/storage/meta_mem/ob_flying_tablet_pointer_map.h Normal file
View File

@ -0,0 +1,51 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_STORAGE_OB_FLYING_POINTER_MAP_H_
#define OCEANBASE_STORAGE_OB_FLYING_POINTER_MAP_H_
#include "storage/meta_mem/ob_meta_obj_struct.h"
#include "storage/meta_mem/ob_tablet_map_key.h"
#include "storage/meta_mem/ob_tablet_pointer.h"
#include "storage/meta_mem/ob_tablet_pointer_handle.h"
#include "storage/ob_resource_map.h"
namespace oceanbase
{
namespace storage
{
class ObFlyingTabletPointerMap final
{
friend class ObTenantMetaMemMgr;
typedef ObTabletPointerHandle* ObInnerTPHandlePtr;
public:
ObFlyingTabletPointerMap(const int64_t capacity);
int init(const uint64_t tenant_id);
int set(const ObDieingTabletMapKey &key, ObTabletPointerHandle &handle);
int check_exist(const ObDieingTabletMapKey &key, bool &is_exist);
int erase(const ObDieingTabletMapKey &key);
int64_t count() const { return map_.size(); }
void destroy();
private:
int inner_erase_(const ObDieingTabletMapKey &key);
private:
bool is_inited_;
int64_t capacity_;
common::ObBucketLock bucket_lock_;
common::hash::ObHashMap<ObDieingTabletMapKey, ObTabletPointerHandle> map_;
DISALLOW_COPY_AND_ASSIGN(ObFlyingTabletPointerMap);
};
} // end namespace storage
} // end namespace oceanbase
#endif /* OCEANBASE_STORAGE_OB_FLYING_POINTER_MAP_H_ */

5
src/storage/meta_mem/ob_meta_obj_struct.cpp
View File

@ -203,14 +203,15 @@ int64_t ObMetaDiskAddr::to_string(char *buf, const int64_t buf_len) const
break;
default:
databuff_printf(buf, buf_len, pos,
"[2nd=%lu][3rd=%lu][offset=%lu,size=%lu,type=%lu,seq=%lu][5th=%lu][6th=%lu]}",
"[2nd=%lu][3rd=%lu][offset=%lu,size=%lu,type=%lu,seq=%lu][trans_seq=%lu, sec_part=%lu][6th=%lu]}",
second_id_,
third_id_,
(uint64_t) offset_,
(uint64_t) size_,
(uint64_t) type_,
(uint64_t) seq_,
fifth_id_,
(uint64_t) block_id().macro_transfer_seq(),
(uint64_t) block_id().tenant_seq(),
sixth_id_);
break;
};

1
src/storage/meta_mem/ob_meta_obj_struct.h
View File

@ -56,7 +56,6 @@ public:
OB_INLINE bool is_file() const { return FILE == type_; }
OB_INLINE bool is_memory() const { return MEM == type_; }
OB_INLINE bool is_none() const { return NONE == type_; }
OB_INLINE int64_t fifth_id() const { return fifth_id_; }
OB_INLINE void set_none_addr() { type_ = NONE; }
OB_INLINE void set_seq(const uint64_t seq) { seq_ = seq; }
OB_INLINE void set_size(const uint64_t size) { size_ = size; }

17
src/storage/meta_mem/ob_tablet_pointer.cpp
View File

@ -45,6 +45,7 @@ ObTabletPointer::ObTabletPointer()
protected_memtable_mgr_handle_(),
ddl_info_(),
initial_state_(true),
flying_(false),
ddl_kv_mgr_lock_(),
mds_lock_(),
mds_table_handler_(),
@ -65,6 +66,7 @@ ObTabletPointer::ObTabletPointer(
protected_memtable_mgr_handle_(memtable_mgr_handle),
ddl_info_(),
initial_state_(true),
flying_(false),
ddl_kv_mgr_lock_(),
mds_table_handler_(),
old_version_chain_(nullptr)
@ -90,6 +92,7 @@ void ObTabletPointer::reset()
reset_obj();
phy_addr_.reset();
ls_handle_.reset();
flying_ = false;
}
void ObTabletPointer::reset_obj()
@ -124,6 +127,18 @@ void ObTabletPointer::reset_obj()
}
}
bool ObTabletPointer::need_push_to_flying_() const
{
return (is_in_memory() && obj_.ptr_->get_ref() > 1) ||
OB_NOT_NULL(old_version_chain_);
}
bool ObTabletPointer::need_remove_from_flying_() const
{
return is_flying() && is_old_version_chain_empty();
}
int ObTabletPointer::read_from_disk(
const bool is_full_load,
common::ObArenaAllocator &allocator,
@ -325,7 +340,7 @@ int ObTabletPointer::dump_meta_obj(ObMetaObjGuard<ObTablet> &guard, void *&free_
const ObTabletID tablet_id = obj_.ptr_->tablet_meta_.tablet_id_;
const int64_t wash_score = obj_.ptr_->get_wash_score();
guard.get_obj(meta_obj);
const ObTabletPersisterParam param(ls_id, ls_handle_.get_ls()->get_ls_epoch(), tablet_id);
const ObTabletPersisterParam param(ls_id, ls_handle_.get_ls()->get_ls_epoch(), tablet_id, obj_.ptr_->get_transfer_seq());
ObTablet *tmp_obj = obj_.ptr_;
if (OB_NOT_NULL(meta_obj.ptr_) && obj_.ptr_->get_try_cache_size() <= ObTenantMetaMemMgr::NORMAL_TABLET_POOL_SIZE) {
char *buf = reinterpret_cast<char*>(meta_obj.ptr_);

9
src/storage/meta_mem/ob_tablet_pointer.h
View File

@ -117,6 +117,8 @@ class ObTabletPointer final
friend class ObLSTabletService;
friend class ObTenantMetaMemMgr;
friend class ObTabletResidentInfo;
friend class ObTabletPointerMap;
friend class ObFlyingTabletPointerMap;
public:
template <LockMode MODE>
void get_mds_truncate_lock_guard(TabletMdsLockGuard<MODE> &lock_guard) const {
@ -167,7 +169,7 @@ public:
// do not KPC memtable_mgr, may dead lock
TO_STRING_KV(K_(phy_addr), K_(obj), K_(ls_handle), K_(ddl_kv_mgr_handle), K_(attr),
K_(protected_memtable_mgr_handle), K_(ddl_info), K_(initial_state), KP_(old_version_chain));
K_(protected_memtable_mgr_handle), K_(ddl_info), K_(initial_state), KP_(old_version_chain), K_(flying));
public:
bool get_initial_state() const;
ObTabletResidentInfo get_tablet_resident_info(const ObTabletMapKey &key) const;
@ -201,6 +203,10 @@ private:
int wash_obj();
int add_tablet_to_old_version_chain(ObTablet *tablet);
int remove_tablet_from_old_version_chain(ObTablet *tablet);
void set_flying() { flying_ = true; }
bool is_flying() const { return flying_; }
bool need_push_to_flying_() const;
bool need_remove_from_flying_() const;
private:
ObMetaDiskAddr phy_addr_; // 48B
ObMetaObj<ObTablet> obj_; // 40B
@ -209,6 +215,7 @@ private:
ObProtectedMemtableMgrHandle protected_memtable_mgr_handle_; // 32B
ObTabletDDLInfo ddl_info_; // 32B
bool initial_state_; // 1B
bool flying_; // 1B
ObByteLock ddl_kv_mgr_lock_; // 1B
mutable mds::MdsLock mds_lock_;// 12B
mds::ObMdsTableHandler mds_table_handler_;// 48B

12
src/storage/meta_mem/ob_tablet_pointer_map.cpp
View File

@ -68,7 +68,8 @@ int ObTabletPointerMap::erase(const ObTabletMapKey &key, ObMetaObjGuard<ObTablet
int ObTabletPointerMap::inner_erase(const ObTabletMapKey &key)
{
int ret = common::OB_SUCCESS;
ObResourceValueStore<ObTabletPointer> *ptr = NULL;
ObResourceValueStore<ObTabletPointer> *ptr = nullptr;
ObTabletPointer *tablet_ptr = nullptr;
uint64_t hash_val = 0;
if (OB_UNLIKELY(!key.is_valid())) {
ret = common::OB_INVALID_ARGUMENT;
@ -79,11 +80,16 @@ int ObTabletPointerMap::inner_erase(const ObTabletMapKey &key)
common::ObBucketHashWLockGuard lock_guard(ResourceMap::bucket_lock_, hash_val);
if (OB_FAIL(ResourceMap::map_.get_refactored(key, ptr))) {
STORAGE_LOG(WARN, "fail to get from map", K(ret));
} else if (OB_ISNULL(ptr)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "ptr should not be nullptr", K(ret), K(key), KP(ptr));
} else if (OB_ISNULL(tablet_ptr = ptr->get_value_ptr())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "value should not be nullptr", K(ret), K(key), KP(tablet_ptr));
} else if (OB_FAIL(ResourceMap::map_.erase_refactored(key))) {
STORAGE_LOG(WARN, "fail to erase from map", K(ret));
} else {
ObTabletPointer *value = ptr->get_value_ptr();
value->reset_obj();
tablet_ptr->reset_obj();
if (OB_FAIL(ResourceMap::dec_handle_ref(ptr))) {
STORAGE_LOG(WARN, "fail to dec handle ref", K(ret));
}

115
src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp
View File

@ -176,6 +176,7 @@ ObTenantMetaMemMgr::ObTenantMetaMemMgr(const uint64_t tenant_id)
bucket_lock_(),
full_tablet_creator_(),
tablet_map_(),
flying_tablet_map_(FLYING_TABLET_THRESHOLD),
external_tablet_cnt_map_(),
tg_id_(-1),
table_gc_task_(this),
@ -241,6 +242,8 @@ int ObTenantMetaMemMgr::init()
LOG_WARN("fail to initialize tablet map", K(ret), K(bucket_num));
} else if (OB_FAIL(external_tablet_cnt_map_.init(193/*prime bucket_num*/, tenant_id_))) {
LOG_WARN("fail to initialize external tablet cnt map", K(ret), K(bucket_num), K(tenant_id_));
} else if (OB_FAIL(flying_tablet_map_.init(tenant_id_))) {
LOG_WARN("fail to initialize tablet map", K(ret), K(bucket_num));
} else if (OB_FAIL(gc_memtable_map_.create(10, "GCMemtableMap", "GCMemtableMap", tenant_id_))) {
LOG_WARN("fail to initialize gc memtable map", K(ret));
} else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TenantMetaMemMgr, tg_id_))) {
@ -274,6 +277,30 @@ int ObTenantMetaMemMgr::fetch_tenant_config()
return ret;
}
int ObTenantMetaMemMgr::check_allow_tablet_gc(
const ObTabletID &tablet_id,
const int64_t transfer_seq,
bool &allow)
{
int ret = OB_SUCCESS;
bool is_exist = false;
allow = false;
ObDieingTabletMapKey key(tablet_id.id(), transfer_seq);
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObTenantMetaMemMgr hasn't been inited", K(ret));
} else if (OB_FAIL(flying_tablet_map_.check_exist(key, is_exist))) {
LOG_WARN("fail to check tablet exist", K(ret), K(key));
} else if (is_exist) {
allow = false;
} else if (OB_FAIL(external_tablet_cnt_map_.check_exist(key, is_exist))) {
LOG_WARN("fail to check tablet exist", K(ret), K(key));
} else {
allow = !is_exist;
}
return ret;
}
void ObTenantMetaMemMgr::init_pool_arr()
{
pool_arr_[static_cast<int>(ObITable::TableType::DATA_MEMTABLE)] = &memtable_pool_;
@ -347,8 +374,9 @@ void ObTenantMetaMemMgr::destroy()
tg_id_ = -1;
}
full_tablet_creator_.reset(); // must reset after gc_tablets
tablet_map_.destroy();
flying_tablet_map_.destroy();
external_tablet_cnt_map_.destroy();
tablet_map_.destroy();
for (common::hash::ObHashMap<share::ObLSID, memtable::ObMemtableSet*>::iterator iter = gc_memtable_map_.begin();
OB_SUCC(ret) && iter != gc_memtable_map_.end(); ++iter) {
memtable::ObMemtableSet *memtable_set = iter->second;
@ -771,6 +799,7 @@ int ObTenantMetaMemMgr::push_tablet_into_gc_queue(ObTablet *tablet)
} else {
const ObTabletMeta &tablet_meta = tablet->get_tablet_meta();
const ObTabletMapKey key(tablet_meta.ls_id_, tablet_meta.tablet_id_);
const ObDieingTabletMapKey dieing_tablet_key(key, tablet->get_transfer_seq());
ObBucketHashWLockGuard lock_guard(bucket_lock_, key.hash());
const ObTabletPointerHandle &ptr_handle = tablet->get_pointer_handle();
@ -781,6 +810,8 @@ int ObTenantMetaMemMgr::push_tablet_into_gc_queue(ObTablet *tablet)
} else if (OB_FAIL(tablet_ptr->remove_tablet_from_old_version_chain(tablet))) {
LOG_WARN("fail to remove tablet from old version chain", K(ret), K(key), KPC(tablet));
} else if (FALSE_IT(tablet->reset_memtable())) {
} else if (tablet_ptr->need_remove_from_flying_() && OB_FAIL(flying_tablet_map_.erase(dieing_tablet_key))) {
LOG_WARN("Fail to erase tablet_ptr from flying_tablet_map", K(ret), K(dieing_tablet_key));
} else if (OB_FAIL(inner_push_tablet_into_gc_queue(tablet))) {
LOG_WARN("fail to push tablet into gc queue", K(ret), KPC(tablet));
}
@ -1710,6 +1741,7 @@ int ObTenantMetaMemMgr::get_current_version_for_tablet(
const share::ObLSID &ls_id,
const ObTabletID &tablet_id,
int64_t &tablet_version,
int64_t &tablet_transfer_seq,
bool &allow_tablet_version_gc)
{
int ret = OB_SUCCESS;
@ -1741,8 +1773,10 @@ int ObTenantMetaMemMgr::get_current_version_for_tablet(
if (OB_FAIL(external_tablet_cnt_map_.check_exist(dieing_key, exist_in_external))) {
LOG_WARN("fail to check ex_tablet exist or not", K(ret), K(dieing_key), K(exist_in_external));
} else {
tablet_version = tablet_ptr->get_addr().fifth_id();
tablet_version = tablet_ptr->get_addr().block_id().meta_version_id();
tablet_transfer_seq = tablet_ptr->get_addr().block_id().meta_transfer_seq();
allow_tablet_version_gc = tablet_ptr->is_old_version_chain_empty() && !exist_in_external;
FLOG_INFO("PRINT TABLET ADDRESS", K(ret), K(tablet_ptr->get_addr()), K(tablet_handle.get_obj()->get_tablet_addr()));
}
}
}
@ -2031,6 +2065,71 @@ bool ObTenantMetaMemMgr::is_tablet_handle_leak_checker_enabled()
return is_tablet_leak_checker_enabled_;
}
int ObTenantMetaMemMgr::push_tablet_pointer_to_fly_map_if_need_(
const ObTabletMapKey &key)
{
int ret = OB_SUCCESS;
int64_t tablet_transfer_seq = -1;
ObTabletPointer *tablet_ptr = nullptr;
ObTabletPointerHandle tp_handle(tablet_map_);
ObTabletHandle t_handle;
if (!key.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(key));
} else if (OB_FAIL(tablet_map_.get(key, tp_handle))) {
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to get ptr handle", K(ret), K(key));
}
} else if (OB_ISNULL(tablet_ptr = static_cast<ObTabletPointer*>(tp_handle.get_resource_ptr()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet ptr is NULL", K(ret), K(tp_handle));
} else if (!tablet_ptr->need_push_to_flying_()) {
LOG_INFO("need not push tablet_ptr to flying", K(ret), K(key), KPC(tablet_ptr));
} else if (OB_FAIL(get_tablet(WashTabletPriority::WTP_HIGH, key, t_handle))) { // for get tablet_transfer_seq
if (OB_ITEM_NOT_SETTED == ret) {
ret = OB_SUCCESS;
LOG_INFO("tablet has not been persisted, need not push_to_fly", K(ret), K(key));
} else {
LOG_WARN("failed to get tablet", K(ret), K(key), K(t_handle));
}
} else if (!t_handle.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid tablet_handle", K(ret), K(t_handle));
} else { // need push tablet_ptr to flying map
bool exist = false;
tablet_transfer_seq = t_handle.get_obj()->get_transfer_seq();
ObDieingTabletMapKey dieing_tablet_key(key, tablet_transfer_seq);
if (OB_FAIL(flying_tablet_map_.check_exist(dieing_tablet_key, exist))) {
LOG_WARN("failed to check exist", K(ret), K(dieing_tablet_key));
} else if (exist) {
// do nothing
LOG_INFO("tablet_pointer has exist in flying_map", K(ret), K(dieing_tablet_key));
} else if (OB_FAIL(flying_tablet_map_.set(dieing_tablet_key, tp_handle))) {
LOG_WARN("fail to push referred tablet_pointer to flying_tablet_map", K(ret), K(dieing_tablet_key), KPC(tablet_ptr));
} else if (OB_FAIL(tablet_ptr->add_tablet_to_old_version_chain(t_handle.get_obj()))) {
LOG_WARN("failed to add tablet to old_chain", K(ret), KPC(tablet_ptr), KPC(t_handle.get_obj()));
} else {
tablet_ptr->set_flying();
}
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(tablet_ptr)) {
int tmp_ret = OB_SUCCESS;
ObDieingTabletMapKey dieing_tablet_key(key, tablet_transfer_seq);
if (OB_TMP_FAIL(flying_tablet_map_.erase(dieing_tablet_key))) {
LOG_WARN("fail to erase tablet from tablet_pointer", K(tmp_ret), K(dieing_tablet_key));
} else {
tablet_ptr->flying_ = false;
}
}
}
return ret;
}
int ObTenantMetaMemMgr::del_tablet(const ObTabletMapKey &key)
{
int ret = OB_SUCCESS;
@ -2049,7 +2148,9 @@ int ObTenantMetaMemMgr::del_tablet(const ObTabletMapKey &key)
// unregister step need ahead of erase, cause a new tablet maybe created after erase but before unregister
// and that new tablet's register step will failed
mark_mds_table_deleted_(key);
if (OB_FAIL(tablet_map_.erase(key, handle))) {
if (OB_FAIL(push_tablet_pointer_to_fly_map_if_need_(key))) {
LOG_WARN("fail to try push tablet_pointer into flying_map", K(ret), K(key));
} else if (OB_FAIL(tablet_map_.erase(key, handle))) { // reuse handle
LOG_WARN("fail to erase tablet pointer", K(ret), K(key));
} else {
if (OB_NOT_NULL(handle.get_obj()) && OB_ISNULL(handle.get_obj()->get_allocator())) {
@ -2230,9 +2331,12 @@ int ObTenantMetaMemMgr::check_all_meta_mem_released(bool &is_released, const cha
const int64_t tx_ctx_memtable_cnt_ = tx_ctx_memtable_pool_.get_used_obj_cnt();
const int64_t lock_memtable_cnt_ = lock_memtable_pool_.get_used_obj_cnt();
const int64_t full_tablet_cnt = full_tablet_creator_.get_used_obj_cnt();
const int64_t external_tablet_cnt = external_tablet_cnt_map_.count();
const int64_t flying_tablet_pointer_cnt = flying_tablet_map_.count();
if (memtable_cnt != 0 || ddl_kv_cnt != 0 || tablet_cnt != 0 || 0 != large_tablet_cnt || ddl_kv_mgr_cnt != 0
|| tx_data_memtable_cnt_ != 0 || tx_ctx_memtable_cnt_ != 0
|| lock_memtable_cnt_ != 0 || full_tablet_cnt != 0) {
|| lock_memtable_cnt_ != 0 || full_tablet_cnt != 0
|| external_tablet_cnt != 0 || flying_tablet_pointer_cnt != 0) {
is_released = false;
} else {
is_released = true;
@ -2243,7 +2347,8 @@ int ObTenantMetaMemMgr::check_all_meta_mem_released(bool &is_released, const cha
LOG_INFO("check all meta mem in t3m", K(module), K(is_released), K(memtable_cnt), K(ddl_kv_cnt),
K(tablet_cnt), K(large_tablet_cnt), K(ddl_kv_mgr_cnt), K(tx_data_memtable_cnt_),
K(tx_ctx_memtable_cnt_), K(lock_memtable_cnt_), K(full_tablet_cnt),
K(wait_gc_tablets_cnt), K(wait_gc_tables_cnt), K(tablet_cnt_in_map));
K(wait_gc_tablets_cnt), K(wait_gc_tables_cnt), K(tablet_cnt_in_map), K(external_tablet_cnt),
K(flying_tablet_pointer_cnt));
}
return ret;
}

6
src/storage/meta_mem/ob_tenant_meta_mem_mgr.h
View File

@ -29,6 +29,7 @@
#include "storage/memtable/ob_memtable.h"
#include "storage/memtable/ob_memtable_util.h"
#include "storage/meta_mem/ob_external_tablet_cnt_map.h"
#include "storage/meta_mem/ob_flying_tablet_pointer_map.h"
#include "storage/meta_mem/ob_meta_obj_struct.h"
#include "storage/meta_mem/ob_tablet_pointer_map.h"
#include "storage/meta_mem/ob_tablet_pointer_handle.h"
@ -265,7 +266,9 @@ public:
const share::ObLSID &ls_id,
const ObTabletID &tablet_id,
int64_t &tablet_version,
int64_t &tablet_transfer_seq,
bool &allow_tablet_version_gc);
int check_allow_tablet_gc(const ObTabletID &tablet_id, const int64_t transfer_seq, bool &allow);
int get_tablet_buffer_infos(ObIArray<ObTabletBufferInfo> &buffer_infos);
int get_tablet_addr(const ObTabletMapKey &key, ObMetaDiskAddr &addr);
int has_tablet(const ObTabletMapKey &key, bool &is_exist);
@ -481,6 +484,7 @@ private:
static const int64_t DEFAULT_MINOR_SSTABLE_SET_COUNT = 49999;
static const int64_t SSTABLE_GC_MAX_TIME = 500; // 500us
static const int64_t LEAK_CHECKER_CONFIG_REFRESH_TIMEOUT = 10000000 * 10; // 10s
static const int64_t FLYING_TABLET_THRESHOLD = 100000;
typedef common::ObBinaryHeap<CandidateTabletInfo, HeapCompare, DEFAULT_TABLET_WASH_HEAP_COUNT> Heap;
typedef common::ObDList<ObMetaObjBufferNode> TabletBufferList;
typedef common::hash::ObHashSet<MinMinorSSTableInfo, common::hash::NoPthreadDefendMode> SSTableSet;
@ -523,6 +527,7 @@ private:
void batch_gc_memtable_();
void batch_destroy_memtable_(memtable::ObMemtableSet *memtable_set);
bool is_tablet_handle_leak_checker_enabled();
int push_tablet_pointer_to_fly_map_if_need_(const ObTabletMapKey &key);
private:
common::SpinRWLock wash_lock_;
@ -531,6 +536,7 @@ private:
ObBucketLock bucket_lock_;
ObFullTabletCreator full_tablet_creator_;
ObTabletPointerMap tablet_map_;
ObFlyingTabletPointerMap flying_tablet_map_;
ObExternalTabletCntMap external_tablet_cnt_map_;
int tg_id_;
int persist_tg_id_; // since persist task may cost too much time, we use another thread to exec.

7
src/storage/meta_store/ob_server_storage_meta_replayer.cpp
View File

@ -23,6 +23,7 @@
#include "storage/slog_ckpt/ob_server_checkpoint_slog_handler.h"
#ifdef OB_BUILD_SHARED_STORAGE
#include "storage/shared_storage/ob_disk_space_manager.h"
#include "storage/tx_storage/ob_tablet_gc_service.h"
#endif
namespace oceanbase
@ -180,6 +181,10 @@ int ObServerStorageMetaReplayer::handle_tenant_create_commit_(const ObTenantMeta
#ifdef OB_BUILD_SHARED_STORAGE
if (OB_FAIL(ret)) {
} else if (GCTX.is_shared_storage_mode()) {
MTL_SWITCH(tenant_id) {
// for macro check in observer start
MTL(checkpoint::ObTabletGCService*)->set_observer_start_macro_block_id_trigger();
}
// when restart observer, if current sys tenant is hidden, hidden_sys_data_disk_size is hidden_sys_data_disk_config_size
const bool is_hidden = tenant_meta.super_block_.is_hidden_;
if ((OB_SYS_TENANT_ID == tenant_id) && is_hidden) {
@ -346,4 +351,4 @@ int ObServerStorageMetaReplayer::ss_read_tenant_unit_(
} // namespace storage
} // namespace oceanbase
} // namespace oceanbase

155
src/storage/meta_store/ob_tenant_storage_meta_persister.cpp
View File

@ -351,7 +351,7 @@ int ObTenantStorageMetaPersister::write_active_tablet_array(ObLS *ls)
LOG_WARN("tablet key or addr is invalid", K(ret), K(tablet_key), K(addr));
} else if (!addr.is_disked()) {
FLOG_INFO("skip MEM and NONE type", K(ret), K(tablet_key), K(addr));
} else if (OB_FAIL(active_tablet_arr.items_.push_back(ObActiveTabletItem(tablet_key.tablet_id_, addr.fifth_id())))) {
} else if (OB_FAIL(active_tablet_arr.items_.push_back(ObActiveTabletItem(tablet_key.tablet_id_, addr.block_id().meta_version_id())))) {
LOG_WARN("fail to push back active tablet item", K(ret), K(tablet_key), K(addr));
}
}
@ -852,6 +852,7 @@ int ObTenantStorageMetaPersister::ss_check_and_delete_tablet_current_version(
const ObLSID &ls_id,
const uint64_t ls_epoch,
const int64_t deleted_tablet_version,
const int64_t deleted_tablet_transfer_seq,
ObArenaAllocator &allocator)
{
int ret = OB_SUCCESS;
@ -868,9 +869,14 @@ int ObTenantStorageMetaPersister::ss_check_and_delete_tablet_current_version(
LOG_WARN("fail to read cur version",
K(ret), K(ls_id), K(tablet_id), K(deleted_tablet_version), K(current_version_opt));
}
} else if (latest_addr.tablet_addr_.fifth_id() > deleted_tablet_version) {
} else if (latest_addr.tablet_addr_.block_id().meta_transfer_seq() > deleted_tablet_transfer_seq) {
// newer tablet_transfer_seq need not process
} else if (latest_addr.tablet_addr_.block_id().meta_transfer_seq() < deleted_tablet_transfer_seq) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet transfer_seq in rebooting should not larger than curr_version", K(ret), K(deleted_tablet_transfer_seq), K(latest_addr));
} else if (latest_addr.tablet_addr_.block_id().meta_version_id() > deleted_tablet_version) {
// may be transfer_in tablet, need not delete current_version tablet;
} else if (latest_addr.tablet_addr_.fifth_id() < deleted_tablet_version) {
} else if (latest_addr.tablet_addr_.block_id().meta_version_id() < deleted_tablet_version) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet version in current_version is the last version of tablet, should equal to the version in pending_free_arr",
K(ret), K(ls_id), K(tablet_id), K(deleted_tablet_version), K(latest_addr));
@ -941,8 +947,8 @@ int ObTenantStorageMetaPersister::ss_remove_tablet_(
ObLSPendingFreeTabletArray tmp_array;
lib::ObMutexGuard guard(array_info->lock_);
const int64_t curr_t = ObTimeUtility::fast_current_time();
const ObPendingFreeTabletItem tablet_item(tablet_id, tablet_addr.fifth_id(),
ObPendingFreeTabletStatus::WAIT_GC, curr_t, gc_type);
const ObPendingFreeTabletItem tablet_item(tablet_id, tablet_addr.block_id().meta_version_id(),
ObPendingFreeTabletStatus::WAIT_GC, curr_t, gc_type, tablet_addr.block_id().meta_transfer_seq());
if (OB_FAIL(tmp_array.assign(array_info->pending_free_tablet_arr_))) {
LOG_WARN("fail to assign pending free tablet array", K(ret));
} else if (has_exist_in_array(tmp_array.items_, tablet_item)) {
@ -977,38 +983,53 @@ int ObTenantStorageMetaPersister::create_tenant_ls_item_(
const ObLSID ls_id, int64_t &ls_epoch)
{
int ret = OB_SUCCESS;
lib::ObMutexGuard guard(super_block_lock_);
omt::ObTenant *tenant = static_cast<omt::ObTenant*>(MTL_CTX());
ObTenantSuperBlock tenant_super_block = tenant->get_super_block();
int64_t i = 0;
for (; i < tenant_super_block.ls_cnt_; i++) {
const ObLSItem &item = tenant_super_block.ls_item_arr_[i];
if (ls_id == item.ls_id_ && item.status_ != ObLSItemStatus::CREATE_ABORT &&
item.status_ != ObLSItemStatus::DELETED) {
break;
}
}
if (OB_UNLIKELY(i != tenant_super_block.ls_cnt_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls item already exist", K(ret), "ls_item", tenant_super_block.ls_item_arr_[i]);
} else if (OB_UNLIKELY(ObTenantSuperBlock::MAX_LS_COUNT == i)) {
ret = OB_SIZE_OVERFLOW;
LOG_WARN("too many ls", K(ret), K(ls_id), K(tenant_super_block));
uint64_t inf_seq = 0;
// have to get macro_seq before get sputer_block_lock
// update preallocate.. need the lock either
if (OB_FAIL(TENANT_SEQ_GENERATOR.get_private_object_seq(inf_seq))) {
LOG_WARN("fail to get tenant_object_seq", K(ret));
} else {
ObLSItem &item = tenant_super_block.ls_item_arr_[i];
tenant_super_block.ls_cnt_ = i + 1;
item.ls_id_ = ls_id;
item.status_ = ObLSItemStatus::CREATING;
item.epoch_ = tenant_super_block.auto_inc_ls_epoch_++;
ls_epoch = item.epoch_;
if (OB_FAIL(ss_write_tenant_super_block_(tenant_super_block))) {
LOG_WARN("fail to write tenant super block", K(ret), K(tenant_super_block));
lib::ObMutexGuard guard(super_block_lock_);
omt::ObTenant *tenant = static_cast<omt::ObTenant*>(MTL_CTX());
ObTenantSuperBlock tenant_super_block = tenant->get_super_block();
int64_t i = 0;
for (; i < tenant_super_block.ls_cnt_; i++) {
const ObLSItem &item = tenant_super_block.ls_item_arr_[i];
if (ls_id == item.ls_id_ && item.status_ != ObLSItemStatus::CREATE_ABORT &&
item.status_ != ObLSItemStatus::DELETED) {
break;
}
}
if (OB_UNLIKELY(i != tenant_super_block.ls_cnt_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls item already exist", K(ret), "ls_item", tenant_super_block.ls_item_arr_[i]);
} else if (OB_UNLIKELY(ObTenantSuperBlock::MAX_LS_COUNT == i)) {
ret = OB_SIZE_OVERFLOW;
LOG_WARN("too many ls", K(ret), K(ls_id), K(tenant_super_block));
} else {
tenant->set_tenant_super_block(tenant_super_block);
ObLSItem &item = tenant_super_block.ls_item_arr_[i];
tenant_super_block.ls_cnt_ = i + 1;
item.ls_id_ = ls_id;
item.min_macro_seq_ = inf_seq;
item.max_macro_seq_ = UINT64_MAX;
item.status_ = ObLSItemStatus::CREATING;
item.epoch_ = tenant_super_block.auto_inc_ls_epoch_++;
ls_epoch = item.epoch_;
if (!item.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected new ls_item", K(ret), K(item));
} else if (OB_FAIL(ss_write_tenant_super_block_(tenant_super_block))) {
LOG_WARN("fail to write tenant super block", K(ret), K(tenant_super_block));
} else {
tenant->set_tenant_super_block(tenant_super_block);
FLOG_INFO("create tenant ls item", K(ret), K(item), K(tenant_super_block), K(i));
}
}
}
FLOG_INFO("create tenant ls item", K(ret), K(ls_id));
if (OB_FAIL(ret)) {
FLOG_INFO("create tenant ls item failed", K(ret), K(ls_id));
}
return ret;
}
@ -1016,29 +1037,52 @@ int ObTenantStorageMetaPersister::update_tenant_ls_item_(
const ObLSID ls_id, const int64_t ls_epoch, const ObLSItemStatus status)
{
int ret = OB_SUCCESS;
lib::ObMutexGuard guard(super_block_lock_);
omt::ObTenant *tenant = static_cast<omt::ObTenant*>(MTL_CTX());
ObTenantSuperBlock tenant_super_block = tenant->get_super_block();
int64_t i = 0;
for (; i < tenant_super_block.ls_cnt_; i++) {
const ObLSItem &item = tenant_super_block.ls_item_arr_[i];
if (ls_id == item.ls_id_ && ls_epoch == item.epoch_) {
break;
uint64_t sup_seq = 0;
if (ObLSItemStatus::DELETED == status) {
// have to get macro_seq before get sputer_block_lock
// update preallocate.. need the lock either
if (OB_FAIL(TENANT_SEQ_GENERATOR.get_private_object_seq(sup_seq))) {
LOG_WARN("fail to get tenant_object_seq", K(ret));
}
}
if (OB_UNLIKELY(i == tenant_super_block.ls_cnt_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls not exist", K(ret), K(ls_id), K(ls_epoch), K(status));
if (OB_FAIL(ret)) {
} else {
const ObLSItem old_item = tenant_super_block.ls_item_arr_[i];
ObLSItem &new_item = tenant_super_block.ls_item_arr_[i];
new_item.status_ = status;
if (OB_FAIL(ss_write_tenant_super_block_(tenant_super_block))) {
LOG_WARN("fail to write tenant super block", K(ret), K(tenant_super_block));
} else {
tenant->set_tenant_super_block(tenant_super_block);
FLOG_INFO("update tenant super block ls item", K(ret), K(old_item), K(new_item));
lib::ObMutexGuard guard(super_block_lock_);
omt::ObTenant *tenant = static_cast<omt::ObTenant*>(MTL_CTX());
ObTenantSuperBlock tenant_super_block = tenant->get_super_block();
int64_t i = 0;
for (; i < tenant_super_block.ls_cnt_; i++) {
const ObLSItem &item = tenant_super_block.ls_item_arr_[i];
if (ls_id == item.ls_id_ && ls_epoch == item.epoch_) {
break;
}
}
if (OB_UNLIKELY(i == tenant_super_block.ls_cnt_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls not exist", K(ret), K(ls_id), K(ls_epoch), K(status));
} else {
const ObLSItem old_item = tenant_super_block.ls_item_arr_[i];
ObLSItem &new_item = tenant_super_block.ls_item_arr_[i];
new_item.status_ = status;
if (ObLSItemStatus::DELETED == status) {
// update the supremum seq of the deleted_ls_item
new_item.max_macro_seq_ = sup_seq;
}
if (!new_item.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected new ls_item", K(ret), K(new_item), K(old_item));
} else if (OB_FAIL(ss_write_tenant_super_block_(tenant_super_block))) {
LOG_WARN("fail to write tenant super block", K(ret), K(tenant_super_block));
} else {
tenant->set_tenant_super_block(tenant_super_block);
FLOG_INFO("update tenant super block ls item", K(ret), K(old_item), K(new_item), K(tenant_super_block), K(i));
}
}
}
if (OB_FAIL(ret)) {
FLOG_INFO("update tenant ls item failed", K(ret), K(ls_id), K(ls_epoch), K(status));
}
return ret;
}
@ -1077,7 +1121,7 @@ int ObTenantStorageMetaPersister::delete_tenant_ls_item_(
LOG_WARN("fail to write tenant super block", K(ret), K(ls_id), K(ls_epoch), K(tenant_super_block), K(tmp_super_block));
} else {
tenant->set_tenant_super_block(tmp_super_block);
FLOG_INFO("update tenant super block ls item", K(ret), K(ls_id), K(ls_epoch), K(tenant_super_block), K(tmp_super_block));
FLOG_INFO("update tenant super block ls item (delete)", K(ret), K(ls_id), K(ls_epoch), K(tenant_super_block), K(tmp_super_block));
}
} else {
ret = OB_ERR_UNEXPECTED;
@ -1222,8 +1266,9 @@ int ObTenantStorageMetaPersister::ss_batch_remove_ls_tablets(
common::ObTabletID tablet_id = tablet_id_arr.at(i);
ObMetaDiskAddr tablet_addr = tablet_addr_arr.at(i);
ObPendingFreeTabletItem tablet_item(tablet_id, tablet_addr.fifth_id(),
ObPendingFreeTabletStatus::WAIT_GC, INT64_MAX /* delete_time */ , GCTabletType::DropLS);
ObPendingFreeTabletItem tablet_item(tablet_id, tablet_addr.block_id().meta_version_id(),
ObPendingFreeTabletStatus::WAIT_GC, INT64_MAX /* delete_time */ , GCTabletType::DropLS,
tablet_addr.block_id().meta_transfer_seq());
if (has_exist_in_array(tmp_pending_free_array.items_, tablet_item)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet_item has existed in pending free tablet arr", K(ret), K(tmp_pending_free_array), K(tablet_item));

1
src/storage/meta_store/ob_tenant_storage_meta_persister.h
View File

@ -92,6 +92,7 @@ public:
const ObLSID &ls_id,
const uint64_t ls_epoch,
const int64_t deleted_tablet_version,
const int64_t deleted_tablet_transfer_seq,
ObArenaAllocator &allocator);
int ss_delete_tenant_ls_item(
const share::ObLSID ls_id, const int64_t ls_epoch)

8
src/storage/meta_store/ob_tenant_storage_meta_replayer.cpp
View File

@ -236,12 +236,15 @@ int ObTenantStorageMetaReplayer::s2_replay_ls_tablets_(
ls->get_ls_id(),
ls->get_ls_epoch(),
deleting_item.tablet_meta_version_,
deleting_item.tablet_transfer_seq_,
allocator))) {
LOG_WARN("failed to check and delete the current_version file of the tablet", K(ret), K(deleting_item), KPC(ls));
}
}
// 3. replay create tablets recorded in active_tablets (except had deleted in pending_free_arr);
// Even though in different transfer_seq, the tablet_version is unique.
// Therefore the compare is safe.
for (int64_t i = 0; OB_SUCC(ret) && i < active_tablets.items_.count(); i++) {
const ObActiveTabletItem &active_item = active_tablets.items_.at(i);
bool has_deleted = false;
@ -259,7 +262,7 @@ int ObTenantStorageMetaReplayer::s2_replay_ls_tablets_(
blocksstable::MacroBlockId object_id;
ObStorageObjectOpt opt;
const int64_t object_size = OB_DEFAULT_MACRO_BLOCK_SIZE;
opt.set_ss_private_tablet_meta_object_opt(ls_id.id(), active_item.tablet_id_.id(), ObStorageObjectOpt::INVALID_TABLET_VERSION);
opt.set_ss_private_tablet_meta_object_opt(ls_id.id(), active_item.tablet_id_.id(), ObStorageObjectOpt::INVALID_TABLET_VERSION, ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ);
if (OB_FAIL(OB_STORAGE_OBJECT_MGR.ss_get_object_id(opt, object_id))) {
LOG_WARN("fail to get object id", K(ret), K(opt));
} else if (OB_FAIL(inaccurate_addr.set_block_addr(object_id, 0/*offset*/, object_size, ObMetaDiskAddr::DiskType::RAW_BLOCK))) {
@ -325,6 +328,7 @@ int ObTenantStorageMetaReplayer::s2_replay_ls_tablets_for_trans_info_tmp_(
ls->get_ls_id(),
ls->get_ls_epoch(),
deleting_item.tablet_meta_version_,
deleting_item.tablet_transfer_seq_,
allocator))) {
LOG_WARN("failed to check and delete the current_version file of the tablet", K(ret), K(deleting_item), KPC(ls));
}
@ -354,7 +358,7 @@ int ObTenantStorageMetaReplayer::s2_replay_ls_tablets_for_trans_info_tmp_(
LOG_WARN("fail to read cur version", K(ret), K(item), K(current_version_opt));
}
} else if (ObStorageObjectOpt::INVALID_TABLET_VERSION != deleted_tablet_meta_version
&& latest_addr.tablet_addr_.fifth_id() <= deleted_tablet_meta_version) {
&& latest_addr.tablet_addr_.block_id().meta_version_id() <= deleted_tablet_meta_version) {
ret = OB_ERR_UNEXPECTED;
LOG_INFO("this tablet has been deleted, but current_version has not been deleted", K(ret), K(item), K(current_version_opt), K(latest_addr), K(deleted_tablet_meta_version));
} else if (OB_FAIL(ls_tablet_svr->s2_replay_create_tablet_for_trans_info_tmp(latest_addr.tablet_addr_, ls_handle, ObTabletID(tablet_id)))) {

35
src/storage/meta_store/ob_tenant_storage_meta_service.cpp
View File

@ -83,6 +83,7 @@ int ObTenantStorageMetaService::start()
int ret = OB_SUCCESS;
omt::ObTenant *tenant = static_cast<omt::ObTenant*>(share::ObTenantEnv::get_tenant());
const ObTenantSuperBlock super_block = tenant->get_super_block();
uint64_t macro_block_id = 0;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -97,6 +98,13 @@ int ObTenantStorageMetaService::start()
LOG_WARN("fail to seq generator", K(ret));
} else if (OB_FAIL(seq_generator_.start())) {
LOG_WARN("fail to seq generator", K(ret));
#ifdef OB_BUILD_SHARED_STORAGE
} else if (is_shared_storage_ && OB_FAIL(seq_generator_.get_private_object_seq(macro_block_id))) {
LOG_WARN("failed to get_private_object_seq", KR(ret));
} else if (is_shared_storage_) {
// for macro check in observer start
MTL(checkpoint::ObTabletGCService*)->set_mtl_start_max_block_id(macro_block_id);
#endif
}
FLOG_INFO("finish start ObTenantStorageMetaService", K(ret));
return ret;
@ -304,7 +312,7 @@ int ObTenantStorageMetaService::ObLSItemIterator::get_next_ls_item(
#ifdef OB_BUILD_SHARED_STORAGE
int ObTenantStorageMetaService::inner_get_blocks_for_tablet_(
const ObMetaDiskAddr &addr,
const ObMetaDiskAddr &tablet_addr,
const int64_t ls_epoch,
const bool is_shared,
ObIArray<blocksstable::MacroBlockId> &block_ids/*OUT*/) const
@ -317,14 +325,14 @@ int ObTenantStorageMetaService::inner_get_blocks_for_tablet_(
int64_t pos = 0;
ObSEArray<blocksstable::MacroBlockId, 100> tmp_print_arr;
if (OB_FAIL(MTL(ObTenantStorageMetaService*)->read_from_disk(addr, ls_epoch, allocator, buf, buf_len))) {
LOG_WARN("fail to read tablet buf from disk", K(ret), K(addr), K(ls_epoch));
} else if (FALSE_IT(tablet.set_tablet_addr(addr))) {
if (OB_FAIL(MTL(ObTenantStorageMetaService*)->read_from_disk(tablet_addr, ls_epoch, allocator, buf, buf_len))) {
LOG_WARN("fail to read tablet buf from disk", K(ret), K(tablet_addr), K(ls_epoch));
} else if (FALSE_IT(tablet.set_tablet_addr(tablet_addr))) {
} else if (OB_FAIL(tablet.deserialize_for_replay(allocator, buf, buf_len, pos))) {
LOG_WARN("fail to deserialize tablet", K(ret), KP(buf), K(buf_len));
} else if (!tablet.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("talbet is invalid", K(ret), K(tablet), K(addr));
LOG_WARN("talbet is invalid", K(ret), K(tablet), K(tablet_addr));
} else {
bool in_memory = true;
ObTabletMacroInfo *macro_info = nullptr;
@ -344,7 +352,7 @@ int ObTenantStorageMetaService::inner_get_blocks_for_tablet_(
} else {
ret = OB_SUCCESS;
if (0 != tmp_print_arr.count()) {
FLOG_INFO("iter get blocks", K(ret), K(ls_epoch), K(addr), K(is_shared), K(tmp_print_arr));
FLOG_INFO("iter get blocks", K(ret), K(ls_epoch), K(tablet_addr), K(is_shared), K(tmp_print_arr));
tmp_print_arr.reuse();
}
break;
@ -364,7 +372,7 @@ int ObTenantStorageMetaService::inner_get_blocks_for_tablet_(
} else if (OB_FAIL(tmp_print_arr.push_back(block_info.macro_id_))) {
LOG_WARN("fail to push shared macro id", K(ret), K(block_info));
} else if (30 == tmp_print_arr.count()) {
FLOG_INFO("iter get blocks", K(ret), K(ls_epoch), K(addr), K(is_shared), K(tmp_print_arr));
FLOG_INFO("iter get blocks", K(ret), K(ls_epoch), K(tablet_addr), K(is_shared), K(tmp_print_arr));
tmp_print_arr.reuse();
}
}
@ -381,21 +389,22 @@ int ObTenantStorageMetaService::get_private_blocks_for_tablet(
const int64_t ls_epoch,
const ObTabletID &tablet_id,
const int64_t tablet_version,
const int64_t tablet_transfer_seq,
ObIArray<blocksstable::MacroBlockId> &block_ids)
{
int ret = OB_SUCCESS;
ObMetaDiskAddr addr;
ObMetaDiskAddr tablet_addr;
blocksstable::MacroBlockId object_id;
blocksstable::ObStorageObjectOpt opt;
const int64_t object_size = OB_DEFAULT_MACRO_BLOCK_SIZE;
opt.set_ss_private_tablet_meta_object_opt(ls_id.id(), tablet_id.id(), tablet_version);
opt.set_ss_private_tablet_meta_object_opt(ls_id.id(), tablet_id.id(), tablet_version, tablet_transfer_seq);
if (OB_FAIL(OB_STORAGE_OBJECT_MGR.ss_get_object_id(opt, object_id))) {
LOG_WARN("fail to get object id", K(ret), K(opt));
} else if (OB_FAIL(addr.set_block_addr(object_id, 0/*offset*/, object_size, ObMetaDiskAddr::DiskType::RAW_BLOCK))) {
LOG_WARN("fail to set initial tablet meta addr", K(ret), K(addr));
} else if (OB_FAIL(inner_get_blocks_for_tablet_(addr, ls_epoch, false/*is_shared*/, block_ids))) {
LOG_WARN("fail to deserialize tablet", K(ret), K(addr), K(ls_epoch), K(block_ids));
} else if (OB_FAIL(tablet_addr.set_block_addr(object_id, 0/*offset*/, object_size, ObMetaDiskAddr::DiskType::RAW_BLOCK))) {
LOG_WARN("fail to set initial tablet meta addr", K(ret), K(tablet_addr));
} else if (OB_FAIL(inner_get_blocks_for_tablet_(tablet_addr, ls_epoch, false/*is_shared*/, block_ids))) {
LOG_WARN("fail to deserialize tablet", K(ret), K(tablet_addr), K(ls_epoch), K(block_ids));
}
return ret;
}

3
src/storage/meta_store/ob_tenant_storage_meta_service.h
View File

@ -71,6 +71,7 @@ public:
const int64_t ls_epoch,
const ObTabletID &tablet_id,
const int64_t tablet_version,
const int64_t tablet_transfer_seq,
ObIArray<blocksstable::MacroBlockId> &block_ids);
int get_shared_blocks_for_tablet(
const ObTabletID &tablet_id,
@ -119,7 +120,7 @@ public:
private:
#ifdef OB_BUILD_SHARED_STORAGE
int inner_get_blocks_for_tablet_(
const ObMetaDiskAddr &addr,
const ObMetaDiskAddr &tablet_addr,
const int64_t ls_epoch,
const bool is_shared,
ObIArray<blocksstable::MacroBlockId> &block_ids/*OUT*/) const;

1
src/storage/multi_data_source/ob_mds_table_merge_task.cpp
View File

@ -138,6 +138,7 @@ int ObMdsTableMergeTask::process()
ctx.time_guard_click(ObStorageCompactionTimeGuard::EXECUTE);
share::dag_yield();
} else if (FALSE_IT(ctx.static_param_.scn_range_.start_scn_ = tablet->get_mds_checkpoint_scn())) {
} else if (FALSE_IT(ctx.static_desc_.tablet_transfer_seq_ = tablet->get_transfer_seq())) {
} else if (MDS_FAIL(build_mds_sstable(ctx, mds_construct_sequence, table_handle))) {
LOG_WARN("fail to build mds sstable", K(ret), K(ls_id), K(tablet_id), KPC(mds_merge_dag_));
} else if (MDS_FAIL(ls->build_new_tablet_from_mds_table(

1
src/storage/multi_data_source/ob_tablet_mds_merge_ctx.cpp
View File

@ -158,6 +158,7 @@ int ObTabletCrossLSMdsMinorMergeCtx::get_merge_tables(ObGetMergeTablesResult &ge
get_merge_table_result.scn_range_.start_scn_ = static_param_.tables_handle_.get_table(0)->get_start_scn();
get_merge_table_result.scn_range_.end_scn_ = static_param_.tables_handle_.get_table(static_param_.tables_handle_.get_count() - 1)->get_end_scn();
get_merge_table_result.version_range_.snapshot_version_ = tablet->get_snapshot_version();
get_merge_table_result.transfer_seq_ = tablet->get_transfer_seq();
}
return ret;
}

17
src/storage/ob_storage_struct.cpp
View File

@ -254,16 +254,21 @@ ObGetMergeTablesResult::ObGetMergeTablesResult()
error_location_(nullptr),
snapshot_info_(),
is_backfill_(false),
backfill_scn_()
backfill_scn_(),
transfer_seq_(ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ)
{
}
bool ObGetMergeTablesResult::is_valid() const
{
return scn_range_.is_valid()
&& (is_simplified_ || handle_.get_count() >= 1)
&& merge_version_ >= 0
&& (!is_backfill_ || backfill_scn_.is_valid());
bool valid = scn_range_.is_valid()
&& (is_simplified_ || handle_.get_count() >= 1)
&& merge_version_ >= 0
&& (!is_backfill_ || backfill_scn_.is_valid());
if (valid && GCTX.is_shared_storage_mode()) {
valid &= (ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ != transfer_seq_);
}
return valid;
}
void ObGetMergeTablesResult::reset_handle_and_range()
@ -291,6 +296,7 @@ void ObGetMergeTablesResult::reset()
snapshot_info_.reset();
is_backfill_ = false;
backfill_scn_.reset();
transfer_seq_ = ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ;
}
int ObGetMergeTablesResult::copy_basic_info(const ObGetMergeTablesResult &src)
@ -309,6 +315,7 @@ int ObGetMergeTablesResult::copy_basic_info(const ObGetMergeTablesResult &src)
is_backfill_ = src.is_backfill_;
backfill_scn_ = src.backfill_scn_;
snapshot_info_ = src.snapshot_info_;
transfer_seq_ = src.transfer_seq_;
}
return ret;
}

3
src/storage/ob_storage_struct.h
View File

@ -313,6 +313,7 @@ struct ObGetMergeTablesResult
//for backfill
bool is_backfill_;
share::SCN backfill_scn_;
int64_t transfer_seq_; // is_used for write_macro_block in ss, used for all compaction.
ObGetMergeTablesResult();
bool is_valid() const;
void reset_handle_and_range();
@ -322,7 +323,7 @@ struct ObGetMergeTablesResult
int copy_basic_info(const ObGetMergeTablesResult &src);
share::SCN get_merge_scn() const;
TO_STRING_KV(K_(version_range), K_(scn_range), K_(merge_version), K_(is_simplified),
K_(handle), K_(update_tablet_directly), K_(schedule_major), K_(is_backfill), K_(backfill_scn));
K_(handle), K_(update_tablet_directly), K_(schedule_major), K_(is_backfill), K_(backfill_scn), K_(transfer_seq));
};
OB_INLINE bool is_valid_migrate_status(const ObMigrateStatus &status)

12
src/storage/ob_super_block_struct.h
View File

@ -207,7 +207,7 @@ public:
bool is_valid() const
{
return ls_id_.is_valid() && epoch_ >= 0 && ObLSItemStatus::MAX != status_;
return ls_id_.is_valid() && epoch_ >= 0 && ObLSItemStatus::MAX != status_ && min_macro_seq_ < max_macro_seq_;
}
TO_STRING_KV(K_(ls_id), K_(epoch), K_(status), K_(min_macro_seq), K_(max_macro_seq));
@ -341,17 +341,19 @@ public:
const common::ObTabletID tablet_id,
const int64_t tablet_meta_version,
const ObPendingFreeTabletStatus status,
int64_t free_time,
GCTabletType gc_type)
const int64_t free_time,
const GCTabletType gc_type,
const int64_t tablet_transfer_seq)
: tablet_id_(tablet_id), tablet_meta_version_(tablet_meta_version),
status_(status), free_time_(free_time),
gc_type_(gc_type), tablet_transfer_seq_(share::OB_INVALID_TRANSFER_SEQ)
gc_type_(gc_type), tablet_transfer_seq_(tablet_transfer_seq)
{}
bool is_valid() const
{
return tablet_id_.is_valid() && tablet_meta_version_ > 0 &&
ObPendingFreeTabletStatus::MAX != status_;
ObPendingFreeTabletStatus::MAX != status_ &&
tablet_transfer_seq_ != share::OB_INVALID_TRANSFER_SEQ;
}
bool operator == (const ObPendingFreeTabletItem &other) const {
return tablet_id_ == other.tablet_id_ &&

9
src/storage/slog/ob_storage_logger_manager.cpp
View File

@ -314,8 +314,13 @@ int ObStorageLoggerManager::get_tenant_slog_dir(
{
int ret = OB_SUCCESS;
int pret = 0;
pret = snprintf(tenant_slog_dir, MAX_PATH_SIZE, "%s/tenant_%" PRIu64,
log_dir_, tenant_id);
if (is_server_tenant(tenant_id)) {
pret = snprintf(tenant_slog_dir, MAX_PATH_SIZE, "%s/server",
log_dir_);
} else {
pret = snprintf(tenant_slog_dir, MAX_PATH_SIZE, "%s/tenant_%" PRIu64,
log_dir_, tenant_id);
}
if (pret < 0 || pret >= MAX_PATH_SIZE) {
ret = OB_BUF_NOT_ENOUGH;
STORAGE_REDO_LOG(ERROR, "construct tenant slog path fail", K(ret), K(tenant_id));

16
src/storage/slog_ckpt/ob_linked_macro_block_writer.cpp
View File

@ -29,18 +29,17 @@ using namespace oceanbase::blocksstable;
ObLinkedMacroBlockWriter::ObLinkedMacroBlockWriter()
: is_inited_(false), write_ctx_(), handle_(), entry_block_id_(),
tablet_id_(0), snapshot_version_(0), cur_macro_seq_(-1)
tablet_id_(0), tablet_transfer_seq_(0), snapshot_version_(0), cur_macro_seq_(-1)
{
}
int ObLinkedMacroBlockWriter::init(const uint64_t tablet_id)
int ObLinkedMacroBlockWriter::init()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("ObLinkedMacroBlockWriter has not been inited", K(ret));
} else {
tablet_id_ = tablet_id;
snapshot_version_ = 0;
cur_macro_seq_ = -1;
is_inited_ = true;
@ -50,6 +49,7 @@ int ObLinkedMacroBlockWriter::init(const uint64_t tablet_id)
int ObLinkedMacroBlockWriter::init_for_object(
const uint64_t tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
const int64_t start_macro_seq)
{
@ -62,6 +62,7 @@ int ObLinkedMacroBlockWriter::init_for_object(
LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(snapshot_version), K(start_macro_seq));
} else {
tablet_id_ = tablet_id;
tablet_transfer_seq_ = tablet_transfer_seq;
snapshot_version_ = snapshot_version;
cur_macro_seq_ = start_macro_seq;
is_inited_ = true;
@ -90,7 +91,7 @@ int ObLinkedMacroBlockWriter::write_block(
if (snapshot_version_ > 0) {
opt.set_ss_share_meta_macro_object_opt(tablet_id_, cur_macro_seq_++, 0);
} else {
opt.set_private_meta_macro_object_opt(tablet_id_);
opt.set_private_meta_macro_object_opt(tablet_id_, tablet_transfer_seq_);
}
const uint64_t tenant_id = MTL_ID();
ObStorageObjectWriteInfo write_info;
@ -223,14 +224,14 @@ ObLinkedMacroBlockItemWriter::ObLinkedMacroBlockItemWriter()
{
}
int ObLinkedMacroBlockItemWriter::init(const bool need_disk_addr, const ObMemAttr &mem_attr, const uint64_t tablet_id /*0 default*/)
int ObLinkedMacroBlockItemWriter::init(const bool need_disk_addr, const ObMemAttr &mem_attr)
{
int ret = OB_SUCCESS;
const int64_t macro_block_size = OB_STORAGE_OBJECT_MGR.get_macro_block_size();
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("ObLinkedMacroBlockItemWriter has already been inited", K(ret));
} else if (OB_FAIL(block_writer_.init(tablet_id))) {
} else if (OB_FAIL(block_writer_.init())) {
LOG_WARN("fail to init meta block writer", K(ret));
} else if (OB_ISNULL(io_buf_ = static_cast<char *>(allocator_.alloc(macro_block_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -252,6 +253,7 @@ int ObLinkedMacroBlockItemWriter::init(const bool need_disk_addr, const ObMemAtt
int ObLinkedMacroBlockItemWriter::init_for_object(
const uint64_t tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
const int64_t start_macro_seq,
ObIMacroBlockFlushCallback *write_callback)
@ -266,7 +268,7 @@ int ObLinkedMacroBlockItemWriter::init_for_object(
} else if (OB_UNLIKELY(0 == tablet_id || (snapshot_version > 0 && start_macro_seq < 0))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(snapshot_version), K(start_macro_seq));
} else if (OB_FAIL(block_writer_.init_for_object(tablet_id, snapshot_version, start_macro_seq))) {
} else if (OB_FAIL(block_writer_.init_for_object(tablet_id, tablet_transfer_seq, snapshot_version, start_macro_seq))) {
LOG_WARN("fail to init meta block writer", K(ret));
} else if (OB_ISNULL(io_buf_ = static_cast<char *>(allocator_.alloc(macro_block_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;

10
src/storage/slog_ckpt/ob_linked_macro_block_writer.h
View File

@ -34,9 +34,10 @@ public:
ObLinkedMacroBlockWriter(const ObLinkedMacroBlockWriter &) = delete;
ObLinkedMacroBlockWriter &operator=(const ObLinkedMacroBlockWriter &) = delete;
int init(const uint64_t talbet_id);
int init();
int init_for_object(
const uint64_t tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
const int64_t start_macro_seq);
int write_block(
@ -59,6 +60,7 @@ private:
blocksstable::ObStorageObjectHandle handle_;
blocksstable::MacroBlockId entry_block_id_;
uint64_t tablet_id_;
int64_t tablet_transfer_seq_;
int64_t snapshot_version_;
int64_t cur_macro_seq_;
};
@ -70,13 +72,15 @@ public:
~ObLinkedMacroBlockItemWriter() = default;
ObLinkedMacroBlockItemWriter(const ObLinkedMacroBlockItemWriter &) = delete;
ObLinkedMacroBlockItemWriter &operator=(const ObLinkedMacroBlockItemWriter &) = delete;
int init(const bool need_disk_addr, const ObMemAttr &mem_attr, const uint64_t tablet_id = 0);
// used for writing macro_info both in shared_nothing and shared_storage
int init_for_object(
const uint64_t tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
const int64_t start_macro_seq,
blocksstable::ObIMacroBlockFlushCallback *write_callback = nullptr);
// only used for ckpt_slog in shared_nothing
int init(const bool need_disk_addr, const ObMemAttr &mem_attr);
int write_item(const char *item_buf, const int64_t item_buf_len, int64_t *item_idx = nullptr);
int close();
inline bool is_closed() const { return is_closed_; };

6
src/storage/slog_ckpt/ob_tenant_storage_checkpoint_writer.cpp
View File

@ -381,7 +381,8 @@ int ObTenantStorageCheckpointWriter::persist_and_copy_tablet(
slog.ls_id_ = tablet_key.ls_id_;
slog.tablet_id_ = tablet_key.tablet_id_;
bool has_slog = false;
const ObTabletPersisterParam param(tablet_key.ls_id_, 0, tablet_key.tablet_id_);
int64_t transfer_seq = 0; // useless in shared_nothing
const ObTabletPersisterParam param(tablet_key.ls_id_, 0, tablet_key.tablet_id_, transfer_seq);
if (OB_FAIL(OB_E(EventTable::EN_SLOG_CKPT_ERROR) OB_SUCCESS)) {
} else if (OB_FAIL(ckpt_slog_handler_->check_slog(tablet_key, has_slog))) {
@ -456,7 +457,8 @@ int ObTenantStorageCheckpointWriter::copy_tablet(
slog.ls_id_ = tablet_key.ls_id_;
slog.tablet_id_ = tablet_key.tablet_id_;
ObMetaDiskAddr old_addr;
const ObTabletPersisterParam param(tablet_key.ls_id_, 0, tablet_key.tablet_id_);
int64_t transfer_seq = 0; // useless in shared_nothing
const ObTabletPersisterParam param(tablet_key.ls_id_, 0, tablet_key.tablet_id_, transfer_seq);
if (OB_FAIL(MTL(ObTenantMetaMemMgr*)->get_tablet_with_allocator(WashTabletPriority::WTP_LOW, tablet_key, allocator, tablet_handle))) {
if (OB_ENTRY_NOT_EXIST == ret) {

1
src/storage/tablet/ob_tablet.cpp
View File

@ -2129,7 +2129,6 @@ int ObTablet::load_deserialize(
int ObTablet::deserialize_post_work(common::ObArenaAllocator &allocator)
{
int ret = OB_SUCCESS;
ObLinkedMacroBlockItemWriter linked_writer; // it needs to hold ref cnt until inner_inc_macro_ref_cnt is called
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("cannot deserialize inited tablet meta", K(ret), K_(is_inited));

2
src/storage/tablet/ob_tablet.h
View File

@ -909,7 +909,7 @@ private:
int64_t memtable_count_; // size: 8B, alignment: 8B
ObDDLKV **ddl_kvs_; // size: 8B, alignment: 8B
int64_t ddl_kv_count_; // size: 8B, alignment: 8B
ObTabletPointerHandle pointer_hdl_; // size: 24B, alignment: 8B
ObTabletPointerHandle pointer_hdl_; // size: 24B, alignment: 8B
ObMetaDiskAddr tablet_addr_; // size: 48B, alignment: 8B 1080
// NOTICE: these two pointers: memtable_mgr_ and log_handler_,
// are considered as cache for tablet.

4
src/storage/tablet/ob_tablet_mds_table_mini_merger.cpp
View File

@ -383,7 +383,7 @@ int ObMdsTableMiniMerger::init(compaction::ObTabletMergeCtx &ctx, ObMdsMiniMerge
LOG_WARN("mds storage schema is invalid", K(ret), KP(storage_schema), KPC(storage_schema));
} else if (OB_FAIL(data_desc_.init(false/*is ddl*/, *storage_schema, ls_id, tablet_id,
ctx.get_merge_type(), ctx.get_snapshot(), data_version, ctx.static_desc_.micro_index_clustered_,
ctx.static_param_.scn_range_.end_scn_))) {
ctx.static_desc_.tablet_transfer_seq_, ctx.static_param_.scn_range_.end_scn_))) {
LOG_WARN("fail to init whole desc", KR(ret), K(ctx), K(ls_id), K(tablet_id));
} else if (OB_FAIL(macro_start_seq.set_parallel_degree(0))) {
LOG_WARN("Failed to set parallel degree to macro start seq", K(ret));
@ -481,6 +481,7 @@ int ObMdsDataCompatHelper::generate_mds_mini_sstable(
ctx->static_param_.scn_range_.end_scn_ = mig_param.mds_checkpoint_scn_;
ctx->static_param_.version_range_.snapshot_version_ = mig_param.mds_checkpoint_scn_.get_val_for_tx();
ctx->static_param_.pre_warm_param_.type_ = ObPreWarmerType::MEM_PRE_WARM;
ctx->static_desc_.tablet_transfer_seq_ = mig_param.transfer_info_.transfer_seq_;
}
if (OB_FAIL(ret)) {
@ -535,6 +536,7 @@ int ObMdsDataCompatHelper::generate_mds_mini_sstable(
ctx->static_param_.scn_range_.end_scn_ = tablet.get_mds_checkpoint_scn();
ctx->static_param_.version_range_.snapshot_version_ = tablet.get_mds_checkpoint_scn().get_val_for_tx();
ctx->static_param_.pre_warm_param_.type_ = ObPreWarmerType::MEM_PRE_WARM;
ctx->static_desc_.tablet_transfer_seq_ = tablet.get_transfer_seq();
if (CLICK_FAIL(tablet.build_full_memory_mds_data(allocator, data))) {
LOG_WARN("fail to build full memory mds data", K(ret));

11
src/storage/tablet/ob_tablet_persister.cpp
View File

@ -300,7 +300,7 @@ void ObTabletPersister::print_time_stats(
LOG_WARN("fail to get initial tablet meta version", K(ret), K(ls_id), K(tablet_id));
}
} else if (old_tablet_addr.is_block()) {
meta_version = old_tablet_addr.block_id().fourth_id() + 1;
meta_version = old_tablet_addr.block_id().meta_version_id() + 1;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected old tablet addr", K(ret), K(tablet_id), K(old_tablet_addr));
@ -310,14 +310,14 @@ void ObTabletPersister::print_time_stats(
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected tablet meta version", K(ret), K(meta_version));
} else {
opt.set_ss_private_tablet_meta_object_opt(ls_id.id(), tablet_id.id(), meta_version);
opt.set_ss_private_tablet_meta_object_opt(ls_id.id(), tablet_id.id(), meta_version, persist_param.tablet_transfer_seq_);
}
} else {
opt.set_ss_share_tablet_meta_object_opt(persist_param.tablet_id_.id(), persist_param.snapshot_version_);
}
#endif
} else {
opt.set_private_meta_macro_object_opt(persist_param.tablet_id_.id());
opt.set_private_meta_macro_object_opt(persist_param.tablet_id_.id(), persist_param.tablet_transfer_seq_);
}
return ret;
}
@ -695,7 +695,7 @@ int ObTabletPersister::persist_and_fill_tablet(
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(linked_writer.init_for_object(param_.tablet_id_.id(), param_.snapshot_version_,
} else if (OB_FAIL(linked_writer.init_for_object(param_.tablet_id_.id(), param_.tablet_transfer_seq_, param_.snapshot_version_,
cur_macro_seq_, param_.ddl_redo_callback_))) {
LOG_WARN("fail to init linked writer", K(ret), K(old_tablet));
} else if (OB_FAIL(tablet_macro_info.init(allocator_, block_info_set, &linked_writer))) {
@ -1184,7 +1184,7 @@ int ObTabletPersister::transform(const ObTabletTransformArg &arg, char *buf, con
void ObTabletPersister::build_async_write_start_opt_(blocksstable::ObStorageObjectOpt &start_opt) const
{
if (!param_.is_shared_object()) {
start_opt.set_private_meta_macro_object_opt(param_.tablet_id_.id());
start_opt.set_private_meta_macro_object_opt(param_.tablet_id_.id(), param_.tablet_transfer_seq_);
} else {
start_opt.set_ss_share_meta_macro_object_opt(
param_.tablet_id_.id(), cur_macro_seq_, 0/*cg_id*/);
@ -1396,6 +1396,7 @@ int ObTabletPersister::persist_sstable_linked_block_if_need(
if (OB_FAIL(sstable->persist_linked_block_if_need(
allocator,
param_.tablet_id_,
param_.tablet_transfer_seq_,
param_.snapshot_version_,
param_.ddl_redo_callback_,
macro_start_seq,

11
src/storage/tablet/ob_tablet_persister.h
View File

@ -170,8 +170,9 @@ public:
ObTabletPersisterParam(
const share::ObLSID ls_id,
const int64_t ls_epoch,
const ObTabletID tablet_id)
: ls_id_(ls_id), ls_epoch_(ls_epoch), tablet_id_(tablet_id),
const ObTabletID tablet_id,
const int64_t tablet_transfer_seq)
: ls_id_(ls_id), ls_epoch_(ls_epoch), tablet_id_(tablet_id), tablet_transfer_seq_(tablet_transfer_seq),
snapshot_version_(0), start_macro_seq_(0),
ddl_redo_callback_(nullptr), ddl_finish_callback_(nullptr)
{}
@ -179,11 +180,12 @@ public:
// shared tablet meta persistence
ObTabletPersisterParam(
const ObTabletID tablet_id,
const int64_t tablet_transfer_seq,
const int64_t snapshot_version,
const int64_t start_macro_seq,
blocksstable::ObIMacroBlockFlushCallback *ddl_redo_callback = nullptr,
blocksstable::ObIMacroBlockFlushCallback *ddl_finish_callback = nullptr)
: ls_id_(), ls_epoch_(0), tablet_id_(tablet_id),
: ls_id_(), ls_epoch_(0), tablet_id_(tablet_id), tablet_transfer_seq_(tablet_transfer_seq),
snapshot_version_(snapshot_version), start_macro_seq_(start_macro_seq),
ddl_redo_callback_(ddl_redo_callback), ddl_finish_callback_(ddl_finish_callback)
{}
@ -193,12 +195,13 @@ public:
bool is_shared_object() const { return snapshot_version_ > 0; }
TO_STRING_KV(K_(ls_id), K_(ls_epoch), K_(tablet_id),
TO_STRING_KV(K_(ls_id), K_(ls_epoch), K_(tablet_id), K_(tablet_transfer_seq),
K_(snapshot_version), K_(start_macro_seq), KP_(ddl_redo_callback), KP_(ddl_finish_callback));
share::ObLSID ls_id_;
int64_t ls_epoch_;
ObTabletID tablet_id_;
int64_t tablet_transfer_seq_;
int64_t snapshot_version_;
int64_t start_macro_seq_;
blocksstable::ObIMacroBlockFlushCallback *ddl_redo_callback_;

3
src/storage/tx_storage/ob_empty_shell_task.cpp
View File

@ -129,6 +129,7 @@ int ObTabletEmptyShellHandler::get_empty_shell_tablet_ids(common::ObTabletIDArra
int64_t tablet_ref_cnt = 0;
bool allow_tablet_version_gc = false;
int64_t current_tablet_version = OB_INVALID_VERSION;
int64_t curr_transfer_seq = OB_INVALID_TRANSFER_SEQ; // unused
ObLSTabletIterator tablet_iter(ObMDSGetTabletMode::READ_WITHOUT_CHECK);
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -176,7 +177,7 @@ int ObTabletEmptyShellHandler::get_empty_shell_tablet_ids(common::ObTabletIDArra
} else if (!can_become_shell) {
STORAGE_LOG(INFO, "tablet can not become shell", KR(ret), "tablet_meta", tablet->get_tablet_meta());
} else if (OB_FAIL(t3m->get_current_version_for_tablet(ls_->get_ls_id(), tablet->get_tablet_id(), current_tablet_version,
allow_tablet_version_gc))) {
curr_transfer_seq, allow_tablet_version_gc))) {
STORAGE_LOG(WARN, "failed to check tablet ref status", KR(ret), K(ls_->get_ls_id()), K(tablet->get_tablet_id()));
} else if (FALSE_IT(tablet_ref_cnt = tablet->get_ref())) {
} else if (GCTX.is_shared_storage_mode() && (!allow_tablet_version_gc || (2 < tablet_ref_cnt))) {

2
src/storage/tx_storage/ob_ls_service.h
View File

@ -172,7 +172,7 @@ public:
if (OB_FAIL(get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) {
STORAGE_LOG(WARN, "failed to get ls", K(ret), K(ls_id), K(tablet_id));
} else {
ls_handle.get_ls()->get_ls_private_block_gc_handler().report_tablet_id_for_gc(tablet_info);
ls_handle.get_ls()->get_ls_private_block_gc_handler().report_tablet_id_for_gc_service(tablet_info);
}
}
#endif

2
src/storage/tx_storage/ob_tablet_gc_service.cpp
View File

@ -59,7 +59,7 @@ int ObTabletGCService::mtl_init(ObTabletGCService* &m)
int ObTabletGCService::init()
{
int ret = OB_SUCCESS;
bool is_shared_storage = GCTX.is_shared_storage_mode();
const bool is_shared_storage = GCTX.is_shared_storage_mode();
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObTabletGCService init twice.", KR(ret));

6
src/storage/tx_storage/ob_tablet_gc_service.h
View File

@ -165,6 +165,12 @@ public:
}
ObPrivateBlockGCThread* get_private_block_gc_thread()
{ return &private_block_gc_thread_; }
void set_mtl_start_max_block_id(const uint64_t mtl_start_max_block_id)
{ private_block_gc_task_.set_mtl_start_max_block_id(mtl_start_max_block_id); }
void set_observer_start_macro_block_id_trigger()
{ private_block_gc_task_.set_observer_start_macro_block_id_trigger(); }
uint64_t get_mtl_start_max_block_id()
{ return private_block_gc_task_.get_mtl_start_max_block_id(); }
#endif
private:

3
tools/ob_admin/slog_tool/ob_admin_slog_executor.cpp
View File

@ -217,7 +217,8 @@ void ObAdminSlogExecutor::print_usage()
" -f --log_file_id specify the slog file id which is to be dumped\n"
" -o --offset specify the offset from which to start scanning / default value is 0, meaning reading from the begining of the file\n"
" -c --parse_count specify the count of slogs needed to be read / default value is -1, meaning reading all slogs left\n"
"eg. 1.ob_admin slog_tool -d /home/fenggu.yh/ob1.obs0/store -u 1 -f 1 \n\n");
"eg. 1.ob_admin slog_tool -d/home/fenggu.yh/ob1.obs0/store -u1 -f1\n"
" (NOTICE: no space between -d/home...)\n\n");
}
ObLogDirEntryOperation::~ObLogDirEntryOperation()

8
unittest/share/scheduler/test_dag_net_in_dag_scheduler.cpp
View File

@ -1721,7 +1721,7 @@ TEST_F(TestDagScheduler, test_add_multi_co_merge_dag_net)
param.tablet_id_ = tablet_id;
param.merge_type_ = compaction::ObMergeType::CONVERT_CO_MAJOR_MERGE;
param.compat_mode_ = lib::Worker::CompatMode::MYSQL;
param.transfer_seq_ = 0;
param.schedule_transfer_seq_ = 0;
ret = scheduler->create_and_add_dag_net<compaction::ObCOMergeDagNet>(&param);
EXPECT_EQ(OB_SUCCESS, ret);
COMMON_LOG(INFO, "Success to create and add co convert dag net", K(ret));
@ -1732,7 +1732,7 @@ TEST_F(TestDagScheduler, test_add_multi_co_merge_dag_net)
param.tablet_id_ = tablet_id;
param.merge_type_ = compaction::ObMergeType::CONVERT_CO_MAJOR_MERGE;
param.compat_mode_ = lib::Worker::CompatMode::MYSQL;
param.transfer_seq_ = 0;
param.schedule_transfer_seq_ = 0;
ret = scheduler->create_and_add_dag_net<compaction::ObCOMergeDagNet>(&param);
EXPECT_EQ(OB_TASK_EXIST, ret);
COMMON_LOG(INFO, "Success to create and add co convert dag net", K(ret));
@ -1743,7 +1743,7 @@ TEST_F(TestDagScheduler, test_add_multi_co_merge_dag_net)
param.tablet_id_ = tablet_id;
param.merge_type_ = compaction::ObMergeType::MAJOR_MERGE;
param.compat_mode_ = lib::Worker::CompatMode::MYSQL;
param.transfer_seq_ = 0;
param.schedule_transfer_seq_ = 0;
ret = scheduler->create_and_add_dag_net<compaction::ObCOMergeDagNet>(&param);
EXPECT_EQ(OB_SUCCESS, ret);
COMMON_LOG(INFO, "Success to create and add co major dag net", K(ret));
@ -1753,7 +1753,7 @@ TEST_F(TestDagScheduler, test_add_multi_co_merge_dag_net)
param.tablet_id_ = tablet_id;
param.merge_type_ = compaction::ObMergeType::MEDIUM_MERGE;
param.compat_mode_ = lib::Worker::CompatMode::MYSQL;
param.transfer_seq_ = 0;
param.schedule_transfer_seq_ = 0;
ret = scheduler->create_and_add_dag_net<compaction::ObCOMergeDagNet>(&param);
EXPECT_EQ(OB_SUCCESS, ret);
COMMON_LOG(INFO, "Success to create and add co medium dag net", K(ret));

2
unittest/storage/blocksstable/ob_multi_version_sstable_test.h
View File

@ -425,7 +425,7 @@ void ObMultiVersionSSTableTest::reset_writer(
SCN scn;
scn.convert_for_tx(snapshot_version);
ASSERT_EQ(OB_SUCCESS, data_desc_.init(false/*is_ddl*/, table_schema_, ls_id, tablet_id, merge_type, snapshot_version, DATA_VERSION_4_1_0_0,
table_schema_.get_micro_index_clustered(), scn));
table_schema_.get_micro_index_clustered(), 0 /*transfer_seq*/, scn));
void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder(false /* not need writer buffer*/);
ASSERT_NE(nullptr, root_index_builder_);

16
unittest/storage/blocksstable/test_data_store_desc.cpp
View File

@ -71,11 +71,11 @@ TEST_F(TestObDataStoreDesc, test_static_desc)
share::SCN scn;
scn.convert_for_tx(100);
ASSERT_EQ(OB_INVALID_ARGUMENT,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, 0 /*transfer_seq*/,
MINI_MERGE, snapshot, share::SCN::invalid_scn(),
1 /*cluster_version*/, EXEC_MODE_LOCAL, false /* micro_index_clustered */));
ASSERT_EQ(OB_SUCCESS,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, 0 /*transfer_seq*/,
MINI_MERGE, snapshot, scn, 1 /*cluster_version*/,
EXEC_MODE_LOCAL, false /* micro_index_clustered */));
ASSERT_TRUE(static_desc.is_valid());
@ -94,7 +94,7 @@ TEST_F(TestObDataStoreDesc, test_static_desc)
ObStaticDataStoreDesc static_desc2;
ASSERT_EQ(OB_SUCCESS,
static_desc2.init(true/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_,
static_desc2.init(true/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, 0 /*transfer_seq*/,
MAJOR_MERGE, snapshot, scn, DATA_VERSION_4_2_0_0, EXEC_MODE_LOCAL, false /* micro_index_clustered */));
ASSERT_TRUE(static_desc2.is_valid());
@ -154,17 +154,18 @@ TEST_F(TestObDataStoreDesc, test_whole_data_desc)
whole_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_,
MAJOR_MERGE, snapshot, DATA_VERSION_4_2_0_0,
table_schema.get_micro_index_clustered(),
0 /*transfer_seq*/,
share::SCN::invalid_scn()));
ASSERT_TRUE(whole_desc.is_valid());
// point to other static desc member
ObStaticDataStoreDesc static_desc;
ASSERT_EQ(OB_INVALID_ARGUMENT,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, 0 /*transfer_seq*/,
MINI_MERGE, snapshot,
share::SCN::invalid_scn(), 0/*cluster_version*/, EXEC_MODE_LOCAL, false /* micro_index_clustered */));
ASSERT_EQ(OB_SUCCESS,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, 0 /*transfer_seq*/,
MAJOR_MERGE, snapshot,
share::SCN::invalid_scn(), DATA_VERSION_4_2_0_0, EXEC_MODE_LOCAL, false /* micro_index_clustered */));
whole_desc.desc_.static_desc_ = &static_desc;
@ -184,7 +185,8 @@ TEST_F(TestObDataStoreDesc, gen_index_desc)
ASSERT_EQ(OB_SUCCESS,
data_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_,
MAJOR_MERGE, snapshot, 1/*clsuter_version*/,
table_schema.get_micro_index_clustered()));
table_schema.get_micro_index_clustered(),
0 /*transfer_seq*/));
ASSERT_TRUE(data_desc.is_valid());
const ObDataStoreDesc &data_store_desc = data_desc.get_desc();
@ -215,7 +217,7 @@ TEST_F(TestObDataStoreDesc, test_cg)
share::SCN scn;
scn.convert_for_tx(100);
ASSERT_EQ(OB_SUCCESS,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_,
static_desc.init(false/*is_ddl*/, table_schema, mock_ls_id_, mock_tablet_id_, 0 /*transfer_seq*/,
MAJOR_MERGE, snapshot, share::SCN::invalid_scn(),
DATA_VERSION_4_3_2_0/*cluster_version*/, EXEC_MODE_LOCAL, false /* micro_index_clustered */));
ASSERT_TRUE(static_desc.is_valid());

72
unittest/storage/blocksstable/test_macro_block_id.cpp
View File

@ -15,6 +15,7 @@
#define protected public
#define private public
#include "storage/blocksstable/ob_macro_block_id.h"
#include "storage/blocksstable/ob_object_manager.h"
namespace oceanbase
@ -96,6 +97,77 @@ TEST_F(TestMacroBlockId, verification)
ASSERT_EQ(OB_DESERIALIZE_ERROR, ret);
ASSERT_EQ(0, pos);
}
TEST_F(TestMacroBlockId, test_transfer_seq)
{
int ret = OB_SUCCESS;
blocksstable::ObStorageObjectOpt curr_opt;
MacroBlockId test_block_id;
uint64_t test_ls_id = 1001;
uint64_t test_tablet_id = 200001;
uint64_t test_tablet_version = ObStorageObjectOpt::INVALID_TABLET_VERSION;
int64_t test_transfer_seq = ObStorageObjectOpt::INVALID_TABLET_TRANSFER_SEQ;
curr_opt.set_ss_private_tablet_meta_object_opt(test_ls_id, test_tablet_id, test_tablet_version, test_transfer_seq);
OB_LOG(INFO, "before set");
hex_dump(&test_block_id.fourth_id_,
sizeof(int64_t),
true,
OB_LOG_LEVEL_WARN);
// in ss mode
test_block_id.set_version_v2();
test_block_id.set_id_mode((uint64_t)ObMacroBlockIdMode::ID_MODE_SHARE);
test_block_id.set_storage_object_type(static_cast<int64_t>(curr_opt.object_type_));
test_block_id.set_incarnation_id(0);
test_block_id.set_column_group_id(0);
test_block_id.set_second_id(curr_opt.ss_private_tablet_opt_.ls_id_);
test_block_id.set_third_id(curr_opt.ss_private_tablet_opt_.tablet_id_);
test_block_id.set_meta_version_id(curr_opt.ss_private_tablet_opt_.version_);
test_block_id.set_meta_transfer_seq(curr_opt.ss_private_tablet_opt_.tablet_transfer_seq_);
OB_LOG(INFO, "after set");
hex_dump(&test_block_id.fourth_id_,
sizeof(int64_t),
true,
OB_LOG_LEVEL_WARN);
OB_LOG(INFO, "show test_block_id", K(test_block_id), K(test_block_id.meta_transfer_seq()), K(test_block_id.meta_version_id()));
int64_t transfer_seq1 = test_block_id.meta_transfer_seq();
OB_LOG(INFO, "transfer_seq1");
hex_dump(&transfer_seq1,
sizeof(int64_t),
true,
OB_LOG_LEVEL_WARN);
int64_t tablet_version1 = test_block_id.meta_version_id();
OB_LOG(INFO, "tablet_version1");
hex_dump(&tablet_version1,
sizeof(int64_t),
true,
OB_LOG_LEVEL_WARN);
ASSERT_EQ(-1, transfer_seq1);
ASSERT_EQ(17592186044415, tablet_version1); // compile failed when using ObStorageObjectOpt::INVALID_TABLET_VERSION
ASSERT_FALSE(test_block_id.is_valid());
const int64_t buf_len = 50;
char buf[buf_len] = {0};
int64_t pos = 0;
ret = test_block_id.serialize(buf, 50, pos);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
ASSERT_EQ(0, pos);
test_block_id.set_meta_transfer_seq(0);
test_block_id.set_meta_version_id(100002);
ret = test_block_id.serialize(buf, 50, pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(32, pos);
pos = 0;
ret = test_block_id.deserialize(buf, 50, pos);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(32, pos);
}
}
}

7
unittest/storage/blocksstable/test_sstable_meta.cpp
View File

@ -546,6 +546,7 @@ TEST_F(TestSSTableMacroInfo, test_huge_block_ids)
ObSharedObjectsWriteCtx linked_block_write_ctx;
ObSArray<ObSharedObjectsWriteCtx> total_ctxs;
ASSERT_EQ(OB_SUCCESS, sstable_macro_info.persist_block_ids(tablet_id,
0, // tablet_transfer_seq
0, // snapshot_version
allocator_,
&link_write_info,
@ -622,6 +623,7 @@ TEST_F(TestSSTableMeta, test_common_sstable_persister_linked_block)
ASSERT_EQ(OB_SUCCESS, sstable.persist_linked_block_if_need(
allocator_,
tablet_id,
0, // tablet_transfer_seq
snapshot_version,
nullptr,
macro_start_seq,
@ -638,6 +640,7 @@ TEST_F(TestSSTableMeta, test_common_sstable_persister_linked_block)
ASSERT_EQ(OB_SUCCESS, sstable.persist_linked_block_if_need(
allocator_,
tablet_id,
0, // tablet_transfer_seq
snapshot_version,
nullptr,
macro_start_seq,
@ -669,6 +672,7 @@ TEST_F(TestSSTableMeta, test_common_sstable_persister_linked_block)
ASSERT_EQ(OB_SUCCESS, tmp_sstable.persist_linked_block_if_need(
allocator_,
tablet_id,
0, // tablet_transfer_seq
snapshot_version,
nullptr,
macro_start_seq,
@ -707,6 +711,7 @@ TEST_F(TestSSTableMeta, test_huge_sstable_persister_linked_block)
ASSERT_EQ(OB_SUCCESS, sstable.persist_linked_block_if_need(
allocator_,
tablet_id,
0, // tablet_transfer_seq
snapshot_version,
nullptr,
macro_start_seq,
@ -723,6 +728,7 @@ TEST_F(TestSSTableMeta, test_huge_sstable_persister_linked_block)
ASSERT_EQ(OB_SUCCESS, sstable.persist_linked_block_if_need(
allocator_,
tablet_id,
0, // tablet_transfer_seq
snapshot_version,
nullptr,
macro_start_seq,
@ -754,6 +760,7 @@ TEST_F(TestSSTableMeta, test_huge_sstable_persister_linked_block)
ASSERT_EQ(OB_SUCCESS, tmp_sstable.persist_linked_block_if_need(
allocator_,
tablet_id,
0, // tablet_transfer_seq
snapshot_version,
nullptr,
macro_start_seq,

2
unittest/storage/ddl/test_ddl_kv.cpp
View File

@ -70,7 +70,7 @@ void TestBlockMetaTree::SetUp()
ObTenantEnv::set_tenant(&tenant_base_);
prepare_schema();
ASSERT_OK(data_desc_.init(table_schema_, ObLSID(1), ObTabletID(1), MAJOR_MERGE,
1/*snapshot*/, 1/*data_version*/, table_schema_.get_micro_index_clustered()));
1/*snapshot*/, 1/*data_version*/, table_schema_.get_micro_index_clustered(), 0 /*transfer_seq*/));
data_desc_.get_col_desc().allocator_.set_tenant_id(500);
ASSERT_OK(row_generate_.init(table_schema_, &allocator_));
}

81
unittest/storage/test_block_gc_handler.cpp
View File

@ -18,7 +18,7 @@
// #include "storage/blocksstable/ob_data_file_prepare.h"
// #include "share/ob_simple_mem_limit_getter.h"
#include "close_modules/shared_storage/storage/shared_storage/ob_public_block_gc_service.h"
#include "storage/ls/ob_ls.h"
namespace oceanbase
{
using namespace common;
@ -135,7 +135,7 @@ TEST_F(TestBlockGCHandler, test_block_gc)
}
class TestPublicBlockGCHandler : public ::testing::Test,
public storage::ObPublicBlockGCHandler
public storage::ObPublicBlockGCHandler
{
public:
TestPublicBlockGCHandler()
@ -213,6 +213,83 @@ TEST_F(TestPublicBlockGCHandler, test_detect_and_gc)
LOG_INFO("test detect_and_gc finish", KPC(this));
}
class TestPrivateBlockGCHandler : public ::testing::Test,
public storage::ObPrivateBlockGCHandler
{
public:
TestPrivateBlockGCHandler()
: storage::ObPrivateBlockGCHandler(share::ObLSID(1), 1, ObTabletID(1), 1, 1, 10, 20)
{
for (int i = 0; i < 30; i++) {
macro_block_ids_[i].id_mode_ = 2; // ID_MODE_SHARE
macro_block_ids_[i].storage_object_type_ = 0; //blocksstable::ObStorageObjectType::PRIVATE_DATA_MACRO;
macro_block_ids_[i].third_id_ = 1;
macro_block_ids_[i].tenant_seq_ = i;
}
}
virtual ~TestPrivateBlockGCHandler() {}
TO_STRING_KV(K(macro_block_ids_[0].tenant_seq_), K(macro_block_ids_[11].tenant_seq_), K(macro_block_ids_[12].tenant_seq_), K(macro_block_ids_[21].tenant_seq_));
int get_blocks_for_tablet(
int64_t tablet_meta_version,
ObIArray<blocksstable::MacroBlockId> &block_ids)
{
for (int i = 11; i < 20; i++) {
if (0 == i % 2) {
block_ids.push_back(macro_block_ids_[i]);
}
}
return OB_SUCCESS;
}
int list_tablet_meta_version(
ObIArray<int64_t> &tablet_versions)
{
tablet_versions.push_back(1);
return OB_SUCCESS;
}
int get_block_ids_from_dir(
ObIArray<blocksstable::MacroBlockId> &block_ids)
{
for (int i = 0; i < 30; i++) {
if (10 != i && 20 != i) {
block_ids.push_back(macro_block_ids_[i]);
}
}
return OB_SUCCESS;
}
int delete_macro_blocks(
ObIArray<blocksstable::MacroBlockId> &block_ids)
{
int ret = OB_SUCCESS;
for (int64_t i = 0; i < block_ids.count(); i++) {
macro_block_ids_[block_ids.at(i).tenant_seq_].third_id_ = 2;
}
return ret;
}
blocksstable::MacroBlockId macro_block_ids_[30];
};
TEST_F(TestPrivateBlockGCHandler, test_macro_check)
{
LOG_INFO("test detect_and_gc start", KPC(this));
ASSERT_EQ(OB_SUCCESS, macro_block_check());
for (int i = 11; i < 20; i++) {
if (0 == i % 2) {
ASSERT_EQ(1, macro_block_ids_[i].third_id_);
} else {
ASSERT_EQ(2, macro_block_ids_[i].third_id_);
}
}
for (int i = 0; i < 30; i++) {
if (i < 10 || i >= 20) {
ASSERT_EQ(1, macro_block_ids_[i].third_id_);
}
}
LOG_INFO("test test_macro_check finish", KPC(this));
}
} // end namespace unittest
} // end namespace oceanbase