hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix migration cut sstable left scn but new tablet table store init failed bug.

Browse Source
This commit is contained in:
godyangfight 2022-12-29 14:41:54 +00:00 committed by ob-robot
parent 4f6f8b11f3
commit 94e0e5067a
4 changed files with 157 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
src/storage/tablet/ob_tablet_table_store.cpp
View File

@ -1379,6 +1379,8 @@ int ObTabletTableStore::replace_ha_minor_sstables_(
ObSEArray<ObITable *, MAX_SSTABLE_CNT> need_add_minor_tables;
ObSEArray<ObITable *, MAX_SSTABLE_CNT> old_minor_tables;
const int64_t inc_pos = 0;
ObTablesHandleArray tables_handle;
ObArray<ObITable *> minor_tables;
if (OB_FAIL(param.tables_handle_.get_all_minor_sstables(need_add_minor_tables))) {
LOG_WARN("failed to add need add minor tables", K(ret), K(param));
@ -1402,18 +1404,20 @@ int ObTabletTableStore::replace_ha_minor_sstables_(
}
} else if (OB_FAIL(ObTableStoreUtil::sort_minor_tables(new_minor_tables))) {
LOG_WARN("failed to sort minor tables", K(ret));
} else if (OB_FAIL(cut_ha_sstable_scn_range_(new_minor_tables))) {
} else if (OB_FAIL(cut_ha_sstable_scn_range_(new_minor_tables, tables_handle))) {
LOG_WARN("failed to cut ha sstable log ts range", K(ret), K(old_store), K(param));
} else if (OB_FAIL(check_minor_tables_continue_(new_minor_tables.count(), new_minor_tables.get_data()))) {
LOG_WARN("minor tables is not continue", K(ret), K(param), K(new_minor_tables), K(old_store));
} else if (new_minor_tables.at(0)->get_start_scn() != tablet_ptr_->get_tablet_meta().start_scn_
|| new_minor_tables.at(new_minor_tables.count() - 1)->get_end_scn() != tablet_ptr_->get_tablet_meta().clog_checkpoint_scn_) {
} else if (OB_FAIL(tables_handle.get_tables(minor_tables))) {
LOG_WARN("failed to get minor tables", K(ret), K(tables_handle));
} else if (OB_FAIL(check_minor_tables_continue_(minor_tables.count(), minor_tables.get_data()))) {
LOG_WARN("minor tables is not continue", K(ret), K(param), K(minor_tables), K(old_store));
} else if (minor_tables.at(0)->get_start_scn() != tablet_ptr_->get_tablet_meta().start_scn_
|| minor_tables.at(minor_tables.count() - 1)->get_end_scn() != tablet_ptr_->get_tablet_meta().clog_checkpoint_scn_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet meta is not match with minor sstables", K(ret), K(new_minor_tables), K(param), K(old_store));
} else if (OB_FAIL(minor_tables_.init_and_copy(allocator, new_minor_tables, inc_pos))) {
LOG_WARN("tablet meta is not match with minor sstables", K(ret), K(minor_tables), K(param), K(old_store));
} else if (OB_FAIL(minor_tables_.init_and_copy(allocator, minor_tables, inc_pos))) {
LOG_WARN("failed to init minor_tables", K(ret));
} else {
LOG_INFO("succeed build ha minor sstables", K(old_store), K(new_minor_tables));
LOG_INFO("succeed build ha minor sstables", K(old_store), K(minor_tables));
}
return ret;
}
@ -1481,12 +1485,14 @@ int ObTabletTableStore::build_ha_ddl_tables_(
}
int ObTabletTableStore::cut_ha_sstable_scn_range_(
common::ObIArray<ObITable *> &minor_sstables)
common::ObIArray<ObITable *> &minor_sstables,
ObTablesHandleArray &tables_handle)
{
int ret = OB_SUCCESS;
SCN last_end_scn = SCN::min_scn();
for (int64_t i = 0; OB_SUCC(ret) && i < minor_sstables.count(); ++i) {
ObITable *table = minor_sstables.at(i);
ObTableHandleV2 new_table_handle;
if (OB_ISNULL(table) || !table->is_multi_version_minor_sstable()) {
ret = OB_ERR_UNEXPECTED;
@ -1500,14 +1506,28 @@ int ObTabletTableStore::cut_ha_sstable_scn_range_(
last_end_scn = table->get_end_scn();
} else {
ObSSTable *sstable = static_cast<ObSSTable *>(table);
ObScnRange new_scn_range;
ObScnRange original_scn_range = sstable->get_scn_range();
new_scn_range.start_scn_ = last_end_scn;
new_scn_range.end_scn_ = table->get_end_scn();
if (OB_FAIL(deep_copy_sstable_(*sstable, new_table_handle))) {
LOG_WARN("failed to deep copy sstable", KPC(sstable), K(new_table_handle));
} else {
table = new_table_handle.get_table();
ObScnRange new_scn_range;
ObScnRange original_scn_range = table->get_scn_range();
new_scn_range.start_scn_ = last_end_scn;
new_scn_range.end_scn_ = table->get_end_scn();
sstable->set_scn_range(new_scn_range);
last_end_scn = table->get_end_scn();
LOG_INFO("cut ha sstable log ts range", KPC(sstable), K(new_scn_range), K(original_scn_range));
table->set_scn_range(new_scn_range);
last_end_scn = table->get_end_scn();
LOG_INFO("cut ha sstable log ts range", KPC(sstable), K(new_scn_range), K(original_scn_range));
}
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(table)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table should not be NULL", K(ret), KP(table));
} else if (OB_FAIL(tables_handle.add_table(table))) {
LOG_WARN("failed to add table into array", K(ret), KPC(table));
}
}
}
return ret;
@ -1817,6 +1837,46 @@ int ObTabletTableStore::get_recycle_version(
}
int ObTabletTableStore::deep_copy_sstable_(
const blocksstable::ObSSTable &old_sstable,
ObTableHandleV2 &new_table_handle)
{
int ret = OB_SUCCESS;
new_table_handle.reset();
ObArenaAllocator allocator;
int64_t pos = 0;
int64_t size = 0;
char *buf = nullptr;
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr *);
ObSSTable *new_sstable = nullptr;
if (!old_sstable.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("deep copy sstable get invalid argument", K(ret), K(old_sstable));
} else if (OB_ISNULL(t3m)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("t3m should not be NULL", K(ret), KP(t3m));
} else if (FALSE_IT(size = old_sstable.get_serialize_size())) {
} else if (OB_ISNULL(buf = static_cast<char *>(allocator.alloc(size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory", K(ret), K(size));
} else if (OB_FAIL(old_sstable.serialize(buf, size, pos))) {
LOG_WARN("failed to serialize sstable", K(ret), K(old_sstable), K(size));
} else if (OB_FAIL(t3m->acquire_sstable(new_table_handle))) {
LOG_WARN("failed to acquire sstable", K(ret));
} else if (OB_ISNULL(new_sstable = static_cast<ObSSTable *>(new_table_handle.get_table()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table should not be NULL", K(ret), K(new_table_handle));
} else if (FALSE_IT(pos = 0)) {
} else if (OB_FAIL(new_sstable->deserialize(t3m->get_tenant_allocator(), buf, size, pos))) {
LOG_WARN("failed to deserialize new sstable", K(ret), K(size), K(pos), K(old_sstable));
} else if (OB_FAIL(new_sstable->deserialize_post_work())) {
LOG_WARN("failed to deserialize post work", K(ret), KPC(new_sstable), K(old_sstable));
}
return ret;
}
ObPrintTableStore::ObPrintTableStore(const ObTabletTableStore &table_store)
: major_tables_(table_store.major_tables_),
minor_tables_(table_store.minor_tables_),

5
src/storage/tablet/ob_tablet_table_store.h
View File

@ -188,7 +188,8 @@ private:
const ObBatchUpdateTableStoreParam &param,
const ObTabletTableStore &old_store);
int cut_ha_sstable_scn_range_(
common::ObIArray<ObITable *> &minor_sstables);
common::ObIArray<ObITable *> &minor_sstables,
ObTablesHandleArray &tables_handle);
int check_minor_tables_continue_(
const int64_t count,
ObITable **minor_sstables) const;
@ -217,6 +218,8 @@ private:
const ObIArray<ObTableHandleV2> &table_handles,
const ObITableArray &old_tables,
ObSEArray<ObITable *, 8> &replaced_tables) const;
int deep_copy_sstable_(const blocksstable::ObSSTable &old_sstable,
ObTableHandleV2 &new_table_handle);
public:
static const int64_t TABLE_STORE_VERSION = 0x0100;

103
unittest/storage/test_sstable_log_ts_range_cut.cpp
View File

@ -21,12 +21,16 @@
#include "storage/ob_i_table.h"
#include "storage/ob_storage_struct.h"
#include "share/scn.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
namespace oceanbase
{
using namespace common;
using namespace blocksstable;
using namespace storage;
using namespace omt;
using namespace share;
namespace unittest
{
@ -42,17 +46,55 @@ public:
sstable.key_.tablet_id_ = 1;
sstable.key_.scn_range_.start_scn_.convert_for_gts(start_log_ts);
sstable.key_.scn_range_.end_scn_.convert_for_gts(end_log_ts);
sstable.meta_.basic_meta_.row_store_type_ = ObRowStoreType::FLAT_ROW_STORE;
sstable.valid_for_reading_ = true;
sstable.meta_.basic_meta_.status_ = SSTABLE_WRITE_BUILDING;
sstable.meta_.data_root_info_.addr_.set_none_addr();
sstable.meta_.macro_info_.macro_meta_info_.addr_.set_none_addr();
sstable.meta_.basic_meta_.compressor_type_ = ObCompressorType::NONE_COMPRESSOR;
}
};
class TestSSTableScnRangeCut : public ::testing::Test
{
public:
TestSSTableScnRangeCut()
: tenant_id_(1001),
t3m_(nullptr),
tenant_base_(1001)
{ }
~TestSSTableScnRangeCut() {}
void SetUp()
{
t3m_ = OB_NEW(ObTenantMetaMemMgr, ObModIds::TEST, 1001);
t3m_->init();
tenant_base_.set(t3m_);
ObTenantEnv::set_tenant(&tenant_base_);
ASSERT_EQ(OB_SUCCESS, tenant_base_.init());
}
void TearDown()
{
t3m_->~ObTenantMetaMemMgr();
t3m_ = nullptr;
tenant_base_.destroy();
ObTenantEnv::set_tenant(nullptr);
}
private:
const uint64_t tenant_id_;
ObTenantMetaMemMgr *t3m_;
ObTenantBase tenant_base_;
ObTabletTableStore tablet_table_store_;
DISALLOW_COPY_AND_ASSIGN(TestSSTableScnRangeCut);
};
//normal condition
//sstable1 log_ts: [0,100)
//sstable2 log_ts: [100, 200)
//sstable3 log_ts: [200,300)
TEST(ObTabletTableStore, sstable_scn_range_no_cross_and_continue)
TEST_F(TestSSTableScnRangeCut, sstable_scn_range_no_cross_and_continue)
{
int ret = OB_SUCCESS;
ObTabletTableStore tablet_table_store;
@ -65,25 +107,26 @@ TEST(ObTabletTableStore, sstable_scn_range_no_cross_and_continue)
ObSSTable sstable3;
TestMockSSTable::generate_mock_sstable(200, 300, sstable3);
ObArray<ObITable *> minor_sstables;
ObTablesHandleArray tables_handle;
ret = minor_sstables.push_back(&sstable1);
ASSERT_EQ(OB_SUCCESS, ret);
ret = minor_sstables.push_back(&sstable2);
ASSERT_EQ(OB_SUCCESS, ret);
ret = minor_sstables.push_back(&sstable3);
ASSERT_EQ(OB_SUCCESS, ret);
ret = tablet_table_store.cut_ha_sstable_scn_range_(minor_sstables);
ret = tablet_table_store.cut_ha_sstable_scn_range_(minor_sstables, tables_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(0, sstable1.key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, sstable1.key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(0, tables_handle.get_table(0)->key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, tables_handle.get_table(0)->key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, sstable2.key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, sstable2.key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, tables_handle.get_table(1)->key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, tables_handle.get_table(1)->key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, sstable3.key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(300, sstable3.key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, tables_handle.get_table(2)->key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(300, tables_handle.get_table(2)->key_.scn_range_.end_scn_.get_val_for_inner_table_field());
}
@ -92,7 +135,7 @@ TEST(ObTabletTableStore, sstable_scn_range_no_cross_and_continue)
//sstable2 log_ts: [200, 300)
//sstable3 log_ts: [300,500)
TEST(ObTabletTableStore, sstable_scn_range_is_not_continue)
TEST_F(TestSSTableScnRangeCut, sstable_scn_range_is_not_continue)
{
int ret = OB_SUCCESS;
ObTabletTableStore tablet_table_store;
@ -107,13 +150,15 @@ TEST(ObTabletTableStore, sstable_scn_range_is_not_continue)
TestMockSSTable::generate_mock_sstable(300, 500, sstable3);
ObArray<ObITable *> minor_sstables;
ObTablesHandleArray tables_handle;
ret = minor_sstables.push_back(&sstable1);
ASSERT_EQ(OB_SUCCESS, ret);
ret = minor_sstables.push_back(&sstable2);
ASSERT_EQ(OB_SUCCESS, ret);
ret = minor_sstables.push_back(&sstable3);
ASSERT_EQ(OB_SUCCESS, ret);
ret = tablet_table_store.cut_ha_sstable_scn_range_(minor_sstables);
ret = tablet_table_store.cut_ha_sstable_scn_range_(minor_sstables, tables_handle);
ASSERT_EQ(OB_ERR_UNEXPECTED, ret);
}
@ -123,7 +168,7 @@ TEST(ObTabletTableStore, sstable_scn_range_is_not_continue)
//sstable2 log_ts: [0, 200)
//sstable3 log_ts: [200,500)
TEST(ObTabletTableStore, sstable_scn_range_contain)
TEST_F(TestSSTableScnRangeCut, sstable_scn_range_contain)
{
int ret = OB_SUCCESS;
ObTabletTableStore tablet_table_store;
@ -138,23 +183,25 @@ TEST(ObTabletTableStore, sstable_scn_range_contain)
TestMockSSTable::generate_mock_sstable(200, 500, sstable3);
ObArray<ObITable *> minor_sstables;
ObTablesHandleArray tables_handle;
ret = minor_sstables.push_back(&sstable1);
ASSERT_EQ(OB_SUCCESS, ret);
ret = minor_sstables.push_back(&sstable2);
ASSERT_EQ(OB_SUCCESS, ret);
ret = minor_sstables.push_back(&sstable3);
ASSERT_EQ(OB_SUCCESS, ret);
ret = tablet_table_store.cut_ha_sstable_scn_range_(minor_sstables);
ret = tablet_table_store.cut_ha_sstable_scn_range_(minor_sstables, tables_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(0, sstable1.key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, sstable1.key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(0, tables_handle.get_table(0)->key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, tables_handle.get_table(0)->key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, sstable2.key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, sstable2.key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, tables_handle.get_table(1)->key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, tables_handle.get_table(1)->key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, sstable3.key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(500, sstable3.key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, tables_handle.get_table(2)->key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(500, tables_handle.get_table(2)->key_.scn_range_.end_scn_.get_val_for_inner_table_field());
}
@ -163,7 +210,7 @@ TEST(ObTabletTableStore, sstable_scn_range_contain)
//sstable2 log_ts: [50, 200)
//sstable3 log_ts: [200,500)
TEST(ObTabletTableStore, sstable_scn_range_has_overlap)
TEST_F(TestSSTableScnRangeCut, sstable_scn_range_has_overlap)
{
int ret = OB_SUCCESS;
ObTabletTableStore tablet_table_store;
@ -178,23 +225,25 @@ TEST(ObTabletTableStore, sstable_scn_range_has_overlap)
TestMockSSTable::generate_mock_sstable(200, 500, sstable3);
ObArray<ObITable *> minor_sstables;
ObTablesHandleArray tables_handle;
ret = minor_sstables.push_back(&sstable1);
ASSERT_EQ(OB_SUCCESS, ret);
ret = minor_sstables.push_back(&sstable2);
ASSERT_EQ(OB_SUCCESS, ret);
ret = minor_sstables.push_back(&sstable3);
ASSERT_EQ(OB_SUCCESS, ret);
ret = tablet_table_store.cut_ha_sstable_scn_range_(minor_sstables);
ret = tablet_table_store.cut_ha_sstable_scn_range_(minor_sstables, tables_handle);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(0, sstable1.key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, sstable1.key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(0, tables_handle.get_table(0)->key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, tables_handle.get_table(0)->key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, sstable2.key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, sstable2.key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(100, tables_handle.get_table(1)->key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, tables_handle.get_table(1)->key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, sstable3.key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(500, sstable3.key_.scn_range_.end_scn_.get_val_for_inner_table_field());
ASSERT_EQ(200, tables_handle.get_table(2)->key_.scn_range_.start_scn_.get_val_for_inner_table_field());
ASSERT_EQ(500, tables_handle.get_table(2)->key_.scn_range_.end_scn_.get_val_for_inner_table_field());
}

2
unittest/storage/test_table_scan_pure_index_table.cpp
View File

@ -102,4 +102,4 @@ int main(int argc, char **argv)
OB_LOGGER.set_log_level(OB_LOG_LEVEL_INFO);
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
}