hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] instant ddl commit

Browse Source 
Co-authored-by: hiddenbomb <bwgui203@gmail.com>
Co-authored-by: Charles0429 <xiezhenjiang@gmail.com>
Co-authored-by: rolandqi <qikai456@126.com>
This commit is contained in:
simonjoylet 2023-01-28 20:34:20 +08:00 committed by ob-robot
parent 5dbb8ee1a2
commit 0f14606391
112 changed files with 2988 additions and 1757 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h vendored
View File

@ -436,7 +436,7 @@ PCODE_DEF(OB_DDL_BUILD_SINGLE_REPLICA_REQUEST, 0x48D)
PCODE_DEF(OB_DDL_BUILD_SINGLE_REPLICA_RESPONSE, 0x48E)
//PCODE_DEF(OB_STANDBY_CUTDATA_BATCH_TASK, 0x490)
PCODE_DEF(OB_FETCH_REUSED_BLOCKS, 0x491)
PCODE_DEF(OB_WRITE_DDL_SSTABLE_COMMIT_LOG, 0x492)
//PCODE_DEF(OB_WRITE_DDL_SSTABLE_COMMIT_LOG, 0x492) // deprecated
//PCODE_DEF(OB_DDL_CHECK_MINOR, 0x493) // 4.0 not supported
// log stream related
PCODE_DEF(OB_REPORT_SYS_LS, 0x494)
@ -750,10 +750,11 @@ PCODE_DEF(OB_HIGH_PRIORITY_BATCH_TABLE_LOCK_TASK, 0x932)
// ddl
PCODE_DEF(OB_REMOTE_WRITE_DDL_REDO_LOG, 0x950)
PCODE_DEF(OB_REMOTE_WRITE_DDL_COMMIT_LOG, 0x951)
//PCODE_DEF(OB_REMOTE_WRITE_DDL_COMMIT_LOG, 0x951)
PCODE_DEF(OB_BATCH_GET_TABLET_AUTOINC_SEQ, 0x952)
PCODE_DEF(OB_BATCH_SET_TABLET_AUTOINC_SEQ, 0x953)
PCODE_DEF(OB_REMOTE_WRITE_DDL_PREPARE_LOG, 0x954)
// rename from OB_REMOTE_WRITE_DDL_PREPARE_LOG
PCODE_DEF(OB_REMOTE_WRITE_DDL_COMMIT_LOG, 0x954)
PCODE_DEF(OB_DDL_CHECK_TABLET_MERGE_STATUS, 0x957)

51
mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h
View File

@ -21,6 +21,7 @@
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "storage/tablet/ob_tablet_create_sstable_param.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
#include "storage/blocksstable/ob_row_generate.h"
#include "share/rc/ob_tenant_base.h"
#include "observer/omt/ob_tenant_node_balancer.h"
@ -70,6 +71,7 @@ public:
virtual ObITable::TableType get_merged_table_type() const;
void prepare_query_param(const bool is_reverse_scan, const ObTableReadInfo &full_read_info);
void destroy_query_param();
void prepare_ddl_kv();
protected:
static const int64_t TEST_COLUMN_CNT = ObExtendType - 1;
static const int64_t MAX_TEST_COLUMN_CNT = TEST_COLUMN_CNT + 3;
@ -88,6 +90,7 @@ protected:
ObRowGenerate row_generate_;
int64_t row_cnt_;
ObSSTable sstable_;
storage::ObDDLKV ddl_kv_;
ObSSTableIndexBuilder *root_index_builder_;
ObMicroBlockData root_block_data_buf_;
ObRowStoreType row_store_type_;
@ -220,6 +223,7 @@ void TestIndexBlockDataPrepare::SetUp()
void TestIndexBlockDataPrepare::TearDown()
{
sstable_.reset();
ddl_kv_.reset();
if (nullptr != root_block_data_buf_.buf_) {
allocator_.free((void *)root_block_data_buf_.buf_);
}
@ -263,7 +267,7 @@ ObITable::TableType TestIndexBlockDataPrepare::get_merged_table_type() const
} else if (META_MAJOR_MERGE == merge_type_) {
table_type = ObITable::TableType::META_MAJOR_SSTABLE;
} else if (DDL_KV_MERGE == merge_type_) {
table_type = ObITable::TableType::KV_DUMP_SSTABLE;
table_type = ObITable::TableType::DDL_DUMP_SSTABLE;
} else { // MINOR_MERGE || HISTORY_MINOR_MERGE
table_type = ObITable::TableType::MINOR_SSTABLE;
}
@ -510,12 +514,57 @@ void TestIndexBlockDataPrepare::prepare_data()
}
ASSERT_EQ(OB_SUCCESS, sstable_.init(param, &allocator_));
STORAGE_LOG(INFO, "create sstable param", K(param));
prepare_ddl_kv();
root_block_data_buf_.buf_ = root_buf;
root_block_data_buf_.size_ = root_size;
row_store_type_ = root_row_store_type;
}
void TestIndexBlockDataPrepare::prepare_ddl_kv()
{
ddl_kv_.reset();
ObTabletHandle tablet_handle;
int ret = OB_SUCCESS;
ObLSID ls_id(ls_id_);
ObTabletID tablet_id(tablet_id_);
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));
share::SCN ddl_start_scn;
ddl_start_scn.convert_from_ts(ObTimeUtility::current_time());
ASSERT_EQ(OB_SUCCESS, ddl_kv_.init(ls_id, tablet_id, ddl_start_scn, sstable_.get_data_version(), ddl_start_scn, 4000));
SMART_VAR(ObSSTableSecMetaIterator, meta_iter) {
ObDatumRange query_range;
query_range.set_whole_range();
ObDataMacroBlockMeta data_macro_meta;
ASSERT_EQ(OB_SUCCESS, meta_iter.open(query_range,
ObMacroBlockMetaType::DATA_BLOCK_META,
sstable_,
tablet_handle.get_obj()->get_index_read_info(),
allocator_));
while (OB_SUCC(ret)) {
if (OB_FAIL(meta_iter.get_next(data_macro_meta))) {
if (OB_ITER_END != ret) {
STORAGE_LOG(WARN, "get data macro meta failed", K(ret));
}
} else {
ObDDLMacroHandle macro_handle;
macro_handle.set_block_id(data_macro_meta.get_macro_id());
ObDataMacroBlockMeta *copied_meta = nullptr;
ASSERT_EQ(OB_SUCCESS, data_macro_meta.deep_copy(copied_meta, allocator_));
ASSERT_EQ(OB_SUCCESS, ddl_kv_.insert_block_meta_tree(macro_handle, copied_meta));
}
}
ASSERT_EQ(OB_ITER_END, ret);
ASSERT_EQ(OB_SUCCESS, ddl_kv_.block_meta_tree_.build_sorted_rowkeys());
}
}
void TestIndexBlockDataPrepare::insert_data(ObMacroBlockWriter &data_writer)
{
row_cnt_ = 0;

9
mittest/mtlenv/storage/blocksstable/test_sstable_row_exister.cpp
View File

@ -83,6 +83,7 @@ TEST_F(TestSSTableRowExister, single_exist)
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, TEST_COLUMN_CNT));
ObStoreCtx ctx;
ObSSTableRowExister row_exister;
ObSSTableRowExister kv_row_exister;
//exist check
bool is_exist = false;
@ -90,15 +91,23 @@ TEST_F(TestSSTableRowExister, single_exist)
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row_cnt_ - 1, row));
ASSERT_EQ(OB_SUCCESS, rowkey.assign(row.storage_datums_, TEST_ROWKEY_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, row_exister.inner_open(iter_param_, context_, &sstable_, &rowkey));
ASSERT_EQ(OB_SUCCESS, kv_row_exister.inner_open(iter_param_, context_, &ddl_kv_, &rowkey));
const ObDatumRow *prow = nullptr;
const ObDatumRow *kv_prow = nullptr;
ASSERT_EQ(OB_SUCCESS, row_exister.inner_get_next_row(prow));
ASSERT_TRUE(prow->row_flag_.is_exist());
ASSERT_EQ(OB_SUCCESS, kv_row_exister.inner_get_next_row(kv_prow));
ASSERT_TRUE(kv_prow->row_flag_.is_exist());
row_exister.reuse();
kv_row_exister.reuse();
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row_cnt_, row));
ASSERT_EQ(OB_SUCCESS, row_exister.inner_open(iter_param_, context_, &sstable_, &rowkey));
ASSERT_EQ(OB_SUCCESS, row_exister.inner_get_next_row(prow));
ASSERT_TRUE(prow->row_flag_.is_not_exist());
ASSERT_EQ(OB_SUCCESS, kv_row_exister.inner_open(iter_param_, context_, &ddl_kv_, &rowkey));
ASSERT_EQ(OB_SUCCESS, kv_row_exister.inner_get_next_row(kv_prow));
ASSERT_TRUE(kv_prow->row_flag_.is_not_exist());
}
}

55
mittest/mtlenv/storage/blocksstable/test_sstable_row_getter.cpp
View File

@ -35,7 +35,7 @@ public:
virtual void SetUp();
virtual void TearDown();
void test_one_rowkey(ObSSTableRowGetter &getter, const int64_t seed);
void test_one_rowkey(const int64_t seed);
};
TestSSTableRowGetter::TestSSTableRowGetter()
@ -77,8 +77,10 @@ void TestSSTableRowGetter::TearDown()
TestIndexBlockDataPrepare::TearDown();
}
void TestSSTableRowGetter::test_one_rowkey(ObSSTableRowGetter &getter, const int64_t seed)
void TestSSTableRowGetter::test_one_rowkey(const int64_t seed)
{
ObSSTableRowGetter getter;
ObSSTableRowGetter kv_getter;
ObDatumRow query_row;
ASSERT_EQ(OB_SUCCESS, query_row.init(allocator_, TEST_COLUMN_CNT));
row_generate_.get_next_row(seed, query_row);
@ -86,54 +88,39 @@ void TestSSTableRowGetter::test_one_rowkey(ObSSTableRowGetter &getter, const int
query_rowkey.assign(query_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
STORAGE_LOG(INFO, "Query rowkey", K(query_row));
ASSERT_EQ(OB_SUCCESS, getter.inner_open(iter_param_, context_, &sstable_, &query_rowkey));
ASSERT_EQ(OB_SUCCESS, kv_getter.inner_open(iter_param_, context_, &ddl_kv_, &query_rowkey));
const ObDatumRow *prow = nullptr;
const ObDatumRow *kv_prow = nullptr;
ASSERT_EQ(OB_SUCCESS, getter.inner_get_next_row(prow));
STORAGE_LOG(INFO, "debug datum row1", KPC(prow));
ASSERT_TRUE(*prow == query_row);
ASSERT_EQ(OB_SUCCESS, kv_getter.inner_get_next_row(kv_prow));
STORAGE_LOG(INFO, "debug datum row1", KPC(prow), KPC(kv_prow));
if (seed >= row_cnt_) {
ASSERT_TRUE(prow->row_flag_.is_not_exist());
ASSERT_TRUE(kv_prow->row_flag_.is_not_exist());
} else {
ASSERT_TRUE(*prow == query_row);
ASSERT_TRUE(*kv_prow == query_row);
}
ASSERT_EQ(OB_ITER_END, getter.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, kv_getter.inner_get_next_row(kv_prow));
getter.reuse();
ASSERT_EQ(OB_SUCCESS, getter.inner_open(iter_param_, context_, &sstable_, &query_rowkey));
ASSERT_EQ(OB_SUCCESS, getter.inner_get_next_row(prow));
STORAGE_LOG(INFO, "debug datum row2", KPC(prow));
ASSERT_TRUE(*prow == query_row);
ASSERT_EQ(OB_ITER_END, getter.inner_get_next_row(prow));
kv_getter.reuse();
}
TEST_F(TestSSTableRowGetter, get)
{
const ObDatumRow *prow = nullptr;
ObSSTableRowGetter getter;
//left border rowkey
test_one_rowkey(getter, 0);
ASSERT_EQ(OB_ITER_END, getter.inner_get_next_row(prow));
getter.reuse();
test_one_rowkey(0);
// right border rowkey
test_one_rowkey(getter, row_cnt_ - 1);
ASSERT_EQ(OB_ITER_END, getter.inner_get_next_row(prow));
getter.reuse();
test_one_rowkey(row_cnt_ - 1);
// mid border rowkey
test_one_rowkey(getter, row_cnt_ / 2);
ASSERT_EQ(OB_ITER_END, getter.inner_get_next_row(prow));
getter.reuse();
test_one_rowkey(row_cnt_ / 2);
// non-exist rowkey
ObDatumRow row;
ObDatumRowkey rowkey;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, TEST_COLUMN_CNT + 1));
row_generate_.get_next_row(row_cnt_, row);
rowkey.assign(row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, getter.inner_open(iter_param_, context_, &sstable_, &rowkey));
ASSERT_EQ(OB_SUCCESS, getter.inner_get_next_row(prow));
ASSERT_TRUE(prow->row_flag_.is_not_exist());
test_one_rowkey(row_cnt_);
}
}

25
mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_getter.cpp
View File

@ -107,7 +107,9 @@ void TestSSTableRowMultiGetter::test_one_case(
ObDatumRowkey rowkey;
ObDatumRow row;
const ObDatumRow *prow = NULL;
const ObDatumRow *kv_prow = NULL;
ObSSTableRowMultiGetter getter;
ObSSTableRowMultiGetter kv_getter;
// prepare rowkeys
ObDatumRowkey mget_rowkeys[TEST_MULTI_GET_CNT];
@ -134,51 +136,74 @@ void TestSSTableRowMultiGetter::test_one_case(
}
if (part_rowkeys.count() > 0) {
ASSERT_EQ(OB_SUCCESS, getter.inner_open(iter_param_, context_, &sstable_, &part_rowkeys));
ASSERT_EQ(OB_SUCCESS, kv_getter.inner_open(iter_param_, context_, &ddl_kv_, &part_rowkeys));
for (int64_t i = 0; i < part_rowkeys.count(); ++i) {
ret = getter.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_getter.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
}
ret = getter.inner_get_next_row(prow);
ASSERT_EQ(OB_ITER_END, ret);
ret = kv_getter.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_ITER_END, ret);
}
getter.reuse();
kv_getter.reuse();
}
// in io
ASSERT_EQ(OB_SUCCESS, getter.inner_open(iter_param_, context_, &sstable_, &rowkeys));
ASSERT_EQ(OB_SUCCESS, kv_getter.inner_open(iter_param_, context_, &ddl_kv_, &rowkeys));
for (int64_t i = 0; i < seeds.count(); ++i) {
ret = getter.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_getter.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
if (seeds.at(i) >= row_cnt_) {
ASSERT_TRUE(prow->row_flag_.is_not_exist());
ASSERT_TRUE(kv_prow->row_flag_.is_not_exist());
} else {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seeds.at(i), check_row_));
ASSERT_EQ(OB_SUCCESS, (const_cast<ObDatumRow *>(prow))->prepare_new_row(schema_cols_));
ASSERT_EQ(OB_SUCCESS, (const_cast<ObDatumRow *>(kv_prow))->prepare_new_row(schema_cols_));
ASSERT_TRUE(check_row_ == *prow);
ASSERT_TRUE(check_row_ == *kv_prow);
}
}
ret = getter.inner_get_next_row(prow);
ASSERT_EQ(OB_ITER_END, ret);
ret = kv_getter.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_ITER_END, ret);
getter.reuse();
kv_getter.reuse();
// in cache
if (hit_mode == HIT_ALL) {
ASSERT_EQ(OB_SUCCESS, getter.inner_open(iter_param_, context_, &sstable_, &rowkeys));
ASSERT_EQ(OB_SUCCESS, kv_getter.inner_open(iter_param_, context_, &ddl_kv_, &rowkeys));
for (int64_t i = 0; i < seeds.count(); ++i) {
ret = getter.inner_get_next_row(prow);
ret = kv_getter.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
if (seeds.at(i) >= row_cnt_) {
ASSERT_TRUE(prow->row_flag_.is_not_exist());
ASSERT_TRUE(kv_prow->row_flag_.is_not_exist());
} else {
ASSERT_EQ(OB_SUCCESS, (const_cast<ObDatumRow *>(prow))->prepare_new_row(schema_cols_));
ASSERT_EQ(OB_SUCCESS, (const_cast<ObDatumRow *>(kv_prow))->prepare_new_row(schema_cols_));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seeds.at(i), check_row_));
ASSERT_TRUE(check_row_ == *prow);
ASSERT_TRUE(check_row_ == *kv_prow);
}
}
ret = getter.inner_get_next_row(prow);
ASSERT_EQ(OB_ITER_END, ret);
ret = kv_getter.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_ITER_END, ret);
}
getter.reuse();
kv_getter.reuse();
}
void TestSSTableRowMultiGetter::test_border(const bool is_reverse_scan)

92
mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp
View File

@ -134,8 +134,10 @@ void TestSSTableRowMultiScanner::test_one_case(
int ret = OB_SUCCESS;
ObDatumRange mscan_ranges[TEST_MULTI_GET_CNT];
ObSSTableRowMultiScanner scanner;
ObSSTableRowMultiScanner kv_scanner;
ObSEArray<ObDatumRange, TEST_MULTI_GET_CNT> ranges;
const ObDatumRow *prow = NULL;
const ObDatumRow *kv_prow = NULL;
//if (HIT_PART == hit_mode) {
//ObArray<ObDatumRange> part_ranges;
//ObArray<ObDatumRange> tmp_ranges;
@ -182,6 +184,7 @@ void TestSSTableRowMultiScanner::test_one_case(
ASSERT_EQ(OB_SUCCESS, ranges.push_back(mscan_ranges[i]));
}
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(iter_param_, context_, &sstable_, &ranges));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(iter_param_, context_, &ddl_kv_, &ranges));
for (int64_t i = 0; i < start_seeds.count(); ++i) {
for (int64_t j = 0; j < count_per_range; ++j) {
const int64_t k = is_reverse_scan ? start_seeds.at(i) + count_per_range - j - 1 : start_seeds.at(i) + j;
@ -190,12 +193,16 @@ void TestSSTableRowMultiScanner::test_one_case(
if (k < row_cnt_) {
ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
ASSERT_TRUE(*prow == check_row);
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_get_next_row(kv_prow));
ASSERT_TRUE(*kv_prow == check_row);
}
}
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
scanner.reuse();
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
kv_scanner.reuse();
if (HIT_ALL == hit_mode) {
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
@ -203,6 +210,11 @@ void TestSSTableRowMultiScanner::test_one_case(
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&ranges));
for (int64_t i = 0; i < start_seeds.count(); ++i) {
for (int64_t j = 0; j < count_per_range; ++j) {
const int64_t k = is_reverse_scan ? start_seeds.at(i) + count_per_range - j - 1 : start_seeds.at(i) + j;
@ -211,13 +223,18 @@ void TestSSTableRowMultiScanner::test_one_case(
if (k < row_cnt_) {
ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
ASSERT_TRUE(*prow == check_row);
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_get_next_row(kv_prow));
ASSERT_TRUE(*kv_prow == check_row);
}
}
}
}
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_ITER_END, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_ITER_END, ret);
scanner.reuse();
kv_scanner.reuse();
}
}
@ -670,8 +687,10 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
ObDatumRow row;
ObDatumRowkey rowkey;
const ObDatumRow *prow = NULL;
const ObDatumRow *kv_prow = NULL;
int64_t row_cnt = 0;
ObSSTableRowMultiScanner scanner;
ObSSTableRowMultiScanner kv_scanner;
// prepare query param
prepare_query_param(is_reverse_scan, tablet_handle_.get_obj()->get_full_read_info());
@ -706,6 +725,11 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&ranges));
row_cnt = 0;
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
@ -713,19 +737,25 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
for (int64_t j = 0; j < count_per_range; ++j) {
const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_get_next_row(kv_prow));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(k, check_row));
++row_cnt;
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
} else {
ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_get_next_row(kv_prow));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(p, check_row));
++row_cnt;
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
kv_scanner.reuse();
// first half multi scan, second half multi get
ranges.reuse();
@ -746,6 +776,11 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&ranges));
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
if (p < TEST_MULTI_GET_CNT / 2) {
@ -753,20 +788,28 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(k, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
} else {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(p, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
kv_scanner.reuse();
// first half multi get, second half multi scan
ranges.reuse();
@ -788,6 +831,11 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&ranges));
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
@ -796,21 +844,30 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(k, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
} else {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(p, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_ITER_END, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_ITER_END, ret);
scanner.reuse();
kv_scanner.reuse();
// first one multi get, others multi scan
ranges.reuse();
@ -831,6 +888,11 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&ranges));
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
if (p != 0) {
@ -838,20 +900,28 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(k, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
} else {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(p, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
kv_scanner.reuse();
// first one multi scan, others multi get
ranges.reuse();
@ -872,6 +942,11 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&ranges));
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
if (p == 0) {
@ -879,20 +954,28 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(k, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
} else {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(p, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
kv_scanner.reuse();
// multi scan not exist row
STORAGE_LOG(DEBUG, "multi_scan_not_exist_row");
@ -914,6 +997,11 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&ranges));
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
if (p % 2) {
@ -921,12 +1009,16 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
} else {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(p, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
destroy_query_param();
}

38
mittest/mtlenv/storage/blocksstable/test_sstable_row_scanner.cpp
View File

@ -124,7 +124,9 @@ void TestSSTableRowScanner::test_one_case(
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, TEST_COLUMN_CNT));
const ObDatumRow *prow = nullptr;
const ObDatumRow *kv_prow = nullptr;
ObSSTableRowScanner scanner;
ObSSTableRowScanner kv_scanner;
if (HIT_PART == hit_mode) {
const int64_t part_start = start + (end - start) / 3;
@ -148,15 +150,25 @@ void TestSSTableRowScanner::test_one_case(
context_,
&sstable_,
&part_range));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&part_range));
for (int64_t i = part_start; i <= part_end; ++i) {
if (i < row_cnt_) {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret) << "i: " << i << " part_start: " << part_start
<< " part_end: " << part_end << " prow: " << prow;
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret) << "i: " << i << " part_start: " << part_start
<< " part_end: " << part_end << " kv_prow: " << kv_prow;
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
kv_scanner.reuse();
}
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
@ -164,6 +176,11 @@ void TestSSTableRowScanner::test_one_case(
context_,
&sstable_,
&range));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&range));
for (int64_t i = start; i <= end; ++i) {
int64_t index = 0;
if (is_reverse_scan) {
@ -180,11 +197,19 @@ void TestSSTableRowScanner::test_one_case(
<< " end: " << end << " prow: " << prow;
ASSERT_TRUE(row == *prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " prow: " << prow;
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
ASSERT_TRUE(row == *kv_prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
kv_scanner.reuse();
if (HIT_ALL == hit_mode) {
int64_t index = 0;
@ -193,6 +218,11 @@ void TestSSTableRowScanner::test_one_case(
context_,
&sstable_,
&range));
ASSERT_EQ(OB_SUCCESS, kv_scanner.inner_open(
iter_param_,
context_,
&ddl_kv_,
&range));
for (int64_t i = start; i <= end; ++i) {
if (is_reverse_scan) {
ret = row_generate_.get_next_row(end - i + start, row);
@ -208,11 +238,19 @@ void TestSSTableRowScanner::test_one_case(
<< " end: " << end << " prow: " << prow;
ASSERT_TRUE(row == *prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " prow: " << prow;
ret = kv_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
ASSERT_TRUE(row == *kv_prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
ASSERT_EQ(OB_ITER_END, kv_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
kv_scanner.reuse();
}
}

2
mittest/simple_server/fast_commit_report.cpp
View File

@ -20,6 +20,7 @@
#include "storage/ls/ob_ls.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/tablet/ob_tablet.h"
static const char *TEST_FILE_NAME = "fast_commit_report";
const int64_t TOTAL_FC_ROW_COUNT = 1000000;
@ -565,7 +566,6 @@ TEST_F(ObFastCommitReport, fast_commit_report)
}
std::cout << "Data reading has been finished, you can generate it with command: sudo perf script -i perf.data -F ip,sym -f > data.viz\n";
std::cout << "And finally you can graph it using url: http://tools.obdev.alibaba-inc.com:8888/perf/ \n";
}
}

1
mittest/simple_server/test_mvcc_gc.cpp
View File

@ -25,6 +25,7 @@
#include "storage/ob_relative_table.h"
#include "storage/compaction/ob_tenant_freeze_info_mgr.h"
#include "storage/concurrency_control/ob_multi_version_garbage_collector.h"
#include "storage/tablet/ob_tablet.h"
static const char *TEST_FILE_NAME = "test_mvcc_gc";

1
src/logservice/leader_coordinator/ob_failure_detector.cpp
View File

@ -29,6 +29,7 @@
#include "ob_leader_coordinator.h"
#include "logservice/ob_log_service.h"
#include "observer/ob_server_event_history_table_operator.h"
#include "storage/slog/ob_storage_logger.h"
namespace oceanbase
{

75
src/observer/ob_rpc_processor_simple.cpp
View File

@ -42,6 +42,7 @@
#include "share/ob_ls_id.h"
#include "storage/ddl/ob_tablet_ddl_kv_mgr.h"
#include "storage/ls/ob_ls.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/tx/ob_trans_service.h"
#include "share/ob_tablet_autoincrement_service.h"
#include "share/sequence/ob_sequence_cache.h"
@ -1912,18 +1913,6 @@ int ObHandlePartTransCtxP::process()
return ret;
}
int ObWriteDDLSSTableCommitLogP::process()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(gctx_.ob_service_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid arguments", K(ret), KP(gctx_.ob_service_));
} else {
ret = gctx_.ob_service_->write_ddl_sstable_commit_log(arg_);
}
return ret;
}
int ObFlushLocalOptStatMonitoringInfoP::process()
{
int ret = OB_SUCCESS;
@ -2013,51 +2002,6 @@ int ObRpcRemoteWriteDDLRedoLogP::process()
return ret;
}
int ObRpcRemoteWriteDDLPrepareLogP::process()
{
int ret = OB_SUCCESS;
uint64_t tenant_id = arg_.tenant_id_;
MTL_SWITCH(tenant_id) {
const ObITable::TableKey &table_key = arg_.table_key_;
ObDDLSSTableRedoWriter sstable_redo_writer;
ObLSService *ls_service = MTL(ObLSService*);
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
ObDDLKvMgrHandle ddl_kv_mgr_handle;
if (OB_UNLIKELY(!arg_.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K_(arg));
} else if (OB_FAIL(ls_service->get_ls(arg_.ls_id_, ls_handle, ObLSGetMod::OBSERVER_MOD))) {
LOG_WARN("get ls failed", K(ret), K(arg_));
} else if (OB_FAIL(ls_handle.get_ls()->get_tablet(table_key.tablet_id_, tablet_handle))) {
LOG_WARN("get tablet failed", K(ret));
} else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) {
LOG_WARN("get ddl kv manager failed", K(ret));
} else if (OB_FAIL(sstable_redo_writer.init(arg_.ls_id_, table_key.tablet_id_))) {
LOG_WARN("init sstable redo writer", K(ret), K(table_key));
} else if (FALSE_IT(sstable_redo_writer.set_start_scn(arg_.start_scn_))) {
} else {
SCN prepare_scn;
if (OB_FAIL(sstable_redo_writer.write_prepare_log(table_key,
arg_.table_id_,
arg_.execution_id_,
arg_.ddl_task_id_,
prepare_scn))) {
LOG_WARN("fail to remote write commit log", K(ret), K(table_key), K_(arg));
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_prepare(arg_.start_scn_,
prepare_scn,
arg_.table_id_,
arg_.ddl_task_id_))) {
LOG_WARN("failed to do ddl kv prepare", K(ret), K(arg_));
} else {
result_ = prepare_scn.get_val_for_tx();
}
}
}
return ret;
}
int ObRpcRemoteWriteDDLCommitLogP::process()
{
int ret = OB_SUCCESS;
@ -2083,11 +2027,20 @@ int ObRpcRemoteWriteDDLCommitLogP::process()
LOG_WARN("init sstable redo writer", K(ret), K(table_key));
} else if (FALSE_IT(sstable_redo_writer.set_start_scn(arg_.start_scn_))) {
} else {
// wait in rpc framework may cause rpc timeout, need sync commit via rs @xiajin
if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->wait_ddl_commit(arg_.start_scn_, arg_.prepare_scn_))) {
LOG_WARN("failed to wait ddl kv commit", K(ret), K(arg_));
} else if (OB_FAIL(sstable_redo_writer.write_commit_log(table_key, arg_.prepare_scn_))) {
SCN commit_scn;
if (OB_FAIL(sstable_redo_writer.write_commit_log(table_key,
arg_.table_id_,
arg_.execution_id_,
arg_.ddl_task_id_,
commit_scn))) {
LOG_WARN("fail to remote write commit log", K(ret), K(table_key), K_(arg));
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_commit(arg_.start_scn_,
commit_scn,
arg_.table_id_,
arg_.ddl_task_id_))) {
LOG_WARN("failed to do ddl kv commit", K(ret), K(arg_));
} else {
result_ = commit_scn.get_val_for_tx();
}
}
}

2
src/observer/ob_rpc_processor_simple.h
View File

@ -195,14 +195,12 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_PRE_PROCESS_SERVER, ObPreProcessServerP);
OB_DEFINE_PROCESSOR_S(Srv, OB_PRE_BOOTSTRAP_CREATE_SERVER_WORKING_DIR, ObPreBootstrapCreateServerWorkingDirP);
OB_DEFINE_PROCESSOR_S(Srv, OB_HANDLE_PART_TRANS_CTX, ObHandlePartTransCtxP);
OB_DEFINE_PROCESSOR_S(Srv, OB_SERVER_FLUSH_OPT_STAT_MONITORING_INFO, ObFlushLocalOptStatMonitoringInfoP);
OB_DEFINE_PROCESSOR_S(Srv, OB_WRITE_DDL_SSTABLE_COMMIT_LOG, ObWriteDDLSSTableCommitLogP);
OB_DEFINE_PROCESSOR_S(Srv, OB_SET_MEMBER_LIST, ObRpcSetMemberListP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CREATE_LS, ObRpcCreateLSP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CREATE_TABLET, ObRpcCreateTabletP);
OB_DEFINE_PROCESSOR_S(Srv, OB_BATCH_BROADCAST_SCHEMA, ObBatchBroadcastSchemaP);
OB_DEFINE_PROCESSOR_S(Srv, OB_DROP_TABLET, ObRpcDropTabletP);
OB_DEFINE_PROCESSOR_S(Srv, OB_REMOTE_WRITE_DDL_REDO_LOG, ObRpcRemoteWriteDDLRedoLogP);
OB_DEFINE_PROCESSOR_S(Srv, OB_REMOTE_WRITE_DDL_PREPARE_LOG, ObRpcRemoteWriteDDLPrepareLogP);
OB_DEFINE_PROCESSOR_S(Srv, OB_REMOTE_WRITE_DDL_COMMIT_LOG, ObRpcRemoteWriteDDLCommitLogP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_LS_CAN_OFFLINE, ObRpcCheckLSCanOfflineP);
OB_DEFINE_PROCESSOR_S(Srv, OB_CLEAN_SEQUENCE_CACHE, ObCleanSequenceCacheP);

21
src/observer/ob_service.cpp
View File

@ -2090,27 +2090,6 @@ int ObService::build_ddl_single_replica_request(const ObDDLBuildSingleReplicaReq
return ret;
}
int ObService::write_ddl_sstable_commit_log(const ObDDLWriteSSTableCommitLogArg &arg)
{
int ret = OB_SUCCESS;
LOG_INFO("receive write ddl sstable commit log", K(arg));
if (OB_UNLIKELY(!arg.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(arg));
} else {
// ObDDLSSTableRedoWriter redo_writer;
// const ObITable::TableKey &table_key = arg.table_key_;
// if (OB_FAIL(ObDDLTableMergeTask::commit_ddl_sstable(table_key))) {
// LOG_WARN("fail to commit ddl sstable", K(ret), K(table_key));
// } else if (OB_FAIL(redo_writer.init(table_key.get_partition_key().get_tenant_id(), table_key.tablet_id_))) {
// LOG_WARN("fail to init ObDDLSSTableRedoWriter", K(ret), K(table_key));
// } else if (OB_FAIL(redo_writer.write_commit_log(table_key))) {
// LOG_WARN("fail to write commit log", K(ret), K(table_key));
// }
}
return ret;
}
int ObService::inner_fill_tablet_info_(
const int64_t tenant_id,
const ObTabletID &tablet_id,

1
src/observer/ob_service.h
View File

@ -128,7 +128,6 @@ public:
int switch_schema(const obrpc::ObSwitchSchemaArg &arg, obrpc::ObSwitchSchemaResult &result);
int calc_column_checksum_request(const obrpc::ObCalcColumnChecksumRequestArg &arg, obrpc::ObCalcColumnChecksumRequestRes &res);
int build_ddl_single_replica_request(const obrpc::ObDDLBuildSingleReplicaRequestArg &arg, obrpc::ObDDLBuildSingleReplicaRequestResult &res);
int write_ddl_sstable_commit_log(const obrpc::ObDDLWriteSSTableCommitLogArg &arg);
int stop_partition_write(const obrpc::Int64 &switchover_timestamp, obrpc::Int64 &result);
int check_partition_log(const obrpc::Int64 &switchover_timestamp, obrpc::Int64 &result);
int get_wrs_info(const obrpc::ObGetWRSArg &arg, obrpc::ObGetWRSResult &result);

2
src/observer/ob_srv_xlator_storage.cpp
View File

@ -93,12 +93,10 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) {
RPC_PROCESSOR(ObRpcBroadcastRsListP, gctx_);
RPC_PROCESSOR(ObPreBootstrapCreateServerWorkingDirP, gctx_);
RPC_PROCESSOR(ObRpcBuildDDLSingleReplicaRequestP, gctx_);
RPC_PROCESSOR(ObWriteDDLSSTableCommitLogP, gctx_);
RPC_PROCESSOR(ObRpcFetchTabletAutoincSeqCacheP, gctx_);
RPC_PROCESSOR(ObRpcBatchGetTabletAutoincSeqP, gctx_);
RPC_PROCESSOR(ObRpcBatchSetTabletAutoincSeqP, gctx_);
RPC_PROCESSOR(ObRpcRemoteWriteDDLRedoLogP, gctx_);
RPC_PROCESSOR(ObRpcRemoteWriteDDLPrepareLogP, gctx_);
RPC_PROCESSOR(ObRpcRemoteWriteDDLCommitLogP, gctx_);
RPC_PROCESSOR(ObRpcLSMigrateReplicaP, gctx_);
RPC_PROCESSOR(ObRpcLSAddReplicaP, gctx_);

1
src/observer/report/ob_tenant_meta_checker.cpp
View File

@ -23,6 +23,7 @@
#include "storage/tx_storage/ob_ls_service.h" // ObLSService, ObLSIterator
#include "storage/tx_storage/ob_ls_handle.h" // ObLSHandle
#include "share/ob_tablet_replica_checksum_operator.h" // ObTabletReplicaChecksumItem
#include "storage/tablet/ob_tablet.h" // ObTablet
namespace oceanbase
{

1
src/observer/virtual_table/ob_all_virtual_memstore_info.cpp
View File

@ -14,6 +14,7 @@
#include "observer/virtual_table/ob_all_virtual_memstore_info.h"
#include "storage/memtable/ob_memtable.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/tablet/ob_tablet.h"
using namespace oceanbase::common;
using namespace oceanbase::memtable;

36
src/observer/virtual_table/ob_all_virtual_table_mgr.cpp
View File

@ -152,21 +152,35 @@ int ObAllVirtualTableMgr::get_next_table(ObITable *&table)
SERVER_LOG(WARN, "unexpected invalid tablet", K(ret), K_(tablet_handle));
} else if (OB_FAIL(tablet_handle_.get_obj()->get_memtables(all_tables_, true/*need_active*/))) {
SERVER_LOG(WARN, "fail to get mem tables", K(ret), K(tablet_handle_));
} else if (OB_FAIL(tablet_handle_.get_obj()->get_all_sstables(sst_tables))) {
SERVER_LOG(WARN, "fail to get sstables", K(ret), K(tablet_handle_));
} else {
for (int i = 0; OB_SUCC(ret) && i < sst_tables.count(); ++i) {
if (OB_FAIL(all_tables_.push_back(sst_tables.at(i)))) {
SERVER_LOG(WARN, "fail to push back sstables", K(ret), K(tablet_handle_));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(tablet_handle_.get_obj()->get_all_sstables(sst_tables))) {
SERVER_LOG(WARN, "fail to get sstables", K(ret), K(tablet_handle_));
} else {
for (int i = 0; OB_SUCC(ret) && i < sst_tables.count(); ++i) {
if (OB_FAIL(all_tables_.push_back(sst_tables.at(i)))) {
SERVER_LOG(WARN, "fail to push back sstables", K(ret), K(tablet_handle_));
}
}
}
if (OB_SUCC(ret) && all_tables_.count() > 0) {
table_idx_ = 0;
table = all_tables_.at(table_idx_);
++table_idx_;
break;
}
if (OB_SUCC(ret)) {
if (OB_FAIL(tablet_handle_.get_obj()->get_ddl_memtables(sst_tables))) {
SERVER_LOG(WARN, "fail to get ddl memtables", K(ret), K(tablet_handle_));
} else {
for (int i = 0; OB_SUCC(ret) && i < sst_tables.count(); ++i) {
if (OB_FAIL(all_tables_.push_back(sst_tables.at(i)))) {
SERVER_LOG(WARN, "fail to push back sstables", K(ret), K(tablet_handle_));
}
}
}
}
if (OB_SUCC(ret) && all_tables_.count() > 0) {
table_idx_ = 0;
table = all_tables_.at(table_idx_);
++table_idx_;
break;
}
}
}
return ret;

5
src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.cpp
View File

@ -12,6 +12,7 @@
#include "observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
#include "storage/ddl/ob_tablet_ddl_kv_mgr.h"
using namespace oceanbase::common;
@ -135,7 +136,9 @@ int ObAllVirtualTabletDDLKVInfo::get_next_ddl_kv(ObDDLKV *&ddl_kv)
}
if (OB_SUCC(ret) && ddl_kv_idx_ >= 0 && ddl_kv_idx_ < ddl_kvs_handle_.get_count()) {
if (OB_FAIL(ddl_kvs_handle_.get_ddl_kv(ddl_kv_idx_, ddl_kv))) {
ddl_kv = static_cast<ObDDLKV *>(ddl_kvs_handle_.get_table(ddl_kv_idx_));
if (OB_ISNULL(ddl_kv)) {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "fail to get ddl kv", K(ret), K(ddl_kv_idx_));
} else {
ddl_kv_idx_++;

2
src/observer/virtual_table/ob_all_virtual_tablet_ddl_kv_info.h
View File

@ -48,7 +48,7 @@ private:
int64_t ls_id_;
ObSharedGuard<storage::ObLSIterator> ls_iter_guard_;
storage::ObLSTabletIterator ls_tablet_iter_;
storage::ObDDLKVsHandle ddl_kvs_handle_;
ObTablesHandleArray ddl_kvs_handle_;
common::ObTabletID curr_tablet_id_;
int64_t ddl_kv_idx_;
char ip_buf_[common::OB_IP_STR_BUFF];

1
src/observer/virtual_table/ob_all_virtual_tx_data_table.cpp
View File

@ -13,6 +13,7 @@
#include "observer/virtual_table/ob_all_virtual_tx_data_table.h"
#include "observer/ob_server.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/tablet/ob_tablet.h"
using namespace oceanbase::common;
using namespace oceanbase::memtable;

2
src/rootserver/ddl_task/ob_column_redefinition_task.cpp
View File

@ -149,7 +149,7 @@ int ObColumnRedefinitionTask::wait_data_complement(const ObDDLTaskStatus next_ta
DEBUG_SYNC(COLUMN_REDEFINITION_REPLICA_BUILD);
if (is_build_replica_end) {
ret = complete_sstable_job_ret_code_;
if (OB_SUCC(ret) && OB_FAIL(check_data_dest_tables_columns_checksum(1/*execution_id*/))) {
if (OB_SUCC(ret) && OB_FAIL(check_data_dest_tables_columns_checksum(1))) {
LOG_WARN("fail to check the columns checkum between data table and hidden one", K(ret));
}
if (OB_FAIL(switch_status(next_task_status, true, ret))) {

4
src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp
View File

@ -671,9 +671,9 @@ int ObDDLRedefinitionTask::check_data_dest_tables_columns_checksum(const int64_t
} else if (OB_UNLIKELY(OB_INVALID_ID == target_object_id_ || !dest_table_column_checksums.created())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), "dest_table_id", target_object_id_, K(dest_table_column_checksums.created()));
} else if (OB_FAIL(ObDDLChecksumOperator::get_table_column_checksum(tenant_id_, execution_id, object_id_, task_id_, data_table_column_checksums, GCTX.root_service_->get_sql_proxy()))) {
} else if (OB_FAIL(ObDDLChecksumOperator::get_table_column_checksum(tenant_id_, execution_id, object_id_, task_id_, false/*replica build*/, data_table_column_checksums, GCTX.root_service_->get_sql_proxy()))) {
LOG_WARN("fail to get table column checksum", K(ret), K(execution_id), "table_id", object_id_, K_(task_id), K(data_table_column_checksums.created()), KP(GCTX.root_service_));
} else if (OB_FAIL(ObDDLChecksumOperator::get_table_column_checksum(tenant_id_, execution_id, target_object_id_, task_id_, dest_table_column_checksums, GCTX.root_service_->get_sql_proxy()))) {
} else if (OB_FAIL(ObDDLChecksumOperator::get_table_column_checksum(tenant_id_, execution_id, target_object_id_, task_id_, false /*replica build*/, dest_table_column_checksums, GCTX.root_service_->get_sql_proxy()))) {
LOG_WARN("fail to get table column checksum", K(ret), K(execution_id), "table_id", target_object_id_, K_(task_id), K(dest_table_column_checksums.created()), KP(GCTX.root_service_));
} else {
uint64_t dest_column_id = 0;

4
src/rootserver/ddl_task/ob_index_build_task.cpp
View File

@ -830,7 +830,7 @@ int ObIndexBuildTask::wait_data_complement()
if (OB_SUCC(ret) && state_finished && !create_index_arg_.is_spatial_index()) {
bool dummy_equal = false;
if (OB_FAIL(ObDDLChecksumOperator::check_column_checksum(
tenant_id_, execution_id_, object_id_, index_table_id_, task_id_, dummy_equal, root_service_->get_sql_proxy()))) {
tenant_id_, execution_id_, object_id_, index_table_id_, task_id_, false/*index build*/, dummy_equal, root_service_->get_sql_proxy()))) {
if (OB_ITER_END != ret) {
LOG_WARN("fail to check column checksum", K(ret), K(index_table_id_), K(object_id_), K(task_id_));
state_finished = true;
@ -986,7 +986,7 @@ int ObIndexBuildTask::verify_checksum()
// do nothing
} else {
if (OB_FAIL(ObDDLChecksumOperator::check_column_checksum(
tenant_id_, execution_id_, object_id_, index_table_id_, task_id_, dummy_equal, root_service_->get_sql_proxy()))) {
tenant_id_, execution_id_, object_id_, index_table_id_, task_id_, true/*check unique index*/, dummy_equal, root_service_->get_sql_proxy()))) {
if (OB_CHECKSUM_ERROR == ret && is_unique_index_) {
ret = OB_ERR_DUPLICATED_UNIQUE_KEY;
}

1
src/rootserver/ob_rs_async_rpc_proxy.h
View File

@ -49,7 +49,6 @@ RPC_F(obrpc::OB_DETECT_MASTER_RS_LS, obrpc::ObDetectMasterRsArg,
obrpc::ObDetectMasterRsLSResult, ObDetectMasterRsLSProxy);
RPC_F(obrpc::OB_GET_ROOT_SERVER_ROLE, obrpc::ObDetectMasterRsArg,
obrpc::ObGetRootserverRoleResult, ObGetRootserverRoleProxy);
RPC_F(obrpc::OB_WRITE_DDL_SSTABLE_COMMIT_LOG, obrpc::ObDDLWriteSSTableCommitLogArg, obrpc::ObDDLWriteCommitLogResult, ObWriteDDLSSTableCommitLogProxy);
RPC_F(obrpc::OB_CREATE_LS, obrpc::ObCreateLSArg, obrpc::ObCreateLSResult, ObLSCreatorProxy);
RPC_F(obrpc::OB_CREATE_TABLET, obrpc::ObBatchCreateTabletArg, obrpc::ObCreateTabletBatchRes, ObTabletCreatorProxy);
RPC_F(obrpc::OB_SET_MEMBER_LIST, obrpc::ObSetMemberListArgV2, obrpc::ObSetMemberListResult, ObSetMemberListProxy);

13
src/share/ob_ddl_checksum.cpp
View File

@ -382,6 +382,7 @@ int ObDDLChecksumOperator::get_table_column_checksum(
const int64_t execution_id,
const uint64_t table_id,
const int64_t ddl_task_id,
const bool is_unique_index_checking,
common::hash::ObHashMap<int64_t, int64_t> &column_checksum_map,
ObMySQLProxy &sql_proxy)
{
@ -395,10 +396,10 @@ int ObDDLChecksumOperator::get_table_column_checksum(
K(column_checksum_map.created()));
} else if (OB_FAIL(sql.assign_fmt(
"SELECT column_id, checksum FROM %s "
"WHERE execution_id = %ld AND tenant_id = %ld AND table_id = %ld AND ddl_task_id = %ld "
"WHERE execution_id = %ld AND tenant_id = %ld AND table_id = %ld AND ddl_task_id = %ld AND task_id %s "
"ORDER BY column_id", OB_ALL_DDL_CHECKSUM_TNAME,
execution_id, ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id),
ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), ddl_task_id))) {
ObSchemaUtils::get_extract_schema_id(exec_tenant_id, table_id), ddl_task_id, is_unique_index_checking ? "< 0" : ">= 0"))) {
LOG_WARN("fail to assign fmt", K(ret));
} else if (OB_FAIL(get_column_checksum(sql, tenant_id, column_checksum_map, sql_proxy))) {
LOG_WARN("fail to get column checksum", K(ret), K(sql));
@ -412,6 +413,7 @@ int ObDDLChecksumOperator::check_column_checksum(
const uint64_t data_table_id,
const uint64_t index_table_id,
const int64_t ddl_task_id,
const bool is_unique_index_checking,
bool &is_equal,
common::ObMySQLProxy &sql_proxy)
{
@ -427,10 +429,11 @@ int ObDDLChecksumOperator::check_column_checksum(
LOG_WARN("fail to create column checksum map", K(ret));
} else if (OB_FAIL(index_table_column_checksums.create(OB_MAX_COLUMN_NUMBER / 2, ObModIds::OB_SSTABLE_CREATE_INDEX))) {
LOG_WARN("fail to create column checksum map", K(ret));
} else if (OB_FAIL(get_table_column_checksum(tenant_id, execution_id, data_table_id, ddl_task_id, data_table_column_checksums, sql_proxy))) {
} else if (OB_FAIL(get_table_column_checksum(tenant_id, execution_id, data_table_id,
ddl_task_id, is_unique_index_checking, data_table_column_checksums, sql_proxy))) {
LOG_WARN("fail to get table column checksum", K(ret), K(execution_id), K(data_table_id), K(ddl_task_id));
} else if (OB_FAIL(get_table_column_checksum(tenant_id, execution_id, index_table_id, ddl_task_id, index_table_column_checksums,
sql_proxy))) {
} else if (OB_FAIL(get_table_column_checksum(tenant_id, execution_id, index_table_id,
ddl_task_id, is_unique_index_checking, index_table_column_checksums, sql_proxy))) {
LOG_WARN("fail to get table column checksum", K(ret), K(execution_id), K(index_table_id), K(ddl_task_id));
} else {
for (hash::ObHashMap<int64_t, int64_t>::const_iterator iter = index_table_column_checksums.begin();

2
src/share/ob_ddl_checksum.h
View File

@ -70,6 +70,7 @@ public:
const int64_t execution_id,
const uint64_t table_id,
const int64_t ddl_task_id,
const bool is_unique_index_checking,
common::hash::ObHashMap<int64_t, int64_t> &column_checksums, common::ObMySQLProxy &sql_proxy);
static int get_tablet_checksum_record(
const uint64_t tenant_id,
@ -85,6 +86,7 @@ public:
const uint64_t data_table_id,
const uint64_t index_table_id,
const int64_t ddl_task_id,
const bool is_unique_index_checking,
bool &is_equal,
common::ObMySQLProxy &sql_proxy);
static int delete_checksum(

47
src/share/ob_rpc_struct.cpp
View File

@ -6441,21 +6441,6 @@ int ObDropDirectoryArg::assign(const ObDropDirectoryArg &other)
OB_SERIALIZE_MEMBER((ObDropDirectoryArg, ObDDLArg), tenant_id_, directory_name_);
OB_SERIALIZE_MEMBER(ObDDLWriteSSTableCommitLogArg, table_key_);
int ObDDLWriteSSTableCommitLogArg::assign(const ObDDLWriteSSTableCommitLogArg &other)
{
int ret = OB_SUCCESS;
table_key_ = other.table_key_;
return ret;
}
OB_SERIALIZE_MEMBER(ObDDLWriteCommitLogResult, ret_code_);
int ObDDLWriteCommitLogResult::assign(const ObDDLWriteCommitLogResult &other)
{
int ret = OB_SUCCESS;
ret_code_ = other.ret_code_;
return ret;
}
bool ObCreateLSArg::is_valid() const
@ -7380,12 +7365,12 @@ int ObRpcRemoteWriteDDLRedoLogArg::init(const uint64_t tenant_id,
OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLRedoLogArg, tenant_id_, ls_id_, redo_info_);
ObRpcRemoteWriteDDLPrepareLogArg::ObRpcRemoteWriteDDLPrepareLogArg()
ObRpcRemoteWriteDDLCommitLogArg::ObRpcRemoteWriteDDLCommitLogArg()
: tenant_id_(OB_INVALID_ID), ls_id_(), table_key_(), start_scn_(SCN::min_scn()),
table_id_(0), execution_id_(-1), ddl_task_id_(0)
{}
int ObRpcRemoteWriteDDLPrepareLogArg::init(const uint64_t tenant_id,
int ObRpcRemoteWriteDDLCommitLogArg::init(const uint64_t tenant_id,
const share::ObLSID &ls_id,
const storage::ObITable::TableKey &table_key,
const SCN &start_scn,
@ -7411,35 +7396,9 @@ int ObRpcRemoteWriteDDLPrepareLogArg::init(const uint64_t tenant_id,
return ret;
}
OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLPrepareLogArg, tenant_id_, ls_id_, table_key_, start_scn_,
OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLCommitLogArg, tenant_id_, ls_id_, table_key_, start_scn_,
table_id_, execution_id_, ddl_task_id_);
ObRpcRemoteWriteDDLCommitLogArg::ObRpcRemoteWriteDDLCommitLogArg()
: tenant_id_(OB_INVALID_ID), ls_id_(), table_key_(), start_scn_(SCN::min_scn()), prepare_scn_(SCN::min_scn())
{}
int ObRpcRemoteWriteDDLCommitLogArg::init(const uint64_t tenant_id,
const share::ObLSID &ls_id,
const storage::ObITable::TableKey &table_key,
const SCN &start_scn,
const SCN &prepare_scn)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(tenant_id == OB_INVALID_ID || !ls_id.is_valid() || !table_key.is_valid() || !start_scn.is_valid_and_not_min()
|| !prepare_scn.is_valid_and_not_min())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tablet id is not valid", K(ret), K(tenant_id), K(ls_id), K(table_key), K(start_scn), K(prepare_scn));
} else {
tenant_id_ = tenant_id;
ls_id_ = ls_id;
table_key_ = table_key;
start_scn_ = start_scn;
prepare_scn_ = prepare_scn;
}
return ret;
}
OB_SERIALIZE_MEMBER(ObRpcRemoteWriteDDLCommitLogArg, tenant_id_, ls_id_, table_key_, start_scn_, prepare_scn_);
bool ObCheckLSCanOfflineArg::is_valid() const
{

62
src/share/ob_rpc_struct.h
View File

@ -7961,34 +7961,6 @@ public:
int progress_;
};
struct ObDDLWriteSSTableCommitLogArg final
{
OB_UNIS_VERSION(1);
public:
ObDDLWriteSSTableCommitLogArg()
: table_key_()
{}
~ObDDLWriteSSTableCommitLogArg() = default;
bool is_valid() const { return table_key_.is_valid(); };
int assign(const ObDDLWriteSSTableCommitLogArg &other);
TO_STRING_KV(K_(table_key));
public:
storage::ObITable::TableKey table_key_;
};
struct ObDDLWriteCommitLogResult final
{
OB_UNIS_VERSION(1);
public:
ObDDLWriteCommitLogResult(): ret_code_(OB_SUCCESS)
{}
~ObDDLWriteCommitLogResult() = default;
int assign(const ObDDLWriteCommitLogResult &other);
TO_STRING_KV(K_(ret_code));
public:
int64_t ret_code_;
};
struct ObDetectMasterRsArg
{
@ -8108,12 +8080,12 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLRedoLogArg);
};
struct ObRpcRemoteWriteDDLPrepareLogArg final
struct ObRpcRemoteWriteDDLCommitLogArg final
{
OB_UNIS_VERSION(1);
public:
ObRpcRemoteWriteDDLPrepareLogArg();
~ObRpcRemoteWriteDDLPrepareLogArg() = default;
ObRpcRemoteWriteDDLCommitLogArg();
~ObRpcRemoteWriteDDLCommitLogArg() = default;
int init(const uint64_t tenant_id,
const share::ObLSID &ls_id,
const storage::ObITable::TableKey &table_key,
@ -8136,37 +8108,11 @@ public:
int64_t table_id_;
int64_t execution_id_;
int64_t ddl_task_id_;
private:
DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLPrepareLogArg);
};
struct ObRpcRemoteWriteDDLCommitLogArg final
{
OB_UNIS_VERSION(1);
public:
ObRpcRemoteWriteDDLCommitLogArg();
~ObRpcRemoteWriteDDLCommitLogArg() = default;
int init(const uint64_t tenant_id,
const share::ObLSID &ls_id,
const storage::ObITable::TableKey &table_key,
const share::SCN &start_scn,
const share::SCN &prepare_scn);
bool is_valid() const
{
return tenant_id_ != OB_INVALID_ID && ls_id_.is_valid() && table_key_.is_valid() && start_scn_.is_valid_and_not_min()
&& prepare_scn_.is_valid_and_not_min();
}
TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(table_key), K_(start_scn), K_(prepare_scn));
public:
uint64_t tenant_id_;
share::ObLSID ls_id_;
storage::ObITable::TableKey table_key_;
share::SCN start_scn_;
share::SCN prepare_scn_;
private:
DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLCommitLogArg);
};
struct ObCheckLSCanOfflineArg
{
OB_UNIS_VERSION(1);

2
src/share/ob_srv_rpc_proxy.h
View File

@ -128,7 +128,6 @@ public:
RPC_S(PR5 fetch_tablet_autoinc_seq_cache, OB_FETCH_TABLET_AUTOINC_SEQ_CACHE, (obrpc::ObFetchTabletSeqArg), obrpc::ObFetchTabletSeqRes);
RPC_AP(PR5 batch_get_tablet_autoinc_seq, OB_BATCH_GET_TABLET_AUTOINC_SEQ, (obrpc::ObBatchGetTabletAutoincSeqArg), obrpc::ObBatchGetTabletAutoincSeqRes);
RPC_AP(PR5 batch_set_tablet_autoinc_seq, OB_BATCH_SET_TABLET_AUTOINC_SEQ, (obrpc::ObBatchSetTabletAutoincSeqArg), obrpc::ObBatchSetTabletAutoincSeqRes);
RPC_AP(PR5 write_ddl_sstable_commit_log, OB_WRITE_DDL_SSTABLE_COMMIT_LOG, (obrpc::ObDDLWriteSSTableCommitLogArg), obrpc::ObDDLWriteCommitLogResult);
RPC_S(PRD force_create_sys_table, OB_FORCE_CREATE_SYS_TABLE, (ObForceCreateSysTableArg));
RPC_S(PRD schema_revise, OB_SCHEMA_REVISE, (ObSchemaReviseArg));
RPC_S(PRD force_set_locality, OB_FORCE_SET_LOCALITY, (ObForceSetLocalityArg));
@ -172,7 +171,6 @@ public:
RPC_S(PR4 admin_remove_lock_op, OB_REMOVE_OBJ_LOCK, (transaction::tablelock::ObAdminRemoveLockOpArg));
RPC_S(PR4 admin_update_lock_op, OB_UPDATE_OBJ_LOCK, (transaction::tablelock::ObAdminUpdateLockOpArg));
RPC_S(PR5 remote_write_ddl_redo_log, OB_REMOTE_WRITE_DDL_REDO_LOG, (obrpc::ObRpcRemoteWriteDDLRedoLogArg));
RPC_S(PR5 remote_write_ddl_prepare_log, OB_REMOTE_WRITE_DDL_PREPARE_LOG, (obrpc::ObRpcRemoteWriteDDLPrepareLogArg), obrpc::Int64);
RPC_S(PR5 remote_write_ddl_commit_log, OB_REMOTE_WRITE_DDL_COMMIT_LOG, (obrpc::ObRpcRemoteWriteDDLCommitLogArg), obrpc::Int64);
RPC_S(PR5 check_ls_can_offline, OB_CHECK_LS_CAN_OFFLINE, (obrpc::ObCheckLSCanOfflineArg));
RPC_S(PR5 clean_sequence_cache, obrpc::OB_CLEAN_SEQUENCE_CACHE, (obrpc::UInt64));

1
src/share/ob_tablet_autoincrement_service.cpp
View File

@ -19,6 +19,7 @@
#include "share/ob_rpc_struct.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "share/scn.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{

2
src/storage/CMakeLists.txt
View File

@ -379,6 +379,7 @@ ob_set_subtarget(ob_storage ddl
ddl/ob_ddl_struct.cpp
ddl/ob_direct_insert_sstable_ctx.cpp
ddl/ob_tablet_barrier_log.cpp
ddl/ob_tablet_ddl_kv.cpp
ddl/ob_tablet_ddl_kv_mgr.cpp
ddl/ob_ddl_heart_beat_task.cpp
ddl/ob_ddl_server_client.cpp
@ -480,7 +481,6 @@ ob_set_subtarget(ob_storage memtable
)
ob_set_subtarget(ob_storage memtable_mvcc
memtable/mvcc/ob_keybtree.cpp
memtable/mvcc/ob_multi_version_iterator.cpp
memtable/mvcc/ob_mvcc_ctx.cpp
memtable/mvcc/ob_mvcc_engine.cpp

2
src/storage/access/ob_block_sample_iterator.cpp
View File

@ -404,7 +404,7 @@ int ObBlockSampleRangeIterator::init_and_push_endkey_iterator(ObGetTableParam &g
} else {
STORAGE_LOG(WARN, "Fail to get next table iter", K(ret), K(get_table_param.tablet_iter_.table_iter_));
}
} else if (!table->is_sstable()) {
} else if (!table->is_sstable() || table->is_ddl_mem_sstable()) {
} else if (OB_ISNULL(table) || OB_ISNULL(sample_range_) || OB_ISNULL(allocator_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected null sstable", K(ret), KP(table), KP(sample_range_), KP(allocator_));

6
src/storage/access/ob_index_tree_prefetcher.cpp
View File

@ -71,7 +71,7 @@ int ObIndexTreePrefetcher::init(
access_ctx_ = &access_ctx;
iter_param_ = &iter_param;
index_read_info_ = iter_param.get_full_read_info()->get_index_read_info();
data_version_ = sstable_->is_major_sstable() ? sstable_->get_snapshot_version() : sstable_->get_key().get_end_scn().get_val_for_tx();
data_version_ = sstable_->get_data_version();
data_block_cache_ = &(OB_STORE_CACHE.get_block_cache());
index_block_cache_ = &(OB_STORE_CACHE.get_index_block_cache());
is_inited_ = true;
@ -98,7 +98,7 @@ int ObIndexTreePrefetcher::switch_context(
sstable_ = &sstable;
access_ctx_ = &access_ctx;
index_read_info_ = &index_read_info;
data_version_ = sstable_->is_major_sstable() ? sstable_->get_snapshot_version() : sstable_->get_key().get_end_scn().get_val_for_tx();
data_version_ = sstable_->get_data_version();
if (!is_rescan_) {
is_rescan_ = true;
for (int64_t i = 0; i < DEFAULT_GET_MICRO_DATA_HANDLE_CNT; ++i) {
@ -795,7 +795,7 @@ int ObIndexTreeMultiPassPrefetcher::init_basic_info(
int32_t range_count = 0;
sstable_ = &sstable;
access_ctx_ = &access_ctx;
data_version_ = sstable_->is_major_sstable() ? sstable_->get_snapshot_version() : sstable_->get_key().get_end_scn().get_val_for_tx();
data_version_ = sstable_->get_data_version();
cur_level_ = 0;
iter_type_ = iter_type;
index_tree_height_ = sstable_->get_meta().get_index_tree_height();

2
src/storage/backup/ob_backup_reader.cpp
View File

@ -590,7 +590,7 @@ int ObSSTableMetaBackupReader::get_meta_data(blocksstable::ObBufferReader &buffe
ObBackupSSTableMeta backup_sstable_meta;
backup_sstable_meta.tablet_id_ = tablet_id_;
if ((backup_data_type_.is_major_backup() && !table_ptr->is_major_sstable())
|| (backup_data_type_.is_minor_backup() && !table_ptr->is_minor_sstable() && !table_ptr->is_ddl_sstable())) {
|| (backup_data_type_.is_minor_backup() && !table_ptr->is_minor_sstable() && !table_ptr->is_ddl_dump_sstable())) {
ret = OB_ERR_SYS;
LOG_WARN("get incorrect table type", K(ret), K(i), K_(backup_data_type), KP(table_ptr));
} else if (FALSE_IT(sstable_ptr = static_cast<ObSSTable *>(table_ptr))) {

2
src/storage/backup/ob_backup_utils.cpp
View File

@ -283,7 +283,7 @@ int ObBackupUtils::check_tablet_ddl_sstable_validity_(const storage::ObTabletHan
} else if (OB_ISNULL(last_table_ptr = ddl_sstable_array.at(ddl_sstable_array.count() - 1))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid table ptr", K(ret), K(ddl_sstable_array));
} else if (!last_table_ptr->is_ddl_sstable()) {
} else if (!last_table_ptr->is_ddl_dump_sstable()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table ptr not correct", K(ret), KPC(last_table_ptr));
} else {

18
src/storage/blocksstable/ob_datum_rowkey.h
View File

@ -37,7 +37,7 @@ public:
OB_INLINE bool is_valid() const { return nullptr != datums_ && datum_cnt_ > 0; }
OB_INLINE bool is_memtable_valid() const { return store_rowkey_.is_valid() && is_valid(); }
OB_INLINE int32_t get_datum_cnt() const { return datum_cnt_; }
OB_INLINE const ObStorageDatum *get_datum_ptr() { return datums_; }
OB_INLINE const ObStorageDatum *get_datum_ptr() const { return datums_; }
OB_INLINE const ObStorageDatum& get_datum(const int64_t idx) const { OB_ASSERT(idx < datum_cnt_); return datums_[idx]; }
OB_INLINE int64_t get_deep_copy_size() const;
OB_INLINE int deep_copy(ObDatumRowkey &dest, common::ObIAllocator &allocator) const;
@ -127,13 +127,19 @@ private:
struct ObDatumRowkeyWrapper
{
public:
ObDatumRowkeyWrapper() = delete;
ObDatumRowkeyWrapper(const ObDatumRowkey &rowkey, const ObStorageDatumUtils &datum_utils)
ObDatumRowkeyWrapper() : rowkey_(nullptr), datum_utils_(nullptr)
{}
ObDatumRowkeyWrapper(const ObDatumRowkey *rowkey, const ObStorageDatumUtils *datum_utils)
: rowkey_(rowkey), datum_utils_(datum_utils)
{}
TO_STRING_KV(K_(rowkey), K_(datum_utils));
const ObDatumRowkey &rowkey_;
const ObStorageDatumUtils &datum_utils_;
bool is_valid() const { return nullptr != rowkey_ && nullptr != datum_utils_; }
const ObDatumRowkey *get_rowkey() const { return rowkey_; }
int compare(const ObDatumRowkeyWrapper &other, int &cmp) const { return rowkey_->compare(*(other.get_rowkey()), *datum_utils_, cmp); }
const ObStorageDatum *get_ptr() const { return rowkey_->get_datum_ptr(); }
const char *repr() const { return to_cstring(rowkey_); }
TO_STRING_KV(KPC_(rowkey), KPC_(datum_utils));
const ObDatumRowkey *rowkey_;
const ObStorageDatumUtils *datum_utils_;
};

2
src/storage/blocksstable/ob_imicro_block_reader.h
View File

@ -118,6 +118,7 @@ struct ObMicroBlockData
{
DATA_BLOCK,
INDEX_BLOCK,
DDL_BLOCK_TREE,
MAX_TYPE
};
public:
@ -140,6 +141,7 @@ public:
int64_t &get_extra_size() { return extra_size_; }
int64_t total_size() const { return size_ + extra_size_; }
bool is_index_block() const { return INDEX_BLOCK == type_ || DDL_BLOCK_TREE == type_;}
void reset() { *this = ObMicroBlockData(); }
OB_INLINE const ObMicroBlockHeader *get_micro_header() const

3
src/storage/blocksstable/ob_index_block_builder.h
View File

@ -378,13 +378,13 @@ public:
int append_macro_row(const ObDataMacroBlockMeta &macro_meta);
int close();
void reset();
private:
static int get_macro_meta(
const char *buf,
const int64_t size,
const MacroBlockId &macro_id,
common::ObIAllocator &allocator,
ObDataMacroBlockMeta *&macro_meta);
private:
static int get_meta_block_and_read_info(
const char *buf,
const int64_t buf_size,
@ -438,6 +438,7 @@ public:
ObMacroMetasArray *&macro_meta_list);
int append_root(ObIndexMicroBlockDesc &root_micro_block_desc);
int close(const int64_t column_cnt, ObSSTableMergeRes &res);
const ObDataStoreDesc &get_index_store_desc() const { return index_store_desc_; }
TO_STRING_KV(K(roots_.count()));
private:
int check_and_rewrite_sstable(ObSSTableMergeRes &res);

178
src/storage/blocksstable/ob_index_block_row_scanner.cpp
View File

@ -14,6 +14,7 @@
#include "share/schema/ob_column_schema.h"
#include "ob_index_block_row_scanner.h"
#include "ob_index_block_row_struct.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
namespace oceanbase
{
@ -207,12 +208,13 @@ int ObIndexBlockDataTransformer::get_reader(
ObIndexBlockRowScanner::ObIndexBlockRowScanner()
: query_range_(nullptr), agg_projector_(nullptr), agg_column_schema_(nullptr),
idx_data_header_(nullptr), macro_id_(), allocator_(nullptr),
micro_reader_helper_(), micro_reader_(nullptr), datum_row_(nullptr), endkey_(),
micro_reader_helper_(), micro_reader_(nullptr),
block_meta_tree_(nullptr), datum_row_(nullptr), endkey_(),
idx_row_parser_(), index_read_info_(nullptr),
current_(ObIMicroBlockReaderInfo::INVALID_ROW_INDEX),
start_(ObIMicroBlockReaderInfo::INVALID_ROW_INDEX),
end_(ObIMicroBlockReaderInfo::INVALID_ROW_INDEX),
step_(1), range_idx_(0), nested_offset_(0), is_transformed_(false), is_get_(false),
step_(1), range_idx_(0), nested_offset_(0), index_format_(IndexFormat::INVALID), is_get_(false),
is_reverse_scan_(false), is_left_border_(false), is_right_border_(false), is_inited_(false)
{}
@ -225,7 +227,8 @@ void ObIndexBlockRowScanner::reuse()
current_ = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
start_ = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
end_ = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
is_transformed_ = false;
block_meta_tree_ = nullptr;
index_format_ = IndexFormat::INVALID;
is_left_border_ = false;
is_right_border_ = false;
}
@ -236,6 +239,7 @@ void ObIndexBlockRowScanner::reset()
idx_data_header_ = nullptr;
micro_reader_helper_.reset();
micro_reader_ = nullptr;
block_meta_tree_ = nullptr;
if (nullptr != datum_row_) {
datum_row_->~ObDatumRow();
if (nullptr != allocator_) {
@ -250,7 +254,7 @@ void ObIndexBlockRowScanner::reset()
step_ = 1;
range_idx_ = 0;
nested_offset_ = 0;
is_transformed_ = false;
index_format_ = IndexFormat::INVALID;
is_get_ = false;
is_reverse_scan_ = false;
is_left_border_ = false;
@ -299,7 +303,7 @@ int ObIndexBlockRowScanner::open(
ret = OB_NOT_INIT;
LOG_WARN("Not inited", K(ret));
} else if (OB_UNLIKELY(!macro_id.is_valid() || !idx_block_data.is_valid() || !rowkey.is_valid()
|| idx_block_data.type_ != ObMicroBlockData::INDEX_BLOCK)) {
|| !idx_block_data.is_index_block())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("Invalid argument to open an index micro block", K(ret),
K(macro_id), K(idx_block_data), K(rowkey));
@ -329,7 +333,7 @@ int ObIndexBlockRowScanner::open(
ret = OB_NOT_INIT;
LOG_WARN("Not inited", K(ret));
} else if (OB_UNLIKELY(!macro_id.is_valid() || !idx_block_data.is_valid() || !range.is_valid()
|| idx_block_data.type_ != ObMicroBlockData::INDEX_BLOCK)) {
|| !idx_block_data.is_index_block())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("Invalid argument to open an index micro block", K(ret), K(idx_block_data), K(range));
} else if (OB_FAIL(init_by_micro_data(idx_block_data))) {
@ -361,22 +365,22 @@ int ObIndexBlockRowScanner::get_next(ObMicroIndexInfo &idx_block_row)
LOG_WARN("Not inited", K(ret));
} else if (end_of_block()) {
ret = OB_ITER_END;
} else if (OB_FAIL(read_curr_idx_row())) {
LOG_WARN("Fail to read currend index row", K(ret), K_(is_transformed), K_(current));
} else if (OB_FAIL(idx_row_parser_.get_header(idx_row_header))) {
LOG_WARN("Fail to get index block row header", K(ret));
} else if (OB_ISNULL(idx_row_header)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected null index block row header", K(ret));
} else if (idx_row_header->is_data_index() && !idx_row_header->is_major_node()) {
if (OB_FAIL(idx_row_parser_.get_minor_meta(idx_minor_info))) {
LOG_WARN("Fail to get minor meta info", K(ret));
} else if (OB_FAIL(read_curr_idx_row(idx_row_header))) {
LOG_WARN("Fail to read currend index row", K(ret), K_(index_format), K_(current));
} else {
if (OB_ISNULL(idx_row_header)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected null index block row header", K(ret), K(index_format_));
} else if (idx_row_header->is_data_index() && !idx_row_header->is_major_node()) {
if (OB_FAIL(idx_row_parser_.get_minor_meta(idx_minor_info))) {
LOG_WARN("Fail to get minor meta info", K(ret));
}
}
}
if (OB_SUCC(ret)) {
idx_block_row.flag_ = 0;
idx_block_row.endkey_ = is_transformed_ ? &idx_data_header_->rowkey_array_[current_] : &endkey_;
idx_block_row.endkey_ = &endkey_;
idx_block_row.row_header_ = idx_row_header;
idx_block_row.minor_meta_info_ = idx_minor_info;
idx_block_row.is_get_ = is_get_;
@ -402,7 +406,7 @@ bool ObIndexBlockRowScanner::end_of_block() const
int ObIndexBlockRowScanner::get_index_row_count(int64_t &index_row_count) const
{
int ret = OB_SUCCESS;
index_row_count = 0;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("Not inited", K(ret));
@ -422,6 +426,8 @@ int ObIndexBlockRowScanner::check_blockscan(
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("Not init", K(ret));
} else if (IndexFormat::BLOCK_TREE == index_format_) {
can_blockscan = false;
} else if (is_reverse_scan_) {
if (rowkey.is_min_rowkey()) {
can_blockscan = true;
@ -431,7 +437,7 @@ int ObIndexBlockRowScanner::check_blockscan(
}
} else {
int cmp_ret = 0;
if (!is_transformed_) {
if (IndexFormat::RAW_DATA == index_format_) {
ObDatumRowkey last_endkey;
ObDatumRow tmp_datum_row; // Normally will use local datum buf, won't allocate memory
if (OB_FAIL(tmp_datum_row.init(index_read_info_->get_request_count()))) {
@ -460,21 +466,30 @@ int ObIndexBlockRowScanner::check_blockscan(
int ObIndexBlockRowScanner::init_by_micro_data(const ObMicroBlockData &idx_block_data)
{
int ret = OB_SUCCESS;
if (nullptr != idx_block_data.get_extra_buf()) {
idx_data_header_ = reinterpret_cast<const ObIndexBlockDataHeader *>(idx_block_data.get_extra_buf());
if (OB_UNLIKELY(!idx_data_header_->is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Invalid index block data header", K(ret), KPC(idx_data_header_));
if (ObMicroBlockData::INDEX_BLOCK == idx_block_data.type_) {
if (nullptr == idx_block_data.get_extra_buf()) {
if (OB_FAIL(micro_reader_helper_.get_reader(idx_block_data.get_store_type(), micro_reader_))) {
LOG_WARN("Fail to get micro block reader", K(ret),
K(idx_block_data), K(idx_block_data.get_store_type()));
} else if (OB_FAIL(micro_reader_->init(idx_block_data, *index_read_info_))) {
LOG_WARN("Fail to init micro reader", K(ret), K(idx_block_data), KPC(index_read_info_));
} else if (OB_FAIL(init_datum_row())) {
LOG_WARN("Fail to init datum row", K(ret));
} else {
index_format_ = IndexFormat::RAW_DATA;
}
} else {
is_transformed_ = true;
idx_data_header_ = reinterpret_cast<const ObIndexBlockDataHeader *>(idx_block_data.get_extra_buf());
if (OB_UNLIKELY(!idx_data_header_->is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Invalid index block data header", K(ret), KPC(idx_data_header_));
} else {
index_format_ = IndexFormat::TRANSFORMED;
}
}
} else if (OB_FAIL(micro_reader_helper_.get_reader(idx_block_data.get_store_type(), micro_reader_))) {
LOG_WARN("Fail to get micro block reader", K(ret),
K(idx_block_data), K(idx_block_data.get_store_type()));
} else if (OB_FAIL(micro_reader_->init(idx_block_data, *index_read_info_))) {
LOG_WARN("Fail to init micro reader", K(ret), K(idx_block_data), KPC(index_read_info_));
} else if (OB_FAIL(init_datum_row())) {
LOG_WARN("Fail to init datum row", K(ret));
} else if (ObMicroBlockData::DDL_BLOCK_TREE == idx_block_data.type_) {
block_meta_tree_ = reinterpret_cast<ObBlockMetaTree *>(const_cast<char *>(idx_block_data.buf_));
index_format_ = IndexFormat::BLOCK_TREE;
}
return ret;
}
@ -484,7 +499,7 @@ int ObIndexBlockRowScanner::locate_key(const ObDatumRowkey &rowkey)
int ret = OB_SUCCESS;
int64_t begin_idx = -1;
int64_t end_idx = -1;
if (!is_transformed_) {
if (IndexFormat::RAW_DATA == index_format_) {
ObDatumRange range;
range.set_start_key(rowkey);
range.end_key_.set_max_rowkey();
@ -499,7 +514,7 @@ int ObIndexBlockRowScanner::locate_key(const ObDatumRowkey &rowkey)
}
}
LOG_TRACE("Binary search rowkey with micro reader", K(ret), K(range), K(begin_idx), K(rowkey));
} else {
} else if (IndexFormat::TRANSFORMED == index_format_) {
ObDatumComparor<ObDatumRowkey> cmp(index_read_info_->get_datum_utils(), ret);
const ObDatumRowkey *first = idx_data_header_->rowkey_array_;
const ObDatumRowkey *last = idx_data_header_->rowkey_array_ + idx_data_header_->row_cnt_;
@ -513,6 +528,31 @@ int ObIndexBlockRowScanner::locate_key(const ObDatumRowkey &rowkey)
}
LOG_TRACE("Binary search rowkey in transformed block", K(ret), KP(found), KPC(first), KP(last),
K(current_), K(rowkey), KPC(idx_data_header_));
} else if (IndexFormat::BLOCK_TREE == index_format_) {
ObDatumRange range;
range.set_start_key(rowkey);
range.set_end_key(rowkey);
range.set_left_closed();
range.set_right_closed();
if (OB_ISNULL(block_meta_tree_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("block meta tree is null", K(ret));
} else if (OB_FAIL(block_meta_tree_->locate_range(range,
index_read_info_->get_datum_utils(),
true,// is_left_border
true,// is_right_border
begin_idx,
end_idx))) {
if (OB_UNLIKELY(OB_BEYOND_THE_RANGE != ret)) {
LOG_WARN("locate rowkey failed", K(ret), K(range), KPC(index_read_info_));
} else {
current_ = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
ret = OB_SUCCESS; // return OB_ITER_END on get_next() for get
}
}
} else {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported index format", K(ret), K(index_format_));
}
if (OB_SUCC(ret)) {
@ -532,7 +572,7 @@ int ObIndexBlockRowScanner::locate_range(
int64_t begin_idx = -1;
int64_t end_idx = -1;
current_ = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
if (is_transformed_) {
if (IndexFormat::TRANSFORMED == index_format_) {
bool is_begin_equal = false;
ObDatumComparor<ObDatumRowkey> lower_bound_cmp(index_read_info_->get_datum_utils(), ret);
ObDatumComparor<ObDatumRowkey> upper_bound_cmp(index_read_info_->get_datum_utils(), ret, false, false);
@ -591,13 +631,30 @@ int ObIndexBlockRowScanner::locate_range(
}
LOG_TRACE("Locate range in index block by range", K(ret), K(range), K(begin_idx), K(end_idx),
K(is_left_border), K(is_right_border), K_(current), KPC(idx_data_header_));
} else if (OB_FAIL(micro_reader_->locate_range(
range, is_left_border, is_right_border, begin_idx, end_idx, true))) {
if (OB_UNLIKELY(OB_BEYOND_THE_RANGE != ret)) {
LOG_WARN("Fail to locate range with micro reader", K(ret));
} else if (IndexFormat::RAW_DATA == index_format_) {
if (OB_FAIL(micro_reader_->locate_range(
range, is_left_border, is_right_border, begin_idx, end_idx, true))) {
if (OB_UNLIKELY(OB_BEYOND_THE_RANGE != ret)) {
LOG_WARN("Fail to locate range with micro reader", K(ret));
}
} else {
LOG_TRACE("Binary search range with micro reader", K(ret), K(range), K(begin_idx), K(end_idx));
}
} else if (IndexFormat::BLOCK_TREE == index_format_) {
if (OB_ISNULL(block_meta_tree_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("block meta tree is null", K(ret));
} else if (OB_FAIL(block_meta_tree_->locate_range(range,
index_read_info_->get_datum_utils(),
is_left_border,
is_right_border,
begin_idx,
end_idx))) {
LOG_WARN("locate rowkey failed", K(ret), K(range), KPC(index_read_info_));
}
} else {
LOG_TRACE("Binary search range with micro reader", K(ret), K(range), K(begin_idx), K(end_idx));
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported index format", K(ret), K(index_format_));
}
if (OB_SUCC(ret)) {
@ -636,26 +693,45 @@ int ObIndexBlockRowScanner::init_datum_row()
return ret;
}
int ObIndexBlockRowScanner::read_curr_idx_row()
int ObIndexBlockRowScanner::read_curr_idx_row(const ObIndexBlockRowHeader *&idx_row_header)
{
int ret = OB_SUCCESS;
idx_row_header = nullptr;
const int64_t rowkey_column_count = index_read_info_->get_rowkey_count();
if (is_transformed_) {
if (IndexFormat::TRANSFORMED == index_format_) {
const char *idx_data_buf = nullptr;
if (OB_FAIL(idx_data_header_->get_index_data(current_, idx_data_buf))) {
LOG_WARN("Fail to get index data", K(ret), K_(current), KPC_(idx_data_header));
} else if (OB_FAIL(idx_row_parser_.init(idx_data_buf))) {
LOG_WARN("Fail to parse index block row", K(ret), K_(current), KPC(idx_data_header_));
} else if (OB_FAIL(idx_row_parser_.get_header(idx_row_header))) {
LOG_WARN("Fail to get index block row header", K(ret));
} else {
endkey_ = idx_data_header_->rowkey_array_[current_];
}
} else if (OB_ISNULL(datum_row_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected null pointer to index row", K(ret));
} else if (OB_FAIL(micro_reader_->get_row(current_, *datum_row_))) {
LOG_WARN("Fail to read index row from block", K(ret), K(current_));
} else if (OB_FAIL(idx_row_parser_.init(rowkey_column_count, *datum_row_))) {
LOG_WARN("Fail to parser index block row", K(ret), K_(datum_row), K(rowkey_column_count));
} else if (OB_FAIL(endkey_.assign(datum_row_->storage_datums_, rowkey_column_count))) {
LOG_WARN("Fail to assign storage datum to endkey", K(ret), KPC(datum_row_), K(rowkey_column_count));
} else if (IndexFormat::RAW_DATA == index_format_) {
if (OB_ISNULL(datum_row_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected null pointer to index row", K(ret));
} else if (OB_FAIL(micro_reader_->get_row(current_, *datum_row_))) {
LOG_WARN("Fail to read index row from block", K(ret), K(current_));
} else if (OB_FAIL(idx_row_parser_.init(rowkey_column_count, *datum_row_))) {
LOG_WARN("Fail to parser index block row", K(ret), K_(datum_row), K(rowkey_column_count));
} else if (OB_FAIL(idx_row_parser_.get_header(idx_row_header))) {
LOG_WARN("Fail to get index block row header", K(ret));
} else if (OB_FAIL(endkey_.assign(datum_row_->storage_datums_, rowkey_column_count))) {
LOG_WARN("Fail to assign storage datum to endkey", K(ret), KPC(datum_row_), K(rowkey_column_count));
}
} else if (IndexFormat::BLOCK_TREE == index_format_) {
if (OB_ISNULL(block_meta_tree_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("block meta iterator is null", K(ret));
} else if (OB_FAIL(block_meta_tree_->get_index_block_row_header(current_, idx_row_header, endkey_))) {
LOG_WARN("get index block row header failed", K(ret), K(current_));
}
} else {
ret = OB_NOT_SUPPORTED;
LOG_WARN("not supported index format", K(ret), K(index_format_));
}
return ret;
}

14
src/storage/blocksstable/ob_index_block_row_scanner.h
View File

@ -25,6 +25,7 @@ namespace storage
{
struct ObTableIterParam;
struct ObTableAccessContext;
class ObBlockMetaTree;
}
namespace blocksstable
{
@ -119,19 +120,25 @@ public:
TO_STRING_KV(K_(current), K_(start), K_(end), K_(step),
K_(range_idx), K_(is_get), K_(is_reverse_scan),
K_(is_left_border), K_(is_right_border),
K_(is_inited), K_(macro_id), KPC_(idx_data_header), KPC_(index_read_info));
K_(is_inited), K_(index_format), K_(macro_id), KPC_(idx_data_header), KPC_(index_read_info));
private:
int init_by_micro_data(const ObMicroBlockData &idx_block_data);
int locate_key(const ObDatumRowkey &rowkey);
int locate_range(const ObDatumRange &range, const bool is_left_border, const bool is_right_border);
int init_datum_row();
int read_curr_idx_row();
int read_curr_idx_row(const ObIndexBlockRowHeader *&idx_row_header);
private:
union {
const ObDatumRowkey *rowkey_;
const ObDatumRange *range_;
const void *query_range_;
};
enum IndexFormat {
INVALID = 0,
RAW_DATA,
TRANSFORMED,
BLOCK_TREE
};
const ObIArray<int32_t> *agg_projector_;
const ObIArray<share::schema::ObColumnSchemaV2> *agg_column_schema_;
const ObIndexBlockDataHeader *idx_data_header_;
@ -139,6 +146,7 @@ private:
ObIAllocator *allocator_;
ObMicroBlockReaderHelper micro_reader_helper_;
ObIMicroBlockReader *micro_reader_;
storage::ObBlockMetaTree *block_meta_tree_;
ObDatumRow *datum_row_;
ObDatumRowkey endkey_;
ObIndexBlockRowParser idx_row_parser_;
@ -149,7 +157,7 @@ private:
int64_t step_;
int16_t range_idx_;
int64_t nested_offset_;
bool is_transformed_;
IndexFormat index_format_;
bool is_get_;
bool is_reverse_scan_;
bool is_left_border_;

2
src/storage/blocksstable/ob_micro_block_row_getter.cpp
View File

@ -378,7 +378,7 @@ int ObMicroBlockRowGetter::inner_get_row(
param_->tablet_id_,
rowkey,
read_info_->get_datum_utils(),
sstable_->is_major_sstable() ? sstable_->get_snapshot_version() : sstable_->get_key().get_end_scn().get_val_for_tx(),
sstable_->get_data_version(),
sstable_->get_key().table_type_);
if (OB_SUCCESS == OB_STORE_CACHE.get_row_cache().put_row(row_cache_key, row_cache_value)) {
context_->table_store_stat_.row_cache_put_cnt_++;

5
src/storage/blocksstable/ob_row_cache.cpp
View File

@ -137,7 +137,10 @@ int ObRowCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&k
bool ObRowCacheKey::is_valid() const
{
return OB_LIKELY(0 != tenant_id_ && tablet_id_.is_valid() && rowkey_size_ > 0
&& data_version_ > -1 && (ObITable::is_minor_sstable(table_type_) || ObITable::is_major_sstable(table_type_) || ObITable::is_meta_major_sstable(table_type_))
&& data_version_ > -1 && (ObITable::is_minor_sstable(table_type_)
|| ObITable::is_major_sstable(table_type_)
|| ObITable::is_ddl_sstable(table_type_)
|| ObITable::is_meta_major_sstable(table_type_))
&& rowkey_.is_valid());
}

29
src/storage/blocksstable/ob_sstable.cpp
View File

@ -31,6 +31,7 @@
#include "storage/compaction/ob_tenant_tablet_scheduler.h"
#include "storage/ob_tenant_tablet_stat_mgr.h"
#include "storage/blocksstable/ob_shared_macro_block_manager.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
namespace oceanbase
{
@ -1229,15 +1230,27 @@ int ObSSTable::get_last_rowkey(
sstable_endkey = &ObDatumRowkey::MAX_ROWKEY;
} else if (OB_FAIL(get_index_tree_root(index_read_info, root_block))) {
LOG_WARN("Fail to get index tree root", K(ret), K(root_block));
} else if (OB_ISNULL(idx_data_header = reinterpret_cast<const ObIndexBlockDataHeader *>(
root_block.get_extra_buf()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected null extra buf after transform", K(ret), K(root_block));
} else if (OB_UNLIKELY(!idx_data_header->is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Invalid index data header", KP(idx_data_header));
} else {
sstable_endkey = &(idx_data_header->rowkey_array_[idx_data_header->row_cnt_ - 1]);
if (ObMicroBlockData::DDL_BLOCK_TREE == root_block.type_) {
ObBlockMetaTree *block_meta_tree = reinterpret_cast<ObBlockMetaTree *>(const_cast<char *>(root_block.buf_));
if (OB_ISNULL(block_meta_tree)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, block meta tree can not be null", K(ret), K(root_block));
} else if (OB_FAIL(block_meta_tree->get_last_rowkey(sstable_endkey))) {
LOG_WARN("get last rowkey failed", K(ret));
}
} else {
if (OB_ISNULL(idx_data_header = reinterpret_cast<const ObIndexBlockDataHeader *>(
root_block.get_extra_buf()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected null extra buf after transform", K(ret), K(root_block));
} else if (OB_UNLIKELY(!idx_data_header->is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Invalid index data header", KP(idx_data_header));
} else {
sstable_endkey = &(idx_data_header->rowkey_array_[idx_data_header->row_cnt_ - 1]);
}
}
}
return ret;
}

10
src/storage/blocksstable/ob_sstable.h
View File

@ -17,6 +17,7 @@
#include "storage/blocksstable/ob_macro_block_meta.h"
#include "share/scn.h"
#include "ob_datum_range.h"
#include "storage/ddl/ob_ddl_struct.h"
namespace oceanbase
{
@ -129,7 +130,12 @@ public:
{
return meta_.basic_meta_.filled_tx_scn_;
}
int64_t get_data_version() const
{
return is_major_sstable() ? get_snapshot_version() :
is_ddl_sstable() ? get_upper_trans_version() :
get_key().get_end_scn().get_val_for_tx();
}
virtual int get_frozen_schema_version(int64_t &schema_version) const override;
int add_disk_ref();
int dec_disk_ref();
@ -198,7 +204,7 @@ private:
void dec_macro_ref();
int get_last_rowkey(const ObTableReadInfo &index_read_info, const ObDatumRowkey *&sstable_endkey);
private:
protected:
blocksstable::ObSSTableMeta meta_;
bool valid_for_reading_;
bool hold_macro_ref_;

40
src/storage/blocksstable/ob_sstable_meta_info.cpp
View File

@ -43,11 +43,13 @@ bool ObRootBlockInfo::is_valid() const
void ObRootBlockInfo::reset()
{
if (OB_NOT_NULL(block_data_.buf_) && OB_NOT_NULL(block_data_allocator_)) {
block_data_allocator_->free(static_cast<void *>(const_cast<char *>(block_data_.buf_)));
}
if (OB_NOT_NULL(block_data_.extra_buf_) && OB_NOT_NULL(block_data_allocator_)) {
block_data_allocator_->free(static_cast<void *>(const_cast<char *>(block_data_.extra_buf_)));
if (ObMicroBlockData::INDEX_BLOCK == block_data_.type_) {
if (OB_NOT_NULL(block_data_.buf_) && OB_NOT_NULL(block_data_allocator_)) {
block_data_allocator_->free(static_cast<void *>(const_cast<char *>(block_data_.buf_)));
}
if (OB_NOT_NULL(block_data_.extra_buf_) && OB_NOT_NULL(block_data_allocator_)) {
block_data_allocator_->free(static_cast<void *>(const_cast<char *>(block_data_.extra_buf_)));
}
}
block_data_.reset();
block_data_allocator_ = nullptr;
@ -154,23 +156,29 @@ int ObRootBlockInfo::init_root_block_info(
} else if (FALSE_IT(addr_ = addr)) {
} else if (FALSE_IT(block_data_allocator_ = allocator)) {
} else if (!addr.is_memory()) {
// nothing to do.
block_data_.type_ = ObMicroBlockData::INDEX_BLOCK;
} else if (OB_FAIL(addr.get_mem_addr(offset, size))) {
LOG_WARN("fail to get memory address", K(ret), K(addr));
} else if (OB_ISNULL(dst_buf = static_cast<char *>(allocator->alloc(size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc buf", K(ret), K(size));
} else {
MEMCPY(dst_buf, block_data.buf_, size);
block_data_.buf_ = dst_buf;
block_data_.size_ = size;
LOG_DEBUG("block data type", K(block_data.type_));
if (ObMicroBlockData::DDL_BLOCK_TREE == block_data.type_) {
block_data_ = block_data;
} else {
if (OB_ISNULL(dst_buf = static_cast<char *>(allocator->alloc(size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc buf", K(ret), K(size));
} else {
MEMCPY(dst_buf, block_data.buf_, size);
block_data_.buf_ = dst_buf;
block_data_.size_ = size;
block_data_.type_ = ObMicroBlockData::INDEX_BLOCK;
}
}
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(dst_buf)) {
allocator->free(dst_buf);
}
} else {
block_data_.type_ = ObMicroBlockData::INDEX_BLOCK;
}
return ret;
}
@ -210,7 +218,9 @@ int ObRootBlockInfo::load_root_block_data(const ObMicroBlockDesMeta &des_meta)
int ObRootBlockInfo::transform_root_block_data(const ObTableReadInfo &read_info)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(block_data_.get_extra_buf()) && OB_NOT_NULL(block_data_.get_buf())) {
if (ObMicroBlockData::INDEX_BLOCK == block_data_.type_
&& OB_ISNULL(block_data_.get_extra_buf())
&& OB_NOT_NULL(block_data_.get_buf())) {
ObIndexBlockDataTransformer *transformer = nullptr;
ObDecoderAllocator *allocator = nullptr;
if (OB_UNLIKELY(!read_info.is_valid())) {

33
src/storage/blocksstable/ob_sstable_sec_meta_iterator.cpp
View File

@ -17,6 +17,7 @@
#include "ob_sstable_sec_meta_iterator.h"
#include "storage/blocksstable/ob_shared_macro_block_manager.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
namespace oceanbase
{
@ -26,7 +27,7 @@ namespace blocksstable
ObSSTableSecMetaIterator::ObSSTableSecMetaIterator()
: sstable_meta_(nullptr), index_read_info_(nullptr), tenant_id_(OB_INVALID_TENANT_ID),
prefetch_flag_(), idx_cursor_(), macro_reader_(), block_cache_(nullptr),
micro_reader_(nullptr), micro_reader_helper_(),
micro_reader_(nullptr), micro_reader_helper_(), block_meta_tree_(nullptr),
query_range_(nullptr), start_bound_micro_block_(), end_bound_micro_block_(),
micro_handles_(), row_(), curr_handle_idx_(0), prefetch_handle_idx_(0), prev_block_row_cnt_(0),
curr_block_start_idx_(0), curr_block_end_idx_(0), curr_block_idx_(0), step_cnt_(0),
@ -43,6 +44,7 @@ void ObSSTableSecMetaIterator::reset()
block_cache_ = nullptr;
micro_reader_ = nullptr;
micro_reader_helper_.reset();
block_meta_tree_ = nullptr;
row_.reset();
query_range_ = nullptr;
curr_handle_idx_ = 0;
@ -94,6 +96,27 @@ int ObSSTableSecMetaIterator::open(
}
if (OB_FAIL(ret) || is_prefetch_end_) {
} else if (sstable.is_ddl_mem_sstable()) {
const ObMicroBlockData &root_block = sstable.get_meta().get_root_info().get_block_data();
if (ObMicroBlockData::DDL_BLOCK_TREE != root_block.type_ || nullptr == root_block.buf_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("block type is not ddl block tree", K(ret), K(root_block));
} else {
block_meta_tree_ = reinterpret_cast<ObBlockMetaTree *>(const_cast<char *>(root_block.buf_));
if (OB_FAIL(block_meta_tree_->locate_range(query_range,
index_read_info.get_datum_utils(),
true, //is_left_border
true, //is_right_border,
curr_block_start_idx_,
curr_block_end_idx_))) {
LOG_WARN("locate range failed", K(ret));
} else {
const int64_t step = max(1, sample_step);
step_cnt_ = !is_reverse_scan ? step : -step;
curr_block_idx_ = !is_reverse_scan ? curr_block_start_idx_ : curr_block_end_idx_;
is_inited_ = true;
}
}
} else if (OB_FAIL(idx_cursor_.init(sstable, allocator, index_read_info_,
get_index_tree_type_map()[meta_type]))) {
LOG_WARN("Fail to init index block tree cursor", K(ret), K(meta_type));
@ -181,6 +204,14 @@ int ObSSTableSecMetaIterator::get_next(ObDataMacroBlockMeta &macro_meta)
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("Secondary meta iterator not inited", K(ret));
} else if (nullptr != block_meta_tree_) {
if (!is_target_row_in_curr_block()) {
ret = OB_ITER_END;
} else if (OB_FAIL(block_meta_tree_->get_macro_block_meta(curr_block_idx_, macro_meta))) {
LOG_WARN("get next macro block meta failed", K(ret), K(curr_block_idx_));
} else {
curr_block_idx_ += step_cnt_;
}
} else {
while (OB_SUCC(ret) && !is_target_row_in_curr_block()) {
if (is_prefetch_end_ && is_handle_buffer_empty()) {

5
src/storage/blocksstable/ob_sstable_sec_meta_iterator.h
View File

@ -18,6 +18,10 @@
namespace oceanbase
{
namespace storage
{
class ObBlockMetaTree;
}
namespace blocksstable
{
@ -86,6 +90,7 @@ private:
ObDataMicroBlockCache *block_cache_;
ObIMicroBlockReader *micro_reader_;
ObMicroBlockReaderHelper micro_reader_helper_;
storage::ObBlockMetaTree *block_meta_tree_;
const ObDatumRange *query_range_;
ObMicroBlockId start_bound_micro_block_;
ObMicroBlockId end_bound_micro_block_;

1
src/storage/compaction/ob_partition_merger.cpp
View File

@ -20,6 +20,7 @@
#include "ob_i_compaction_filter.h"
#include "storage/tx/ob_trans_service.h"
#include "storage/blocksstable/ob_data_macro_block_merge_writer.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{

3
src/storage/compaction/ob_tablet_merge_ctx.cpp
View File

@ -22,6 +22,7 @@
#include "storage/compaction/ob_tenant_tablet_scheduler.h"
#include "observer/omt/ob_multi_tenant.h"
#include "share/scheduler/ob_dag_warning_history_mgr.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/compaction/ob_medium_compaction_mgr.h"
#include "storage/compaction/ob_medium_compaction_func.h"
#include "src/storage/meta_mem/ob_tenant_meta_mem_mgr.h"
@ -615,7 +616,7 @@ ObITable::TableType ObTabletMergeCtx::get_merged_table_type() const
} else if (META_MAJOR_MERGE == param_.merge_type_) {
table_type = ObITable::TableType::META_MAJOR_SSTABLE;
} else if (DDL_KV_MERGE == param_.merge_type_) {
table_type = ObITable::TableType::KV_DUMP_SSTABLE;
table_type = ObITable::TableType::DDL_DUMP_SSTABLE;
} else { // MINOR_MERGE || HISTORY_MINOR_MERGE
table_type = ObITable::TableType::MINOR_SSTABLE;
}

34
src/storage/compaction/ob_tenant_tablet_scheduler.cpp
View File

@ -36,6 +36,7 @@
#include "storage/compaction/ob_tenant_compaction_progress.h"
#include "storage/compaction/ob_server_compaction_event_history.h"
#include "share/scn.h"
#include "storage/ddl/ob_ddl_merge_task.h"
namespace oceanbase
{
@ -749,6 +750,32 @@ int ObTenantTabletScheduler::schedule_tablet_minor_merge(
return ret;
}
int ObTenantTabletScheduler::schedule_tablet_ddl_major_merge(ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
ObDDLKvMgrHandle kv_mgr_handle;
if (!tablet_handle.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(tablet_handle));
} else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(kv_mgr_handle))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("get ddl kv mgr failed", K(ret), K(tablet_handle));
} else {
ret = OB_SUCCESS;
}
} else if (kv_mgr_handle.is_valid() && kv_mgr_handle.get_obj()->can_schedule_major_compaction()) {
ObDDLTableMergeDagParam param;
if (OB_FAIL(kv_mgr_handle.get_obj()->get_ddl_major_merge_param(param))) {
LOG_WARN("get ddl major merge param failed", K(ret));
} else if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) {
if (OB_SIZE_OVERFLOW != ret && OB_EAGAIN != ret) {
LOG_WARN("schedule ddl merge dag failed", K(ret), K(param));
}
}
}
return ret;
}
template <class T>
int ObTenantTabletScheduler::schedule_merge_execute_dag(
const ObTabletMergeDagParam &param,
@ -844,6 +871,13 @@ int ObTenantTabletScheduler::schedule_ls_minor_merge(
LOG_WARN("failed to schedule tablet merge", K(tmp_ret), K(ls_id), K(tablet_id));
}
}
if (OB_TMP_FAIL(schedule_tablet_ddl_major_merge(tablet_handle))) {
if (OB_SIZE_OVERFLOW != tmp_ret && OB_EAGAIN != tmp_ret) {
LOG_WARN("failed to schedule tablet ddl merge", K(tmp_ret), K(ls_id), K(tablet_handle));
}
}
if (OB_SUCC(ret)) {
need_fast_freeze = false;
if (!fast_freeze_checker_.need_check()) {

3
src/storage/compaction/ob_tenant_tablet_scheduler.h
View File

@ -35,6 +35,7 @@ namespace storage
class ObLS;
class ObTablet;
class ObITable;
class ObTabletDDLKvMgr;
class ObFastFreezeChecker
{
@ -153,6 +154,8 @@ public:
const ObMergeType merge_type,
const int64_t &merge_snapshot_version,
const bool is_tenant_major_merge = false);
static int schedule_tablet_ddl_major_merge(
ObTabletHandle &tablet_handle);
int get_min_dependent_schema_version(int64_t &min_schema_version);

2
src/storage/ddl/ob_build_index_task.cpp
View File

@ -149,7 +149,7 @@ int ObUniqueIndexChecker::scan_table_with_column_checksum(
ObQueryFlag query_flag(ObQueryFlag::Forward,
true, /*is daily merge scan*/
true, /*is read multiple macro block*/
true, /*sys task scan, read one macro block in single io*/
false, /*sys task scan, read one macro block in single io*/
false, /*is full row scan?*/
false,
false);

33
src/storage/ddl/ob_complement_data_task.cpp
View File

@ -868,7 +868,7 @@ int ObComplementWriteTask::do_local_scan()
ObQueryFlag query_flag(ObQueryFlag::Forward,
true, /*is daily merge scan*/
true, /*is read multiple macro block*/
true, /*sys task scan, read one macro block in single io*/
false, /*sys task scan, read one macro block in single io*/
false /*is full row scan?*/,
false,
false);
@ -1200,7 +1200,7 @@ int ObComplementMergeTask::add_build_hidden_table_sstable()
ObTablet *tablet = nullptr;
ObTabletHandle tablet_handle;
ObITable::TableKey hidden_table_key;
SCN prepare_scn;
SCN commit_scn;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObComplementMergetask has not been inited", K(ret));
@ -1224,15 +1224,15 @@ int ObComplementMergeTask::add_build_hidden_table_sstable()
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(context_->data_sstable_redo_writer_.write_prepare_log(hidden_table_key,
param_->hidden_table_schema_->get_table_id(),
param_->execution_id_,
param_->task_id_,
prepare_scn))) {
} else if (OB_FAIL(context_->data_sstable_redo_writer_.write_commit_log(hidden_table_key,
param_->hidden_table_schema_->get_table_id(),
param_->execution_id_,
param_->task_id_,
commit_scn))) {
if (OB_TASK_EXPIRED == ret) {
LOG_INFO("ddl task expired", K(ret), K(hidden_table_key), KPC(param_));
} else {
LOG_WARN("fail write ddl prepare log", K(ret), K(hidden_table_key));
LOG_WARN("fail write ddl commit log", K(ret), K(hidden_table_key));
}
} else {
ObTabletHandle new_tablet_handle; // no use here
@ -1244,18 +1244,15 @@ int ObComplementMergeTask::add_build_hidden_table_sstable()
const int64_t ddl_task_id = param_->task_id_;
if (OB_FAIL(tablet->get_ddl_kv_mgr(ddl_kv_mgr_handle))) {
LOG_WARN("get ddl kv manager failed", K(ret));
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_prepare(ddl_start_scn,
prepare_scn,
table_id,
ddl_task_id))) {
LOG_WARN("commit ddl log failed", K(ret), K(ls_id), K(tablet_id), K(prepare_scn), K(hidden_table_key),
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_commit(ddl_start_scn,
commit_scn,
table_id,
ddl_task_id))) {
LOG_WARN("commit ddl log failed", K(ret), K(ls_id), K(tablet_id), K(commit_scn), K(hidden_table_key),
K(ddl_start_scn), "new_ddl_start_scn", ddl_kv_mgr_handle.get_obj()->get_start_scn());
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->wait_ddl_commit(ddl_start_scn, prepare_scn))) {
LOG_WARN("wait ddl commit failed", K(ret), K(ls_id), K(tablet_id), K(hidden_table_key),
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->wait_ddl_merge_success(ddl_start_scn, commit_scn))) {
LOG_WARN("wait ddl merge failed", K(ret), K(ls_id), K(tablet_id), K(hidden_table_key),
K(ddl_start_scn), "new_ddl_start_scn", ddl_kv_mgr_handle.get_obj()->get_start_scn());
} else if (OB_FAIL(context_->data_sstable_redo_writer_.write_commit_log(hidden_table_key,
prepare_scn))) {
LOG_WARN("fail write ddl commit log", K(ret), K(hidden_table_key));
}
}
return ret;

50
src/storage/ddl/ob_ddl_clog.cpp
View File

@ -18,6 +18,7 @@
#include "storage/meta_mem/ob_tablet_handle.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/ddl/ob_tablet_ddl_kv_mgr.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{
@ -183,23 +184,6 @@ int ObDDLCommitClogCb::init(const share::ObLSID &ls_id,
int ObDDLCommitClogCb::on_success()
{
int ret = OB_SUCCESS;
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
ObDDLKvMgrHandle ddl_kv_mgr_handle;
if (!is_inited_) {
// ddl prepare will not init this cb, so do nothing
} else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) {
LOG_WARN("failed to get log stream", K(ret), K(ls_id_));
} else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle,
tablet_id_,
tablet_handle,
ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) {
LOG_WARN("get tablet handle failed", K(ret), K(ls_id_), K(tablet_id_));
} else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) {
LOG_WARN("get ddl kv manager failed", K(ret), K(ls_id_), K(tablet_id_));
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->unregister_from_tablet(start_scn_, ddl_kv_mgr_handle))) {
LOG_WARN("unregister ddl kv manager from tablet failed", K(ret), K(ls_id_), K(tablet_id_));
}
status_.set_ret_code(ret);
status_.set_state(STATE_SUCCESS);
try_release();
@ -304,13 +288,13 @@ int ObDDLRedoLog::init(const blocksstable::ObDDLMacroBlockRedoInfo &redo_info)
OB_SERIALIZE_MEMBER(ObDDLRedoLog, redo_info_);
ObDDLPrepareLog::ObDDLPrepareLog()
ObDDLCommitLog::ObDDLCommitLog()
: table_key_(), start_scn_(SCN::min_scn())
{
}
int ObDDLPrepareLog::init(const ObITable::TableKey &table_key,
const SCN &start_scn)
int ObDDLCommitLog::init(const ObITable::TableKey &table_key,
const SCN &start_scn)
{
int ret = OB_SUCCESS;
if (!table_key.is_valid() || !start_scn.is_valid_and_not_min()) {
@ -323,32 +307,8 @@ int ObDDLPrepareLog::init(const ObITable::TableKey &table_key,
return ret;
}
OB_SERIALIZE_MEMBER(ObDDLPrepareLog, table_key_, start_scn_);
OB_SERIALIZE_MEMBER(ObDDLCommitLog, table_key_, start_scn_);
ObDDLCommitLog::ObDDLCommitLog()
: table_key_(), start_scn_(SCN::min_scn()), prepare_scn_(SCN::min_scn())
{
}
int ObDDLCommitLog::init(const ObITable::TableKey &table_key,
const SCN &start_scn,
const SCN &prepare_scn)
{
int ret = OB_SUCCESS;
if (!table_key.is_valid() ||
!start_scn.is_valid_and_not_min() ||
!prepare_scn.is_valid_and_not_min()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(table_key), K(start_scn), K(prepare_scn));
} else {
table_key_ = table_key;
start_scn_ = start_scn;
prepare_scn_ = prepare_scn;
}
return ret;
}
OB_SERIALIZE_MEMBER(ObDDLCommitLog, table_key_, start_scn_, prepare_scn_);
ObTabletSchemaVersionChangeLog::ObTabletSchemaVersionChangeLog()
: tablet_id_(), schema_version_(-1)

33
src/storage/ddl/ob_ddl_clog.h
View File

@ -27,10 +27,10 @@ enum class ObDDLClogType : int64_t
{
UNKNOWN = -1,
DDL_REDO_LOG = 0x1,
DDL_COMMIT_LOG = 0x2,
// DDL_COMMIT_LOG = 0x2, // deprecated
DDL_TABLET_SCHEMA_VERSION_CHANGE_LOG = 0x10,
DDL_START_LOG = 0x20,
DDL_PREPARE_LOG = 0x40
DDL_COMMIT_LOG = 0x40,// rename from DDL_PREPARE_LOG
};
enum ObDDLClogState : uint8_t
@ -171,12 +171,12 @@ private:
blocksstable::ObDDLMacroBlockRedoInfo redo_info_;
};
class ObDDLPrepareLog final
class ObDDLCommitLog final
{
OB_UNIS_VERSION_V(1);
public:
ObDDLPrepareLog();
~ObDDLPrepareLog() = default;
ObDDLCommitLog();
~ObDDLCommitLog() = default;
int init(const ObITable::TableKey &table_key,
const share::SCN &start_scn);
bool is_valid() const { return table_key_.is_valid() && start_scn_.is_valid(); }
@ -188,29 +188,6 @@ private:
share::SCN start_scn_;
};
class ObDDLCommitLog final
{
OB_UNIS_VERSION_V(1);
public:
ObDDLCommitLog();
~ObDDLCommitLog() = default;
int init(const ObITable::TableKey &table_key,
const share::SCN &start_scn,
const share::SCN &prepare_scn);
bool is_valid() const {
return table_key_.is_valid() &&
start_scn_.is_valid() &&
prepare_scn_.is_valid(); }
ObITable::TableKey get_table_key() const { return table_key_; }
share::SCN get_start_scn() const { return start_scn_; }
share::SCN get_prepare_scn() const { return prepare_scn_; }
TO_STRING_KV(K_(table_key), K_(start_scn), K_(prepare_scn));
private:
ObITable::TableKey table_key_;
share::SCN start_scn_;
share::SCN prepare_scn_;
};
class ObTabletSchemaVersionChangeLog final
{
public:

187
src/storage/ddl/ob_ddl_merge_task.cpp
View File

@ -94,7 +94,7 @@ int ObDDLTableMergeDag::create_first_task()
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
ObDDLKvMgrHandle ddl_kv_mgr_handle;
ObDDLKVsHandle ddl_kvs_handle;
ObTablesHandleArray ddl_kvs_handle;
ObDDLTableMergeTask *merge_task = nullptr;
if (OB_FAIL(ls_service->get_ls(ddl_param_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) {
LOG_WARN("get ls failed", K(ret), K(ddl_param_));
@ -127,9 +127,10 @@ int ObDDLTableMergeDag::create_first_task()
// use chain task to ensure log ts continuious in table store
ObDDLTableDumpTask *last_task = nullptr;
for (int64_t i = 0; OB_SUCC(ret) && i < ddl_kvs_handle.get_count(); ++i) {
ObDDLKV *ddl_kv = nullptr;
ObDDLKV *ddl_kv = static_cast<ObDDLKV *>(ddl_kvs_handle.get_table(i));
ObDDLTableDumpTask *task = nullptr;
if (OB_FAIL(ddl_kvs_handle.get_ddl_kv(i, ddl_kv))) {
if (OB_ISNULL(ddl_kv)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get ddl kv failed", K(ret), K(i));
} else if (OB_FAIL(alloc_task(task))) {
LOG_WARN("Fail to alloc task", K(ret));
@ -260,14 +261,15 @@ int ObDDLTableDumpTask::process()
LOG_WARN("get ddl kv mgr failed", K(ret), K(ls_id_), K(tablet_id_));
}
} else {
ObDDLKVHandle ddl_kv_handle;
ObTableHandleV2 ddl_kv_handle;
ObDDLKV *ddl_kv = nullptr;
ObTablesHandleArray ddl_sstable_handles;
bool need_compact = false;
ObArray<ObITable *> candidate_sstables;
if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_freezed_ddl_kv(freeze_scn_, ddl_kv_handle))) {
LOG_WARN("get ddl kv handle failed", K(ret), K(freeze_scn_));
} else if (OB_FAIL(ddl_kv_handle.get_ddl_kv(ddl_kv))) {
} else if (OB_ISNULL(ddl_kv = static_cast<ObDDLKV *>(ddl_kv_handle.get_table()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get ddl kv failed", K(ret));
} else if (OB_FAIL(ddl_kv->close())) {
if (OB_EAGAIN != ret) {
@ -357,34 +359,35 @@ int ObDDLTableMergeTask::process()
ObTabletDDLParam ddl_param;
ObTableHandleV2 table_handle;
bool is_data_complete = false;
if (ddl_sstable_handles.empty()) {
const ObSSTable *latest_major_sstable = nullptr;
if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable(merge_param_.ls_id_, merge_param_.tablet_id_, latest_major_sstable))) {
LOG_WARN("check if major sstable exist failed", K(ret));
} else if (nullptr != latest_major_sstable) {
LOG_INFO("major sstable has been created before", K(merge_param_), K(ddl_param.table_key_));
sstable = static_cast<ObSSTable *>(tablet_handle.get_obj()->get_table_store().get_major_sstables().get_boundary_table(false/*first*/));
} else if (tablet_handle.get_obj()->get_tablet_meta().table_store_flag_.with_major_sstable()) {
skip_major_process = true;
LOG_INFO("tablet me says with major but no major, meaning its a migrated deleted tablet, skip");
}
bool is_scn_overlap = false;
const ObSSTable *latest_major_sstable = nullptr;
if (OB_FAIL(ObTabletDDLUtil::check_and_get_major_sstable(merge_param_.ls_id_, merge_param_.tablet_id_, latest_major_sstable))) {
LOG_WARN("check if major sstable exist failed", K(ret));
} else if (nullptr != latest_major_sstable) {
LOG_INFO("major sstable has been created before", K(merge_param_), K(ddl_param.table_key_));
sstable = static_cast<ObSSTable *>(tablet_handle.get_obj()->get_table_store().get_major_sstables().get_boundary_table(false/*first*/));
} else if (tablet_handle.get_obj()->get_tablet_meta().table_store_flag_.with_major_sstable()) {
skip_major_process = true;
LOG_INFO("tablet me says with major but no major, meaning its a migrated deleted tablet, skip");
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_ddl_param(ddl_param))) {
LOG_WARN("get tablet ddl param failed", K(ret));
} else if (merge_param_.start_scn_ > SCN::min_scn() && merge_param_.start_scn_ < ddl_param.start_scn_) {
LOG_INFO("ddl merge task expired, do nothing", K(merge_param_), "new_start_scn", ddl_param.start_scn_);
} else if (merge_param_.is_commit_ && OB_FAIL(check_data_integrity(ddl_sstable_handles,
ddl_param.start_scn_,
merge_param_.rec_scn_,
is_data_complete))) {
} else if (merge_param_.is_commit_ && OB_FAIL(ObTabletDDLUtil::check_data_integrity(ddl_sstable_handles,
ddl_param.start_scn_,
merge_param_.rec_scn_,
is_data_complete,
is_scn_overlap))) {
LOG_WARN("check ddl sstable integrity failed", K(ret), K(ddl_sstable_handles), K(ddl_param), K(merge_param_));
} else if (merge_param_.is_commit_ && !is_data_complete) {
ret = OB_NOT_ENOUGH_STORE;
LOG_WARN("current ddl sstables not contain all data", K(ddl_sstable_handles), K(ddl_param), K(merge_param_));
} else if (FALSE_IT(ddl_param.table_key_.table_type_ = merge_param_.is_commit_ ?
ObITable::TableType::MAJOR_SSTABLE : ObITable::TableType::KV_DUMP_SSTABLE)) {
ObITable::TableType::MAJOR_SSTABLE : ObITable::TableType::DDL_DUMP_SSTABLE)) {
} else if (OB_FAIL(ObTabletDDLUtil::compact_ddl_sstable(ddl_sstable_handles.get_tables(),
tablet_handle.get_obj()->get_index_read_info(),
ddl_param,
is_scn_overlap,
table_handle))) {
LOG_WARN("compact sstables failed", K(ret));
} else {
@ -429,20 +432,26 @@ int ObDDLTableMergeTask::process()
return ret;
}
int ObDDLTableMergeTask::check_data_integrity(const ObTablesHandleArray &ddl_sstables,
// the input ddl sstable is sorted with start_scn
int ObTabletDDLUtil::check_data_integrity(const ObTablesHandleArray &ddl_sstables,
const SCN &start_scn,
const SCN &prepare_scn,
bool &is_data_complete)
bool &is_data_complete,
bool &is_scn_overlap)
{
int ret = OB_SUCCESS;
is_data_complete = false;
if (OB_UNLIKELY(ddl_sstables.empty() || !start_scn.is_valid_and_not_min() || !prepare_scn.is_valid_and_not_min())) {
is_scn_overlap = false;
if (OB_UNLIKELY(!start_scn.is_valid_and_not_min() || !prepare_scn.is_valid_and_not_min() || prepare_scn <= start_scn)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ddl_sstables.get_count()), K(start_scn), K(prepare_scn));
} else if (ddl_sstables.empty()) {
is_data_complete = false;
is_scn_overlap = false;
} else {
const ObSSTable *first_ddl_sstable = static_cast<const ObSSTable *>(ddl_sstables.get_tables().at(0));
const ObSSTable *last_ddl_sstable = static_cast<const ObSSTable *>(ddl_sstables.get_tables().at(ddl_sstables.get_count() - 1));
if (first_ddl_sstable->get_start_scn() != start_scn) {
if (first_ddl_sstable->get_start_scn() != SCN::scn_dec(start_scn)) {
LOG_INFO("start log ts not match", K(first_ddl_sstable->get_key()), K(start_scn));
} else if (last_ddl_sstable->get_end_scn() != prepare_scn) {
LOG_INFO("prepare log ts not match", K(last_ddl_sstable->get_key()), K(prepare_scn));
@ -451,12 +460,15 @@ int ObDDLTableMergeTask::check_data_integrity(const ObTablesHandleArray &ddl_sst
SCN last_end_scn = first_ddl_sstable->get_end_scn();
for (int64_t i = 1; OB_SUCC(ret) && i < ddl_sstables.get_count(); ++i) {
const ObSSTable *cur_ddl_sstable = static_cast<const ObSSTable *>(ddl_sstables.get_tables().at(i));
if (cur_ddl_sstable->get_start_scn() != last_end_scn) {
if (cur_ddl_sstable->get_start_scn() < last_end_scn) {
is_scn_overlap = true;
last_end_scn = SCN::max(last_end_scn, cur_ddl_sstable->get_end_scn());
} else if (cur_ddl_sstable->get_start_scn() == last_end_scn) {
last_end_scn = SCN::max(last_end_scn, cur_ddl_sstable->get_end_scn());
} else {
is_data_complete = false;
LOG_INFO("ddl sstable not continue", K(i), K(cur_ddl_sstable->get_key()), K(last_end_scn));
break;
} else {
last_end_scn = cur_ddl_sstable->get_end_scn();
}
}
}
@ -489,6 +501,7 @@ int ObTabletDDLUtil::prepare_index_data_desc(const share::ObLSID &ls_id,
const ObTabletID &tablet_id,
const int64_t snapshot_version,
const int64_t cluster_version,
const ObSSTable *first_ddl_sstable,
ObDataStoreDesc &data_desc)
{
int ret = OB_SUCCESS;
@ -517,6 +530,17 @@ int ObTabletDDLUtil::prepare_index_data_desc(const share::ObLSID &ls_id,
cluster_version))) {
LOG_WARN("init data store desc failed", K(ret), K(tablet_id));
} else {
if (nullptr != first_ddl_sstable) {
// use the param in first ddl sstable, which persist the param when ddl start
const ObSSTableBasicMeta &basic_meta = first_ddl_sstable->get_meta().get_basic_meta();
data_desc.row_store_type_ = basic_meta.row_store_type_;
data_desc.compressor_type_ = basic_meta.compressor_type_;
data_desc.master_key_id_ = basic_meta.master_key_id_;
data_desc.encrypt_id_ = basic_meta.encrypt_id_;
data_desc.encoder_opt_.set_store_type(basic_meta.row_store_type_);
MEMCPY(data_desc.encrypt_key_, basic_meta.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH);
data_desc.need_prebuild_bloomfilter_ = false;
}
data_desc.is_ddl_ = true;
data_desc.row_column_count_ = data_desc.rowkey_column_count_ + 1;
// prepare col_desc_array for index block and macro meta block
@ -547,6 +571,7 @@ int ObTabletDDLUtil::prepare_index_data_desc(const share::ObLSID &ls_id,
int ObTabletDDLUtil::prepare_index_builder(const ObTabletDDLParam &ddl_param,
ObIAllocator &allocator,
const ObSSTableIndexBuilder::ObSpaceOptimizationMode mode,
const blocksstable::ObSSTable *first_ddl_sstable,
ObSSTableIndexBuilder *&sstable_index_builder,
ObIndexBlockRebuilder *&index_block_rebuilder)
{
@ -559,6 +584,7 @@ int ObTabletDDLUtil::prepare_index_builder(const ObTabletDDLParam &ddl_param,
ddl_param.table_key_.get_tablet_id(),
ddl_param.table_key_.version_range_.snapshot_version_,
ddl_param.cluster_version_,
first_ddl_sstable,
data_desc))) {
LOG_WARN("prepare data desc failed", K(ret), K(ddl_param));
} else if (OB_ISNULL(buf = allocator.alloc(sizeof(ObSSTableIndexBuilder)))) {
@ -594,6 +620,7 @@ int ObTabletDDLUtil::prepare_index_builder(const ObTabletDDLParam &ddl_param,
int ObTabletDDLUtil::create_ddl_sstable(ObSSTableIndexBuilder *sstable_index_builder,
const ObTabletDDLParam &ddl_param,
const ObSSTable *first_ddl_sstable,
ObTableHandleV2 &table_handle)
{
int ret = OB_SUCCESS;
@ -618,8 +645,14 @@ int ObTabletDDLUtil::create_ddl_sstable(ObSSTableIndexBuilder *sstable_index_bui
} else {
const ObStorageSchema &storage_schema = tablet_handle.get_obj()->get_storage_schema();
int64_t column_count = 0;
if (OB_FAIL(storage_schema.get_stored_column_count_in_sstable(column_count))) {
LOG_WARN("fail to get stored column count in sstable", K(ret));
if (nullptr != first_ddl_sstable) {
column_count = first_ddl_sstable->get_meta().get_basic_meta().column_cnt_;
} else {
if (OB_FAIL(storage_schema.get_stored_column_count_in_sstable(column_count))) {
LOG_WARN("fail to get stored column count in sstable", K(ret));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(sstable_index_builder->close(column_count, res))) {
LOG_WARN("close sstable index builder close failed", K(ret));
} else {
@ -660,24 +693,16 @@ int ObTabletDDLUtil::create_ddl_sstable(ObSSTableIndexBuilder *sstable_index_bui
if (OB_FAIL(res.fill_column_checksum(&storage_schema, param.column_checksums_))) {
LOG_WARN("fail to fill column checksum for empty major", K(ret), K(param));
} else {
for (int64_t i = param.column_checksums_.count(); OB_SUCC(ret) && i > column_count; --i) {
param.column_checksums_.pop_back();
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, table_handle))) {
LOG_WARN("create sstable failed", K(ret), K(param));
} else {
const int64_t rebuild_seq = ls_handle.get_ls()->get_rebuild_seq();
ObTabletHandle new_tablet_handle;
ObUpdateTableStoreParam table_store_param(table_handle,
tablet_handle.get_obj()->get_snapshot_version(),
ddl_param.table_key_.is_major_sstable() ? false: true, // keep_old_ddl_sstable
&storage_schema,
rebuild_seq,
ddl_param.cluster_version_,
ddl_param.table_key_.is_major_sstable() ? true : false, // update_with_major_flag
ddl_param.table_key_.is_major_sstable() ? true : false); // need report checksum
if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(ddl_param.table_key_.get_tablet_id(), table_store_param, new_tablet_handle))) {
LOG_WARN("failed to update tablet table store", K(ret), K(ddl_param.table_key_), K(table_store_param));
} else {
LOG_INFO("create ddl sstable success", K(ddl_param), K(param), K(table_store_param));
}
LOG_INFO("create ddl sstable success", K(ddl_param), K(table_handle));
}
}
}
@ -685,9 +710,50 @@ int ObTabletDDLUtil::create_ddl_sstable(ObSSTableIndexBuilder *sstable_index_bui
return ret;
}
int ObTabletDDLUtil::update_ddl_table_store(ObSSTableIndexBuilder *sstable_index_builder,
const ObTabletDDLParam &ddl_param,
const ObSSTable *first_ddl_sstable,
ObTableHandleV2 &table_handle)
{
int ret = OB_SUCCESS;
if (OB_FAIL(create_ddl_sstable(sstable_index_builder, ddl_param, first_ddl_sstable, table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret));
} else {
ObLSService *ls_service = MTL(ObLSService *);
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
if (OB_FAIL(ls_service->get_ls(ddl_param.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) {
LOG_WARN("get ls failed", K(ret), K(ddl_param));
} else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle,
ddl_param.table_key_.tablet_id_,
tablet_handle,
ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) {
LOG_WARN("get tablet failed", K(ret), K(ddl_param));
} else {
const int64_t rebuild_seq = ls_handle.get_ls()->get_rebuild_seq();
ObTabletHandle new_tablet_handle;
ObUpdateTableStoreParam table_store_param(table_handle,
tablet_handle.get_obj()->get_snapshot_version(),
&tablet_handle.get_obj()->get_storage_schema(),
rebuild_seq,
ddl_param.table_key_.is_major_sstable(), // update_with_major_flag
ddl_param.table_key_.is_major_sstable()); // need report checksum
table_store_param.ddl_info_.keep_old_ddl_sstable_ = !ddl_param.table_key_.is_major_sstable();
table_store_param.ddl_info_.ddl_cluster_version_ = ddl_param.cluster_version_;
if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(ddl_param.table_key_.get_tablet_id(), table_store_param, new_tablet_handle))) {
LOG_WARN("failed to update tablet table store", K(ret), K(ddl_param.table_key_), K(table_store_param));
} else {
LOG_INFO("ddl update table store success", K(ddl_param), K(table_store_param), K(table_handle));
}
}
}
return ret;
}
int ObTabletDDLUtil::compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstables,
const ObTableReadInfo &read_info,
const ObTabletDDLParam &ddl_param,
const bool is_scn_overlap,
ObTableHandleV2 &table_handle)
{
int ret = OB_SUCCESS;
@ -701,10 +767,12 @@ int ObTabletDDLUtil::compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstable
if (OB_UNLIKELY(!ddl_param.is_valid() || ddl_sstables.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ddl_param), K(ddl_sstables.count()));
} else if (OB_FAIL(prepare_index_builder(ddl_param, arena, mode, sstable_index_builder, index_block_rebuilder))) {
} else if (OB_FAIL(prepare_index_builder(ddl_param, arena, mode,
static_cast<ObSSTable *>(ddl_sstables.at(0)),
sstable_index_builder, index_block_rebuilder))) {
LOG_WARN("prepare sstable index builder failed", K(ret));
} else {
// campact
ObDatumRowkey last_rowkey;
SMART_VAR(ObSSTableSecMetaIterator, meta_iter) {
ObDatumRange query_range;
query_range.set_whole_range();
@ -723,12 +791,28 @@ int ObTabletDDLUtil::compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstable
if (OB_ITER_END != ret) {
LOG_WARN("get data macro meta failed", K(ret));
}
} else if (OB_FAIL(index_block_rebuilder->append_macro_row(data_macro_meta))) {
LOG_WARN("append macro row failed", K(ret));
} else {
int cmp_ret = 1; // defaut 1 for last_rowkey is invalid
const ObDatumRowkey &cur_rowkey = data_macro_meta.end_key_;
if (is_scn_overlap && last_rowkey.is_valid()
&& OB_FAIL(cur_rowkey.compare(last_rowkey,
sstable_index_builder->get_index_store_desc().datum_utils_,
cmp_ret))) {
LOG_WARN("compare rowkey failed", K(ret), K(cur_rowkey), K(last_rowkey));
} else if (cmp_ret <= 0) { // cur_rowkey <= last_rowkey
// exist rowkey, skip
} else if (OB_FAIL(index_block_rebuilder->append_macro_row(data_macro_meta))) {
LOG_WARN("append macro row failed", K(ret));
}
}
}
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
if (is_scn_overlap && data_macro_meta.end_key_.is_valid()) {
if (OB_FAIL(data_macro_meta.end_key_.deep_copy(last_rowkey, arena))) {
LOG_WARN("deep copy to last rowkey failed", K(ret));
}
}
}
#ifdef ERRSIM
if (OB_SUCC(ret) && ddl_param.table_key_.is_major_sstable()) {
@ -749,7 +833,10 @@ int ObTabletDDLUtil::compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstable
if (OB_SUCC(ret)) {
if (OB_FAIL(index_block_rebuilder->close())) {
LOG_WARN("index block rebuilder close failed", K(ret));
} else if (OB_FAIL(create_ddl_sstable(sstable_index_builder, ddl_param, table_handle))) {
} else if (OB_FAIL(update_ddl_table_store(sstable_index_builder,
ddl_param,
static_cast<ObSSTable *>(ddl_sstables.at(0)),
table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret));
} else {
LOG_INFO("compact ddl sstable success", K(ddl_param));

19
src/storage/ddl/ob_ddl_merge_task.h
View File

@ -19,6 +19,7 @@
#include "storage/blocksstable/ob_index_block_builder.h"
#include "storage/blocksstable/ob_macro_block_struct.h"
#include "storage/ddl/ob_ddl_struct.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/blocksstable/ob_macro_block_struct.h"
#include "storage/blocksstable/ob_index_block_builder.h"
@ -117,10 +118,6 @@ public:
virtual ~ObDDLTableMergeTask();
int init(const ObDDLTableMergeDagParam &ddl_dag_param);
virtual int process() override;
static int check_data_integrity(const ObTablesHandleArray &ddl_sstables,
const share::SCN &start_scn,
const share::SCN &prepare_scn,
bool &is_data_complete);
TO_STRING_KV(K_(is_inited), K_(merge_param));
private:
bool is_inited_;
@ -151,21 +148,30 @@ public:
const ObTabletID &tablet_id,
const int64_t snapshot_version,
const int64_t cluster_version,
const blocksstable::ObSSTable *first_ddl_sstable,
blocksstable::ObDataStoreDesc &data_desc);
static int prepare_index_builder(const ObTabletDDLParam &ddl_param,
ObIAllocator &allocator,
const blocksstable::ObSSTableIndexBuilder::ObSpaceOptimizationMode mode,
const blocksstable::ObSSTable *first_ddl_sstable,
blocksstable::ObSSTableIndexBuilder *&sstable_index_builder,
blocksstable::ObIndexBlockRebuilder *&index_block_rebuilder);
static int create_ddl_sstable(blocksstable::ObSSTableIndexBuilder *sstable_index_builder,
const ObTabletDDLParam &ddl_param,
const blocksstable::ObSSTable *first_ddl_sstable,
ObTableHandleV2 &table_handle);
static int update_ddl_table_store(blocksstable::ObSSTableIndexBuilder *sstable_index_builder,
const ObTabletDDLParam &ddl_param,
const blocksstable::ObSSTable *first_ddl_sstable,
ObTableHandleV2 &table_handle);
static int compact_ddl_sstable(const ObIArray<ObITable *> &ddl_sstables,
const ObTableReadInfo &read_info,
const ObTabletDDLParam &ddl_param,
const bool is_scn_overlap,
ObTableHandleV2 &table_handle);
static int report_ddl_checksum(const share::ObLSID &ls_id,
@ -178,6 +184,11 @@ public:
const ObTabletID &tablet_id,
const blocksstable::ObSSTable *&latest_major_sstable);
static int check_data_integrity(const ObTablesHandleArray &ddl_sstables,
const share::SCN &start_scn,
const share::SCN &prepare_scn,
bool &is_data_complete,
bool &is_scn_overlap);
};
} // namespace storage

52
src/storage/ddl/ob_ddl_redo_log_replayer.cpp
View File

@ -151,13 +151,12 @@ int ObDDLRedoLogReplayer::replay_redo(const ObDDLRedoLog &log, const SCN &scn)
return ret;
}
int ObDDLRedoLogReplayer::replay_prepare(const ObDDLPrepareLog &log, const SCN &scn)
int ObDDLRedoLogReplayer::replay_commit(const ObDDLCommitLog &log, const SCN &scn)
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObITable::TableKey table_key = log.get_table_key();
ObDDLKvMgrHandle ddl_kv_mgr_handle;
ObDDLKVHandle ddl_kv_handle;
ObDDLKV *ddl_kv = nullptr;
bool need_replay = true;
@ -180,53 +179,14 @@ int ObDDLRedoLogReplayer::replay_prepare(const ObDDLPrepareLog &log, const SCN &
LOG_WARN("need replay but tablet handle is invalid", K(ret), K(need_replay), K(tablet_handle), K(log), K(scn));
} else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) {
LOG_WARN("get ddl kv mgr failed", K(ret), K(log), K(scn));
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_prepare(log.get_start_scn(), scn))) {
LOG_WARN("replay ddl prepare log failed", K(ret), K(log), K(scn));
} else {
LOG_INFO("replay ddl prepare log success", K(ret), K(log), K(scn));
}
LOG_INFO("finish replay ddl prepare log", K(ret), K(need_replay), K(log), K(scn));
return ret;
}
int ObDDLRedoLogReplayer::replay_commit(const ObDDLCommitLog &log, const SCN &scn)
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObITable::TableKey table_key = log.get_table_key();
ObDDLKvMgrHandle ddl_kv_mgr_handle;
ObDDLKVHandle ddl_kv_handle;
ObDDLKV *ddl_kv = nullptr;
bool need_replay = true;
DEBUG_SYNC(BEFORE_REPLAY_DDL_COMMIT);
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObDDLRedoLogReplayer has not been inited", K(ret));
} else if (OB_UNLIKELY(!log.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(log));
} else if (OB_FAIL(check_need_replay_ddl_log(table_key, log.get_start_scn(), scn, need_replay, tablet_handle))) {
if (OB_EAGAIN != ret) {
LOG_WARN("fail to check need replay ddl log", K(ret), K(table_key), K(scn), K(log));
}
} else if (!need_replay) {
// do nothing
} else if (OB_UNLIKELY(!tablet_handle.is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("need replay but tablet handle is invalid", K(ret), K(need_replay), K(tablet_handle), K(log), K(scn));
} else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) {
LOG_WARN("get ddl kv mgr failed", K(ret), K(log), K(scn));
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_commit(log.get_start_scn(), log.get_prepare_scn(), true/*is_replay*/))) {
if (OB_EAGAIN != ret) {
LOG_WARN("replay ddl commit log failed", K(ret), K(log), K(scn));
}
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_commit(log.get_start_scn(), scn))) {
if (OB_TABLET_NOT_EXIST == ret || OB_TASK_EXPIRED == ret) {
ret = OB_SUCCESS; // exit when tablet not exist or task expired
} else {
LOG_WARN("replay ddl commit log failed", K(ret), K(log), K(scn));
}
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->unregister_from_tablet(log.get_start_scn(), ddl_kv_mgr_handle))) {
LOG_WARN("unregister ddl kv mgr from tablet failed", K(ret), K(log), K(scn));
} else {
LOG_INFO("replay ddl commit log success", K(ret), K(log), K(scn));
}
LOG_INFO("finish replay ddl commit log", K(ret), K(need_replay), K(log), K(scn));
return ret;

1
src/storage/ddl/ob_ddl_redo_log_replayer.h
View File

@ -33,7 +33,6 @@ public:
void reset() { destroy(); }
int replay_start(const ObDDLStartLog &log, const share::SCN &scn);
int replay_redo(const ObDDLRedoLog &log, const share::SCN &scn);
int replay_prepare(const ObDDLPrepareLog &log, const share::SCN &scn);
int replay_commit(const ObDDLCommitLog &log, const share::SCN &scn);
private:
void destroy();

77
src/storage/ddl/ob_ddl_redo_log_writer.cpp
View File

@ -26,6 +26,7 @@
#include "storage/ddl/ob_tablet_ddl_kv_mgr.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "observer/ob_server_event_history_table_operator.h"
#include "storage/tablet/ob_tablet.h"
using namespace oceanbase::common;
using namespace oceanbase::storage;
@ -930,7 +931,7 @@ int ObDDLMacroBlockRedoWriter::remote_write_macro_redo(const ObAddr &leader_addr
if (OB_FAIL(arg.init(MTL_ID(), leader_ls_id, redo_info))) {
LOG_WARN("fail to init ObRpcRemoteWriteDDLRedoLogArg", K(ret));
} else if (OB_FAIL(srv_rpc_proxy->to(leader_addr).remote_write_ddl_redo_log(arg))) {
LOG_WARN("fail to remote write ddl redo log", K(ret), K(arg));
LOG_WARN("fail to remote write ddl redo log", K(ret), K(leader_addr), K(arg));
}
}
return ret;
@ -1059,11 +1060,11 @@ int ObDDLSSTableRedoWriter::wait_redo_log_finish(const ObDDLMacroBlockRedoInfo &
return ret;
}
int ObDDLSSTableRedoWriter::write_prepare_log(const ObITable::TableKey &table_key,
const int64_t table_id,
const int64_t execution_id,
const int64_t ddl_task_id,
SCN &prepare_scn)
int ObDDLSSTableRedoWriter::write_commit_log(const ObITable::TableKey &table_key,
const int64_t table_id,
const int64_t execution_id,
const int64_t ddl_task_id,
SCN &commit_scn)
{
int ret = OB_SUCCESS;
@ -1075,10 +1076,10 @@ int ObDDLSSTableRedoWriter::write_prepare_log(const ObITable::TableKey &table_ke
"ddl_task_id", ddl_task_id);
DEBUG_SYNC(BEFORE_DDL_WRITE_PREPARE_LOG);
#endif
prepare_scn.set_min();
commit_scn.set_min();
ObLSHandle ls_handle;
ObLS *ls = nullptr;
ObDDLPrepareLog log;
ObDDLCommitLog log;
ObDDLCommitLogHandle handle;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
@ -1093,60 +1094,6 @@ int ObDDLSSTableRedoWriter::write_prepare_log(const ObITable::TableKey &table_ke
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("ls should not be null", K(ret), K(table_key));
} else if (!remote_write_) {
if (OB_FAIL(ObDDLRedoLogWriter::get_instance().write_ddl_finish_log(log, ObDDLClogType::DDL_PREPARE_LOG, ls_id_, ls->get_log_handler(), handle))) {
if (ObDDLUtil::need_remote_write(ret)) {
if (OB_FAIL(switch_to_remote_write())) {
LOG_WARN("fail to switch to remote write", K(ret), K(table_key));
}
} else {
LOG_WARN("fail to write ddl commit log", K(ret), K(table_key));
}
} else if (OB_FAIL(handle.wait())) {
LOG_WARN("wait ddl commit log finish failed", K(ret), K(table_key));
} else {
prepare_scn = handle.get_commit_scn();
}
}
if (OB_SUCC(ret) && remote_write_) {
ObSrvRpcProxy *srv_rpc_proxy = GCTX.srv_rpc_proxy_;
obrpc::ObRpcRemoteWriteDDLPrepareLogArg arg;
obrpc::Int64 log_ns;
if (OB_FAIL(arg.init(MTL_ID(), leader_ls_id_, table_key, get_start_scn(), table_id, execution_id, ddl_task_id))) {
LOG_WARN("fail to init ObRpcRemoteWriteDDLPrepareLogArg", K(ret));
} else if (OB_ISNULL(srv_rpc_proxy)) {
ret = OB_ERR_SYS;
LOG_WARN("srv rpc proxy or location service is null", K(ret), KP(srv_rpc_proxy));
} else if (OB_FAIL(srv_rpc_proxy->to(leader_addr_).remote_write_ddl_prepare_log(arg, log_ns))) {
LOG_WARN("fail to remote write ddl redo log", K(ret), K(arg));
} else if (OB_FAIL(prepare_scn.convert_for_tx(log_ns))) {
LOG_WARN("convert for tx failed", K(ret));
}
}
return ret;
}
int ObDDLSSTableRedoWriter::write_commit_log(const ObITable::TableKey &table_key,
const SCN &prepare_scn)
{
int ret = OB_SUCCESS;
ObLSHandle ls_handle;
ObLS *ls = nullptr;
ObDDLCommitLog log;
ObDDLCommitLogHandle handle;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObDDLSSTableRedoWriter has not been inited", K(ret));
} else if (OB_UNLIKELY(!table_key.is_valid() || !start_scn_.is_valid_and_not_min() || !prepare_scn.is_valid_and_not_min())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(table_key), K(start_scn_), K(prepare_scn));
} else if (OB_FAIL(log.init(table_key, get_start_scn(), prepare_scn))) {
LOG_WARN("fail to init DDLCommitLog", K(ret), K(table_key), K(start_scn_), K(prepare_scn));
} else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) {
LOG_WARN("get ls failed", K(ret), K(ls_id_));
} else if (OB_ISNULL(ls = ls_handle.get_ls())) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("ls should not be null", K(ret), K(table_key));
} else if (!remote_write_) {
if (OB_FAIL(ObDDLRedoLogWriter::get_instance().write_ddl_finish_log(log, ObDDLClogType::DDL_COMMIT_LOG, ls_id_, ls->get_log_handler(), handle))) {
if (ObDDLUtil::need_remote_write(ret)) {
@ -1158,19 +1105,23 @@ int ObDDLSSTableRedoWriter::write_commit_log(const ObITable::TableKey &table_key
}
} else if (OB_FAIL(handle.wait())) {
LOG_WARN("wait ddl commit log finish failed", K(ret), K(table_key));
} else {
commit_scn = handle.get_commit_scn();
}
}
if (OB_SUCC(ret) && remote_write_) {
ObSrvRpcProxy *srv_rpc_proxy = GCTX.srv_rpc_proxy_;
obrpc::ObRpcRemoteWriteDDLCommitLogArg arg;
obrpc::Int64 log_ns;
if (OB_FAIL(arg.init(MTL_ID(), leader_ls_id_, table_key, get_start_scn(), prepare_scn))) {
if (OB_FAIL(arg.init(MTL_ID(), leader_ls_id_, table_key, get_start_scn(), table_id, execution_id, ddl_task_id))) {
LOG_WARN("fail to init ObRpcRemoteWriteDDLCommitLogArg", K(ret));
} else if (OB_ISNULL(srv_rpc_proxy)) {
ret = OB_ERR_SYS;
LOG_WARN("srv rpc proxy or location service is null", K(ret), KP(srv_rpc_proxy));
} else if (OB_FAIL(srv_rpc_proxy->to(leader_addr_).remote_write_ddl_commit_log(arg, log_ns))) {
LOG_WARN("fail to remote write ddl redo log", K(ret), K(arg));
} else if (OB_FAIL(commit_scn.convert_for_tx(log_ns))) {
LOG_WARN("convert for tx failed", K(ret));
}
}
return ret;

11
src/storage/ddl/ob_ddl_redo_log_writer.h
View File

@ -21,6 +21,7 @@
#include "storage/tx_storage/ob_ls_map.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "storage/meta_mem/ob_tablet_pointer.h"
namespace oceanbase
{
@ -253,13 +254,11 @@ public:
const blocksstable::MacroBlockId &macro_block_id);
int wait_redo_log_finish(const blocksstable::ObDDLMacroBlockRedoInfo &redo_info,
const blocksstable::MacroBlockId &macro_block_id);
int write_prepare_log(const ObITable::TableKey &table_key,
const int64_t table_id,
const int64_t execution_id,
const int64_t ddl_task_id,
share::SCN &prepare_scn);
int write_commit_log(const ObITable::TableKey &table_key,
const share::SCN &prepare_scn);
const int64_t table_id,
const int64_t execution_id,
const int64_t ddl_task_id,
share::SCN &commit_scn);
OB_INLINE void set_start_scn(const share::SCN &start_scn) { start_scn_.atomic_set(start_scn); }
OB_INLINE share::SCN get_start_scn() const { return start_scn_.atomic_get(); }
private:

399
src/storage/ddl/ob_ddl_struct.cpp
View File

@ -12,26 +12,16 @@
#define USING_LOG_PREFIX STORAGE
#include "ob_ddl_struct.h"
#include "share/scn.h"
#include "storage/blocksstable/ob_block_manager.h"
#include "storage/blocksstable/ob_block_sstable_struct.h"
#include "storage/blocksstable/ob_index_block_builder.h"
#include "storage/blocksstable/ob_macro_block_struct.h"
#include "share/ob_force_print_log.h"
#include "share/schema/ob_multi_version_schema_service.h"
#include "storage/ddl/ob_ddl_struct.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
#include "storage/ddl/ob_tablet_ddl_kv_mgr.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/meta_mem/ob_tablet_handle.h"
#include "storage/ddl/ob_ddl_merge_task.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "storage/compaction/ob_schedule_dag_func.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/blocksstable/ob_block_manager.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
using namespace oceanbase::storage;
using namespace oceanbase::blocksstable;
using namespace oceanbase::clog;
using namespace oceanbase::share;
using namespace oceanbase::share::schema;
ObDDLMacroHandle::ObDDLMacroHandle()
: block_id_()
@ -130,368 +120,6 @@ bool ObDDLMacroBlock::is_valid() const
}
ObDDLKV::ObDDLKV()
: is_inited_(false), ls_id_(), tablet_id_(), ddl_start_scn_(SCN::min_scn()), snapshot_version_(0),
lock_(), allocator_("DDL_KV"), is_freezed_(false), is_closed_(false), last_freezed_scn_(SCN::min_scn()),
min_scn_(SCN::max_scn()), max_scn_(SCN::min_scn()), freeze_scn_(SCN::max_scn()), pending_cnt_(0), cluster_version_(0),
ref_cnt_(0), sstable_index_builder_(nullptr), index_block_rebuilder_(nullptr), is_rebuilder_closed_(false)
{
}
ObDDLKV::~ObDDLKV()
{
destroy();
}
int ObDDLKV::init(const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const SCN &ddl_start_scn,
const int64_t snapshot_version,
const SCN &last_freezed_scn,
const int64_t cluster_version)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDDLKV has been inited twice", K(ret));
} else if (OB_UNLIKELY(!ls_id.is_valid()
|| !tablet_id.is_valid()
|| !ddl_start_scn.is_valid_and_not_min()
|| snapshot_version <= 0
|| !last_freezed_scn.is_valid_and_not_min()
|| cluster_version < 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(ddl_start_scn), K(snapshot_version), K(last_freezed_scn), K(cluster_version));
} else {
ObTabletDDLParam ddl_param;
ddl_param.tenant_id_ = MTL_ID();
ddl_param.ls_id_ = ls_id;
ddl_param.table_key_.tablet_id_ = tablet_id;
ddl_param.table_key_.table_type_ = ObITable::TableType::MAJOR_SSTABLE;
ddl_param.table_key_.version_range_.base_version_ = 0;
ddl_param.table_key_.version_range_.snapshot_version_ = snapshot_version;
ddl_param.start_scn_ = ddl_start_scn;
ddl_param.snapshot_version_ = snapshot_version;
ddl_param.cluster_version_ = cluster_version;
if (OB_FAIL(ObTabletDDLUtil::prepare_index_builder(ddl_param, allocator_, ObSSTableIndexBuilder::DISABLE, sstable_index_builder_, index_block_rebuilder_))) {
LOG_WARN("prepare index builder failed", K(ret));
} else {
ls_id_ = ls_id;
tablet_id_ = tablet_id;
ddl_start_scn_ = ddl_start_scn;
snapshot_version_ = snapshot_version;
last_freezed_scn_ = last_freezed_scn;
cluster_version_ = cluster_version;
is_inited_ = true;
LOG_INFO("ddl kv init success", K(ls_id_), K(tablet_id_), K(ddl_start_scn_), K(snapshot_version_), K(last_freezed_scn_), K(cluster_version_));
}
}
return ret;
}
void ObDDLKV::destroy()
{
if (nullptr != index_block_rebuilder_) {
index_block_rebuilder_->~ObIndexBlockRebuilder();
allocator_.free(index_block_rebuilder_);
index_block_rebuilder_ = nullptr;
}
if (nullptr != sstable_index_builder_) {
sstable_index_builder_->~ObSSTableIndexBuilder();
allocator_.free(sstable_index_builder_);
sstable_index_builder_ = nullptr;
}
allocator_.reset();
}
int ObDDLKV::set_macro_block(const ObDDLMacroBlock &macro_block)
{
int ret = OB_SUCCESS;
const int64_t MAX_DDL_BLOCK_COUNT = 10L * 1024L * 1024L * 1024L / OB_SERVER_BLOCK_MGR.get_macro_block_size();
int64_t freeze_block_count = MAX_DDL_BLOCK_COUNT;
#ifdef ERRSIM
if (0 != GCONF.errsim_max_ddl_block_count) {
freeze_block_count = GCONF.errsim_max_ddl_block_count;
LOG_INFO("ddl set macro block count", K(freeze_block_count));
}
#endif
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ddl kv is not init", K(ret));
} else if (OB_UNLIKELY(!macro_block.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(macro_block));
} else {
const uint64_t tenant_id = MTL_ID();
ObUnitInfoGetter::ObTenantConfig unit;
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(GCTX.omt_->get_tenant_unit(tenant_id, unit))) {
LOG_WARN("get tenant unit failed", K(tmp_ret), K(tenant_id));
} else {
const int64_t log_allowed_block_count = unit.config_.log_disk_size() * 0.2 / OB_SERVER_BLOCK_MGR.get_macro_block_size();
if (log_allowed_block_count <= 0) {
tmp_ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid macro block count by log disk size", K(tmp_ret), K(tenant_id), K(unit.config_));
} else {
freeze_block_count = min(freeze_block_count, log_allowed_block_count);
}
}
}
if (OB_SUCC(ret) && ddl_blocks_.count() >= freeze_block_count) {
ObDDLTableMergeDagParam param;
param.ls_id_ = ls_id_;
param.tablet_id_ = tablet_id_;
param.start_scn_ = ddl_start_scn_;
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) {
LOG_WARN("try schedule ddl merge dag failed when ddl kv is full ",
K(tmp_ret), K(ls_id_), K(tablet_id_), K(ddl_blocks_.count()));
}
}
if (OB_SUCC(ret)) {
ObDataMacroBlockMeta *data_macro_meta = nullptr;
ObArenaAllocator meta_allocator;
TCWLockGuard guard(lock_);
if (macro_block.ddl_start_scn_ != ddl_start_scn_) {
if (macro_block.ddl_start_scn_ > ddl_start_scn_) {
ret = OB_EAGAIN;
LOG_INFO("ddl start scn too large, retry", K(ret),
K(ls_id_), K(tablet_id_), K(ddl_start_scn_), K(macro_block));
} else {
// filter out and do nothing
LOG_INFO("ddl start scn too small, maybe from old build task, ignore", K(ret),
K(ls_id_), K(tablet_id_), K(ddl_start_scn_), K(macro_block));
}
} else if (macro_block.scn_ > freeze_scn_) {
ret = OB_EAGAIN;
LOG_INFO("this ddl kv is freezed, retry other ddl kv", K(ret), K(ls_id_), K(tablet_id_), K(macro_block), K(freeze_scn_));
} else if (OB_FAIL(index_block_rebuilder_->append_macro_row(
macro_block.buf_, macro_block.size_, macro_block.get_block_id()))) {
LOG_WARN("append macro meta failed", K(ret), K(macro_block));
} else if (OB_FAIL(ddl_blocks_.push_back(macro_block.block_handle_))) {
LOG_WARN("push back block handle failed", K(ret), K(macro_block.block_handle_));
} else {
min_scn_ = SCN::min(min_scn_, macro_block.scn_);
max_scn_ = SCN::max(max_scn_, macro_block.scn_);
LOG_INFO("succeed to set macro block into ddl kv", K(macro_block));
}
}
return ret;
}
int ObDDLKV::freeze(const SCN &freeze_scn)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ddl kv is not init", K(ret));
} else {
TCWLockGuard guard(lock_);
if (is_freezed_) {
// do nothing
} else {
if (freeze_scn.is_valid_and_not_min()) {
freeze_scn_ = freeze_scn;
} else if (max_scn_.is_valid_and_not_min()) {
freeze_scn_ = max_scn_;
} else {
ret = OB_EAGAIN;
LOG_INFO("ddl kv not freezed, try again", K(ret), K(ls_id_), K(tablet_id_), K(ddl_blocks_.count()));
}
if (OB_SUCC(ret)) {
ATOMIC_SET(&is_freezed_, true);
LOG_INFO("ddl kv freezed", K(ret), K(ls_id_), K(tablet_id_), K(ddl_blocks_.count()));
}
}
}
return ret;
}
int ObDDLKV::close()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ddl kv is not init", K(ret));
} else if (is_closed_) {
// do nothing
LOG_INFO("ddl kv already closed", K(*this));
} else if (OB_FAIL(wait_pending())) {
if (OB_EAGAIN != ret) {
LOG_WARN("wait pending failed", K(ret));
}
} else if (!is_rebuilder_closed_) {
if (OB_FAIL(index_block_rebuilder_->close())) {
LOG_WARN("index block rebuilder close failed", K(ret));
} else {
is_rebuilder_closed_ = true;
}
}
if (OB_SUCC(ret) && !is_closed_) {
ObTableHandleV2 table_handle;
ObTabletDDLParam ddl_param;
ddl_param.tenant_id_ = MTL_ID();
ddl_param.ls_id_ = ls_id_;
ddl_param.table_key_.tablet_id_ = tablet_id_;
ddl_param.table_key_.table_type_ = ObITable::TableType::KV_DUMP_SSTABLE;
ddl_param.table_key_.scn_range_.start_scn_ = last_freezed_scn_;
ddl_param.table_key_.scn_range_.end_scn_ = freeze_scn_;
ddl_param.start_scn_ = ddl_start_scn_;
ddl_param.snapshot_version_ = snapshot_version_;
ddl_param.cluster_version_ = cluster_version_;
if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(sstable_index_builder_, ddl_param, table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret));
} else {
is_closed_ = true;
LOG_INFO("ddl kv closed success", K(*this));
}
}
return ret;
}
void ObDDLKV::inc_pending_cnt()
{
ATOMIC_INC(&pending_cnt_);
}
void ObDDLKV::dec_pending_cnt()
{
ATOMIC_DEC(&pending_cnt_);
}
int ObDDLKV::wait_pending()
{
int ret = OB_SUCCESS;
ObLSService *ls_service = MTL(ObLSService *);
ObLSHandle ls_handle;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
} else if (OB_UNLIKELY(!is_freezed())) {
ret = OB_STATE_NOT_MATCH;
LOG_WARN("ddl kv not freezed", K(ret));
} else if (OB_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) {
LOG_WARN("get ls handle failed", K(ret), K(ls_id_));
} else {
SCN max_decided_scn;
if (OB_FAIL(ls_handle.get_ls()->get_max_decided_scn(max_decided_scn))) {
LOG_WARN("get max decided log ts failed", K(ret), K(ls_id_));
} else {
// max_decided_scn is the left border scn - 1
// the min deciding(replay or apply) scn (aka left border) is max_decided_scn + 1
const bool pending_finished = SCN::plus(max_decided_scn, 1) >= freeze_scn_ && !is_pending();
if (!pending_finished) {
ret = OB_EAGAIN;
if (REACH_TIME_INTERVAL(1000L * 1000L)) {
LOG_INFO("wait pending not finish", K(ret), K(*this), K(max_decided_scn));
}
}
}
}
return ret;
}
ObDDLKVHandle::ObDDLKVHandle()
: kv_(nullptr)
{
}
ObDDLKVHandle::~ObDDLKVHandle()
{
reset();
}
int ObDDLKVHandle::set_ddl_kv(ObDDLKV *kv)
{
int ret = OB_SUCCESS;
if (nullptr != kv_) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDDLKVHandle cannot be inited twice", K(ret));
} else if (OB_ISNULL(kv)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), KP(kv));
} else {
kv->inc_ref();
kv_ = kv;
}
return ret;
}
int ObDDLKVHandle::get_ddl_kv(ObDDLKV *&kv)
{
int ret = OB_SUCCESS;
kv = nullptr;
if (!is_valid()) {
ret = OB_ENTRY_NOT_EXIST;
} else {
kv = kv_;
}
return ret;
}
void ObDDLKVHandle::reset()
{
if (nullptr != kv_) {
if (0 == kv_->dec_ref()) {
op_free(kv_);
STORAGE_LOG(INFO, "free a ddl kv", KP(kv_));
}
kv_ = nullptr;
}
}
ObDDLKVsHandle::ObDDLKVsHandle()
: kv_array_()
{
}
ObDDLKVsHandle::~ObDDLKVsHandle()
{
reset();
}
void ObDDLKVsHandle::reset()
{
for (int64_t i = 0; i < kv_array_.count(); ++i) {
ObDDLKV *kv = kv_array_.at(i);
if (nullptr != kv) {
if (0 == kv->dec_ref()) {
op_free(kv);
STORAGE_LOG(INFO, "free a ddl kv");
}
kv_array_.at(i) = nullptr;
}
}
kv_array_.reset();
}
int ObDDLKVsHandle::add_ddl_kv(ObDDLKV *kv)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(kv)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), KP(kv));
} else if (OB_FAIL(kv_array_.push_back(kv))) {
LOG_WARN("fail to push back kv array", K(ret));
} else {
kv->inc_ref();
}
return ret;
}
int ObDDLKVsHandle::get_ddl_kv(const int64_t idx, ObDDLKV *&kv)
{
int ret = OB_SUCCESS;
if (idx >= kv_array_.count()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(idx), K(kv_array_.count()));
} else {
kv = kv_array_.at(idx);
}
return ret;
}
ObDDLKVPendingGuard::ObDDLKVPendingGuard(ObTablet *tablet, const SCN &scn)
: tablet_(tablet), scn_(scn), kv_handle_(), ret_(OB_SUCCESS)
{
@ -505,9 +133,7 @@ ObDDLKVPendingGuard::ObDDLKVPendingGuard(ObTablet *tablet, const SCN &scn)
LOG_WARN("get ddl kv mgr failed", K(ret));
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_or_create_ddl_kv(scn, kv_handle_))) {
LOG_WARN("acquire ddl kv failed", K(ret));
} else if (OB_FAIL(kv_handle_.get_ddl_kv(curr_kv))) {
LOG_WARN("fail to get ddl kv", K(ret));
} else if (OB_ISNULL(curr_kv)) {
} else if (OB_ISNULL(curr_kv = static_cast<ObDDLKV *>(kv_handle_.get_table()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, active ddl kv must not be nullptr", K(ret));
} else {
@ -525,8 +151,8 @@ int ObDDLKVPendingGuard::get_ddl_kv(ObDDLKV *&kv)
kv = nullptr;
if (OB_FAIL(ret_)) {
// do nothing
} else if (OB_FAIL(kv_handle_.get_ddl_kv(kv))) {
LOG_WARN("fail to get ddl kv", K(ret));
} else {
kv = static_cast<ObDDLKV *>(kv_handle_.get_table());
}
return ret;
}
@ -535,10 +161,8 @@ ObDDLKVPendingGuard::~ObDDLKVPendingGuard()
{
int ret = OB_SUCCESS;
if (OB_SUCCESS == ret_) {
ObDDLKV *curr_kv = nullptr;
if (OB_FAIL(kv_handle_.get_ddl_kv(curr_kv))) {
LOG_ERROR("error unexpected, can not get ddl kv", K(ret));
} else if (nullptr != curr_kv) {
ObDDLKV *curr_kv = static_cast<ObDDLKV *>(kv_handle_.get_table());
if (nullptr != curr_kv) {
curr_kv->dec_pending_cnt();
}
}
@ -559,6 +183,9 @@ int ObDDLKVPendingGuard::set_macro_block(ObTablet *tablet, const ObDDLMacroBlock
ObDDLKVPendingGuard guard(tablet, macro_block.scn_);
if (OB_FAIL(guard.get_ddl_kv(ddl_kv))) {
LOG_WARN("get ddl kv failed", K(ret));
} else if (OB_ISNULL(ddl_kv)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ddl kv is null", K(ret), KP(ddl_kv), K(guard));
} else if (OB_FAIL(ddl_kv->set_macro_block(macro_block))) {
LOG_WARN("fail to set macro block info", K(ret));
} else {

102
src/storage/ddl/ob_ddl_struct.h
View File

@ -13,22 +13,12 @@
#ifndef OCEANBASE_STORAGE_OB_DDL_STRUCT_H_
#define OCEANBASE_STORAGE_OB_DDL_STRUCT_H_
#include "lib/hash/ob_hashmap.h"
#include "lib/lock/ob_mutex.h"
#include "lib/container/ob_array.h"
#include "share/scn.h"
#include "storage/ob_i_table.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/blocksstable/ob_block_sstable_struct.h"
#include "storage/blocksstable/ob_index_block_builder.h"
#include "storage/checkpoint/ob_freeze_checkpoint.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
namespace oceanbase
{
namespace blocksstable
{
struct ObSSTableMergeRes;
}
namespace storage
{
@ -41,6 +31,7 @@ public:
ObDDLMacroHandle(const ObDDLMacroHandle &other);
ObDDLMacroHandle &operator=(const ObDDLMacroHandle &other);
~ObDDLMacroHandle();
bool is_valid() const { return block_id_.is_valid(); }
int set_block_id(const blocksstable::MacroBlockId &block_id);
int reset_macro_block_ref();
const blocksstable::MacroBlockId &get_block_id() const { return block_id_; }
@ -69,92 +60,10 @@ public:
};
class ObDDLKV
{
public:
ObDDLKV();
~ObDDLKV();
int init(const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const share::SCN &ddl_start_scn,
const int64_t snapshot_version,
const share::SCN &last_freezed_scn,
const int64_t cluster_version);
void destroy();
int set_macro_block(const ObDDLMacroBlock &macro_block);
class ObDDLKV;
int freeze(const share::SCN &freeze_scn);
bool is_freezed() const { return ATOMIC_LOAD(&is_freezed_); }
int close();
bool is_closed() const { return is_closed_; }
share::SCN get_min_scn() const { return min_scn_; }
share::SCN get_freeze_scn() const { return freeze_scn_; }
share::SCN get_ddl_start_scn() const { return ddl_start_scn_; }
int64_t get_macro_block_cnt() const { return ddl_blocks_.count(); }
void inc_pending_cnt(); // used by ddl kv pending guard
void dec_pending_cnt();
bool is_pending() const { return ATOMIC_LOAD(&pending_cnt_) > 0; }
int wait_pending();
void inc_ref() { ATOMIC_INC(&ref_cnt_); }
int64_t dec_ref() { return ATOMIC_SAF(&ref_cnt_, 1 /* just sub 1 */); }
int64_t get_ref() { return ATOMIC_LOAD(&ref_cnt_); }
TO_STRING_KV(K_(is_inited), K_(ls_id), K_(tablet_id), K_(ddl_start_scn), K_(snapshot_version),
K_(is_freezed), K_(is_closed),
K_(last_freezed_scn), K_(min_scn), K_(max_scn), K_(freeze_scn),
K_(pending_cnt), K_(cluster_version), K_(ref_cnt), K(ddl_blocks_.count()),
KP_(sstable_index_builder), KP_(index_block_rebuilder), K_(is_rebuilder_closed));
private:
static const int64_t TOTAL_LIMIT = 10 * 1024 * 1024 * 1024L;
static const int64_t HOLD_LIMIT = 10 * 1024 * 1024 * 1024L;
friend class ObDDLSSTableIterator;
bool is_inited_;
share::ObLSID ls_id_;
common::ObTabletID tablet_id_;
share::SCN ddl_start_scn_; // the log ts of ddl start log
int64_t snapshot_version_; // the snapshot version for major sstable which is completed by ddl
common::TCRWLock lock_; // lock for ddl_blocks_ and freeze_log_ts_
common::ObArenaAllocator allocator_;
bool is_freezed_;
bool is_closed_;
share::SCN last_freezed_scn_; // the freezed log ts of last ddl kv. the log ts range of this ddl kv is (last_freezed_log_ts_, freeze_log_ts_]
share::SCN min_scn_; // the min log ts of macro blocks
share::SCN max_scn_; // the max log ts of macro blocks
share::SCN freeze_scn_; // ddl kv refuse data larger than freeze log ts, freeze_log_ts >= max_log_ts
int64_t pending_cnt_; // the amount of kvs that are replaying
int64_t cluster_version_;
int64_t ref_cnt_;
ObArray<ObDDLMacroHandle> ddl_blocks_;
blocksstable::ObSSTableIndexBuilder *sstable_index_builder_;
blocksstable::ObIndexBlockRebuilder *index_block_rebuilder_;
bool is_rebuilder_closed_;
};
class ObDDLKVHandle final
{
public:
ObDDLKVHandle();
~ObDDLKVHandle();
int set_ddl_kv(ObDDLKV *kv);
int get_ddl_kv(ObDDLKV *&kv);
bool is_valid() const { return nullptr != kv_; }
void reset();
private:
ObDDLKV *kv_;
DISALLOW_COPY_AND_ASSIGN(ObDDLKVHandle);
};
class ObDDLKVsHandle final
{
public:
ObDDLKVsHandle();
~ObDDLKVsHandle();
int add_ddl_kv(ObDDLKV *ddl_kv);
void reset();
int64_t get_count() const { return kv_array_.count(); }
int get_ddl_kv(const int64_t idx, ObDDLKV *&kv);
private:
ObArray<ObDDLKV *> kv_array_;
};
class ObTablet;
class ObDDLKVPendingGuard final
{
@ -165,10 +74,11 @@ public:
~ObDDLKVPendingGuard();
int get_ret() const { return ret_; }
int get_ddl_kv(ObDDLKV *&kv);
TO_STRING_KV(KP(tablet_), K(scn_), K(kv_handle_), K(ret_));
private:
ObTablet *tablet_;
share::SCN scn_;
ObDDLKVHandle kv_handle_;
ObTableHandleV2 kv_handle_;
int ret_;
};

27
src/storage/ddl/ob_direct_insert_sstable_ctx.cpp
View File

@ -827,7 +827,7 @@ int ObSSTableInsertTabletContext::create_sstable_with_clog(
share::schema::ObMultiVersionSchemaService *schema_service = nullptr;
const share::schema::ObTableSchema *table_schema = nullptr;
const uint64_t tenant_id = MTL_ID();
SCN prepare_scn;
SCN commit_scn;
ObSchemaGetterGuard schema_guard;
if (OB_UNLIKELY(!table_key.is_valid())) {
ret = OB_INVALID_ARGUMENT;
@ -848,15 +848,15 @@ int ObSSTableInsertTabletContext::create_sstable_with_clog(
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(data_sstable_redo_writer_.write_prepare_log(table_key,
table_schema->get_table_id(),
build_param_.execution_id_,
build_param_.ddl_task_id_,
prepare_scn))) {
} else if (OB_FAIL(data_sstable_redo_writer_.write_commit_log(table_key,
table_schema->get_table_id(),
build_param_.execution_id_,
build_param_.ddl_task_id_,
commit_scn))) {
if (OB_TASK_EXPIRED == ret) {
LOG_INFO("ddl task expired", K(ret), K(table_key), K(build_param_));
} else {
LOG_WARN("fail write ddl prepare log", K(ret), K(table_key));
LOG_WARN("fail write ddl commit log", K(ret), K(table_key));
}
} else {
DEBUG_SYNC(AFTER_REMOTE_WRITE_DDL_PREPARE_LOG);
@ -872,27 +872,24 @@ int ObSSTableInsertTabletContext::create_sstable_with_clog(
LOG_WARN("get tablet failed", K(ret));
} else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) {
LOG_WARN("get ddl kv manager failed", K(ret), K(ls_id), K(tablet_id));
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_prepare(ddl_start_scn,
prepare_scn,
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_commit(ddl_start_scn,
commit_scn,
table_id,
ddl_task_id))) {
if (OB_TASK_EXPIRED == ret) {
LOG_INFO("ddl task expired", K(ret), K(ls_id), K(tablet_id),
K(ddl_start_scn), "new_ddl_start_scn", ddl_kv_mgr_handle.get_obj()->get_start_scn());
} else {
LOG_WARN("failed to do ddl kv prepare", K(ret), K(ddl_start_scn), K(prepare_scn), K(build_param_));
LOG_WARN("failed to do ddl kv commit", K(ret), K(ddl_start_scn), K(commit_scn), K(build_param_));
}
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->wait_ddl_commit(ddl_start_scn, prepare_scn))) {
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->wait_ddl_merge_success(ddl_start_scn, commit_scn))) {
if (OB_TASK_EXPIRED == ret) {
LOG_INFO("ddl task expired, but return success", K(ret), K(ls_id), K(tablet_id),
K(ddl_start_scn), "new_ddl_start_scn",
ddl_kv_mgr_handle.get_obj()->get_start_scn(), K(build_param_));
} else {
LOG_WARN("failed to wait ddl kv commit", K(ret), K(ddl_start_scn), K(build_param_));
LOG_WARN("failed to wait ddl merge", K(ret), K(ddl_start_scn), K(build_param_));
}
} else if (OB_FAIL(data_sstable_redo_writer_.write_commit_log(table_key,
prepare_scn))) {
LOG_WARN("fail write ddl commit log", K(ret), K(table_key));
}
}
return ret;

707
src/storage/ddl/ob_tablet_ddl_kv.cpp Normal file
View File

@ -0,0 +1,707 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include "storage/ddl/ob_ddl_struct.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
#include "share/scn.h"
#include "storage/blocksstable/ob_block_manager.h"
#include "storage/blocksstable/ob_block_sstable_struct.h"
#include "storage/blocksstable/ob_index_block_builder.h"
#include "storage/blocksstable/ob_macro_block_struct.h"
#include "share/ob_force_print_log.h"
#include "share/schema/ob_multi_version_schema_service.h"
#include "storage/ddl/ob_tablet_ddl_kv_mgr.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/meta_mem/ob_tablet_handle.h"
#include "storage/ddl/ob_ddl_merge_task.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "storage/compaction/ob_schedule_dag_func.h"
#include "storage/blocksstable/ob_datum_rowkey.h"
#include "storage/tablet/ob_tablet_create_delete_helper.h"
using namespace oceanbase::storage;
using namespace oceanbase::blocksstable;
using namespace oceanbase::clog;
using namespace oceanbase::share;
using namespace oceanbase::share::schema;
ObBlockMetaTree::ObBlockMetaTree()
: is_inited_(false), fifo_allocator_(), tree_allocator_(fifo_allocator_), block_tree_(tree_allocator_)
{
}
ObBlockMetaTree::~ObBlockMetaTree()
{
destroy();
}
int ObBlockMetaTree::init(const share::ObLSID &ls_id,
const ObITable::TableKey &table_key,
const share::SCN &ddl_start_scn,
const int64_t cluster_version)
{
int ret = OB_SUCCESS;
const ObMemAttr mem_attr(MTL_ID(), "DDL_KV");
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", K(ret));
} else if (OB_UNLIKELY(!ls_id.is_valid() || !table_key.is_valid() || cluster_version <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(table_key));
} else if (OB_FAIL(fifo_allocator_.init(ObMallocAllocator::get_instance(), OB_MALLOC_MIDDLE_BLOCK_SIZE, mem_attr))) {
LOG_WARN("init fifo allocator failed", K(ret));
} else if (OB_FAIL(block_tree_.init())) {
LOG_WARN("init block tree failed", K(ret));
} else if (OB_FAIL(ObTabletDDLUtil::prepare_index_data_desc(ls_id,
table_key.tablet_id_,
table_key.get_snapshot_version(),
cluster_version,
nullptr, // first ddl sstable
data_desc_))) {
LOG_WARN("prepare data store desc failed", K(ret), K(ls_id), K(table_key), K(cluster_version));
} else {
is_inited_ = true;
}
return ret;
}
int ObDDLKV::init_sstable_param(const share::ObLSID &ls_id,
const ObITable::TableKey &table_key,
const share::SCN &ddl_start_scn,
ObTabletCreateSSTableParam &sstable_param)
{
int ret = OB_SUCCESS;
ObLSService *ls_service = MTL(ObLSService *);
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
if (OB_UNLIKELY(!ls_id.is_valid() || !table_key.is_valid() || !ddl_start_scn.is_valid_and_not_min())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ls_id), K(table_key), K(ddl_start_scn));
} else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) {
LOG_WARN("get ls failed", K(ret), K(ls_id));
} else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle,
table_key.tablet_id_,
tablet_handle,
ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) {
LOG_WARN("get tablet failed", K(ret));
} else {
int64_t column_count = 0;
const ObStorageSchema &storage_schema = tablet_handle.get_obj()->get_storage_schema();
const int64_t root_block_size = sizeof(ObBlockMetaTree);
const ObDataStoreDesc &data_desc = block_meta_tree_.get_data_desc();
if (OB_FAIL(storage_schema.get_stored_column_count_in_sstable(column_count))) {
LOG_WARN("fail to get stored column count in sstable", K(ret));
} else {
sstable_param.table_key_ = table_key;
sstable_param.table_key_.table_type_ = ObITable::DDL_MEM_SSTABLE;
sstable_param.is_ready_for_read_ = true;
sstable_param.table_mode_ = storage_schema.get_table_mode_struct();
sstable_param.index_type_ = storage_schema.get_index_type();
sstable_param.rowkey_column_cnt_ = storage_schema.get_rowkey_column_num() + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt();
sstable_param.schema_version_ = storage_schema.get_schema_version();
sstable_param.create_snapshot_version_ = table_key.get_snapshot_version();
sstable_param.ddl_scn_ = ddl_start_scn;
sstable_param.root_row_store_type_ = data_desc.row_store_type_;
sstable_param.data_index_tree_height_ = 2; // fixed tree height, because there is only one root block
sstable_param.column_cnt_ = column_count;
sstable_param.contain_uncommitted_row_ = false; // ddl build major sstable with committed rows only
sstable_param.compressor_type_ = data_desc.compressor_type_;
sstable_param.encrypt_id_ = data_desc.encrypt_id_;
sstable_param.master_key_id_ = data_desc.master_key_id_;
MEMCPY(sstable_param.encrypt_key_, data_desc.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH);
sstable_param.use_old_macro_block_count_ = 0; // all new, no reuse
sstable_param.index_blocks_cnt_ = 0; // index macro block count, the index is in memory, so be 0.
sstable_param.other_block_ids_.reset(); // other blocks contains only index macro blocks now, so empty.
}
if (OB_SUCC(ret)) {
// set root block for data tree
if (OB_FAIL(sstable_param.root_block_addr_.set_mem_addr(0/*offset*/, root_block_size/*size*/))) {
LOG_WARN("set root block address for data tree failed", K(ret));
} else {
sstable_param.root_block_data_.type_ = ObMicroBlockData::DDL_BLOCK_TREE;
sstable_param.root_block_data_.buf_ = reinterpret_cast<char *>(&block_meta_tree_);
sstable_param.root_block_data_.size_ = root_block_size;
}
}
if (OB_SUCC(ret)) {
// set root block for secondary meta tree
if (OB_FAIL(sstable_param.data_block_macro_meta_addr_.set_mem_addr(0/*offset*/, root_block_size/*size*/))) {
LOG_WARN("set root block address for secondary meta tree failed", K(ret));
} else {
sstable_param.data_block_macro_meta_.type_ = ObMicroBlockData::DDL_BLOCK_TREE;
sstable_param.data_block_macro_meta_.buf_ = reinterpret_cast<char *>(&block_meta_tree_);
sstable_param.data_block_macro_meta_.size_ = root_block_size;
}
}
}
return ret;
}
void ObBlockMetaTree::destroy()
{
is_inited_ = false;
macro_blocks_.reset();
block_tree_.destroy();
data_desc_.reset();
sorted_rowkeys_.reset();
fifo_allocator_.reset();
}
int ObBlockMetaTree::insert_macro_block(const ObDDLMacroHandle &macro_handle,
const blocksstable::ObDatumRowkeyWrapper &rowkey,
const blocksstable::ObDataMacroBlockMeta *meta)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(!macro_handle.is_valid() || !rowkey.is_valid() || nullptr == meta)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(macro_handle), K(rowkey), KP(meta));
} else if (OB_FAIL(macro_blocks_.push_back(macro_handle))) {
LOG_WARN("push back macro handle failed", K(ret), K(macro_handle));
} else if (OB_FAIL(block_tree_.insert(rowkey, meta))) {
LOG_WARN("insert block tree failed", K(ret), K(rowkey), KPC(meta));
}
return ret;
}
// TODO@wenqu: direct use btree iterator
int ObBlockMetaTree::build_sorted_rowkeys()
{
int ret = OB_SUCCESS;
const int64_t version = INT64_MAX;
sorted_rowkeys_.reuse();
BtreeIterator iter;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_FAIL(block_tree_.set_key_range(iter,
ObDatumRowkeyWrapper(&ObDatumRowkey::MIN_ROWKEY, &data_desc_.datum_utils_),
false,
ObDatumRowkeyWrapper(&ObDatumRowkey::MAX_ROWKEY, &data_desc_.datum_utils_),
false,
version))) {
LOG_WARN("locate range failed", K(ret));
} else if (OB_FAIL(sorted_rowkeys_.reserve(get_macro_block_cnt()))) {
LOG_WARN("reserve sorted rowkeys failed", K(ret), K(get_macro_block_cnt()));
} else {
while (OB_SUCC(ret)) {
ObDatumRowkeyWrapper rowkey_wrapper;
const ObDataMacroBlockMeta *block_meta = nullptr;
if (OB_FAIL(iter.get_next(rowkey_wrapper, block_meta))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next failed", K(ret));
} else {
ret = OB_SUCCESS;
break;
}
} else if (OB_ISNULL(block_meta)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("block_meta is null", K(ret), KP(block_meta));
} else {
IndexItem cur_item(rowkey_wrapper.rowkey_, block_meta);
cur_item.header_.version_ = ObIndexBlockRowHeader::INDEX_BLOCK_HEADER_V1;
cur_item.header_.row_store_type_ = static_cast<uint8_t>(data_desc_.row_store_type_);
cur_item.header_.compressor_type_ = static_cast<uint8_t>(data_desc_.compressor_type_);
cur_item.header_.is_data_index_ = true;
cur_item.header_.is_data_block_ = false;
cur_item.header_.is_leaf_block_ = true;
cur_item.header_.is_macro_node_ = true;
cur_item.header_.is_major_node_ = true;
cur_item.header_.is_deleted_ = block_meta->val_.is_deleted_;
cur_item.header_.contain_uncommitted_row_ = block_meta->val_.contain_uncommitted_row_;
cur_item.header_.macro_id_ = block_meta->val_.macro_id_;
cur_item.header_.block_offset_ = block_meta->val_.block_offset_;
cur_item.header_.block_size_ = block_meta->val_.block_size_;
cur_item.header_.macro_block_count_ = 1;
cur_item.header_.micro_block_count_ = block_meta->val_.micro_block_count_;
cur_item.header_.master_key_id_ = data_desc_.master_key_id_;
cur_item.header_.encrypt_id_ = data_desc_.encrypt_id_;
MEMCPY(cur_item.header_.encrypt_key_, data_desc_.encrypt_key_, sizeof(cur_item.header_.encrypt_key_));
cur_item.header_.schema_version_ = data_desc_.schema_version_;
cur_item.header_.row_count_ = block_meta->val_.row_count_;
if (OB_UNLIKELY(!cur_item.header_.is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Built an invalid index block row", K(ret), K(cur_item));
} else if (OB_FAIL(sorted_rowkeys_.push_back(cur_item))) {
LOG_WARN("push back index item failed", K(ret), K(rowkey_wrapper), KPC(block_meta));
}
}
}
}
return ret;
}
bool ObBlockMetaTree::CompareFunctor::operator ()(const IndexItem &item,
const blocksstable::ObDatumRowkey &rowkey)
{
int cmp_ret = 0;
item.rowkey_->compare(rowkey, datum_utils_, cmp_ret);
return cmp_ret < 0;
}
bool ObBlockMetaTree::CompareFunctor::operator ()(const blocksstable::ObDatumRowkey &rowkey,
const IndexItem &item)
{
int cmp_ret = 0;
item.rowkey_->compare(rowkey, datum_utils_, cmp_ret);
return cmp_ret > 0;
}
int ObBlockMetaTree::locate_range(const blocksstable::ObDatumRange &range,
const blocksstable::ObStorageDatumUtils &datum_utils,
const bool is_left_border,
const bool is_right_border,
int64_t &begin_idx,
int64_t &end_idx)
{
int ret = OB_SUCCESS;
begin_idx = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
end_idx = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (sorted_rowkeys_.empty()) {
// do nothing
} else {
CompareFunctor cmp(datum_utils);
if (!is_left_border || range.get_start_key().is_min_rowkey()) {
begin_idx = 0;
} else {
if (range.is_left_closed()) {
begin_idx = std::lower_bound(sorted_rowkeys_.begin(), sorted_rowkeys_.end(), range.get_start_key(), cmp) - sorted_rowkeys_.begin();
} else {
begin_idx = std::upper_bound(sorted_rowkeys_.begin(), sorted_rowkeys_.end(), range.get_start_key(), cmp) - sorted_rowkeys_.begin();
}
if (sorted_rowkeys_.count() == begin_idx) {
ret = OB_BEYOND_THE_RANGE;
}
}
if (OB_SUCC(ret)) {
if (!is_right_border || range.get_end_key().is_max_rowkey()) {
end_idx = sorted_rowkeys_.count() - 1;
} else {
if (range.is_right_closed()) {
end_idx = std::lower_bound(sorted_rowkeys_.begin(), sorted_rowkeys_.end(), range.get_end_key(), cmp) - sorted_rowkeys_.begin();
} else {
end_idx = std::upper_bound(sorted_rowkeys_.begin(), sorted_rowkeys_.end(), range.get_end_key(), cmp) - sorted_rowkeys_.begin();
}
if (sorted_rowkeys_.count() == end_idx) {
end_idx = sorted_rowkeys_.count() - 1;
}
}
}
}
if (OB_FAIL(ret)) {
begin_idx = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
end_idx = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
}
return ret;
}
int ObBlockMetaTree::get_index_block_row_header(const int64_t idx,
const ObIndexBlockRowHeader *&idx_header,
blocksstable::ObDatumRowkey &endkey)
{
int ret = OB_SUCCESS;
idx_header = nullptr;
endkey.reset();
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(idx < 0 || idx >= sorted_rowkeys_.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(idx), K(sorted_rowkeys_.count()));
} else {
IndexItem &cur_item = sorted_rowkeys_.at(idx);
if (OB_FAIL(cur_item.block_meta_->get_rowkey(endkey))) {
LOG_WARN("get endkey failed", K(ret), K(cur_item));
} else {
idx_header = &cur_item.header_;
}
}
return ret;
}
int ObBlockMetaTree::get_macro_block_meta(const int64_t idx,
ObDataMacroBlockMeta &macro_meta)
{
int ret = OB_SUCCESS;
macro_meta.reset();
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_UNLIKELY(idx < 0 || idx >= sorted_rowkeys_.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret));
} else {
const ObDataMacroBlockMeta &found_meta = *sorted_rowkeys_.at(idx).block_meta_;
if (OB_FAIL(macro_meta.assign(found_meta))) {
LOG_WARN("assign macro meta failed", K(ret), K(found_meta));
}
}
return ret;
}
int ObBlockMetaTree::get_last_rowkey(const ObDatumRowkey *&last_rowkey)
{
int ret = OB_SUCCESS;
if (sorted_rowkeys_.count() > 0) {
last_rowkey = sorted_rowkeys_.at(sorted_rowkeys_.count() - 1).rowkey_;
} else {
last_rowkey = &ObDatumRowkey::MAX_ROWKEY;
}
return ret;
}
ObDDLKV::ObDDLKV()
: is_inited_(false), ls_id_(), tablet_id_(), ddl_start_scn_(SCN::min_scn()), snapshot_version_(0),
lock_(), arena_allocator_("DDL_KV"), is_freezed_(false), is_closed_(false), last_freezed_scn_(SCN::min_scn()),
min_scn_(SCN::max_scn()), max_scn_(SCN::min_scn()), freeze_scn_(SCN::max_scn()), pending_cnt_(0), cluster_version_(0),
sstable_index_builder_(nullptr), index_block_rebuilder_(nullptr), is_rebuilder_closed_(false)
{
}
ObDDLKV::~ObDDLKV()
{
reset();
}
int ObDDLKV::init(const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const SCN &ddl_start_scn,
const int64_t snapshot_version,
const SCN &last_freezed_scn,
const int64_t cluster_version)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("ObDDLKV has been inited twice", K(ret), KP(this));
} else if (OB_UNLIKELY(!ls_id.is_valid()
|| !tablet_id.is_valid()
|| !ddl_start_scn.is_valid_and_not_min()
|| snapshot_version <= 0
|| !last_freezed_scn.is_valid_and_not_min()
|| cluster_version < 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(ddl_start_scn), K(snapshot_version), K(last_freezed_scn), K(cluster_version));
} else {
ObTabletDDLParam ddl_param;
ddl_param.tenant_id_ = MTL_ID();
ddl_param.ls_id_ = ls_id;
ddl_param.table_key_.tablet_id_ = tablet_id;
ddl_param.table_key_.table_type_ = ObITable::TableType::MAJOR_SSTABLE;
ddl_param.table_key_.version_range_.base_version_ = 0;
ddl_param.table_key_.version_range_.snapshot_version_ = snapshot_version;
ddl_param.start_scn_ = ddl_start_scn;
ddl_param.snapshot_version_ = snapshot_version;
ddl_param.cluster_version_ = cluster_version;
ObTabletCreateSSTableParam sstable_param;
if (OB_FAIL(ObTabletDDLUtil::prepare_index_builder(ddl_param, arena_allocator_, ObSSTableIndexBuilder::DISABLE,
nullptr/*first_ddl_sstable*/, sstable_index_builder_, index_block_rebuilder_))) {
LOG_WARN("prepare index builder failed", K(ret));
} else if (OB_FAIL(block_meta_tree_.init(ls_id, ddl_param.table_key_, ddl_start_scn, cluster_version))) {
LOG_WARN("init mem index sstable failed", K(ret), K(ddl_param));
} else if (OB_FAIL(init_sstable_param(ls_id, ddl_param.table_key_, ddl_start_scn, sstable_param))) {
LOG_WARN("init sstable param failed", K(ret));
} else if (OB_FAIL(ObSSTable::init(sstable_param, &arena_allocator_))) {
LOG_WARN("init sstable failed", K(ret));
} else {
ls_id_ = ls_id;
tablet_id_ = tablet_id;
ddl_start_scn_ = ddl_start_scn;
snapshot_version_ = snapshot_version;
last_freezed_scn_ = last_freezed_scn;
cluster_version_ = cluster_version;
is_inited_ = true;
LOG_INFO("ddl kv init success", K(ls_id_), K(tablet_id_), K(ddl_start_scn_), K(snapshot_version_), K(last_freezed_scn_), K(cluster_version_), KP(this));
}
}
return ret;
}
void ObDDLKV::reset()
{
FLOG_INFO("ddl kv reset", KP(this), K(*this));
is_inited_ = false;
ls_id_.reset();
tablet_id_.reset();
ddl_start_scn_ = SCN::min_scn();
snapshot_version_ = 0;
is_freezed_ = false;
is_closed_ = false;
last_freezed_scn_ = SCN::min_scn();
min_scn_ = SCN::max_scn();
max_scn_ = SCN::min_scn();
freeze_scn_ = SCN::max_scn();
pending_cnt_ = 0;
cluster_version_ = 0;
is_rebuilder_closed_ = false;
if (nullptr != index_block_rebuilder_) {
index_block_rebuilder_->~ObIndexBlockRebuilder();
arena_allocator_.free(index_block_rebuilder_);
index_block_rebuilder_ = nullptr;
}
if (nullptr != sstable_index_builder_) {
sstable_index_builder_->~ObSSTableIndexBuilder();
arena_allocator_.free(sstable_index_builder_);
sstable_index_builder_ = nullptr;
}
block_meta_tree_.destroy();
ObSSTable::reset();
arena_allocator_.reset();
}
int ObDDLKV::set_macro_block(const ObDDLMacroBlock &macro_block)
{
int ret = OB_SUCCESS;
const int64_t MAX_DDL_BLOCK_COUNT = 10L * 1024L * 1024L * 1024L / OB_SERVER_BLOCK_MGR.get_macro_block_size();
int64_t freeze_block_count = MAX_DDL_BLOCK_COUNT;
#ifdef ERRSIM
if (0 != GCONF.errsim_max_ddl_block_count) {
freeze_block_count = GCONF.errsim_max_ddl_block_count;
LOG_INFO("ddl set macro block count", K(freeze_block_count));
}
#endif
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ddl kv is not init", K(ret));
} else if (OB_UNLIKELY(!macro_block.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(macro_block));
} else {
const uint64_t tenant_id = MTL_ID();
ObUnitInfoGetter::ObTenantConfig unit;
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(GCTX.omt_->get_tenant_unit(tenant_id, unit))) {
LOG_WARN("get tenant unit failed", K(tmp_ret), K(tenant_id));
} else {
const int64_t log_allowed_block_count = unit.config_.log_disk_size() * 0.2 / OB_SERVER_BLOCK_MGR.get_macro_block_size();
if (log_allowed_block_count <= 0) {
tmp_ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid macro block count by log disk size", K(tmp_ret), K(tenant_id), K(unit.config_));
} else {
freeze_block_count = min(freeze_block_count, log_allowed_block_count);
}
}
}
if (OB_SUCC(ret) && get_macro_block_cnt() >= freeze_block_count) {
ObDDLTableMergeDagParam param;
param.ls_id_ = ls_id_;
param.tablet_id_ = tablet_id_;
param.start_scn_ = ddl_start_scn_;
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) {
LOG_WARN("try schedule ddl merge dag failed when ddl kv is full ",
K(tmp_ret), K(ls_id_), K(tablet_id_), K(get_macro_block_cnt()));
}
}
if (OB_SUCC(ret)) {
ObDataMacroBlockMeta *data_macro_meta = nullptr;
TCWLockGuard guard(lock_);
if (macro_block.ddl_start_scn_ != ddl_start_scn_) {
if (macro_block.ddl_start_scn_ > ddl_start_scn_) {
ret = OB_EAGAIN;
LOG_INFO("ddl start scn too large, retry", K(ret),
K(ls_id_), K(tablet_id_), K(ddl_start_scn_), K(macro_block));
} else {
// filter out and do nothing
LOG_INFO("ddl start scn too small, maybe from old build task, ignore", K(ret),
K(ls_id_), K(tablet_id_), K(ddl_start_scn_), K(macro_block));
}
} else if (macro_block.scn_ > freeze_scn_) {
ret = OB_EAGAIN;
LOG_INFO("this ddl kv is freezed, retry other ddl kv", K(ret), K(ls_id_), K(tablet_id_), K(macro_block), K(freeze_scn_));
} else if (OB_FAIL(index_block_rebuilder_->get_macro_meta(macro_block.buf_, macro_block.size_, macro_block.get_block_id(), arena_allocator_, data_macro_meta))) {
LOG_WARN("get macro meta failed", K(ret), K(macro_block));
} else if (OB_FAIL(index_block_rebuilder_->append_macro_row(
macro_block.buf_, macro_block.size_, macro_block.get_block_id()))) {
LOG_WARN("append macro meta failed", K(ret), K(macro_block));
} else if (OB_FAIL(insert_block_meta_tree(macro_block.block_handle_, data_macro_meta))) {
LOG_WARN("insert macro block failed", K(ret), K(macro_block), KPC(data_macro_meta));
} else {
min_scn_ = SCN::min(min_scn_, macro_block.scn_);
max_scn_ = SCN::max(max_scn_, macro_block.scn_);
LOG_INFO("succeed to set macro block into ddl kv", K(macro_block));
}
}
return ret;
}
int ObDDLKV::insert_block_meta_tree(const ObDDLMacroHandle &macro_handle, blocksstable::ObDataMacroBlockMeta *data_macro_meta)
{
int ret = OB_SUCCESS;
if (OB_FAIL(block_meta_tree_.insert_macro_block(macro_handle,
ObDatumRowkeyWrapper(&data_macro_meta->end_key_, &sstable_index_builder_->get_index_store_desc().datum_utils_),
data_macro_meta))) {
LOG_WARN("insert macro block failed", K(ret), K(macro_handle), KPC(data_macro_meta));
} else {
const ObDataBlockMetaVal &meta_val = data_macro_meta->get_meta_val();
meta_.get_basic_meta().data_macro_block_count_ += 1;
meta_.get_basic_meta().data_micro_block_count_ += meta_val.micro_block_count_;
meta_.get_basic_meta().max_merged_trans_version_ = max(meta_.get_basic_meta().max_merged_trans_version_, meta_val.max_merged_trans_version_);
meta_.get_basic_meta().row_count_ += meta_val.row_count_;
meta_.get_basic_meta().data_checksum_ = ob_crc64_sse42(meta_.get_basic_meta().data_checksum_, &meta_val.data_checksum_, sizeof(meta_val.data_checksum_));
meta_.get_basic_meta().occupy_size_ += meta_val.occupy_size_;
meta_.get_basic_meta().original_size_ += meta_val.original_size_;
}
return ret;
}
int ObDDLKV::freeze(const SCN &freeze_scn)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ddl kv is not init", K(ret));
} else {
TCWLockGuard guard(lock_);
if (is_freezed_) {
// do nothing
} else {
if (freeze_scn.is_valid_and_not_min()) {
freeze_scn_ = freeze_scn;
} else if (max_scn_.is_valid_and_not_min()) {
freeze_scn_ = max_scn_;
} else {
ret = OB_EAGAIN;
LOG_INFO("ddl kv not freezed, try again", K(ret), K(ls_id_), K(tablet_id_), K(get_macro_block_cnt()));
}
if (OB_SUCC(ret)) {
ATOMIC_SET(&is_freezed_, true);
LOG_INFO("ddl kv freezed", K(ret), K(ls_id_), K(tablet_id_), K(get_macro_block_cnt()));
}
}
}
return ret;
}
int ObDDLKV::prepare_sstable()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ddl kv is not init", K(ret));
} else if (!is_freezed()) {
ret = OB_STATE_NOT_MATCH;
LOG_WARN("ddl kv not freezed", K(ret), K(*this));
} else if (OB_FAIL(wait_pending())) {
if (OB_EAGAIN != ret) {
LOG_WARN("wait pending failed", K(ret));
}
} else if (OB_FAIL(block_meta_tree_.build_sorted_rowkeys())) {
LOG_WARN("build sorted keys failed", K(ret), K(block_meta_tree_));
} else {
key_.scn_range_.start_scn_ = last_freezed_scn_;
key_.scn_range_.end_scn_ = freeze_scn_;
}
return ret;
}
int ObDDLKV::close()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ddl kv is not init", K(ret));
} else if (is_closed_) {
// do nothing
LOG_INFO("ddl kv already closed", K(*this));
} else if (OB_FAIL(wait_pending())) {
if (OB_EAGAIN != ret) {
LOG_WARN("wait pending failed", K(ret));
}
} else if (!is_rebuilder_closed_) {
if (OB_FAIL(index_block_rebuilder_->close())) {
LOG_WARN("index block rebuilder close failed", K(ret));
} else {
is_rebuilder_closed_ = true;
}
}
if (OB_SUCC(ret) && !is_closed_) {
ObTableHandleV2 table_handle;
ObTabletDDLParam ddl_param;
ddl_param.tenant_id_ = MTL_ID();
ddl_param.ls_id_ = ls_id_;
ddl_param.table_key_.tablet_id_ = tablet_id_;
ddl_param.table_key_.table_type_ = ObITable::TableType::DDL_DUMP_SSTABLE;
ddl_param.table_key_.scn_range_.start_scn_ = last_freezed_scn_;
ddl_param.table_key_.scn_range_.end_scn_ = freeze_scn_;
ddl_param.start_scn_ = ddl_start_scn_;
ddl_param.snapshot_version_ = snapshot_version_;
ddl_param.cluster_version_ = cluster_version_;
if (OB_FAIL(ObTabletDDLUtil::update_ddl_table_store(sstable_index_builder_,
ddl_param,
nullptr/*first_ddl_sstable*/,
table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret));
} else {
is_closed_ = true;
LOG_INFO("ddl kv closed success", K(*this));
}
}
return ret;
}
void ObDDLKV::inc_pending_cnt()
{
ATOMIC_INC(&pending_cnt_);
}
void ObDDLKV::dec_pending_cnt()
{
ATOMIC_DEC(&pending_cnt_);
}
int ObDDLKV::wait_pending()
{
int ret = OB_SUCCESS;
ObLSService *ls_service = MTL(ObLSService *);
ObLSHandle ls_handle;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
} else if (OB_UNLIKELY(!is_freezed())) {
ret = OB_STATE_NOT_MATCH;
LOG_WARN("ddl kv not freezed", K(ret));
} else if (OB_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) {
LOG_WARN("get ls handle failed", K(ret), K(ls_id_));
} else {
SCN max_decided_scn;
if (OB_FAIL(ls_handle.get_ls()->get_max_decided_scn(max_decided_scn))) {
LOG_WARN("get max decided log ts failed", K(ret), K(ls_id_));
} else {
// max_decided_scn is the left border scn - 1
// the min deciding(replay or apply) scn (aka left border) is max_decided_scn + 1
const bool pending_finished = SCN::plus(max_decided_scn, 1) >= freeze_scn_ && !is_pending();
if (!pending_finished) {
ret = OB_EAGAIN;
//if (REACH_TIME_INTERVAL(1000L * 1000L)) {
LOG_INFO("wait pending not finish", K(ret), K(*this), K(max_decided_scn));
//}
}
}
}
return ret;
}

177
src/storage/ddl/ob_tablet_ddl_kv.h Normal file
View File

@ -0,0 +1,177 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_STORAGE_DDL_OB_TABLET_DDL_KV_H_
#define OCEANBASE_STORAGE_DDL_OB_TABLET_DDL_KV_H_
#include "lib/hash/ob_hashmap.h"
#include "lib/lock/ob_mutex.h"
#include "share/scn.h"
#include "storage/ob_i_table.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/blocksstable/ob_block_sstable_struct.h"
#include "storage/blocksstable/ob_index_block_builder.h"
#include "storage/checkpoint/ob_freeze_checkpoint.h"
#include "storage/memtable/mvcc/ob_keybtree.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "storage/ddl/ob_ddl_struct.h"
#include "storage/blocksstable/ob_sstable.h"
namespace oceanbase
{
namespace blocksstable
{
struct ObSSTableMergeRes;
struct ObDatumRowkey;
struct ObDatumRowkeyWrapper;
struct ObStorageDatumUtils;
class ObDataMacroBlockMeta;
}
namespace storage
{
class ObBlockMetaTree
{
typedef keybtree::ObKeyBtree<blocksstable::ObDatumRowkeyWrapper, const blocksstable::ObDataMacroBlockMeta *> KeyBtree;
typedef keybtree::BtreeIterator<blocksstable::ObDatumRowkeyWrapper, const blocksstable::ObDataMacroBlockMeta *> BtreeIterator;
typedef keybtree::BtreeNodeAllocator<blocksstable::ObDatumRowkeyWrapper, const blocksstable::ObDataMacroBlockMeta *> BtreeNodeAllocator;
typedef keybtree::BtreeRawIterator<blocksstable::ObDatumRowkeyWrapper, const blocksstable::ObDataMacroBlockMeta *> BtreeRawIterator;
public:
ObBlockMetaTree();
virtual ~ObBlockMetaTree();
int init(const share::ObLSID &ls_id,
const ObITable::TableKey &table_key,
const share::SCN &ddl_start_scn,
const int64_t cluster_version);
void destroy();
int insert_macro_block(const ObDDLMacroHandle &macro_handle,
const blocksstable::ObDatumRowkeyWrapper &rowkey,
const blocksstable::ObDataMacroBlockMeta *meta);
int locate_range(const blocksstable::ObDatumRange &range,
const blocksstable::ObStorageDatumUtils &datum_utils,
const bool is_left_border,
const bool is_right_border,
int64_t &begin_idx,
int64_t &end_idx);
int get_index_block_row_header(const int64_t idx,
const blocksstable::ObIndexBlockRowHeader *&header,
blocksstable::ObDatumRowkey &endkey);
int get_macro_block_meta(const int64_t idx,
blocksstable::ObDataMacroBlockMeta &macro_meta);
int64_t get_macro_block_cnt() const { return macro_blocks_.count(); }
int get_last_rowkey(const blocksstable::ObDatumRowkey *&last_rowkey);
int build_sorted_rowkeys();
const blocksstable::ObDataStoreDesc &get_data_desc() const { return data_desc_; }
TO_STRING_KV(K(is_inited_), K(macro_blocks_.count()), K(sorted_rowkeys_.count()), K(data_desc_));
private:
struct IndexItem final
{
public:
IndexItem() : rowkey_(nullptr), block_meta_(nullptr), is_header_valid_(false) {}
IndexItem(const blocksstable::ObDatumRowkey *rowkey,
const blocksstable::ObDataMacroBlockMeta *block_meta)
: rowkey_(rowkey), block_meta_(block_meta), is_header_valid_(false) {}
TO_STRING_KV(KPC_(rowkey), KPC_(block_meta), K_(header));
const blocksstable::ObDatumRowkey *rowkey_;
const blocksstable::ObDataMacroBlockMeta *block_meta_;
blocksstable::ObIndexBlockRowHeader header_;
bool is_header_valid_;
};
struct CompareFunctor
{
CompareFunctor(const blocksstable::ObStorageDatumUtils &datum_utils) : datum_utils_(datum_utils) {}
bool operator ()(const IndexItem &item, const blocksstable::ObDatumRowkey &rowkey);
bool operator ()(const blocksstable::ObDatumRowkey &rowkey, const IndexItem &item);
const blocksstable::ObStorageDatumUtils &datum_utils_;
};
private:
bool is_inited_;
ObArray<ObDDLMacroHandle> macro_blocks_;
ObFIFOAllocator fifo_allocator_;
BtreeNodeAllocator tree_allocator_;
KeyBtree block_tree_;
blocksstable::ObDataStoreDesc data_desc_;
ObArray<IndexItem> sorted_rowkeys_;
};
class ObDDLKV : public blocksstable::ObSSTable
{
public:
ObDDLKV();
virtual ~ObDDLKV();
int init(const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const share::SCN &ddl_start_scn,
const int64_t snapshot_version,
const share::SCN &last_freezed_scn,
const int64_t cluster_version);
void reset();
int set_macro_block(const ObDDLMacroBlock &macro_block);
int freeze(const share::SCN &freeze_scn);
bool is_freezed() const { return ATOMIC_LOAD(&is_freezed_); }
int close();
int prepare_sstable();
bool is_closed() const { return is_closed_; }
share::SCN get_min_scn() const { return min_scn_; }
share::SCN get_freeze_scn() const { return freeze_scn_; }
share::SCN get_ddl_start_scn() const { return ddl_start_scn_; }
int64_t get_macro_block_cnt() const { return block_meta_tree_.get_macro_block_cnt(); }
void inc_pending_cnt(); // used by ddl kv pending guard
void dec_pending_cnt();
bool is_pending() const { return ATOMIC_LOAD(&pending_cnt_) > 0; }
int wait_pending();
TO_STRING_KV(K_(is_inited), K_(ls_id), K_(tablet_id), K_(ddl_start_scn), K_(snapshot_version),
K_(is_freezed), K_(is_closed),
K_(last_freezed_scn), K_(min_scn), K_(max_scn), K_(freeze_scn),
K_(pending_cnt), K_(cluster_version), K_(ref_cnt),
KP_(sstable_index_builder), KP_(index_block_rebuilder), K_(is_rebuilder_closed),
K_(block_meta_tree));
private:
int insert_block_meta_tree(const ObDDLMacroHandle &macro_handle,
blocksstable::ObDataMacroBlockMeta *data_macro_meta);
int init_sstable_param(const share::ObLSID &ls_id,
const ObITable::TableKey &table_key,
const share::SCN &ddl_start_scn,
ObTabletCreateSSTableParam &sstable_param);
private:
static const int64_t TOTAL_LIMIT = 10 * 1024 * 1024 * 1024L;
static const int64_t HOLD_LIMIT = 10 * 1024 * 1024 * 1024L;
bool is_inited_;
share::ObLSID ls_id_;
common::ObTabletID tablet_id_;
share::SCN ddl_start_scn_; // the log ts of ddl start log
int64_t snapshot_version_; // the snapshot version for major sstable which is completed by ddl
common::TCRWLock lock_; // lock for block_meta_tree_ and freeze_log_ts_
common::ObArenaAllocator arena_allocator_;
bool is_freezed_;
bool is_closed_;
share::SCN last_freezed_scn_; // the freezed log ts of last ddl kv. the log ts range of this ddl kv is (last_freezed_log_ts_, freeze_log_ts_]
share::SCN min_scn_; // the min log ts of macro blocks
share::SCN max_scn_; // the max log ts of macro blocks
share::SCN freeze_scn_; // ddl kv refuse data larger than freeze log ts, freeze_log_ts >= max_log_ts
int64_t pending_cnt_; // the amount of kvs that are replaying
int64_t cluster_version_;
blocksstable::ObSSTableIndexBuilder *sstable_index_builder_;
blocksstable::ObIndexBlockRebuilder *index_block_rebuilder_;
bool is_rebuilder_closed_;
ObBlockMetaTree block_meta_tree_;
};
} // end namespace storage
} // end namespace oceanbase
#endif

347
src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp
View File

@ -16,11 +16,13 @@
#include "share/scn.h"
#include "share/ob_force_print_log.h"
#include "storage/ddl/ob_ddl_struct.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
#include "storage/ddl/ob_ddl_merge_task.h"
#include "storage/blocksstable/ob_sstable_sec_meta_iterator.h"
#include "storage/compaction/ob_schedule_dag_func.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
using namespace oceanbase::common;
using namespace oceanbase::blocksstable;
@ -29,10 +31,10 @@ using namespace oceanbase::storage;
ObTabletDDLKvMgr::ObTabletDDLKvMgr()
: is_inited_(false), success_start_scn_(SCN::min_scn()), ls_id_(), tablet_id_(), table_key_(), cluster_version_(0),
start_scn_(SCN::min_scn()), max_freeze_scn_(SCN::min_scn()),
table_id_(0), execution_id_(-1), head_(0), tail_(0), lock_(ObLatchIds::TABLET_DDL_KV_MGR_LOCK), ref_cnt_(0)
start_scn_(SCN::min_scn()), commit_scn_(SCN::min_scn()), max_freeze_scn_(SCN::min_scn()),
table_id_(0), execution_id_(-1), head_(0), tail_(0), lock_(ObLatchIds::TABLET_DDL_KV_MGR_LOCK), ref_cnt_(0),
can_schedule_major_compaction_(false)
{
MEMSET(ddl_kvs_, 0, MAX_DDL_KV_CNT_IN_STORAGE * sizeof(ddl_kvs_[0]));
}
ObTabletDDLKvMgr::~ObTabletDDLKvMgr()
@ -53,17 +55,21 @@ void ObTabletDDLKvMgr::destroy()
}
head_ = 0;
tail_ = 0;
MEMSET(ddl_kvs_, 0, sizeof(ddl_kvs_));
for (int64_t i = 0; i < MAX_DDL_KV_CNT_IN_STORAGE; ++i) {
ddl_kv_handles_[i].reset();
}
ls_id_.reset();
tablet_id_.reset();
table_key_.reset();
cluster_version_ = 0;
start_scn_.set_min();
commit_scn_.set_min();
max_freeze_scn_.set_min();
table_id_ = 0;
execution_id_ = -1;
success_start_scn_.set_min();
is_inited_ = false;
can_schedule_major_compaction_ = false;
}
int ObTabletDDLKvMgr::init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id)
@ -148,14 +154,12 @@ int ObTabletDDLKvMgr::ddl_start(const ObITable::TableKey &table_key,
return ret;
}
int ObTabletDDLKvMgr::ddl_prepare(
const SCN &start_scn,
const SCN &prepare_scn,
const uint64_t table_id,
const int64_t ddl_task_id)
int ObTabletDDLKvMgr::ddl_commit(const SCN &start_scn,
const SCN &commit_scn,
const uint64_t table_id,
const int64_t ddl_task_id)
{
int ret = OB_SUCCESS;
ObDDLKVHandle kv_handle;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
@ -164,17 +168,27 @@ int ObTabletDDLKvMgr::ddl_prepare(
LOG_WARN("ddl not started", K(ret));
} else if (start_scn < start_scn_) {
ret = OB_TASK_EXPIRED;
LOG_INFO("skip ddl prepare log", K(start_scn), K(*this));
} else if (OB_FAIL(freeze_ddl_kv(prepare_scn))) {
LOG_WARN("freeze ddl kv failed", K(ret), K(prepare_scn));
LOG_INFO("skip ddl commit log", K(start_scn), K(*this));
} else if (OB_FAIL(freeze_ddl_kv(commit_scn))) {
LOG_WARN("freeze ddl kv failed", K(ret), K(commit_scn));
} else {
ret = OB_EAGAIN;
while (OB_EAGAIN == ret) {
if (OB_FAIL(update_ddl_major_sstable())) {
LOG_WARN("update ddl major sstable failed", K(ret));
}
if (OB_EAGAIN == ret) {
usleep(1000L);
}
}
table_id_ = table_id;
ddl_task_id_ = ddl_task_id;
commit_scn_ = commit_scn;
ObDDLTableMergeDagParam param;
param.ls_id_ = ls_id_;
param.tablet_id_ = tablet_id_;
param.rec_scn_ = prepare_scn;
param.rec_scn_ = commit_scn;
param.is_commit_ = true;
param.start_scn_ = start_scn;
param.table_id_ = table_id;
@ -191,12 +205,12 @@ int ObTabletDDLKvMgr::ddl_prepare(
ob_usleep(10L * 1000L);
if (REACH_TIME_INTERVAL(10L * 1000L * 1000L)) {
LOG_INFO("retry schedule ddl commit task",
K(start_scn), K(prepare_scn), K(table_id), K(ddl_task_id), K(*this),
K(start_scn), K(commit_scn), K(table_id), K(ddl_task_id), K(*this),
"wait_elpased_s", (ObTimeUtility::fast_current_time() - start_ts) / 1000000L);
}
}
} else {
LOG_INFO("schedule ddl commit task success", K(start_scn), K(prepare_scn), K(table_id), K(ddl_task_id), K(*this));
LOG_INFO("schedule ddl commit task success", K(start_scn), K(commit_scn), K(table_id), K(ddl_task_id), K(*this));
break;
}
}
@ -204,10 +218,7 @@ int ObTabletDDLKvMgr::ddl_prepare(
return ret;
}
int ObTabletDDLKvMgr::ddl_commit(
const SCN &start_scn,
const SCN &prepare_scn,
const bool is_replay)
int ObTabletDDLKvMgr::schedule_ddl_merge_task(const SCN &start_scn, const SCN &commit_scn, const bool is_replay)
{
int ret = OB_SUCCESS;
ObLSHandle ls_handle;
@ -215,10 +226,10 @@ int ObTabletDDLKvMgr::ddl_commit(
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
} else if (is_commit_success()) {
FLOG_INFO("ddl commit already succeed", K(start_scn), K(prepare_scn), K(*this));
FLOG_INFO("ddl commit already succeed", K(start_scn), K(commit_scn), K(*this));
} else if (start_scn < start_scn_) {
ret = OB_TASK_EXPIRED;
LOG_INFO("skip ddl commit log", K(start_scn), K(prepare_scn), K(*this));
LOG_INFO("skip ddl commit log", K(start_scn), K(commit_scn), K(*this));
} else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) {
LOG_WARN("failed to get log stream", K(ret), K(ls_id_));
} else {
@ -226,7 +237,7 @@ int ObTabletDDLKvMgr::ddl_commit(
ObDDLTableMergeDagParam param;
param.ls_id_ = ls_id_;
param.tablet_id_ = tablet_id_;
param.rec_scn_ = prepare_scn;
param.rec_scn_ = commit_scn;
param.is_commit_ = true;
param.start_scn_ = start_scn;
param.table_id_ = table_id_;
@ -248,7 +259,7 @@ int ObTabletDDLKvMgr::ddl_commit(
ret = OB_SUCCESS; // think as succcess for replay
} else {
if (REACH_TIME_INTERVAL(10L * 1000L * 1000L)) {
LOG_INFO("replay ddl commit", K(ret), K(start_scn), K(prepare_scn), K(*this));
LOG_INFO("replay ddl commit", K(ret), K(start_scn), K(commit_scn), K(*this));
}
ret = OB_EAGAIN; // retry by replay service
}
@ -256,17 +267,15 @@ int ObTabletDDLKvMgr::ddl_commit(
return ret;
}
int ObTabletDDLKvMgr::wait_ddl_commit(
const SCN &start_scn,
const SCN &prepare_scn)
int ObTabletDDLKvMgr::wait_ddl_merge_success(const SCN &start_scn, const SCN &commit_scn)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
} else if (OB_UNLIKELY(!start_scn.is_valid_and_not_min() || !prepare_scn.is_valid_and_not_min())) {
} else if (OB_UNLIKELY(!start_scn.is_valid_and_not_min() || !commit_scn.is_valid_and_not_min())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(start_scn), K(prepare_scn));
LOG_WARN("invalid argument", K(ret), K(start_scn), K(commit_scn));
} else if (!is_started()) {
ret = OB_STATE_NOT_MATCH;
LOG_WARN("ddl not started", K(ret));
@ -276,18 +285,18 @@ int ObTabletDDLKvMgr::wait_ddl_commit(
} else {
const int64_t wait_start_ts = ObTimeUtility::fast_current_time();
while (OB_SUCC(ret)) {
if (OB_FAIL(ddl_commit(start_scn, prepare_scn, false/*is_replay*/ ))) {
if (OB_FAIL(schedule_ddl_merge_task(start_scn, commit_scn, false/*is_replay*/))) {
if (OB_EAGAIN == ret) {
ob_usleep(10L * 1000L);
ret = OB_SUCCESS; // retry
} else {
LOG_WARN("commit ddl log failed", K(ret), K(start_scn), K(prepare_scn), K(ls_id_), K(tablet_id_));
LOG_WARN("commit ddl log failed", K(ret), K(start_scn), K(commit_scn), K(ls_id_), K(tablet_id_));
}
} else {
break;
}
if (REACH_TIME_INTERVAL(10L * 1000L * 1000L)) {
LOG_INFO("wait build ddl sstable", K(ret), K(ls_id_), K(tablet_id_), K(start_scn_), K(prepare_scn), K(max_freeze_scn_),
LOG_INFO("wait build ddl sstable", K(ret), K(ls_id_), K(tablet_id_), K(start_scn_), K(commit_scn), K(max_freeze_scn_),
"wait_elpased_s", (ObTimeUtility::fast_current_time() - wait_start_ts) / 1000000L);
}
}
@ -295,6 +304,20 @@ int ObTabletDDLKvMgr::wait_ddl_commit(
return ret;
}
int ObTabletDDLKvMgr::get_ddl_major_merge_param(ObDDLTableMergeDagParam &param)
{
int ret = OB_SUCCESS;
param.ls_id_ = ls_id_;
param.tablet_id_ = tablet_id_;
param.rec_scn_ = commit_scn_;
param.is_commit_ = true;
param.start_scn_ = start_scn_;
param.table_id_ = table_id_;
param.execution_id_ = execution_id_;
param.ddl_task_id_ = ddl_task_id_;
return ret;
}
int ObTabletDDLKvMgr::set_commit_success(const SCN &start_scn)
{
int ret = OB_SUCCESS;
@ -329,9 +352,20 @@ int ObTabletDDLKvMgr::set_commit_success(const SCN &start_scn)
bool ObTabletDDLKvMgr::is_commit_success() const
{
TCRLockGuard guard(lock_);
return is_commit_success_unlock();
}
bool ObTabletDDLKvMgr::is_commit_success_unlock() const
{
return success_start_scn_ > SCN::min_scn() && success_start_scn_ == start_scn_;
}
bool ObTabletDDLKvMgr::can_schedule_major_compaction() const
{
TCRLockGuard guard(lock_);
return can_schedule_major_compaction_ && !is_commit_success_unlock();
}
int ObTabletDDLKvMgr::cleanup()
{
int ret = OB_SUCCESS;
@ -354,14 +388,18 @@ void ObTabletDDLKvMgr::cleanup_unlock()
}
head_ = 0;
tail_ = 0;
MEMSET(ddl_kvs_, 0, sizeof(ddl_kvs_));
for (int64_t i = 0; i < MAX_DDL_KV_CNT_IN_STORAGE; ++i) {
ddl_kv_handles_[i].reset();
}
table_key_.reset();
cluster_version_ = 0;
start_scn_.set_min();
commit_scn_.set_min();
max_freeze_scn_.set_min();
table_id_ = 0;
execution_id_ = -1;
success_start_scn_.set_min();
can_schedule_major_compaction_ = false;
}
bool ObTabletDDLKvMgr::is_execution_id_older(const int64_t execution_id)
@ -511,13 +549,16 @@ int ObTabletDDLKvMgr::update_tablet(const SCN &start_scn, const int64_t snapshot
ObVersionRange::MIN_VERSION, // multi_version_start
&tablet_handle.get_obj()->get_storage_schema(),
rebuild_seq);
param.keep_old_ddl_sstable_ = false;
param.ddl_start_scn_ = start_scn;
param.ddl_snapshot_version_ = snapshot_version;
param.ddl_checkpoint_scn_ = ddl_checkpoint_scn;
param.ddl_execution_id_ = execution_id_;
param.ddl_cluster_version_ = cluster_version_;
if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(tablet_id_, param, new_tablet_handle))) {
param.ddl_info_.keep_old_ddl_sstable_ = false;
param.ddl_info_.ddl_start_scn_ = start_scn;
param.ddl_info_.ddl_snapshot_version_ = snapshot_version;
param.ddl_info_.ddl_checkpoint_scn_ = ddl_checkpoint_scn;
param.ddl_info_.ddl_execution_id_ = execution_id_;
param.ddl_info_.ddl_cluster_version_ = cluster_version_;
if (OB_FAIL(create_empty_ddl_sstable(table_handle))) {
LOG_WARN("create empty ddl sstable failed", K(ret));
} else if (FALSE_IT(param.table_handle_ = table_handle)) {
} else if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(tablet_id_, param, new_tablet_handle))) {
LOG_WARN("failed to update tablet table store", K(ret), K(ls_id_), K(tablet_id_), K(param));
} else {
LOG_INFO("update tablet success", K(ls_id_), K(tablet_id_), K(param), K(start_scn), K(snapshot_version), K(ddl_checkpoint_scn));
@ -526,6 +567,76 @@ int ObTabletDDLKvMgr::update_tablet(const SCN &start_scn, const int64_t snapshot
return ret;
}
int ObTabletDDLKvMgr::create_empty_ddl_sstable(ObTableHandleV2 &table_handle)
{
int ret = OB_SUCCESS;
table_handle.reset();
ObArenaAllocator arena;
ObSSTableIndexBuilder *sstable_index_builder = nullptr;
ObIndexBlockRebuilder *index_block_rebuilder = nullptr;
const ObSSTableIndexBuilder::ObSpaceOptimizationMode mode = ObSSTableIndexBuilder::DISABLE;
ObTabletDDLParam ddl_param;
if (OB_FAIL(get_ddl_param(ddl_param))) {
LOG_WARN("get ddl param failed", K(ret));
} else {
ddl_param.table_key_.table_type_ = ObITable::DDL_DUMP_SSTABLE;
ddl_param.table_key_.scn_range_.start_scn_ = SCN::scn_dec(start_scn_);
ddl_param.table_key_.scn_range_.end_scn_ = start_scn_;
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObTabletDDLUtil::prepare_index_builder(ddl_param, arena, mode, nullptr/*first_ddl_sstable*/,
sstable_index_builder, index_block_rebuilder))) {
LOG_WARN("prepare sstable index builder failed", K(ret), K(ddl_param));
} else if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(sstable_index_builder, ddl_param, nullptr/*first_ddl_sstable*/, table_handle))) {
LOG_WARN("create ddl sstable failed", K(ret), K(ddl_param));
}
if (nullptr != index_block_rebuilder) {
index_block_rebuilder->~ObIndexBlockRebuilder();
arena.free(index_block_rebuilder);
index_block_rebuilder = nullptr;
}
if (nullptr != sstable_index_builder) {
sstable_index_builder->~ObSSTableIndexBuilder();
arena.free(sstable_index_builder);
sstable_index_builder = nullptr;
}
return ret;
}
int ObTabletDDLKvMgr::update_ddl_major_sstable()
{
int ret = OB_SUCCESS;
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) {
LOG_WARN("failed to get log stream", K(ret), K(ls_id_));
} else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle,
tablet_id_,
tablet_handle,
ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) {
LOG_WARN("get tablet handle failed", K(ret), K(ls_id_), K(tablet_id_));
} else {
{
TCWLockGuard guard(lock_);
can_schedule_major_compaction_ = true;
}
ObTabletHandle new_tablet_handle;
ObUpdateTableStoreParam param(tablet_handle.get_obj()->get_snapshot_version(),
ObVersionRange::MIN_VERSION, // multi_version_start
&tablet_handle.get_obj()->get_storage_schema(),
ls_handle.get_ls()->get_rebuild_seq());
param.ddl_info_.keep_old_ddl_sstable_ = true;
if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(tablet_id_, param, new_tablet_handle))) {
LOG_WARN("failed to update tablet table store", K(ret), K(ls_id_), K(tablet_id_), K(param));
}
}
return ret;
}
int ObTabletDDLKvMgr::get_ddl_param(ObTabletDDLParam &ddl_param)
{
int ret = OB_SUCCESS;
@ -547,7 +658,7 @@ int ObTabletDDLKvMgr::get_ddl_param(ObTabletDDLParam &ddl_param)
return ret;
}
int ObTabletDDLKvMgr::get_freezed_ddl_kv(const SCN &freeze_scn, ObDDLKVHandle &kv_handle)
int ObTabletDDLKvMgr::get_freezed_ddl_kv(const SCN &freeze_scn, ObTableHandleV2 &kv_handle)
{
int ret = OB_SUCCESS;
kv_handle.reset();
@ -559,15 +670,14 @@ int ObTabletDDLKvMgr::get_freezed_ddl_kv(const SCN &freeze_scn, ObDDLKVHandle &k
TCRLockGuard guard(lock_);
for (int64_t i = head_; OB_SUCC(ret) && !found && i < tail_; ++i) {
const int64_t idx = get_idx(i);
ObDDLKV *cur_kv = ddl_kvs_[idx];
ObTableHandleV2 &cur_kv_handle = ddl_kv_handles_[idx];
ObDDLKV *cur_kv = static_cast<ObDDLKV *>(cur_kv_handle.get_table());
if (OB_ISNULL(cur_kv)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(cur_kv), K(i), K(head_), K(tail_));
} else if (freeze_scn == cur_kv->get_freeze_scn()) {
found = true;
if (OB_FAIL(kv_handle.set_ddl_kv(cur_kv))) {
LOG_WARN("hold ddl kv failed", K(ret));
}
kv_handle = cur_kv_handle;
}
}
if (OB_SUCC(ret) && !found) {
@ -588,32 +698,32 @@ int64_t ObTabletDDLKvMgr::get_idx(const int64_t pos) const
return pos & (MAX_DDL_KV_CNT_IN_STORAGE - 1);
}
int ObTabletDDLKvMgr::get_active_ddl_kv_impl(ObDDLKVHandle &kv_handle)
int ObTabletDDLKvMgr::get_active_ddl_kv_impl(ObTableHandleV2 &kv_handle)
{
int ret = OB_SUCCESS;
ObDDLKV *kv = nullptr;
kv_handle.reset();
if (get_count() == 0) {
ret = OB_ENTRY_NOT_EXIST;
} else {
kv = ddl_kvs_[get_idx(tail_ - 1)];
ObTableHandleV2 &tail_kv_handle = ddl_kv_handles_[get_idx(tail_ - 1)];
ObDDLKV *kv = static_cast<ObDDLKV *>(tail_kv_handle.get_table());
if (nullptr == kv) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, kv must not be nullptr", K(ret));
} else if (kv->is_freezed()) {
kv = nullptr;
ret = OB_SUCCESS;
} else if (OB_FAIL(kv_handle.set_ddl_kv(kv))) {
LOG_WARN("fail to set ddl kv", K(ret));
} else {
kv_handle = tail_kv_handle;
}
}
return ret;
}
int ObTabletDDLKvMgr::get_or_create_ddl_kv(const SCN &scn, ObDDLKVHandle &kv_handle)
int ObTabletDDLKvMgr::get_or_create_ddl_kv(const SCN &scn, ObTableHandleV2 &kv_handle)
{
int ret = OB_SUCCESS;
kv_handle.reset();
ObDDLKV *kv = nullptr;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTabletDDLKvMgr is not inited", K(ret));
@ -621,44 +731,35 @@ int ObTabletDDLKvMgr::get_or_create_ddl_kv(const SCN &scn, ObDDLKVHandle &kv_han
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(scn));
} else {
ObTableHandleV2 tmp_kv_handle;
TCRLockGuard guard(lock_);
try_get_ddl_kv_unlock(scn, kv);
if (nullptr != kv) {
// increase or decrease the reference count must be under the lock
if (OB_FAIL(kv_handle.set_ddl_kv(kv))) {
LOG_WARN("fail to set ddl kv", K(ret));
}
}
try_get_ddl_kv_unlock(scn, kv_handle);
}
if (OB_SUCC(ret) && nullptr == kv) {
if (OB_SUCC(ret) && !kv_handle.is_valid()) {
TCWLockGuard guard(lock_);
try_get_ddl_kv_unlock(scn, kv);
if (nullptr != kv) {
try_get_ddl_kv_unlock(scn, kv_handle);
if (kv_handle.is_valid()) {
// do nothing
} else if (OB_FAIL(alloc_ddl_kv(kv))) {
} else if (OB_FAIL(alloc_ddl_kv(kv_handle))) {
LOG_WARN("create ddl kv failed", K(ret));
}
if (OB_SUCC(ret) && nullptr != kv) {
// increase or decrease the reference count must be under the lock
if (OB_FAIL(kv_handle.set_ddl_kv(kv))) {
LOG_WARN("fail to set ddl kv", K(ret));
}
}
}
return ret;
}
void ObTabletDDLKvMgr::try_get_ddl_kv_unlock(const SCN &scn, ObDDLKV *&kv)
void ObTabletDDLKvMgr::try_get_ddl_kv_unlock(const SCN &scn, ObTableHandleV2 &kv_handle)
{
int ret = OB_SUCCESS;
kv_handle.reset();
if (get_count() > 0) {
for (int64_t i = tail_ - 1; OB_SUCC(ret) && i >= head_ && nullptr == kv; ++i) {
ObDDLKV *tmp_kv = ddl_kvs_[get_idx(i)];
for (int64_t i = tail_ - 1; OB_SUCC(ret) && i >= head_ && !kv_handle.is_valid(); ++i) {
ObTableHandleV2 &tmp_kv_handle = ddl_kv_handles_[get_idx(i)];
ObDDLKV *tmp_kv = static_cast<ObDDLKV *>(tmp_kv_handle.get_table());
if (OB_ISNULL(tmp_kv)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(tmp_kv), K(i), K(head_), K(tail_));
} else if (scn <= tmp_kv->get_freeze_scn()) {
kv = tmp_kv;
kv_handle = tmp_kv_handle;
break;
}
}
@ -668,8 +769,7 @@ void ObTabletDDLKvMgr::try_get_ddl_kv_unlock(const SCN &scn, ObDDLKV *&kv)
int ObTabletDDLKvMgr::freeze_ddl_kv(const SCN &freeze_scn)
{
int ret = OB_SUCCESS;
ObDDLKVHandle kv_handle;
ObDDLKV *kv = nullptr;
ObTableHandleV2 kv_handle;
TCWLockGuard guard(lock_);
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -678,22 +778,20 @@ int ObTabletDDLKvMgr::freeze_ddl_kv(const SCN &freeze_scn)
// do nothing
} else if (OB_FAIL(get_active_ddl_kv_impl(kv_handle))) {
LOG_WARN("fail to get active ddl kv", K(ret));
} else if (OB_FAIL(kv_handle.get_ddl_kv(kv))) {
if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("fail to get ddl kv", K(ret));
} else {
ret = OB_SUCCESS;
}
}
if (OB_SUCC(ret) && nullptr == kv && freeze_scn > max_freeze_scn_) {
// freeze_scn > 0 only occured when ddl prepare
if (OB_SUCC(ret) && !kv_handle.is_valid() && freeze_scn > max_freeze_scn_) {
// freeze_scn > 0 only occured when ddl commit
// assure there is an alive ddl kv, for waiting pre-logs
if (OB_FAIL(alloc_ddl_kv(kv))) {
if (OB_FAIL(alloc_ddl_kv(kv_handle))) {
LOG_WARN("create ddl kv failed", K(ret));
}
}
if (OB_SUCC(ret) && nullptr != kv) {
if (OB_FAIL(kv->freeze(freeze_scn))) {
if (OB_SUCC(ret) && kv_handle.is_valid()) {
ObDDLKV *kv = static_cast<ObDDLKV *>(kv_handle.get_table());
if (OB_ISNULL(kv)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ddl kv is null", K(ret), KP(kv), K(kv_handle));
} else if (OB_FAIL(kv->freeze(freeze_scn))) {
if (OB_EAGAIN != ret) {
LOG_WARN("fail to freeze active ddl kv", K(ret));
} else {
@ -718,7 +816,7 @@ int ObTabletDDLKvMgr::release_ddl_kvs(const SCN &end_scn)
} else {
for (int64_t i = head_; OB_SUCC(ret) && i < tail_; ++i) {
const int64_t idx = get_idx(head_);
ObDDLKV *kv = ddl_kvs_[idx];
ObDDLKV *kv = static_cast<ObDDLKV *>(ddl_kv_handles_[idx].get_table());
LOG_INFO("try release ddl kv", K(end_scn), KPC(kv));
#ifdef ERRSIM
if (OB_SUCC(ret)) {
@ -754,7 +852,7 @@ int ObTabletDDLKvMgr::get_ddl_kv_min_scn(SCN &min_scn)
} else {
for (int64_t i = head_; OB_SUCC(ret) && i < tail_; ++i) {
const int64_t idx = get_idx(head_);
ObDDLKV *kv = ddl_kvs_[idx];
ObDDLKV *kv = static_cast<ObDDLKV *>(ddl_kv_handles_[idx].get_table());
if (OB_ISNULL(kv)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(kv), K(i), K(head_), K(tail_));
@ -766,23 +864,23 @@ int ObTabletDDLKvMgr::get_ddl_kv_min_scn(SCN &min_scn)
return ret;
}
int ObTabletDDLKvMgr::get_ddl_kvs(const bool frozen_only, ObDDLKVsHandle &ddl_kvs_handle)
int ObTabletDDLKvMgr::get_ddl_kvs_unlock(const bool frozen_only, ObTablesHandleArray &kv_handle_array)
{
int ret = OB_SUCCESS;
ddl_kvs_handle.reset();
TCRLockGuard guard(lock_);
kv_handle_array.reset();
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTabletDDLKvMgr is not inited", K(ret));
} else {
for (int64_t pos = head_; OB_SUCC(ret) && pos < tail_; ++pos) {
const int64_t idx = get_idx(pos);
ObDDLKV *cur_kv = ddl_kvs_[idx];
ObTableHandleV2 &cur_kv_handle = ddl_kv_handles_[idx];
ObDDLKV *cur_kv = static_cast<ObDDLKV *>(cur_kv_handle.get_table());
if (OB_ISNULL(cur_kv)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(cur_kv), K(pos), K(head_), K(tail_));
} else if (!frozen_only || cur_kv->is_freezed()) {
if (OB_FAIL(ddl_kvs_handle.add_ddl_kv(cur_kv))) {
if (OB_FAIL(kv_handle_array.add_table(cur_kv_handle))) {
LOG_WARN("fail to push back ddl kv", K(ret));
}
}
@ -791,6 +889,36 @@ int ObTabletDDLKvMgr::get_ddl_kvs(const bool frozen_only, ObDDLKVsHandle &ddl_kv
return ret;
}
int ObTabletDDLKvMgr::get_ddl_kvs(const bool frozen_only, ObTablesHandleArray &kv_handle_array)
{
int ret = OB_SUCCESS;
kv_handle_array.reset();
TCRLockGuard guard(lock_);
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTabletDDLKvMgr is not inited", K(ret));
} else if (OB_FAIL(get_ddl_kvs_unlock(frozen_only, kv_handle_array))) {
LOG_WARN("get ddl kv unlock failed", K(ret));
}
return ret;
}
int ObTabletDDLKvMgr::get_ddl_kvs_for_query(ObTablesHandleArray &kv_handle_array)
{
int ret = OB_SUCCESS;
kv_handle_array.reset();
TCRLockGuard guard(lock_);
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTabletDDLKvMgr is not inited", K(ret));
} else if (!can_schedule_major_compaction_) {
// do nothing
} else if (OB_FAIL(get_ddl_kvs_unlock(true/*frozen_only*/, kv_handle_array))) {
LOG_WARN("get ddl kv unlock failed", K(ret));
}
return ret;
}
int ObTabletDDLKvMgr::check_has_effective_ddl_kv(bool &has_ddl_kv)
{
int ret = OB_SUCCESS;
@ -804,10 +932,13 @@ int ObTabletDDLKvMgr::check_has_effective_ddl_kv(bool &has_ddl_kv)
return ret;
}
int ObTabletDDLKvMgr::alloc_ddl_kv(ObDDLKV *&kv)
int ObTabletDDLKvMgr::alloc_ddl_kv(ObTableHandleV2 &kv_handle)
{
int ret = OB_SUCCESS;
kv = nullptr;
kv_handle.reset();
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr *);
ObTableHandleV2 tmp_kv_handle;
ObDDLKV *kv = nullptr;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ddl kv manager not init", K(ret));
@ -817,9 +948,11 @@ int ObTabletDDLKvMgr::alloc_ddl_kv(ObDDLKV *&kv)
} else if (get_count() == MAX_DDL_KV_CNT_IN_STORAGE) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, too much ddl kv count", K(ret));
} else if (OB_ISNULL(kv = op_alloc(ObDDLKV))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory", K(ret));
} else if (OB_FAIL(t3m->acquire_ddl_kv(tmp_kv_handle))) {
LOG_WARN("acquire ddl kv failed", K(ret));
} else if (OB_ISNULL(kv = static_cast<ObDDLKV *>(tmp_kv_handle.get_table()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ddl kv is null", K(ret));
} else if (OB_FAIL(kv->init(ls_id_,
tablet_id_,
start_scn_,
@ -831,14 +964,10 @@ int ObTabletDDLKvMgr::alloc_ddl_kv(ObDDLKV *&kv)
} else {
const int64_t idx = get_idx(tail_);
tail_++;
ddl_kvs_[idx] = kv;
kv->inc_ref();
ddl_kv_handles_[idx] = tmp_kv_handle;
kv_handle = tmp_kv_handle;
FLOG_INFO("succeed to add ddl kv", K(ls_id_), K(tablet_id_), K(head_), K(tail_), "ddl_kv_cnt", get_count(), KP(kv));
}
if (OB_FAIL(ret) && nullptr != kv) {
op_free(kv);
kv = nullptr;
}
return ret;
}
@ -852,14 +981,8 @@ void ObTabletDDLKvMgr::free_ddl_kv(const int64_t idx)
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(idx));
} else {
ObDDLKV *kv = ddl_kvs_[idx];
ddl_kvs_[idx] = nullptr;
if (nullptr != kv) {
if (0 == kv->dec_ref()) {
op_free(kv);
FLOG_INFO("free ddl kv", K(ls_id_), K(tablet_id_), KP(kv));
}
}
FLOG_INFO("free ddl kv", K(ls_id_), K(tablet_id_), KPC(ddl_kv_handles_[idx].get_table()));
ddl_kv_handles_[idx].reset();
}
}

36
src/storage/ddl/ob_tablet_ddl_kv_mgr.h
View File

@ -27,10 +27,8 @@ namespace oceanbase
namespace storage
{
class ObDDLKV;
class ObDDLKVHandle;
class ObDDLKVsHandle;
struct ObTabletDDLParam;
struct ObDDLTableMergeDagParam;
class ObTabletDDLKvMgr final
{
@ -39,18 +37,20 @@ public:
~ObTabletDDLKvMgr();
int init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); // init before memtable mgr
int ddl_start(const ObITable::TableKey &table_key, const share::SCN &start_scn, const int64_t cluster_version, const int64_t execution_id, const share::SCN &checkpoint_scn);
int ddl_prepare(const share::SCN &start_scn, const share::SCN &commit_scn, const uint64_t table_id = 0, const int64_t ddl_task_id = 0); // schedule build a major sstable
int ddl_commit(const share::SCN &start_scn, const share::SCN &prepare_scn, const bool is_replay); // try wait build major sstable
int wait_ddl_commit(const share::SCN &start_scn, const share::SCN &prepare_scn);
int ddl_commit(const share::SCN &start_scn, const share::SCN &commit_scn, const uint64_t table_id = 0, const int64_t ddl_task_id = 0); // schedule build a major sstable
int schedule_ddl_merge_task(const share::SCN &start_scn, const share::SCN &commit_scn, const bool is_replay); // try wait build major sstable
int wait_ddl_merge_success(const share::SCN &start_scn, const share::SCN &commit_scn);
int get_ddl_param(ObTabletDDLParam &ddl_param);
int get_or_create_ddl_kv(const share::SCN &scn, ObDDLKVHandle &kv_handle); // used in active ddl kv guard
int get_freezed_ddl_kv(const share::SCN &freeze_scn, ObDDLKVHandle &kv_handle); // locate ddl kv with exeact freeze log ts
int get_ddl_kvs(const bool frozen_only, ObDDLKVsHandle &ddl_kvs_handle); // get all freeze ddl kvs
int get_or_create_ddl_kv(const share::SCN &scn, ObTableHandleV2 &kv_handle); // used in active ddl kv guard
int get_freezed_ddl_kv(const share::SCN &freeze_scn, ObTableHandleV2 &kv_handle); // locate ddl kv with exeact freeze log ts
int get_ddl_kvs(const bool frozen_only, ObTablesHandleArray &kv_handle_array); // get all freeze ddl kvs
int get_ddl_kvs_for_query(ObTablesHandleArray &kv_handle_array);
int freeze_ddl_kv(const share::SCN &freeze_scn = share::SCN::min_scn()); // freeze the active ddl kv, when memtable freeze or ddl commit
int release_ddl_kvs(const share::SCN &rec_scn); // release persistent ddl kv, used in ddl merge task for free ddl kv
int check_has_effective_ddl_kv(bool &has_ddl_kv); // used in ddl log handler for checkpoint
int get_ddl_kv_min_scn(share::SCN &min_scn); // for calculate rec_scn of ls
share::SCN get_start_scn() const { return start_scn_; }
share::SCN get_commit_scn() const { return commit_scn_; }
bool is_started() const { return share::SCN::min_scn() != start_scn_; }
int set_commit_success(const share::SCN &start_scn);
bool is_commit_success() const;
@ -64,20 +64,26 @@ public:
OB_INLINE int64_t dec_ref() { return ATOMIC_SAF(&ref_cnt_, 1 /* just sub 1 */); }
OB_INLINE int64_t get_ref() const { return ATOMIC_LOAD(&ref_cnt_); }
OB_INLINE void reset() { destroy(); }
bool can_schedule_major_compaction() const;
int get_ddl_major_merge_param(ObDDLTableMergeDagParam &merge_param);
TO_STRING_KV(K_(is_inited), K_(success_start_scn), K_(ls_id), K_(tablet_id), K_(table_key),
K_(cluster_version), K_(start_scn), K_(max_freeze_scn),
K_(cluster_version), K_(start_scn), K_(commit_scn), K_(max_freeze_scn),
K_(table_id), K_(execution_id), K_(ddl_task_id), K_(head), K_(tail), K_(ref_cnt));
private:
int64_t get_idx(const int64_t pos) const;
int64_t get_count() const;
int alloc_ddl_kv(ObDDLKV *&kv);
int alloc_ddl_kv(ObTableHandleV2 &kv_handle);
void free_ddl_kv(const int64_t idx);
int get_active_ddl_kv_impl(ObDDLKVHandle &kv_handle);
void try_get_ddl_kv_unlock(const share::SCN &scn, ObDDLKV *&kv);
int get_active_ddl_kv_impl(ObTableHandleV2 &kv_handle);
void try_get_ddl_kv_unlock(const share::SCN &scn, ObTableHandleV2 &kv_handle);
int get_ddl_kvs_unlock(const bool frozen_only, ObTablesHandleArray &kv_handle_array);
int update_tablet(const share::SCN &start_scn, const int64_t snapshot_version, const share::SCN &ddl_checkpoint_scn);
int update_ddl_major_sstable();
int create_empty_ddl_sstable(ObTableHandleV2 &table_handle);
void cleanup_unlock();
void destroy();
bool is_commit_success_unlock() const;
private:
static const int64_t MAX_DDL_KV_CNT_IN_STORAGE = 64;
bool is_inited_;
@ -87,15 +93,17 @@ private:
ObITable::TableKey table_key_;
int64_t cluster_version_;
share::SCN start_scn_;
share::SCN commit_scn_;
share::SCN max_freeze_scn_;
uint64_t table_id_; // used for ddl checksum
int64_t execution_id_; // used for ddl checksum
int64_t ddl_task_id_; // used for ddl checksum
ObDDLKV *ddl_kvs_[MAX_DDL_KV_CNT_IN_STORAGE];
ObTableHandleV2 ddl_kv_handles_[MAX_DDL_KV_CNT_IN_STORAGE];
int64_t head_;
int64_t tail_;
common::TCRWLock lock_;
volatile int64_t ref_cnt_ CACHE_ALIGNED;
bool can_schedule_major_compaction_;
DISALLOW_COPY_AND_ASSIGN(ObTabletDDLKvMgr);
};

3
src/storage/high_availability/ob_ls_migration.cpp
View File

@ -24,6 +24,7 @@
#include "ob_storage_ha_src_provider.h"
#include "storage/tablet/ob_tablet_iterator.h"
#include "ob_storage_ha_utils.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{
@ -2318,7 +2319,7 @@ int ObTabletMigrationTask::generate_ddl_copy_tasks_(
} else if (OB_ISNULL(tablet_copy_finish_task) || OB_ISNULL(parent_task)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("generate minor task get invalid argument", K(ret), KP(tablet_copy_finish_task), KP(parent_task));
} else if (OB_FAIL(generate_copy_tasks_(ObITable::is_ddl_sstable, tablet_copy_finish_task, parent_task))) {
} else if (OB_FAIL(generate_copy_tasks_(ObITable::is_ddl_dump_sstable, tablet_copy_finish_task, parent_task))) {
LOG_WARN("failed to generate copy ddl tasks", K(ret), KPC(copy_tablet_ctx_));
}
return ret;

5
src/storage/high_availability/ob_physical_copy_task.cpp
View File

@ -18,6 +18,7 @@
#include "share/ob_cluster_version.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/high_availability/ob_storage_ha_tablet_builder.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{
@ -834,7 +835,7 @@ int ObPhysicalCopyFinishTask::get_merge_type_(
merge_type = ObMergeType::MAJOR_MERGE;
} else if (sstable_param->table_key_.is_minor_sstable()) {
merge_type = ObMergeType::MINOR_MERGE;
} else if (sstable_param->table_key_.is_ddl_sstable()) {
} else if (sstable_param->table_key_.is_ddl_dump_sstable()) {
merge_type = ObMergeType::DDL_KV_MERGE;
} else {
ret = OB_ERR_UNEXPECTED;
@ -971,7 +972,7 @@ int ObPhysicalCopyFinishTask::build_create_sstable_param_(
param.rowkey_column_cnt_ = sstable_param_->basic_meta_.rowkey_column_count_;
param.ddl_scn_ = sstable_param_->basic_meta_.ddl_scn_;
MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH);
if (param.table_key_.is_major_sstable() || param.table_key_.is_ddl_sstable()) {
if (param.table_key_.is_major_sstable() || param.table_key_.is_ddl_dump_sstable()) {
if (OB_FAIL(res.fill_column_checksum(sstable_param_->column_default_checksums_,
param.column_checksums_))) {
LOG_WARN("fail to fill column checksum", K(ret), K(res));

5
src/storage/high_availability/ob_storage_ha_dag.cpp
View File

@ -16,6 +16,7 @@
#include "share/rc/ob_tenant_base.h"
#include "share/scn.h"
#include "observer/ob_server_event_history_table_operator.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{
@ -569,7 +570,7 @@ int ObStorageHATaskUtils::check_need_copy_sstable(
if (OB_FAIL(check_minor_sstable_need_copy_(param, tablet_handle, need_copy))) {
LOG_WARN("failed to check minor sstable need copy", K(ret), K(param), K(tablet_handle));
}
} else if (param.table_key_.is_ddl_sstable()) {
} else if (param.table_key_.is_ddl_dump_sstable()) {
if (OB_FAIL(check_ddl_sstable_need_copy_(param, tablet_handle, need_copy))) {
LOG_WARN("failed to check ddl sstable need copy", K(ret), K(param), K(tablet_handle));
}
@ -674,7 +675,7 @@ int ObStorageHATaskUtils::check_ddl_sstable_need_copy_(
const ObSSTable *sstable = nullptr;
ObTablesHandleArray tables_handle_array;
if (!param.table_key_.is_ddl_sstable()) {
if (!param.table_key_.is_ddl_dump_sstable()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("check ddl sstable need copy get invalid argument", K(ret), K(param));
} else if (OB_ISNULL(tablet = tablet_handle.get_obj())) {

32
src/storage/high_availability/ob_storage_ha_reader.cpp
View File

@ -17,6 +17,7 @@
#include "share/rc/ob_tenant_base.h"
#include "share/scn.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{
@ -1658,15 +1659,26 @@ int ObCopySSTableInfoObProducer::init(
ret = OB_SSTABLE_NOT_EXIST;
LOG_WARN("src tablet start clog ts is bigger than dest needed log ts",
K(ret), K(tablet_sstable_info), KPC(tablet));
} else if (!tablet_sstable_info_.ddl_sstable_scn_range_.is_empty()
&& (tablet->get_tablet_meta().ddl_start_scn_ != tablet_sstable_info_.ddl_sstable_scn_range_.start_scn_
|| tablet->get_tablet_meta().ddl_checkpoint_scn_ < tablet_sstable_info_.ddl_sstable_scn_range_.end_scn_)) {
ret = OB_DDL_SSTABLE_RANGE_CROSS;
LOG_WARN("ddl sstable not exist", K(ret), K(tablet_sstable_info), KPC(tablet));
} else if (OB_FAIL(tablet->get_ha_tables(iter_, is_ready_for_read))) {
LOG_WARN("failed to get read tables", K(ret));
} else {
status_ = ObCopyTabletStatus::TABLET_EXIST;
} else if (!tablet_sstable_info.ddl_sstable_scn_range_.is_empty()) {
if (tablet->get_tablet_meta().get_ddl_sstable_start_scn() < tablet_sstable_info.ddl_sstable_scn_range_.start_scn_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ddl start scn fall back", K(ret), K(tablet->get_tablet_meta()), K(tablet_sstable_info));
} else if (tablet->get_tablet_meta().get_ddl_sstable_start_scn() == tablet_sstable_info.ddl_sstable_scn_range_.start_scn_) {
if (tablet->get_tablet_meta().ddl_checkpoint_scn_ < tablet_sstable_info.ddl_sstable_scn_range_.end_scn_) {
ret = OB_DDL_SSTABLE_RANGE_CROSS;
LOG_WARN("ddl sstable not exist", K(ret), K(tablet_sstable_info), KPC(tablet));
}
} else {
LOG_INFO("ddl start scn advanced, the expired ddl sstable has been cleaned", "tablet_id", tablet_sstable_info.tablet_id_,
K(tablet->get_tablet_meta().ddl_start_scn_), K(tablet_sstable_info.ddl_sstable_scn_range_));
}
}
if (OB_SUCC(ret) && nullptr != tablet) {
if (OB_FAIL(tablet->get_ha_tables(iter_, is_ready_for_read))) {
LOG_WARN("failed to get read tables", K(ret));
} else {
status_ = ObCopyTabletStatus::TABLET_EXIST;
}
}
if (OB_FAIL(ret)) {
@ -1748,7 +1760,7 @@ int ObCopySSTableInfoObProducer::check_need_copy_sstable_(
} else {
need_copy_sstable = true;
}
} else if (sstable->is_ddl_sstable()) {
} else if (sstable->is_ddl_dump_sstable()) {
const SCN ddl_sstable_start_scn = tablet_sstable_info_.ddl_sstable_scn_range_.start_scn_;
const SCN ddl_sstable_end_scn = tablet_sstable_info_.ddl_sstable_scn_range_.end_scn_;
if (tablet_sstable_info_.ddl_sstable_scn_range_.is_empty()) {

61
src/storage/high_availability/ob_storage_ha_tablet_builder.cpp
View File

@ -18,6 +18,7 @@
#include "storage/tablet/ob_tablet_create_delete_helper.h"
#include "share/scn.h"
#include "observer/ob_server_event_history_table_operator.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{
@ -707,14 +708,10 @@ int ObStorageHATabletsBuilder::get_remote_logical_minor_scn_range_(
return ret;
}
// 1. ddl_start_scn == ddl_checkpoint_scn
// 1.1 no need copy sstable
// 2. ddl_start_scn < ddl_checkpoint_scn
// 2.1 ddl_checkpoint_scn has pushed on dest ls tablet
// 2.1.1 we need to copy sstable in [ddl_start_scn, ddl_checkpoint_scn]
// 2.2 ddl_checkpoint_scn has not pushed on dest ls tablet
// 2.2.1 we need find the orignal checkpoint ts of tablet meta, mark as ddl_checkpoint_scn_1
// 2.2.2 and the need copy sstable is in [ddl_start_scn, ddl_checkpoint_scn_1]
// the tablet meta if the one copied from the source server
// ddl_sstable_array is the sstable of the destination server
// the first ddl sstable is an empty one with scn range: (ddl_start_scn - 1, ddl_start_scn]
// the scn range of ddl_sstable_array is continuous, so get the min ddl start scn as the end scn of need_copy_scn_range
int ObStorageHATabletsBuilder::get_need_copy_ddl_sstable_range_(
const ObTablet *tablet,
const ObSSTableArray &ddl_sstable_array,
@ -730,41 +727,33 @@ int ObStorageHATabletsBuilder::get_need_copy_ddl_sstable_range_(
} else {
const SCN ddl_start_scn = tablet->get_tablet_meta().ddl_start_scn_;
const SCN ddl_checkpoint_scn = tablet->get_tablet_meta().ddl_checkpoint_scn_;
if (ddl_start_scn == ddl_checkpoint_scn) {
need_copy_scn_range.start_scn_ = ddl_start_scn;
need_copy_scn_range.end_scn_ = ddl_checkpoint_scn;
} else if (ddl_start_scn < ddl_checkpoint_scn) {
bool ddl_checkpoint_pushed = !ddl_sstable_array.empty();
if (ddl_checkpoint_pushed) {
need_copy_scn_range.start_scn_ = ddl_start_scn;
SCN max_start_scn = SCN::max_scn();
if (OB_FAIL(get_ddl_sstable_max_start_scn_(ddl_sstable_array, max_start_scn))) {
LOG_WARN("failed to get ddl sstable min start log ts", K(ret));
} else {
need_copy_scn_range.end_scn_ = max_start_scn;
}
} else {
need_copy_scn_range.start_scn_ = ddl_start_scn;
need_copy_scn_range.end_scn_ = ddl_checkpoint_scn;
}
#ifdef ERRSIM
LOG_INFO("ddl checkpoint pushed", K(ddl_checkpoint_pushed), K(ddl_sstable_array), K(ddl_start_scn), K(ddl_checkpoint_scn));
SERVER_EVENT_SYNC_ADD("storage_ha", "get_need_copy_ddl_sstable_range",
"tablet_id", tablet->get_tablet_meta().tablet_id_,
"dest_ddl_checkpoint_pushed", ddl_checkpoint_pushed,
"start_scn", need_copy_scn_range.start_scn_,
"end_scn", need_copy_scn_range.end_scn_);
#endif
} else {
need_copy_scn_range.start_scn_ = tablet->get_tablet_meta().get_ddl_sstable_start_scn();
if (ddl_start_scn > ddl_checkpoint_scn) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("checkpoint ts should be greater than start ts",
K(ret), "tablet_meta", tablet->get_tablet_meta());
} else {
if (!ddl_sstable_array.empty()) {
if (OB_FAIL(get_ddl_sstable_min_start_scn_(ddl_sstable_array, need_copy_scn_range.end_scn_))) {
LOG_WARN("failed to get ddl sstable min start scn", K(ret));
}
} else {
need_copy_scn_range.end_scn_ = ddl_checkpoint_scn;
}
#ifdef ERRSIM
LOG_INFO("get_need_copy_ddl_sstable_range", K(ddl_sstable_array), K(ddl_start_scn), K(ddl_checkpoint_scn));
SERVER_EVENT_SYNC_ADD("storage_ha", "get_need_copy_ddl_sstable_range",
"tablet_id", tablet->get_tablet_meta().tablet_id_,
"dest_ddl_sstable_count", ddl_sstable_array.count(),
"start_scn", need_copy_scn_range.start_scn_,
"end_scn", need_copy_scn_range.end_scn_);
#endif
}
}
return ret;
}
int ObStorageHATabletsBuilder::get_ddl_sstable_max_start_scn_(
int ObStorageHATabletsBuilder::get_ddl_sstable_min_start_scn_(
const ObSSTableArray &ddl_sstable_array,
SCN &max_start_scn)
{
@ -787,7 +776,7 @@ int ObStorageHATabletsBuilder::get_ddl_sstable_max_start_scn_(
if (OB_ISNULL(table)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sstable should not be NULL", K(ret), KP(table), K(param_));
} else if (!table->is_ddl_sstable()) {
} else if (!table->is_ddl_dump_sstable()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sstable type is unexpected", K(ret), KP(table), K(param_));
} else {

2
src/storage/high_availability/ob_storage_ha_tablet_builder.h
View File

@ -104,7 +104,7 @@ private:
const ObTablet *tablet,
const ObSSTableArray &ddl_sstable_array,
share::ObScnRange &scn_range);
int get_ddl_sstable_max_start_scn_(
int get_ddl_sstable_min_start_scn_(
const ObSSTableArray &ddl_sstable_array,
share::SCN &max_start_scn);
int hold_local_reuse_sstable_(

2
src/storage/high_availability/ob_storage_restore_struct.cpp
View File

@ -244,7 +244,7 @@ int ObRestoreUtils::get_backup_data_type(
data_type.set_minor_data_backup();
} else if (table_key.is_major_sstable()) {
data_type.set_major_data_backup();
} else if (table_key.is_ddl_sstable()) {
} else if (table_key.is_ddl_dump_sstable()) {
data_type.set_minor_data_backup();
} else {
ret = OB_ERR_UNEXPECTED;

3
src/storage/high_availability/ob_tablet_group_restore.cpp
View File

@ -19,6 +19,7 @@
#include "ob_ls_restore.h"
#include "share/backup/ob_backup_data_store.h"
#include "observer/ob_server_event_history_table_operator.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{
@ -2234,7 +2235,7 @@ int ObTabletRestoreTask::generate_ddl_restore_tasks_(
} else if (!ObTabletRestoreAction::is_restore_minor(tablet_restore_ctx_->action_)) {
LOG_INFO("tablet not restore minor, skip ddl restore tasks",
K(ret), KPC(ha_dag_net_ctx_), KPC(tablet_restore_ctx_));
} else if (OB_FAIL(generate_restore_task_(ObITable::is_ddl_sstable, tablet_copy_finish_task, parent_task))) {
} else if (OB_FAIL(generate_restore_task_(ObITable::is_ddl_dump_sstable, tablet_copy_finish_task, parent_task))) {
LOG_WARN("failed to generate ddl restore task", K(ret), KPC(ha_dag_net_ctx_), KPC(tablet_restore_ctx_));
}
return ret;

25
src/storage/ls/ob_ls_ddl_log_handler.cpp
View File

@ -159,10 +159,6 @@ int ObLSDDLLogHandler::replay(const void *buffer,
ret = replay_ddl_redo_log_(log_buf, buf_size, tmp_pos, log_scn);
break;
}
case ObDDLClogType::DDL_PREPARE_LOG: {
ret = replay_ddl_prepare_log_(log_buf, buf_size, tmp_pos, log_scn);
break;
}
case ObDDLClogType::DDL_COMMIT_LOG: {
ret = replay_ddl_commit_log_(log_buf, buf_size, tmp_pos, log_scn);
break;
@ -261,6 +257,7 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn)
param.tablet_id_ = ddl_kv_mgr_handle.get_obj()->get_tablet_id();
param.start_scn_ = ddl_kv_mgr_handle.get_obj()->get_start_scn();
param.rec_scn_ = rec_scn;
LOG_INFO("schedule ddl merge dag", K(param));
if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) {
if (OB_EAGAIN != ret && OB_SIZE_OVERFLOW != ret) {
LOG_WARN("failed to schedule ddl kv merge dag", K(ret));
@ -329,28 +326,10 @@ int ObLSDDLLogHandler::replay_ddl_redo_log_(const char *log_buf,
return ret;
}
int ObLSDDLLogHandler::replay_ddl_prepare_log_(const char *log_buf,
int ObLSDDLLogHandler::replay_ddl_commit_log_(const char *log_buf,
const int64_t buf_size,
int64_t pos,
const SCN &log_scn)
{
int ret = OB_SUCCESS;
ObDDLPrepareLog log;
if (OB_FAIL(log.deserialize(log_buf, buf_size, pos))) {
LOG_WARN("fail to deserialize ddl commit log", K(ret));
} else if (OB_FAIL(ddl_log_replayer_.replay_prepare(log, log_scn))) {
if (OB_TABLET_NOT_EXIST != ret && OB_EAGAIN != ret) {
LOG_WARN("fail to replay ddl prepare log", K(ret), K(log));
ret = OB_EAGAIN;
}
}
return ret;
}
int ObLSDDLLogHandler::replay_ddl_commit_log_(const char *log_buf,
const int64_t buf_size,
int64_t pos,
const SCN &log_scn)
{
int ret = OB_SUCCESS;
ObDDLCommitLog log;

1
src/storage/ls/ob_ls_ddl_log_handler.h
View File

@ -57,7 +57,6 @@ public:
share::SCN get_rec_scn() override final;
private:
int replay_ddl_redo_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn);
int replay_ddl_prepare_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn);
int replay_ddl_commit_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn);
int replay_ddl_tablet_schema_version_change_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn);
int replay_ddl_start_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn);

384
src/storage/memtable/mvcc/ob_keybtree.cpp
View File

@ -10,14 +10,10 @@
* See the Mulan PubL v2 for more details.
*/
#include "ob_keybtree.h"
#include "ob_keybtree_deps.h"
#include "lib/allocator/ob_qsync.h"
#include "lib/allocator/ob_retire_station.h"
#include "lib/oblog/ob_log_module.h"
#include "storage/memtable/ob_memtable_key.h"
#include "share/ob_errno.h"
namespace oceanbase
{
@ -25,49 +21,8 @@ namespace keybtree
{
using namespace oceanbase::common;
STATIC_ASSERT(sizeof(Iterator) == 376, "Iterator size changed");
// ob_keybtree_deps.h begin
bool RWLock::try_rdlock()
{
bool lock_succ = true;
uint32_t lock = ATOMIC_LOAD(&lock_);
if (0 != (lock >> 16)) {
lock_succ = false;
} else {
lock_succ = ATOMIC_BCAS(&lock_, lock, (lock + 2));
}
return lock_succ;
}
bool RWLock::try_wrlock(const uint16_t uid)
{
bool lock_succ = true;
while (!ATOMIC_BCAS(&writer_id_, 0, uid)) {
if (1 == (ATOMIC_LOAD(&read_ref_) & 0x1)) {
//sched_yield();
} else {
lock_succ = false;
break;
}
}
if (lock_succ) {
while (ATOMIC_LOAD(&read_ref_) > 1) {
sched_yield();
}
}
return lock_succ;
}
WeightEstimate::WeightEstimate(int64_t node_cnt) {
for(int64_t i = 0, k = 1; i < MAX_LEVEL; i++) {
weight_[i] = k;
k *= node_cnt/2;
}
}
void BtreeNode::reset()
template<typename BtreeKey, typename BtreeVal>
void BtreeNode<BtreeKey, BtreeVal>::reset()
{
index_.reset();
magic_num_ = MAGIC_NUM;
@ -78,7 +33,8 @@ void BtreeNode::reset()
ObLink::reset();
}
int BtreeNode::make_new_root(BtreeKey key1, BtreeNode *node_1, BtreeKey key2, BtreeNode *node_2, int16_t level, int64_t version)
template<typename BtreeKey, typename BtreeVal>
int BtreeNode<BtreeKey, BtreeVal>::make_new_root(BtreeKey key1, BtreeNode *node_1, BtreeKey key2, BtreeNode *node_2, int16_t level, int64_t version)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(node_1) || OB_ISNULL(node_2)) {
@ -98,7 +54,8 @@ int BtreeNode::make_new_root(BtreeKey key1, BtreeNode *node_1, BtreeKey key2, Bt
return ret;
}
void BtreeNode::print(FILE *file, const int depth) const
template<typename BtreeKey, typename BtreeVal>
void BtreeNode<BtreeKey, BtreeVal>::print(FILE *file, const int depth) const
{
if (OB_ISNULL(file)) {
// do nothing
@ -108,7 +65,7 @@ void BtreeNode::print(FILE *file, const int depth) const
int count = index.size();
fprintf(file, " index=%lx %dV:%ld ", index.get(), count, max_del_version_);
for (int i = 0; i < count; i++) {
fprintf(file, " %lx->%lx", (uint64_t)kvs_[i].key_.get_rowkey()->get_obj_ptr(), (uint64_t)kvs_[i].val_);
fprintf(file, " %lx->%lx", (uint64_t)kvs_[i].key_.get_ptr(), (uint64_t)kvs_[i].val_);
}
for (int i = 0; i < count; i++) {
fprintf(file, "\n%*s %s%s->%lx", depth * 2 + 2, "|-", get_tag(i, &index) ? "#": "+", get_key(i, &index).repr(), (uint64_t)get_val(i, &index));
@ -117,7 +74,7 @@ void BtreeNode::print(FILE *file, const int depth) const
int count = size();
fprintf(file, " index=%lx %dC:%ld", index_.get(), count, max_del_version_);
for (int i = 0; i < count; i++) {
fprintf(file, " %lx->%lx", (uint64_t)kvs_[i].key_.get_rowkey()->get_obj_ptr(), (uint64_t)kvs_[i].val_);
fprintf(file, " %lx->%lx", (uint64_t)kvs_[i].key_.get_ptr(), (uint64_t)kvs_[i].val_);
}
for (int i = 0; i < count; i++) {
fprintf(file, "\n%*s %s%s->%lx", depth * 2 + 2, "|-", get_tag(i)? "#": "+", get_key(i).repr(), (uint64_t)get_val(i));
@ -127,7 +84,8 @@ void BtreeNode::print(FILE *file, const int depth) const
}
}
int BtreeNode::get_next_active_child(int pos, int64_t version, int64_t* cnt, MultibitSet *index)
template<typename BtreeKey, typename BtreeVal>
int BtreeNode<BtreeKey, BtreeVal>::get_next_active_child(int pos, int64_t version, int64_t* cnt, MultibitSet *index)
{
if (version < max_del_version_) {
++pos;
@ -144,7 +102,8 @@ int BtreeNode::get_next_active_child(int pos, int64_t version, int64_t* cnt, Mul
return pos;
}
int BtreeNode::get_prev_active_child(int pos, int64_t version, int64_t* cnt, MultibitSet *index)
template<typename BtreeKey, typename BtreeVal>
int BtreeNode<BtreeKey, BtreeVal>::get_prev_active_child(int pos, int64_t version, int64_t* cnt, MultibitSet *index)
{
if (version < max_del_version_) {
--pos;
@ -161,7 +120,8 @@ int BtreeNode::get_prev_active_child(int pos, int64_t version, int64_t* cnt, Mul
return pos;
}
void BtreeNode::copy(BtreeNode &dest, const int dest_start, const int start, const int end)
template<typename BtreeKey, typename BtreeVal>
void BtreeNode<BtreeKey, BtreeVal>::copy(BtreeNode &dest, const int dest_start, const int start, const int end)
{
if (OB_LIKELY(start < end)) {
for (int i = 0; i < end - start; ++i) {
@ -176,7 +136,8 @@ void BtreeNode::copy(BtreeNode &dest, const int dest_start, const int start, con
}
}
void BtreeNode::copy_and_insert(BtreeNode &dest_node, const int start, const int end, int pos,
template<typename BtreeKey, typename BtreeVal>
void BtreeNode<BtreeKey, BtreeVal>::copy_and_insert(BtreeNode &dest_node, const int start, const int end, int pos,
BtreeKey key_1, BtreeVal val_1, BtreeKey key_2, BtreeVal val_2)
{
if (pos - start >= 0) {
@ -186,7 +147,9 @@ void BtreeNode::copy_and_insert(BtreeNode &dest_node, const int start, const int
dest_node.insert_into_node(pos + 1 - start, key_2, val_2);
copy(dest_node, (pos + 2) - start, pos + 1, end);
}
uint64_t BtreeNode::check_tag(MultibitSet *index) const
template<typename BtreeKey, typename BtreeVal>
uint64_t BtreeNode<BtreeKey, BtreeVal>::check_tag(MultibitSet *index) const
{
uint64_t tag = 1;
MultibitSet temp;
@ -203,7 +166,8 @@ uint64_t BtreeNode::check_tag(MultibitSet *index) const
return tag;
}
void BtreeNode::replace_child(BtreeNode *new_node, const int pos, BtreeNode *child, int64_t del_version)
template<typename BtreeKey, typename BtreeVal>
void BtreeNode<BtreeKey, BtreeVal>::replace_child(BtreeNode *new_node, const int pos, BtreeNode *child, int64_t del_version)
{
new_node->level_ = level_;
new_node->max_del_version_ = std::max(max_del_version_, del_version);
@ -211,7 +175,8 @@ void BtreeNode::replace_child(BtreeNode *new_node, const int pos, BtreeNode *chi
new_node->set_val(pos, (BtreeVal)child);
}
void BtreeNode::replace_child_and_key(BtreeNode *new_node, const int pos, BtreeKey key, BtreeNode *child, int64_t del_version)
template<typename BtreeKey, typename BtreeVal>
void BtreeNode<BtreeKey, BtreeVal>::replace_child_and_key(BtreeNode *new_node, const int pos, BtreeKey key, BtreeNode *child, int64_t del_version)
{
new_node->level_ = level_;
new_node->max_del_version_ = std::max(max_del_version_, del_version);
@ -219,7 +184,8 @@ void BtreeNode::replace_child_and_key(BtreeNode *new_node, const int pos, BtreeK
new_node->insert_into_node(pos, key, (BtreeVal)child);
}
void BtreeNode::split_child_no_overflow(BtreeNode *new_node, const int pos, BtreeKey key_1, BtreeVal val_1,
template<typename BtreeKey, typename BtreeVal>
void BtreeNode<BtreeKey, BtreeVal>::split_child_no_overflow(BtreeNode *new_node, const int pos, BtreeKey key_1, BtreeVal val_1,
BtreeKey key_2, BtreeVal val_2, int64_t del_version)
{
new_node->level_ = level_;
@ -227,7 +193,8 @@ void BtreeNode::split_child_no_overflow(BtreeNode *new_node, const int pos, Btre
copy_and_insert(*new_node, 0, size(), pos, key_1, val_1, key_2, val_2);
}
void BtreeNode::split_child_cause_recursive_split(BtreeNode *new_node_1, BtreeNode *new_node_2,
template<typename BtreeKey, typename BtreeVal>
void BtreeNode<BtreeKey, BtreeVal>::split_child_cause_recursive_split(BtreeNode *new_node_1, BtreeNode *new_node_2,
const int pos,
BtreeKey key_1,
BtreeVal val_1, BtreeKey key_2, BtreeVal val_2, int64_t del_version)
@ -248,13 +215,15 @@ void BtreeNode::split_child_cause_recursive_split(BtreeNode *new_node_1, BtreeNo
}
}
void Path::reset()
template<typename BtreeKey, typename BtreeVal>
void Path<BtreeKey, BtreeVal>::reset()
{
depth_ = 0;
is_found_ = false;
}
int Path::push(BtreeNode *node, const int pos)
template<typename BtreeKey, typename BtreeVal>
int Path<BtreeKey, BtreeVal>::push(BtreeNode *node, const int pos)
{
int ret = OB_SUCCESS;
if (depth_ >= MAX_DEPTH) {
@ -267,7 +236,8 @@ int Path::push(BtreeNode *node, const int pos)
return ret;
}
int Path::top(BtreeNode *&node, int &pos)
template<typename BtreeKey, typename BtreeVal>
int Path<BtreeKey, BtreeVal>::top(BtreeNode *&node, int &pos)
{
int ret = OB_SUCCESS;
if (depth_ <= 0) {
@ -281,7 +251,8 @@ int Path::top(BtreeNode *&node, int &pos)
return ret;
}
int Path::top_k(int k, BtreeNode*& node, int& pos)
template<typename BtreeKey, typename BtreeVal>
int Path<BtreeKey, BtreeVal>::top_k(int k, BtreeNode*& node, int& pos)
{
int ret = OB_SUCCESS;
if (depth_ < k) {
@ -295,14 +266,16 @@ int Path::top_k(int k, BtreeNode*& node, int& pos)
return ret;
}
int BaseHandle::acquire_ref()
template<typename BtreeKey, typename BtreeVal>
int BaseHandle<BtreeKey, BtreeVal>::acquire_ref()
{
int ret = OB_SUCCESS;
qc_slot_ = qclock_.enter_critical();
return ret;
}
int GetHandle::get(BtreeNode *root, BtreeKey key, BtreeVal &val)
template<typename BtreeKey, typename BtreeVal>
int GetHandle<BtreeKey, BtreeVal>::get(BtreeNode *root, BtreeKey key, BtreeVal &val)
{
int ret = OB_SUCCESS;
BtreeNode *leaf = nullptr;
@ -340,7 +313,8 @@ int GetHandle::get(BtreeNode *root, BtreeKey key, BtreeVal &val)
return ret;
}
int ScanHandle::get(BtreeKey &key, BtreeVal &val)
template<typename BtreeKey, typename BtreeVal>
int ScanHandle<BtreeKey, BtreeVal>::get(BtreeKey &key, BtreeVal &val)
{
int ret = OB_SUCCESS;
BtreeNode *leaf = nullptr;
@ -354,7 +328,8 @@ int ScanHandle::get(BtreeKey &key, BtreeVal &val)
return ret;
}
int ScanHandle::get(BtreeKey &key, BtreeVal &val, bool is_backward, BtreeKey*& last_key)
template<typename BtreeKey, typename BtreeVal>
int ScanHandle<BtreeKey, BtreeVal>::get(BtreeKey &key, BtreeVal &val, bool is_backward, BtreeKey*& last_key)
{
int ret = OB_SUCCESS;
BtreeNode *leaf = nullptr;
@ -373,7 +348,8 @@ int ScanHandle::get(BtreeKey &key, BtreeVal &val, bool is_backward, BtreeKey*& l
return ret;
}
int ScanHandle::pop_level_node(const bool is_backward, const int64_t level, const double ratio,
template<typename BtreeKey, typename BtreeVal>
int ScanHandle<BtreeKey, BtreeVal>::pop_level_node(const bool is_backward, const int64_t level, const double ratio,
const int64_t gap_limit, int64_t &element_count, int64_t &phy_element_count,
BtreeKey*& last_key, int64_t &gap_size)
{
@ -428,7 +404,8 @@ int ScanHandle::pop_level_node(const bool is_backward, const int64_t level, cons
return ret;
}
int ScanHandle::pop_level_node(const int64_t level)
template<typename BtreeKey, typename BtreeVal>
int ScanHandle<BtreeKey, BtreeVal>::pop_level_node(const int64_t level)
{
int ret = OB_SUCCESS;
while(true) {
@ -445,7 +422,8 @@ int ScanHandle::pop_level_node(const int64_t level)
return ret;
}
int ScanHandle::find_path(BtreeNode *root, BtreeKey key, int64_t version)
template<typename BtreeKey, typename BtreeVal>
int ScanHandle<BtreeKey, BtreeVal>::find_path(BtreeNode *root, BtreeKey key, int64_t version)
{
int ret = OB_SUCCESS;
int pos = -1;
@ -479,29 +457,8 @@ int ScanHandle::find_path(BtreeNode *root, BtreeKey key, int64_t version)
return ret;
}
int ScanHandle::skip_gap(BtreeKey& end, bool reverse, int64_t& size)
{
int ret = OB_SUCCESS;
BtreeVal val = nullptr;
if (reverse) {
if (OB_SUCCESS != scan_backward(true, &size)) {
OB_LOG(ERROR, "skip_gap: backward to start, this should not happen");
end = BtreeKey::get_min_key();
} else {
get(end, val);
}
} else {
if (OB_SUCCESS != scan_forward(true, &size)) {
OB_LOG(ERROR, "skip_gap: foward to end, this should not happen");
end = BtreeKey::get_max_key();
} else {
get(end, val);
}
}
return ret;
}
bool ScanHandle::maybe_big_gap(bool is_backward) {
template<typename BtreeKey, typename BtreeVal>
bool ScanHandle<BtreeKey, BtreeVal>::maybe_big_gap(bool is_backward) {
bool ret = false;
BtreeNode* node = nullptr;
int pos = 0;
@ -524,7 +481,8 @@ bool ScanHandle::maybe_big_gap(bool is_backward) {
return ret;
}
int ScanHandle::scan_forward(const int64_t level) {
template<typename BtreeKey, typename BtreeVal>
int ScanHandle<BtreeKey, BtreeVal>::scan_forward(const int64_t level) {
int ret = OB_SUCCESS;
BtreeNode *node = nullptr;
int pos = 0;
@ -552,7 +510,8 @@ int ScanHandle::scan_forward(const int64_t level) {
return ret;
}
int ScanHandle::scan_backward(const int64_t level)
template<typename BtreeKey, typename BtreeVal>
int ScanHandle<BtreeKey, BtreeVal>::scan_backward(const int64_t level)
{
int ret = OB_SUCCESS;
BtreeNode *node = nullptr;
@ -581,7 +540,8 @@ int ScanHandle::scan_backward(const int64_t level)
return ret;
}
int ScanHandle::scan_forward(bool skip_inactive, int64_t* skip_cnt) {
template<typename BtreeKey, typename BtreeVal>
int ScanHandle<BtreeKey, BtreeVal>::scan_forward(bool skip_inactive, int64_t* skip_cnt) {
int ret = OB_SUCCESS;
BtreeNode* node = nullptr;
int pos = 0;
@ -608,7 +568,8 @@ int ScanHandle::scan_forward(bool skip_inactive, int64_t* skip_cnt) {
return ret;
}
int ScanHandle::scan_backward(bool skip_inactive, int64_t* skip_cnt)
template<typename BtreeKey, typename BtreeVal>
int ScanHandle<BtreeKey, BtreeVal>::scan_backward(bool skip_inactive, int64_t* skip_cnt)
{
int ret = OB_SUCCESS;
BtreeNode *node = nullptr;
@ -636,7 +597,8 @@ int ScanHandle::scan_backward(bool skip_inactive, int64_t* skip_cnt)
return ret;
}
BtreeNode *WriteHandle::alloc_node()
template<typename BtreeKey, typename BtreeVal>
BtreeNode<BtreeKey, BtreeVal> *WriteHandle<BtreeKey, BtreeVal>::alloc_node()
{
BtreeNode *p = nullptr;
if (OB_NOT_NULL(p = (BtreeNode *)base_.alloc_node(get_is_in_delete()))) {
@ -645,7 +607,8 @@ BtreeNode *WriteHandle::alloc_node()
return p;
}
void WriteHandle::free_node(BtreeNode *p)
template<typename BtreeKey, typename BtreeVal>
void WriteHandle<BtreeKey, BtreeVal>::free_node(BtreeNode *p)
{
if (OB_NOT_NULL(p)) {
base_.free_node(p);
@ -653,7 +616,8 @@ void WriteHandle::free_node(BtreeNode *p)
}
}
void WriteHandle::free_list()
template<typename BtreeKey, typename BtreeVal>
void WriteHandle<BtreeKey, BtreeVal>::free_list()
{
BtreeNode *p = nullptr;
while (OB_NOT_NULL(p = (BtreeNode *)retire_list_.pop())) {
@ -664,7 +628,8 @@ void WriteHandle::free_list()
}
}
void WriteHandle::retire(const int btree_err)
template<typename BtreeKey, typename BtreeVal>
void WriteHandle<BtreeKey, BtreeVal>::retire(const int btree_err)
{
if (OB_SUCCESS != btree_err) {
free_list();
@ -673,7 +638,8 @@ void WriteHandle::retire(const int btree_err)
}
}
int WriteHandle::insert_and_split_upward(BtreeKey key, BtreeVal &val, BtreeNode *&new_root)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::insert_and_split_upward(BtreeKey key, BtreeVal &val, BtreeNode *&new_root)
{
int ret = OB_SUCCESS;
int pos = -1;
@ -732,7 +698,8 @@ int WriteHandle::insert_and_split_upward(BtreeKey key, BtreeVal &val, BtreeNode
return ret;
}
int WriteHandle::tag_delete(BtreeKey key, BtreeVal& val, int64_t version, BtreeNode *&new_root)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::tag_delete(BtreeKey key, BtreeVal& val, int64_t version, BtreeNode *&new_root)
{
int ret = OB_SUCCESS;
BtreeNode *old_node = nullptr;
@ -746,7 +713,8 @@ int WriteHandle::tag_delete(BtreeKey key, BtreeVal& val, int64_t version, BtreeN
return ret;
}
int WriteHandle::tag_insert(BtreeKey key, BtreeNode *&new_root)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::tag_insert(BtreeKey key, BtreeNode *&new_root)
{
int ret = OB_SUCCESS;
BtreeNode *old_node = nullptr;
@ -760,7 +728,8 @@ int WriteHandle::tag_insert(BtreeKey key, BtreeNode *&new_root)
return ret;
}
int WriteHandle::insert_into_node(BtreeNode *old_node, int pos, BtreeKey key, BtreeVal val, BtreeNode *&new_node_1, BtreeNode *&new_node_2)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::insert_into_node(BtreeNode *old_node, int pos, BtreeKey key, BtreeVal val, BtreeNode *&new_node_1, BtreeNode *&new_node_2)
{
int ret = OB_SUCCESS;
int tmp_pos = -1;
@ -794,7 +763,8 @@ int WriteHandle::insert_into_node(BtreeNode *old_node, int pos, BtreeKey key, Bt
return ret;
}
int WriteHandle::try_wrlock(BtreeNode *node)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::try_wrlock(BtreeNode *node)
{
int ret = OB_SUCCESS;
uint16_t uid = (uint16_t)(get_itid() + 1);
@ -810,7 +780,8 @@ int WriteHandle::try_wrlock(BtreeNode *node)
return ret;
}
int WriteHandle::tag_delete_on_leaf(BtreeNode* old_node, int pos, BtreeKey key, BtreeVal& val, int64_t version, BtreeNode*& new_node)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::tag_delete_on_leaf(BtreeNode* old_node, int pos, BtreeKey key, BtreeVal& val, int64_t version, BtreeNode*& new_node)
{
int ret = OB_SUCCESS;
UNUSED(key);
@ -823,7 +794,8 @@ int WriteHandle::tag_delete_on_leaf(BtreeNode* old_node, int pos, BtreeKey key,
return ret;
}
int WriteHandle::tag_insert_on_leaf(BtreeNode* old_node, int pos, BtreeKey key, BtreeNode*& new_node)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::tag_insert_on_leaf(BtreeNode* old_node, int pos, BtreeKey key, BtreeNode*& new_node)
{
int ret = OB_SUCCESS;
UNUSED(key);
@ -835,7 +807,8 @@ int WriteHandle::tag_insert_on_leaf(BtreeNode* old_node, int pos, BtreeKey key,
return ret;
}
int WriteHandle::tag_upward(BtreeNode* new_node, BtreeNode*& new_root)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::tag_upward(BtreeNode* new_node, BtreeNode*& new_root)
{
int ret = OB_SUCCESS;
BtreeNode* old_node = nullptr;
@ -858,7 +831,8 @@ int WriteHandle::tag_upward(BtreeNode* new_node, BtreeNode*& new_root)
return ret;
}
int WriteHandle::tag_leaf(BtreeNode *old_node, const int pos, BtreeNode*& new_node, uint64_t tag, int64_t version)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::tag_leaf(BtreeNode *old_node, const int pos, BtreeNode*& new_node, uint64_t tag, int64_t version)
{
int ret = OB_SUCCESS;
uint64_t old_tag = 1;
@ -894,7 +868,8 @@ int WriteHandle::tag_leaf(BtreeNode *old_node, const int pos, BtreeNode*& new_no
return ret;
}
int WriteHandle::replace_child_by_copy(BtreeNode *old_node, const int pos, BtreeVal val, int64_t version, BtreeNode*& new_node)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::replace_child_by_copy(BtreeNode *old_node, const int pos, BtreeVal val, int64_t version, BtreeNode*& new_node)
{
int ret = OB_SUCCESS;
if (OB_FAIL(try_wrlock(old_node))) {
@ -909,7 +884,8 @@ int WriteHandle::replace_child_by_copy(BtreeNode *old_node, const int pos, Btree
return ret;
}
int WriteHandle::replace_child(BtreeNode *old_node, const int pos, BtreeVal val)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::replace_child(BtreeNode *old_node, const int pos, BtreeVal val)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(old_node)) {
@ -923,7 +899,8 @@ int WriteHandle::replace_child(BtreeNode *old_node, const int pos, BtreeVal val)
return ret;
}
int WriteHandle::replace_child_and_key(BtreeNode *old_node, const int pos, BtreeKey key, BtreeNode *child, BtreeNode *&new_node, int64_t version)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::replace_child_and_key(BtreeNode *old_node, const int pos, BtreeKey key, BtreeNode *child, BtreeNode *&new_node, int64_t version)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(old_node) || OB_ISNULL(child)) {
@ -940,7 +917,8 @@ int WriteHandle::replace_child_and_key(BtreeNode *old_node, const int pos, Btree
return ret;
}
int WriteHandle::make_new_root(BtreeNode *&root, BtreeKey key1, BtreeNode *node_1, BtreeKey key2, BtreeNode *node_2, int16_t level, int64_t version)
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::make_new_root(BtreeNode *&root, BtreeKey key1, BtreeNode *node_1, BtreeKey key2, BtreeNode *node_2, int16_t level, int64_t version)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(node_1) || OB_ISNULL(node_2)) {
@ -953,7 +931,8 @@ int WriteHandle::make_new_root(BtreeNode *&root, BtreeKey key1, BtreeNode *node_
return ret;
}
int WriteHandle::split_child(BtreeNode *old_node, const int pos, BtreeKey key_1, BtreeVal val_1, BtreeKey key_2,
template<typename BtreeKey, typename BtreeVal>
int WriteHandle<BtreeKey, BtreeVal>::split_child(BtreeNode *old_node, const int pos, BtreeKey key_1, BtreeVal val_1, BtreeKey key_2,
BtreeVal val_2, BtreeNode *&new_node_1, BtreeNode *&new_node_2, int64_t version)
{
int ret = OB_SUCCESS;
@ -976,7 +955,8 @@ int WriteHandle::split_child(BtreeNode *old_node, const int pos, BtreeKey key_1,
return ret;
}
void Iterator::reset()
template<typename BtreeKey, typename BtreeVal>
void Iterator<BtreeKey, BtreeVal>::reset()
{
scan_handle_.reset();
jump_key_ = nullptr;
@ -990,7 +970,8 @@ void Iterator::reset()
iter_count_ = 0;
}
int Iterator::set_key_range(const BtreeKey min_key, const bool start_exclude,
template<typename BtreeKey, typename BtreeVal>
int Iterator<BtreeKey, BtreeVal>::set_key_range(const BtreeKey min_key, const bool start_exclude,
const BtreeKey max_key, const bool end_exclude, int64_t version)
{
int ret = OB_SUCCESS;
@ -1012,7 +993,8 @@ int Iterator::set_key_range(const BtreeKey min_key, const bool start_exclude,
return ret;
}
int Iterator::get_next(BtreeKey &key, BtreeVal &value)
template<typename BtreeKey, typename BtreeVal>
int Iterator<BtreeKey, BtreeVal>::get_next(BtreeKey &key, BtreeVal &value)
{
int ret = OB_SUCCESS;
if (OB_FAIL(iter_next(key, value))){
@ -1032,7 +1014,8 @@ int Iterator::get_next(BtreeKey &key, BtreeVal &value)
return ret;
}
int Iterator::next_on_level(const int64_t level, BtreeKey& key, BtreeVal& value)
template<typename BtreeKey, typename BtreeVal>
int Iterator<BtreeKey, BtreeVal>::next_on_level(const int64_t level, BtreeKey& key, BtreeVal& value)
{
int ret = OB_SUCCESS;
BtreeKey* jump_key = nullptr;
@ -1063,7 +1046,8 @@ int Iterator::next_on_level(const int64_t level, BtreeKey& key, BtreeVal& value)
return ret;
}
int Iterator::estimate_one_level(const int64_t level, const int64_t start_batch_count, const int64_t end_batch_count,
template<typename BtreeKey, typename BtreeVal>
int Iterator<BtreeKey, BtreeVal>::estimate_one_level(const int64_t level, const int64_t start_batch_count, const int64_t end_batch_count,
const int64_t max_node_count, const int64_t skip_range_limit, const double gap_ratio,
int64_t &level_physical_row_count, int64_t &level_element_count, int64_t &node_count)
{
@ -1092,7 +1076,8 @@ int Iterator::estimate_one_level(const int64_t level, const int64_t start_batch_
return ret;
}
int Iterator::estimate_element_count(int64_t &physical_row_count, int64_t &element_count, const double ratio)
template<typename BtreeKey, typename BtreeVal>
int Iterator<BtreeKey, BtreeVal>::estimate_element_count(int64_t &physical_row_count, int64_t &element_count, const double ratio)
{
int ret = OB_SUCCESS;
static const int64_t MAX_SAMPLE_LEAF_COUNT = 500;
@ -1131,7 +1116,8 @@ int Iterator::estimate_element_count(int64_t &physical_row_count, int64_t &eleme
return ret;
}
int Iterator::iter_next(BtreeKey &key, BtreeVal &value)
template<typename BtreeKey, typename BtreeVal>
int Iterator<BtreeKey, BtreeVal>::iter_next(BtreeKey &key, BtreeVal &value)
{
int ret = OB_SUCCESS;
bool skip_inactive = false;
@ -1164,7 +1150,8 @@ int Iterator::iter_next(BtreeKey &key, BtreeVal &value)
return ret;
}
int Iterator::iter_next_batch_level_node(int64_t &element_count, const int64_t batch_count,
template<typename BtreeKey, typename BtreeVal>
int Iterator<BtreeKey, BtreeVal>::iter_next_batch_level_node(int64_t &element_count, const int64_t batch_count,
const int64_t level, int64_t &gap_size,
const int64_t gap_limit, int64_t &phy_element_count,
const double ratio)
@ -1222,7 +1209,8 @@ int Iterator::iter_next_batch_level_node(int64_t &element_count, const int64_t b
}
// cmp < 0: iter end
int Iterator::comp(BtreeKey& cur_key, BtreeKey* jump_key, int &cmp)
template<typename BtreeKey, typename BtreeVal>
int Iterator<BtreeKey, BtreeVal>::comp(BtreeKey& cur_key, BtreeKey* jump_key, int &cmp)
{
int ret = OB_SUCCESS;
cmp = 0;
@ -1241,13 +1229,15 @@ int Iterator::comp(BtreeKey& cur_key, BtreeKey* jump_key, int &cmp)
return ret;
}
void BtreeIterator::KVQueue::reset()
template<typename BtreeKey, typename BtreeVal>
void BtreeIterator<BtreeKey, BtreeVal>::KVQueue::reset()
{
push_ = 0;
pop_ = 0;
}
int BtreeIterator::KVQueue::push(const BtreeKV &data)
template<typename BtreeKey, typename BtreeVal>
int BtreeIterator<BtreeKey, BtreeVal>::KVQueue::push(const BtreeKV &data)
{
int ret = 0;
if (push_ >= pop_ + capacity) {
@ -1258,7 +1248,8 @@ int BtreeIterator::KVQueue::push(const BtreeKV &data)
return ret;
}
int BtreeIterator::KVQueue::pop(BtreeKV &data)
template<typename BtreeKey, typename BtreeVal>
int BtreeIterator<BtreeKey, BtreeVal>::KVQueue::pop(BtreeKV &data)
{
int ret = 0;
if (pop_ >= push_) {
@ -1269,7 +1260,8 @@ int BtreeIterator::KVQueue::pop(BtreeKV &data)
return ret;
}
int BtreeIterator::init(ObKeyBtree &btree)
template<typename BtreeKey, typename BtreeVal>
int BtreeIterator<BtreeKey, BtreeVal>::init(ObKeyBtree &btree)
{
int ret = OB_SUCCESS;
if (OB_NOT_NULL(iter_)) {
@ -1280,7 +1272,8 @@ int BtreeIterator::init(ObKeyBtree &btree)
return ret;
}
void BtreeIterator::reset()
template<typename BtreeKey, typename BtreeVal>
void BtreeIterator<BtreeKey, BtreeVal>::reset()
{
kv_queue_.reset();
is_iter_end_ = false;
@ -1296,7 +1289,8 @@ void BtreeIterator::reset()
}
}
int BtreeIterator::set_key_range(const BtreeKey min_key, const bool start_exclude,
template<typename BtreeKey, typename BtreeVal>
int BtreeIterator<BtreeKey, BtreeVal>::set_key_range(const BtreeKey min_key, const bool start_exclude,
const BtreeKey max_key, const bool end_exclude, int64_t version)
{
int ret = OB_SUCCESS;
@ -1312,7 +1306,8 @@ int BtreeIterator::set_key_range(const BtreeKey min_key, const bool start_exclud
return ret;
}
int BtreeIterator::get_next(BtreeKey &key, BtreeVal &value)
template<typename BtreeKey, typename BtreeVal>
int BtreeIterator<BtreeKey, BtreeVal>::get_next(BtreeKey &key, BtreeVal &value)
{
int ret = OB_SUCCESS;
BtreeKV item;
@ -1335,7 +1330,8 @@ int BtreeIterator::get_next(BtreeKey &key, BtreeVal &value)
return ret;
}
int BtreeIterator::scan_batch()
template<typename BtreeKey, typename BtreeVal>
int BtreeIterator<BtreeKey, BtreeVal>::scan_batch()
{
int ret = OB_SUCCESS;
if (is_iter_end_) {
@ -1370,13 +1366,15 @@ int BtreeIterator::scan_batch()
// ob_keybtree.h begin
void BtreeNodeList::bulk_push(BtreeNode* first, BtreeNode* last) {
template<typename BtreeKey, typename BtreeVal>
void BtreeNodeList<BtreeKey, BtreeVal>::bulk_push(BtreeNode* first, BtreeNode* last) {
BtreeNode* tail = load_lock();
last->next_ = tail;
ATOMIC_STORE(&tail_, first);
}
BtreeNode* BtreeNodeList::pop() {
template<typename BtreeKey, typename BtreeVal>
BtreeNode<BtreeKey, BtreeVal>* BtreeNodeList<BtreeKey, BtreeVal>::pop() {
BtreeNode* tail = nullptr;
if (OB_NOT_NULL(ATOMIC_LOAD(&tail_))) {
tail = load_lock();
@ -1385,7 +1383,8 @@ BtreeNode* BtreeNodeList::pop() {
return tail;
}
BtreeNode* BtreeNodeList::load_lock() {
template<typename BtreeKey, typename BtreeVal>
BtreeNode<BtreeKey, BtreeVal>* BtreeNodeList<BtreeKey, BtreeVal>::load_lock() {
BtreeNode* tail = nullptr;
BtreeNode* LOCK = (BtreeNode*)~0UL;
while(LOCK == (tail = ATOMIC_TAS(&tail_, LOCK)))
@ -1393,7 +1392,8 @@ BtreeNode* BtreeNodeList::load_lock() {
return tail;
}
int64_t BtreeNodeAllocator::push_idx()
template<typename BtreeKey, typename BtreeVal>
int64_t BtreeNodeAllocator<BtreeKey, BtreeVal>::push_idx()
{
RLOCAL(int64_t, push_idx);
if (0 == push_idx) {
@ -1401,7 +1401,9 @@ int64_t BtreeNodeAllocator::push_idx()
}
return (push_idx++) % MAX_LIST_COUNT;
}
int64_t BtreeNodeAllocator::pop_idx()
template<typename BtreeKey, typename BtreeVal>
int64_t BtreeNodeAllocator<BtreeKey, BtreeVal>::pop_idx()
{
RLOCAL(int64_t, pop_idx);
if (0 == pop_idx) {
@ -1410,7 +1412,8 @@ int64_t BtreeNodeAllocator::pop_idx()
return (pop_idx++) % MAX_LIST_COUNT;
}
BtreeNode *BtreeNodeAllocator::alloc_node(const bool is_emergency)
template<typename BtreeKey, typename BtreeVal>
BtreeNode<BtreeKey, BtreeVal> *BtreeNodeAllocator<BtreeKey, BtreeVal>::alloc_node(const bool is_emergency)
{
BtreeNode *p = nullptr;
int ret = OB_SUCCESS;
@ -1421,9 +1424,11 @@ BtreeNode *BtreeNodeAllocator::alloc_node(const bool is_emergency)
return p;
}
int BtreeNodeAllocator::pop(BtreeNode*& p)
template<typename BtreeKey, typename BtreeVal>
int BtreeNodeAllocator<BtreeKey, BtreeVal>::pop(BtreeNode*& p)
{
int64_t pop_list_idx = pop_idx();
const int64_t NODE_SIZE = sizeof(BtreeNode);
if (OB_ISNULL(p = free_list_array_[pop_list_idx].pop())) {
// queue is empty, fill nodes.
char *block = nullptr;
@ -1456,7 +1461,8 @@ int BtreeNodeAllocator::pop(BtreeNode*& p)
return OB_NOT_NULL(p) ? OB_SUCCESS : OB_ALLOCATE_MEMORY_FAILED;
}
int BtreeRawIterator::init(ObKeyBtree &btree)
template<typename BtreeKey, typename BtreeVal>
int BtreeRawIterator<BtreeKey, BtreeVal>::init(ObKeyBtree &btree)
{
int ret = OB_SUCCESS;
if (OB_NOT_NULL(iter_)) {
@ -1467,7 +1473,8 @@ int BtreeRawIterator::init(ObKeyBtree &btree)
return ret;
}
void BtreeRawIterator::reset()
template<typename BtreeKey, typename BtreeVal>
void BtreeRawIterator<BtreeKey, BtreeVal>::reset()
{
if (OB_NOT_NULL(iter_)) {
iter_->~Iterator();
@ -1475,13 +1482,15 @@ void BtreeRawIterator::reset()
}
}
int BtreeRawIterator::set_key_range(const BtreeKey min_key, const bool start_exclude,
template<typename BtreeKey, typename BtreeVal>
int BtreeRawIterator<BtreeKey, BtreeVal>::set_key_range(const BtreeKey min_key, const bool start_exclude,
const BtreeKey max_key, const bool end_exclude, int64_t version)
{
return iter_->set_key_range(min_key, start_exclude, max_key, end_exclude, version);
}
int BtreeRawIterator::get_next(BtreeKey &key, BtreeVal &val)
template<typename BtreeKey, typename BtreeVal>
int BtreeRawIterator<BtreeKey, BtreeVal>::get_next(BtreeKey &key, BtreeVal &val)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(iter_)) {
@ -1494,7 +1503,8 @@ int BtreeRawIterator::get_next(BtreeKey &key, BtreeVal &val)
return ret;
}
int BtreeRawIterator::estimate_key_count(int64_t top_level, int64_t& child_count, int64_t& key_count)
template<typename BtreeKey, typename BtreeVal>
int BtreeRawIterator<BtreeKey, BtreeVal>::estimate_key_count(int64_t top_level, int64_t& child_count, int64_t& key_count)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(iter_)) {
@ -1523,7 +1533,8 @@ int BtreeRawIterator::estimate_key_count(int64_t top_level, int64_t& child_count
return ret;
}
int BtreeRawIterator::split_range(int64_t top_level, int64_t branch_count, int64_t part_count, BtreeKey* key_array)
template<typename BtreeKey, typename BtreeVal>
int BtreeRawIterator<BtreeKey, BtreeVal>::split_range(int64_t top_level, int64_t branch_count, int64_t part_count, BtreeKey* key_array)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(iter_)) {
@ -1559,7 +1570,8 @@ int BtreeRawIterator::split_range(int64_t top_level, int64_t branch_count, int64
return ret;
}
int BtreeRawIterator::estimate_element_count(int64_t &physical_row_count, int64_t &element_count, const double ratio)
template<typename BtreeKey, typename BtreeVal>
int BtreeRawIterator<BtreeKey, BtreeVal>::estimate_element_count(int64_t &physical_row_count, int64_t &element_count, const double ratio)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(iter_)) {
@ -1574,24 +1586,30 @@ int BtreeRawIterator::estimate_element_count(int64_t &physical_row_count, int64_
return ret;
}
bool BtreeRawIterator::is_reverse_scan() const { return iter_->is_reverse_scan(); }
template<typename BtreeKey, typename BtreeVal>
bool BtreeRawIterator<BtreeKey, BtreeVal>::is_reverse_scan() const { return iter_->is_reverse_scan(); }
int ObKeyBtree::init()
template<typename BtreeKey, typename BtreeVal>
int ObKeyBtree<BtreeKey, BtreeVal>::init()
{
UNUSED(update_split_info(NODE_KEY_COUNT / 2));
return OB_SUCCESS;
}
void ObKeyBtree::print(FILE *file) const
template<typename BtreeKey, typename BtreeVal>
void ObKeyBtree<BtreeKey, BtreeVal>::print(FILE *file) const
{
if (OB_NOT_NULL(file)) {
fprintf(file, "\n|root=%p node_size=%d node_key_count=%d total_size=%ld\n", root_, NODE_SIZE,
fprintf(file, "\n|root=%p node_size=%lu node_key_count=%d total_size=%ld\n", root_, sizeof(BtreeNode),
NODE_KEY_COUNT, size());
root_->print(file, 0);
if (OB_NOT_NULL(root_)) {
root_->print(file, 0);
}
}
}
int ObKeyBtree::destroy()
template<typename BtreeKey, typename BtreeVal>
int ObKeyBtree<BtreeKey, BtreeVal>::destroy()
{
destroy(ATOMIC_SET(&root_, nullptr));
{
@ -1608,7 +1626,8 @@ int ObKeyBtree::destroy()
return OB_SUCCESS;
}
int ObKeyBtree::del(const BtreeKey key, BtreeVal &value, int64_t version)
template<typename BtreeKey, typename BtreeVal>
int ObKeyBtree<BtreeKey, BtreeVal>::del(const BtreeKey key, BtreeVal &value, int64_t version)
{
int ret = OB_EAGAIN;
while (OB_EAGAIN == ret) {
@ -1633,7 +1652,8 @@ int ObKeyBtree::del(const BtreeKey key, BtreeVal &value, int64_t version)
return ret;
}
int ObKeyBtree::re_insert(const BtreeKey key, BtreeVal value)
template<typename BtreeKey, typename BtreeVal>
int ObKeyBtree<BtreeKey, BtreeVal>::re_insert(const BtreeKey key, BtreeVal value)
{
int ret = OB_EAGAIN;
UNUSED(value);
@ -1659,7 +1679,8 @@ int ObKeyBtree::re_insert(const BtreeKey key, BtreeVal value)
return ret;
}
int ObKeyBtree::insert(const BtreeKey key, BtreeVal &value)
template<typename BtreeKey, typename BtreeVal>
int ObKeyBtree<BtreeKey, BtreeVal>::insert(const BtreeKey key, BtreeVal &value)
{
int ret = OB_SUCCESS;
BtreeNode *old_root = nullptr;
@ -1699,21 +1720,8 @@ int ObKeyBtree::insert(const BtreeKey key, BtreeVal &value)
return ret;
}
int ObKeyBtree::skip_gap(const BtreeKey start, BtreeKey &end, int64_t version, bool reverse, int64_t &size)
{
int ret = OB_SUCCESS;
ScanHandle handle(*this);
if (OB_FAIL(handle.acquire_ref())) {
OB_LOG(ERROR, "acquire_ref fail", K(ret));
} else if (OB_FAIL(handle.find_path(ATOMIC_LOAD(&root_), start, version))) {
// do nothing
} else if (OB_FAIL(handle.skip_gap(end, reverse, size))) {
// do nothing
}
return ret;
}
int ObKeyBtree::get(const BtreeKey key, BtreeVal &value)
template<typename BtreeKey, typename BtreeVal>
int ObKeyBtree<BtreeKey, BtreeVal>::get(const BtreeKey key, BtreeVal &value)
{
int ret = OB_SUCCESS;
GetHandle handle(*this);
@ -1727,7 +1735,8 @@ int ObKeyBtree::get(const BtreeKey key, BtreeVal &value)
return ret;
}
int ObKeyBtree::set_key_range(BtreeIterator &iter, const BtreeKey min_key, const bool start_exclude,
template<typename BtreeKey, typename BtreeVal>
int ObKeyBtree<BtreeKey, BtreeVal>::set_key_range(BtreeIterator &iter, const BtreeKey min_key, const bool start_exclude,
const BtreeKey max_key, const bool end_exclude, int64_t version)
{
int ret = OB_SUCCESS;
@ -1739,7 +1748,8 @@ int ObKeyBtree::set_key_range(BtreeIterator &iter, const BtreeKey min_key, const
return ret;
}
int ObKeyBtree::set_key_range(BtreeRawIterator &iter, const BtreeKey min_key, const bool start_exclude,
template<typename BtreeKey, typename BtreeVal>
int ObKeyBtree<BtreeKey, BtreeVal>::set_key_range(BtreeRawIterator &iter, const BtreeKey min_key, const bool start_exclude,
const BtreeKey max_key, const bool end_exclude, int64_t version)
{
int ret = OB_SUCCESS;
@ -1751,7 +1761,8 @@ int ObKeyBtree::set_key_range(BtreeRawIterator &iter, const BtreeKey min_key, co
return ret;
}
BtreeNode *ObKeyBtree::alloc_node(const bool is_emergency)
template<typename BtreeKey, typename BtreeVal>
BtreeNode<BtreeKey, BtreeVal> *ObKeyBtree<BtreeKey, BtreeVal>::alloc_node(const bool is_emergency)
{
BtreeNode *p = nullptr;
if (OB_NOT_NULL(p = (BtreeNode *)node_allocator_.alloc_node(is_emergency))) {
@ -1761,7 +1772,8 @@ BtreeNode *ObKeyBtree::alloc_node(const bool is_emergency)
return p;
}
void ObKeyBtree::free_node(BtreeNode *p)
template<typename BtreeKey, typename BtreeVal>
void ObKeyBtree<BtreeKey, BtreeVal>::free_node(BtreeNode *p)
{
if (OB_NOT_NULL(p)) {
ObKeyBtree *host = (ObKeyBtree *)p->get_host();
@ -1772,7 +1784,8 @@ void ObKeyBtree::free_node(BtreeNode *p)
}
}
void ObKeyBtree::retire(HazardList &retire_list)
template<typename BtreeKey, typename BtreeVal>
void ObKeyBtree<BtreeKey, BtreeVal>::retire(HazardList &retire_list)
{
HazardList reclaim_list;
BtreeNode *p = nullptr;
@ -1784,7 +1797,8 @@ void ObKeyBtree::retire(HazardList &retire_list)
}
}
int32_t ObKeyBtree::update_split_info(int32_t split_pos)
template<typename BtreeKey, typename BtreeVal>
int32_t ObKeyBtree<BtreeKey, BtreeVal>::update_split_info(int32_t split_pos)
{
if (split_pos < 0) {
split_pos = 0;
@ -1794,25 +1808,29 @@ int32_t ObKeyBtree::update_split_info(int32_t split_pos)
return (ret < 1) ? 1 : ret;
}
RetireStation &ObKeyBtree::get_retire_station()
template<typename BtreeKey, typename BtreeVal>
RetireStation &ObKeyBtree<BtreeKey, BtreeVal>::get_retire_station()
{
static RetireStation retire_station_(get_qclock(), RETIRE_LIMIT);
return retire_station_;
}
QClock& ObKeyBtree::get_qclock()
template<typename BtreeKey, typename BtreeVal>
QClock& ObKeyBtree<BtreeKey, BtreeVal>::get_qclock()
{
static QClock qclock_;
return qclock_;
}
ObQSync& ObKeyBtree::get_qsync()
template<typename BtreeKey, typename BtreeVal>
ObQSync& ObKeyBtree<BtreeKey, BtreeVal>::get_qsync()
{
static ObQSync qsync;
return qsync;
}
void ObKeyBtree::destroy(BtreeNode *root)
template<typename BtreeKey, typename BtreeVal>
void ObKeyBtree<BtreeKey, BtreeVal>::destroy(BtreeNode *root)
{
ObKeyBtree *host = nullptr;
if (OB_NOT_NULL(root) && OB_NOT_NULL(host = (ObKeyBtree *)(root->get_host()))) {

66
src/storage/memtable/mvcc/ob_keybtree.h
View File

@ -14,7 +14,7 @@
#define __OB_KEYBTREE_H__
#include "lib/metrics/ob_counter.h"
#include "storage/memtable/ob_memtable_key.h"
#include "storage/memtable/mvcc/ob_keybtree_deps.h"
namespace oceanbase
{
@ -26,31 +26,35 @@ class ObQSync;
class HazardList;
class ObIAllocator;
}
namespace memtable
{
class ObStoreRowkeyWrapper;
struct ObMvccRow;
}
namespace keybtree
{
template<typename BtreeKey, typename BtreeVal>
class BtreeNode;
class HazardLessIterator;
template<typename BtreeKey, typename BtreeVal>
class Iterator;
template<typename BtreeKey, typename BtreeVal>
class ObKeyBtree;
class ScanHandle;
template<typename BtreeKey, typename BtreeVal>
class WriteHandle;
template<typename BtreeKey, typename BtreeVal>
class GetHandle;
using BtreeKey = memtable::ObStoreRowkeyWrapper;
using BtreeVal = memtable::ObMvccRow *;
template<typename BtreeKey, typename BtreeVal>
struct BtreeKV
{
BtreeKey key_; // 8byte
BtreeVal val_; // 8byte
};
template<typename BtreeKey, typename BtreeVal>
struct BtreeNodeList
{
private:
typedef BtreeNode<BtreeKey, BtreeVal> BtreeNode;
public:
BtreeNodeList(): tail_(nullptr) {}
void bulk_push(BtreeNode* first, BtreeNode* last);
void push(BtreeNode* p) { bulk_push(p, p); }
@ -59,11 +63,14 @@ struct BtreeNodeList
BtreeNode* tail_;
};
template<typename BtreeKey, typename BtreeVal>
class BtreeNodeAllocator
{
private:
typedef BtreeNode<BtreeKey, BtreeVal> BtreeNode;
typedef BtreeNodeList<BtreeKey, BtreeVal> BtreeNodeList;
enum {
MAX_LIST_COUNT = common::MAX_CPU_NUM
MAX_LIST_COUNT = MAX_CPU_NUM
};
public:
BtreeNodeAllocator(common::ObIAllocator &allocator) : allocator_(allocator), alloc_memory_(0) {}
@ -96,7 +103,13 @@ private:
* Interface of Iterator
* it will do batch_scan, so we can keep it without release for a long time.
*/
template<typename BtreeKey, typename BtreeVal>
class BtreeIterator {
private:
typedef BtreeKV<BtreeKey, BtreeVal> BtreeKV;
typedef Iterator<BtreeKey, BtreeVal> Iterator;
typedef ObKeyBtree<BtreeKey, BtreeVal> ObKeyBtree;
class KVQueue {
enum {
capacity = 225
@ -133,6 +146,7 @@ public:
const BtreeKey max_key, const bool end_exclude, int64_t version);
int get_next(BtreeKey &key, BtreeVal &val);
bool is_reverse_scan() const { return scan_backward_; }
bool is_iter_end() const { return is_iter_end_; }
private:
int scan_batch();
private:
@ -144,16 +158,20 @@ private:
bool end_exclude_; // 1byte
bool is_iter_end_; // 1byte
bool scan_backward_; // 1byte
KVQueue kv_queue_; // 3616 == 16 + 16 * n, n == 225
char buf_[376]; // sizeof(Iterator) == 376
KVQueue kv_queue_; // 16 + sizeof(BtreeKV) * n, n == 225, 3616 when sizeof(BtreeKV) is 16
char buf_[sizeof(Iterator)]; // 376 when sizeof(BtreeKV) is 16
};
// sizezof(BtreeIterator) == 4032, some extra memory for QueryEngine Iterator, do not larger than 4k.
// sizezof(BtreeIterator) == 4032 when sizeof(BtreeKV) is 16, some extra memory for QueryEngine Iterator, do not larger than 4k.
/*
* Use for estimate row count.
* DO NOT Keep for a long time, otherwise writing will be blocked.
*/
template<typename BtreeKey, typename BtreeVal>
class BtreeRawIterator {
private:
typedef Iterator<BtreeKey, BtreeVal> Iterator;
typedef ObKeyBtree<BtreeKey, BtreeVal> ObKeyBtree;
public:
explicit BtreeRawIterator(): iter_(NULL) {}
~BtreeRawIterator() { reset(); }
@ -168,14 +186,25 @@ public:
bool is_reverse_scan() const;
private:
Iterator *iter_; // 8byte
char buf_[376]; // sizeof(Iterator) == 376
char buf_[sizeof(Iterator)]; // 376 when sizeof(BtreeKV) is 16
};
template<typename BtreeKey, typename BtreeVal>
class ObKeyBtree
{
friend class BtreeIterator;
friend class BtreeRawIterator;
friend class Iterator;
private:
friend class BtreeIterator<BtreeKey, BtreeVal>;
friend class BtreeRawIterator<BtreeKey, BtreeVal>;
friend class Iterator<BtreeKey, BtreeVal>;
typedef BtreeIterator<BtreeKey, BtreeVal> BtreeIterator;
typedef BtreeRawIterator<BtreeKey, BtreeVal> BtreeRawIterator;
typedef Iterator<BtreeKey, BtreeVal> Iterator;
typedef BtreeNodeAllocator<BtreeKey, BtreeVal> BtreeNodeAllocator;
typedef BtreeNode<BtreeKey, BtreeVal> BtreeNode;
typedef WriteHandle<BtreeKey, BtreeVal> WriteHandle;
typedef ScanHandle<BtreeKey, BtreeVal> ScanHandle;
typedef GetHandle<BtreeKey, BtreeVal> GetHandle;
public:
ObKeyBtree(BtreeNodeAllocator &node_allocator)
: split_info_(0),
@ -191,7 +220,6 @@ public:
int del(const BtreeKey key, BtreeVal &value, int64_t version);
int re_insert(const BtreeKey key, BtreeVal value);
int insert(const BtreeKey key, BtreeVal &value);
int skip_gap(const BtreeKey start, BtreeKey &end, int64_t version, bool reverse, int64_t &size);
int get(const BtreeKey key, BtreeVal &value);
int set_key_range(BtreeIterator &iter, const BtreeKey min_key, const bool start_exclude,
const BtreeKey max_key, const bool end_exclude, int64_t version);
@ -223,4 +251,6 @@ private:
}; // end namespace common
}; // end namespace oceanbase
#include "ob_keybtree.cpp"
#endif /* __OB_KEYBTREE_H__ */

91
src/storage/memtable/mvcc/ob_keybtree_deps.h
View File

@ -13,8 +13,8 @@
#ifndef __OCEANBASE_KEYBTREE_DEPS_H_
#define __OCEANBASE_KEYBTREE_DEPS_H_
#include "lib/ob_abort.h"
#include "lib/allocator/ob_retire_station.h"
#include "storage/memtable/mvcc/ob_mvcc_row.h"
#define BTREE_ASSERT(x) if (OB_UNLIKELY(!(x))) { ob_abort(); }
@ -22,16 +22,25 @@ namespace oceanbase
{
namespace keybtree
{
template<typename BtreeKey, typename BtreeVal>
struct BtreeKV;
template<typename BtreeKey, typename BtreeVal>
class ScanHandle;
template<typename BtreeKey, typename BtreeVal>
class ObKeyBtree;
using RawType = uint64_t;
enum
{
NODE_SIZE = 280,
MAX_CPU_NUM = 64,
RETIRE_LIMIT = 1024,
NODE_KEY_COUNT = 15,
NODE_COUNT_PER_ALLOC = 128
};
template<typename BtreeKey, typename BtreeVal>
struct CompHelper
{
OB_INLINE int compare(const BtreeKey search_key, const BtreeKey idx_key, int &cmp) const
@ -46,9 +55,34 @@ public:
RWLock(): lock_(0) {}
~RWLock() {}
void set_spin() { UNUSED(ATOMIC_AAF(&read_ref_, 1)); }
bool try_rdlock();
bool try_rdlock() {
bool lock_succ = true;
uint32_t lock = ATOMIC_LOAD(&lock_);
if (0 != (lock >> 16)) {
lock_succ = false;
} else {
lock_succ = ATOMIC_BCAS(&lock_, lock, (lock + 2));
}
return lock_succ;
}
bool is_hold_wrlock(const uint16_t uid) const { return ATOMIC_LOAD(&writer_id_) == uid; }
bool try_wrlock(const uint16_t uid);
bool try_wrlock(const uint16_t uid) {
bool lock_succ = true;
while (!ATOMIC_BCAS(&writer_id_, 0, uid)) {
if (1 == (ATOMIC_LOAD(&read_ref_) & 0x1)) {
//sched_yield();
} else {
lock_succ = false;
break;
}
}
if (lock_succ) {
while (ATOMIC_LOAD(&read_ref_) > 1) {
sched_yield();
}
}
return lock_succ;
}
void rdunlock() { UNUSED(ATOMIC_FAA(&read_ref_, -2)); }
void wrunlock() { ATOMIC_STORE(&lock_, 0); }
private:
@ -118,7 +152,12 @@ class WeightEstimate
{
public:
enum { MAX_LEVEL = 64 };
WeightEstimate(int64_t node_cnt);
WeightEstimate(int64_t node_cnt) {
for(int64_t i = 0, k = 1; i < MAX_LEVEL; i++) {
weight_[i] = k;
k *= node_cnt/2;
}
}
int64_t get_weight(int64_t level) { return weight_[level]; }
private:
int64_t weight_[MAX_LEVEL];
@ -130,9 +169,14 @@ static int64_t estimate_level_weight(int64_t level)
return weight_estimate.get_weight(level);
}
template<typename BtreeKey, typename BtreeVal>
class BtreeNode: public common::ObLink
{
friend class ScanHandle;
private:
friend class ScanHandle<BtreeKey, BtreeVal>;
typedef BtreeKV<BtreeKey, BtreeVal> BtreeKV;
typedef ObKeyBtree<BtreeKey, BtreeVal> ObKeyBtree;
typedef CompHelper<BtreeKey, BtreeVal> CompHelper;
private:
enum {
MAGIC_NUM = 0xb7ee //47086
@ -280,8 +324,11 @@ private:
BtreeKV kvs_[NODE_KEY_COUNT]; // 16 * 15 = 240byte
};
template<typename BtreeKey, typename BtreeVal>
class Path
{
private:
typedef BtreeNode<BtreeKey, BtreeVal> BtreeNode;
enum { MAX_DEPTH = 16 };
struct Item
{
@ -335,8 +382,11 @@ private:
bool is_found_;
};
template<typename BtreeKey, typename BtreeVal>
class BaseHandle
{
private:
typedef CompHelper<BtreeKey, BtreeVal> CompHelper;
public:
BaseHandle(QClock& qclock): index_(), qclock_(qclock), qc_slot_(UINT64_MAX), comp_() {}
~BaseHandle() { release_ref(); }
@ -356,16 +406,27 @@ private:
CompHelper comp_;
};
class GetHandle: public BaseHandle
template<typename BtreeKey, typename BtreeVal>
class GetHandle: public BaseHandle<BtreeKey, BtreeVal>
{
private:
typedef BaseHandle<BtreeKey, BtreeVal> BaseHandle;
typedef BtreeNode<BtreeKey, BtreeVal> BtreeNode;
typedef ObKeyBtree<BtreeKey, BtreeVal> ObKeyBtree;
public:
GetHandle(ObKeyBtree &tree): BaseHandle(tree.get_qclock()) { UNUSED(tree); }
~GetHandle() {}
int get(BtreeNode *root, BtreeKey key, BtreeVal &val);
};
class ScanHandle: public BaseHandle
template<typename BtreeKey, typename BtreeVal>
class ScanHandle: public BaseHandle<BtreeKey, BtreeVal>
{
private:
typedef BaseHandle<BtreeKey, BtreeVal> BaseHandle;
typedef Path<BtreeKey, BtreeVal> Path;
typedef BtreeNode<BtreeKey, BtreeVal> BtreeNode;
typedef ObKeyBtree<BtreeKey, BtreeVal> ObKeyBtree;
private:
Path path_;
int64_t version_;
@ -386,7 +447,6 @@ public:
BtreeKey*& last_key, int64_t &gap_size);
int pop_level_node(const int64_t level);
int find_path(BtreeNode *root, BtreeKey key, int64_t version);
int skip_gap(BtreeKey& end, bool reverse, int64_t& size);
bool maybe_big_gap(bool is_backward);
int scan_forward(const int64_t level);
int scan_backward(const int64_t level);
@ -394,8 +454,14 @@ public:
int scan_backward(bool skip_inactive=false, int64_t* skip_cnt=NULL);
};
class WriteHandle: public BaseHandle
template<typename BtreeKey, typename BtreeVal>
class WriteHandle: public BaseHandle<BtreeKey, BtreeVal>
{
private:
typedef BaseHandle<BtreeKey, BtreeVal> BaseHandle;
typedef Path<BtreeKey, BtreeVal> Path;
typedef BtreeNode<BtreeKey, BtreeVal> BtreeNode;
typedef ObKeyBtree<BtreeKey, BtreeVal> ObKeyBtree;
private:
ObKeyBtree &base_;
Path path_;
@ -484,8 +550,13 @@ private:
};
// when modify this, pls modify buf_size in ob_keybtree.h.
template<typename BtreeKey, typename BtreeVal>
class Iterator
{
private:
typedef ObKeyBtree<BtreeKey, BtreeVal> ObKeyBtree;
typedef CompHelper<BtreeKey, BtreeVal> CompHelper;
typedef ScanHandle<BtreeKey, BtreeVal> ScanHandle;
public:
explicit Iterator(ObKeyBtree &btree): btree_(btree), scan_handle_(btree), jump_key_(nullptr),
cmp_result_(0), comp_(scan_handle_.get_comp()),

10
src/storage/memtable/mvcc/ob_mvcc_iterator.cpp
View File

@ -482,16 +482,6 @@ int ObMvccRowIterator::try_purge(const ObTxSnapshot &snapshot_info,
return ret;
}
int ObMvccRowIterator::get_end_gap_key(const ObTxSnapshot &snapshot_info, const ObStoreRowkey *&key, int64_t& size)
{
ObIQueryEngineIterator *iter = query_engine_iter_;
bool is_reverse = iter->is_reverse_scan();
return query_engine_->skip_gap(iter->get_key(),
key,
snapshot_info.version_.get_val_for_tx(),
is_reverse,
size);
}
} // namespace memtable
} // namespace oceanbase

2
src/storage/memtable/mvcc/ob_mvcc_iterator.h
View File

@ -179,8 +179,6 @@ public:
int get_key_val(const ObMemtableKey*& key, ObMvccRow*& row);
int try_purge(const transaction::ObTxSnapshot &snapshot_info,
const ObMemtableKey* key, ObMvccRow* row);
int get_end_gap_key(const transaction::ObTxSnapshot &snapshot_info,
const ObStoreRowkey *&key, int64_t& size);
uint8_t get_iter_flag()
{
return query_engine_iter_? query_engine_iter_->get_iter_flag(): 0;

41
src/storage/memtable/mvcc/ob_query_engine.cpp
View File

@ -25,7 +25,8 @@ ObQueryEngine::TableIndex * const ObQueryEngine::PLACE_HOLDER =
(ObQueryEngine::TableIndex *)0x1;
// modify buf size in ob_keybtree.h together, otherwise there may be memory waste or overflow.
STATIC_ASSERT(sizeof(ObQueryEngine::Iterator<keybtree::BtreeIterator>) <= 5120, "Iterator size exceeded");
STATIC_ASSERT(sizeof(ObQueryEngine::Iterator<keybtree::BtreeIterator<ObStoreRowkeyWrapper, ObMvccRow *>>) <= 5120, "Iterator size exceeded");
STATIC_ASSERT(sizeof(keybtree::Iterator<ObStoreRowkeyWrapper, ObMvccRow *>) == 376, "Iterator size changed");
int ObQueryEngine::TableIndex::init()
{
@ -55,7 +56,7 @@ void ObQueryEngine::TableIndex::check_cleanout(bool &is_all_cleanout,
int64_t &count)
{
int ret = OB_SUCCESS;
Iterator<keybtree::BtreeIterator> iter;
Iterator<keybtree::BtreeIterator<ObStoreRowkeyWrapper, ObMvccRow *>> iter;
ObStoreRowkeyWrapper scan_start_key_wrapper(&ObStoreRowkey::MIN_STORE_ROWKEY);
ObStoreRowkeyWrapper scan_end_key_wrapper(&ObStoreRowkey::MAX_STORE_ROWKEY);
iter.reset();
@ -90,7 +91,7 @@ void ObQueryEngine::TableIndex::check_cleanout(bool &is_all_cleanout,
void ObQueryEngine::TableIndex::dump2text(FILE* fd)
{
int ret = OB_SUCCESS;
Iterator<keybtree::BtreeIterator> iter;
Iterator<BtreeIterator> iter;
ObStoreRowkeyWrapper scan_start_key_wrapper(&ObStoreRowkey::MIN_STORE_ROWKEY);
ObStoreRowkeyWrapper scan_end_key_wrapper(&ObStoreRowkey::MAX_STORE_ROWKEY);
iter.reset();
@ -170,7 +171,7 @@ int64_t ObQueryEngine::TableIndex::btree_size() const
int64_t ObQueryEngine::TableIndex::btree_alloc_memory() const
{
int64_t alloc_mem = sizeof(keybtree::ObKeyBtree);
int64_t alloc_mem = sizeof(KeyBtree);
return alloc_mem;
}
@ -329,24 +330,6 @@ int ObQueryEngine::ensure(const ObMemtableKey *key, ObMvccRow *value)
return ret;
}
int ObQueryEngine::skip_gap(const ObMemtableKey *start, const ObStoreRowkey *&end, int64_t version, bool is_reverse, int64_t& size)
{
int ret = OB_SUCCESS;
TableIndex *node_ptr = nullptr;
ObStoreRowkeyWrapper start_btk(start->get_rowkey());
ObStoreRowkeyWrapper end_btk;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (OB_FAIL(get_table_index(node_ptr))) {
// do nothing
} else if (OB_FAIL(node_ptr->get_keybtree().skip_gap(start_btk, end_btk, version, is_reverse, size))) {
// do nothing
} else {
end = end_btk.get_rowkey();
}
return ret;
}
int ObQueryEngine::check_and_purge(const ObMemtableKey *key, ObMvccRow *row, int64_t version, bool &purged)
{
int ret = OB_SUCCESS;
@ -396,7 +379,7 @@ int ObQueryEngine::scan(const ObMemtableKey *start_key, const bool start_exclude
const bool end_exclude, const int64_t version, ObIQueryEngineIterator *&ret_iter)
{
int ret = OB_SUCCESS;
Iterator<keybtree::BtreeIterator> *iter = nullptr;
Iterator<BtreeIterator> *iter = nullptr;
TableIndex *node_ptr = nullptr;
if (IS_NOT_INIT) {
TRANS_LOG(WARN, "not init", "this", this);
@ -436,11 +419,11 @@ int ObQueryEngine::scan(const ObMemtableKey *start_key, const bool start_exclude
void ObQueryEngine::revert_iter(ObIQueryEngineIterator *iter)
{
iter_alloc_.free((Iterator<keybtree::BtreeIterator> *)iter);
iter_alloc_.free((Iterator<BtreeIterator> *)iter);
iter = NULL;
}
int ObQueryEngine::sample_rows(Iterator<keybtree::BtreeRawIterator> *iter, const ObMemtableKey *start_key,
int ObQueryEngine::sample_rows(Iterator<BtreeRawIterator> *iter, const ObMemtableKey *start_key,
const int start_exclude, const ObMemtableKey *end_key, const int end_exclude,
int64_t &logical_row_count, int64_t &physical_row_count, double &ratio)
{
@ -523,7 +506,7 @@ int ObQueryEngine::sample_rows(Iterator<keybtree::BtreeRawIterator> *iter, const
return ret;
}
int ObQueryEngine::init_raw_iter_for_estimate(Iterator<keybtree::BtreeRawIterator>*& iter,
int ObQueryEngine::init_raw_iter_for_estimate(Iterator<BtreeRawIterator>*& iter,
const ObMemtableKey *start_key,
const ObMemtableKey *end_key)
{
@ -560,7 +543,7 @@ int ObQueryEngine::estimate_size(const ObMemtableKey *start_key,
int64_t& total_rows)
{
int ret = OB_SUCCESS;
Iterator<keybtree::BtreeRawIterator> *iter = nullptr;
Iterator<BtreeRawIterator> *iter = nullptr;
branch_count = 0;
for(level = 0; branch_count < ESTIMATE_CHILD_COUNT_THRESHOLD && OB_SUCC(ret); ) {
level++;
@ -596,7 +579,7 @@ int ObQueryEngine::split_range(const ObMemtableKey *start_key,
ObIArray<ObStoreRange> &range_array)
{
int ret = OB_SUCCESS;
Iterator<keybtree::BtreeRawIterator> *iter = nullptr;
Iterator<BtreeRawIterator> *iter = nullptr;
int64_t level = 0;
int64_t branch_count = 0;
int64_t total_bytes = 0;
@ -676,7 +659,7 @@ int ObQueryEngine::estimate_row_count(const ObMemtableKey *start_key, const int
int64_t &logical_row_count, int64_t &physical_row_count)
{
int ret = OB_SUCCESS;
Iterator<keybtree::BtreeRawIterator> *iter = nullptr;
Iterator<BtreeRawIterator> *iter = nullptr;
int64_t remaining_row_count = 0;
int64_t element_count = 0;
int64_t phy_row_count1 = 0;

18
src/storage/memtable/mvcc/ob_query_engine.h
View File

@ -56,7 +56,10 @@ public:
INIT_TABLE_INDEX_COUNT = (1 << 10),
MAX_SAMPLE_ROW_COUNT = 500
};
typedef keybtree::ObKeyBtree KeyBtree;
typedef keybtree::ObKeyBtree<ObStoreRowkeyWrapper, ObMvccRow *> KeyBtree;
typedef keybtree::BtreeIterator<ObStoreRowkeyWrapper, ObMvccRow *> BtreeIterator;
typedef keybtree::BtreeNodeAllocator<ObStoreRowkeyWrapper, ObMvccRow *> BtreeNodeAllocator;
typedef keybtree::BtreeRawIterator<ObStoreRowkeyWrapper, ObMvccRow *> BtreeRawIterator;
typedef ObMtHash KeyHash;
template <typename BtreeIterator>
@ -140,7 +143,7 @@ public:
class TableIndex
{
public:
explicit TableIndex(keybtree::BtreeNodeAllocator &btree_allocator,
explicit TableIndex(BtreeNodeAllocator &btree_allocator,
common::ObIAllocator &memstore_allocator,
int64_t obj_cnt)
: is_inited_(false),
@ -184,7 +187,6 @@ public:
int set(const ObMemtableKey *key, ObMvccRow *value);
int get(const ObMemtableKey *parameter_key, ObMvccRow *&row, ObMemtableKey *returned_key);
int ensure(const ObMemtableKey *key, ObMvccRow *value);
int skip_gap(const ObMemtableKey *start, const ObStoreRowkey *&end, int64_t version, bool is_reverse, int64_t& size);
int check_and_purge(const ObMemtableKey *key, ObMvccRow *row, int64_t version, bool &purged);
int purge(const ObMemtableKey *key, int64_t version);
int scan(const ObMemtableKey *start_key, const bool start_exclude, const ObMemtableKey *end_key,
@ -243,10 +245,10 @@ public:
int set_table_index(const int64_t obj_cnt, TableIndex *&return_ptr);
bool is_partition_memtable_empty(const uint64_t table_id) const;
private:
int sample_rows(Iterator<keybtree::BtreeRawIterator> *iter, const ObMemtableKey *start_key,
int sample_rows(Iterator<BtreeRawIterator> *iter, const ObMemtableKey *start_key,
const int start_exclude, const ObMemtableKey *end_key, const int end_exclude,
int64_t &logical_row_count, int64_t &physical_row_count, double &ratio);
int init_raw_iter_for_estimate(Iterator<keybtree::BtreeRawIterator>*& iter,
int init_raw_iter_for_estimate(Iterator<BtreeRawIterator>*& iter,
const ObMemtableKey *start_key,
const ObMemtableKey *end_key);
int set_table_index_(const int64_t obj_cnt, TableIndex *&return_ptr);
@ -258,9 +260,9 @@ private:
uint64_t tenant_id_;
TableIndex *index_;
ObIAllocator &memstore_allocator_;
keybtree::BtreeNodeAllocator btree_allocator_;
IteratorAlloc<keybtree::BtreeIterator> iter_alloc_;
IteratorAlloc<keybtree::BtreeRawIterator> raw_iter_alloc_;
BtreeNodeAllocator btree_allocator_;
IteratorAlloc<BtreeIterator> iter_alloc_;
IteratorAlloc<BtreeRawIterator> raw_iter_alloc_;
};
} // namespace memtable

19
src/storage/memtable/ob_memtable_key.h
View File

@ -60,14 +60,6 @@ public:
rowkey_ = nullptr;
hash_val_ = 0;
}
static const ObMemtableKey& get_min_key() {
static ObMemtableKey key(&common::ObStoreRowkey::MIN_STORE_ROWKEY);
return key;
}
static const ObMemtableKey& get_max_key() {
static ObMemtableKey key(&common::ObStoreRowkey::MAX_STORE_ROWKEY);
return key;
}
public:
int compare(const ObMemtableKey &other, int &cmp) const
{
@ -304,17 +296,8 @@ public:
uint64_t hash() const { return rowkey_->hash(); }
int checksum(common::ObBatchChecksum &bc) const { return rowkey_->checksum(bc); }
int64_t to_string(char *buf, const int64_t buf_len) const { return rowkey_->to_string(buf, buf_len); }
const ObObj *get_ptr() const { return rowkey_->get_obj_ptr(); }
const char *repr() const { return rowkey_->repr(); }
static const ObStoreRowkeyWrapper& get_min_key()
{
static ObStoreRowkeyWrapper key_wrapper(&common::ObStoreRowkey::MIN_STORE_ROWKEY);
return key_wrapper;
}
static const ObStoreRowkeyWrapper& get_max_key()
{
static ObStoreRowkeyWrapper key_wrapper(&common::ObStoreRowkey::MAX_STORE_ROWKEY);
return key_wrapper;
}
public:
const common::ObStoreRowkey *rowkey_;
};

62
src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp
View File

@ -28,6 +28,7 @@
#include "storage/tx_storage/ob_ls_handle.h"
#include "storage/tx_storage/ob_ls_map.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/ddl/ob_tablet_ddl_kv.h"
namespace oceanbase
{
@ -96,6 +97,7 @@ ObTenantMetaMemMgr::ObTenantMetaMemMgr(const uint64_t tenant_id)
pinned_tablet_set_(),
memtable_pool_(tenant_id, get_default_memtable_pool_count(), "MemTblObj", ObCtxIds::DEFAULT_CTX_ID, wash_func_),
sstable_pool_(tenant_id, get_default_sstable_pool_count(), "SSTblObj", ObCtxIds::META_OBJ_CTX_ID, wash_func_),
ddl_kv_pool_(tenant_id, MAX_DDL_KV_IN_OBJ_POOL, "DDLKVObj", ObCtxIds::DEFAULT_CTX_ID, wash_func_),
tablet_pool_(tenant_id, get_default_tablet_pool_count(), "TabletObj", ObCtxIds::META_OBJ_CTX_ID, wash_func_),
tablet_ddl_kv_mgr_pool_(tenant_id, get_default_tablet_pool_count(), "DDLKvMgrObj", ObCtxIds::DEFAULT_CTX_ID, wash_func_),
tablet_memtable_mgr_pool_(tenant_id, get_default_tablet_pool_count(), "MemTblMgrObj", ObCtxIds::DEFAULT_CTX_ID,wash_func_),
@ -170,9 +172,10 @@ void ObTenantMetaMemMgr::init_pool_arr()
pool_arr_[static_cast<int>(ObITable::TableType::TX_DATA_MEMTABLE)] = &tx_data_memtable_pool_;
pool_arr_[static_cast<int>(ObITable::TableType::TX_CTX_MEMTABLE)] = &tx_ctx_memtable_pool_;
pool_arr_[static_cast<int>(ObITable::TableType::LOCK_MEMTABLE)] = &lock_memtable_pool_;
pool_arr_[static_cast<int>(ObITable::TableType::DDL_MEM_SSTABLE)] = &ddl_kv_pool_;
for (int64_t i = ObITable::TableType::MAJOR_SSTABLE; i < ObITable::TableType::MAX_TABLE_TYPE; i++) {
if (ObITable::is_sstable((ObITable::TableType)i)) {
if (ObITable::is_sstable((ObITable::TableType)i) && !ObITable::is_ddl_mem_sstable((ObITable::TableType)i)) {
pool_arr_[i] = &sstable_pool_;
}
}
@ -340,13 +343,13 @@ int ObTenantMetaMemMgr::gc_tables_in_queue(bool &all_table_cleaned)
FLOG_INFO("Successfully finish table gc", K(sstable_cnt), K(data_memtable_cnt),
K(tx_data_memtable_cnt), K(tx_ctx_memtable_cnt), K(lock_memtable_cnt),
K(pending_cnt), K(recycled_cnt), K(allocator_),
K(tablet_pool_), K(sstable_pool_), K(memtable_pool_),
K(tablet_pool_), K(sstable_pool_), K(ddl_kv_pool_), K(memtable_pool_),
"tablet count", tablet_map_.count(),
"min_minor_cnt", last_min_minor_sstable_set_.size(),
"pinned_tablet_cnt", pinned_tablet_set_.size());
} else if (REACH_COUNT_INTERVAL(100)) {
FLOG_INFO("Recycle 0 table", K(ret), K(allocator_),
K(tablet_pool_), K(sstable_pool_), K(memtable_pool_),
K(tablet_pool_), K(sstable_pool_), K(ddl_kv_pool_), K(memtable_pool_),
"tablet count", tablet_map_.count(),
"min_minor_cnt", last_min_minor_sstable_set_.size(),
"pinned_tablet_cnt", pinned_tablet_set_.size());
@ -363,7 +366,11 @@ void ObTenantMetaMemMgr::gc_sstable(ObSSTable *sstable)
} else {
const int64_t block_cnt = sstable->get_meta().get_macro_info().get_total_block_cnt();
const int64_t start_time = ObTimeUtility::current_time();
sstable_pool_.free_obj(sstable);
if (sstable->is_ddl_mem_sstable()) {
ddl_kv_pool_.free_obj(sstable);
} else {
sstable_pool_.free_obj(sstable);
}
const int64_t end_time = ObTimeUtility::current_time();
if (end_time - start_time > SSTABLE_GC_MAX_TIME) {
LOG_WARN("sstable gc costs too much time", K(start_time), K(end_time), K(block_cnt));
@ -530,6 +537,42 @@ void ObTenantMetaMemMgr::release_sstable(ObSSTable *sstable)
}
}
int ObTenantMetaMemMgr::acquire_ddl_kv(ObTableHandleV2 &handle)
{
int ret = OB_SUCCESS;
ObDDLKV *ddl_kv = nullptr;
handle.reset();
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObTenantMetaMemMgr hasn't been initialized", K(ret));
} else if (OB_FAIL(ddl_kv_pool_.acquire(ddl_kv))) {
LOG_WARN("fail to acquire ddl kv object", K(ret));
} else if (OB_FAIL(handle.set_table(ddl_kv, this, ObITable::TableType::DDL_MEM_SSTABLE))) {
LOG_WARN("fail to set table", K(ret), KP(ddl_kv));
} else {
ddl_kv = nullptr;
}
if (OB_FAIL(ret)) {
handle.reset();
if (OB_NOT_NULL(ddl_kv)) {
release_ddl_kv(ddl_kv);
}
}
return ret;
}
void ObTenantMetaMemMgr::release_ddl_kv(ObDDLKV *ddl_kv)
{
if (OB_NOT_NULL(ddl_kv)) {
if (0 != ddl_kv->get_ref()) {
LOG_ERROR("ddl kv reference count may be leak", KPC(ddl_kv));
} else {
ddl_kv_pool_.release(ddl_kv);
}
}
}
int ObTenantMetaMemMgr::acquire_memtable(ObTableHandleV2 &handle)
{
int ret = OB_SUCCESS;
@ -1061,6 +1104,8 @@ int ObTenantMetaMemMgr::get_meta_mem_status(common::ObIArray<ObTenantMetaMemStat
LOG_WARN("fail to get memtable pool's info", K(ret), K(info));
} else if (OB_FAIL(get_obj_pool_info(sstable_pool_, "SSTABLE POOL", info))) {
LOG_WARN("fail to get sstable pool's info", K(ret), K(info));
} else if (OB_FAIL(get_obj_pool_info(ddl_kv_pool_, "DDL KV POOL", info))) {
LOG_WARN("fail to get ddl kv pool's info", K(ret), K(info));
} else if (OB_FAIL(get_obj_pool_info(tablet_pool_, "TABLET POOL", info))) {
LOG_WARN("fail to get tablet pool's info", K(ret), K(info));
} else if (OB_FAIL(get_obj_pool_info(tablet_ddl_kv_mgr_pool_, "KV MGR POOL", info))) {
@ -1214,20 +1259,21 @@ int ObTenantMetaMemMgr::check_all_meta_mem_released(ObLSService &ls_service, boo
} else {
int64_t memtable_cnt = memtable_pool_.get_used_obj_cnt();
int64_t sstable_cnt = sstable_pool_.get_used_obj_cnt();
int64_t ddl_kv_cnt = ddl_kv_pool_.get_used_obj_cnt();
int64_t tablet_cnt = tablet_pool_.get_used_obj_cnt();
int64_t ddl_kv_mgr_cnt = tablet_ddl_kv_mgr_pool_.get_used_obj_cnt();
int64_t tablet_memtable_mgr_cnt = tablet_memtable_mgr_pool_.get_used_obj_cnt();
int64_t tx_data_memtable_cnt_ = tx_data_memtable_pool_.get_used_obj_cnt();
int64_t tx_ctx_memtable_cnt_ = tx_ctx_memtable_pool_.get_used_obj_cnt();
int64_t lock_memtable_cnt_ = lock_memtable_pool_.get_used_obj_cnt();
if (memtable_cnt != 0 || sstable_cnt != 0 || tablet_cnt != 0 || ddl_kv_mgr_cnt != 0
if (memtable_cnt != 0 || sstable_cnt != 0 || ddl_kv_cnt != 0 || tablet_cnt != 0 || ddl_kv_mgr_cnt != 0
|| tablet_memtable_mgr_cnt != 0 || tx_data_memtable_cnt_ != 0 || tx_ctx_memtable_cnt_ != 0
|| lock_memtable_cnt_ != 0) {
is_released = false;
} else {
is_released = true;
}
LOG_INFO("check all meta mem in t3m", K(module), K(is_released), K(memtable_cnt), K(sstable_cnt),
LOG_INFO("check all meta mem in t3m", K(module), K(is_released), K(memtable_cnt), K(sstable_cnt), K(ddl_kv_cnt),
K(tablet_cnt), K(ddl_kv_mgr_cnt), K(tablet_memtable_mgr_cnt), K(tx_data_memtable_cnt_),
K(tx_ctx_memtable_cnt_), K(lock_memtable_cnt_),
"min_minor_cnt", last_min_minor_sstable_set_.size(),
@ -1585,7 +1631,7 @@ int ObTenantMetaMemMgr::try_wash_tablet(const int64_t expect_wash_cnt)
LOG_WARN("no object can be washed", K(ret), K(wash_inner_cnt), K(wash_user_cnt),
K(wash_mem_addr_cnt), K(expect_wash_cnt),
"tablet count", tablet_map_.count(), K(allocator_), K(tablet_pool_),
K(sstable_pool_), K(memtable_pool_), K(tablet_ddl_kv_mgr_pool_),
K(sstable_pool_), K(ddl_kv_pool_), K(memtable_pool_), K(tablet_ddl_kv_mgr_pool_),
K(tablet_memtable_mgr_pool_), K(tx_data_memtable_pool_), K(tx_ctx_memtable_pool_),
K(lock_memtable_pool_), K(op),
"min_minor_cnt", last_min_minor_sstable_set_.size(),
@ -1594,7 +1640,7 @@ int ObTenantMetaMemMgr::try_wash_tablet(const int64_t expect_wash_cnt)
} else {
FLOG_INFO("succeed to wash tablet", K(wash_inner_cnt), K(wash_user_cnt),
K(wash_mem_addr_cnt), K(expect_wash_cnt), "tablet count", tablet_map_.count(),
K(allocator_), K(tablet_pool_), K(sstable_pool_), K(memtable_pool_), K(op),
K(allocator_), K(tablet_pool_), K(sstable_pool_), K(ddl_kv_pool_), K(memtable_pool_), K(op),
"min_minor_cnt", last_min_minor_sstable_set_.size(),
"pinned_tablet_cnt", pinned_tablet_set_.size(),
K(time_guard));

6
src/storage/meta_mem/ob_tenant_meta_mem_mgr.h
View File

@ -93,6 +93,7 @@ private:
static const int64_t MAX_TX_DATA_MEMTABLE_CNT_IN_OBJ_POOL = MAX_MEMSTORE_CNT * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER;
static const int64_t MAX_TX_CTX_MEMTABLE_CNT_IN_OBJ_POOL = OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER;
static const int64_t MAX_LOCK_MEMTABLE_CNT_IN_OBJ_POOL = OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER;
static const int64_t MAX_DDL_KV_IN_OBJ_POOL = 5000;
static int64_t get_default_tablet_pool_count()
{
@ -133,6 +134,9 @@ public:
int acquire_sstable(ObTableHandleV2 &handle);
int acquire_sstable(ObTableHandleV2 &handle, common::ObIAllocator &allocator);
// ddl kv interface
int acquire_ddl_kv(ObTableHandleV2 &handle);
// memtable interfaces
int acquire_memtable(ObTableHandleV2 &handle);
int acquire_tx_data_memtable(ObTableHandleV2 &handle);
@ -408,6 +412,7 @@ private:
void init_pool_arr();
void release_memtable(memtable::ObMemtable *memtable);
void release_sstable(blocksstable::ObSSTable *sstable);
void release_ddl_kv(ObDDLKV *ddl_kv);
void release_tablet(ObTablet *tablet);
void release_tablet_ddl_kv_mgr(ObTabletDDLKvMgr *ddl_kv_mgr);
void release_tablet_memtable_mgr(ObTabletMemtableMgr *memtable_mgr);
@ -443,6 +448,7 @@ private:
ObTenantMetaObjPool<memtable::ObMemtable> memtable_pool_;
ObTenantMetaObjPool<blocksstable::ObSSTable> sstable_pool_;
ObTenantMetaObjPool<ObDDLKV> ddl_kv_pool_;
ObTenantMetaObjPool<ObTablet> tablet_pool_;
ObTenantMetaObjPool<ObTabletDDLKvMgr> tablet_ddl_kv_mgr_pool_;
ObTenantMetaObjPool<ObTabletMemtableMgr> tablet_memtable_mgr_pool_;

1
src/storage/ob_i_table.cpp
View File

@ -64,6 +64,7 @@ const char* ObITable::table_type_name_[] =
"META_MAJOR",
"DDL_DUMP",
"REMOTE_LOGICAL_MINOR",
"DDL_MEM",
};
uint64_t ObITable::TableKey::hash() const

18
src/storage/ob_i_table.h
View File

@ -88,8 +88,9 @@ public:
MINOR_SSTABLE = 11,
MINI_SSTABLE = 12,
META_MAJOR_SSTABLE = 13,
KV_DUMP_SSTABLE = 14,
DDL_DUMP_SSTABLE = 14,
REMOTE_LOGICAL_MINOR_SSTABLE = 15,
DDL_MEM_SSTABLE = 16,
// < add new sstable before here, See is_sstable()
MAX_TABLE_TYPE
};
@ -121,6 +122,8 @@ public:
OB_INLINE bool is_meta_major_sstable() const { return ObITable::is_meta_major_sstable(table_type_); }
OB_INLINE bool is_multi_version_table() const { return ObITable::is_multi_version_table(table_type_); }
OB_INLINE bool is_ddl_sstable() const { return ObITable::is_ddl_sstable(table_type_); }
OB_INLINE bool is_ddl_dump_sstable() const { return ObITable::is_ddl_dump_sstable(table_type_); }
OB_INLINE bool is_ddl_mem_sstable() const { return ObITable::is_ddl_mem_sstable(table_type_); }
OB_INLINE bool is_table_with_scn_range() const { return ObITable::is_table_with_scn_range(table_type_); }
OB_INLINE bool is_remote_logical_minor_sstable() const { return ObITable::is_remote_logical_minor_sstable(table_type_); }
@ -231,6 +234,8 @@ public:
OB_INLINE bool is_table_with_scn_range() const { return is_table_with_scn_range(key_.table_type_); }
virtual OB_INLINE int64_t get_timestamp() const { return 0; }
virtual bool is_ddl_sstable() const { return is_ddl_sstable(key_.table_type_); }
virtual bool is_ddl_dump_sstable() const { return is_ddl_dump_sstable(key_.table_type_); }
virtual bool is_ddl_mem_sstable() const { return is_ddl_mem_sstable(key_.table_type_); }
virtual bool is_remote_logical_minor_sstable() const { return is_remote_logical_minor_sstable(key_.table_type_); }
virtual bool is_empty() const = 0;
DECLARE_VIRTUAL_TO_STRING;
@ -315,7 +320,16 @@ public:
}
static bool is_ddl_sstable(const TableType table_type)
{
return ObITable::TableType::KV_DUMP_SSTABLE == table_type;
return ObITable::TableType::DDL_DUMP_SSTABLE == table_type
|| ObITable::TableType::DDL_MEM_SSTABLE == table_type;
}
static bool is_ddl_dump_sstable(const TableType table_type)
{
return ObITable::TableType::DDL_DUMP_SSTABLE == table_type;
}
static bool is_ddl_mem_sstable(const TableType table_type)
{
return ObITable::TableType::DDL_MEM_SSTABLE == table_type;
}
static bool is_table_with_scn_range(const TableType table_type)
{

42
src/storage/ob_storage_struct.cpp
View File

@ -208,6 +208,25 @@ int ObGetMergeTablesResult::assign(const ObGetMergeTablesResult &src)
}
return ret;
}
ObDDLTableStoreParam::ObDDLTableStoreParam()
: keep_old_ddl_sstable_(true),
ddl_start_scn_(SCN::min_scn()),
ddl_checkpoint_scn_(SCN::min_scn()),
ddl_snapshot_version_(0),
ddl_execution_id_(-1),
ddl_cluster_version_(0)
{
}
bool ObDDLTableStoreParam::is_valid() const
{
return ddl_start_scn_.is_valid()
&& ddl_checkpoint_scn_.is_valid()
&& ddl_snapshot_version_ >= 0
&& ddl_execution_id_ >= 0
&& ddl_cluster_version_ >= 0;
}
ObUpdateTableStoreParam::ObUpdateTableStoreParam(
const int64_t snapshot_version,
@ -218,17 +237,12 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam(
snapshot_version_(snapshot_version),
clog_checkpoint_scn_(),
multi_version_start_(multi_version_start),
keep_old_ddl_sstable_(true),
need_report_(false),
storage_schema_(storage_schema),
rebuild_seq_(rebuild_seq),
update_with_major_flag_(false),
need_check_sstable_(false),
ddl_checkpoint_scn_(SCN::min_scn()),
ddl_start_scn_(SCN::min_scn()),
ddl_snapshot_version_(0),
ddl_execution_id_(-1),
ddl_cluster_version_(0),
ddl_info_(),
allow_duplicate_sstable_(false),
tx_data_(),
binding_info_(),
@ -255,17 +269,12 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam(
snapshot_version_(snapshot_version),
clog_checkpoint_scn_(),
multi_version_start_(multi_version_start),
keep_old_ddl_sstable_(true),
need_report_(need_report),
storage_schema_(storage_schema),
rebuild_seq_(rebuild_seq),
update_with_major_flag_(false),
need_check_sstable_(need_check_sstable),
ddl_checkpoint_scn_(SCN::min_scn()),
ddl_start_scn_(SCN::min_scn()),
ddl_snapshot_version_(0),
ddl_execution_id_(-1),
ddl_cluster_version_(0),
ddl_info_(),
allow_duplicate_sstable_(allow_duplicate_sstable),
tx_data_(),
binding_info_(),
@ -279,27 +288,20 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam(
ObUpdateTableStoreParam::ObUpdateTableStoreParam(
const ObTableHandleV2 &table_handle,
const int64_t snapshot_version,
const bool keep_old_ddl_sstable,
const ObStorageSchema *storage_schema,
const int64_t rebuild_seq,
const int64_t ddl_cluster_version,
const bool update_with_major_flag,
const bool need_report)
: table_handle_(table_handle),
snapshot_version_(snapshot_version),
clog_checkpoint_scn_(),
multi_version_start_(0),
keep_old_ddl_sstable_(keep_old_ddl_sstable),
need_report_(need_report),
storage_schema_(storage_schema),
rebuild_seq_(rebuild_seq),
update_with_major_flag_(update_with_major_flag),
need_check_sstable_(false),
ddl_checkpoint_scn_(SCN::min_scn()),
ddl_start_scn_(SCN::min_scn()),
ddl_snapshot_version_(0),
ddl_execution_id_(-1),
ddl_cluster_version_(ddl_cluster_version),
ddl_info_(),
allow_duplicate_sstable_(false),
tx_data_(),
binding_info_(),

32
src/storage/ob_storage_struct.h
View File

@ -27,6 +27,7 @@
#include "share/scn.h"
#include "storage/tablet/ob_tablet_multi_source_data.h"
#include "storage/tablet/ob_tablet_binding_helper.h"
#include "storage/ddl/ob_ddl_struct.h"
namespace oceanbase
{
@ -293,6 +294,23 @@ OB_INLINE bool is_valid_migrate_status(const ObMigrateStatus &status)
return status >= OB_MIGRATE_STATUS_NONE && status < OB_MIGRATE_STATUS_MAX;
}
struct ObDDLTableStoreParam final
{
public:
ObDDLTableStoreParam();
~ObDDLTableStoreParam() = default;
bool is_valid() const;
TO_STRING_KV(K_(keep_old_ddl_sstable), K_(ddl_start_scn), K_(ddl_checkpoint_scn),
K_(ddl_snapshot_version), K_(ddl_execution_id), K_(ddl_cluster_version));
public:
bool keep_old_ddl_sstable_;
share::SCN ddl_start_scn_;
share::SCN ddl_checkpoint_scn_;
int64_t ddl_snapshot_version_;
int64_t ddl_execution_id_;
int64_t ddl_cluster_version_;
};
struct ObUpdateTableStoreParam
{
ObUpdateTableStoreParam(
@ -316,35 +334,27 @@ struct ObUpdateTableStoreParam
ObUpdateTableStoreParam( // for ddl merge task only
const ObTableHandleV2 &table_handle,
const int64_t snapshot_version,
const bool keep_old_ddl_sstable,
const ObStorageSchema *storage_schema,
const int64_t rebuild_seq,
const int64_t ddl_cluster_version,
const bool update_with_major_flag,
const bool need_report = false);
bool is_valid() const;
TO_STRING_KV(K_(table_handle), K_(snapshot_version), K_(clog_checkpoint_scn), K_(multi_version_start),
K_(keep_old_ddl_sstable), K_(need_report), KPC_(storage_schema), K_(rebuild_seq), K_(update_with_major_flag),
K_(need_check_sstable), K_(ddl_checkpoint_scn), K_(ddl_start_scn), K_(ddl_snapshot_version),
K_(ddl_execution_id), K_(ddl_cluster_version), K_(allow_duplicate_sstable), K_(tx_data), K_(binding_info), K_(auto_inc_seq),
K_(need_report), KPC_(storage_schema), K_(rebuild_seq), K_(update_with_major_flag),
K_(need_check_sstable), K_(ddl_info), K_(allow_duplicate_sstable), K_(tx_data), K_(binding_info), K_(auto_inc_seq),
KPC_(medium_info_list), "merge_type", merge_type_to_str(merge_type_));
ObTableHandleV2 table_handle_;
int64_t snapshot_version_;
share::SCN clog_checkpoint_scn_;
int64_t multi_version_start_;
bool keep_old_ddl_sstable_;
bool need_report_;
const ObStorageSchema *storage_schema_;
int64_t rebuild_seq_;
bool update_with_major_flag_;
bool need_check_sstable_;
share::SCN ddl_checkpoint_scn_;
share::SCN ddl_start_scn_;
int64_t ddl_snapshot_version_;
int64_t ddl_execution_id_;
int64_t ddl_cluster_version_;
ObDDLTableStoreParam ddl_info_;
bool allow_duplicate_sstable_;
// msd

1
src/storage/restore/ob_ls_restore_handler.cpp
View File

@ -31,6 +31,7 @@
#include "share/backup/ob_backup_data_store.h"
#include "observer/ob_server_event_history_table_operator.h"
#include "share/restore/ob_restore_persist_helper.h"
#include "storage/tablet/ob_tablet.h"
using namespace oceanbase;
using namespace share;

1
src/storage/restore/ob_ls_restore_task_mgr.cpp
View File

@ -14,6 +14,7 @@
#include "ob_ls_restore_task_mgr.h"
#include "storage/ls/ob_ls.h"
#include "lib/lock/ob_mutex.h"
#include "storage/tablet/ob_tablet.h"
using namespace oceanbase;
using namespace share;

1
src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp
View File

@ -30,6 +30,7 @@
#include "observer/ob_server_event_history_table_operator.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "sql/das/ob_das_id_service.h"
#include "storage/tablet/ob_tablet.h"
namespace oceanbase
{

2
src/storage/tablet/ob_table_store_util.cpp
View File

@ -79,7 +79,7 @@ void ObITableArray::reset_table(const int64_t pos)
&& reinterpret_cast<ObSSTable*>(array_[pos])->is_small_sstable())) {
FLOG_INFO("this thread doesn't have MTL ctx, push sstable into gc queue", KP(array_[pos]), K(array_[pos]->get_key()));
meta_mem_mgr_->push_table_into_gc_queue(array_[pos], array_[pos]->get_key().table_type_);
} else if (array_[pos]->is_sstable()) {
} else if (array_[pos]->is_sstable() && !array_[pos]->is_ddl_mem_sstable()) {
meta_mem_mgr_->gc_sstable(reinterpret_cast<ObSSTable*>(array_[pos]));
} else {
meta_mem_mgr_->push_table_into_gc_queue(array_[pos], array_[pos]->get_key().table_type_);

45
src/storage/tablet/ob_tablet.cpp
View File

@ -222,8 +222,7 @@ int ObTablet::init(
param.snapshot_version_, param.multi_version_start_,
tx_data, ddl_data, autoinc_seq, input_max_sync_schema_version,
MAX(max_serialized_medium_scn, old_tablet.tablet_meta_.max_serialized_medium_scn_),
param.clog_checkpoint_scn_, param.ddl_checkpoint_scn_, param.ddl_start_scn_, param.ddl_snapshot_version_,
param.ddl_execution_id_, param.ddl_cluster_version_))) {
param.clog_checkpoint_scn_, param.ddl_info_))) {
LOG_WARN("failed to init tablet meta", K(ret), K(old_tablet), K(param),
K(tx_data), K(ddl_data), K(autoinc_seq), K(input_max_sync_schema_version));
} else if (OB_FAIL(table_store_.init(*allocator_, this, param, old_tablet.table_store_))) {
@ -1153,6 +1152,22 @@ int ObTablet::get_read_major_sstable(
return table_store_.get_read_major_sstable(major_snapshot_version, iter);
}
int ObTablet::get_ddl_memtables(common::ObIArray<ObITable *> &ddl_memtables) const
{
int ret = OB_SUCCESS;
ddl_memtables.reset();
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret), K_(is_inited));
} else {
const ObSSTableArray &tmp_tables = table_store_.get_ddl_memtables();
if (!tmp_tables.empty() && OB_FAIL(tmp_tables.get_all_tables(ddl_memtables))) {
LOG_WARN("fail to get ddl memtables", K(ret));
}
}
return ret;
}
int ObTablet::get_all_sstables(common::ObIArray<ObITable *> &sstables) const
{
int ret = OB_SUCCESS;
@ -1974,16 +1989,26 @@ int ObTablet::get_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool try_creat
{
int ret = OB_SUCCESS;
ddl_kv_mgr_handle.reset();
ObTabletPointer *tablet_ptr = static_cast<ObTabletPointer*>(pointer_hdl_.get_resource_ptr());
if (try_create) {
if (OB_FAIL(tablet_ptr->create_ddl_kv_mgr(tablet_meta_.ls_id_, tablet_meta_.tablet_id_, ddl_kv_mgr_handle))) {
LOG_WARN("create ddl kv mgr failed", K(ret), K(tablet_meta_));
if (!pointer_hdl_.is_valid()) {
if (try_create) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet pointer not valid", K(ret));
} else {
ret = OB_ENTRY_NOT_EXIST;
LOG_DEBUG("tablet pointer not valid", K(ret));
}
} else {
tablet_ptr->get_ddl_kv_mgr(ddl_kv_mgr_handle);
if (!ddl_kv_mgr_handle.is_valid()) {
ret = OB_ENTRY_NOT_EXIST;
LOG_DEBUG("ddl kv mgr not exist", K(ret), K(ddl_kv_mgr_handle));
ObTabletPointer *tablet_ptr = static_cast<ObTabletPointer*>(pointer_hdl_.get_resource_ptr());
if (try_create) {
if (OB_FAIL(tablet_ptr->create_ddl_kv_mgr(tablet_meta_.ls_id_, tablet_meta_.tablet_id_, ddl_kv_mgr_handle))) {
LOG_WARN("create ddl kv mgr failed", K(ret), K(tablet_meta_));
}
} else {
tablet_ptr->get_ddl_kv_mgr(ddl_kv_mgr_handle);
if (!ddl_kv_mgr_handle.is_valid()) {
ret = OB_ENTRY_NOT_EXIST;
LOG_DEBUG("ddl kv mgr not exist", K(ret), K(ddl_kv_mgr_handle));
}
}
}
return ret;

3
src/storage/tablet/ob_tablet.h
View File

@ -76,8 +76,6 @@ class ObTableHandleV2;
class ObFreezer;
class ObTabletDDLInfo;
class ObTabletDDLKvMgr;
class ObDDLKVHandle;
class ObDDLKVsHandle;
class ObStorageSchema;
class ObTabletTableIterator;
class ObMetaDiskAddr;
@ -225,6 +223,7 @@ public:
int get_all_sstables(common::ObIArray<ObITable *> &sstables) const;
int get_sstables_size(int64_t &used_size) const;
int get_memtables(common::ObIArray<storage::ObITable *> &memtables, const bool need_active = false) const;
int get_ddl_memtables(common::ObIArray<ObITable *> &ddl_memtables) const;
int check_need_remove_old_table(const int64_t multi_version_start, bool &need_remove) const;
int update_upper_trans_version(ObLS &ls, bool &is_updated);

3
src/storage/tablet/ob_tablet_create_sstable_param.h
View File

@ -19,6 +19,7 @@
#include "storage/blocksstable/ob_imicro_block_reader.h"
#include "storage/meta_mem/ob_meta_obj_struct.h"
#include "share/scn.h"
#include "storage/ddl/ob_ddl_struct.h"
namespace oceanbase
{
@ -103,7 +104,7 @@ public:
int64_t occupy_size_;
int64_t original_size_;
int64_t max_merged_trans_version_;
share::SCN ddl_scn_;
share::SCN ddl_scn_; // saved into sstable meta
share::SCN filled_tx_scn_;
bool contain_uncommitted_row_;
bool is_meta_root_;

Some files were not shown because too many files have changed in this diff Show More