hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] DML stong type refactoring and optimization

Browse Source
This commit is contained in:
mjhmllover 2024-08-27 08:31:08 +00:00 committed by ob-robot
parent 24f761a6da
commit 4edb7b4b37
82 changed files with 4340 additions and 2806 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp
View File

@ -769,7 +769,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
// first half multi scan, second half multi get
ranges.reuse();
test_allocator.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ObDatumRowkey tmp_rowkey;
mget_ranges[i].border_flag_.set_inclusive_start();
@ -824,7 +823,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
// first half multi get, second half multi scan
ranges.reuse();
test_allocator.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ObDatumRowkey tmp_rowkey;
mget_ranges[i].border_flag_.set_inclusive_start();
@ -883,7 +881,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
// first one multi get, others multi scan
ranges.reuse();
test_allocator.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ObDatumRowkey tmp_rowkey;
mget_ranges[i].border_flag_.set_inclusive_start();
@ -938,7 +935,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
// first one multi scan, others multi get
ranges.reuse();
test_allocator.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ObDatumRowkey tmp_rowkey;
mget_ranges[i].border_flag_.set_inclusive_start();
@ -994,7 +990,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan(
// multi scan not exist row
STORAGE_LOG(DEBUG, "multi_scan_not_exist_row");
ranges.reuse();
test_allocator.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ObDatumRowkey tmp_rowkey;
mget_ranges[i].border_flag_.set_inclusive_start();

39
mittest/mtlenv/storage/test_co_merge.cpp
View File

@ -620,7 +620,8 @@ TEST_F(TestCOMerge, test_merge_default_row_store_with_empty_major)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -722,7 +723,8 @@ TEST_F(TestCOMerge, test_merge_default_row_store_with_empty_major)
// ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
// ObMockDirectReadIterator sstable_iter;
// ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
// ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
// bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
// ASSERT_TRUE(is_equal);
// scanner->~ObStoreRowIterator();
// handle1.reset();
// handle2.reset();
@ -819,7 +821,8 @@ TEST_F(TestCOMerge, test_column_store_merge_with_empty_co_table)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, *cg_read_info));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
merger.reset();
@ -946,7 +949,8 @@ TEST_F(TestCOMerge, test_co_merge_with_twice_major)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
handle1.reset();
@ -1049,7 +1053,8 @@ TEST_F(TestCOMerge, test_co_merge_with_twice_major)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(new_result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
}
@ -1205,7 +1210,8 @@ TEST_F(TestCOMerge, test_merge_range)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
}
@ -1355,7 +1361,8 @@ TEST_F(TestCOMerge, test_merge_range_with_open)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
}
@ -1508,7 +1515,8 @@ TEST_F(TestCOMerge, test_merge_range_with_left_open)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
}
@ -1661,7 +1669,8 @@ TEST_F(TestCOMerge, test_merge_range_with_right_open)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
}
@ -1823,7 +1832,8 @@ TEST_F(TestCOMerge, test_merge_range_left_is_min)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
}
@ -1988,7 +1998,8 @@ TEST_F(TestCOMerge, test_merge_range_with_right_max)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
}
@ -2268,7 +2279,8 @@ TEST_F(TestCOMerge, test_merge_range_is_whole_range)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
}
@ -2568,7 +2580,8 @@ TEST_F(TestCOMerge, test_rebuild_sstable)
ObMockIterator res_iter;
res_iter.reset();
ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i]));
ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, false/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
}
}

4
mittest/mtlenv/storage/test_lob_manager.cpp
View File

@ -328,7 +328,7 @@ void TestLobManager::insert_lob_piece(
ASSERT_NE(nullptr, tablet);
// insert rows
ObMockNewRowIterator mock_iter;
ObMockDatumRowIterator mock_iter;
ObSEArray<uint64_t, 512> column_ids;
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 0); // pk
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 1); // c1
@ -452,7 +452,7 @@ void TestLobManager::insert_lob_meta(
ASSERT_NE(nullptr, tablet);
// insert rows
ObMockNewRowIterator mock_iter;
ObMockDatumRowIterator mock_iter;
ObSEArray<uint64_t, 512> column_ids;
for (int i = 0; i < ObLobMetaUtil::LOB_META_COLUMN_CNT; i++) {
column_ids.push_back(OB_APP_MIN_COLUMN_ID + i);

101
mittest/mtlenv/storage/test_memtable_v2.cpp
View File

@ -467,7 +467,7 @@ public:
void write_tx(ObStoreCtx *wtx,
ObMemtable *memtable,
const int64_t snapshot,
const ObStoreRow &write_row,
ObDatumRow &write_row,
const int expect_ret = OB_SUCCESS,
const int64_t expire_time = 10000000000)
{
@ -737,20 +737,23 @@ public:
int mock_row(const int64_t key,
const int64_t value,
ObDatumRowkey &rowkey,
ObStoreRow &row)
ObDatumRow &row)
{
rowkey_datums_[0].set_int(key);
rowkey_datums_[1].set_int(value);
rowkey.assign(rowkey_datums_, 1);
ObStorageDatum *datum = new ObStorageDatum[2];
datum[0].set_int(key);
datum[1].set_int(value);
ObObj *obj = new ObObj[2];
obj[0].set_int(key);
obj[1].set_int(value);
row.row_val_.cells_ = obj;
row.row_val_.count_ = 2;
row.row_val_.projector_ = NULL;
row.flag_.set_flag(ObDmlFlag::DF_INSERT);
row.storage_datums_ = datum;
row.count_ = 2;
row.row_flag_.set_flag(ObDmlFlag::DF_INSERT);
rowkey.store_rowkey_.assign(obj, 1);
return OB_SUCCESS;
@ -758,18 +761,20 @@ public:
int mock_delete(const int64_t key,
ObDatumRowkey &rowkey,
ObStoreRow &row)
ObDatumRow &row)
{
rowkey_datums_[0].set_int(key);
rowkey.assign(rowkey_datums_, 1);
ObStorageDatum *datum = new ObStorageDatum[1];
datum[0].set_int(key);
ObObj *obj = new ObObj[1];
obj[0].set_int(key);
row.row_val_.cells_ = obj;
row.row_val_.count_ = 2;
row.row_val_.projector_ = NULL;
row.flag_.set_flag(ObDmlFlag::DF_DELETE);
row.storage_datums_ = datum;
row.count_ = 2;
row.row_flag_.set_flag(ObDmlFlag::DF_DELETE);
rowkey.store_rowkey_.assign(obj, 1);
return OB_SUCCESS;
@ -1240,7 +1245,7 @@ TEST_F(TestMemtableV2, test_write_read_conflict)
TRANS_LOG(INFO, "######## CASE1: write row into memtable");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -1384,7 +1389,7 @@ TEST_F(TestMemtableV2, test_tx_abort)
TRANS_LOG(INFO, "######## CASE1: write row into memtable");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -1497,7 +1502,7 @@ TEST_F(TestMemtableV2, test_write_write_conflict)
TRANS_LOG(INFO, "######## CASE1: txn1 write row into memtable");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -1542,7 +1547,7 @@ TEST_F(TestMemtableV2, test_write_write_conflict)
TRANS_LOG(INFO, "######## CASE2: txn2 write row into memtable, lock for write failed");
ObDatumRowkey rowkey2;
ObStoreRow write_row2;
ObDatumRow write_row2;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
3, /*value*/
rowkey2,
@ -1564,7 +1569,7 @@ TEST_F(TestMemtableV2, test_write_write_conflict)
TRANS_LOG(INFO, "######## CASE3: txn1 write row into memtable, lock for write succeed");
ObDatumRowkey rowkey3;
ObStoreRow write_row3;
ObDatumRow write_row3;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
4, /*value*/
rowkey3,
@ -1713,7 +1718,7 @@ TEST_F(TestMemtableV2, test_lock)
TRANS_LOG(INFO, "######## CASE1: txn1 lock row in memtable");
ObDatumRowkey rowkey;
ObStoreRow tmp_row;
ObDatumRow tmp_row;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -1768,7 +1773,7 @@ TEST_F(TestMemtableV2, test_lock)
TRANS_LOG(INFO, "######## CASE3: txn2 write row in memtable with lock for write failed");
ObDatumRowkey rowkey2;
ObStoreRow write_row2;
ObDatumRow write_row2;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
3, /*value*/
rowkey2,
@ -1889,7 +1894,7 @@ TEST_F(TestMemtableV2, test_lock)
ObTransID write_tx_id3 = ObTransID(3);
ObStoreCtx *wtx3 = start_tx(write_tx_id3);
ObDatumRowkey rowkey3;
ObStoreRow write_row3;
ObDatumRow write_row3;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
4, /*value*/
rowkey3,
@ -1975,7 +1980,7 @@ TEST_F(TestMemtableV2, test_sstable_lock)
is_sstable_contains_lock_ = true;
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -2007,8 +2012,8 @@ TEST_F(TestMemtableV2, test_rollback_to)
TRANS_LOG(INFO, "######## CASE1: write row into memtable");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObStoreRow write_row2;
ObDatumRow write_row;
ObDatumRow write_row2;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -2070,7 +2075,7 @@ TEST_F(TestMemtableV2, test_rollback_to)
wtx_seq_no1 + 1 /*to*/);
ObDatumRowkey rowkey3;
ObStoreRow write_row3;
ObDatumRow write_row3;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
4, /*value*/
rowkey3,
@ -2099,11 +2104,11 @@ TEST_F(TestMemtableV2, test_replay)
TRANS_LOG(INFO, "######## CASE1: txn1 and txn3 write row in lmemtable");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
ObDatumRowkey rowkey2;
ObStoreRow write_row2;
ObDatumRow write_row2;
ObDatumRowkey rowkey3;
ObStoreRow write_row3;
ObDatumRow write_row3;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
@ -2286,9 +2291,9 @@ TEST_F(TestMemtableV2, test_replay)
// TRANS_LOG(INFO, "######## CASE1: txn1 write row in lmemtable");
// ObDatumRowkey rowkey;
// ObStoreRow write_row;
// ObDatumRow write_row;
// ObDatumRowkey rowkey2;
// ObStoreRow write_row2;
// ObDatumRow write_row2;
// EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
// 2, /*value*/
@ -2421,11 +2426,11 @@ TEST_F(TestMemtableV2, test_compact)
TRANS_LOG(INFO, "######## CASE1: txn1 and txn3 write row in lmemtable");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
ObDatumRowkey rowkey2;
ObStoreRow write_row2;
ObDatumRow write_row2;
ObDatumRowkey rowkey3;
ObStoreRow write_row3;
ObDatumRow write_row3;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
@ -2653,11 +2658,11 @@ TEST_F(TestMemtableV2, test_compact_v2)
TRANS_LOG(INFO, "######## CASE1: txn1 write two rows and txn3 write a row in memtable");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
ObDatumRowkey rowkey2;
ObStoreRow write_row2;
ObDatumRow write_row2;
ObDatumRowkey rowkey3;
ObStoreRow write_row3;
ObDatumRow write_row3;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
@ -2820,13 +2825,13 @@ TEST_F(TestMemtableV2, test_compact_v3)
TRANS_LOG(INFO, "######## CASE1: txn1 write two row and txn2 write row in lmemtable");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
ObDatumRowkey rowkey2;
ObStoreRow write_row2;
ObDatumRow write_row2;
ObDatumRowkey rowkey3;
ObStoreRow write_row3;
ObDatumRow write_row3;
ObDatumRowkey rowkey4;
ObStoreRow write_row4;
ObDatumRow write_row4;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
@ -2958,11 +2963,11 @@ TEST_F(TestMemtableV2, test_dml_flag)
TRANS_LOG(INFO, "######## CASE1: txns write and row in lmemtable, test its dml flag");
ObDatumRowkey rowkey;
ObStoreRow write_row1;
ObDatumRow write_row1;
ObDatumRowkey rowkey2;
ObStoreRow write_row2;
ObDatumRow write_row2;
ObDatumRowkey rowkey3;
ObStoreRow write_row3;
ObDatumRow write_row3;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
@ -3121,7 +3126,7 @@ TEST_F(TestMemtableV2, test_fast_commit)
TRANS_LOG(INFO, "######## CASE1: write row into memtable and fast commit, then check result is ok");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -3241,7 +3246,7 @@ TEST_F(TestMemtableV2, test_fast_commit_with_no_delay_cleanout)
TRANS_LOG(INFO, "######## CASE1: write row into memtable and not fast commit, then check result is ok");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -3369,8 +3374,8 @@ TEST_F(TestMemtableV2, test_seq_set_violation)
TRANS_LOG(INFO, "######## CASE1: write row into memtable");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObStoreRow write_row2;
ObDatumRow write_row;
ObDatumRow write_row2;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -3427,7 +3432,7 @@ TEST_F(TestMemtableV2, test_parallel_lock_with_same_txn)
TRANS_LOG(INFO, "######## CASE1: lock row into memtable parallelly");
ObDatumRowkey rowkey;
ObStoreRow write_row;
ObDatumRow write_row;
EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/
2, /*value*/
rowkey,
@ -3490,14 +3495,14 @@ TEST_F(TestMemtableV2, test_sync_log_fail_on_frozen_memtable)
ObStoreCtx *tx_1 = start_tx(txid_1);
int i = 1;
for (; i <= 10; i++) {
ObStoreRow row1;
ObDatumRow row1;
EXPECT_EQ(OB_SUCCESS, mock_row(i, i*2, rowkey, row1));
write_tx(tx_1, memtable, 10000, row1);
}
ObTransID txid_2 = ObTransID(2);
ObStoreCtx *tx_2 = start_tx(txid_2);
for (; i <= 20; i++) {
ObStoreRow row1;
ObDatumRow row1;
EXPECT_EQ(OB_SUCCESS, mock_row(i, i*4, rowkey, row1));
write_tx(tx_2, memtable, 10000, row1);
}

24
mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp
View File

@ -305,7 +305,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycle_macro)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -403,7 +404,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycle_after_reuse)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -499,7 +501,8 @@ TEST_F(TestMultiVersionMergeRecycle, reuse_after_recycle)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -599,7 +602,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycled_micros_after_reuse)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -708,7 +712,8 @@ TEST_F(TestMultiVersionMergeRecycle, rowkeys_across_micros)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -828,7 +833,8 @@ TEST_F(TestMultiVersionMergeRecycle, rowkeys_across_macro)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -925,7 +931,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycle_macro_with_last_row)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -1025,7 +1032,8 @@ TEST_F(TestMultiVersionMergeRecycle, reuse_after_recycle_with_last)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();

81
mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp
View File

@ -345,7 +345,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_two_macro_and_second_macro_is_filtere
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -452,7 +453,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_three_macro_inc_merge)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -575,7 +577,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_committed_in_minor)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -667,7 +670,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_in_one_macro_committed_is_last)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -787,7 +791,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_in_one_macro_committed_following_l
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -913,7 +918,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_in_one_macro_committed_following_s
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -1021,7 +1027,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_three_macro_full_merge)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -1175,7 +1182,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -1337,7 +1345,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans_can_compact)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -1504,7 +1513,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans_can_not_compact)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -1610,7 +1620,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_macro_reused_with_shadow)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -1736,7 +1747,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_macro_reused_without_shadow)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -1820,7 +1832,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_greater_multi_version)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -1931,7 +1944,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_greater_multi_version_and_uncommit
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -2066,7 +2080,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_ghost_row)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -2151,7 +2166,8 @@ TEST_F(TestMultiVersionMerge, compare_dml_flag)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -2250,7 +2266,8 @@ TEST_F(TestMultiVersionMerge, get_last_after_reuse)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -2340,7 +2357,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_two_macro_with_commit_scn_less_multi_
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -2461,7 +2479,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_macro_with_last_shadow_version_less_t
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -2562,7 +2581,8 @@ TEST_F(TestMultiVersionMerge, shadow_row_is_last_in_macro)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -2684,7 +2704,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_macro_without_open_next_macro)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -2792,7 +2813,8 @@ TEST_F(TestMultiVersionMerge, range_cross_macro)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -2933,7 +2955,8 @@ TEST_F(TestMultiVersionMerge, test_merge_base_iter_have_ghost_row)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -3033,7 +3056,8 @@ TEST_F(TestMultiVersionMerge, test_major_range_cross_macro)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
scanner->~ObStoreRowIterator();
handle1.reset();
handle2.reset();
@ -3150,7 +3174,8 @@ TEST_F(TestMultiVersionMerge, test_trans_cross_macro_with_ghost_row)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -3272,7 +3297,8 @@ TEST_F(TestMultiVersionMerge, test_trans_cross_macro_with_ghost_row2)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();
@ -3387,7 +3413,8 @@ TEST_F(TestMultiVersionMerge, test_running_trans_cross_macro_with_abort_sql_seq)
ASSERT_EQ(OB_SUCCESS, res_iter.from(result1));
ObMockDirectReadIterator sstable_iter;
ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_));
ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/));
bool is_equal = res_iter.equals<ObMockDirectReadIterator, ObStoreRow>(sstable_iter, true/*cmp multi version row flag*/);
ASSERT_TRUE(is_equal);
ASSERT_EQ(OB_SUCCESS, clear_tx_data());
scanner->~ObStoreRowIterator();
handle1.reset();

4
mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp
View File

@ -97,7 +97,7 @@ void TestTableScanPureDataTable::insert_data_to_tablet(MockObAccessService *acce
ASSERT_NE(nullptr, tablet);
// insert rows
ObMockNewRowIterator mock_iter;
ObMockDatumRowIterator mock_iter;
ObSEArray<uint64_t, 512> column_ids;
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 0); // pk
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 1); // c1
@ -105,7 +105,7 @@ void TestTableScanPureDataTable::insert_data_to_tablet(MockObAccessService *acce
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 3); // c3
column_ids.push_back(OB_APP_MIN_COLUMN_ID + 4); // c4
ASSERT_EQ(OB_SUCCESS, mock_iter.from(TestDmlCommon::data_row_str));
ASSERT_EQ(OB_SUCCESS, mock_iter.from_for_datum(TestDmlCommon::data_row_str));
transaction::ObTransService *tx_service = MTL(transaction::ObTransService*);
// 1. get tx desc

4
mittest/mtlenv/storage/test_trans.cpp
View File

@ -164,8 +164,8 @@ void TestTrans::create_ls(uint64_t tenant_id, ObLSID &ls_id, ObLS *&ls)
void TestTrans::insert_rows(ObLSID &ls_id, ObTabletID &tablet_id, ObTxDesc &tx_desc, ObTxReadSnapshot snapshot, const char* ins_str)
{
int64_t affected_rows = 0;
ObMockNewRowIterator ins_iter;
ASSERT_EQ(OB_SUCCESS, ins_iter.from(ins_str));
ObMockDatumRowIterator ins_iter;
ASSERT_EQ(OB_SUCCESS, ins_iter.from_for_datum(ins_str));
ObArenaAllocator allocator;
share::schema::ObTableDMLParam table_dml_param(allocator);

2
mittest/simple_server/test_callbacks_with_reverse_order.cpp
View File

@ -36,7 +36,7 @@ int ObLSTabletService::insert_tablet_rows(
const int64_t row_count,
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
ObStoreRow *rows,
ObDatumRow *rows,
ObRowsInfo &rows_info)
{
int ret = OB_SUCCESS;

4
src/observer/table/ob_table_modify_executor.cpp
View File

@ -321,11 +321,11 @@ int ObTableApiModifyExecutor::get_next_conflict_rowkey(DASTaskIter &task_iter,
bool got_row = false;
while (OB_SUCC(ret) && !got_row) {
ObNewRow *dup_row = nullptr;
ObDatumRow *dup_row = nullptr;
ObChunkDatumStore::StoredRow *stored_row = nullptr;
ObDASWriteBuffer::DmlShadowRow ssr;
ObDASInsertOp *ins_op = static_cast<ObDASInsertOp*>(*task_iter);
ObNewRowIterator *conflict_result = ins_op->get_duplicated_result();
ObDatumRowIterator *conflict_result = ins_op->get_duplicated_result();
const ObDASInsCtDef *ins_ctdef = static_cast<const ObDASInsCtDef*>(ins_op->get_ctdef());
if (OB_ISNULL(conflict_result)) {
ret = OB_ERR_UNEXPECTED;

10
src/observer/table_load/ob_table_load_store_trans_px_writer.cpp
View File

@ -186,7 +186,7 @@ int ObTableLoadStoreTransPXWriter::check_status()
return ret;
}
int ObTableLoadStoreTransPXWriter::write(const ObNewRow &row)
int ObTableLoadStoreTransPXWriter::write(const blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -199,11 +199,13 @@ int ObTableLoadStoreTransPXWriter::write(const ObNewRow &row)
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", KR(ret), K(row), K(column_count_));
} else {
ObNewRow new_row;
ObDatumRow new_row;
if (is_heap_table_) {
new_row.assign(row.cells_ + 1, row.count_ - 1);
new_row.storage_datums_ = row.storage_datums_ + 1;
new_row.count_ = row.count_ - 1;
} else {
new_row.assign(row.cells_, row.count_);
new_row.storage_datums_ = row.storage_datums_;
new_row.count_ = row.count_;
}
if (OB_FAIL(writer_->px_write(tablet_id_, new_row))) {
LOG_WARN("fail to px write", KR(ret), K(row), K(new_row));

2
src/observer/table_load/ob_table_load_store_trans_px_writer.h
View File

@ -38,7 +38,7 @@ public:
ObTableLoadTransStoreWriter *writer);
int prepare_write(const common::ObTabletID &tablet_id,
const common::ObIArray<uint64_t> &column_ids);
int write(const common::ObNewRow &row);
int write(const blocksstable::ObDatumRow &row);
TO_STRING_KV(KP_(store_ctx),
KP_(trans),

25
src/observer/table_load/ob_table_load_trans_store.cpp
View File

@ -317,7 +317,7 @@ int ObTableLoadTransStoreWriter::write(int32_t session_id,
return ret;
}
int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const ObNewRow &row)
int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -330,22 +330,13 @@ int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const ObN
} else {
ObTableLoadSequenceNo seq_no(0); // pdml导入的行目前不存在主键冲突，先都用一个默认的seq_no
SessionContext &session_ctx = session_ctx_array_[0];
for (int64_t i = 0; OB_SUCC(ret) && i < table_data_desc_->column_count_; ++i) {
ObStorageDatum &datum = session_ctx.datum_row_.storage_datums_[i];
const ObObj &obj = row.cells_[i];
if (OB_FAIL(datum.from_obj_enhance(obj))) {
LOG_WARN("fail to from obj enhance", KR(ret), K(obj));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(write_row_to_table_store(session_ctx.table_store_,
tablet_id,
seq_no,
session_ctx.datum_row_))) {
LOG_WARN("fail to write row", KR(ret), K(tablet_id));
} else {
ATOMIC_AAF(&trans_ctx_->ctx_->job_stat_->store_.processed_rows_, 1);
}
if (OB_FAIL(write_row_to_table_store(session_ctx.table_store_,
tablet_id,
seq_no,
row))) {
LOG_WARN("fail to write row", KR(ret), K(tablet_id), K(row));
} else {
ATOMIC_AAF(&trans_ctx_->ctx_->job_stat_->store_.processed_rows_, 1);
}
}
return ret;

2
src/observer/table_load/ob_table_load_trans_store.h
View File

@ -74,7 +74,7 @@ public:
public:
// 只在对应工作线程中调用, 串行执行
int write(int32_t session_id, const table::ObTableLoadTabletObjRowArray &row_array);
int px_write(const ObTabletID &tablet_id, const common::ObNewRow &row);
int px_write(const ObTabletID &tablet_id, const blocksstable::ObDatumRow &row);
int flush(int32_t session_id);
int clean_up(int32_t session_id);
public:

10
src/share/datum/ob_datum.h
View File

@ -128,6 +128,7 @@ struct ObDatumPtr {
const char *inner_enumset_;
const ObLobCommon *lob_data_;
const ObLobLocator *lob_locator_;
const ObMemLobCommon *mem_lob_;
const ObObj *extend_obj_; // for extend type
const ObDecimalInt *decimal_int_;
};
@ -283,6 +284,10 @@ public:
return ObURowIDData(pack_, (const uint8_t *)ptr_);
}
inline const ObLobLocator &get_lob_locator() const { return *lob_locator_; }
inline void get_mem_lob(ObLobLocatorV2 &lob_locator) const
{
lob_locator.assign_ptr(mem_lob_, len_, has_lob_header());
}
inline const ObLobCommon &get_lob_data() const { return *lob_data_; }
inline const ObDecimalInt *get_decimal_int() const { return decimal_int_; }
@ -389,11 +394,6 @@ public:
ptr_ = reinterpret_cast<const char *>(urowid_data.rowid_content_);
pack_ = static_cast<uint32_t>(urowid_data.rowid_len_);
}
inline void set_urowid(const char *ptr, const int64_t size)
{
ptr_ = ptr;
pack_ = static_cast<uint32_t>(size);
}
inline void set_lob_locator(const ObLobLocator &value)
{
lob_locator_ = &value;

90
src/sql/das/ob_das_dml_ctx_define.cpp
View File

@ -19,6 +19,7 @@
#include "sql/engine/dml/ob_dml_service.h"
#include "sql/engine/expr/ob_expr_lob_utils.h"
#include "storage/access/ob_dml_param.h"
#include "storage/blocksstable/ob_datum_row_utils.h"
namespace oceanbase
{
namespace sql
@ -107,7 +108,7 @@ ObDASDMLIterator::~ObDASDMLIterator()
}
}
int ObDASDMLIterator::get_next_domain_index_row(ObNewRow *&row)
int ObDASDMLIterator::get_next_domain_index_row(ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(domain_iter_) && OB_FAIL(ObDomainDMLIterator::create_domain_dml_iterator(
@ -121,7 +122,7 @@ int ObDASDMLIterator::get_next_domain_index_row(ObNewRow *&row)
return ret;
}
int ObDASDMLIterator::get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_count)
int ObDASDMLIterator::get_next_domain_index_rows(ObDatumRow *&rows, int64_t &row_count)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(domain_iter_) && OB_FAIL(ObDomainDMLIterator::create_domain_dml_iterator(
@ -135,25 +136,25 @@ int ObDASDMLIterator::get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_c
return ret;
}
int ObDASDMLIterator::get_next_row(ObNewRow *&row)
int ObDASDMLIterator::get_next_row(blocksstable::ObDatumRow *&datum_row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(cur_row_)) {
if (OB_FAIL(ob_create_row(allocator_, row_projector_->count(), cur_row_))) {
LOG_WARN("create current row failed", K(ret), K(row_projector_));
if (OB_ISNULL(cur_datum_row_)) {
if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(allocator_, row_projector_->count(), cur_datum_row_))) {
LOG_WARN("create current datum row failed", K(ret), K(row_projector_));
} else if (OB_FAIL(write_buffer_.begin(write_iter_))) {
LOG_WARN("begin write iterator failed", K(ret));
}
}
if (OB_SUCC(ret) && das_ctdef_->table_param_.get_data_table().is_domain_index()) {
if (OB_FAIL(get_next_domain_index_row(row))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next domain index row", K(ret), K(das_ctdef_->table_param_.get_data_table()));
if (OB_SUCC(ret)) {
if (das_ctdef_->table_param_.get_data_table().is_domain_index()) {
if (OB_FAIL(get_next_domain_index_row(datum_row))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next domain index row", K(ret), K(das_ctdef_->table_param_.get_data_table()));
}
}
}
} else {
if (OB_SUCC(ret)) {
} else {
const ObChunkDatumStore::StoredRow *sr = nullptr;
if (OB_FAIL(write_iter_.get_next_row(sr))) {
if (OB_ITER_END != ret) {
@ -163,23 +164,18 @@ int ObDASDMLIterator::get_next_row(ObNewRow *&row)
*sr,
*row_projector_,
allocator_,
*cur_row_))) {
*cur_datum_row_))) {
LOG_WARN("project storage row failed", K(ret));
} else {
row = cur_row_;
LOG_TRACE("get next row from dml das iterator", KPC(sr), KPC(row), K(das_ctdef_));
datum_row = cur_datum_row_;
LOG_TRACE("get next row from dml das iterator", KPC(sr), KPC(datum_row), K(das_ctdef_));
}
}
}
return ret;
}
int ObDASDMLIterator::get_next_row()
{
return OB_NOT_IMPLEMENT;
}
int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count)
int ObDASDMLIterator::get_next_rows(blocksstable::ObDatumRow *&rows, int64_t &row_count)
{
int ret = OB_SUCCESS;
const bool is_domain_index = das_ctdef_->table_param_.get_data_table().is_domain_index();
@ -193,8 +189,8 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count)
row_count = 1;
}
} else {
if (OB_ISNULL(cur_rows_)) {
if (OB_FAIL(ob_create_rows(allocator_, batch_size_, row_projector_->count(), cur_rows_))) {
if (OB_ISNULL(cur_datum_rows_)) {
if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_rows(allocator_, batch_size_, row_projector_->count(), cur_datum_rows_))) {
LOG_WARN("Failed to create rows", K(ret), K_(row_projector));
} else if (OB_FAIL(write_buffer_.begin(write_iter_))) {
LOG_WARN("Failed to begin write iterator", K(ret));
@ -215,18 +211,18 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count)
*sr,
*row_projector_,
allocator_,
cur_rows_[row_count]))) {
cur_datum_rows_[row_count]))) {
LOG_WARN("Failed to project storage row", K(ret));
} else {
++row_count;
LOG_TRACE("Get next rows from dml das iterator", KPC(sr), K(cur_rows_[row_count - 1]), K_(das_ctdef));
LOG_TRACE("Get next rows from dml das iterator", KPC(sr), K(cur_datum_rows_[row_count - 1]), K_(das_ctdef));
}
}
if (OB_SUCC(ret) || OB_LIKELY(OB_ITER_END == ret)) {
if (0 == row_count) {
ret = OB_ITER_END;
} else {
rows = cur_rows_;
rows = cur_datum_rows_;
ret = OB_SUCCESS;
}
}
@ -238,8 +234,8 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count)
int ObDASDMLIterator::rewind(const ObDASDMLBaseCtDef *das_ctdef)
{
int ret = common::OB_SUCCESS;
cur_row_ = nullptr;
cur_rows_ = nullptr;
cur_datum_row_ = nullptr;
cur_datum_rows_ = nullptr;
set_ctdef(das_ctdef);
if (OB_NOT_NULL(domain_iter_)) {
if (OB_FAIL(domain_iter_->rewind())) {
@ -269,7 +265,7 @@ void ObDASDMLIterator::set_ctdef(const ObDASDMLBaseCtDef *das_ctdef)
}
}
int ObDASMLogDMLIterator::get_next_row(ObNewRow *&row)
int ObDASMLogDMLIterator::get_next_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(row_iter_)) {
@ -368,27 +364,29 @@ int ObDASWriteBuffer::DmlShadowRow::shadow_copy(const ObIArray<ObExpr*> &exprs,
return ret;
}
int ObDASWriteBuffer::DmlShadowRow::shadow_copy(const ObNewRow &row)
int ObDASWriteBuffer::DmlShadowRow::shadow_copy(const blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(store_row_) || OB_UNLIKELY(store_row_->cnt_ != row.get_count())) {
if (OB_ISNULL(store_row_) || OB_UNLIKELY(store_row_->cnt_ != row.get_column_count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL datums or count mismatch", K(ret), KPC(store_row_), K(row));
} else {
ObDatum *cells = store_row_->cells();
for (int64_t i = 0; OB_SUCC(ret) && i < row.get_count(); ++i) {
if (lib::is_oracle_mode() && row.get_cell(i).is_lob_locator() && strip_lob_locator_) {
ObObjDatumMapType map_type = OBJ_DATUM_MAPPING_MAX;
for (int64_t i = 0; OB_SUCC(ret) && i < row.get_column_count(); ++i) {
if (lib::is_oracle_mode() && column_types_ != nullptr && column_types_->at(i).is_lob_locator() && strip_lob_locator_) {
ObString payload;
if (column_types_ != nullptr && !column_types_->at(i).is_lob()) {
if (!column_types_->at(i).is_lob()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid column type", K(ret), K(column_types_->at(i)));
} else if (OB_FAIL(row.get_cell(i).get_lob_locator()->get_payload(payload))) {
} else if (OB_FAIL(row.storage_datums_[i].get_lob_locator().get_payload(payload))) {
LOG_WARN("get lob payload failed", K(ret));
} else {
cells[i].set_string(payload);
}
} else if (OB_FAIL(cells[i].from_obj(row.get_cell(i)))) {
LOG_WARN("shadow copy obj failed", K(ret), K(i), K(row));
} else if (FALSE_IT(map_type = ObDatum::get_obj_datum_map_type(column_types_->at(i).get_type()))) {
} else if (OB_FAIL(cells[i].from_storage_datum(row.storage_datums_[i], map_type))) {
LOG_WARN("shadow copy storage datum failed", K(ret), K(i), K(row));
}
if (OB_SUCC(ret)) {
//add the data length of datum
@ -671,7 +669,7 @@ int ObDASWriteBuffer::begin(NewRowIterator &it, const ObIArray<ObObjMeta> &col_t
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ob_create_row(*das_alloc_, col_types.count(), it.cur_new_row_))) {
if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(*das_alloc_, col_types.count(), it.cur_new_row_))) {
LOG_WARN("create new row failed", K(ret));
} else {
it.col_types_ = &col_types;
@ -683,8 +681,8 @@ int ObDASWriteBuffer::begin(NewRowIterator &it, const ObIArray<ObObjMeta> &col_t
int ObDASWriteBuffer::dump_data(const ObDASDMLBaseCtDef &das_base_ctdef) const
{
int ret = OB_SUCCESS;
ObNewRow *new_row = NULL;
ObNewRow *old_row = NULL;
blocksstable::ObDatumRow *new_row = NULL;
blocksstable::ObDatumRow *old_row = NULL;
const ObChunkDatumStore::StoredRow *store_row = NULL;
int64_t rownum = 0;
ObArenaAllocator tmp_alloc;
@ -705,7 +703,7 @@ int ObDASWriteBuffer::dump_data(const ObDASDMLBaseCtDef &das_base_ctdef) const
if (!das_base_ctdef.old_row_projector_.empty()) {
// create old row
if (OB_ISNULL(old_row)
&& OB_FAIL(ob_create_row(tmp_alloc, das_base_ctdef.old_row_projector_.count(), old_row))) {
&& OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(tmp_alloc, das_base_ctdef.old_row_projector_.count(), old_row))) {
LOG_WARN("create old row buffer failed", K(ret), K(das_base_ctdef.old_row_projector_.count()));
} else if (OB_FAIL(ObDASUtils::project_storage_row(das_base_ctdef,
*store_row,
@ -720,7 +718,7 @@ int ObDASWriteBuffer::dump_data(const ObDASDMLBaseCtDef &das_base_ctdef) const
if (OB_SUCC(ret)) {
if (!das_base_ctdef.new_row_projector_.empty()) {
if (OB_ISNULL(new_row)
&& OB_FAIL(ob_create_row(tmp_alloc, das_base_ctdef.new_row_projector_.count(), new_row))) {
&& OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(tmp_alloc, das_base_ctdef.new_row_projector_.count(), new_row))) {
LOG_WARN("create new row buffer failed", K(ret), K(das_base_ctdef.new_row_projector_.count()));
} else if (OB_FAIL(ObDASUtils::project_storage_row(das_base_ctdef,
*store_row,
@ -905,7 +903,7 @@ int ObDASWriteBuffer::Iterator::get_next_row_skip_const(ObEvalCtx &ctx, const Ob
return ret;
}
int ObDASWriteBuffer::NewRowIterator::get_next_row(ObNewRow *&row)
int ObDASWriteBuffer::NewRowIterator::get_next_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
const DmlRow *sr = nullptr;
@ -924,9 +922,7 @@ int ObDASWriteBuffer::NewRowIterator::get_next_row(ObNewRow *&row)
if (OB_SUCC(ret) && sr != nullptr) {
for (int64_t i = 0; OB_SUCC(ret) && i < sr->cnt_; ++i) {
if (OB_FAIL(sr->cells()[i].to_obj(cur_new_row_->cells_[i], col_types_->at(i)))) {
LOG_WARN("convert datum to ObNewRow failed", K(ret));
}
cur_new_row_->storage_datums_[i].shallow_copy_from_datum(sr->cells()[i]);
}
if (OB_SUCC(ret)) {
row = cur_new_row_;

40
src/sql/das/ob_das_dml_ctx_define.h
View File

@ -22,6 +22,7 @@
#include "sql/engine/ob_operator.h"
#include "sql/resolver/dml/ob_hint.h"
#include "storage/fts/ob_fts_plugin_helper.h"
#include "storage/blocksstable/ob_datum_row_iterator.h"
namespace oceanbase
{
namespace storage
@ -33,7 +34,8 @@ namespace sql
typedef common::ObFixedArray<common::ObObjMeta, common::ObIAllocator> ObjMetaFixedArray;
typedef common::ObFixedArray<common::ObAccuracy, common::ObIAllocator> AccuracyFixedArray;
static const int64_t SAPTIAL_INDEX_DEFAULT_ROW_COUNT = 32; // 一个wkb生成的cellid数量（设定值）
typedef common::ObSEArray<common::ObNewRow, SAPTIAL_INDEX_DEFAULT_ROW_COUNT> ObDomainIndexRow;
static const int64_t SAPTIAL_INDEX_DEFAULT_COL_COUNT = 2;
typedef common::ObSEArray<blocksstable::ObDatumRow*, SAPTIAL_INDEX_DEFAULT_ROW_COUNT> ObDomainIndexRow;
class ObDomainDMLIterator;
struct ObDASDMLBaseRtDef;
@ -286,7 +288,7 @@ public:
const common::ObIArray<common::ObObjMeta> &col_types,
bool strip_lob_locator);
virtual int shadow_copy(const common::ObIArray<ObExpr*> &exprs, ObEvalCtx &ctx) override;
int shadow_copy(const common::ObNewRow &row);
int shadow_copy(const blocksstable::ObDatumRow &row);
// reset && release referenced memory
virtual void reset()
{
@ -355,13 +357,13 @@ public:
{ }
~NewRowIterator() { }
int get_next_row(ObNewRow *&row);
int get_next_row(blocksstable::ObDatumRow *&row);
inline bool is_inited() { return nullptr != col_types_; }
private:
DmlRow *cur_row_;
ObChunkDatumStore::Iterator *datum_iter_;
const common::ObIArray<ObObjMeta> *col_types_;
ObNewRow *cur_new_row_;
blocksstable::ObDatumRow *cur_new_row_;
};
OB_UNIS_VERSION(1);
@ -459,7 +461,7 @@ private:
uint32_t row_extend_size_;
};
class ObDASDMLIterator : public common::ObNewRowIterator
class ObDASDMLIterator : public blocksstable::ObDatumRowIterator
{
public:
static const int64_t DEFAULT_BATCH_SIZE = 256;
@ -471,8 +473,8 @@ public:
das_ctdef_(das_ctdef),
row_projector_(nullptr),
allocator_(alloc),
cur_row_(nullptr),
cur_rows_(nullptr),
cur_datum_row_(nullptr),
cur_datum_rows_(nullptr),
main_ctdef_(das_ctdef),
domain_iter_(nullptr)
{
@ -480,37 +482,37 @@ public:
batch_size_ = MIN(write_buffer_.get_row_cnt(), DEFAULT_BATCH_SIZE);
}
virtual ~ObDASDMLIterator();
virtual int get_next_row(common::ObNewRow *&row) override;
virtual int get_next_row() override;
virtual int get_next_rows(ObNewRow *&rows, int64_t &row_count);
virtual int get_next_row(blocksstable::ObDatumRow *&datum_row) override;
virtual int get_next_rows(blocksstable::ObDatumRow *&rows, int64_t &row_count);
ObDASWriteBuffer &get_write_buffer() { return write_buffer_; }
virtual void reset() override { }
int rewind(const ObDASDMLBaseCtDef *das_ctdef);
private:
void set_ctdef(const ObDASDMLBaseCtDef *das_ctdef);
int get_next_domain_index_row(ObNewRow *&row);
int get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_count);
int get_next_domain_index_row(blocksstable::ObDatumRow *&row);
int get_next_domain_index_rows(blocksstable::ObDatumRow *&rows, int64_t &row_count);
private:
ObDASWriteBuffer &write_buffer_;
const ObDASDMLBaseCtDef *das_ctdef_;
const IntFixedArray *row_projector_;
ObDASWriteBuffer::Iterator write_iter_;
common::ObIAllocator &allocator_;
common::ObNewRow *cur_row_;
common::ObNewRow *cur_rows_;
blocksstable::ObDatumRow *cur_datum_row_;
blocksstable::ObDatumRow *cur_datum_rows_;
const ObDASDMLBaseCtDef *main_ctdef_;
ObDomainDMLIterator *domain_iter_;
int64_t batch_size_;
};
class ObDASMLogDMLIterator : public ObNewRowIterator
class ObDASMLogDMLIterator : public blocksstable::ObDatumRowIterator
{
public:
// support get next datum row
ObDASMLogDMLIterator(
const ObTabletID &tablet_id,
const storage::ObDMLBaseParam &dml_param,
ObNewRowIterator *iter,
ObDatumRowIterator *iter,
ObDASOpType op_type)
: tablet_id_(tablet_id),
dml_param_(dml_param),
@ -524,14 +526,12 @@ public:
}
}
virtual ~ObDASMLogDMLIterator() {}
virtual int get_next_row(ObNewRow *&row) override;
virtual int get_next_row() override { return OB_NOT_IMPLEMENT; }
virtual void reset() override {}
virtual int get_next_row(blocksstable::ObDatumRow *&datum_row) override;
private:
const ObTabletID &tablet_id_;
const storage::ObDMLBaseParam &dml_param_;
ObNewRowIterator *row_iter_;
ObDatumRowIterator *row_iter_;
ObDASOpType op_type_;
bool is_old_row_;
};

199
src/sql/das/ob_das_domain_utils.cpp
View File

@ -19,6 +19,7 @@
#include "sql/das/ob_das_utils.h"
#include "sql/engine/expr/ob_expr_lob_utils.h"
#include "observer/omt/ob_tenant_srs.h"
#include "storage/blocksstable/ob_datum_row_utils.h"
using namespace oceanbase::common;
@ -59,10 +60,11 @@ int ObDASDomainUtils::generate_spatial_index_rows(
} else {
ObS2Adapter s2object(&allocator, srid != 0 ? srs_item->is_geographical_srs() : false);
ObSpatialMBR spa_mbr;
ObObj *obj_arr = NULL;
ObS2Cellids cellids;
char *mbr = NULL;
char *mbr = nullptr;
int64_t mbr_len = 0;
void *rows_buf = nullptr;
blocksstable::ObDatumRow *rows = nullptr;
if (OB_FAIL(s2object.init(wkb_str, srs_bound))) {
LOG_WARN("Init s2object failed", K(ret));
} else if (OB_FAIL(s2object.get_cellids(cellids, false))) {
@ -81,38 +83,35 @@ int ObDASDomainUtils::generate_spatial_index_rows(
LOG_WARN("failed to alloc memory for spatial index row mbr", K(ret));
} else if (OB_FAIL(spa_mbr.to_char(mbr, mbr_len))) {
LOG_WARN("failed transform ObSpatialMBR to string", K(ret));
} else if (OB_ISNULL(rows_buf = reinterpret_cast<char *>(allocator.alloc(cellids.size() * sizeof(blocksstable::ObDatumRow))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory for spatial index rows buffer", K(ret));
} else {
rows = new (rows_buf) blocksstable::ObDatumRow[cellids.size()];
int64_t cellid_col_idx = 0;
int64_t mbr_col_idx = 1;
for (uint64_t i = 0; OB_SUCC(ret) && i < cellids.size(); i++) {
if (OB_ISNULL(obj_arr = reinterpret_cast<ObObj *>(allocator.alloc(sizeof(ObObj) * rowkey_num)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory for spatial index row cells", K(ret));
if (OB_FAIL(rows[i].init(allocator, rowkey_num))) {
LOG_WARN("init datum row failed", K(ret), K(rowkey_num));
} else {
// 索引行[cellid_obj][mbr_obj][rowkey_obj]
for(uint64_t j = 0; OB_SUCC(ret) && j < rowkey_num; j++) {
obj_arr[j].set_nop_value();
const ObObjMeta &col_type = das_ctdef.column_types_.at(j);
const ObAccuracy &col_accuracy = das_ctdef.column_accuracys_.at(j);
int64_t projector_idx = row_projector.at(j);
if (OB_FAIL(dml_row.cells()[projector_idx].to_obj(obj_arr[j], col_type))) {
LOG_WARN("stored row to new row obj failed", K(ret),
K(dml_row.cells()[projector_idx]), K(col_type), K(projector_idx), K(j));
} else if (OB_FAIL(ObDASUtils::reshape_storage_value(col_type, col_accuracy, allocator, obj_arr[j]))) {
if (FALSE_IT(rows[i].storage_datums_[j].shallow_copy_from_datum(dml_row.cells()[projector_idx]))) {
} else if (rows[i].storage_datums_[j].is_null()) {
} else if (OB_FAIL(ObDASUtils::reshape_datum_value(col_type, col_accuracy, true, allocator, rows[i].storage_datums_[j]))) {
LOG_WARN("reshape storage value failed", K(ret), K(col_type), K(projector_idx), K(j));
}
}
if (OB_SUCC(ret)) {
int64_t cellid_col_idx = 0;
int64_t mbr_col_idx = 1;
obj_arr[cellid_col_idx].set_uint64(cellids.at(i));
rows[i].storage_datums_[cellid_col_idx].set_uint(cellids.at(i));
ObString mbr_val(mbr_len, mbr);
obj_arr[mbr_col_idx].set_varchar(mbr_val);
obj_arr[mbr_col_idx].set_collation_type(CS_TYPE_BINARY);
obj_arr[mbr_col_idx].set_collation_level(CS_LEVEL_IMPLICIT);
ObNewRow row;
row.cells_ = obj_arr;
row.count_ = rowkey_num;
if (OB_FAIL(spat_rows.push_back(row))) {
LOG_WARN("failed to push back spatial index row", K(ret), K(row));
rows[i].storage_datums_[mbr_col_idx].set_string(mbr_val);
// not set_collation_type(CS_TYPE_BINARY) and set_collation_level(CS_LEVEL_IMPLICIT)
if (OB_FAIL(spat_rows.push_back(&rows[i]))) {
LOG_WARN("failed to push back spatial index row", K(ret), K(rows[i]));
}
}
}
@ -137,7 +136,8 @@ int ObDASDomainUtils::generate_spatial_index_rows(
const int64_t ft_word_bkt_cnt = MAX(fulltext.length() / 10, 2);
int64_t doc_length = 0;
ObFTWordMap ft_word_map;
ObObj *obj_arr = nullptr;
void *rows_buf = nullptr;
blocksstable::ObDatumRow *rows = nullptr;
if (OB_ISNULL(helper)
|| OB_UNLIKELY(!ft_obj_meta.is_valid())
|| OB_UNLIKELY(doc_id.length() != sizeof(ObDocId) || doc_id.empty()) ) {
@ -153,47 +153,41 @@ int ObDASDomainUtils::generate_spatial_index_rows(
K(ft_obj_meta.get_collation_type()), K(fulltext));
} else if (0 == ft_word_map.size()) {
ret = OB_ITER_END;
} else {
const int64_t obj_cnt = ft_word_map.size() * FT_WORD_DOC_COL_CNT;
const int64_t obj_arr_size = sizeof(ObObj) * obj_cnt;
if (OB_ISNULL(obj_arr = reinterpret_cast<ObObj *>(allocator.alloc(obj_arr_size)))) {
} else if (OB_ISNULL(rows_buf = reinterpret_cast<char *>(allocator.alloc(ft_word_map.size() * sizeof(blocksstable::ObDatumRow))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate obj array", K(ret), K(obj_arr_size));
} else {
for (int64_t i = 0; i < obj_cnt; ++i) {
new (obj_arr + i) ObObj();
}
}
LOG_WARN("failed to alloc memory for full text index rows buffer", K(ret));
} else {
int64_t i = 0;
rows = new (rows_buf) blocksstable::ObDatumRow[ft_word_map.size()];
for (ObFTWordMap::const_iterator iter = ft_word_map.begin();
OB_SUCC(ret) && iter != ft_word_map.end();
++iter) {
const ObFTWord &ft_word = iter->first;
const int64_t word_cnt = iter->second;
// index row format
// - FTS_INDEX: [WORD], [DOC_ID], [WORD_COUNT]
// - FTS_DOC_WORD: [DOC_ID], [WORD], [WORD_COUNT]
const int64_t word_idx = is_fts_index_aux ? 0 : 1;
const int64_t doc_id_idx = is_fts_index_aux ? 1 : 0;
const int64_t word_cnt_idx = 2;
const int64_t doc_len_idx = 3;
obj_arr[i * FT_WORD_DOC_COL_CNT + word_idx].set_varchar(ft_word.get_word());
obj_arr[i * FT_WORD_DOC_COL_CNT + word_idx].set_meta_type(ft_obj_meta);
obj_arr[i * FT_WORD_DOC_COL_CNT + doc_id_idx].set_varbinary(doc_id);
obj_arr[i * FT_WORD_DOC_COL_CNT + word_cnt_idx].set_uint64(word_cnt);
obj_arr[i * FT_WORD_DOC_COL_CNT + doc_len_idx].set_uint64(doc_length);
ObNewRow row;
row.cells_ = &(obj_arr[i * FT_WORD_DOC_COL_CNT]);
row.count_ = FT_WORD_DOC_COL_CNT;
if (OB_FAIL(word_rows.push_back(row))) {
LOG_WARN("fail to push back row", K(ret), K(row));
if (OB_FAIL(rows[i].init(allocator, FT_WORD_DOC_COL_CNT))) {
LOG_WARN("init datum row failed", K(ret), K(FT_WORD_DOC_COL_CNT));
} else {
ObDocId tmp_doc_id;
tmp_doc_id.tablet_id_ = ((const uint64_t *)doc_id.ptr())[0];
tmp_doc_id.seq_id_ = ((const uint64_t *)doc_id.ptr())[1];
STORAGE_FTS_LOG(DEBUG, "succeed to add word row", K(ret), K(is_fts_index_aux), "doc_id", tmp_doc_id,
K(ft_word), K(word_cnt), K(i), K(row));
++i;
const ObFTWord &ft_word = iter->first;
const int64_t word_cnt = iter->second;
// index row format
// - FTS_INDEX: [WORD], [DOC_ID], [WORD_COUNT]
// - FTS_DOC_WORD: [DOC_ID], [WORD], [WORD_COUNT]
const int64_t word_idx = is_fts_index_aux ? 0 : 1;
const int64_t doc_id_idx = is_fts_index_aux ? 1 : 0;
const int64_t word_cnt_idx = 2;
const int64_t doc_len_idx = 3;
rows[i].storage_datums_[word_idx].set_string(ft_word.get_word());
rows[i].storage_datums_[doc_id_idx].set_string(doc_id);
rows[i].storage_datums_[word_cnt_idx].set_uint(word_cnt);
rows[i].storage_datums_[doc_len_idx].set_uint(doc_length);
if (OB_FAIL(word_rows.push_back(&rows[i]))) {
LOG_WARN("fail to push back row", K(ret), K(rows[i]));
} else {
ObDocId tmp_doc_id;
tmp_doc_id.tablet_id_ = ((const uint64_t *)doc_id.ptr())[0];
tmp_doc_id.seq_id_ = ((const uint64_t *)doc_id.ptr())[1];
STORAGE_FTS_LOG(DEBUG, "succeed to add word row", K(ret), K(is_fts_index_aux), "doc_id", tmp_doc_id,
K(ft_word), K(word_cnt), K(i), K(rows[i]));
++i;
}
}
}
}
@ -332,65 +326,66 @@ int ObDASDomainUtils::generate_multivalue_index_rows(ObIAllocator &allocator,
uint64_t rowkey_column_start = column_num - 1 - data_table_rowkey_cnt;
uint64_t rowkey_column_end = column_num - 1;
void *rows_buf = nullptr;
if (OB_FAIL(get_pure_mutivalue_data(json_str, data, data_len, record_num))) {
LOG_WARN("failed to parse binary.", K(ret), K(json_str));
} else if (record_num == 0 && (is_unique_index && rowkey_column_start == 1)) {
} else if (OB_FAIL(calc_save_rowkey_policy(allocator, das_ctdef, row_projector,
dml_row, record_num, is_save_rowkey))) {
LOG_WARN("failed to calc store policy.", K(ret), K(data_table_rowkey_cnt));
} else if (OB_ISNULL(rows_buf = reinterpret_cast<char *>(allocator.alloc(record_num * sizeof(blocksstable::ObDatumRow))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory for multi value index rows buffer", K(ret));
} else {
ObObj *obj_arr = nullptr;
int64_t pos = sizeof(uint32_t);
blocksstable::ObDatumRow *rows = new (rows_buf) blocksstable::ObDatumRow[record_num];
ObObj obj;
for (int i = 0; OB_SUCC(ret) && (i < record_num || !is_none_unique_done) ; ++i) {
if (OB_ISNULL(obj_arr = reinterpret_cast<ObObj *>(allocator.alloc(sizeof(ObObj) * column_num)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory for multivalue index row cells", K(ret));
}
for(uint64_t j = 0; OB_SUCC(ret) && j < column_num; j++) {
obj_arr[j].set_nop_value();
ObObjMeta col_type = das_ctdef.column_types_.at(j);
const ObAccuracy &col_accuracy = das_ctdef.column_accuracys_.at(j);
int64_t projector_idx = row_projector.at(j);
if (OB_FAIL(rows[i].init(allocator, column_num))) {
LOG_WARN("init datum row failed", K(ret), K(column_num));
} else {
for(uint64_t j = 0; OB_SUCC(ret) && j < column_num; j++) {
ObObjMeta col_type = das_ctdef.column_types_.at(j);
const ObAccuracy &col_accuracy = das_ctdef.column_accuracys_.at(j);
int64_t projector_idx = row_projector.at(j);
if (multivalue_idx == projector_idx) {
if (OB_FAIL(obj_arr[j].deserialize(data, data_len, pos))) {
LOG_WARN("failed to deserialize datum.", K(ret), K(json_str));
} else {
if (ob_is_number_or_decimal_int_tc(col_type.get_type()) || ob_is_temporal_type(col_type.get_type())) {
col_type.set_collation_level(CS_LEVEL_NUMERIC);
if (multivalue_idx == projector_idx) {
// TODO: change obj to datum when do deserialize@xuanxi
obj.set_nop_value();
if (OB_FAIL(obj.deserialize(data, data_len, pos))) {
LOG_WARN("failed to deserialize datum", K(ret), K(json_str));
} else if (OB_FAIL(rows[i].storage_datums_[j].from_obj_enhance(obj))) {
LOG_WARN("failed to convert datum from obj", K(ret), K(obj));
} else {
col_type.set_collation_level(CS_LEVEL_IMPLICIT);
if (ob_is_number_or_decimal_int_tc(col_type.get_type()) || ob_is_temporal_type(col_type.get_type())) {
col_type.set_collation_level(CS_LEVEL_NUMERIC);
} else {
col_type.set_collation_level(CS_LEVEL_IMPLICIT);
}
is_none_unique_done = true;
}
obj_arr[j].set_collation_level(col_type.get_collation_level());
obj_arr[j].set_collation_type(col_type.get_collation_type());
obj_arr[j].set_type(col_type.get_type());
is_none_unique_done = true;
} else if (!is_save_rowkey && (rowkey_column_start >= j && j < rowkey_column_end)) {
rows[i].storage_datums_[j].set_null();
} else if (multivalue_arr_idx == projector_idx) {
rows[i].storage_datums_[j].set_null();
} else {
rows[i].storage_datums_[j].shallow_copy_from_datum(dml_row.cells()[projector_idx]);
}
if (OB_SUCC(ret) && OB_FAIL(ObDASUtils::reshape_datum_value(col_type, col_accuracy, true, allocator, rows[i].storage_datums_[j]))) {
LOG_WARN("reshape storage value failed", K(ret), K(col_type), K(projector_idx), K(j));
}
} else if (!is_save_rowkey && (rowkey_column_start >= j && j < rowkey_column_end)) {
obj_arr[j].set_null();
} else if (multivalue_arr_idx == projector_idx) {
obj_arr[j].set_null();
} else if (OB_FAIL(dml_row.cells()[projector_idx].to_obj(obj_arr[j], col_type))) {
LOG_WARN("stored row to new row obj failed", K(ret),
K(dml_row.cells()[projector_idx]), K(col_type), K(projector_idx), K(j));
}
if (OB_SUCC(ret) && OB_FAIL(ObDASUtils::reshape_storage_value(col_type, col_accuracy, allocator, obj_arr[j]))) {
LOG_WARN("reshape storage value failed", K(ret), K(col_type), K(projector_idx), K(j));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(mvi_rows.push_back(&rows[i]))) {
LOG_WARN("failed to push back spatial index row", K(ret), K(rows[i]));
}
} // end if (OB_SUCC(ret))
}
if (OB_SUCC(ret)) {
ObNewRow row;
row.cells_ = obj_arr;
row.count_ = column_num;
if (OB_FAIL(mvi_rows.push_back(row))) {
LOG_WARN("failed to push back spatial index row", K(ret), K(row));
}
} // end if (OB_SUCC(ret))
}
}
@ -501,7 +496,7 @@ bool ObDomainDMLIterator::is_same_domain_type(const ObDASDMLBaseCtDef *das_ctdef
return is_same_domain_type;
}
int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row)
int ObDomainDMLIterator::get_next_domain_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
const ObChunkDatumStore::StoredRow *sr = nullptr;
@ -529,7 +524,7 @@ int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row)
}
}
if (OB_SUCC(ret) && row_idx_ < rows_.count()) {
row = &(rows_[row_idx_]);
row = rows_[row_idx_];
++row_idx_;
got_row = true;
}
@ -538,7 +533,7 @@ int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row)
return ret;
}
int ObDomainDMLIterator::get_next_domain_rows(ObNewRow *&row, int64_t &row_count)
int ObDomainDMLIterator::get_next_domain_rows(blocksstable::ObDatumRow *&row, int64_t &row_count)
{
int ret = OB_SUCCESS;
const ObChunkDatumStore::StoredRow *sr = nullptr;
@ -572,7 +567,7 @@ int ObDomainDMLIterator::get_next_domain_rows(ObNewRow *&row, int64_t &row_count
}
}
if (OB_SUCC(ret) && row_idx_ < rows_.count()) {
row = &(rows_[row_idx_]);
row = rows_[row_idx_];
row_count = rows_.count() - row_idx_;
row_idx_ = rows_.count();
got_row = true;

6
src/sql/das/ob_das_domain_utils.h
View File

@ -84,7 +84,7 @@ class ObDomainDMLIterator
{
public:
static const int64_t DEFAULT_DOMAIN_ROW_COUNT = 32;
typedef common::ObSEArray<common::ObNewRow, DEFAULT_DOMAIN_ROW_COUNT> ObDomainIndexRow;
typedef common::ObSEArray<blocksstable::ObDatumRow*, DEFAULT_DOMAIN_ROW_COUNT> ObDomainIndexRow;
static int create_domain_dml_iterator(
common::ObIAllocator &allocator,
@ -104,8 +104,8 @@ public:
}
void set_ctdef(const ObDASDMLBaseCtDef *das_ctdef, const IntFixedArray *row_projector);
void set_row_projector(const IntFixedArray *row_projector) { row_projector_ = row_projector; }
int get_next_domain_row(ObNewRow *&row);
int get_next_domain_rows(ObNewRow *&row, int64_t &row_count);
int get_next_domain_row(blocksstable::ObDatumRow *&row);
int get_next_domain_rows(blocksstable::ObDatumRow *&row, int64_t &row_count);
bool is_same_domain_type(const ObDASDMLBaseCtDef *das_ctdef) const;
TO_STRING_KV(K_(row_idx), K_(rows), KPC_(row_projector), KPC_(das_ctdef), K_(main_ctdef));

39
src/sql/das/ob_das_insert_op.cpp
View File

@ -149,7 +149,7 @@ int ObDASInsertOp::insert_rows()
}
int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param,
ObAccessService *as,
ObNewRowIterator &dml_iter,
ObDatumRowIterator &dml_iter,
ObDASConflictIterator *result_iter,
const ObDASInsCtDef *ins_ctdef,
ObDASInsRtDef *ins_rtdef,
@ -158,7 +158,7 @@ int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param,
common::ObTabletID tablet_id)
{
int ret = OB_SUCCESS;
ObNewRow *insert_row = NULL;
blocksstable::ObDatumRow *insert_row = NULL;
int64_t affected_rows = 0;
if (OB_FAIL(ObDMLService::init_dml_param(*ins_ctdef,
*ins_rtdef,
@ -170,7 +170,7 @@ int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param,
LOG_WARN("init index dml param failed", K(ret), KPC(ins_ctdef), KPC(ins_rtdef));
}
while (OB_SUCC(ret) && OB_SUCC(dml_iter.get_next_row(insert_row))) {
ObNewRowIterator *duplicated_rows = NULL;
blocksstable::ObDatumRowIterator *duplicated_rows = NULL;
if (OB_ISNULL(insert_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("insert_row is null", K(ret));
@ -297,7 +297,7 @@ int ObDASInsertOp::insert_row_with_fetch()
LOG_WARN("rewind dml iter failed", K(ret));
} else {
ObDASMLogDMLIterator mlog_iter(index_tablet_id, dml_param, &dml_iter, DAS_OP_TABLE_INSERT);
ObNewRowIterator *new_iter = nullptr;
ObDatumRowIterator *new_iter = nullptr;
if (index_ins_ctdef->table_param_.get_data_table().is_mlog_table()
&& !index_ins_ctdef->is_access_mlog_as_master_table_) {
new_iter = &mlog_iter;
@ -331,7 +331,7 @@ int ObDASInsertOp::store_conflict_row(ObDASInsertResult &ins_result)
{
int ret = OB_SUCCESS;
bool added = false;
ObNewRow *dup_row = nullptr;
ObDatumRow *dup_row = nullptr;
ObDASWriteBuffer &result_buffer = ins_result.get_result_buffer();
ObDASWriteBuffer::DmlShadowRow ssr;
if (OB_ISNULL(result_)) {
@ -483,10 +483,10 @@ ObDASInsertResult::~ObDASInsertResult()
{
}
int ObDASInsertResult::get_next_row(ObNewRow *&row)
int ObDASInsertResult::get_next_row(ObDatumRow *&row)
{
int ret = OB_SUCCESS;
ObNewRow *result_row = NULL;
ObDatumRow *result_row = NULL;
if (OB_FAIL(result_newrow_iter_.get_next_row(result_row))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from result iter failed", K(ret));
@ -497,19 +497,6 @@ int ObDASInsertResult::get_next_row(ObNewRow *&row)
return ret;
}
int ObDASInsertResult::get_next_rows(int64_t &count, int64_t capacity)
{
UNUSED(count);
UNUSED(capacity);
return OB_NOT_IMPLEMENT;
}
int ObDASInsertResult::get_next_row()
{
int ret = OB_NOT_IMPLEMENT;
return ret;
}
int ObDASInsertResult::link_extra_result(ObDASExtraData &extra_result)
{
UNUSED(extra_result);
@ -585,11 +572,11 @@ void ObDASConflictIterator::reset()
duplicated_iter_list_.reset();
}
int ObDASConflictIterator::get_next_row(common::ObNewRow *&row)
int ObDASConflictIterator::get_next_row(ObDatumRow *&row)
{
int ret = OB_SUCCESS;
bool find_next_iter = false;
ObNewRow *dup_row = NULL;
ObDatumRow *dup_row = NULL;
do {
if (find_next_iter) {
++curr_iter_;
@ -599,7 +586,7 @@ int ObDASConflictIterator::get_next_row(common::ObNewRow *&row)
ret = OB_ITER_END;
LOG_DEBUG("fetch conflict row iterator end");
} else {
ObNewRowIterator *dup_row_iter = *curr_iter_;
blocksstable::ObDatumRowIterator *dup_row_iter = *curr_iter_;
if (OB_ISNULL(dup_row_iter)) {
find_next_iter = true;
} else if (OB_FAIL(dup_row_iter->get_next_row(dup_row))) {
@ -624,11 +611,5 @@ int ObDASConflictIterator::get_next_row(common::ObNewRow *&row)
return ret;
}
int ObDASConflictIterator::get_next_row()
{
int ret = OB_NOT_IMPLEMENT;
return ret;
}
} // namespace sql
} // namespace oceanbase

23
src/sql/das/ob_das_insert_op.h
View File

@ -22,8 +22,8 @@ namespace sql
{
class ObDASInsertResult;
typedef common::ObList<ObNewRowIterator *, common::ObIAllocator> ObDuplicatedIterList;
class ObDASConflictIterator : public common::ObNewRowIterator
typedef common::ObList<blocksstable::ObDatumRowIterator *, common::ObIAllocator> ObDuplicatedIterList;
class ObDASConflictIterator : public blocksstable::ObDatumRowIterator
{
public:
ObDASConflictIterator(const ObjMetaFixedArray &output_types,
@ -36,9 +36,8 @@ public:
~ObDASConflictIterator() {};
virtual int get_next_row(common::ObNewRow *&row) override;
virtual int get_next_row() override;
virtual void reset() override;
void reset();
virtual int get_next_row(blocksstable::ObDatumRow *&row) override;
void init_curr_iter()
{ curr_iter_ = duplicated_iter_list_.begin(); }
@ -78,7 +77,7 @@ public:
return insert_buffer_.dump_data(*ins_ctdef_);
}
common::ObNewRowIterator *get_duplicated_result()
blocksstable::ObDatumRowIterator *get_duplicated_result()
{ return result_; }
INHERIT_TO_STRING_KV("parent", ObIDASTaskOp,
@ -93,7 +92,7 @@ private:
int insert_index_with_fetch(ObDMLBaseParam &dml_param,
ObAccessService *as,
common::ObNewRowIterator &dml_iter,
blocksstable::ObDatumRowIterator &dml_iter,
ObDASConflictIterator *result_iter,
const ObDASInsCtDef *ins_ctdef,
ObDASInsRtDef *ins_rtdef,
@ -105,12 +104,12 @@ private:
const ObDASInsCtDef *ins_ctdef_;
ObDASInsRtDef *ins_rtdef_;
ObDASWriteBuffer insert_buffer_;
common::ObNewRowIterator *result_;
blocksstable::ObDatumRowIterator *result_;
int64_t affected_rows_; // local execute result, no need to serialize
bool is_duplicated_; // local execute result, no need to serialize
};
class ObDASInsertResult : public ObIDASTaskResult, public common::ObNewRowIterator
class ObDASInsertResult : public ObIDASTaskResult, public blocksstable::ObDatumRowIterator
{
OB_UNIS_VERSION_V(1);
public:
@ -118,10 +117,8 @@ public:
virtual ~ObDASInsertResult();
virtual int init(const ObIDASTaskOp &op, common::ObIAllocator &alloc) override;
virtual int reuse() override;
virtual int get_next_row(ObNewRow *&row) override;
virtual int get_next_row() override;
virtual int get_next_rows(int64_t &count, int64_t capacity) override;
virtual void reset() override;
virtual int get_next_row(blocksstable::ObDatumRow *&row) override;
virtual void reset();
virtual int link_extra_result(ObDASExtraData &extra_result) override;
int init_result_newrow_iter(const ObjMetaFixedArray *output_types);
ObDASWriteBuffer &get_result_buffer() { return result_buffer_; }

21
src/sql/das/ob_das_update_op.cpp
View File

@ -18,6 +18,7 @@
#include "sql/das/ob_das_utils.h"
#include "sql/das/ob_das_domain_utils.h"
#include "storage/tx_storage/ob_access_service.h"
#include "storage/blocksstable/ob_datum_row_utils.h"
#include "sql/engine/expr/ob_expr_lob_utils.h"
namespace oceanbase
{
@ -43,7 +44,7 @@ using namespace share;
namespace sql
{
class ObDASUpdIterator : public ObNewRowIterator
class ObDASUpdIterator : public blocksstable::ObDatumRowIterator
{
public:
ObDASUpdIterator(const ObDASUpdCtDef *das_ctdef,
@ -60,10 +61,8 @@ public:
{
}
virtual ~ObDASUpdIterator();
virtual int get_next_row(ObNewRow *&row) override;
virtual int get_next_row() override { return OB_NOT_IMPLEMENT; }
virtual int get_next_row(blocksstable::ObDatumRow *&row) override;
ObDASWriteBuffer &get_write_buffer() { return write_buffer_; }
virtual void reset() override { }
int rewind(const ObDASDMLBaseCtDef *das_ctdef)
{
int ret = common::OB_SUCCESS;
@ -92,12 +91,12 @@ public:
}
private:
// domain index
int get_next_domain_index_row(ObNewRow *&row);
int get_next_domain_index_row(blocksstable::ObDatumRow *&row);
private:
const ObDASUpdCtDef *das_ctdef_;
ObDASWriteBuffer &write_buffer_;
ObNewRow *old_row_;
ObNewRow *new_row_;
blocksstable::ObDatumRow *old_row_;
blocksstable::ObDatumRow *new_row_;
ObDASWriteBuffer::Iterator result_iter_;
ObDomainDMLIterator *domain_iter_;
bool got_old_row_;
@ -105,7 +104,7 @@ private:
common::ObIAllocator &allocator_;
};
int ObDASUpdIterator::get_next_row(ObNewRow *&row)
int ObDASUpdIterator::get_next_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
const ObChunkDatumStore::StoredRow *sr = NULL;
@ -119,7 +118,7 @@ int ObDASUpdIterator::get_next_row(ObNewRow *&row)
} else if (!got_old_row_) {
got_old_row_ = true;
if (OB_ISNULL(old_row_)) {
if (OB_FAIL(ob_create_row(allocator_, das_ctdef_->old_row_projector_.count(), old_row_))) {
if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(allocator_, das_ctdef_->old_row_projector_.count(), old_row_))) {
LOG_WARN("create row buffer failed", K(ret), K(das_ctdef_->old_row_projector_.count()));
} else if (OB_FAIL(write_buffer_.begin(result_iter_))) {
LOG_WARN("begin datum result iterator failed", K(ret));
@ -127,7 +126,7 @@ int ObDASUpdIterator::get_next_row(ObNewRow *&row)
}
if (OB_SUCC(ret) && OB_ISNULL(new_row_)) {
if (OB_FAIL(ob_create_row(allocator_, das_ctdef_->new_row_projector_.count(), new_row_))) {
if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(allocator_, das_ctdef_->new_row_projector_.count(), new_row_))) {
LOG_WARN("create row buffer failed", K(ret), K(das_ctdef_->new_row_projector_.count()));
}
}
@ -182,7 +181,7 @@ ObDASUpdIterator::~ObDASUpdIterator()
}
}
int ObDASUpdIterator::get_next_domain_index_row(ObNewRow *&row)
int ObDASUpdIterator::get_next_domain_index_row(ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (!iter_has_built_) {

47
src/sql/das/ob_das_utils.cpp
View File

@ -23,6 +23,7 @@
#include "observer/omt/ob_tenant_srs.h"
#include "share/ob_tablet_autoincrement_service.h"
#include "storage/access/ob_dml_param.h"
#include "storage/blocksstable/ob_datum_row_utils.h"
namespace oceanbase
{
using namespace common;
@ -203,7 +204,7 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef,
const ObDASWriteBuffer::DmlRow &dml_row,
const IntFixedArray &row_projector,
ObIAllocator &allocator,
ObNewRow &storage_row)
blocksstable::ObDatumRow &storage_row)
{
int ret = OB_SUCCESS;
for (int64_t i = 0; OB_SUCC(ret) && i < row_projector.count(); ++i) {
@ -212,14 +213,17 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef,
const ObAccuracy &col_accuracy = dml_ctdef.column_accuracys_.at(i);
if (projector_idx < 0) {
//this column is not touched by query, only need to be marked as nop
storage_row.cells_[i].set_nop_value();
} else if (OB_FAIL(dml_row.cells()[projector_idx].to_obj(storage_row.cells_[i], col_type))) {
LOG_WARN("stored row to new row obj failed", K(ret),
K(dml_row.cells()[projector_idx]), K(col_type), K(projector_idx), K(i));
} else if (OB_FAIL(reshape_storage_value(col_type, col_accuracy, allocator, storage_row.cells_[i]))) {
storage_row.storage_datums_[i].set_nop();
} else if (FALSE_IT(storage_row.storage_datums_[i].shallow_copy_from_datum(dml_row.cells()[projector_idx]))) {
} else if (storage_row.storage_datums_[i].is_null()) {
//nothing to do
} else if (OB_FAIL(reshape_datum_value(col_type, col_accuracy, true, allocator, storage_row.storage_datums_[i]))) {
LOG_WARN("reshape storage value failed", K(ret));
} else if (col_type.is_lob_storage() && col_type.has_lob_header()) {
storage_row.storage_datums_[i].set_has_lob_header();
}
}
//to project shadow rowkey with unique index
if (OB_SUCC(ret) && dml_ctdef.spk_cnt_) {
bool need_shadow_columns = false;
@ -229,7 +233,7 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef,
//need to fill shadow pk with the real pk value
bool rowkey_has_null = false;
for (int64_t i = 0; !rowkey_has_null && i < index_key_cnt; i++) {
rowkey_has_null = storage_row.cells_[i].is_null();
rowkey_has_null = storage_row.storage_datums_[i].is_null();
}
need_shadow_columns = rowkey_has_null;
} else {
@ -237,14 +241,14 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef,
//need to fill shadow pk with the real pk value
bool is_rowkey_all_null = true;
for (int64_t i = 0; is_rowkey_all_null && i < index_key_cnt; i++) {
is_rowkey_all_null = storage_row.cells_[i].is_null();
is_rowkey_all_null = storage_row.storage_datums_[i].is_null();
}
need_shadow_columns = is_rowkey_all_null;
}
if (!need_shadow_columns) {
for (int64_t i = 0; i < dml_ctdef.spk_cnt_; ++i) {
int64_t spk_idx = index_key_cnt + i;
storage_row.cells_[spk_idx].set_null();
storage_row.storage_datums_[spk_idx].set_null();
}
}
}
@ -398,7 +402,7 @@ int ObDASUtils::find_child_das_def(const ObDASBaseCtDef *root_ctdef,
int ObDASUtils::generate_mlog_row(const ObTabletID &tablet_id,
const storage::ObDMLBaseParam &dml_param,
ObNewRow &row,
blocksstable::ObDatumRow &row,
ObDASOpType op_type,
bool is_old_row)
{
@ -432,23 +436,26 @@ int ObDASUtils::generate_mlog_row(const ObTabletID &tablet_id,
}
}
row.cells_[sequence_col].set_int(ObObjType::ObIntType, static_cast<int64_t>(autoinc_seq));
row.storage_datums_[sequence_col].reuse();
row.storage_datums_[dmltype_col].reuse();
row.storage_datums_[old_new_col].reuse();
row.storage_datums_[sequence_col].set_int(static_cast<int64_t>(autoinc_seq));
if (sql::DAS_OP_TABLE_DELETE == op_type) {
row.cells_[dmltype_col].set_varchar("D");
row.cells_[old_new_col].set_varchar("O");
row.storage_datums_[dmltype_col].set_string(ObString("D"));
row.storage_datums_[old_new_col].set_string(ObString("O"));
} else if (sql::DAS_OP_TABLE_UPDATE == op_type) {
row.cells_[dmltype_col].set_varchar("U");
row.storage_datums_[dmltype_col].set_string(ObString("U"));
if (is_old_row) {
row.cells_[old_new_col].set_varchar("O");
row.storage_datums_[old_new_col].set_string(ObString("O"));
} else {
row.cells_[old_new_col].set_varchar("N");
row.storage_datums_[old_new_col].set_string(ObString("N"));
}
} else {
row.cells_[dmltype_col].set_varchar("I");
row.cells_[old_new_col].set_varchar("N");
row.storage_datums_[dmltype_col].set_string(ObString("I"));
row.storage_datums_[old_new_col].set_string(ObString("N"));
}
row.cells_[dmltype_col].set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI);
row.cells_[old_new_col].set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI);
// TODO: if we need to update col_type_ in col_descs with ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI ?@xuanxi
}
return ret;
}

4
src/sql/das/ob_das_utils.h
View File

@ -53,7 +53,7 @@ public:
const ObDASWriteBuffer::DmlRow &dml_row,
const IntFixedArray &row_projector,
common::ObIAllocator &allocator,
common::ObNewRow &storage_row);
blocksstable::ObDatumRow &storage_row);
static int reshape_storage_value(const common::ObObjMeta &col_type,
const common::ObAccuracy &col_accuracy,
common::ObIAllocator &allocator,
@ -93,7 +93,7 @@ public:
}
static int generate_mlog_row(const common::ObTabletID &tablet_id,
const storage::ObDMLBaseParam &dml_param,
common::ObNewRow &row,
blocksstable::ObDatumRow &row,
ObDASOpType op_type,
bool is_old_row);
};

4
src/sql/engine/dml/ob_table_insert_up_op.cpp
View File

@ -981,11 +981,11 @@ int ObTableInsertUpOp::get_next_conflict_rowkey(DASTaskIter &task_iter)
int ret = OB_SUCCESS;
bool got_row = false;
while (OB_SUCC(ret) && !got_row) {
ObNewRow *dup_row = nullptr;
ObDatumRow *dup_row = nullptr;
ObChunkDatumStore::StoredRow *stored_row = nullptr;
ObDASWriteBuffer::DmlShadowRow ssr;
ObDASInsertOp *ins_op = static_cast<ObDASInsertOp*>(*task_iter);
ObNewRowIterator *conflict_result = ins_op->get_duplicated_result();
ObDatumRowIterator *conflict_result = ins_op->get_duplicated_result();
const ObDASInsCtDef *ins_ctdef = static_cast<const ObDASInsCtDef*>(ins_op->get_ctdef());
// 因为返回的都是主表的主键，主表的主键一定是在存储层有储存的，是不需要再收起来层再做运算的，
// 所以这里不需要clear eval flag

4
src/sql/engine/dml/ob_table_replace_op.cpp
View File

@ -413,11 +413,11 @@ int ObTableReplaceOp::get_next_conflict_rowkey(DASTaskIter &task_iter)
int ret = OB_SUCCESS;
bool got_row = false;
while (OB_SUCC(ret) && !got_row) {
ObNewRow *dup_row = nullptr;
ObDatumRow *dup_row = nullptr;
ObChunkDatumStore::StoredRow *stored_row = nullptr;
ObDASWriteBuffer::DmlShadowRow ssr;
ObDASInsertOp *ins_op = static_cast<ObDASInsertOp*>(*task_iter);
ObNewRowIterator *conflict_result = ins_op->get_duplicated_result();
ObDatumRowIterator *conflict_result = ins_op->get_duplicated_result();
const ObDASInsCtDef *ins_ctdef = static_cast<const ObDASInsCtDef*>(ins_op->get_ctdef());
// 因为返回的都是主表的主键，主表的主键一定是在存储层有储存的，是不需要再收起来层再做运算的，
// 所以这里不需要clear eval flag

2
src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp
View File

@ -292,7 +292,7 @@ int ObExprPLGetCursorAttr::calc_pl_get_cursor_attr(
} else if (OB_UNLIKELY(rowid.empty())) {
expr_datum.set_null();
} else {
expr_datum.set_urowid(rowid.ptr(), rowid.length());
expr_datum.set_string(rowid.ptr(), rowid.length());
}
}
break;

55
src/sql/engine/table/ob_table_scan_op.cpp
View File

@ -3135,25 +3135,17 @@ int ObTableScanOp::inner_get_next_spatial_index_row()
} else if (cellids.size() > SAPTIAL_INDEX_DEFAULT_ROW_COUNT) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("cellid over size", K(ret), K(cellids.size()));
} else if (OB_ISNULL(spat_index_.obj_buffer_)) {
} else if (OB_ISNULL(spat_index_.rows_)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory for spatial index row cells", K(ret));
LOG_WARN("failed to alloc memory for spatial index datum row", K(ret));
} else {
ObObj *obj_arr = reinterpret_cast<ObObj *>(spat_index_.obj_buffer_);
uint64_t obj_idx = 0;
for (uint64_t i = 0; OB_SUCC(ret) && i < cellids.size(); i++) {
obj_arr[obj_idx].set_nop_value();
obj_arr[obj_idx].set_uint64(cellids.at(i));
obj_arr[obj_idx + 1].set_nop_value();
obj_arr[obj_idx + 1].set_varchar(mbr_val);
obj_arr[obj_idx + 1].set_collation_type(CS_TYPE_BINARY);
obj_arr[obj_idx + 1].set_collation_level(CS_LEVEL_IMPLICIT);
ObNewRow row;
row.cells_ = &obj_arr[obj_idx];
row.count_ = 2;
obj_idx += 2;
if (OB_FAIL(spat_index_.spat_rows_->push_back(row))) {
LOG_WARN("failed to push back spatial index row", K(ret), K(row));
for (uint64_t i = 0, datum_idx = 0; OB_SUCC(ret) && i < cellids.size(); i++) {
spat_index_.rows_[i].reuse();
spat_index_.rows_[i].storage_datums_[datum_idx].set_uint(cellids.at(i));
spat_index_.rows_[i].storage_datums_[datum_idx + 1].set_string(mbr_val);
// not set_collation_type(CS_TYPE_BINARY) and set_collation_level(CS_LEVEL_IMPLICIT)
if (OB_FAIL(spat_index_.spat_rows_->push_back(spat_index_.rows_ + i))) {
LOG_WARN("failed to push back spatial index row", K(ret), K(spat_index_.rows_[i]));
}
}
}
@ -3162,10 +3154,10 @@ int ObTableScanOp::inner_get_next_spatial_index_row()
}
}
}
if (OB_SUCC(ret) && !need_ignore_null) {
ObNewRow &row = (*(spat_index_.spat_rows_))[spat_index_.spat_row_index_++];
ObObj &cellid= row.get_cell(0);
ObObj &mbr = row.get_cell(1);
if (OB_SUCC(ret)) {
ObDatumRow *row = (*(spat_index_.spat_rows_))[spat_index_.spat_row_index_++];
ObStorageDatum &cellid = row->storage_datums_[0];
ObStorageDatum &mbr = row->storage_datums_[1];
if (OB_FAIL(fill_generated_cellid_mbr(cellid, mbr))) {
LOG_WARN("fill cellid mbr failed", K(ret), K(cellid), K(mbr));
}
@ -3178,19 +3170,24 @@ int ObTableScanOp::init_spatial_index_rows()
{
int ret = OB_SUCCESS;
void *buf = ctx_.get_allocator().alloc(sizeof(ObDomainIndexRow));
void *row_buf = ctx_.get_allocator().alloc(sizeof(blocksstable::ObDatumRow) * SAPTIAL_INDEX_DEFAULT_ROW_COUNT);
void *mbr_buffer = ctx_.get_allocator().alloc(OB_DEFAULT_MBR_SIZE);
void *obj_buf = ctx_.get_allocator().alloc(sizeof(ObObj) * 2 * SAPTIAL_INDEX_DEFAULT_ROW_COUNT);
if (OB_ISNULL(buf) || OB_ISNULL(mbr_buffer) || OB_ISNULL(obj_buf)) {
if (OB_ISNULL(buf) || OB_ISNULL(mbr_buffer) || OB_ISNULL(row_buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate spatial row store failed", K(ret), K(buf), K(mbr_buffer));
} else {
spat_index_.spat_rows_ = new(buf) ObDomainIndexRow();
spat_index_.rows_ = new(row_buf) blocksstable::ObDatumRow[SAPTIAL_INDEX_DEFAULT_ROW_COUNT];
spat_index_.mbr_buffer_ = mbr_buffer;
spat_index_.obj_buffer_ = obj_buf;
const ObExprPtrIArray &exprs = MY_SPEC.output_;
const uint8_t spatial_expr_cnt = 3;
uint8_t cnt = 0;
for (uint32_t i = 0; i < exprs.count() && cnt < spatial_expr_cnt; i++) {
for (uint32_t i = 0; OB_SUCC(ret) && i < SAPTIAL_INDEX_DEFAULT_ROW_COUNT; i++) {
if (OB_FAIL(spat_index_.rows_[i].init(SAPTIAL_INDEX_DEFAULT_COL_COUNT))) {
LOG_WARN("init datum row failed", K(ret));
}
}
for (uint32_t i = 0; OB_SUCC(ret) && i < exprs.count() && cnt < spatial_expr_cnt; i++) {
if (exprs.at(i)->type_ == T_FUN_SYS_SPATIAL_CELLID) {
spat_index_.cell_idx_ = i;
cnt++;
@ -3202,7 +3199,7 @@ int ObTableScanOp::init_spatial_index_rows()
cnt++;
}
}
if (cnt != spatial_expr_cnt) {
if (OB_FAIL(ret) || cnt != spatial_expr_cnt) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid spatial index exprs", K(ret), K(cnt));
}
@ -3210,7 +3207,7 @@ int ObTableScanOp::init_spatial_index_rows()
return ret;
}
int ObTableScanOp::fill_generated_cellid_mbr(const ObObj &cellid, const ObObj &mbr)
int ObTableScanOp::fill_generated_cellid_mbr(const ObStorageDatum &cellid, const ObStorageDatum &mbr)
{
int ret = OB_SUCCESS;
const ObExprPtrIArray &exprs = MY_SPEC.output_;
@ -3220,12 +3217,12 @@ int ObTableScanOp::fill_generated_cellid_mbr(const ObObj &cellid, const ObObj &m
} else {
for (uint8_t i = 0; i < 2 && OB_SUCC(ret); i++) {
ObObjDatumMapType type = i == 0 ? OBJ_DATUM_8BYTE_DATA : OBJ_DATUM_STRING;
const ObObj &value = i == 0 ? cellid : mbr;
const ObStorageDatum &value = i == 0 ? cellid : mbr;
uint32_t idx = i == 0 ? spat_index_.cell_idx_ : spat_index_.mbr_idx_;
ObExpr *expr = exprs.at(idx);
ObDatum *datum = &expr->locate_datum_for_write(get_eval_ctx());
ObEvalInfo *eval_info = &expr->get_eval_info(get_eval_ctx());
if (OB_FAIL(datum->from_obj(value, type))) {
if (OB_FAIL(datum->from_storage_datum(value, type))) {
LOG_WARN("fill spatial index row failed", K(ret));
} else {
eval_info->evaluated_ = true;

6
src/sql/engine/table/ob_table_scan_op.h
View File

@ -71,18 +71,18 @@ struct ObSpatialIndexCache
public:
ObSpatialIndexCache() :
spat_rows_(nullptr),
rows_(nullptr),
spat_row_index_(0),
mbr_buffer_(nullptr),
obj_buffer_(nullptr),
geo_idx_(0),
cell_idx_(0),
mbr_idx_(0)
{}
~ObSpatialIndexCache() {};
ObDomainIndexRow *spat_rows_;
blocksstable::ObDatumRow *rows_;
uint8_t spat_row_index_;
void *mbr_buffer_;
void *obj_buffer_;
uint32_t geo_idx_;
uint32_t cell_idx_;
uint32_t mbr_idx_;
@ -509,7 +509,7 @@ protected:
ObTableScanStat &scan_stat) const;
void set_cache_stat(const ObPlanStat &plan_stat);
int inner_get_next_row_implement();
int fill_generated_cellid_mbr(const ObObj &cellid, const ObObj &mbr);
int fill_generated_cellid_mbr(const ObStorageDatum &cellid, const ObStorageDatum &mbr);
int inner_get_next_spatial_index_row();
int init_spatial_index_rows();
void set_real_rescan_cnt(int64_t real_rescan_cnt) { group_rescan_cnt_ = real_rescan_cnt; }

3
src/storage/CMakeLists.txt
View File

@ -43,8 +43,11 @@ ob_set_subtarget(ob_storage blocksstable
blocksstable/ob_sstable_printer.cpp
blocksstable/ob_storage_cache_suite.cpp
blocksstable/ob_super_block_buffer_holder.cpp
blocksstable/ob_storage_datum.cpp
blocksstable/ob_datum_row.cpp
blocksstable/ob_datum_rowkey.cpp
blocksstable/ob_datum_row_store.cpp
blocksstable/ob_datum_row_utils.cpp
blocksstable/ob_data_store_desc.cpp
blocksstable/ob_table_flag.cpp
blocksstable/ob_datum_rowkey_vector.cpp

14
src/storage/access/ob_rows_info.cpp
View File

@ -14,6 +14,7 @@
#include "storage/ob_storage_struct.h"
#include "storage/ob_relative_table.h"
#include "storage/ob_storage_schema.h"
#include "storage/blocksstable/ob_datum_row_utils.h"
#include "ob_store_row_iterator.h"
namespace oceanbase
@ -93,6 +94,7 @@ ObRowsInfo::ObRowsInfo()
tablet_id_(),
datum_utils_(nullptr),
min_key_(),
col_descs_(nullptr),
conflict_rowkey_idx_(-1),
error_code_(0),
delete_count_(0),
@ -124,9 +126,11 @@ void ObRowsInfo::reset()
error_code_ = 0;
conflict_rowkey_idx_ = -1;
is_inited_ = false;
col_descs_ = nullptr;
}
int ObRowsInfo::init(
const ObColDescIArray &column_descs,
const ObRelativeTable &table,
ObStoreCtx &store_ctx,
const ObITableReadInfo &rowkey_read_info)
@ -139,6 +143,7 @@ int ObRowsInfo::init(
} else if (OB_FAIL(exist_helper_.init(table, store_ctx, rowkey_read_info, exist_allocator_, scan_mem_allocator_))) {
STORAGE_LOG(WARN, "Failed to init exist helper", K(ret));
} else {
col_descs_ = &column_descs;
datum_utils_ = &rowkey_read_info.get_datum_utils();
tablet_id_ = table.get_tablet_id();
rowkey_column_num_ = table.get_rowkey_column_num();
@ -160,7 +165,7 @@ void ObRowsInfo::reuse()
}
//not only checking duplicate, but also assign rowkeys
int ObRowsInfo::check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRelativeTable &table)
int ObRowsInfo::check_duplicate(ObDatumRow *rows, const int64_t row_count, ObRelativeTable &table)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
@ -192,10 +197,9 @@ int ObRowsInfo::check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRel
} else {
ObMarkedRowkeyAndLockState marked_rowkey_and_lock_state;
marked_rowkey_and_lock_state.row_idx_ = i;
ObRowkey rowkey(rows_[i].row_val_.cells_, rowkey_column_num_);
if (OB_FAIL(marked_rowkey_and_lock_state.marked_rowkey_.get_rowkey().from_rowkey(rowkey,
key_allocator_))) {
STORAGE_LOG(WARN, "Failed to transfer rowkey", K(ret), K(rowkey));
ObDatumRowkey &datum_rowkey = marked_rowkey_and_lock_state.marked_rowkey_.get_rowkey();
if (OB_FAIL(blocksstable::ObDatumRowUtils::prepare_rowkey(rows[i], rowkey_column_num_, *col_descs_, key_allocator_, datum_rowkey))) {
STORAGE_LOG(WARN, "Failed to prepare rowkey", K(ret), K(rowkey_column_num_), K(rows_[i]));
} else if (OB_FAIL(rowkeys_.push_back(marked_rowkey_and_lock_state))) {
STORAGE_LOG(WARN, "Failed to push back rowkey", K(ret), K(marked_rowkey_and_lock_state));
}

6
src/storage/access/ob_rows_info.h
View File

@ -90,10 +90,11 @@ public:
return delete_count_ == rowkeys_.count();
}
int init(
const ObColDescIArray &column_descs,
const ObRelativeTable &table,
ObStoreCtx &store_ctx,
const ObITableReadInfo &rowkey_read_info);
int check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRelativeTable &table);
int check_duplicate(blocksstable::ObDatumRow *rows, const int64_t row_count, ObRelativeTable &table);
blocksstable::ObDatumRowkey& get_duplicate_rowkey()
{
return min_key_;
@ -265,12 +266,13 @@ private:
public:
ObRowkeyAndLockStates rowkeys_;
ObPermutation permutation_;
ObStoreRow *rows_;
blocksstable::ObDatumRow *rows_;
ExistHelper exist_helper_;
ObTabletID tablet_id_;
private:
const blocksstable::ObStorageDatumUtils *datum_utils_;
blocksstable::ObDatumRowkey min_key_;
const ObColDescIArray *col_descs_;
int64_t conflict_rowkey_idx_;
int error_code_;
int64_t delete_count_;

5
src/storage/access/ob_table_access_context.cpp
View File

@ -266,10 +266,12 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag,
}
return ret;
}
int ObTableAccessContext::init(const common::ObQueryFlag &query_flag,
ObStoreCtx &ctx,
common::ObIAllocator &allocator,
const common::ObVersionRange &trans_version_range)
const common::ObVersionRange &trans_version_range,
CachedIteratorNode *cached_iter_node)
{
int ret = OB_SUCCESS;
if (is_inited_) {
@ -288,6 +290,7 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag,
ls_id_ = ctx.ls_id_;
tablet_id_ = ctx.tablet_id_;
lob_locator_helper_ = nullptr;
cached_iter_node_ = cached_iter_node;
if (!micro_block_handle_mgr_.is_valid()
&& OB_FAIL(micro_block_handle_mgr_.init(enable_limit, table_store_stat_, query_flag_))) {
LOG_WARN("Fail to init micro block handle mgr", K(ret));

3
src/storage/access/ob_table_access_context.h
View File

@ -199,7 +199,8 @@ struct ObTableAccessContext
int init(const common::ObQueryFlag &query_flag,
ObStoreCtx &ctx,
common::ObIAllocator &allocator,
const common::ObVersionRange &trans_version_range);
const common::ObVersionRange &trans_version_range,
CachedIteratorNode *cached_iter_node = nullptr);
int alloc_iter_pool(const bool use_column_store);
void inc_micro_access_cnt();
int init_scan_allocator(ObTableScanParam &scan_param);

390
src/storage/blocksstable/ob_datum_row.cpp
View File

@ -22,73 +22,6 @@ namespace oceanbase
using namespace common;
namespace blocksstable
{
static int nonext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret)
{
int ret = cmp_func.cmp_func_(left, right, cmp_ret);
STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret));
return ret;
}
static int nonext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret)
{
int ret = OB_SUCCESS;
UNUSEDx(left, cmp_func);
if (right.is_max()) {
cmp_ret = -1;
} else if (right.is_min()) {
cmp_ret = 1;
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(right));
}
STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret));
return ret;
}
static int ext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret)
{
int ret = OB_SUCCESS;
UNUSEDx(right, cmp_func);
if (left.is_max()) {
cmp_ret = 1;
} else if (left.is_min()) {
cmp_ret = -1;
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left));
}
STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret));
return ret;
}
static int ext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret)
{
int ret = OB_SUCCESS;
UNUSEDx(cmp_func);
int64_t lv = left.is_max() - left.is_min();
int64_t rv = right.is_max() - right.is_min();
if (OB_UNLIKELY(0 == lv || 0 == rv)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left), K(right));
} else {
cmp_ret = lv - rv;
}
STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret));
return ret;
}
typedef int (*ExtSafeCompareFunc)(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret);
static ExtSafeCompareFunc ext_safe_cmp_funcs[2][2] = {
{nonext_nonext_compare, nonext_ext_compare},
{ext_nonext_compare, ext_ext_compare}
};
int ObStorageDatumCmpFunc::compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const
{
return ext_safe_cmp_funcs[left.is_ext()][right.is_ext()](left, right, cmp_func_, cmp_ret);
}
const char *get_dml_str(ObDmlFlag dml_flag)
{
@ -106,86 +39,6 @@ OB_SERIALIZE_MEMBER(ObDmlRowFlag, whole_flag_);
OB_SERIALIZE_MEMBER(ObMultiVersionRowFlag, flag_);
/*
*ObStorageDatumBuffer
*/
ObStorageDatumBuffer::ObStorageDatumBuffer(common::ObIAllocator *allocator)
: capacity_(LOCAL_BUFFER_ARRAY),
local_datums_(),
datums_(local_datums_),
allocator_(allocator),
is_inited_(nullptr != allocator)
{}
ObStorageDatumBuffer::~ObStorageDatumBuffer()
{
if (datums_ != local_datums_ && nullptr != allocator_) {
allocator_->free(datums_);
}
}
void ObStorageDatumBuffer::reset()
{
if (datums_ != local_datums_ && nullptr != allocator_) {
allocator_->free(datums_);
}
allocator_ = nullptr;
datums_ = local_datums_;
capacity_ = LOCAL_BUFFER_ARRAY;
for (int64_t i = 0; i < capacity_; i++) {
datums_[i].reuse();
}
is_inited_ = false;
}
int ObStorageDatumBuffer::init(common::ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObStorageDatumBuffer init twice", K(ret), K(*this));
} else {
OB_ASSERT(datums_ == local_datums_);
allocator_ = &allocator;
is_inited_ = true;
}
return ret;
}
int ObStorageDatumBuffer::reserve(const int64_t count, const bool keep_data)
{
int ret = OB_SUCCESS;
void *buf = nullptr;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObStorageDatumBuffer is not inited", K(ret), K(*this));
} else if (OB_UNLIKELY(count <= 0)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to reserve datum buffer", K(ret), K(count));
} else if (count <= capacity_){
} else if (OB_ISNULL(buf = allocator_->alloc(sizeof(ObStorageDatum) * count))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(count));
} else {
ObStorageDatum *new_datums = new (buf) ObStorageDatum [count];
if (keep_data) {
for (int64_t i = 0; i < capacity_; i++) {
new_datums[i] = datums_[i];
}
}
if (nullptr != datums_ && datums_ != local_datums_) {
allocator_->free(datums_);
}
datums_ = new_datums;
capacity_ = count;
}
return ret;
}
/*
*ObConstDatumRow
*/
@ -232,8 +85,7 @@ ObDatumRow::ObDatumRow(const uint64_t tenant_id)
}
ObDatumRow::~ObDatumRow()
{
}
{}
int ObDatumRow::init(ObIAllocator &allocator, const int64_t capacity, char *trans_info_ptr)
{
@ -256,7 +108,9 @@ int ObDatumRow::init(ObIAllocator &allocator, const int64_t capacity, char *tran
}
return ret;
}
int ObDatumRow::init(const int64_t capacity)
{
int ret = OB_SUCCESS;
@ -442,6 +296,27 @@ int ObDatumRow::is_datums_changed(const ObDatumRow &other, bool &is_changed) con
return ret;
}
int ObDatumRow::shallow_copy(const ObDatumRow &other)
{
int ret = OB_SUCCESS;
if (!other.is_valid()) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected error for shallow copy invalid datum row", K(ret), K(*this), K(other));
} else {
trans_info_ = nullptr;
fast_filter_skipped_ = other.fast_filter_skipped_;
storage_datums_ = other.storage_datums_;
snapshot_version_ = other.snapshot_version_;
group_idx_ = other.group_idx_;
scan_index_ = other.scan_index_;
trans_id_ = other.trans_id_;
mvcc_row_flag_ = other.mvcc_row_flag_;
row_flag_ = other.row_flag_;
count_ = other.count_;
}
return ret;
}
OB_DEF_SERIALIZE(ObDatumRow)
{
int ret = OB_SUCCESS;
@ -503,8 +378,8 @@ DEF_TO_STRING(ObDatumRow)
{
int64_t pos = 0;
J_OBJ_START();
J_KV(K_(row_flag), K_(trans_id), K_(scan_index), K_(mvcc_row_flag),
K_(snapshot_version), K_(fast_filter_skipped), K_(have_uncommited_row), K_(group_idx), K_(count), K_(datum_buffer));
J_KV(K_(row_flag), K_(trans_id), K_(scan_index), K_(mvcc_row_flag), K_(snapshot_version), K_(fast_filter_skipped),
K_(have_uncommited_row), K_(group_idx), K_(count), K_(datum_buffer));
if (NULL != buf && buf_len >= 0) {
if (NULL != storage_datums_) {
J_COMMA();
@ -615,219 +490,6 @@ int ObNewRowBuilder::build_store_row(
return ret;
}
/*
*ObStorageDatumUtils
*/
ObStorageDatumUtils::ObStorageDatumUtils()
: rowkey_cnt_(0),
cmp_funcs_(),
hash_funcs_(),
ext_hash_func_(),
is_oracle_mode_(false),
is_inited_(false)
{}
ObStorageDatumUtils::~ObStorageDatumUtils()
{}
int ObStorageDatumUtils::transform_multi_version_col_desc(const ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
ObIArray<share::schema::ObColDesc> &mv_col_descs)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(schema_rowkey_cnt > col_descs.count())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to transform mv col descs", K(ret), K(schema_rowkey_cnt), K(col_descs));
} else {
mv_col_descs.reuse();
for (int64_t i = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; i++) {
if (OB_FAIL(mv_col_descs.push_back(col_descs.at(i)))) {
STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(i));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(storage::ObMultiVersionRowkeyHelpper::add_extra_rowkey_cols(mv_col_descs))) {
STORAGE_LOG(WARN, "Fail to add extra_rowkey_cols", K(ret), K(schema_rowkey_cnt));
} else {
for (int64_t i = schema_rowkey_cnt; OB_SUCC(ret) && i < col_descs.count(); i++) {
const share::schema::ObColDesc &col_desc = col_descs.at(i);
if (col_desc.col_id_ == common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID
|| col_desc.col_id_ == common::OB_HIDDEN_SQL_SEQUENCE_COLUMN_ID) {
continue;
} else if (OB_FAIL(mv_col_descs.push_back(col_desc))) {
STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(col_desc));
}
}
}
}
return ret;
}
int ObStorageDatumUtils::init(const ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
const bool is_oracle_mode,
ObIAllocator &allocator,
const bool is_column_store)
{
int ret = OB_SUCCESS;
ObSEArray<share::schema::ObColDesc, 32> mv_col_descs;
int64_t mv_rowkey_cnt = 0;
int64_t mv_column_cnt = 0;
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this));
} else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER
|| schema_rowkey_cnt > col_descs.count())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(col_descs), K(schema_rowkey_cnt));
} else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) {
STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret));
} else if (FALSE_IT(mv_column_cnt = is_column_store ? 0 : storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) {
} else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + mv_column_cnt)) {
} else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, allocator))) {
STORAGE_LOG(WARN, "Failed to reserve cmp func array", K(ret));
} else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, allocator))) {
STORAGE_LOG(WARN, "Failed to reserve hash func array", K(ret));
} else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) {
STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt));
}
return ret;
}
int ObStorageDatumUtils::init(const common::ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
const bool is_oracle_mode,
const int64_t arr_buf_len,
char *arr_buf)
{
int ret = OB_SUCCESS;
ObSEArray<share::schema::ObColDesc, 32> mv_col_descs;
int64_t pos = 0;
int64_t mv_rowkey_cnt = 0;
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this));
} else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER
|| schema_rowkey_cnt > col_descs.count())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(col_descs), K(schema_rowkey_cnt));
} else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) {
STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret));
} else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) {
} else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) {
STORAGE_LOG(WARN, "Failed to init compare function array", K(ret));
} else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) {
STORAGE_LOG(WARN, "Failed to init hash function array", K(ret));
} else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) {
STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt));
}
return ret;
}
int ObStorageDatumUtils::inner_init(
const common::ObIArray<share::schema::ObColDesc> &mv_col_descs,
const int64_t mv_rowkey_col_cnt,
const bool is_oracle_mode)
{
int ret = OB_SUCCESS;
is_oracle_mode_ = is_oracle_mode;
// support column order index until next task done
//
// we could use the cmp funcs in the basic funcs directlly
bool is_null_last = is_oracle_mode_;
ObCmpFunc cmp_func;
ObHashFunc hash_func;
for (int64_t i = 0; OB_SUCC(ret) && i < mv_rowkey_col_cnt; i++) {
const share::schema::ObColDesc &col_desc = mv_col_descs.at(i);
//TODO @hanhui support desc rowkey
bool is_ascending = true || col_desc.col_order_ == ObOrderType::ASC;
bool has_lob_header = is_lob_storage(col_desc.col_type_.get_type());
ObPrecision precision = PRECISION_UNKNOWN_YET;
if (col_desc.col_type_.is_decimal_int()) {
precision = col_desc.col_type_.get_stored_precision();
OB_ASSERT(precision != PRECISION_UNKNOWN_YET);
}
sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(col_desc.col_type_.get_type(),
col_desc.col_type_.get_collation_type(),
col_desc.col_type_.get_scale(),
is_oracle_mode,
has_lob_header,
precision);
if (OB_UNLIKELY(nullptr == basic_funcs
|| nullptr == basic_funcs->null_last_cmp_
|| nullptr == basic_funcs->murmur_hash_)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected null basic funcs", K(ret), K(col_desc));
} else {
cmp_func.cmp_func_ = is_null_last ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_;
hash_func.hash_func_ = basic_funcs->murmur_hash_;
if (OB_FAIL(hash_funcs_.push_back(hash_func))) {
STORAGE_LOG(WARN, "Failed to push back hash func", K(ret), K(i), K(col_desc));
} else if (is_ascending) {
if (OB_FAIL(cmp_funcs_.push_back(ObStorageDatumCmpFunc(cmp_func)))) {
STORAGE_LOG(WARN, "Failed to push back cmp func", K(ret), K(i), K(col_desc));
}
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(WARN, "Unsupported desc column order", K(ret), K(col_desc), K(i));
}
}
}
if (OB_SUCC(ret)) {
sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(ObExtendType, CS_TYPE_BINARY);
if (OB_UNLIKELY(nullptr == basic_funcs || nullptr == basic_funcs->murmur_hash_)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected null basic funcs for extend type", K(ret));
} else {
ext_hash_func_.hash_func_ = basic_funcs->murmur_hash_;
rowkey_cnt_ = mv_rowkey_col_cnt;
is_inited_ = true;
}
}
return ret;
}
int ObStorageDatumUtils::assign(const ObStorageDatumUtils &other_utils, ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!other_utils.is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to assign datum utils", K(ret), K(other_utils));
} else {
rowkey_cnt_ = other_utils.get_rowkey_count();
is_oracle_mode_ = other_utils.is_oracle_mode();
ext_hash_func_ = other_utils.get_ext_hash_funcs();
if (OB_FAIL(cmp_funcs_.init_and_assign(other_utils.get_cmp_funcs(), allocator))) {
STORAGE_LOG(WARN, "Failed to assign cmp func array", K(ret));
} else if (OB_FAIL(hash_funcs_.init_and_assign(other_utils.get_hash_funcs(), allocator))) {
STORAGE_LOG(WARN, "Failed to assign hash func array", K(ret));
} else {
is_inited_ = true;
}
}
return ret;
}
void ObStorageDatumUtils::reset()
{
rowkey_cnt_ = 0;
cmp_funcs_.reset();
hash_funcs_.reset();
ext_hash_func_.hash_func_ = nullptr;
is_inited_ = false;
}
int64_t ObStorageDatumUtils::get_deep_copy_size() const
{
return cmp_funcs_.get_deep_copy_size() + hash_funcs_.get_deep_copy_size();
}
int ObGhostRowUtil::is_ghost_row(
const blocksstable::ObMultiVersionRowFlag &flag,
bool &is_ghost_row)

389
src/storage/blocksstable/ob_datum_row.h
View File

@ -21,6 +21,7 @@
#include "storage/tx/ob_trans_define.h"
#include "common/row/ob_row.h"
#include "storage/ob_storage_util.h"
#include "storage/blocksstable/ob_datum_rowkey.h"
namespace oceanbase
{
@ -38,6 +39,7 @@ namespace blocksstable
{
struct ObDmlRowFlag;
struct ObDatumRowkey;
enum ObDmlFlag
{
@ -307,65 +309,6 @@ public:
K_(flag));
};
//TODO optimize number buffer
struct ObStorageDatum : public common::ObDatum
{
ObStorageDatum() { set_nop(); }
ObStorageDatum(const ObStorageDatum &datum) { reuse(); *this = datum; }
~ObStorageDatum() = default;
// ext value section
OB_INLINE void reuse() { ptr_ = buf_; reserved_ = 0; pack_ = 0; }
OB_INLINE void set_ext_value(const int64_t ext_value)
{ reuse(); set_ext(); no_cv(extend_obj_)->set_ext(ext_value); }
OB_INLINE void set_nop() { set_ext_value(ObActionFlag::OP_NOP); }
OB_INLINE void set_min() { set_ext_value(common::ObObj::MIN_OBJECT_VALUE); }
OB_INLINE void set_max() { set_ext_value(common::ObObj::MAX_OBJECT_VALUE); }
OB_INLINE bool is_nop_value() const { return is_nop(); } // temp solution
// transfer section
OB_INLINE bool is_local_buf() const { return ptr_ == buf_; }
OB_INLINE int from_buf_enhance(const char *buf, const int64_t buf_len);
OB_INLINE int from_obj_enhance(const common::ObObj &obj);
OB_INLINE int to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const;
OB_INLINE int deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator);
OB_INLINE int deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos);
OB_INLINE void shallow_copy_from_datum(const ObDatum &src);
OB_INLINE int64_t get_deep_copy_size() const;
OB_INLINE ObStorageDatum& operator=(const ObStorageDatum &other);
OB_INLINE int64_t storage_to_string(char *buf, int64_t buf_len, const bool for_dump = false) const;
OB_INLINE bool need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const;
OB_INLINE const char *to_cstring(const bool for_dump = false) const;
//only for unittest
OB_INLINE bool operator==(const ObStorageDatum &other) const;
OB_INLINE bool operator==(const ObObj &other) const;
//datum 12 byte
int32_t reserved_;
// buf 16 byte
char buf_[common::OBJ_DATUM_NUMBER_RES_SIZE];
};
struct ObStorageDatumBuffer
{
public:
ObStorageDatumBuffer(common::ObIAllocator *allocator = nullptr);
~ObStorageDatumBuffer();
void reset();
int init(common::ObIAllocator &allocator);
int reserve(const int64_t count, const bool keep_data = false);
OB_INLINE bool is_valid() const { return is_inited_; }
OB_INLINE ObStorageDatum *get_datums() { return datums_; }
OB_INLINE int64_t get_capacity() const { return capacity_; }
TO_STRING_KV(K_(capacity), KP_(datums), KP_(local_datums));
private:
static const int64_t LOCAL_BUFFER_ARRAY = common::OB_ROW_DEFAULT_COLUMNS_COUNT;
int64_t capacity_;
ObStorageDatum local_datums_[LOCAL_BUFFER_ARRAY];
ObStorageDatum *datums_;
common::ObIAllocator *allocator_;
bool is_inited_;
};
struct ObDatumRow
{
OB_UNIS_VERSION(1);
@ -379,6 +322,7 @@ public:
int reserve(const int64_t capacity, const bool keep_data = false);
int deep_copy(const ObDatumRow &src, common::ObIAllocator &allocator);
int from_store_row(const storage::ObStoreRow &store_row);
int shallow_copy(const ObDatumRow &other);
//only for unittest
bool operator==(const ObDatumRow &other) const;
bool operator==(const common::ObNewRow &other) const;
@ -487,333 +431,6 @@ public:
int64_t datum_row_offset_;
};
struct ObStorageDatumCmpFunc
{
public:
ObStorageDatumCmpFunc(common::ObCmpFunc &cmp_func) : cmp_func_(cmp_func) {}
ObStorageDatumCmpFunc() = default;
~ObStorageDatumCmpFunc() = default;
int compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const;
OB_INLINE const common::ObCmpFunc &get_cmp_func() const { return cmp_func_; }
TO_STRING_KV(K_(cmp_func));
private:
common::ObCmpFunc cmp_func_;
};
typedef storage::ObFixedMetaObjArray<ObStorageDatumCmpFunc> ObStoreCmpFuncs;
typedef storage::ObFixedMetaObjArray<common::ObHashFunc> ObStoreHashFuncs;
struct ObStorageDatumUtils
{
public:
ObStorageDatumUtils();
~ObStorageDatumUtils();
// init with array memory from allocator
int init(const common::ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
const bool is_oracle_mode,
common::ObIAllocator &allocator,
const bool is_column_store = false);
// init with array memory on fixed size memory buffer
int init(const common::ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
const bool is_oracle_mode,
const int64_t arr_buf_len,
char *arr_buf);
int assign(const ObStorageDatumUtils &other_utils, common::ObIAllocator &allocator);
void reset();
OB_INLINE bool is_valid() const
{
return is_inited_ && cmp_funcs_.count() >= rowkey_cnt_ && hash_funcs_.count() >= rowkey_cnt_;
}
OB_INLINE bool is_oracle_mode() const { return is_oracle_mode_; }
OB_INLINE int64_t get_rowkey_count() const { return rowkey_cnt_; }
OB_INLINE const ObStoreCmpFuncs &get_cmp_funcs() const { return cmp_funcs_; }
OB_INLINE const ObStoreHashFuncs &get_hash_funcs() const { return hash_funcs_; }
OB_INLINE const common::ObHashFunc &get_ext_hash_funcs() const { return ext_hash_func_; }
int64_t get_deep_copy_size() const;
TO_STRING_KV(K_(is_oracle_mode), K_(rowkey_cnt), K_(is_inited), K_(is_oracle_mode));
private:
//TODO to be removed by @hanhui
int transform_multi_version_col_desc(const common::ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
common::ObIArray<share::schema::ObColDesc> &mv_col_descs);
int inner_init(
const common::ObIArray<share::schema::ObColDesc> &mv_col_descs,
const int64_t mv_rowkey_col_cnt,
const bool is_oracle_mode);
private:
int32_t rowkey_cnt_; // multi version rowkey
ObStoreCmpFuncs cmp_funcs_; // multi version rowkey cmp funcs
ObStoreHashFuncs hash_funcs_; // multi version rowkey cmp funcs
common::ObHashFunc ext_hash_func_;
bool is_oracle_mode_;
bool is_inited_;
DISALLOW_COPY_AND_ASSIGN(ObStorageDatumUtils);
};
OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator)
{
int ret = common::OB_SUCCESS;
reuse();
pack_ = src.pack_;
if (is_null()) {
} else if (src.len_ == 0) {
} else if (src.is_local_buf()) {
OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE);
MEMCPY(buf_, src.ptr_, src.len_);
ptr_ = buf_;
} else {
char * buf = static_cast<char *>(allocator.alloc(src.len_));
if (OB_ISNULL(buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
COMMON_LOG(WARN, "allocate memory failed", K(ret), K(src));
pack_ = 0;
} else {
MEMCPY(buf, src.ptr_, src.len_);
// need set ptr_ after memory copy, if this == &src
ptr_ = buf;
}
}
return ret;
}
OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos)
{
int ret = common::OB_SUCCESS;
reuse();
pack_ = src.pack_;
if (is_null()) {
} else if (src.len_ == 0) {
} else if (src.is_local_buf()) {
OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE);
MEMCPY(buf_, src.ptr_, src.len_);
ptr_ = buf_;
} else if (OB_UNLIKELY(nullptr == buf || buf_len < pos + src.len_)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to deep copy datum", K(ret), K(src), KP(buf), K(buf_len), K(pos));
pack_ = 0;
} else {
MEMCPY(buf + pos, src.ptr_, src.len_);
// need set ptr_ after memory copy, if this == &src
ptr_ = buf + pos;
pos += src.len_;
}
return ret;
}
OB_INLINE void ObStorageDatum::shallow_copy_from_datum(const ObDatum &src)
{
if (this != &src) {
reuse();
pack_ = src.pack_;
if (is_null()) {
} else if (src.len_ == 0) {
} else {
ptr_ = src.ptr_;
}
}
}
OB_INLINE int64_t ObStorageDatum::get_deep_copy_size() const
{
int64_t deep_copy_len = 0;
if (is_null()) {
} else if (is_local_buf()) {
OB_ASSERT(len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE);
} else {
deep_copy_len = len_;
}
return deep_copy_len;
}
OB_INLINE int ObStorageDatum::from_buf_enhance(const char *buf, const int64_t buf_len)
{
int ret = common::OB_SUCCESS;
if (OB_UNLIKELY(nullptr == buf || buf_len < 0 || buf_len > UINT32_MAX)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to transfer from buf", K(ret), KP(buf), K(buf_len));
} else {
reuse();
len_ = static_cast<uint32_t>(buf_len);
if (buf_len > 0) {
ptr_ = buf;
}
}
return ret;
}
OB_INLINE int ObStorageDatum::from_obj_enhance(const common::ObObj &obj)
{
int ret = common::OB_SUCCESS;
reuse();
if (obj.is_ext()) {
set_ext_value(obj.get_ext());
} else if (OB_FAIL(from_obj(obj))) {
STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(obj));
}
STORAGE_LOG(DEBUG, "chaser debug from obj", K(obj), K(*this));
return ret;
}
OB_INLINE int ObStorageDatum::to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const
{
int ret = common::OB_SUCCESS;
if (is_outrow()) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "lob should not set outrow in datum", K(ret), K(*this), K(obj), K(meta));
} else if (is_ext()) {
obj.set_ext(get_ext());
} else if (OB_FAIL(to_obj(obj, meta))) {
STORAGE_LOG(WARN, "Failed to transfer datum to obj", K(ret), K(*this), K(obj), K(meta));
}
return ret;
}
OB_INLINE ObStorageDatum& ObStorageDatum::operator=(const ObStorageDatum &other)
{
if (&other != this) {
reuse();
pack_ = other.pack_;
if (is_null()) {
} else if (len_ == 0) {
} else if (other.is_local_buf()) {
OB_ASSERT(other.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE);
MEMCPY(buf_, other.ptr_, other.len_);
ptr_ = buf_;
} else {
ptr_ = other.ptr_;
}
}
return *this;
}
OB_INLINE bool ObStorageDatum::operator==(const ObStorageDatum &other) const
{
bool bret = true;
if (is_null()) {
bret = other.is_null();
} else if (is_ext()) {
bret = other.is_ext() && extend_obj_->get_ext() == other.extend_obj_->get_ext();
} else {
bret = ObDatum::binary_equal(*this, other);
}
if (!bret) {
STORAGE_LOG(DEBUG, "obj and datum no equal", K(other), K(*this));
}
return bret;
}
OB_INLINE bool ObStorageDatum::operator==(const common::ObObj &other) const
{
int ret = OB_SUCCESS;
bool bret = true;
ObStorageDatum datum;
if (OB_FAIL(datum.from_obj_enhance(other))) {
STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(other), K(datum));
} else {
bret = *this == datum;
}
if (!bret) {
STORAGE_LOG(DEBUG, "obj and datum no equal", K(other), K(datum), KPC(this));
}
return bret;
}
OB_INLINE int64_t ObStorageDatum::storage_to_string(char *buf, int64_t buf_len, const bool for_dump) const
{
int64_t pos = 0;
if (is_ext()) {
if (is_nop()) {
J_NOP();
} else if (is_max()) {
BUF_PRINTF("MAX_OBJ");
} else if (is_min()) {
BUF_PRINTF("MIN_OBJ");
}
} else if(!for_dump) {
pos = to_string(buf, buf_len);
} else {
int ret = OB_SUCCESS;
const static int64_t STR_MAX_PRINT_LEN = 128L;
if (null_) {
J_NULL();
} else {
J_OBJ_START();
BUF_PRINTF("len: %d, flag: %d, null: %d", len_, flag_, null_);
if (len_ > 0) {
OB_ASSERT(NULL != ptr_);
const int64_t plen = std::min(static_cast<int64_t>(len_),
static_cast<int64_t>(STR_MAX_PRINT_LEN));
// print hex value
BUF_PRINTF(", hex: ");
if (OB_FAIL(hex_print(ptr_, plen, buf, buf_len, pos))) {
// no logging in to_string function.
} else {
// maybe ObIntTC
if (sizeof(int64_t) == len_) {
BUF_PRINTF(", int: %ld", *int_);
// maybe number with one digit
if (1 == num_->desc_.len_) {
BUF_PRINTF(", num_digit0: %u", num_->digits_[0]);
}
}
// maybe printable C string
int64_t idx = 0;
while (idx < plen && isprint(ptr_[idx])) {
idx++;
}
if (idx >= plen) {
BUF_PRINTF(", cstr: %.*s", static_cast<int>(plen), ptr_);
}
}
}
J_OBJ_END();
}
}
return pos;
}
OB_INLINE const char *ObStorageDatum::to_cstring(const bool for_dump) const
{
char *buffer = NULL;
int64_t str_len = 0;
CStringBufMgr &mgr = CStringBufMgr::get_thread_local_instance();
mgr.inc_level();
const int64_t buf_len = mgr.acquire(buffer);
if (OB_ISNULL(buffer)) {
LIB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "buffer is NULL");
} else {
str_len = storage_to_string(buffer, buf_len -1, for_dump);
if (str_len >= 0 && str_len < buf_len) {
buffer[str_len] = '\0';
} else {
buffer[0] = '\0';
}
mgr.update_position(str_len + 1);
}
mgr.try_clear_list();
mgr.dec_level();
return buffer;
}
OB_INLINE bool ObStorageDatum::need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const
{
return OBJ_DATUM_STRING == map_type && sizeof(uint64_t) == len_ && is_local_buf();
}
struct ObGhostRowUtil {
public:
ObGhostRowUtil() = delete;

98
src/storage/blocksstable/ob_datum_row_iterator.h Normal file
View File

@ -0,0 +1,98 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_DATUM_ROW_ITERATOR_
#define OCEANBASE_DATUM_ROW_ITERATOR_
#include "src/storage/blocksstable/ob_datum_row.h"
namespace oceanbase
{
namespace blocksstable
{
class ObDatumRowIterator
{
public:
typedef common::ObReserveArenaAllocator<1024> ObStorageReserveAllocator;
public:
ObDatumRowIterator() {}
virtual ~ObDatumRowIterator() {}
/**
* get the next datum row and move the cursor
*
* @param row [out]
*
* @return OB_ITER_END if end of iteration
*/
virtual int get_next_row(ObDatumRow *&row) = 0;
virtual int get_next_rows(ObDatumRow *&rows, int64_t &row_count)
{
int ret = OB_SUCCESS;
if (OB_FAIL(get_next_row(rows))) {
} else {
row_count = 1;
}
return ret;
}
virtual void reset() {}
TO_STRING_EMPTY();
};
/// wrap one datum row as an iterator
class ObSingleDatumRowIteratorWrapper: public ObDatumRowIterator
{
public:
ObSingleDatumRowIteratorWrapper();
ObSingleDatumRowIteratorWrapper(ObDatumRow *row);
virtual ~ObSingleDatumRowIteratorWrapper() {}
void set_row(ObDatumRow *row) { row_ = row; }
virtual int get_next_row(ObDatumRow *&row);
virtual void reset() { iter_end_ = false; }
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObSingleDatumRowIteratorWrapper);
private:
// data members
ObDatumRow *row_;
bool iter_end_;
};
inline ObSingleDatumRowIteratorWrapper::ObSingleDatumRowIteratorWrapper()
:row_(NULL),
iter_end_(false)
{}
inline ObSingleDatumRowIteratorWrapper::ObSingleDatumRowIteratorWrapper(ObDatumRow *row)
:row_(row),
iter_end_(false)
{}
inline int ObSingleDatumRowIteratorWrapper::get_next_row(ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(row_)) {
ret = OB_NOT_INIT;
} else if (iter_end_) {
ret = OB_ITER_END;
} else {
row = row_;
iter_end_ = true;
}
return ret;
}
}
}
#endif //OCEANBASE_DATUM_ROW_ITERATOR_

242
src/storage/blocksstable/ob_datum_row_store.cpp Normal file
View File

@ -0,0 +1,242 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX COMMON
#include "ob_datum_row_store.h"
#include "lib/utility/utility.h"
namespace oceanbase
{
namespace blocksstable
{
////////////////////////////////////////////////////////////////
struct ObDatumRowStore::BlockInfo
{
explicit BlockInfo(int64_t block_size)
:magic_(0xabcd4444abcd4444),
next_(NULL),
curr_data_pos_(0),
block_size_(block_size)
{
}
OB_INLINE int64_t get_remain_size() const { return block_size_ - curr_data_pos_; }
OB_INLINE int64_t get_remain_size_for_read(int64_t pos) const { return curr_data_pos_ - pos; }
OB_INLINE char *get_buffer() { return data_ + curr_data_pos_; }
OB_INLINE const char *get_buffer_head() const { return data_; }
OB_INLINE void advance(const int64_t length) { curr_data_pos_ += length; }
OB_INLINE BlockInfo *get_next_block() { return next_; }
OB_INLINE const BlockInfo *get_next_block() const { return next_; }
OB_INLINE int64_t get_block_size() const { return block_size_; };
int append_row(const ObDatumRow &row, const int64_t length);
friend class ObDatumRowStore::BlockList;
private:
#ifdef __clang__
int64_t magic_ [[gnu::unused]];
#else
int64_t magic_;
#endif
BlockInfo *next_;
/**
* cur_data_pos_ must be set when BlockInfo deserialized
*/
int64_t curr_data_pos_;
int64_t block_size_;
char data_[0];
};
int ObDatumRowStore::BlockInfo::append_row(const ObDatumRow &row, const int64_t length)
{
int ret = OB_SUCCESS;
int64_t pos = 0;
if (OB_UNLIKELY(0 > curr_data_pos_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "out of memory range",
K(ret), K_(block_size), K_(curr_data_pos), K(length));
} else if (OB_FAIL(row.serialize(get_buffer(), get_remain_size(), pos))) {
STORAGE_LOG(WARN, "fail to serialize datum row", K(ret), K(row));
} else {
advance(pos);
}
return ret;
}
////////////////////////////////////////////////////////////////
ObDatumRowStore::BlockList::BlockList()
:first_(NULL),
last_(NULL),
count_(0),
used_mem_size_(0)
{
};
void ObDatumRowStore::BlockList::reset()
{
first_ = NULL;
last_ = NULL;
count_ = 0;
used_mem_size_ = 0;
}
int ObDatumRowStore::BlockList::add_last(BlockInfo *block)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(block)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(block));
} else {
block->next_ = NULL;
if (OB_ISNULL(last_)) {
if (OB_ISNULL(first_)) {
first_ = block;
last_ = block;
} else {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "invalid block list", K(ret), K_(last), K_(first));
}
} else {
last_->next_ = block;
last_ = block;
}
}
if (OB_SUCC(ret)) {
used_mem_size_ += block->get_block_size();
++count_;
}
return ret;
}
////////////////////////////////////////////////////////////////
ObDatumRowStore::Iterator::Iterator(const ObDatumRowStore &row_store)
: row_store_(row_store),
cur_iter_block_(row_store_.blocks_.get_first()),
cur_iter_pos_(0)
{
}
int ObDatumRowStore::Iterator::get_next_row(ObDatumRow &row)
{
int ret = OB_SUCCESS;
int64_t pos = 0;
if (OB_UNLIKELY(row.count_ < row_store_.get_col_count())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "column buffer count is not enough", K(ret), K_(row.count), K(row_store_.get_col_count()));
} else if (OB_ISNULL(cur_iter_block_)) {
// the last block
ret = OB_ITER_END;
} else if (OB_FAIL(row.deserialize(cur_iter_block_->get_buffer_head() + cur_iter_pos_, cur_iter_block_->get_remain_size_for_read(cur_iter_pos_), pos))) {
STORAGE_LOG(WARN, "failed to deserialize datum row", K(ret), K(cur_iter_block_->get_block_size()), K(cur_iter_pos_),
K(row.get_serialize_size()), K(pos), K(row));
} else {
// next
cur_iter_pos_ += pos;
// update current block when current block reach its end
if (cur_iter_block_->get_remain_size_for_read(cur_iter_pos_) <= 0) {
// next block
cur_iter_block_ = cur_iter_block_->get_next_block();
cur_iter_pos_ = 0;
}
} // end else
return ret;
}
////////////////////////////////////////////////////////////////
ObDatumRowStore::ObDatumRowStore()
: inner_alloc_("DatumRowStore", MTL_ID()),
blocks_(),
row_count_(0),
col_count_(0)
{
}
ObDatumRowStore::~ObDatumRowStore()
{
clear_rows();
}
// method for ObAggregateFunction::prepare()
// prepare need to reuse ObDatumRowStore for WRITE,
// it needs to reuse reserved_columns_ which should not be cleared
void ObDatumRowStore::clear_rows()
{
row_count_ = 0;
col_count_ = 0;
// free all blocks
BlockInfo *block = blocks_.get_first();
while (NULL != block) {
BlockInfo *next = block->get_next_block();
block->~BlockInfo();
inner_alloc_.free(block);
block = next;
}
blocks_.reset();
}
int ObDatumRowStore::new_block(int64_t block_size, ObDatumRowStore::BlockInfo *&block)
{
int ret = OB_SUCCESS;
// normalize block size
if (block_size > BIG_BLOCK_SIZE) {
block_size = OB_MAX_ROW_LENGTH_IN_MEMTABLE;
} else if (block_size > NORMAL_BLOCK_SIZE) {
block_size = BIG_BLOCK_SIZE;
} else {
block_size = NORMAL_BLOCK_SIZE;
}
// make sure all memory allocated under the right tenant
block = static_cast<BlockInfo *>(inner_alloc_.alloc(block_size));
if (OB_ISNULL(block)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to alloc block memory", K(ret), K(block_size + sizeof(BlockInfo)));
} else {
block = new(block) BlockInfo(block_size - sizeof(BlockInfo));
if (OB_FAIL(blocks_.add_last(block))) {
STORAGE_LOG(WARN, "failed to add a new block to block list", K(ret));
}
}
return ret;
}
int ObDatumRowStore::add_row(const ObDatumRow &row)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(0 < col_count_) && OB_UNLIKELY(row.count_ != col_count_)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "all rows should have the same columns", K(col_count_), K(row.count_));
} else {
int64_t length = row.get_serialize_size();
BlockInfo *block = blocks_.get_last();
if (OB_ISNULL(block) || block->get_remain_size() < length) {
if (OB_FAIL(new_block(length, block))) {
STORAGE_LOG(WARN, "failed to new block", K(ret), K(length));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(block->append_row(row, length))) {
STORAGE_LOG(WARN, "failed to append row", K(ret), K(row));
} else {
++row_count_;
}
}
}
return ret;
}
ObDatumRowStore::Iterator ObDatumRowStore::begin() const
{
return Iterator(*this);
}
} //end common
} //end oceanbase

92
src/storage/blocksstable/ob_datum_row_store.h Normal file
View File

@ -0,0 +1,92 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_COMMON_OB_DATUM_ROW_STORE_H
#define OCEANBASE_COMMON_OB_DATUM_ROW_STORE_H
#include <stdint.h>
#include <utility>
#include "common/row/ob_row.h"
#include "common/row/ob_row_iterator.h"
#include "lib/container/ob_fixed_array.h"
#include "lib/string/ob_string.h"
#include "ob_datum_row.h"
#include "storage/ob_i_store.h"
namespace oceanbase
{
namespace blocksstable
{
class ObDatumRowStore
{
public:
struct BlockInfo;
class Iterator
{
public:
friend class ObDatumRowStore;
int get_next_row(ObDatumRow &row);
private:
explicit Iterator(const ObDatumRowStore &row_store);
protected:
const ObDatumRowStore &row_store_;
const BlockInfo *cur_iter_block_;
int64_t cur_iter_pos_;
};
public:
ObDatumRowStore();
~ObDatumRowStore();
void clear_rows();
int add_row(const ObDatumRow &row);
inline int64_t get_row_count() const { return row_count_; }
inline int64_t get_col_count() const { return col_count_; }
Iterator begin() const;
TO_STRING_KV(N_BLOCK_NUM, blocks_.get_block_count(),
N_ROW_COUNT, row_count_,
N_COLUMN_COUNT, col_count_);
private:
DISALLOW_COPY_AND_ASSIGN(ObDatumRowStore);
static const int64_t BIG_BLOCK_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
static const int64_t NORMAL_BLOCK_SIZE = OB_MALLOC_NORMAL_BLOCK_SIZE;
// non-circular doubly linked list
class BlockList
{
public:
BlockList();
void reset();
int add_last(BlockInfo *block);
BlockInfo *get_first() { return first_; }
BlockInfo *get_last() { return last_; }
const BlockInfo *get_first() const { return first_; }
const BlockInfo *get_last() const { return last_; }
int64_t get_block_count() const { return count_; }
int64_t get_used_mem_size() const { return used_mem_size_; }
private:
BlockInfo *first_;
BlockInfo *last_;
int64_t count_;
int64_t used_mem_size_; // bytes of all blocks
};
private:
int new_block(int64_t block_size, BlockInfo *&block);
private:
DefaultPageAllocator inner_alloc_;
BlockList blocks_;
int64_t row_count_;
int64_t col_count_;
};
} // end namespace common
} // end namespace oceanbase
#endif /* OCEANBASE_COMMON_OB_DATUM_ROW_STORE_H */

97
src/storage/blocksstable/ob_datum_row_utils.cpp Normal file
View File

@ -0,0 +1,97 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include "ob_datum_row_utils.h"
namespace oceanbase
{
namespace blocksstable
{
int ObDatumRowUtils::ob_create_row(ObIAllocator &allocator, int64_t col_count, ObDatumRow *&datum_row)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(col_count <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("row_count is invalid", K(ret), K(col_count));
} else {
void *row_buf = NULL;
if (OB_ISNULL(row_buf = allocator.alloc(sizeof(blocksstable::ObDatumRow)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate row buffer failed", K(ret), K(sizeof(blocksstable::ObDatumRow)));
} else if (FALSE_IT(datum_row = new(row_buf) blocksstable::ObDatumRow())) {
} else if (OB_FAIL(datum_row->init(allocator, col_count))) {
LOG_WARN("fail to init datum row", K(ret), K(col_count));
}
if (OB_FAIL(ret) && nullptr != datum_row) {
datum_row->~ObDatumRow();
datum_row = nullptr;
allocator.free(row_buf);
}
}
return ret;
}
int ObDatumRowUtils::ob_create_rows(ObIAllocator &allocator, int64_t row_count, int64_t col_count, ObDatumRow *&datum_rows)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(col_count <= 0 || row_count <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("col count or row count is invalid", K(ret), K(col_count), K(row_count));
} else {
void *rows_buf = nullptr;
const size_t rows_buf_len = sizeof(blocksstable::ObDatumRow) * row_count;
if (OB_ISNULL(rows_buf = allocator.alloc(rows_buf_len))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("Failed to allocate row buffer", K(ret), K(rows_buf_len));
} else {
char *row_buf = static_cast<char*>(rows_buf);
datum_rows = new(row_buf) blocksstable::ObDatumRow[row_count]();
int64_t i = 0;
for (; OB_SUCC(ret) && i < row_count; ++i) {
if (OB_FAIL(datum_rows[i].init(allocator, col_count))) {
LOG_WARN("fail to init datum row", K(ret), K(col_count), K(datum_rows[i]));
}
}
if (OB_FAIL(ret)) {
// release storage_datums
for (int64_t j = i; j >= 0; --j) {
datum_rows[j].~ObDatumRow();
}
allocator.free(rows_buf);
}
}
}
return ret;
}
int ObDatumRowUtils::prepare_rowkey(
const ObDatumRow &datum_row,
const int key_datum_cnt,
const ObColDescIArray &col_descs,
common::ObIAllocator &allocator,
ObDatumRowkey &rowkey)
{
int ret = OB_SUCCESS;
if (!datum_row.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid datum row", K(ret), K(datum_row));
} else if (OB_FAIL(rowkey.assign(datum_row.storage_datums_, key_datum_cnt))) {
LOG_WARN("failed to assign datum rowkey", K(ret), K(datum_row), K(key_datum_cnt));
} else if (OB_FAIL(rowkey.prepare_memtable_readable(col_descs, allocator))) {
LOG_WARN("failed to prepare store rowkey to read memtable", K(ret), K(datum_row), K(rowkey));
}
return ret;
}
} // namespace sql
} // namespace oceanbase

38
src/storage/blocksstable/ob_datum_row_utils.h Normal file
View File

@ -0,0 +1,38 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef DEV_SRC_STORAGE_BLOCKSSTABLE_OB_DATUM_ROW_UTILS_H_
#define DEV_SRC_STORAGE_BLOCKSSTABLE_OB_DATUM_ROW_UTILS_H_
#include "share/ob_define.h"
#include "ob_datum_row.h"
namespace oceanbase
{
namespace blocksstable
{
class ObDatumRowUtils
{
typedef common::ObIArray<share::schema::ObColDesc> ObColDescIArray;
public:
static int ob_create_row(ObIAllocator &allocator, int64_t col_count, ObDatumRow *&datum_row);
static int ob_create_rows(ObIAllocator &allocator, int64_t row_count, int64_t col_count, ObDatumRow *&datum_rows);
// TODO@xuanxi: rewrite it when store rowkey is no longer needed
static int prepare_rowkey(
const ObDatumRow &datum_row,
const int key_datum_cnt,
const ObColDescIArray &col_descs,
common::ObIAllocator &allocator,
ObDatumRowkey &rowkey);
};
} // namespace sql
} // namespace oceanbase
#endif /* DEV_SRC_STORAGE_BLOCKSSTABLE_OB_DATUM_ROW_UTILS_H_ */

19
src/storage/blocksstable/ob_datum_rowkey.cpp
View File

@ -616,6 +616,25 @@ int ObDatumRowkeyHelper::convert_store_rowkey(const ObDatumRowkey &datum_rowkey,
return ret;
}
int ObDatumRowkeyHelper::prepare_datum_rowkey(const ObDatumRow &datum_row,
const int key_datum_cnt,
const ObIArray<share::schema::ObColDesc> &col_descs,
ObDatumRowkey &datum_rowkey)
{
int ret = OB_SUCCESS;
if (!datum_row.is_valid() || col_descs.count() < datum_row.get_column_count()) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Get invalid datum row", K(ret), K(datum_row), K(col_descs));
} else if (OB_FAIL(datum_rowkey.assign(datum_row.storage_datums_, key_datum_cnt))) {
STORAGE_LOG(WARN, "Failed to assign datum rowkey", K(ret), K(datum_row), K(key_datum_cnt));
} else if (OB_FAIL(convert_store_rowkey(datum_rowkey, col_descs, datum_rowkey.store_rowkey_))) {
STORAGE_LOG(WARN, "Failed to convert store rowkeyy", K(ret), K(datum_rowkey));
}
return ret;
}
int ObDatumRowkeyHelper::reserve(const int64_t rowkey_cnt)
{

7
src/storage/blocksstable/ob_datum_rowkey.h
View File

@ -13,7 +13,7 @@
#ifndef OB_STORAGE_BLOCKSSTABLE_DATUM_ROWKEY_H
#define OB_STORAGE_BLOCKSSTABLE_DATUM_ROWKEY_H
#include "ob_datum_row.h"
#include "ob_storage_datum.h"
#include "lib/utility/ob_print_kv.h"
//to be removed
#include "common/rowkey/ob_store_rowkey.h"
@ -27,6 +27,7 @@ struct ObDatumRange;
class ObRowkeyVector;
struct ObDiscreteDatumRowkey;
struct ObCommonDatumRowkey;
struct ObDatumRow;
struct ObDatumRowkey
{
@ -123,6 +124,10 @@ public:
int convert_store_rowkey(const ObDatumRowkey &datum_rowkey,
const common::ObIArray<share::schema::ObColDesc> &col_descs,
common::ObStoreRowkey &rowkey);
int prepare_datum_rowkey(const ObDatumRow &datum_row,
const int key_datum_cnt,
const ObIArray<share::schema::ObColDesc> &col_descs,
ObDatumRowkey &datum_rowkey);
int reserve(const int64_t rowkey_cnt);
OB_INLINE ObStorageDatum *get_datums() { return datum_buffer_.get_datums(); }
OB_INLINE int64_t get_capacity() const { return datum_buffer_.get_capacity(); }

37
src/storage/blocksstable/ob_row_writer.cpp
View File

@ -76,17 +76,16 @@ int ObRowWriter::init_common(char *buf, const int64_t buf_size, const int64_t po
}
int ObRowWriter::check_row_valid(
const ObStoreRow &row,
const ObDatumRow &row,
const int64_t rowkey_column_count)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!row.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid row writer input argument", K(row), K(ret));
} else if (OB_UNLIKELY(rowkey_column_count <= 0 || rowkey_column_count > row.row_val_.count_)) {
LOG_WARN("invalid row writer input argument", K(ret), K(row));
} else if (OB_UNLIKELY(rowkey_column_count <= 0 || rowkey_column_count > row.count_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid row writer input argument",
K(rowkey_column_count), K(row.row_val_.count_), K(ret));
LOG_WARN("invalid row writer input argument", K(ret), K(rowkey_column_count), K(row.count_));
}
return ret;
}
@ -230,24 +229,24 @@ int ObRowWriter::write(const int64_t rowkey_column_cnt, const ObDatumRow &datum_
return ret;
}
// when update_idx == nullptr, write full row; else only write rowkey + update cells
// when update_idx == nullptr, write full row; else only write rowkey + update storage_datums
int ObRowWriter::write(
const int64_t rowkey_column_count,
const storage::ObStoreRow &row,
const ObDatumRow &datum_row,
const ObIArray<int64_t> *update_idx,
char *&buf,
int64_t &len)
{
int ret = OB_SUCCESS;
len = 0;
if (OB_UNLIKELY(nullptr != update_idx && update_idx->count() > row.row_val_.count_)) {
if (OB_UNLIKELY(nullptr != update_idx && update_idx->count() > datum_row.count_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("update idx is invalid", K(ret), KPC(update_idx), K_(row.row_val_.count), K(rowkey_column_count));
LOG_WARN("update idx is invalid", K(ret), KPC(update_idx), K_(datum_row.count), K(rowkey_column_count));
} else {
do {
if (OB_FAIL(alloc_buf_and_init(OB_BUF_NOT_ENOUGH == ret))) {
LOG_WARN("row writer fail to alloc and init", K(ret));
} else if (OB_FAIL(inner_write_row(rowkey_column_count, row, update_idx))) {
} else if (OB_FAIL(inner_write_row(rowkey_column_count, datum_row, update_idx))) {
if (OB_BUF_NOT_ENOUGH != ret) {
LOG_WARN("row writer fail to append row header", K(ret), K(row_buffer_), K(pos_));
}
@ -289,29 +288,29 @@ int ObRowWriter::check_update_idx_array_valid(
int ObRowWriter::inner_write_row(
const int64_t rowkey_column_count,
const ObStoreRow &row,
const ObDatumRow &datum_row,
const ObIArray<int64_t> *update_idx)
{
int ret = OB_SUCCESS;
if (OB_FAIL(check_row_valid(row, rowkey_column_count))) {
if (OB_FAIL(check_row_valid(datum_row, rowkey_column_count))) {
LOG_WARN("row writer fail to init store row", K(ret), K(rowkey_column_count));
} else if (nullptr != update_idx && OB_FAIL(check_update_idx_array_valid(rowkey_column_count, update_idx))) {
LOG_WARN("invalid update idx array", K(ret));
} else if (OB_FAIL(append_row_header(
row.flag_.get_serialize_flag(),
row.row_type_flag_.flag_,
row.trans_id_.get_id(),
row.row_val_.count_,
datum_row.row_flag_.get_serialize_flag(),
datum_row.mvcc_row_flag_.flag_,
datum_row.trans_id_.get_id(),
datum_row.count_,
rowkey_column_count))) {
if (OB_BUF_NOT_ENOUGH != ret) {
LOG_WARN("row writer fail to append row header", K(ret), K(row));
LOG_WARN("row writer fail to append row header", K(ret), K(datum_row));
}
} else {
update_idx_array_ = update_idx;
rowkey_column_cnt_ = rowkey_column_count;
if (OB_FAIL(inner_write_cells(row.row_val_.cells_, row.row_val_.count_))) {
if (OB_FAIL(inner_write_cells(datum_row.storage_datums_, datum_row.count_))) {
if (OB_BUF_NOT_ENOUGH != ret) {
LOG_WARN("failed to write cells", K(ret), K(row));
LOG_WARN("failed to write cells", K(ret), K(datum_row));
}
}
}

6
src/storage/blocksstable/ob_row_writer.h
View File

@ -63,7 +63,7 @@ public:
int write_rowkey(const common::ObStoreRowkey &rowkey, char *&buf, int64_t &len);
int write(
const int64_t rowkey_cnt,
const storage::ObStoreRow &row,
const ObDatumRow &datum_row,
const ObIArray<int64_t> *update_idx,
char *&buf,
int64_t &len);
@ -102,7 +102,7 @@ private:
};
int inner_write_row(
const int64_t rowkey_column_count,
const storage::ObStoreRow &row,
const ObDatumRow &row,
const ObIArray<int64_t> *update_idx);
OB_INLINE int write_oracle_timestamp(const common::ObOTimestampData &ot_data, const common::ObOTimestampMetaAttrType otmat);
int append_column(const common::ObObj &obj);
@ -110,7 +110,7 @@ private:
int append_8_bytes_column(const ObStorageDatum &datum);
int init_common(char *buf, const int64_t buf_size, const int64_t pos);
int check_row_valid(
const storage::ObStoreRow &row,
const ObDatumRow &row,
const int64_t rowkey_column_count);
int append_row_header(
const uint8_t row_flag,

386
src/storage/blocksstable/ob_storage_datum.cpp Normal file
View File

@ -0,0 +1,386 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include "ob_storage_datum.h"
#include "share/schema/ob_table_param.h"
#include "share/ob_force_print_log.h"
#include "storage/ob_i_store.h"
#include "share/scheduler/ob_tenant_dag_scheduler.h"
namespace oceanbase
{
using namespace common;
namespace blocksstable
{
static int nonext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret)
{
int ret = cmp_func.cmp_func_(left, right, cmp_ret);
STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret));
return ret;
}
static int nonext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret)
{
int ret = OB_SUCCESS;
UNUSEDx(left, cmp_func);
if (right.is_max()) {
cmp_ret = -1;
} else if (right.is_min()) {
cmp_ret = 1;
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(right));
}
STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret));
return ret;
}
static int ext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret)
{
int ret = OB_SUCCESS;
UNUSEDx(right, cmp_func);
if (left.is_max()) {
cmp_ret = 1;
} else if (left.is_min()) {
cmp_ret = -1;
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left));
}
STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret));
return ret;
}
static int ext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret)
{
int ret = OB_SUCCESS;
UNUSEDx(cmp_func);
int64_t lv = left.is_max() - left.is_min();
int64_t rv = right.is_max() - right.is_min();
if (OB_UNLIKELY(0 == lv || 0 == rv)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left), K(right));
} else {
cmp_ret = lv - rv;
}
STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret));
return ret;
}
typedef int (*ExtSafeCompareFunc)(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret);
static ExtSafeCompareFunc ext_safe_cmp_funcs[2][2] = {
{nonext_nonext_compare, nonext_ext_compare},
{ext_nonext_compare, ext_ext_compare}
};
int ObStorageDatumCmpFunc::compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const
{
return ext_safe_cmp_funcs[left.is_ext()][right.is_ext()](left, right, cmp_func_, cmp_ret);
}
/*
*ObStorageDatumUtils
*/
ObStorageDatumUtils::ObStorageDatumUtils()
: rowkey_cnt_(0),
cmp_funcs_(),
hash_funcs_(),
ext_hash_func_(),
is_oracle_mode_(false),
is_inited_(false)
{}
ObStorageDatumUtils::~ObStorageDatumUtils()
{}
int ObStorageDatumUtils::transform_multi_version_col_desc(const ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
ObIArray<share::schema::ObColDesc> &mv_col_descs)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(schema_rowkey_cnt > col_descs.count())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to transform mv col descs", K(ret), K(schema_rowkey_cnt), K(col_descs));
} else {
mv_col_descs.reuse();
for (int64_t i = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; i++) {
if (OB_FAIL(mv_col_descs.push_back(col_descs.at(i)))) {
STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(i));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(storage::ObMultiVersionRowkeyHelpper::add_extra_rowkey_cols(mv_col_descs))) {
STORAGE_LOG(WARN, "Fail to add extra_rowkey_cols", K(ret), K(schema_rowkey_cnt));
} else {
for (int64_t i = schema_rowkey_cnt; OB_SUCC(ret) && i < col_descs.count(); i++) {
const share::schema::ObColDesc &col_desc = col_descs.at(i);
if (col_desc.col_id_ == common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID
|| col_desc.col_id_ == common::OB_HIDDEN_SQL_SEQUENCE_COLUMN_ID) {
continue;
} else if (OB_FAIL(mv_col_descs.push_back(col_desc))) {
STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(col_desc));
}
}
}
}
return ret;
}
int ObStorageDatumUtils::init(const ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
const bool is_oracle_mode,
ObIAllocator &allocator,
const bool is_column_store)
{
int ret = OB_SUCCESS;
ObSEArray<share::schema::ObColDesc, 32> mv_col_descs;
int64_t mv_rowkey_cnt = 0;
int64_t mv_extra_rowkey_cnt = 0;
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this));
} else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER
|| schema_rowkey_cnt > col_descs.count())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(schema_rowkey_cnt), K(col_descs));
} else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) {
STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret));
} else if (FALSE_IT(mv_extra_rowkey_cnt = is_column_store ? 0 : storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) {
} else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + mv_extra_rowkey_cnt)) {
} else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, allocator))) {
STORAGE_LOG(WARN, "Failed to reserve cmp func array", K(ret));
} else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, allocator))) {
STORAGE_LOG(WARN, "Failed to reserve hash func array", K(ret));
} else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) {
STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt));
}
return ret;
}
int ObStorageDatumUtils::init(const common::ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
const bool is_oracle_mode,
const int64_t arr_buf_len,
char *arr_buf)
{
int ret = OB_SUCCESS;
ObSEArray<share::schema::ObColDesc, 32> mv_col_descs;
int64_t pos = 0;
int64_t mv_rowkey_cnt = 0;
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this));
} else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER
|| schema_rowkey_cnt > col_descs.count())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(col_descs), K(schema_rowkey_cnt));
} else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) {
STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret));
} else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) {
} else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) {
STORAGE_LOG(WARN, "Failed to init compare function array", K(ret));
} else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) {
STORAGE_LOG(WARN, "Failed to init hash function array", K(ret));
} else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) {
STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt));
}
return ret;
}
int ObStorageDatumUtils::inner_init(
const common::ObIArray<share::schema::ObColDesc> &mv_col_descs,
const int64_t mv_rowkey_col_cnt,
const bool is_oracle_mode)
{
int ret = OB_SUCCESS;
is_oracle_mode_ = is_oracle_mode;
// support column order index until next task done
//
// we could use the cmp funcs in the basic funcs directlly
bool is_null_last = is_oracle_mode_;
ObCmpFunc cmp_func;
ObHashFunc hash_func;
for (int64_t i = 0; OB_SUCC(ret) && i < mv_rowkey_col_cnt; i++) {
const share::schema::ObColDesc &col_desc = mv_col_descs.at(i);
//TODO @hanhui support desc rowkey
bool is_ascending = true || col_desc.col_order_ == ObOrderType::ASC;
bool has_lob_header = is_lob_storage(col_desc.col_type_.get_type());
ObPrecision precision = PRECISION_UNKNOWN_YET;
if (col_desc.col_type_.is_decimal_int()) {
precision = col_desc.col_type_.get_stored_precision();
OB_ASSERT(precision != PRECISION_UNKNOWN_YET);
}
sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(col_desc.col_type_.get_type(),
col_desc.col_type_.get_collation_type(),
col_desc.col_type_.get_scale(),
is_oracle_mode,
has_lob_header,
precision);
if (OB_UNLIKELY(nullptr == basic_funcs
|| nullptr == basic_funcs->null_last_cmp_
|| nullptr == basic_funcs->murmur_hash_)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected null basic funcs", K(ret), K(col_desc));
} else {
cmp_func.cmp_func_ = is_null_last ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_;
hash_func.hash_func_ = basic_funcs->murmur_hash_;
if (OB_FAIL(hash_funcs_.push_back(hash_func))) {
STORAGE_LOG(WARN, "Failed to push back hash func", K(ret), K(i), K(col_desc));
} else if (is_ascending) {
if (OB_FAIL(cmp_funcs_.push_back(ObStorageDatumCmpFunc(cmp_func)))) {
STORAGE_LOG(WARN, "Failed to push back cmp func", K(ret), K(i), K(col_desc));
}
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(WARN, "Unsupported desc column order", K(ret), K(col_desc), K(i));
}
}
}
if (OB_SUCC(ret)) {
sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(ObExtendType, CS_TYPE_BINARY);
if (OB_UNLIKELY(nullptr == basic_funcs || nullptr == basic_funcs->murmur_hash_)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected null basic funcs for extend type", K(ret));
} else {
ext_hash_func_.hash_func_ = basic_funcs->murmur_hash_;
rowkey_cnt_ = mv_rowkey_col_cnt;
is_inited_ = true;
}
}
return ret;
}
int ObStorageDatumUtils::assign(const ObStorageDatumUtils &other_utils, ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!other_utils.is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to assign datum utils", K(ret), K(other_utils));
} else {
rowkey_cnt_ = other_utils.get_rowkey_count();
is_oracle_mode_ = other_utils.is_oracle_mode();
ext_hash_func_ = other_utils.get_ext_hash_funcs();
if (OB_FAIL(cmp_funcs_.init_and_assign(other_utils.get_cmp_funcs(), allocator))) {
STORAGE_LOG(WARN, "Failed to assign cmp func array", K(ret));
} else if (OB_FAIL(hash_funcs_.init_and_assign(other_utils.get_hash_funcs(), allocator))) {
STORAGE_LOG(WARN, "Failed to assign hash func array", K(ret));
} else {
is_inited_ = true;
}
}
return ret;
}
void ObStorageDatumUtils::reset()
{
rowkey_cnt_ = 0;
cmp_funcs_.reset();
hash_funcs_.reset();
ext_hash_func_.hash_func_ = nullptr;
is_inited_ = false;
}
int64_t ObStorageDatumUtils::get_deep_copy_size() const
{
return cmp_funcs_.get_deep_copy_size() + hash_funcs_.get_deep_copy_size();
}
/*
*ObStorageDatumBuffer
*/
ObStorageDatumBuffer::ObStorageDatumBuffer(common::ObIAllocator *allocator)
: capacity_(LOCAL_BUFFER_ARRAY),
local_datums_(),
datums_(local_datums_),
allocator_(allocator),
is_inited_(nullptr != allocator)
{}
ObStorageDatumBuffer::~ObStorageDatumBuffer()
{
if (datums_ != local_datums_ && nullptr != allocator_) {
allocator_->free(datums_);
}
}
void ObStorageDatumBuffer::reset()
{
if (datums_ != local_datums_ && nullptr != allocator_) {
allocator_->free(datums_);
}
allocator_ = nullptr;
datums_ = local_datums_;
capacity_ = LOCAL_BUFFER_ARRAY;
for (int64_t i = 0; i < capacity_; i++) {
datums_[i].reuse();
}
is_inited_ = false;
}
int ObStorageDatumBuffer::init(common::ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObStorageDatumBuffer init twice", K(ret), K(*this));
} else {
OB_ASSERT(datums_ == local_datums_);
allocator_ = &allocator;
is_inited_ = true;
}
return ret;
}
int ObStorageDatumBuffer::reserve(const int64_t count, const bool keep_data)
{
int ret = OB_SUCCESS;
void *buf = nullptr;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObStorageDatumBuffer is not inited", K(ret), K(*this));
} else if (OB_UNLIKELY(count <= 0)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to reserve datum buffer", K(ret), K(count));
} else if (count <= capacity_){
} else if (OB_ISNULL(buf = allocator_->alloc(sizeof(ObStorageDatum) * count))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(count));
} else {
ObStorageDatum *new_datums = new (buf) ObStorageDatum [count];
if (keep_data) {
for (int64_t i = 0; i < capacity_; i++) {
new_datums[i] = datums_[i];
}
}
if (nullptr != datums_ && datums_ != local_datums_) {
allocator_->free(datums_);
}
datums_ = new_datums;
capacity_ = count;
}
return ret;
}
} // namespace blocksstable
} // namespace oceanbase

427
src/storage/blocksstable/ob_storage_datum.h Normal file
View File

@ -0,0 +1,427 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_STORAGE_BLOCKSSTABLE_STORAGE_DATUM_H
#define OB_STORAGE_BLOCKSSTABLE_STORAGE_DATUM_H
#include "common/ob_common_types.h"
#include "common/ob_tablet_id.h"
#include "share/datum/ob_datum.h"
#include "share/datum/ob_datum_funcs.h"
#include "storage/meta_mem/ob_fixed_meta_obj_array.h"
#include "storage/tx/ob_trans_define.h"
#include "common/row/ob_row.h"
#include "storage/ob_storage_util.h"
namespace oceanbase
{
namespace share{
namespace schema
{
struct ObColDesc;
}
}
namespace storage
{
struct ObStoreRow;
}
namespace blocksstable
{
//TODO optimize number buffer
struct ObStorageDatum : public common::ObDatum
{
ObStorageDatum() { set_nop(); }
ObStorageDatum(const ObStorageDatum &datum) { reuse(); *this = datum; }
~ObStorageDatum() = default;
// ext value section
OB_INLINE void reuse() { ptr_ = buf_; reserved_ = 0; pack_ = 0; }
OB_INLINE void set_ext_value(const int64_t ext_value)
{ reuse(); set_ext(); no_cv(extend_obj_)->set_ext(ext_value); }
OB_INLINE void set_nop() { set_ext_value(ObActionFlag::OP_NOP); }
OB_INLINE void set_min() { set_ext_value(common::ObObj::MIN_OBJECT_VALUE); }
OB_INLINE void set_max() { set_ext_value(common::ObObj::MAX_OBJECT_VALUE); }
OB_INLINE bool is_nop_value() const { return is_nop(); } // temp solution
// transfer section
OB_INLINE bool is_local_buf() const { return ptr_ == buf_; }
OB_INLINE int from_buf_enhance(const char *buf, const int64_t buf_len);
OB_INLINE int from_obj_enhance(const common::ObObj &obj);
OB_INLINE int to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const;
OB_INLINE int deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator);
OB_INLINE int deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos);
OB_INLINE void shallow_copy_from_datum(const ObDatum &src);
OB_INLINE int64_t get_deep_copy_size() const;
OB_INLINE ObStorageDatum& operator=(const ObStorageDatum &other);
OB_INLINE int64_t storage_to_string(char *buf, int64_t buf_len, const bool for_dump = false) const;
OB_INLINE bool need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const;
OB_INLINE const char *to_cstring(const bool for_dump = false) const;
//only for unittest
OB_INLINE bool operator==(const ObStorageDatum &other) const;
OB_INLINE bool operator==(const ObObj &other) const;
//datum 12 byte
int32_t reserved_;
// buf 16 byte
char buf_[common::OBJ_DATUM_NUMBER_RES_SIZE];
};
struct ObStorageDatumCmpFunc
{
public:
ObStorageDatumCmpFunc(common::ObCmpFunc &cmp_func) : cmp_func_(cmp_func) {}
ObStorageDatumCmpFunc() = default;
~ObStorageDatumCmpFunc() = default;
int compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const;
OB_INLINE const common::ObCmpFunc &get_cmp_func() const { return cmp_func_; }
TO_STRING_KV(K_(cmp_func));
private:
common::ObCmpFunc cmp_func_;
};
typedef storage::ObFixedMetaObjArray<ObStorageDatumCmpFunc> ObStoreCmpFuncs;
typedef storage::ObFixedMetaObjArray<common::ObHashFunc> ObStoreHashFuncs;
struct ObStorageDatumUtils
{
public:
ObStorageDatumUtils();
~ObStorageDatumUtils();
// init with array memory from allocator
int init(const common::ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
const bool is_oracle_mode,
common::ObIAllocator &allocator,
const bool is_column_store = false);
// init with array memory on fixed size memory buffer
int init(const common::ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
const bool is_oracle_mode,
const int64_t arr_buf_len,
char *arr_buf);
int assign(const ObStorageDatumUtils &other_utils, common::ObIAllocator &allocator);
void reset();
OB_INLINE bool is_valid() const
{
return is_inited_ && cmp_funcs_.count() >= rowkey_cnt_ && hash_funcs_.count() >= rowkey_cnt_;
}
OB_INLINE bool is_oracle_mode() const { return is_oracle_mode_; }
OB_INLINE int64_t get_rowkey_count() const { return rowkey_cnt_; }
OB_INLINE const ObStoreCmpFuncs &get_cmp_funcs() const { return cmp_funcs_; }
OB_INLINE const ObStoreHashFuncs &get_hash_funcs() const { return hash_funcs_; }
OB_INLINE const common::ObHashFunc &get_ext_hash_funcs() const { return ext_hash_func_; }
int64_t get_deep_copy_size() const;
TO_STRING_KV(K_(is_oracle_mode), K_(rowkey_cnt), K_(is_inited), K_(is_oracle_mode));
private:
//TODO to be removed by @hanhui
int transform_multi_version_col_desc(const common::ObIArray<share::schema::ObColDesc> &col_descs,
const int64_t schema_rowkey_cnt,
common::ObIArray<share::schema::ObColDesc> &mv_col_descs);
int inner_init(
const common::ObIArray<share::schema::ObColDesc> &mv_col_descs,
const int64_t mv_rowkey_col_cnt,
const bool is_oracle_mode);
private:
int32_t rowkey_cnt_; // multi version rowkey
ObStoreCmpFuncs cmp_funcs_; // multi version rowkey cmp funcs
ObStoreHashFuncs hash_funcs_; // multi version rowkey cmp funcs
common::ObHashFunc ext_hash_func_;
bool is_oracle_mode_;
bool is_inited_;
DISALLOW_COPY_AND_ASSIGN(ObStorageDatumUtils);
};
struct ObStorageDatumBuffer
{
public:
ObStorageDatumBuffer(common::ObIAllocator *allocator = nullptr);
~ObStorageDatumBuffer();
void reset();
int init(common::ObIAllocator &allocator);
int reserve(const int64_t count, const bool keep_data = false);
OB_INLINE bool is_valid() const { return is_inited_; }
OB_INLINE ObStorageDatum *get_datums() { return datums_; }
OB_INLINE int64_t get_capacity() const { return capacity_; }
TO_STRING_KV(K_(capacity), KP_(datums), KP_(local_datums));
private:
static const int64_t LOCAL_BUFFER_ARRAY = common::OB_ROW_DEFAULT_COLUMNS_COUNT >> 1;
int64_t capacity_;
ObStorageDatum local_datums_[LOCAL_BUFFER_ARRAY];
ObStorageDatum *datums_;
common::ObIAllocator *allocator_;
bool is_inited_;
};
OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator)
{
int ret = common::OB_SUCCESS;
reuse();
pack_ = src.pack_;
if (is_null()) {
} else if (src.len_ == 0) {
} else if (src.is_local_buf()) {
OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE);
MEMCPY(buf_, src.ptr_, src.len_);
ptr_ = buf_;
} else {
char * buf = static_cast<char *>(allocator.alloc(src.len_));
if (OB_ISNULL(buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
COMMON_LOG(WARN, "allocate memory failed", K(ret), K(src));
pack_ = 0;
} else {
MEMCPY(buf, src.ptr_, src.len_);
// need set ptr_ after memory copy, if this == &src
ptr_ = buf;
}
}
return ret;
}
OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos)
{
int ret = common::OB_SUCCESS;
reuse();
pack_ = src.pack_;
if (is_null()) {
} else if (src.len_ == 0) {
} else if (src.is_local_buf()) {
OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE);
MEMCPY(buf_, src.ptr_, src.len_);
ptr_ = buf_;
} else if (OB_UNLIKELY(nullptr == buf || buf_len < pos + src.len_)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to deep copy datum", K(ret), K(src), KP(buf), K(buf_len), K(pos));
pack_ = 0;
} else {
MEMCPY(buf + pos, src.ptr_, src.len_);
// need set ptr_ after memory copy, if this == &src
ptr_ = buf + pos;
pos += src.len_;
}
return ret;
}
OB_INLINE void ObStorageDatum::shallow_copy_from_datum(const ObDatum &src)
{
if (this != &src) {
reuse();
pack_ = src.pack_;
if (is_null()) {
} else if (src.len_ == 0) {
} else {
ptr_ = src.ptr_;
}
}
}
OB_INLINE int64_t ObStorageDatum::get_deep_copy_size() const
{
int64_t deep_copy_len = 0;
if (is_null()) {
} else if (is_local_buf()) {
OB_ASSERT(len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE);
} else {
deep_copy_len = len_;
}
return deep_copy_len;
}
OB_INLINE int ObStorageDatum::from_buf_enhance(const char *buf, const int64_t buf_len)
{
int ret = common::OB_SUCCESS;
if (OB_UNLIKELY(nullptr == buf || buf_len < 0 || buf_len > UINT32_MAX)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to transfer from buf", K(ret), KP(buf), K(buf_len));
} else {
reuse();
len_ = static_cast<uint32_t>(buf_len);
if (buf_len > 0) {
ptr_ = buf;
}
}
return ret;
}
OB_INLINE int ObStorageDatum::from_obj_enhance(const common::ObObj &obj)
{
int ret = common::OB_SUCCESS;
reuse();
if (obj.is_ext()) {
set_ext_value(obj.get_ext());
} else if (OB_FAIL(from_obj(obj))) {
STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(obj));
}
STORAGE_LOG(DEBUG, "chaser debug from obj", K(obj), K(*this));
return ret;
}
OB_INLINE int ObStorageDatum::to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const
{
int ret = common::OB_SUCCESS;
if (is_outrow()) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "lob should not set outrow in datum", K(ret), K(*this), K(obj), K(meta));
} else if (is_ext()) {
obj.set_ext(get_ext());
} else if (OB_FAIL(to_obj(obj, meta))) {
STORAGE_LOG(WARN, "Failed to transfer datum to obj", K(ret), K(*this), K(obj), K(meta));
}
return ret;
}
OB_INLINE ObStorageDatum& ObStorageDatum::operator=(const ObStorageDatum &other)
{
if (&other != this) {
reuse();
pack_ = other.pack_;
if (is_null()) {
} else if (len_ == 0) {
} else if (other.is_local_buf()) {
OB_ASSERT(other.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE);
MEMCPY(buf_, other.ptr_, other.len_);
ptr_ = buf_;
} else {
ptr_ = other.ptr_;
}
}
return *this;
}
OB_INLINE bool ObStorageDatum::operator==(const ObStorageDatum &other) const
{
bool bret = true;
if (is_null()) {
bret = other.is_null();
} else if (is_ext()) {
bret = other.is_ext() && extend_obj_->get_ext() == other.extend_obj_->get_ext();
} else {
bret = ObDatum::binary_equal(*this, other);
}
if (!bret) {
STORAGE_LOG(DEBUG, "datum and datum no equal", K(other), K(*this));
}
return bret;
}
OB_INLINE bool ObStorageDatum::operator==(const common::ObObj &other) const
{
int ret = OB_SUCCESS;
bool bret = true;
ObStorageDatum datum;
if (OB_FAIL(datum.from_obj_enhance(other))) {
STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(other), K(datum));
} else {
bret = *this == datum;
}
if (!bret) {
STORAGE_LOG(DEBUG, "obj and datum no equal", K(other), K(datum), KPC(this));
}
return bret;
}
OB_INLINE int64_t ObStorageDatum::storage_to_string(char *buf, int64_t buf_len, const bool for_dump) const
{
int64_t pos = 0;
if (is_ext()) {
if (is_nop()) {
J_NOP();
} else if (is_max()) {
BUF_PRINTF("MAX_OBJ");
} else if (is_min()) {
BUF_PRINTF("MIN_OBJ");
}
} else if(!for_dump) {
pos = to_string(buf, buf_len);
} else {
int ret = OB_SUCCESS;
const static int64_t STR_MAX_PRINT_LEN = 128L;
if (null_) {
J_NULL();
} else {
J_OBJ_START();
BUF_PRINTF("len: %d, flag: %d, null: %d", len_, flag_, null_);
if (len_ > 0) {
OB_ASSERT(NULL != ptr_);
const int64_t plen = std::min(static_cast<int64_t>(len_),
static_cast<int64_t>(STR_MAX_PRINT_LEN));
// print hex value
BUF_PRINTF(", hex: ");
if (OB_FAIL(hex_print(ptr_, plen, buf, buf_len, pos))) {
// no logging in to_string function.
} else {
// maybe ObIntTC
if (sizeof(int64_t) == len_) {
BUF_PRINTF(", int: %ld", *int_);
// maybe number with one digit
if (1 == num_->desc_.len_) {
BUF_PRINTF(", num_digit0: %u", num_->digits_[0]);
}
}
// maybe printable C string
int64_t idx = 0;
while (idx < plen && isprint(ptr_[idx])) {
idx++;
}
if (idx >= plen) {
BUF_PRINTF(", cstr: %.*s", static_cast<int>(plen), ptr_);
}
}
}
J_OBJ_END();
}
}
return pos;
}
OB_INLINE const char *ObStorageDatum::to_cstring(const bool for_dump) const
{
char *buffer = NULL;
int64_t str_len = 0;
CStringBufMgr &mgr = CStringBufMgr::get_thread_local_instance();
mgr.inc_level();
const int64_t buf_len = mgr.acquire(buffer);
if (OB_ISNULL(buffer)) {
LIB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "buffer is NULL");
} else {
str_len = storage_to_string(buffer, buf_len -1, for_dump);
if (str_len >= 0 && str_len < buf_len) {
buffer[str_len] = '\0';
} else {
buffer[0] = '\0';
}
mgr.update_position(str_len + 1);
}
mgr.try_clear_list();
mgr.dec_level();
return buffer;
}
OB_INLINE bool ObStorageDatum::need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const
{
return OBJ_DATUM_STRING == map_type && sizeof(uint64_t) == len_ && is_local_buf();
}
} // namespace blocksstable
} // namespace oceanbase
#endif

19
src/storage/lob/ob_ext_info_callback.cpp
View File

@ -186,7 +186,7 @@ int ObExtInfoCallback::set(
{
int ret = OB_SUCCESS;
ObLobManager *lob_mngr = MTL(ObLobManager*);
ObDatumRow datum_row;
blocksstable::ObDatumRow datum_row;
char *buf = nullptr;
int64_t len = 0;
@ -250,7 +250,8 @@ int ObExtInfoCbRegister::register_cb(
const blocksstable::ObDmlFlag dml_flag,
transaction::ObTxDesc *tx_desc,
transaction::ObTxSEQ &parent_seq_no,
ObObj &index_data,
blocksstable::ObStorageDatum &index_data,
ObObjType &type,
ObObj &ext_info_data)
{
int ret = OB_SUCCESS;
@ -269,7 +270,7 @@ int ObExtInfoCbRegister::register_cb(
} else if (OB_ISNULL(mvcc_ctx_ = ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("data is empty", K(ret), K(ext_info_data));
} else if (OB_FAIL(init_header(index_data, ext_info_data))) {
} else if (OB_FAIL(init_header(type))) {
LOG_WARN("init_header_ fail", K(ret), K(ext_info_data));
} else if (OB_FAIL(build_data_iter(ext_info_data))) {
LOG_WARN("build data iter fail", K(ret));
@ -306,24 +307,24 @@ int ObExtInfoCbRegister::register_cb(
}
if (OB_FAIL(ret)) {
} else if (OB_FALSE_IT(seq_no_cnt_ = cb_cnt)) {
} else if (OB_FAIL(set_index_data(index_data))) {
} else if (OB_FAIL(set_index_data(index_data, type))) {
LOG_WARN("set_index_data fail", K(ret));
}
}
return ret;
}
int ObExtInfoCbRegister::init_header(ObObj& index_data, ObObj &ext_info_data)
int ObExtInfoCbRegister::init_header(ObObjType &type)
{
int ret = OB_SUCCESS;
header_.type_ = get_type(index_data.get_type());
header_.type_ = get_type(type);
return ret;
}
int ObExtInfoCbRegister::set_index_data(ObObj &index_data)
int ObExtInfoCbRegister::set_index_data(blocksstable::ObStorageDatum &index_data, ObObjType &type)
{
int ret = OB_SUCCESS;
if (is_lob_storage(index_data.get_type())) {
if (is_lob_storage(type)) {
if (OB_FAIL(set_outrow_ctx_seq_no(index_data))) {
LOG_WARN("set_outrow_ctx_seq_no fail", K(ret));
}
@ -334,7 +335,7 @@ int ObExtInfoCbRegister::set_index_data(ObObj &index_data)
return ret;
}
int ObExtInfoCbRegister::set_outrow_ctx_seq_no(ObObj& index_data)
int ObExtInfoCbRegister::set_outrow_ctx_seq_no(blocksstable::ObStorageDatum& index_data)
{
int ret = OB_SUCCESS;
ObLobLocatorV2 locator;

11
src/storage/lob/ob_ext_info_callback.h
View File

@ -153,12 +153,13 @@ public:
const blocksstable::ObDmlFlag dml_flag,
transaction::ObTxDesc *tx_desc,
transaction::ObTxSEQ &parent_seq_no,
ObObj &index_data,
blocksstable::ObStorageDatum &index_data,
ObObjType &type,
ObObj &ext_info_data);
private:
static ObExtInfoLogType get_type(ObObjType obj_type)
static ObExtInfoLogType get_type(ObObjType &obj_type)
{
ObExtInfoLogType res = OB_INVALID_EXT_INFO_LOG;
switch (obj_type)
@ -174,12 +175,12 @@ private:
int build_data_iter(ObObj &ext_info_data);
int set_index_data(ObObj &index_data);
int set_outrow_ctx_seq_no(ObObj& index_data);
int set_index_data(blocksstable::ObStorageDatum &index_data, ObObjType &type);
int set_outrow_ctx_seq_no(blocksstable::ObStorageDatum& index_data);
int get_data(ObString &data);
int init_header(ObObj& index_data, ObObj &ext_info_data);
int init_header(ObObjType &type);
public:
TO_STRING_KV(K(timeout_), K(data_size_), K(seq_no_st_), K(seq_no_cnt_), K(header_writed_));

4
src/storage/lob/ob_lob_meta.cpp
View File

@ -972,7 +972,7 @@ int ObLobMetaManager::write(ObLobAccessParam& param, ObLobMetaInfo& in_row)
return ret;
}
int ObLobMetaManager::batch_insert(ObLobAccessParam& param, ObNewRowIterator &iter)
int ObLobMetaManager::batch_insert(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter)
{
int ret = OB_SUCCESS;
if (OB_FAIL(persistent_lob_adapter_.write_lob_meta(param, iter))) {
@ -981,7 +981,7 @@ int ObLobMetaManager::batch_insert(ObLobAccessParam& param, ObNewRowIterator &it
return ret;
}
int ObLobMetaManager::batch_delete(ObLobAccessParam& param, ObNewRowIterator &iter)
int ObLobMetaManager::batch_delete(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter)
{
int ret = OB_SUCCESS;
if (OB_FAIL(persistent_lob_adapter_.erase_lob_meta(param, iter))) {

4
src/storage/lob/ob_lob_meta.h
View File

@ -230,8 +230,8 @@ public:
~ObLobMetaManager() {}
// write one lob meta row
int write(ObLobAccessParam& param, ObLobMetaInfo& in_row);
int batch_insert(ObLobAccessParam& param, ObNewRowIterator &iter);
int batch_delete(ObLobAccessParam& param, ObNewRowIterator &iter);
int batch_insert(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter);
int batch_delete(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter);
// append
int append(ObLobAccessParam& param, ObLobMetaWriteIter& iter);
// return ObLobMetaWriteResult

160
src/storage/lob/ob_lob_persistent_adaptor.cpp
View File

@ -418,15 +418,14 @@ int ObPersistentLobApator::erase_lob_piece_tablet(ObLobAccessParam& param, ObLob
// construct insert data
int64_t affected_rows = 0;
ObObj cell[ObLobPieceUtil::LOB_PIECE_COLUMN_CNT];
char serialize_buf[32] = {0};
// make insert iterator
ObNewRow new_row;
ObDatumRow new_row;
blocksstable::ObSingleDatumRowIteratorWrapper single_iter;
common::ObSingleRowIteratorWrapper single_iter;
single_iter.set_row(&new_row);
if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, cell, new_row, &single_iter, in_row))) {
if (OB_FAIL(new_row.init(ObLobPieceUtil::LOB_PIECE_COLUMN_CNT))) {
LOG_WARN("failed to init new datum row", K(ret));
} else if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, new_row, &single_iter, in_row))) {
LOG_WARN("failed to set insert piece row.", K(ret), K(in_row));
} else if (OB_FAIL(oas->delete_rows(param.ls_id_,
lob_piece_tablet.get_obj()->get_tablet_meta().tablet_id_,
@ -534,15 +533,14 @@ int ObPersistentLobApator::write_lob_piece_tablet(ObLobAccessParam& param, ObLob
// construct insert data
int64_t affected_rows = 0;
ObObj cell[ObLobPieceUtil::LOB_PIECE_COLUMN_CNT];
char serialize_buf[32] = {0};
// make insert iterator
ObNewRow new_row;
ObDatumRow new_row;
blocksstable::ObSingleDatumRowIteratorWrapper single_iter;
common::ObSingleRowIteratorWrapper single_iter;
single_iter.set_row(&new_row);
if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, cell, new_row, &single_iter, in_row))) {
if (OB_FAIL(new_row.init(ObLobPieceUtil::LOB_PIECE_COLUMN_CNT))) {
LOG_WARN("failed to init new datum row", K(ret));
} else if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, new_row, &single_iter, in_row))) {
LOG_WARN("failed to set insert piece row.", K(ret), K(in_row));
} else if (OB_FAIL(oas->insert_rows(param.ls_id_,
lob_piece_tablet.get_obj()->get_tablet_meta().tablet_id_,
@ -604,15 +602,14 @@ int ObPersistentLobApator::update_lob_piece_tablet(ObLobAccessParam& param, ObLo
// construct insert data
int64_t affected_rows = 0;
ObObj cell[ObLobPieceUtil::LOB_PIECE_COLUMN_CNT];
char serialize_buf[32] = {0};
// make insert iterator
ObNewRow new_row;
ObDatumRow new_row;
blocksstable::ObSingleDatumRowIteratorWrapper single_iter;
common::ObSingleRowIteratorWrapper single_iter;
single_iter.set_row(&new_row);
if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, cell, new_row, &single_iter, in_row))) {
if (OB_FAIL(new_row.init(ObLobPieceUtil::LOB_PIECE_COLUMN_CNT))) {
LOG_WARN("failed to init new datum row", K(ret));
} else if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, new_row, &single_iter, in_row))) {
LOG_WARN("failed to set insert piece row.", K(ret), K(in_row));
} else if (OB_FAIL(oas->update_rows(param.ls_id_,
lob_piece_tablet.get_obj()->get_tablet_meta().tablet_id_,
@ -824,43 +821,32 @@ int ObPersistentLobApator::inner_get_tablet(
}
void ObPersistentLobApator::set_lob_meta_row(
ObObj* cell,
ObNewRow& new_row,
blocksstable::ObDatumRow& datum_row,
ObLobMetaInfo& in_row)
{
for (int64_t i = 0; i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) {
cell[i].reset();
cell[i].set_nop_value();
}
cell[ObLobMetaUtil::LOB_ID_COL_ID].set_varchar(reinterpret_cast<char*>(&in_row.lob_id_), sizeof(ObLobId));
cell[ObLobMetaUtil::LOB_ID_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY);
cell[ObLobMetaUtil::SEQ_ID_COL_ID].set_varchar(in_row.seq_id_);
cell[ObLobMetaUtil::SEQ_ID_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY);
cell[ObLobMetaUtil::BYTE_LEN_COL_ID].set_uint32(in_row.byte_len_);
cell[ObLobMetaUtil::CHAR_LEN_COL_ID].set_uint32(in_row.char_len_);
cell[ObLobMetaUtil::PIECE_ID_COL_ID].set_uint64(in_row.piece_id_);
cell[ObLobMetaUtil::LOB_DATA_COL_ID].set_varchar(in_row.lob_data_);
cell[ObLobMetaUtil::LOB_DATA_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY);
new_row.assign(cell, ObLobMetaUtil::LOB_META_COLUMN_CNT);
datum_row.reuse();
datum_row.storage_datums_[ObLobMetaUtil::LOB_ID_COL_ID].set_string(reinterpret_cast<char*>(&in_row.lob_id_), sizeof(ObLobId));
// TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi
datum_row.storage_datums_[ObLobMetaUtil::SEQ_ID_COL_ID].set_string(in_row.seq_id_);
// TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi
datum_row.storage_datums_[ObLobMetaUtil::BYTE_LEN_COL_ID].set_uint32(in_row.byte_len_);
datum_row.storage_datums_[ObLobMetaUtil::CHAR_LEN_COL_ID].set_uint32(in_row.char_len_);
datum_row.storage_datums_[ObLobMetaUtil::PIECE_ID_COL_ID].set_uint(in_row.piece_id_);
datum_row.storage_datums_[ObLobMetaUtil::LOB_DATA_COL_ID].set_string(in_row.lob_data_);
// TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi
}
int ObPersistentLobApator::set_lob_piece_row(
char* buf,
size_t buf_len,
ObObj* cell,
ObNewRow& new_row,
common::ObSingleRowIteratorWrapper* new_row_iter,
ObDatumRow& datum_row,
blocksstable::ObSingleDatumRowIteratorWrapper* new_row_iter,
ObLobPieceInfo& in_row)
{
int ret = OB_SUCCESS;
for (int64_t i = 0; i < ObLobPieceUtil::LOB_PIECE_COLUMN_CNT; ++i) {
cell[i].reset();
cell[i].set_nop_value();
}
cell[0].set_uint64(in_row.piece_id_);
cell[1].set_uint32(in_row.len_);
datum_row.reuse();
datum_row.storage_datums_[0].set_uint(in_row.piece_id_);
datum_row.storage_datums_[1].set_uint32(in_row.len_);
int64_t pos = 0;
if (!in_row.macro_id_.is_valid()) {
@ -868,11 +854,9 @@ int ObPersistentLobApator::set_lob_piece_row(
} else if (OB_FAIL(in_row.macro_id_.serialize(buf, buf_len, pos))) {
LOG_WARN("failed to serialize macro id", K(ret), K(buf_len), K(pos));
} else {
cell[2].set_varchar(buf, pos);
cell[2].set_collation_type(common::ObCollationType::CS_TYPE_BINARY);
new_row.assign(cell, ObLobPieceUtil::LOB_PIECE_COLUMN_CNT);
new_row_iter->set_row(&new_row);
datum_row.storage_datums_[2].set_string(buf, pos);
// TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi
new_row_iter->set_row(&datum_row);
}
return ret;
@ -928,20 +912,20 @@ int ObPersistentLobApator::do_scan_lob_meta(
ObTabletHandle lob_meta_tablet;
ObTabletHandle lob_piece_tablet;
if (OB_FAIL(get_lob_tablets(param, data_tablet, lob_meta_tablet, lob_piece_tablet))) {
LOG_WARN("failed to get tablets.", K(ret), K(param));
LOG_WARN("failed to get tablets", K(ret), K(param));
} else {
ObAccessService *oas = MTL(ObAccessService*);
scan_param.tablet_id_ = param.lob_meta_tablet_id_;
scan_param.schema_version_ = lob_meta_tablet.get_obj()->get_tablet_meta().max_sync_storage_schema_version_;
if (OB_ISNULL(oas)) {
ret = OB_ERR_INTERVAL_INVALID;
LOG_WARN("get access service failed.", K(ret));
LOG_WARN("get access service failed", K(ret));
} else if (OB_FAIL(build_common_scan_param(param, param.has_single_chunk(), ObLobMetaUtil::LOB_META_COLUMN_CNT, scan_param))) {
LOG_WARN("build common scan param failed.", K(ret), K(param));
LOG_WARN("build common scan param failed", K(ret), K(param));
} else if (OB_FAIL(prepare_table_param(param, scan_param, true/*is_meta*/))) {
LOG_WARN("prepare lob meta table param failed.", K(ret), K(param));
LOG_WARN("prepare lob meta table param failed", K(ret), K(param));
} else if (OB_FAIL(oas->table_scan(scan_param, meta_iter))) {
LOG_WARN("do table scan falied.", K(ret), K(scan_param), K(param));
LOG_WARN("do table scan falied", K(ret), K(scan_param), K(param));
}
}
return ret;
@ -967,7 +951,7 @@ int ObPersistentLobApator::scan_lob_meta(
} else if (OB_FAIL(param.get_rowkey_range(rowkey_objs, range))) {
LOG_WARN("get_rowkey_range fail", K(ret));
} else if (OB_FAIL(scan_param.key_ranges_.push_back(range))) {
LOG_WARN("failed to push key range.", K(ret), K(scan_param), K(range));
LOG_WARN("failed to push key range", K(ret), K(scan_param), K(range));
} else if (OB_FAIL(do_scan_lob_meta(param, scan_param, meta_iter))) {
LOG_WARN("do_scan_lob_meta fail", K(ret));
}
@ -1004,28 +988,28 @@ int ObPersistentLobApator::set_dml_seq_no(ObLobAccessParam &param)
LOG_DEBUG("dml lob meta with seq no", K(param.dml_base_param_->spec_seq_no_));
} else {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("failed to get seq no from param.", K(ret), K(param));
LOG_WARN("failed to get seq no from param", K(ret), K(param));
}
} else {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid seq no from param.", K(ret), K(param));
LOG_WARN("invalid seq no from param", K(ret), K(param));
}
return ret;
}
int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam &param, ObNewRowIterator& row_iter)
int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam &param, ObDatumRowIterator& row_iter)
{
int ret = OB_SUCCESS;
int64_t affected_rows = 0;
ObAccessService *oas = MTL(ObAccessService*);
if (OB_ISNULL(oas)) {
ret = OB_ERR_INTERVAL_INVALID;
LOG_WARN("get access service failed.", K(ret), KP(oas));
LOG_WARN("get access service failed", K(ret), KP(oas));
} else if (OB_ISNULL(param.tx_desc_)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get tx desc null.", K(ret), K(param));
LOG_WARN("get tx desc null", K(ret), K(param));
} else if (OB_FAIL(prepare_lob_tablet_id(param))) {
LOG_WARN("failed to get tablets.", K(ret), K(param));
LOG_WARN("failed to get tablets", K(ret), K(param));
} else if (OB_FAIL(prepare_lob_meta_dml(param))) {
LOG_WARN("failed to prepare lob meta dml", K(ret));
} else if (OB_FAIL(oas->delete_rows(
@ -1041,19 +1025,19 @@ int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam &param, ObNewRowItera
return ret;
}
int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObNewRowIterator& row_iter)
int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObDatumRowIterator& row_iter)
{
int ret = OB_SUCCESS;
int64_t affected_rows = 0;
ObAccessService *oas = MTL(ObAccessService*);
if (OB_ISNULL(oas)) {
ret = OB_ERR_INTERVAL_INVALID;
LOG_WARN("get access service failed.", K(ret), KP(oas));
LOG_WARN("get access service failed", K(ret), KP(oas));
} else if (OB_ISNULL(param.tx_desc_)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get tx desc null.", K(ret), K(param));
LOG_WARN("get tx desc null", K(ret), K(param));
} else if (OB_FAIL(prepare_lob_tablet_id(param))) {
LOG_WARN("failed to get tablets.", K(ret), K(param));
LOG_WARN("failed to get tablets", K(ret), K(param));
} else if (OB_FAIL(prepare_lob_meta_dml(param))) {
LOG_WARN("failed to prepare lob meta dml", K(ret));
} else if (OB_FAIL(oas->insert_rows(
@ -1069,26 +1053,26 @@ int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObNewRowItera
return ret;
}
int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObNewRowIterator &row_iter)
int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObDatumRowIterator &row_iter)
{
int ret = OB_SUCCESS;
int64_t affected_rows = 0;
ObAccessService *oas = MTL(ObAccessService*);
if (OB_ISNULL(oas)) {
ret = OB_ERR_INTERVAL_INVALID;
LOG_WARN("get access service failed.", K(ret), KP(oas));
LOG_WARN("get access service failed", K(ret), KP(oas));
} else if (OB_ISNULL(param.tx_desc_)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get tx desc null.", K(ret), K(param));
LOG_WARN("get tx desc null", K(ret), K(param));
} else if (OB_FAIL(prepare_lob_tablet_id(param))) {
LOG_WARN("failed to get tablets.", K(ret), K(param));
LOG_WARN("failed to get tablets", K(ret), K(param));
} else if (OB_FAIL(prepare_lob_meta_dml(param))) {
LOG_WARN("failed to prepare lob meta dml", K(ret));
} else {
ObSEArray<uint64_t, 6> update_column_ids;
for (int i = 2; OB_SUCC(ret) && i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) {
if (OB_FAIL(update_column_ids.push_back(OB_APP_MIN_COLUMN_ID + i))) {
LOG_WARN("push column ids failed.", K(ret), K(i));
LOG_WARN("push column ids failed", K(ret), K(i));
}
}
if (OB_FAIL(ret)) {
@ -1110,11 +1094,12 @@ int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObNewRowIter
int ObPersistentLobApator::write_lob_meta(ObLobAccessParam &param, ObLobMetaInfo& row_info)
{
int ret = OB_SUCCESS;
ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT];
ObNewRow new_row;
ObDatumRow new_row;
ObLobPersistInsertSingleRowIter single_iter;
set_lob_meta_row(cell, new_row, row_info);
if (OB_FAIL(single_iter.init(&param, &new_row))) {
if (OB_FAIL(new_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) {
LOG_WARN("failed to init datum row", K(ret));
} else if (FALSE_IT(set_lob_meta_row(new_row, row_info))) {
} else if (OB_FAIL(single_iter.init(&param, &new_row))) {
LOG_WARN("single_iter init fail", K(ret));
} else if (OB_FAIL(write_lob_meta(param, single_iter))) {
LOG_WARN("write_lob_meta fail", K(ret));
@ -1125,11 +1110,12 @@ int ObPersistentLobApator::write_lob_meta(ObLobAccessParam &param, ObLobMetaInfo
int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam &param, ObLobMetaInfo& row_info)
{
int ret = OB_SUCCESS;
ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT];
ObNewRow new_row;
ObDatumRow new_row;
ObLobPersistDeleteSingleRowIter single_iter;
set_lob_meta_row(cell, new_row, row_info);
if (OB_FAIL(single_iter.init(&param, &new_row))) {
if (OB_FAIL(new_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) {
LOG_WARN("failed to init datum row", K(ret));
} else if (FALSE_IT(set_lob_meta_row(new_row, row_info))) {
} else if (OB_FAIL(single_iter.init(&param, &new_row))) {
LOG_WARN("single_iter init fail", K(ret));
} else if (OB_FAIL(erase_lob_meta(param, single_iter))) {
LOG_WARN("erase_lob_meta fail", K(ret));
@ -1140,14 +1126,16 @@ int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam &param, ObLobMetaInfo
int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& old_row, ObLobMetaInfo& new_row)
{
int ret = OB_SUCCESS;
ObObj new_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT];
ObNewRow new_tbl_row;
set_lob_meta_row(new_row_cell, new_tbl_row, new_row);
ObObj old_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT];
ObNewRow old_tbl_row;
set_lob_meta_row(old_row_cell, old_tbl_row, old_row);
ObDatumRow new_datum_row;
ObDatumRow old_datum_row;
ObLobPersistUpdateSingleRowIter upd_iter;
if (OB_FAIL(upd_iter.init(&param, &old_tbl_row, &new_tbl_row))) {
if (OB_FAIL(new_datum_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) {
LOG_WARN("failed to init new datum row", K(ret));
} else if (OB_FAIL(old_datum_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) {
LOG_WARN("failed to init old datum row", K(ret));
} else if (FALSE_IT(set_lob_meta_row(new_datum_row, new_row))) {
} else if (FALSE_IT(set_lob_meta_row(old_datum_row, old_row))) {
} else if (OB_FAIL(upd_iter.init(&param, &old_datum_row, &new_datum_row))) {
LOG_WARN("upd_iter init fail", K(ret));
} else if (OB_FAIL(update_lob_meta(param, upd_iter))) {
LOG_WARN("update_lob_meta fail", K(ret));

29
src/storage/lob/ob_lob_persistent_adaptor.h
View File

@ -17,6 +17,7 @@
#include "storage/blocksstable/ob_macro_block_id.h"
#include "ob_i_lob_adaptor.h"
#include "common/row/ob_row_iterator.h"
#include "storage/blocksstable/ob_datum_row_iterator.h"
namespace oceanbase
{
@ -25,22 +26,20 @@ namespace storage
class ObStoreCtxGuard;
class ObLobUpdIterator : public ObNewRowIterator
class ObLobUpdIterator : public blocksstable::ObDatumRowIterator
{
public:
ObLobUpdIterator(ObNewRow *old_row,
ObNewRow *new_row)
ObLobUpdIterator(blocksstable::ObDatumRow *old_row,
blocksstable::ObDatumRow *new_row)
: old_row_(old_row),
new_row_(new_row),
got_old_row_(false),
is_iter_end_(false)
{}
virtual int get_next_row(ObNewRow *&row) override;
virtual int get_next_row() override { return OB_NOT_IMPLEMENT; }
virtual void reset() override {}
virtual int get_next_row(blocksstable::ObDatumRow *&row) override;
private:
ObNewRow *old_row_;
ObNewRow *new_row_;
blocksstable::ObDatumRow *old_row_;
blocksstable::ObDatumRow *new_row_;
bool got_old_row_;
bool is_iter_end_;
};
@ -94,13 +93,12 @@ public:
virtual int update_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& old_row, ObLobMetaInfo& new_row) override;
static void set_lob_meta_row(
ObObj* cell,
ObNewRow& new_row,
blocksstable::ObDatumRow& datum_row,
ObLobMetaInfo& in_row);
int write_lob_meta(ObLobAccessParam &param, ObNewRowIterator& row_iter);
int erase_lob_meta(ObLobAccessParam &param, ObNewRowIterator& row_iter);
int update_lob_meta(ObLobAccessParam& param, ObNewRowIterator &row_iter);
int write_lob_meta(ObLobAccessParam &param, blocksstable::ObDatumRowIterator& row_iter);
int erase_lob_meta(ObLobAccessParam &param, blocksstable::ObDatumRowIterator& row_iter);
int update_lob_meta(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &row_iter);
int build_common_scan_param(
const ObLobAccessParam &param,
@ -159,9 +157,8 @@ private:
int set_lob_piece_row(
char* buf,
size_t buf_len,
ObObj* cell,
ObNewRow& new_row,
common::ObSingleRowIteratorWrapper* new_row_iter,
blocksstable::ObDatumRow& datum_row,
blocksstable::ObSingleDatumRowIteratorWrapper* new_row_iter,
ObLobPieceInfo& in_row);
int init_table_param();

24
src/storage/lob/ob_lob_persistent_iterator.cpp
View File

@ -65,7 +65,7 @@ int ObLobPersistWriteIter::dec_lob_size(ObLobMetaInfo &info)
return ret;
}
int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, ObNewRow *old_row, ObNewRow *new_row)
int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, blocksstable::ObDatumRow *old_row, blocksstable::ObDatumRow *new_row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(param) || OB_ISNULL(old_row) || OB_ISNULL(new_row)) {
@ -79,7 +79,7 @@ int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, ObNewRow *old
return ret;
}
int ObLobPersistUpdateSingleRowIter::get_next_row(ObNewRow *&row)
int ObLobPersistUpdateSingleRowIter::get_next_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(old_row_) || OB_ISNULL(new_row_)) {
@ -100,7 +100,7 @@ int ObLobPersistUpdateSingleRowIter::get_next_row(ObNewRow *&row)
return ret;
}
int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row)
int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, blocksstable::ObDatumRow *row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(param) || OB_ISNULL(row)) {
@ -113,7 +113,7 @@ int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row
return ret;
}
int ObLobPersistInsertSingleRowIter::get_next_row(ObNewRow *&row)
int ObLobPersistInsertSingleRowIter::get_next_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(param_) || OB_ISNULL(row_)) {
@ -130,7 +130,7 @@ int ObLobPersistInsertSingleRowIter::get_next_row(ObNewRow *&row)
return ret;
}
int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row)
int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, blocksstable::ObDatumRow *row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(param) || OB_ISNULL(row)) {
@ -143,7 +143,7 @@ int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row
return ret;
}
int ObLobPersistDeleteSingleRowIter::get_next_row(ObNewRow *&row)
int ObLobPersistDeleteSingleRowIter::get_next_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(param_) || OB_ISNULL(row_)) {
@ -166,6 +166,8 @@ int ObLobPersistInsertIter::init(ObLobAccessParam *param, ObLobMetaWriteIter *me
if (OB_ISNULL(param) || OB_ISNULL(meta_iter)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("param or meta_iter is null", K(ret), KP(param), KP(meta_iter));
} else if (OB_FAIL(new_row_.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) {
LOG_WARN("init new datum row failed", K(ret));
} else {
param_ = param;
meta_iter_ = meta_iter;
@ -173,7 +175,7 @@ int ObLobPersistInsertIter::init(ObLobAccessParam *param, ObLobMetaWriteIter *me
return ret;
}
int ObLobPersistInsertIter::get_next_row(ObNewRow *&row)
int ObLobPersistInsertIter::get_next_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (OB_FAIL(meta_iter_->get_next_row(result_))) {
@ -188,7 +190,7 @@ int ObLobPersistInsertIter::get_next_row(ObNewRow *&row)
} else if (OB_FAIL(inc_lob_size(result_.info_))) {
LOG_WARN("inc_lob_size fail", K(ret));
} else {
ObPersistentLobApator::set_lob_meta_row(row_cell_, new_row_, result_.info_);
ObPersistentLobApator::set_lob_meta_row(new_row_, result_.info_);
row = &new_row_;
}
return ret;
@ -200,6 +202,8 @@ int ObLobPersistDeleteIter::init(ObLobAccessParam *param, ObLobMetaScanIter *met
if (OB_ISNULL(param) || OB_ISNULL(meta_iter)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("param or meta_iter is null", K(ret), KP(param), KP(meta_iter));
} else if (OB_FAIL(new_row_.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) {
LOG_WARN("init new datum row failed", K(ret));
} else {
param_ = param;
meta_iter_ = meta_iter;
@ -207,7 +211,7 @@ int ObLobPersistDeleteIter::init(ObLobAccessParam *param, ObLobMetaScanIter *met
return ret;
}
int ObLobPersistDeleteIter::get_next_row(ObNewRow *&row)
int ObLobPersistDeleteIter::get_next_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (OB_FAIL(meta_iter_->get_next_row(result_))) {
@ -221,7 +225,7 @@ int ObLobPersistDeleteIter::get_next_row(ObNewRow *&row)
} else if (OB_FAIL(dec_lob_size(result_.info_))) {
LOG_WARN("dec_lob_size fail", K(ret));
} else {
ObPersistentLobApator::set_lob_meta_row(row_cell_, new_row_, result_.info_);
ObPersistentLobApator::set_lob_meta_row(new_row_, result_.info_);
row = &new_row_;
}
return ret;

41
src/storage/lob/ob_lob_persistent_iterator.h
View File

@ -12,19 +12,22 @@
#ifndef OCEABASE_STORAGE_OB_LOB_PERSISTENT_ITERATOR_
#define OCEABASE_STORAGE_OB_LOB_PERSISTENT_ITERATOR_
#include "storage/blocksstable/ob_datum_row_iterator.h"
#include "storage/lob/ob_lob_util.h"
#include "storage/lob/ob_lob_meta.h"
namespace oceanbase
{
namespace storage
{
class ObLobPersistWriteIter : public ObNewRowIterator
class ObLobPersistWriteIter : public blocksstable::ObDatumRowIterator
{
public:
ObLobPersistWriteIter(): param_(nullptr) {}
virtual ~ObLobPersistWriteIter() {}
virtual int get_next_row() override { return OB_NOT_IMPLEMENT; }
virtual int get_next_row(blocksstable::ObDatumRow *&row) override { return OB_NOT_IMPLEMENT; }
protected:
int update_seq_no();
@ -44,10 +47,10 @@ public:
row_(nullptr),
iter_end_(false)
{}
int init(ObLobAccessParam *param, ObNewRow *row);
int init(ObLobAccessParam *param, blocksstable::ObDatumRow *row);
virtual ~ObLobPersistInsertSingleRowIter() {}
virtual int get_next_row(ObNewRow *&row);
virtual int get_next_row(blocksstable::ObDatumRow *&row);
virtual void reset() { iter_end_ = false; }
private:
@ -55,7 +58,7 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObLobPersistInsertSingleRowIter);
private:
// data members
ObNewRow *row_;
blocksstable::ObDatumRow *row_;
bool iter_end_;
};
@ -67,10 +70,10 @@ public:
row_(nullptr),
iter_end_(false)
{}
int init(ObLobAccessParam *param, ObNewRow *row);
int init(ObLobAccessParam *param, blocksstable::ObDatumRow *row);
virtual ~ObLobPersistDeleteSingleRowIter() {}
virtual int get_next_row(ObNewRow *&row);
virtual int get_next_row(blocksstable::ObDatumRow *&row);
virtual void reset() { iter_end_ = false; }
private:
@ -78,7 +81,7 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObLobPersistDeleteSingleRowIter);
private:
// data members
ObNewRow *row_;
blocksstable::ObDatumRow *row_;
bool iter_end_;
};
@ -94,9 +97,9 @@ public:
virtual ~ObLobPersistUpdateSingleRowIter() {}
int init(ObLobAccessParam *param, ObNewRow *old_row, ObNewRow *new_row);
int init(ObLobAccessParam *param, blocksstable::ObDatumRow *old_row, blocksstable::ObDatumRow *new_row);
virtual int get_next_row(ObNewRow *&row) override;
virtual int get_next_row(blocksstable::ObDatumRow *&row) override;
virtual void reset() override {}
private:
@ -104,8 +107,8 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObLobPersistUpdateSingleRowIter);
private:
ObNewRow *old_row_;
ObNewRow *new_row_;
blocksstable::ObDatumRow *old_row_;
blocksstable::ObDatumRow *new_row_;
bool got_old_row_;
bool is_iter_end_;
};
@ -114,10 +117,10 @@ private:
class ObLobPersistInsertIter: public ObLobPersistWriteIter
{
public:
ObLobPersistInsertIter() : meta_iter_(nullptr), new_row_(), row_cell_(), result_() {}
ObLobPersistInsertIter() : meta_iter_(nullptr), new_row_(), result_() {}
int init(ObLobAccessParam *param, ObLobMetaWriteIter *meta_iter);
virtual ~ObLobPersistInsertIter() {}
virtual int get_next_row(ObNewRow *&row);
virtual int get_next_row(blocksstable::ObDatumRow *&row);
virtual void reset() { new_row_.reset(); }
private:
@ -126,8 +129,7 @@ private:
private:
// data members
ObLobMetaWriteIter *meta_iter_;
ObNewRow new_row_;
ObObj row_cell_[ObLobMetaUtil::LOB_META_COLUMN_CNT];
blocksstable::ObDatumRow new_row_;
ObLobMetaWriteResult result_;
};
@ -135,10 +137,10 @@ private:
class ObLobPersistDeleteIter: public ObLobPersistWriteIter
{
public:
ObLobPersistDeleteIter() : meta_iter_(nullptr), new_row_(), row_cell_(), result_() {}
ObLobPersistDeleteIter() : meta_iter_(nullptr), new_row_(), result_() {}
int init(ObLobAccessParam *param, ObLobMetaScanIter *meta_iter);
virtual ~ObLobPersistDeleteIter() {}
virtual int get_next_row(ObNewRow *&row);
virtual int get_next_row(blocksstable::ObDatumRow *&row);
virtual void reset() { new_row_.reset(); }
@ -148,8 +150,7 @@ private:
private:
// data members
ObLobMetaScanIter *meta_iter_;
ObNewRow new_row_;
ObObj row_cell_[ObLobMetaUtil::LOB_META_COLUMN_CNT];
blocksstable::ObDatumRow new_row_;
ObLobMetaScanResult result_;
};

1101
src/storage/ls/ob_ls_tablet_service.cpp
View File

File diff suppressed because it is too large Load Diff

147
src/storage/ls/ob_ls_tablet_service.h
View File

@ -33,6 +33,7 @@
#include "storage/tablet/ob_tablet_persister.h"
#include "storage/lob/ob_lob_manager.h"
#include "storage/multi_data_source/mds_table_mgr.h"
#include "storage/blocksstable/ob_datum_row_iterator.h"
namespace oceanbase
{
@ -57,12 +58,18 @@ namespace blocksstable
{
class ObMigrationSSTableParam;
struct ObDatumRowkey;
class ObDatumRowStore;
}
namespace compaction
{
class ObTabletMergeCtx;
}
namespace sql
{
class ObDASDMLIterator;
class ObDASUpdIterator;
}
namespace storage
{
@ -317,7 +324,7 @@ public:
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
int insert_row(
ObTabletHandle &tablet_handle,
@ -325,31 +332,31 @@ public:
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObIArray<uint64_t> &duplicated_column_ids,
const common::ObNewRow &row,
blocksstable::ObDatumRow &row,
const ObInsertFlag flag,
int64_t &affected_rows,
common::ObNewRowIterator *&duplicated_rows);
blocksstable::ObDatumRowIterator *&duplicated_rows);
int update_rows(
ObTabletHandle &tablet_handle,
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const ObIArray<uint64_t> &column_ids,
const ObIArray< uint64_t> &updated_column_ids,
ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
int put_rows(
ObTabletHandle &tablet_handle,
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const ObIArray<uint64_t> &column_ids,
ObNewRowIterator *row_iter,
ObDatumRowIterator *row_iter,
int64_t &affected_rows); // for htable, insert or update
int delete_rows(
ObTabletHandle &tablet_handle,
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const ObIArray<uint64_t> &column_ids,
ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
int lock_rows(
ObTabletHandle &tablet_handle,
@ -357,13 +364,13 @@ public:
const ObDMLBaseParam &dml_param,
const ObLockFlag lock_flag,
const bool is_sfu,
ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
int lock_row(
ObTabletHandle &tablet_handle,
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const ObNewRow &row,
blocksstable::ObDatumRow &row,
const ObLockFlag lock_flag,
const bool is_sfu);
int get_multi_ranges_cost(
@ -553,7 +560,7 @@ private:
int offline_build_tablet_without_memtable_();
int offline_gc_tablet_for_create_or_transfer_in_abort_();
int offline_destroy_memtable_and_mds_table_();
int mock_duplicated_rows_(common::ObNewRowIterator *&duplicated_rows);
int mock_duplicated_rows_(blocksstable::ObDatumRowIterator *&duplicated_rows);
private:
static int check_real_leader_for_4377_(const ObLSID ls_id);
static int check_need_rollback_in_transfer_for_4377_(const transaction::ObTxDesc *tx_desc,
@ -562,50 +569,42 @@ private:
static int build_create_sstable_param_for_migration(
const blocksstable::ObMigrationSSTableParam &migrate_sstable_param,
ObTabletCreateSSTableParam &create_sstable_param);
static int need_check_old_row_legitimacy(
ObDMLRunningCtx &run_ctx,
bool &need_check,
bool &is_udf);
static int construct_table_rows(
const ObNewRow *rows,
ObStoreRow *tbl_rows,
int64_t row_count);
static int check_old_row_legitimacy(
const blocksstable::ObStoreCmpFuncs &cmp_funcs,
ObTabletHandle &data_tablet_handle,
ObDMLRunningCtx &run_ctx,
const common::ObNewRow &old_row);
blocksstable::ObDatumRow &old_row);
static int check_new_row_legitimacy(
ObDMLRunningCtx &run_ctx,
const common::ObNewRow &new_row);
const blocksstable::ObDatumRow &datum_row);
static int insert_rows_to_tablet(
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
const common::ObNewRow *const rows,
blocksstable::ObDatumRow *rows,
const int64_t row_count,
ObRowsInfo &rows_info,
storage::ObStoreRow *tbl_rows,
int64_t &afct_num,
int64_t &dup_num);
static int insert_tablet_rows(
const int64_t row_count,
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
ObStoreRow *rows,
blocksstable::ObDatumRow *rows,
ObRowsInfo &rows_info);
static int insert_lob_col(
ObDMLRunningCtx &run_ctx,
const ObColDesc &column,
ObObj &obj,
blocksstable::ObStorageDatum &datum,
ObLobAccessParam *del_param,
ObLobCommon *lob_common);
static int insert_lob_tablet_row(
ObTabletHandle &data_tablet,
ObDMLRunningCtx &run_ctx,
ObStoreRow &row);
blocksstable::ObDatumRow &datum_row);
static int insert_lob_tablet_rows(
ObTabletHandle &data_tablet,
ObDMLRunningCtx &run_ctx,
ObStoreRow *rows,
blocksstable::ObDatumRow *rows,
int64_t row_count);
static int extract_rowkey(
const ObRelativeTable &table,
@ -620,8 +619,8 @@ private:
const int64_t buffer_len,
const common::ObTimeZoneInfo *tz_info = nullptr);
static int get_next_rows(
ObNewRowIterator *row_iter,
ObNewRow *&rows,
blocksstable::ObDatumRowIterator *row_iter,
blocksstable::ObDatumRow *&rows,
int64_t &row_count);
static int construct_update_idx(
const int64_t schema_rowkey_cnt,
@ -634,19 +633,21 @@ private:
bool &rowkey_change,
bool &delay_new);
static int check_rowkey_value_change(
const common::ObNewRow &old_row,
const common::ObNewRow &new_row,
const ObDatumRow &old_row,
const ObDatumRow &new_row,
const int64_t rowkey_len,
const blocksstable::ObStorageDatumUtils &rowkey_datum_utils,
bool &rowkey_change);
static int process_delta_lob(
ObDMLRunningCtx &run_ctx,
const ObColDesc &column,
ObObj &old_obj,
ObStorageDatum &old_datum,
ObLobLocatorV2 &delta_lob,
ObObj &obj);
ObStorageDatum &datum);
static int register_ext_info_commit_cb(
ObDMLRunningCtx &run_ctx,
ObObj &col_data,
ObStorageDatum &col_data,
ObObjType type,
ObObj &ext_info_data);
static int set_lob_storage_params(
ObDMLRunningCtx &run_ctx,
@ -658,9 +659,8 @@ private:
ObDMLRunningCtx &run_ctx,
const ObIArray<int64_t> &update_idx,
bool data_tbl_rowkey_change,
ObStoreRow &old_sql_row,
ObStoreRow &old_row,
ObStoreRow &new_row);
ObDatumRow &old_row,
ObDatumRow &new_row);
static int update_row_to_tablet(
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
@ -668,42 +668,39 @@ private:
const ObIArray<int64_t> &update_idx,
const bool delay_new,
const bool lob_update,
ObStoreRow &old_tbl_row,
ObStoreRow &new_tbl_row,
ObRowStore *row_store,
ObDatumRow &old_row,
ObDatumRow &new_row,
ObDatumRowStore *row_store,
bool &duplicate);
static int process_old_row(
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
const bool data_tbl_rowkey_change,
const bool lob_update,
ObStoreRow &tbl_row);
ObDatumRow &datum_row);
static int process_new_row(
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
const common::ObIArray<int64_t> &update_idx,
const ObStoreRow &old_tbl_row,
const ObStoreRow &new_tbl_row,
const ObDatumRow &old_datum_row,
ObDatumRow &new_datum_row,
const bool rowkey_change);
static int process_data_table_row(
ObTabletHandle &data_tablet,
ObDMLRunningCtx &run_ctx,
const ObIArray<int64_t> &update_idx,
const ObStoreRow &old_tbl_row,
const ObStoreRow &new_tbl_row,
const ObDatumRow &old_datum_row,
ObDatumRow &new_datum_row,
const bool rowkey_change);
static int check_new_row_nullable_value(
const ObIArray<uint64_t> &column_ids,
ObRelativeTable &data_table,
const ObNewRow &new_row);
static int check_new_row_nullable_value(
static int check_datum_row_nullable_value(
const common::ObIArray<share::schema::ObColDesc> &col_descs,
ObRelativeTable &relative_table,
const common::ObNewRow &new_row);
static int check_new_row_shadow_pk(
const blocksstable::ObDatumRow &datum_row);
static int check_datum_row_shadow_pk(
const ObIArray<uint64_t> &column_ids,
ObRelativeTable &data_table,
const ObNewRow &new_row);
const blocksstable::ObDatumRow &datum_row,
const blocksstable::ObStorageDatumUtils &rowkey_datum_utils);
static int check_row_locked_by_myself(
ObTabletHandle &tablet_handle,
ObRelativeTable &relative_table,
@ -715,61 +712,55 @@ private:
ObDMLRunningCtx &run_ctx,
const ObInsertFlag flag,
const common::ObIArray<uint64_t> &out_col_ids,
const common::ObNewRow &row,
common::ObNewRowIterator *&duplicated_rows);
blocksstable::ObDatumRow &row,
blocksstable::ObDatumRowIterator *&duplicated_rows);
static int init_single_row_getter(
ObSingleRowGetter &row_getter,
ObDMLRunningCtx &run_ctx,
const ObIArray<uint64_t> &out_col_ids,
ObRelativeTable &relative_table,
bool skip_read_lob = false);
static int single_get_row(
ObSingleRowGetter &row_getter,
const blocksstable::ObDatumRowkey &rowkey,
ObNewRowIterator *&duplicated_rows);
static int convert_row_to_rowkey(
ObSingleRowGetter &index_row_getter,
ObStoreRowkey &rowkey);
static int add_duplicate_row(
ObDatumRow *storage_row,
const blocksstable::ObStorageDatumUtils &rowkey_datum_utils,
blocksstable::ObDatumRowIterator *&duplicated_rows);
static int get_next_row_from_iter(
ObNewRowIterator *row_iter,
ObStoreRow &store_row,
blocksstable::ObDatumRowIterator *row_iter,
ObDatumRow &datum_row,
const bool need_copy_cells);
static int insert_row_to_tablet(
const bool check_exist,
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
ObStoreRow &tbl_row);
static int process_old_row_lob_col(
blocksstable::ObDatumRow &datum_row);
static int process_old_row_lob_col(
ObTabletHandle &data_tablet_handle,
ObDMLRunningCtx &run_ctx,
ObStoreRow &tbl_row);
blocksstable::ObDatumRow &datum_row);
static int table_refresh_row(
ObTabletHandle &data_tablet_handle,
ObDMLRunningCtx &run_ctx,
ObNewRow &row);
blocksstable::ObDatumRow &datum_row);
static int delete_row_in_tablet(
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
const ObNewRow &row);
blocksstable::ObDatumRow &datum_row);
static int delete_lob_col(
ObDMLRunningCtx &run_ctx,
const ObColDesc &column,
ObObj &obj,
const ObObj &sql_obj,
blocksstable::ObStorageDatum &datum,
ObLobCommon *&lob_common,
ObLobAccessParam &lob_param);
static int delete_lob_tablet_rows(
ObDMLRunningCtx &run_ctx,
ObTabletHandle &data_tablet,
ObStoreRow &tbl_row,
const ObNewRow &row);
blocksstable::ObDatumRow &datum_row);
static int prepare_scan_table_param(
ObTableScanParam &param,
share::schema::ObMultiVersionSchemaService &schema_service);
static void dump_diag_info_for_old_row_loss(
ObRelativeTable &data_table,
ObStoreCtx &store_ctx,
const ObStoreRow &tbl_row);
ObDMLRunningCtx &run_ctx,
blocksstable::ObDatumRow &datum_row);
int set_allow_to_read_(ObLS *ls);
// TODO(chenqingxiang.cqx): remove this
int create_empty_shell_tablet(
@ -786,8 +777,14 @@ private:
const int64_t ddl_task_id,
const common::ObTabletID &tablet_id,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
static int get_storage_row(const ObDatumRow &sql_row,
const ObIArray<uint64_t> &column_ids,
ObSingleRowGetter &row_getter,
ObDMLRunningCtx &run_ctx,
ObDatumRow *&out_row,
bool use_fuse_row_cache = false);
static int check_is_gencol_check_failed(const ObRelativeTable &data_table,
uint64_t error_col_id,
bool &is_virtual_gencol);

5
src/storage/memtable/mvcc/ob_mvcc_ctx.cpp
View File

@ -269,12 +269,13 @@ int ObIMvccCtx::register_ext_info_commit_cb(
const blocksstable::ObDmlFlag dml_flag,
transaction::ObTxDesc *tx_desc,
transaction::ObTxSEQ &parent_seq_no,
ObObj &index_data,
blocksstable::ObStorageDatum &index_data,
ObObjType &type,
ObObj &ext_info_data)
{
int ret = OB_SUCCESS;
storage::ObExtInfoCbRegister cb_register;
if (OB_FAIL(cb_register.register_cb(this, timeout, dml_flag, tx_desc, parent_seq_no, index_data, ext_info_data))) {
if (OB_FAIL(cb_register.register_cb(this, timeout, dml_flag, tx_desc, parent_seq_no, index_data, type, ext_info_data))) {
TRANS_LOG(ERROR, "register ext info callback failed", K(ret), K(cb_register), K(*this));
}
return ret;

3
src/storage/memtable/mvcc/ob_mvcc_ctx.h
View File

@ -178,7 +178,8 @@ public:
const blocksstable::ObDmlFlag dml_flag,
transaction::ObTxDesc *tx_desc,
transaction::ObTxSEQ &parent_seq_no,
ObObj &index_data,
blocksstable::ObStorageDatum &index_data,
ObObjType &type,
ObObj &ext_info_data);
public:
virtual void reset()

2
src/storage/memtable/ob_concurrent_control.cpp
View File

@ -68,7 +68,7 @@ int check_sequence_set_violation(const concurrent_control::ObWriteFlag write_fla
ret = OB_ERR_PRIMARY_KEY_DUPLICATE;
TRANS_LOG(WARN, "pdml duplicate primary key found", K(ret),
K(writer_tx_id), K(writer_dml_flag), K(writer_seq_no),
K(locker_tx_id), K(locker_dml_flag), K(locker_seq_no));
K(locker_tx_id), K(locker_dml_flag), K(locker_seq_no), K(reader_seq_no));
// Case 2.1: For the case of the update in the storage layer, it may be
// split into lock and update in a single statement and fail the check, so
// we need bypass this case(Currently only the update of the lob will cause

200
src/storage/memtable/ob_memtable.cpp
View File

@ -373,7 +373,7 @@ int ObMemtable::multi_set(
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
const common::ObIArray<share::schema::ObColDesc> &columns,
const storage::ObStoreRow *rows,
blocksstable::ObDatumRow *rows,
const int64_t row_count,
const bool check_exist,
const share::ObEncryptMeta *encrypt_meta,
@ -381,7 +381,6 @@ int ObMemtable::multi_set(
{
int ret = OB_SUCCESS;
ObMvccWriteGuard guard(ret);
ObMemtableKeyGenerator mtk_generator;
if (IS_NOT_INIT) {
TRANS_LOG(WARN, "Not inited", K(*this));
ret = OB_NOT_INIT;
@ -389,7 +388,7 @@ int ObMemtable::multi_set(
ret = OB_INVALID_ARGUMENT;
TRANS_LOG(WARN, "Invalid argument", K(ret), K(param), K(context));
} else if (OB_UNLIKELY(nullptr == context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()
|| param.get_schema_rowkey_count() > columns.count())) {
|| param.get_schema_rowkey_count() > columns.count())) {
ret = OB_INVALID_ARGUMENT;
TRANS_LOG(WARN, "Invalid param", K(ret), K(param), K(columns.count()));
#ifdef OB_BUILD_TDE_SECURITY
@ -400,8 +399,6 @@ int ObMemtable::multi_set(
} else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) {
TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret));
#endif
} else if (OB_FAIL(mtk_generator.init(rows, row_count, param.get_schema_rowkey_count(), columns))) {
TRANS_LOG(WARN, "fail to generate memtable keys", KPC(encrypt_meta), K(*context.store_ctx_), KR(ret));
}
if (OB_FAIL(ret)) {
@ -410,17 +407,22 @@ int ObMemtable::multi_set(
} else {
lib::CompatModeGuard compat_guard(mode_);
if (row_count > 1) {
ret = multi_set_(param, columns, rows, row_count, check_exist, mtk_generator, context, rows_info);
ret = multi_set_(param, columns, rows, row_count, check_exist, context, rows_info);
} else {
ret = set_(param,
columns,
rows[0],
nullptr, /*old_row*/
nullptr, /*update_idx*/
mtk_generator[0],
check_exist,
context,
nullptr /*mvcc_row*/);
ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns);
if (OB_FAIL(memtable_key_generator.init())) {
TRANS_LOG(WARN, "fail to init memtable key generator", K(ret));
} else {
ret = set_(param,
columns,
rows[0],
nullptr, /*old_row*/
nullptr, /*update_idx*/
check_exist,
context,
memtable_key_generator,
nullptr /*mvcc_row*/);
}
}
guard.set_memtable(this);
}
@ -529,24 +531,23 @@ int ObMemtable::set(
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
const common::ObIArray<share::schema::ObColDesc> &columns,
const storage::ObStoreRow &row,
blocksstable::ObDatumRow &row,
const share::ObEncryptMeta *encrypt_meta,
const bool check_exist)
{
int ret = OB_SUCCESS;
ObMvccWriteGuard guard(ret);
ObMemtableKeyGenerator mtk_generator;
if (IS_NOT_INIT) {
TRANS_LOG(WARN, "not init", K(*this));
ret = OB_NOT_INIT;
} else if (!param.is_valid() || !context.is_valid()) {
ret = OB_INVALID_ARGUMENT;
TRANS_LOG(WARN, "invalid argument, ", K(ret), K(param), K(context));
ret = OB_INVALID_ARGUMENT;
TRANS_LOG(WARN, "invalid argument, ", K(ret), K(param), K(context));
} else if (NULL == context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()
|| param.get_schema_rowkey_count() > columns.count()
|| row.row_val_.count_ < columns.count()) {
|| param.get_schema_rowkey_count() > columns.count()
|| row.count_ < columns.count()) {
TRANS_LOG(WARN, "invalid param", K(param),
K(columns.count()), K(row.row_val_.count_));
K(columns.count()), K(row.count_));
ret = OB_INVALID_ARGUMENT;
#ifdef OB_BUILD_TDE_SECURITY
//TODO: table_id is just used as encrypt_index, we may rename it in the future.
@ -556,32 +557,28 @@ int ObMemtable::set(
} else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) {
TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret));
#endif
} else if (OB_FAIL(mtk_generator.init(&row, 1, param.get_schema_rowkey_count(), columns))) {
TRANS_LOG(WARN, "fail to generate memtable keys", KPC(encrypt_meta), K(*context.store_ctx_), KR(ret));
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(guard.write_auth(*context.store_ctx_))) {
TRANS_LOG(WARN, "not allow to write", K(*context.store_ctx_));
} else {
lib::CompatModeGuard compat_guard(mode_);
ret = set_(param,
columns,
row,
NULL, /*old_row*/
NULL, /*update_idx*/
mtk_generator[0],
check_exist,
context,
nullptr /*mvcc_row*/);
guard.set_memtable(this);
}
if (OB_SUCC(ret)) {
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(try_report_dml_stat_(param.table_id_))) {
TRANS_LOG_RET(WARN, tmp_ret, "fail to report dml stat", K_(reported_dml_stat));
ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns);
if (OB_FAIL(memtable_key_generator.init())) {
TRANS_LOG(WARN, "fail to init memtable key generator", K(ret));
} else {
lib::CompatModeGuard compat_guard(mode_);
ret = set_(param,
columns,
row,
NULL, /*old_row*/
NULL, /*update_idx*/
check_exist,
context,
memtable_key_generator,
nullptr /*mvcc_row*/);
TRANS_LOG(WARN, "[xuanxi] set row", K(ret), K(row), K(check_exist), K(memtable_key_generator.get_memtable_key()));
guard.set_memtable(this);
}
}
return ret;
@ -592,13 +589,12 @@ int ObMemtable::set(
storage::ObTableAccessContext &context,
const ObIArray<ObColDesc> &columns,
const ObIArray<int64_t> &update_idx,
const storage::ObStoreRow &old_row,
const storage::ObStoreRow &new_row,
const blocksstable::ObDatumRow &old_row,
blocksstable::ObDatumRow &new_row,
const share::ObEncryptMeta *encrypt_meta)
{
int ret = OB_SUCCESS;
ObMvccWriteGuard guard(ret);
ObMemtableKeyGenerator mtk_generator;
if (IS_NOT_INIT) {
TRANS_LOG(WARN, "not init", K(*this));
ret = OB_NOT_INIT;
@ -606,7 +602,7 @@ int ObMemtable::set(
ret = OB_INVALID_ARGUMENT;
TRANS_LOG(WARN, "invalid argument, ", K(ret), K(param), K(context));
} else if (NULL == context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()
|| param.get_schema_rowkey_count() > columns.count()) {
|| param.get_schema_rowkey_count() > columns.count()) {
ret = OB_INVALID_ARGUMENT;
TRANS_LOG(ERROR, "invalid param", K(ret), K(param));
#ifdef OB_BUILD_TDE_SECURITY
@ -617,32 +613,28 @@ int ObMemtable::set(
} else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) {
TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret));
#endif
} else if (OB_FAIL(mtk_generator.init(&new_row, 1, param.get_schema_rowkey_count(), columns))) {
TRANS_LOG(WARN, "fail to generate memtable keys", KPC(encrypt_meta), K(*context.store_ctx_), KR(ret));
}
if (OB_FAIL(ret)){
} else if (OB_FAIL(guard.write_auth(*context.store_ctx_))) {
TRANS_LOG(WARN, "not allow to write", K(*context.store_ctx_));
} else {
lib::CompatModeGuard compat_guard(mode_);
ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns);
if (OB_FAIL(memtable_key_generator.init())) {
TRANS_LOG(WARN, "fail to init memtable key generator", K(ret));
} else {
lib::CompatModeGuard compat_guard(mode_);
ret = set_(param,
columns,
new_row,
&old_row,
&update_idx,
mtk_generator[0],
false/*check_exist*/,
context,
nullptr /*mvcc_row*/);
guard.set_memtable(this);
}
if (OB_SUCC(ret)) {
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(try_report_dml_stat_(param.table_id_))) {
TRANS_LOG_RET(WARN, tmp_ret, "fail to report dml stat", K_(reported_dml_stat));
ret = set_(param,
columns,
new_row,
&old_row,
&update_idx,
false/*check_exist*/,
context,
memtable_key_generator,
nullptr /*mvcc_row*/);
guard.set_memtable(this);
}
}
return ret;
@ -651,10 +643,13 @@ int ObMemtable::set(
int ObMemtable::lock(
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
const common::ObNewRow &row)
ObColDescArray &col_desc,
blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
ObMvccWriteGuard guard(ret);
ObDatumRowkeyHelper rowkey_helper;
ObDatumRowkey datum_rowkey(row.storage_datums_, param.get_schema_rowkey_count());
ObStoreRowkey tmp_key;
ObMemtableKey mtk;
ObMvccAccessCtx &acc_ctx = context.store_ctx_->mvcc_acc_ctx_;
@ -670,9 +665,9 @@ int ObMemtable::lock(
// actually, there is no circumstance in where locking the index table is need.
ret = OB_NOT_SUPPORTED;
TRANS_LOG(WARN, "locking the non-unique local index is not supported", K(ret), K(row), K(param));
} else if (OB_FAIL(tmp_key.assign(row.cells_, param.get_schema_rowkey_count()))) {
TRANS_LOG(WARN, "Failed to assign rowkey", K(row), K(param));
} else if (OB_FAIL(mtk.encode(param.get_read_info()->get_columns_desc(), &tmp_key))) {
} else if (OB_FAIL(rowkey_helper.convert_store_rowkey(datum_rowkey, col_desc, tmp_key))) {
LOG_WARN("Failed to convert store rowkey from datum rowkey", K(ret), K(row), K(datum_rowkey));
} else if (OB_FAIL(mtk.encode(col_desc, &tmp_key))) {
TRANS_LOG(WARN, "encode mtk failed", K(ret), K(param));
} else if (acc_ctx.write_flag_.is_check_row_locked()) {
if (OB_FAIL(ObRowConflictHandler::check_foreign_key_constraint(param, context, tmp_key))) {
@ -1589,7 +1584,6 @@ void ObMemtable::lock_row_on_frozen_stores_on_failure(
int ObMemtable::lock_rows_on_frozen_stores_(
const bool check_exist,
const storage::ObTableIterParam &param,
const ObMemtableKeyGenerator &memtable_keys,
storage::ObTableAccessContext &context,
ObMvccRowAndWriteResults &mvcc_rows,
ObRowsInfo &rows_info)
@ -2499,10 +2493,9 @@ bool ObMemtable::need_for_save(const share::ObEncryptMeta *encrypt_meta)
int ObMemtable::multi_set_(
const storage::ObTableIterParam &param,
const common::ObIArray<share::schema::ObColDesc> &columns,
const storage::ObStoreRow *rows,
const blocksstable::ObDatumRow *rows,
const int64_t row_count,
const bool check_exist,
const ObMemtableKeyGenerator &memtable_keys,
storage::ObTableAccessContext &context,
storage::ObRowsInfo &rows_info)
{
@ -2511,8 +2504,13 @@ int ObMemtable::multi_set_(
int64_t conflict_idx = -1;
int64_t row_size_stat = 0;
ObMvccRowAndWriteResults mvcc_rows;
// TODO(xuanxi): remove it later
ObMemtableKeyGenerator::ObMemtableKeyBuffer memtable_key_buffer;
ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns, &memtable_key_buffer);
if (OB_FAIL(mvcc_rows.prepare_allocate(row_count))) {
TRANS_LOG(WARN, "Failed to prepare allocate mvcc rows", K(ret), K(row_count));
} else if (OB_FAIL(memtable_key_generator.init())) {
TRANS_LOG(WARN, "fail to init memtable key generator", K(ret));
}
// 1. Check write conflict in memtables.
@ -2524,9 +2522,9 @@ int ObMemtable::multi_set_(
rows[i],
nullptr, /*old_row*/
nullptr, /*update_idx*/
memtable_keys[i],
check_exist,
context,
memtable_key_generator,
&(mvcc_rows[permutation_idx])))) {
if (OB_UNLIKELY(OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret)) {
TRANS_LOG(WARN, "Failed to insert new row", K(ret), K(i), K(permutation_idx), K(rows[i]));
@ -2553,7 +2551,7 @@ int ObMemtable::multi_set_(
// 2. Check uniqueness constraint and write conflict in sstables.
if (OB_FAIL(ret)) {
} else if (rows_info.all_rows_found()) {
} else if (OB_FAIL(lock_rows_on_frozen_stores_(check_exist, param, memtable_keys, context, mvcc_rows, rows_info))) {
} else if (OB_FAIL(lock_rows_on_frozen_stores_(check_exist, param, context, mvcc_rows, rows_info))) {
TRANS_LOG(WARN, "Failed to lock rows on frozen stores", K(ret));
} else if (rows_info.have_conflict()) {
conflict_idx = rows_info.get_conflict_idx();
@ -2579,9 +2577,9 @@ int ObMemtable::multi_set_(
if (param.is_non_unique_local_index_) {
// no need to detect deadlock for non-unique local index table
} else {
for (int64_t idx = 0; idx < memtable_keys.count(); ++idx) {
for (int64_t idx = 0; idx < memtable_key_buffer.count(); ++idx) {
MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(),
memtable_keys[idx],
memtable_key_buffer.at(idx),
context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id());
}
}
@ -2601,12 +2599,12 @@ int ObMemtable::multi_set_(
int ObMemtable::set_(
const storage::ObTableIterParam &param,
const common::ObIArray<share::schema::ObColDesc> &columns,
const storage::ObStoreRow &new_row,
const storage::ObStoreRow *old_row,
const blocksstable::ObDatumRow &new_row,
const blocksstable::ObDatumRow *old_row,
const common::ObIArray<int64_t> *update_idx,
const ObMemtableKey &mtk,
const bool check_exist,
storage::ObTableAccessContext &context,
ObMemtableKeyGenerator &memtable_key_generator,
ObMvccRowAndWriteResult *mvcc_row)
{
int ret = OB_SUCCESS;
@ -2645,11 +2643,11 @@ int ObMemtable::set_(
TRANS_LOG(WARN, "get write seq failed", K(ret));
} else if (OB_FAIL(row_writer.write(param.get_schema_rowkey_count(), new_row, update_idx, buf, len))) {
TRANS_LOG(WARN, "Failed to write new row", K(ret), K(new_row));
} else if (OB_UNLIKELY(new_row.flag_.is_not_exist())) {
} else if (OB_UNLIKELY(new_row.row_flag_.is_not_exist())) {
ret = OB_ERR_UNEXPECTED;
TRANS_LOG(ERROR, "Unexpected not exist trans node", K(ret), K(new_row));
} else {
ObMemtableData mtd(new_row.flag_.get_dml_flag(), len, buf);
ObMemtableData mtd(new_row.row_flag_.get_dml_flag(), len, buf);
ObTxNodeArg arg(
ctx.mvcc_acc_ctx_.tx_id_, /*trans id*/
&mtd, /*memtable_data*/
@ -2657,24 +2655,27 @@ int ObMemtable::set_(
init_timestamp_, /*memstore_version*/
write_seq, /*seq_no*/
write_epoch, /*write_epoch*/
new_row.row_val_.count_ /*column_cnt*/);
if (OB_FAIL(mvcc_write_(param,
new_row.count_ /*column_cnt*/);
if (OB_FAIL(memtable_key_generator.generate_memtable_key(new_row))) {
TRANS_LOG(WARN, "generate memtable key fail", K(ret), K(new_row));
} else if (OB_FAIL(mvcc_write_(param,
context,
&mtk,
memtable_key_generator.get_memtable_key(),
arg,
check_exist,
is_new_locked,
memtable_key_generator.get_key_buffer(),
mvcc_row))) {
if (OB_TRY_LOCK_ROW_CONFLICT != ret &&
OB_TRANSACTION_SET_VIOLATION != ret &&
OB_ERR_PRIMARY_KEY_DUPLICATE != ret) {
TRANS_LOG(WARN, "mvcc write fail", K(mtk), K(ret));
TRANS_LOG(WARN, "mvcc write fail", K(memtable_key_generator.get_memtable_key()), K(ret));
}
} else {
TRANS_LOG(DEBUG, "set end, success",
"ret", ret,
"tablet_id_", key_.tablet_id_,
"dml_flag", new_row.flag_.get_dml_flag(),
"dml_flag", new_row.row_flag_.get_dml_flag(),
"columns", strarray<ObColDesc>(columns),
"old_row", to_cstring(old_row),
"new_row", to_cstring(new_row),
@ -2711,9 +2712,8 @@ int ObMemtable::set_(
//set_end(ctx.mvcc_acc_ctx_, ret);
if (OB_SUCC(ret)) {
set_max_data_schema_version(ctx.table_version_);
set_max_column_cnt(new_row.row_val_.count_);
set_max_column_cnt(new_row.count_);
}
return ret;
}
@ -2754,10 +2754,11 @@ int ObMemtable::lock_(
rowkey.get_obj_cnt()); /*column_cnt*/
if (OB_FAIL(mvcc_write_(param,
context,
&mtk,
mtk,
arg,
false, /*check_exist*/
is_new_locked,
nullptr, /*memtable_key_buffer*/
nullptr /*mvcc_row*/))) {
} else if (OB_UNLIKELY(!is_new_locked)) {
TRANS_LOG(DEBUG, "lock twice, no need to store lock trans node");
@ -2810,10 +2811,11 @@ int ObMemtable::mvcc_replay_(storage::ObStoreCtx &ctx,
int ObMemtable::mvcc_write_(
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
const ObMemtableKey *key,
const ObMemtableKey &key,
const ObTxNodeArg &arg,
const bool check_exist,
bool &is_new_locked,
ObMemtableKeyGenerator::ObMemtableKeyBuffer *memtable_key_buffer,
ObMvccRowAndWriteResult *mvcc_row)
{
int ret = OB_SUCCESS;
@ -2826,13 +2828,13 @@ int ObMemtable::mvcc_write_(
SCN snapshot_version = ctx.mvcc_acc_ctx_.get_snapshot_version();
transaction::ObTxSnapshot &snapshot = ctx.mvcc_acc_ctx_.snapshot_;
if (OB_FAIL(mvcc_engine_.create_kv(key,
if (OB_FAIL(mvcc_engine_.create_kv(&key,
// is_insert
blocksstable::ObDmlFlag::DF_INSERT == arg.data_->dml_flag_,
&stored_key,
value,
is_new_add))) {
TRANS_LOG(WARN, "create kv failed", K(ret), K(arg), K(*key));
TRANS_LOG(WARN, "create kv failed", K(ret), K(arg), K(key));
} else if (OB_FAIL(mvcc_engine_.mvcc_write(ctx,
snapshot,
*value,
@ -2840,29 +2842,29 @@ int ObMemtable::mvcc_write_(
res))) {
if (OB_TRY_LOCK_ROW_CONFLICT == ret) {
ret = post_row_write_conflict_(ctx.mvcc_acc_ctx_,
*key,
key,
res.lock_state_,
value->get_last_compact_cnt(),
value->get_total_trans_node_cnt());
} else if (OB_TRANSACTION_SET_VIOLATION == ret) {
mem_ctx->on_tsc_retry(*key,
mem_ctx->on_tsc_retry(key,
snapshot_version,
value->get_max_trans_version(),
value->get_max_trans_id());
} else if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) {
mem_ctx->on_key_duplication_retry(*key);
mem_ctx->on_key_duplication_retry(key);
} else {
TRANS_LOG(WARN, "mvcc write fail", K(ret));
}
} else if (nullptr == mvcc_row && OB_FAIL(lock_row_on_frozen_stores_(param,
arg,
key,
&key,
check_exist,
context,
value,
res))) {
lock_row_on_frozen_stores_on_failure(arg.data_->dml_flag_,
*key,
key,
ret,
value,
context,
@ -2890,6 +2892,8 @@ int ObMemtable::mvcc_write_(
(void)mvcc_engine_.mvcc_undo(value);
res.is_mvcc_undo_ = true;
TRANS_LOG(WARN, "register row commit failed", K(ret));
} else if (nullptr != memtable_key_buffer && OB_FAIL(memtable_key_buffer->push_back(stored_key))) {
TRANS_LOG(WARN, "push back stored memtable key into buffer failed", K(ret));
} else if (nullptr == mvcc_row && res.has_insert()) {
(void)mvcc_engine_.finish_kv(res);
/*****[for deadlock]*****/
@ -2898,7 +2902,7 @@ int ObMemtable::mvcc_write_(
// no need to detect deadlock for non-unique local index table
} else {
MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(),
*key,
key,
context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id());
}
/***********************/

26
src/storage/memtable/ob_memtable.h
View File

@ -245,22 +245,22 @@ public: // derived from ObITable
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
const common::ObIArray<share::schema::ObColDesc> &columns, // TODO: remove columns
const storage::ObStoreRow &row,
blocksstable::ObDatumRow &row,
const share::ObEncryptMeta *encrypt_meta,
const bool check_exist);
virtual int set(
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
const common::ObIArray<share::schema::ObColDesc> &columns, // TODO: remove columns
const common::ObIArray<share::schema::ObColDesc> &columns,
const ObIArray<int64_t> &update_idx,
const storage::ObStoreRow &old_row,
const storage::ObStoreRow &new_row,
const blocksstable::ObDatumRow &old_row,
blocksstable::ObDatumRow &new_row,
const share::ObEncryptMeta *encrypt_meta);
int multi_set(
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
const common::ObIArray<share::schema::ObColDesc> &columns,
const storage::ObStoreRow *rows,
blocksstable::ObDatumRow *rows,
const int64_t row_count,
const bool check_exist,
const share::ObEncryptMeta *encrypt_meta,
@ -287,7 +287,8 @@ public: // derived from ObITable
virtual int lock(
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
const common::ObNewRow &row);
ObColDescArray &col_desc,
blocksstable::ObDatumRow &row);
virtual int lock(
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
@ -476,10 +477,11 @@ private:
int mvcc_write_(
const storage::ObTableIterParam &param,
storage::ObTableAccessContext &context,
const ObMemtableKey *key,
const ObMemtableKey &key,
const ObTxNodeArg &arg,
const bool check_exist,
bool &is_new_locked,
ObMemtableKeyGenerator::ObMemtableKeyBuffer *memtable_key_buffer,
ObMvccRowAndWriteResult *mvcc_row = nullptr);
int mvcc_replay_(storage::ObStoreCtx &ctx,
@ -513,7 +515,6 @@ private:
int lock_rows_on_frozen_stores_(
const bool check_exist,
const storage::ObTableIterParam &param,
const ObMemtableKeyGenerator &memtable_keys,
storage::ObTableAccessContext &context,
ObMvccRowAndWriteResults &mvcc_rows,
ObRowsInfo &rows_info);
@ -544,20 +545,19 @@ private:
int set_(
const storage::ObTableIterParam &param,
const common::ObIArray<share::schema::ObColDesc> &columns,
const storage::ObStoreRow &new_row,
const storage::ObStoreRow *old_row,
const blocksstable::ObDatumRow &new_row,
const blocksstable::ObDatumRow *old_row,
const common::ObIArray<int64_t> *update_idx,
const ObMemtableKey &mtk,
const bool check_exist,
storage::ObTableAccessContext &context,
ObMemtableKeyGenerator &memtable_key_generator,
ObMvccRowAndWriteResult *mvcc_row = nullptr);
int multi_set_(
const storage::ObTableIterParam &param,
const common::ObIArray<share::schema::ObColDesc> &columns,
const storage::ObStoreRow *rows,
const blocksstable::ObDatumRow *rows,
const int64_t row_count,
const bool check_exist,
const ObMemtableKeyGenerator &memtable_keys,
storage::ObTableAccessContext &context,
storage::ObRowsInfo &rows_info);
int lock_(

128
src/storage/memtable/ob_memtable_key.cpp
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 OceanBase
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
@ -11,110 +11,56 @@
*/
#include "ob_memtable_key.h"
#include "lib/ob_errno.h"
#include "rowkey/ob_rowkey.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/ob_i_store.h"
namespace oceanbase
{
namespace memtable
{
constexpr int64_t ObMemtableKeyGenerator::STACK_BUFFER_SIZE;
int ObMemtableKeyGenerator::init(const storage::ObStoreRow *rows,
const int64_t row_count,
const int64_t schema_rowkey_count,
const common::ObIArray<share::schema::ObColDesc> &columns)
int ObMemtableKeyGenerator::init()
{
int ret = OB_SUCCESS;
if (size_ != 0) {
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
TRANS_LOG(WARN, "init ObMemtableKeyGenerator twice", K(ret));
} else if (columns_.count() < rowkey_cnt_) {
ret = OB_INVALID_ARGUMENT;
TRANS_LOG(WARN, "rowkey number mismatched", K(columns_.count()), K(rowkey_cnt_));
} else if (OB_FAIL(obj_buf_.init(&allocator_))) {
TRANS_LOG(WARN, "init obj buffer fail", K(ret));
} else if (OB_FAIL(obj_buf_.reserve(rowkey_cnt_))) {
TRANS_LOG(WARN, "reserve obj buffer fail", K(ret), K(rowkey_cnt_));
} else {
int64_t extra_size = row_count - STACK_BUFFER_SIZE;
if (extra_size > 0) {
if (FALSE_IT(p_extra_store_row_keys_ =
(ObStoreRowkey *)share::mtl_malloc(extra_size * (sizeof(ObStoreRowkey)), "MemTableKey"))) {
} else if (OB_ISNULL(p_extra_store_row_keys_)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
TRANS_LOG(WARN, "Failed to alloc ObStoreRowkey memory", K(ret), K(row_count), K(schema_rowkey_count), K(columns), KP(MTL_CTX()),
KP(lib::ObMallocAllocator::get_instance()), KP(p_extra_store_row_keys_));
} else if (FALSE_IT(p_extra_memtable_keys_ =
(ObMemtableKey *)share::mtl_malloc(extra_size * (sizeof(ObMemtableKey)), "MemTableKey"))) {
} else if (OB_ISNULL(p_extra_memtable_keys_)) {
share::mtl_free(p_extra_store_row_keys_);
p_extra_store_row_keys_ = nullptr;
ret = OB_ALLOCATE_MEMORY_FAILED;
TRANS_LOG(WARN, "Failed to alloc ObMemtableKey memory", K(ret), K(row_count), K(schema_rowkey_count), K(columns), KP(MTL_CTX()),
KP(lib::ObMallocAllocator::get_instance()), KP(p_extra_store_row_keys_));
is_inited_ = true;
}
return ret;
}
int ObMemtableKeyGenerator::generate_memtable_key(const blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
TRANS_LOG(WARN, "ObMemtableKeyGenerator not inited", K(ret));
} else if (OB_UNLIKELY(!row.is_valid())) {
ret = OB_INVALID_ARGUMENT;
TRANS_LOG(WARN, "invalid argument", K(ret), K(row));
} else {
ObObj *objs = obj_buf_.get_data();
for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt_; i++) {
if (OB_FAIL(row.storage_datums_[i].to_obj_enhance(objs[i], columns_.at(i).col_type_))) {
TRANS_LOG(WARN, "failed to transfer datum to obj", K(ret), K(i), K(row));
}
}
for (int i = 0; i < row_count && OB_SUCC(ret); ++i) {
ObStoreRowkey *p_store_row_key = i < STACK_BUFFER_SIZE ? &store_row_key_buffer_[i] : &p_extra_store_row_keys_[i - STACK_BUFFER_SIZE];
ObMemtableKey *p_memtable_key = i < STACK_BUFFER_SIZE ? &memtable_key_buffer_[i] : &p_extra_memtable_keys_[i - STACK_BUFFER_SIZE];
new (p_store_row_key) ObStoreRowkey();
new (p_memtable_key) ObMemtableKey();
if (OB_FAIL(p_store_row_key->assign(rows[i].row_val_.cells_, schema_rowkey_count))) {
p_store_row_key->~ObStoreRowkey();
TRANS_LOG(WARN, "Failed to assign tmp rowkey", K(ret), K(rows[i]), K(row_count), K(schema_rowkey_count));
} else if (OB_FAIL(p_memtable_key->encode(columns, p_store_row_key))) {
p_memtable_key->~ObMemtableKey();
p_store_row_key->~ObStoreRowkey();
TRANS_LOG(WARN, "mtk encode fail", K(ret), K(rows[i]), K(row_count), K(schema_rowkey_count));
} else {
size_++;
if (OB_SUCC(ret)) {
if (OB_FAIL(store_rowkey_.assign(objs, rowkey_cnt_))) {
TRANS_LOG(WARN, "failed to assign rowkey", K(ret), K(row), K(objs), K(rowkey_cnt_));
} else if (OB_FAIL(memtable_key_.encode(columns_, &store_rowkey_))) {
TRANS_LOG(WARN, "memtable key encode failed", K(ret), K(row), K(columns_), K(store_rowkey_));
}
}
if (OB_FAIL(ret)) {
reset();
}
}
return ret;
}
ObMemtableKey &ObMemtableKeyGenerator::operator[](int64_t idx) {
ObMemtableKey *element = nullptr;
if (OB_UNLIKELY(idx < 0 || idx >= size_)) {
ob_abort();
}
if (idx < STACK_BUFFER_SIZE) {
element = &memtable_key_buffer_[idx];
} else {
element = &p_extra_memtable_keys_[idx - STACK_BUFFER_SIZE];
}
return *element;
}
const ObMemtableKey &ObMemtableKeyGenerator::operator[](int64_t idx) const
{
return const_cast<ObMemtableKeyGenerator *>(this)->operator[](idx);
}
void ObMemtableKeyGenerator::reset()
{
int64_t idx = size_ - 1;
for(; idx >= STACK_BUFFER_SIZE; --idx) {
p_extra_memtable_keys_[idx - STACK_BUFFER_SIZE].~ObMemtableKey();
p_extra_store_row_keys_[idx - STACK_BUFFER_SIZE].~ObStoreRowkey();
}
for(; idx >= 0; --idx) {
memtable_key_buffer_[idx].~ObMemtableKey();
store_row_key_buffer_[idx].~ObStoreRowkey();
}
if (OB_UNLIKELY(nullptr != p_extra_memtable_keys_)) {
share::mtl_free(p_extra_memtable_keys_);
}
if (OB_UNLIKELY(nullptr != p_extra_store_row_keys_)) {
share::mtl_free(p_extra_store_row_keys_);
}
new (this) ObMemtableKeyGenerator();
}
ObMemtableKeyGenerator::~ObMemtableKeyGenerator()
{
reset();
}
}
}
} // namespace memtable
} // namespace oceanbase

53
src/storage/memtable/ob_memtable_key.h
View File

@ -283,7 +283,7 @@ public:
private:
common::ObStoreRowkey *rowkey_;
mutable uint64_t hash_val_; // Perf optimization.
DISALLOW_COPY_AND_ASSIGN(ObMemtableKey);
//DISALLOW_COPY_AND_ASSIGN(ObMemtableKey);
};
class ObStoreRowkeyWrapper
@ -308,31 +308,38 @@ public:
const common::ObStoreRowkey *rowkey_;
};
// this is for multi_set pre alloc memory to generate memtable key
class ObMemtableKeyGenerator {// RAII
static constexpr int64_t STACK_BUFFER_SIZE = 32;
// TODO(xuanxi): remove it later
class ObMemtableKeyGenerator {
public:
ObMemtableKeyGenerator() : p_extra_store_row_keys_(nullptr), p_extra_memtable_keys_(nullptr), size_(0) {}
~ObMemtableKeyGenerator();
int init(const storage::ObStoreRow *rows,
const int64_t row_count,
const int64_t schema_rowkey_count,
const common::ObIArray<share::schema::ObColDesc> &columns);
void reset();
int64_t count() const { return size_; }
ObMemtableKey &operator[](int64_t idx);
const ObMemtableKey &operator[](int64_t idx) const;
using ObMemtableKeyBuffer = common::ObSEArray<ObMemtableKey, 16>;
public:
ObMemtableKeyGenerator(
const int64_t rowkey_cnt,
const common::ObIArray<share::schema::ObColDesc> &columns,
ObMemtableKeyBuffer *memtable_key_buffer = nullptr)
: allocator_(common::ObMemAttr(MTL_ID(), "ObMemtableKey")),
rowkey_cnt_(rowkey_cnt),
columns_(columns),
memtable_key_buffer_(memtable_key_buffer),
is_inited_(false)
{}
~ObMemtableKeyGenerator() = default;
int init();
int generate_memtable_key(const blocksstable::ObDatumRow &datum_row);
ObMemtableKey &get_memtable_key() { return memtable_key_; }
ObMemtableKeyBuffer *get_key_buffer() { return memtable_key_buffer_; }
private:
// this is for avoid memory allocation when rows not so much
ObStoreRowkey store_row_key_buffer_[STACK_BUFFER_SIZE];
ObMemtableKey memtable_key_buffer_[STACK_BUFFER_SIZE];
ObStoreRowkey *p_extra_store_row_keys_;
ObMemtableKey *p_extra_memtable_keys_;
int64_t size_;
ObArenaAllocator allocator_;
int64_t rowkey_cnt_;
const common::ObIArray<share::schema::ObColDesc> &columns_;
storage::ObObjBufArray obj_buf_;
ObStoreRowkey store_rowkey_;
ObMemtableKey memtable_key_;
ObMemtableKeyBuffer *memtable_key_buffer_;
bool is_inited_;
};
}
}
} // namespace memtable
} // namespace oceanbase
#endif // OCEANBASE_MEMTABLE_OB_MEMTABLE_KEY2_

112
src/storage/ob_dml_running_ctx.cpp
View File

@ -35,7 +35,8 @@ ObDMLRunningCtx::ObDMLRunningCtx(
ObStoreCtx &store_ctx,
const ObDMLBaseParam &dml_param,
common::ObIAllocator &allocator,
const blocksstable::ObDmlFlag dml_flag)
const blocksstable::ObDmlFlag dml_flag,
bool is_need_row_datum_utils)
: store_ctx_(store_ctx),
dml_param_(dml_param),
allocator_(allocator),
@ -44,13 +45,21 @@ ObDMLRunningCtx::ObDMLRunningCtx(
col_map_(nullptr),
col_descs_(nullptr),
column_ids_(nullptr),
tbl_row_(),
datum_row_(),
cmp_funcs_(),
is_old_row_valid_for_lob_(false),
is_need_check_old_row_(is_need_row_datum_utils),
is_udf_(false),
schema_guard_(share::schema::ObSchemaMgrItem::MOD_RELATIVE_TABLE),
is_need_row_datum_utils_(is_need_row_datum_utils),
is_inited_(false)
{
}
ObDMLRunningCtx::~ObDMLRunningCtx()
{
}
int ObDMLRunningCtx::init(
const common::ObIArray<uint64_t> *column_ids,
const common::ObIArray<uint64_t> *upd_col_ids,
@ -89,6 +98,10 @@ int ObDMLRunningCtx::init(
LOG_WARN("failed to get relative table", K(ret), K(dml_param_));
} else if (NULL != column_ids && OB_FAIL(prepare_column_info(*column_ids))) {
LOG_WARN("fail to get column descriptions and column map", K(ret), K(*column_ids));
} else if (is_need_check_old_row_ && OB_FAIL(check_need_old_row_legitimacy())) {
LOG_WARN("fail to get flag of checking old row legitimacy", K(ret));
} else if (is_need_check_old_row_ && OB_FAIL(init_cmp_funcs())) {
LOG_WARN("fail to init compare functions", K(ret));
} else {
store_ctx_.mvcc_acc_ctx_.mem_ctx_->set_table_version(dml_param_.schema_version_);
store_ctx_.table_version_ = dml_param_.schema_version_;
@ -161,6 +174,101 @@ int ObDMLRunningCtx::prepare_column_info(const common::ObIArray<uint64_t> &colum
return ret;
}
int ObDMLRunningCtx::check_need_old_row_legitimacy()
{
int ret = OB_SUCCESS;
// TODO(jingxing): setting this to true
if (OB_FAIL(relative_table_.has_udf_column(is_need_check_old_row_))) {
LOG_WARN("check has udf column failed", K(ret));
} else if (is_need_check_old_row_) {
is_udf_ = true;
ObTableStoreIterator &table_iter = *relative_table_.tablet_iter_.table_iter();
while (OB_SUCC(ret) && !is_need_check_old_row_) {
ObITable *table_ptr = nullptr;
if (OB_FAIL(table_iter.get_next(table_ptr))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next table failed", K(ret));
}
} else if (OB_ISNULL(table_ptr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, table ptr must not be nullptr", K(ret));
} else {
is_need_check_old_row_ = table_ptr->is_major_sstable();
}
}
} else if (dml_param_.is_batch_stmt_ && !relative_table_.is_index_table()) {
//batch stmt execution dependency defensive check to check
//if the same row was modified multiple times
is_need_check_old_row_ = true;
ret = OB_E(EventTable::EN_INS_MULTI_VALUES_BATCH_OPT) OB_SUCCESS;
// no need to check old row, just for bmsql performance optimization
// TODO yuchen.ywc
if (OB_SUCCESS != ret) {
LOG_INFO("error sim when current statement is batch update", K(ret), K_(is_udf));
is_need_check_old_row_ = false;
ret = OB_SUCCESS;
}
} else if (GCONF.enable_defensive_check()) {
is_need_check_old_row_ = true;
if (relative_table_.is_index_table() && !relative_table_.can_read_index()) {
//index can not be read during building index, so does not check old index row
is_need_check_old_row_ = false;
}
if (ObDmlFlag::DF_LOCK == dml_flag_) {
is_need_check_old_row_ = false;
}
}
return ret;
}
int ObDMLRunningCtx::init_cmp_funcs()
{
int ret = OB_SUCCESS;
const common::ObIArray<share::schema::ObColDesc> &col_descs = dml_param_.table_param_->get_col_descs();
int64_t column_cnt = col_descs.count();
if (OB_UNLIKELY(column_cnt < 0 || column_cnt > OB_ROW_MAX_COLUMNS_COUNT)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument to init compare functions", K(ret), K(column_cnt), K(col_descs));
} else if (OB_FAIL(cmp_funcs_.init(column_cnt, allocator_))) {
STORAGE_LOG(WARN, "Failed to reserve cmp func array", K(ret));
} else {
bool is_oracle_mode = lib::is_oracle_mode();
ObCmpFunc cmp_func;
for (int64_t i = 0; OB_SUCC(ret) && i < col_descs.count(); i++) {
const share::schema::ObColDesc &col_desc = col_descs.at(i);
//TODO @hanhui support desc rowkey
bool is_ascending = true || col_desc.col_order_ == ObOrderType::ASC;
bool has_lob_header = is_lob_storage(col_desc.col_type_.get_type());
ObPrecision precision = PRECISION_UNKNOWN_YET;
if (col_desc.col_type_.is_decimal_int()) {
precision = col_desc.col_type_.get_stored_precision();
OB_ASSERT(precision != PRECISION_UNKNOWN_YET);
}
sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(col_desc.col_type_.get_type(),
col_desc.col_type_.get_collation_type(),
col_desc.col_type_.get_scale(),
is_oracle_mode,
has_lob_header,
precision);
if (OB_UNLIKELY(nullptr == basic_funcs || nullptr == basic_funcs->null_last_cmp_)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "Unexpected null basic funcs", K(ret), K(col_desc));
} else {
if (is_ascending) {
cmp_func.cmp_func_ = is_oracle_mode ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_;
if (OB_FAIL(cmp_funcs_.push_back(ObStorageDatumCmpFunc(cmp_func)))) {
STORAGE_LOG(WARN, "Failed to push back cmp func", K(ret), K(i), K(col_desc));
}
} else {
ret = OB_ERR_SYS;
STORAGE_LOG(WARN, "Unsupported desc column order", K(ret), K(col_desc), K(i));
}
}
}
}
return ret;
}
int ObDMLRunningCtx::check_schema_version(
share::schema::ObMultiVersionSchemaService &schema_service,
const uint64_t tenant_id,

13
src/storage/ob_dml_running_ctx.h
View File

@ -47,8 +47,9 @@ public:
ObStoreCtx &store_ctx,
const ObDMLBaseParam &dml_param,
common::ObIAllocator &allocator,
const blocksstable::ObDmlFlag dml_flag);
~ObDMLRunningCtx() {}
const blocksstable::ObDmlFlag dml_flag,
bool is_need_row_datum_utils = false);
~ObDMLRunningCtx();
int init(
const common::ObIArray<uint64_t> *column_ids,
@ -65,6 +66,8 @@ private:
const share::schema::ObTableSchemaParam &schema,
ObTabletHandle &tablet_handle,
const share::SCN &read_snapshot);
int check_need_old_row_legitimacy();
int init_cmp_funcs();
int check_schema_version(share::schema::ObMultiVersionSchemaService &schema_service,
const uint64_t tenant_id,
const uint64_t table_id,
@ -86,11 +89,15 @@ public:
const share::schema::ColumnMap *col_map_;
const ObColDescIArray *col_descs_;
const common::ObIArray<uint64_t> *column_ids_;
ObStoreRow tbl_row_;
blocksstable::ObDatumRow datum_row_;
blocksstable::ObStoreCmpFuncs cmp_funcs_;
bool is_old_row_valid_for_lob_;
bool is_need_check_old_row_;
bool is_udf_;
private:
share::schema::ObSchemaGetterGuard schema_guard_;
bool is_need_row_datum_utils_;
bool is_inited_;
};
} // namespace storage

2
src/storage/ob_query_iterator_factory.cpp
View File

@ -113,7 +113,7 @@ void ObQueryIteratorFactory::free_table_scan_iter(common::ObNewRowIterator *iter
}
}
void ObQueryIteratorFactory::free_insert_dup_iter(common::ObNewRowIterator *iter)
void ObQueryIteratorFactory::free_insert_dup_iter(blocksstable::ObDatumRowIterator *iter)
{
if (OB_LIKELY(NULL != iter)) {
(void)ATOMIC_FAA(&insert_dup_release_count_, 1);

6
src/storage/ob_query_iterator_factory.h
View File

@ -18,6 +18,10 @@
namespace oceanbase
{
namespace blocksstable
{
class ObDatumRowIterator;
}
namespace storage
{
class ObMultipleScanMerge;
@ -37,7 +41,7 @@ public:
static ObColMap *get_col_map();
static void free_table_scan_iter(common::ObNewRowIterator *iter);
static void free_insert_dup_iter(common::ObNewRowIterator *iter);
static void free_insert_dup_iter(blocksstable::ObDatumRowIterator *iter);
static void free_merge_iter(ObQueryRowIterator *iter);
static void free_col_map(ObColMap *col_map);
static void free_work_row(ObStoreRow *row);

181
src/storage/ob_value_row_iterator.cpp
View File

@ -25,7 +25,7 @@ using namespace blocksstable;
namespace storage
{
ObValueRowIterator::ObValueRowIterator()
: ObNewRowIterator(),
: ObDatumRowIterator(),
is_inited_(false),
unique_(false),
allocator_("ObValueRowAlloc"),
@ -53,10 +53,9 @@ int ObValueRowIterator::init(bool unique)
return ret;
}
int ObValueRowIterator::add_row(common::ObNewRow &row)
int ObValueRowIterator::add_row(ObDatumRow &row, const ObStorageDatumUtils &rowkey_datum_utils)
{
int ret = OB_SUCCESS;
ObNewRow *cur_row = NULL;
if (!row.is_valid()) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid row", K(ret), K(row));
@ -71,29 +70,34 @@ int ObValueRowIterator::add_row(common::ObNewRow &row)
//on multiple unique index is small, so there is usually only one row in the value row iterator
//so using list traversal to deduplicate unique index is more efficiently
//and also saves the CPU overhead that constructs the hash map
ObStoreRowkey rowkey;
ObStoreRowkey tmp_rowkey;
if (OB_FAIL(rowkey.assign(row.cells_, row.count_))) {
ObDatumRowkey rowkey;
ObDatumRowkey tmp_rowkey;
if (OB_FAIL(rowkey.assign(row.storage_datums_, row.count_))) {
STORAGE_LOG(WARN, "Failed to assign rowkey", K(ret), K(row));
}
for (int64_t i = 0; OB_SUCC(ret) && !exist && i < rows_.count(); ++i) {
if (OB_FAIL(tmp_rowkey.assign(rows_.at(i).cells_, rows_.at(i).count_))) {
STORAGE_LOG(WARN, "Failed to assign rowkey", K(ret), K(i), K(rows_.at(i)));
} else if (OB_UNLIKELY(tmp_rowkey == rowkey)) {
exist = true;
if (OB_FAIL(tmp_rowkey.assign(rows_.at(i)->storage_datums_, rows_.at(i)->count_))) {
STORAGE_LOG(WARN, "Failed to assign rowkey", K(ret), K(i), KPC(rows_.at(i)));
} else if (OB_FAIL(tmp_rowkey.equal(rowkey, rowkey_datum_utils, exist))) {
STORAGE_LOG(WARN, "Failed to compare rowkey", K(ret), K(i), K(tmp_rowkey), K(rowkey));
}
}
}
// store non-exist row
if (OB_SUCC(ret)) {
if (!exist) {
if (NULL == (cur_row = rows_.alloc_place_holder())) {
ObDatumRow *cur_row = nullptr;
void *buff = nullptr;
if (OB_ISNULL(buff = allocator_.alloc(sizeof(ObDatumRow)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(ERROR, "add row error", K(ret));
} else if (OB_SUCCESS != (ret = ob_write_row(allocator_,
row,
*cur_row))) {
STORAGE_LOG(WARN, "alloc memory for datum row error", K(ret));
} else if (FALSE_IT(cur_row = new (buff) ObDatumRow())) {
} else if (OB_FAIL(cur_row->init(allocator_, row.count_))) {
STORAGE_LOG(WARN, "init datum row error", K(ret), K(row), KPC(cur_row));
} else if (OB_FAIL(cur_row->deep_copy(row, allocator_))) {
STORAGE_LOG(WARN, "copy row error", K(ret), K(row));
} else if (OB_FAIL(rows_.push_back(cur_row))) {
STORAGE_LOG(WARN, "fail to push datum row to iterator array", K(ret), K(cur_row));
}
}
}
@ -101,27 +105,14 @@ int ObValueRowIterator::add_row(common::ObNewRow &row)
return ret;
}
int ObValueRowIterator::get_next_row(common::ObNewRow *&row)
int ObValueRowIterator::get_next_row(ObDatumRow *&row)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObValueRowIterator is not initialized", K(ret));
} else if (cur_idx_ < rows_.count()) {
row = &rows_.at(cur_idx_++);
} else {
ret = OB_ITER_END;
}
return ret;
}
int ObValueRowIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count)
{
int ret = OB_SUCCESS;
if (cur_idx_ < rows_.count()) {
rows = &(rows_.at(cur_idx_));
row_count = rows_.count() - cur_idx_;
cur_idx_ = rows_.count();
row = rows_.at(cur_idx_++);
} else {
ret = OB_ITER_END;
}
@ -143,23 +134,23 @@ ObSingleRowGetter::ObSingleRowGetter(ObIAllocator &allocator, ObTablet &tablet)
store_ctx_(nullptr),
output_projector_(allocator),
relative_table_(nullptr),
table_param_(nullptr),
allocator_(allocator),
new_row_builder_()
cached_iter_node_(nullptr)
{
}
ObSingleRowGetter::~ObSingleRowGetter()
{
if (single_merge_ != nullptr) {
single_merge_->~ObSingleMerge();
allocator_.free(single_merge_);
if (nullptr != single_merge_) {
if (nullptr == cached_iter_node_) {
single_merge_->~ObSingleMerge();
allocator_.free(single_merge_);
}
single_merge_ = nullptr;
}
if (table_param_ != nullptr) {
table_param_->~ObTableParam();
allocator_.free(table_param_);
table_param_ = nullptr;
if (nullptr != cached_iter_node_) {
ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*);
iter_pool->release(cached_iter_node_);
}
}
@ -171,6 +162,7 @@ int ObSingleRowGetter::init_dml_access_ctx(
int ret = OB_SUCCESS;
common::ObQueryFlag query_flag;
common::ObVersionRange trans_version_range;
query_flag.set_not_use_bloomfilter_cache();
query_flag.read_latest_ = ObQueryFlag::OBSF_MASK_READ_LATEST;
if (skip_read_lob) {
query_flag.skip_read_lob_ = ObQueryFlag::OBSF_MASK_SKIP_READ_LOB;
@ -180,7 +172,7 @@ int ObSingleRowGetter::init_dml_access_ctx(
trans_version_range.multi_version_start_ = 0;
store_ctx_ = &store_ctx;
if (OB_FAIL(access_ctx_.init(query_flag, store_ctx, allocator_, trans_version_range))) {
if (OB_FAIL(access_ctx_.init(query_flag, store_ctx, allocator_, trans_version_range, cached_iter_node_))) {
LOG_WARN("failed to init table access ctx", K(ret));
}
return ret;
@ -233,18 +225,21 @@ int ObSingleRowGetter::init_dml_access_param(ObRelativeTable &relative_table,
return ret;
}
int ObSingleRowGetter::create_table_param()
int ObSingleRowGetter::prepare_cached_iter_node()
{
int ret = OB_SUCCESS;
void *buf = nullptr;
if (table_param_ != nullptr) {
ret = OB_INIT_TWICE;
LOG_WARN("init table param twice", K(ret));
} else if (OB_ISNULL(buf = allocator_.alloc(sizeof(share::schema::ObTableParam)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate table param failed", K(ret), K(sizeof(share::schema::ObTableParam)));
} else {
table_param_ = new(buf) share::schema::ObTableParam(allocator_);
if (OB_UNLIKELY(nullptr != cached_iter_node_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected not null cached iter node", K(ret), KP(cached_iter_node_));
} else if (can_use_global_iter_pool()) {
ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*);
if (OB_FAIL(iter_pool->get(ITER_TYPE, cached_iter_node_))) {
STORAGE_LOG(WARN, "Failed to get from iter pool", K(ret));
} else if (nullptr != cached_iter_node_) {
access_param_.set_use_global_iter_pool();
access_param_.iter_param_.set_use_stmt_iter_pool();
STORAGE_LOG(TRACE, "use global iter pool", K(access_param_));
}
}
return ret;
}
@ -252,28 +247,18 @@ int ObSingleRowGetter::create_table_param()
int ObSingleRowGetter::open(const ObDatumRowkey &rowkey, bool use_fuse_row_cache)
{
int ret = OB_SUCCESS;
void *buf = nullptr;
const common::ObIArray<share::schema::ObColDesc> * col_descs = nullptr;
if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObSingleMerge)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("Fail to allocate memory for multi get merge ", K(ret));
} else {
{
ObStorageTableGuard guard(tablet_, *store_ctx_, false);
if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(*relative_table_))) {
STORAGE_LOG(WARN, "fail to protect table", K(ret));
}
{
ObStorageTableGuard guard(tablet_, *store_ctx_, false);
if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(*relative_table_))) {
STORAGE_LOG(WARN, "fail to protect table", K(ret));
}
single_merge_ = new(buf) ObSingleMerge();
}
if (OB_SUCC(ret)) {
if (OB_FAIL(single_merge_->init(access_param_, access_ctx_, get_table_param_))) {
STORAGE_LOG(WARN, "Fail to init ObSingleMerge, ", K(ret));
ACTIVE_GLOBAL_ITERATOR_GUARD(ret, cached_iter_node_);
if (OB_FAIL(init_single_merge())) {
STORAGE_LOG(WARN, "Fail to init ObSingleMerge", K(ret));
} else if (OB_FAIL(single_merge_->open(rowkey))) {
STORAGE_LOG(WARN, "Fail to open iter, ", K(ret));
} else if (OB_FAIL(new_row_builder_.init(single_merge_->get_out_project_cells(), allocator_))) {
LOG_WARN("Failed to init ObNewRowBuilder", K(ret));
STORAGE_LOG(WARN, "Fail to open iter", K(ret));
}
if (use_fuse_row_cache) {
access_ctx_.use_fuse_row_cache_ = true;
@ -282,9 +267,10 @@ int ObSingleRowGetter::open(const ObDatumRowkey &rowkey, bool use_fuse_row_cache
return ret;
}
int ObSingleRowGetter::get_next_row(ObNewRow *&row)
int ObSingleRowGetter::get_next_row(blocksstable::ObDatumRow *&row)
{
int ret = OB_SUCCESS;
ACTIVE_GLOBAL_ITERATOR_GUARD(ret, cached_iter_node_);
row = nullptr;
while (OB_SUCC(ret)) {
blocksstable::ObDatumRow *store_row = NULL;
@ -293,9 +279,7 @@ int ObSingleRowGetter::get_next_row(ObNewRow *&row)
STORAGE_LOG(WARN, "failed to get next row", K(ret));
}
} else if (store_row->row_flag_.is_exist_without_delete()) {
if (OB_FAIL(new_row_builder_.build(*store_row, row))) {
STORAGE_LOG(WARN, "Failed to build new row", K(ret), KPC(store_row));
}
row = store_row;
break;
}
}
@ -319,5 +303,58 @@ int ObSingleRowGetter::get_next_row(ObNewRow *&row)
}
return ret;
}
bool ObSingleRowGetter::can_use_global_iter_pool() const
{
bool use_pool = false;
if (access_param_.iter_param_.tablet_id_.is_inner_tablet()) {
} else if (access_param_.iter_param_.has_lob_column_out_) {
} else {
const int64_t table_cnt = get_table_param_.tablet_iter_.table_iter()->count();
const int64_t col_cnt = get_table_param_.tablet_iter_.get_tablet()->get_rowkey_read_info().get_schema_column_count();
ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*);
if (OB_NOT_NULL(iter_pool)) {
use_pool = iter_pool->can_use_iter_pool(table_cnt, col_cnt, ITER_TYPE);
}
}
return use_pool;
}
int ObSingleRowGetter::init_single_merge()
{
int ret = OB_SUCCESS;
ObQueryRowIterator *cached_iter = nullptr == cached_iter_node_ ? nullptr : cached_iter_node_->get_iter();
if (OB_NOT_NULL(cached_iter)) {
if (OB_UNLIKELY(cached_iter->get_type() != ITER_TYPE)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "Unexpected cached iter type", K(ret), K(cached_iter->get_type()));
} else {
single_merge_ = static_cast<ObSingleMerge*>(cached_iter);
}
}
if (OB_FAIL(ret)) {
} else if (nullptr == single_merge_) {
void *buf = nullptr;
if (OB_ISNULL(buf = access_ctx_.get_long_life_allocator()->alloc(sizeof(ObSingleMerge)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "Fail to allocate memory", K(ret));
} else {
single_merge_ = new (buf) ObSingleMerge();
if (OB_FAIL(single_merge_->init(access_param_, access_ctx_, get_table_param_))) {
STORAGE_LOG(WARN, "Failed to init multiple merge", K(ret));
}
if (OB_FAIL(ret)) {
single_merge_->~ObSingleMerge();
access_ctx_.get_long_life_allocator()->free(single_merge_);
single_merge_ = nullptr;
} else if (nullptr != cached_iter_node_) {
cached_iter_node_->set_iter(single_merge_);
}
}
} else if (OB_FAIL(single_merge_->switch_table(access_param_, access_ctx_, get_table_param_))) {
STORAGE_LOG(WARN, "Failed to switch table", K(ret), K(access_param_));
}
return ret;
}
} // end namespace storage
} // end namespace oceanbase

20
src/storage/ob_value_row_iterator.h
View File

@ -22,6 +22,7 @@
#include "storage/ob_i_store.h"
#include "storage/access/ob_dml_param.h"
#include "blocksstable/ob_datum_rowkey.h"
#include "blocksstable/ob_datum_row_iterator.h"
namespace oceanbase
{
@ -29,17 +30,16 @@ namespace storage
{
class ObTablet;
class ObValueRowIterator : public common::ObNewRowIterator
class ObValueRowIterator : public blocksstable::ObDatumRowIterator
{
static const int64_t DEFAULT_ROW_NUM = 2;
typedef common::ObSEArray<common::ObNewRow, DEFAULT_ROW_NUM> RowArray;
typedef common::ObSEArray<blocksstable::ObDatumRow*, DEFAULT_ROW_NUM> RowArray;
public:
ObValueRowIterator();
virtual ~ObValueRowIterator();
virtual int init(bool unique);
virtual int get_next_row(common::ObNewRow *&row);
virtual int get_next_rows(common::ObNewRow *&rows, int64_t &row_count);
virtual int add_row(common::ObNewRow &row);
virtual int get_next_row(blocksstable::ObDatumRow *&row);
virtual int add_row(blocksstable::ObDatumRow &row, const blocksstable::ObStorageDatumUtils &rowkey_datum_utils);
virtual void reset();
private:
bool is_inited_;
@ -55,6 +55,7 @@ class ObSingleMerge;
class ObSingleRowGetter
{
typedef common::ObFixedArray<int32_t, common::ObIAllocator> Projector;
const ObQRIterType ITER_TYPE = T_SINGLE_GET;
public:
ObSingleRowGetter(common::ObIAllocator &allocator, ObTablet &tablet);
~ObSingleRowGetter();
@ -68,13 +69,15 @@ public:
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &out_col_ids,
const bool skip_read_lob = false);
int prepare_cached_iter_node();
ObTableAccessParam &get_access_param() { return access_param_; }
ObTableAccessContext &get_access_ctx() { return access_ctx_; }
void set_relative_table(ObRelativeTable *relative_table) { relative_table_ = relative_table; }
int open(const blocksstable::ObDatumRowkey &rowkey, bool use_fuse_row_cache = false);
int get_next_row(common::ObNewRow *&row);
int get_next_row(blocksstable::ObDatumRow *&row);
private:
int create_table_param();
bool can_use_global_iter_pool() const;
int init_single_merge();
private:
ObTablet *tablet_;
ObSingleMerge *single_merge_;
@ -84,9 +87,8 @@ private:
ObTableAccessContext access_ctx_;
ObGetTableParam get_table_param_;
ObRelativeTable *relative_table_;
share::schema::ObTableParam *table_param_;
common::ObIAllocator &allocator_;
blocksstable::ObNewRowBuilder new_row_builder_;
CachedIteratorNode *cached_iter_node_;
};
} // end namespace storage
} // end namespace oceanbase

31
src/storage/tablet/ob_tablet.cpp
View File

@ -91,6 +91,7 @@
#include "storage/tablet/ob_tablet_mds_table_mini_merger.h"
#include "storage/blocksstable/ob_shared_macro_block_manager.h"
#include "storage/ob_direct_load_table_guard.h"
#include "storage/blocksstable/ob_datum_row_utils.h"
namespace oceanbase
{
@ -3639,7 +3640,8 @@ int ObTablet::check_schema_version_for_bounded_staleness_read(
int ObTablet::lock_row(
ObRelativeTable &relative_table,
ObStoreCtx &store_ctx,
const common::ObNewRow &row)
ObColDescArray &col_desc,
blocksstable::ObDatumRow &row)
{
int ret = OB_SUCCESS;
ObStorageTableGuard guard(this, store_ctx, true);
@ -3669,7 +3671,7 @@ int ObTablet::lock_row(
LOG_WARN("prepare write memtable fail", K(ret), K(relative_table));
} else if (OB_FAIL(prepare_param_ctx(allocator, relative_table, store_ctx, param, context))) {
LOG_WARN("prepare param ctx fail, ", K(ret));
} else if (OB_FAIL(write_memtable->lock(param, context, row))) {
} else if (OB_FAIL(write_memtable->lock(param, context, col_desc, row))) {
LOG_WARN("failed to lock write_memtable", K(ret), K(row));
}
}
@ -4101,8 +4103,8 @@ int ObTablet::update_row(
storage::ObStoreCtx &store_ctx,
const common::ObIArray<share::schema::ObColDesc> &col_descs,
const ObIArray<int64_t> &update_idx,
const storage::ObStoreRow &old_row,
const storage::ObStoreRow &new_row,
const blocksstable::ObDatumRow &old_row,
blocksstable::ObDatumRow &new_row,
const common::ObIArray<transaction::ObEncryptMetaCache> *encrypt_meta_arr)
{
int ret = OB_SUCCESS;
@ -4159,7 +4161,7 @@ int ObTablet::update_row(
int ObTablet::insert_rows(
ObRelativeTable &relative_table,
ObStoreCtx &store_ctx,
ObStoreRow *rows,
ObDatumRow *rows,
ObRowsInfo &rows_info,
const bool check_exist,
const ObColDescIArray &col_descs,
@ -4209,7 +4211,7 @@ int ObTablet::insert_row_without_rowkey_check(
ObStoreCtx &store_ctx,
const bool check_exist,
const common::ObIArray<share::schema::ObColDesc> &col_descs,
const storage::ObStoreRow &row,
blocksstable::ObDatumRow &row,
const common::ObIArray<transaction::ObEncryptMetaCache> *encrypt_meta_arr)
{
int ret = OB_SUCCESS;
@ -4373,7 +4375,8 @@ int ObTablet::do_rowkeys_exist(
int ObTablet::rowkey_exists(
ObRelativeTable &relative_table,
ObStoreCtx &store_ctx,
const common::ObNewRow &row,
const ObColDescIArray &col_descs,
ObDatumRow &row,
bool &exists)
{
int ret = OB_SUCCESS;
@ -4399,24 +4402,22 @@ int ObTablet::rowkey_exists(
}
if (OB_SUCC(ret)) {
ObStoreRowkey rowkey;
ObDatumRowkey datum_rowkey;
ObDatumRowkeyHelper rowkey_helper;
ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "rowkey_acc_ctx"));
ObTableIterParam param;
ObTableAccessContext context;
if (OB_FAIL(rowkey.assign(row.cells_, relative_table.get_rowkey_column_num()))) {
LOG_WARN("Failed to assign rowkey", K(ret), K(row));
} else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) {
LOG_WARN("Failed to transfer datum rowkey", K(ret), K(rowkey));
if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(row, relative_table.get_rowkey_column_num(),
col_descs, datum_rowkey))) {
LOG_WARN("Failed to prepare rowkey", K(ret), K(row));
} else if (OB_FAIL(prepare_param_ctx(allocator, relative_table, store_ctx, param, context))) {
LOG_WARN("Failed to prepare param ctx, ", K(ret), K(rowkey));
LOG_WARN("Failed to prepare param ctx, ", K(ret), K(row));
} else if (OB_FAIL(relative_table.tablet_iter_.get_tablet()->do_rowkey_exists(
param, context, datum_rowkey, exists))) {
LOG_WARN("do rowkey exist fail", K(ret), K(rowkey));
LOG_WARN("do rowkey exist fail", K(ret), K(row), K(datum_rowkey));
}
LOG_DEBUG("chaser debug row", K(ret), K(row), K(rowkey));
LOG_DEBUG("chaser debug row", K(ret), K(row));
}
}
return ret;

14
src/storage/tablet/ob_tablet.h
View File

@ -314,7 +314,7 @@ public:
int insert_rows(
ObRelativeTable &relative_table,
ObStoreCtx &store_ctx,
ObStoreRow *rows,
blocksstable::ObDatumRow *rows,
ObRowsInfo &rows_info,
const bool check_exist,
const ObColDescIArray &col_descs,
@ -325,20 +325,21 @@ public:
ObStoreCtx &store_ctx,
const bool check_exist,
const ObColDescIArray &col_descs,
const storage::ObStoreRow &row,
blocksstable::ObDatumRow &row,
const common::ObIArray<transaction::ObEncryptMetaCache> *encrypt_meta_arr);
int update_row(
ObRelativeTable &relative_table,
ObStoreCtx &store_ctx,
const ObColDescIArray &col_descs,
const ObIArray<int64_t> &update_idx,
const storage::ObStoreRow &old_row,
const storage::ObStoreRow &new_row,
const blocksstable::ObDatumRow &old_row,
blocksstable::ObDatumRow &new_row,
const common::ObIArray<transaction::ObEncryptMetaCache> *encrypt_meta_arr);
int lock_row(
ObRelativeTable &relative_table,
ObStoreCtx &store_ctx,
const common::ObNewRow &row);
ObColDescArray &col_desc,
blocksstable::ObDatumRow &row);
int lock_row(
ObRelativeTable &relative_table,
ObStoreCtx &store_ctx,
@ -399,7 +400,8 @@ public:
int rowkey_exists(
ObRelativeTable &relative_table,
ObStoreCtx &store_ctx,
const common::ObNewRow &row,
const ObColDescIArray &col_descs,
blocksstable::ObDatumRow &row,
bool &exists);
int rowkeys_exists(
ObStoreCtx &store_ctx,

1
src/storage/tablet/ob_tablet_table_store_iterator.cpp
View File

@ -96,6 +96,7 @@ void ObTableStoreIterator::reset()
table_ptr_array_.reset();
sstable_handle_array_.reset();
table_store_handle_.reset();
if (nullptr != transfer_src_table_store_handle_) {
transfer_src_table_store_handle_->~ObStorageMetaHandle();
ob_free(transfer_src_table_store_handle_);

18
src/storage/tx_storage/ob_access_service.cpp
View File

@ -754,7 +754,7 @@ int ObAccessService::delete_rows(
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows)
{
ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write);
@ -808,7 +808,7 @@ int ObAccessService::put_rows(
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows)
{
ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write);
@ -861,7 +861,7 @@ int ObAccessService::insert_rows(
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows)
{
ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write);
@ -916,10 +916,10 @@ int ObAccessService::insert_row(
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObIArray<uint64_t> &duplicated_column_ids,
const common::ObNewRow &row,
blocksstable::ObDatumRow &row,
const ObInsertFlag flag,
int64_t &affected_rows,
common::ObNewRowIterator *&duplicated_rows)
blocksstable::ObDatumRowIterator *&duplicated_rows)
{
ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write);
int ret = OB_SUCCESS;
@ -970,7 +970,7 @@ int ObAccessService::insert_row(
return ret;
}
int ObAccessService::revert_insert_iter(common::ObNewRowIterator *iter)
int ObAccessService::revert_insert_iter(blocksstable::ObDatumRowIterator *iter)
{
ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write);
int ret = OB_SUCCESS;
@ -987,7 +987,7 @@ int ObAccessService::update_rows(
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObIArray< uint64_t> &updated_column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows)
{
ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write);
@ -1043,7 +1043,7 @@ int ObAccessService::lock_rows(
const ObDMLBaseParam &dml_param,
const int64_t abs_lock_timeout,
const ObLockFlag lock_flag,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows)
{
ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write);
@ -1098,7 +1098,7 @@ int ObAccessService::lock_row(
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const int64_t abs_lock_timeout,
const common::ObNewRow &row,
blocksstable::ObDatumRow &row,
const ObLockFlag lock_flag)
{
ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write);

18
src/storage/tx_storage/ob_access_service.h
View File

@ -137,7 +137,7 @@ public:
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
int put_rows(
const share::ObLSID &ls_id,
@ -145,7 +145,7 @@ public:
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
int insert_rows(
const share::ObLSID &ls_id,
@ -153,7 +153,7 @@ public:
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
int insert_row(
const share::ObLSID &ls_id,
@ -162,11 +162,11 @@ public:
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObIArray<uint64_t> &duplicated_column_ids,
const common::ObNewRow &row,
blocksstable::ObDatumRow &row,
const ObInsertFlag flag,
int64_t &affected_rows,
common::ObNewRowIterator *&duplicated_rows);
int revert_insert_iter(common::ObNewRowIterator *iter);
blocksstable::ObDatumRowIterator *&duplicated_rows);
int revert_insert_iter(blocksstable::ObDatumRowIterator *iter);
int update_rows(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
@ -174,7 +174,7 @@ public:
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObIArray< uint64_t> &updated_column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
int lock_rows(
const share::ObLSID &ls_id,
@ -183,7 +183,7 @@ public:
const ObDMLBaseParam &dml_param,
const int64_t abs_lock_timeout, /* -1: undefined, 0: nowait */
const ObLockFlag lock_flag,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
int lock_row(
const share::ObLSID &ls_id,
@ -191,7 +191,7 @@ public:
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const int64_t abs_lock_timeout,
const common::ObNewRow &row,
blocksstable::ObDatumRow &row,
const ObLockFlag lock_flag);
int estimate_row_count(
const ObTableScanParam &param,

84
unittest/storage/blocksstable/test_row_reader.cpp
View File

@ -1326,30 +1326,30 @@ TEST_F(TestNewRowReader, test_write_update_row)
const int64_t num = 200;
const int64_t write_col_cnt = 80;
const int64_t rowkey_cnt = 5;
oceanbase::common::ObObj objs[num];
ObStoreRow writer_row;
writer_row.row_val_.cells_ = objs;
writer_row.row_val_.count_ = column_num;
blocksstable::ObStorageDatum objs[num];
ObDatumRow writer_row;
writer_row.storage_datums_ = objs;
writer_row.count_ = column_num;
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(writer_row));
for (int i = writer_row.row_val_.count_; i < write_col_cnt; ++i) {
writer_row.row_val_.cells_[i] = writer_row.row_val_.cells_[i - writer_row.row_val_.count_];
for (int i = writer_row.count_; i < write_col_cnt; ++i) {
writer_row.storage_datums_[i] = writer_row.storage_datums_[i - writer_row.count_];
}
writer_row.row_val_.count_ = write_col_cnt;
writer_row.count_ = write_col_cnt;
memtable::ObNopBitMap nop_bitmap;
bool read_finished = false;
ret = nop_bitmap.init(writer_row.row_val_.count_, rowkey_cnt);
ret = nop_bitmap.init(writer_row.count_, rowkey_cnt);
ObDatumRow reader_row;
ASSERT_EQ(OB_SUCCESS, reader_row.init(allocator_, writer_row.row_val_.count_));
for (int i = 0; i < writer_row.row_val_.count_; ++i) {
ASSERT_EQ(OB_SUCCESS, reader_row.init(allocator_, writer_row.count_));
for (int i = 0; i < writer_row.count_; ++i) {
reader_row.storage_datums_[i].set_nop();
}
int64_t array[] = {5, 18, 29, 45, 75, 78};
writer_row.row_val_.cells_[array[2]].set_int(0);
writer_row.row_val_.cells_[array[4]].set_int(0);
writer_row.storage_datums_[array[2]].set_int(0);
writer_row.storage_datums_[array[4]].set_int(0);
build_column_read_info(rowkey_cnt, writer_row);
STORAGE_LOG(INFO, "write_row", K(writer_row));
@ -1363,11 +1363,11 @@ TEST_F(TestNewRowReader, test_write_update_row)
int64_t len = 0;
char *buf = nullptr;
writer_row.flag_.set_flag(ObDmlFlag::DF_UPDATE);
writer_row.row_val_.cells_[array[2]].set_int(100 * i);
writer_row.row_val_.cells_[array[4]].set_int(100 * i);
writer_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE);
writer_row.storage_datums_[array[2]].set_int(100 * i);
writer_row.storage_datums_[array[4]].set_int(100 * i);
if (i == 4) {
writer_row.flag_.set_flag(ObDmlFlag::DF_INSERT);
writer_row.row_flag_.set_flag(ObDmlFlag::DF_INSERT);
ret = row_writer[i].write(rowkey_cnt, writer_row, nullptr, buf, len);
} else {
ret = row_writer[i].write(rowkey_cnt, writer_row, &update_idx, buf, len);
@ -1387,7 +1387,7 @@ TEST_F(TestNewRowReader, test_write_update_row)
STORAGE_LOG(INFO, "chaser check writer row", K(read_info_));
int update_pos = 0;
for (int i = 0; i < writer_row.row_val_.count_; ++i) {
for (int i = 0; i < writer_row.count_; ++i) {
bool check_cell_flag = false;
if (i < rowkey_cnt) {
check_cell_flag = true;
@ -1397,12 +1397,10 @@ TEST_F(TestNewRowReader, test_write_update_row)
}
if (check_cell_flag) {
STORAGE_LOG(INFO, "check", K(i), K(update_pos), K(reader_row.storage_datums_[i]));
if (ObNumberFloatType != writer_row.row_val_.cells_[i].get_type()) {
if (i == array[2] || i == array[4]) {
ASSERT_TRUE(reader_row.storage_datums_[i].get_int() == 0);
} else {
ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.row_val_.cells_[i]);
}
if (i == array[2] || i == array[4]) {
ASSERT_TRUE(reader_row.storage_datums_[i].get_int() == 0);
} else {
ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.storage_datums_[i]);
}
} else {
ASSERT_TRUE(!reader_row.storage_datums_[i].is_nop());
@ -1416,28 +1414,28 @@ TEST_F(TestNewRowReader, test_write_write_nop_val)
const int64_t num = 400;
const int64_t write_col_cnt = 302;
const int64_t rowkey_cnt = 3;
oceanbase::common::ObObj objs[num];
ObStoreRow writer_row;
writer_row.row_val_.cells_ = objs;
writer_row.row_val_.count_ = column_num;
blocksstable::ObStorageDatum objs[num];
ObDatumRow writer_row;
writer_row.storage_datums_ = objs;
writer_row.count_ = column_num;
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(writer_row));
int64_t idx = 0;
writer_row.row_val_.cells_[idx++].set_int(14);
writer_row.row_val_.cells_[idx++].set_int(-1658240586131896801);
writer_row.row_val_.cells_[idx++].set_int(-INT64_MAX);
writer_row.storage_datums_[idx++].set_int(14);
writer_row.storage_datums_[idx++].set_int(-1658240586131896801);
writer_row.storage_datums_[idx++].set_int(-INT64_MAX);
for (int i = idx; i < write_col_cnt; ++i) {
writer_row.row_val_.cells_[i].set_nop_value();
writer_row.storage_datums_[i].set_nop();
}
writer_row.row_val_.count_ = write_col_cnt;
writer_row.count_ = write_col_cnt;
memtable::ObNopBitMap nop_bitmap;
bool read_finished = false;
ret = nop_bitmap.init(writer_row.row_val_.count_, rowkey_cnt);
ret = nop_bitmap.init(writer_row.count_, rowkey_cnt);
ObDatumRow reader_row;
ASSERT_EQ(OB_SUCCESS, reader_row.init(allocator_, num));
reader_row.count_ = writer_row.row_val_.count_;
reader_row.count_ = writer_row.count_;
build_column_read_info(rowkey_cnt, writer_row);
int64_t len = 0;
@ -1449,11 +1447,9 @@ TEST_F(TestNewRowReader, test_write_write_nop_val)
ret = row_reader.read_row(buf, len, &read_info_, reader_row);
STORAGE_LOG(INFO, "chaser check writer row", K(read_info_), K(reader_row));
for (int i = 0; i < writer_row.row_val_.count_; ++i) {
for (int i = 0; i < writer_row.count_; ++i) {
STORAGE_LOG(INFO, "check", K(i), K(reader_row.storage_datums_[i]));
if (ObNumberFloatType != writer_row.row_val_.cells_[i].get_type()) {
ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.row_val_.cells_[i]);
}
ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.storage_datums_[i]);
}
}
@ -1504,10 +1500,10 @@ TEST_F(TestNewRowReader, test_update_idx_with_rowkey)
const int64_t write_col_cnt = 80;
const int64_t rowkey_cnt = 3;
oceanbase::common::ObObj objs[num];
ObStoreRow writer_row;
writer_row.row_val_.cells_ = objs;
writer_row.row_val_.count_ = column_num;
blocksstable::ObStorageDatum objs[num];
ObDatumRow writer_row;
writer_row.storage_datums_ = objs;
writer_row.count_ = column_num;
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(writer_row));
int64_t update_array[] = {0, 1, 2, 5, 18};
@ -1515,9 +1511,9 @@ TEST_F(TestNewRowReader, test_update_idx_with_rowkey)
build_column_read_info(rowkey_cnt, writer_row);
int64_t array_idx = 0;
for (int i = 0; i < writer_row.row_val_.count_; ++i) {
for (int i = 0; i < writer_row.count_; ++i) {
if (i != update_array[array_idx]) {
writer_row.row_val_.cells_[i].set_nop_value();
writer_row.storage_datums_[i].set_nop();
} else {
update_idx.push_back(update_array[array_idx]);
++array_idx;

2
unittest/storage/mock_access_service.cpp
View File

@ -32,7 +32,7 @@ int MockObAccessService::insert_rows(
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows)
{
int ret = OB_SUCCESS;

2
unittest/storage/mock_access_service.h
View File

@ -33,7 +33,7 @@ public:
transaction::ObTxDesc &tx_desc,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
blocksstable::ObDatumRowIterator *row_iter,
int64_t &affected_rows);
public:
ObLSTabletService *tablet_service_; // different kinds of mock ls tablet service

1283
unittest/storage/mockcontainer/mock_ob_iterator.cpp
View File

File diff suppressed because it is too large Load Diff

253
unittest/storage/mockcontainer/mock_ob_iterator.h
View File

@ -23,6 +23,7 @@
#include "storage/access/ob_store_row_iterator.h"
#include "storage/access/ob_table_read_info.h"
#include "storage/access/ob_sstable_row_whole_scanner.h"
#include "storage/blocksstable/ob_datum_row_iterator.h"
namespace oceanbase
{
@ -39,10 +40,16 @@ public:
int64_t count() const { return rows_.count(); }
// get row at idx, do not move iter
int get_row(const int64_t idx, const blocksstable::ObDatumRow *&row) const;
int get_row(const int64_t idx, blocksstable::ObDatumRow *&row) const;
// get row at idx, do not move iter
int get_row(const int64_t idx, const storage::ObStoreRow *&row) const;
int get_row(const int64_t idx, storage::ObStoreRow *&row) const;
int get_row(const int64_t idx, common::ObNewRow *&row) const;
// get row and move iter to the next
int get_next_row(const blocksstable::ObDatumRow *&row);
int get_next_row(blocksstable::ObDatumRow *&row);
// get row and move iter to the next
int get_next_row(const storage::ObStoreRow *&row);
int get_next_row(storage::ObStoreRow *&row);
int get_next_row(common::ObNewRow *&row);
@ -52,6 +59,7 @@ public:
// rewind iter
void reset_iter();
int add_row(blocksstable::ObDatumRow *row);
int add_row(storage::ObStoreRow *row);
int from(
const char *cstr,
@ -63,6 +71,24 @@ public:
char escape = '\\',
uint16_t* col_id_array = nullptr,
int64_t *result_col_id_array = nullptr);
int from_for_datum(
const char *cstr,
char escape = '\\',
uint16_t* col_id_array = nullptr,
int64_t *result_col_id_array = nullptr);
int from_for_datum(
const common::ObString &str,
char escape = '\\',
uint16_t* col_id_array = nullptr,
int64_t *result_col_id_array = nullptr);
static bool inner_equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2);
static bool equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2,
const bool cmp_multi_version_row_flag = false, const bool cmp_is_get_and_scan_index = false);
bool equals(int64_t idx, blocksstable::ObDatumRow &row) const;
bool equals(int64_t idx, const blocksstable::ObDatumRow &row) const;
bool equals(blocksstable::ObDatumRow &other_row) const { return equals(0, other_row); }
bool equals(const blocksstable::ObDatumRow &other_row) const { return equals(0, other_row); }
static bool equals(const common::ObNewRow &r1, const common::ObNewRow &r2);
static bool equals(const storage::ObStoreRow &r1, const storage::ObStoreRow &r2,
@ -97,15 +123,15 @@ public:
return ret;
}
template<typename T>
template<typename T, typename T_ROW>
bool equals(T &other_iter, const bool cmp_multi_version_row_flag = false,
const bool cmp_is_get_and_scan_index = false)
{
bool bool_ret = true;
int ret1 = common::OB_SUCCESS;
int ret2 = common::OB_SUCCESS;
const storage::ObStoreRow *other_row = NULL;
storage::ObStoreRow *this_row = NULL;
const T_ROW *other_row = NULL;
T_ROW *this_row = NULL;
int64_t idx = 0;
while (bool_ret) {
@ -155,7 +181,7 @@ private:
void setup_start_cursor();
void advance();
bool end_of_row() const;
common::ObSEArray<blocksstable::ObDatumRow *, DEF_ROW_NUM> datum_rows_;
common::ObSEArray<storage::ObStoreRow *, DEF_ROW_NUM> rows_;
int64_t column_cnt_;
common::ObObjMeta metas_[common::OB_ROW_MAX_COLUMNS_COUNT];
@ -177,6 +203,8 @@ public:
void reset_iter() { return iter_.reset_iter(); }
int from(const char *cstr, char escape = '\\') { return iter_.from(cstr, escape); }
int from(const common::ObString &str, char escape = '\\') { return iter_.from(str, escape); }
int from_for_datum(const char *cstr, char escape = '\\') { return iter_.from_for_datum(cstr, escape); }
int from_for_datum(const common::ObString &str, char escape = '\\') { return iter_.from_for_datum(str, escape); }
bool equals(ROW_TYPE &row) const { return iter_.equals(row); }
bool equals(int64_t idx, ROW_TYPE &row) const { return iter_.equals(idx, row); }
// compare to an iterator, rewind the iter before call this function
@ -219,10 +247,10 @@ public:
private:
ObMockIterator iter_;
};
typedef ObMockRowIterator<storage::ObStoreRowIterator, const storage::ObStoreRow>
ObMockStoreRowIterator;
typedef ObMockRowIterator<storage::ObStoreRowIterator, const storage::ObStoreRow> ObMockStoreRowIterator;
typedef ObMockRowIterator<storage::ObQueryRowIterator, storage::ObStoreRow> ObMockQueryRowIterator;
typedef ObMockRowIterator<common::ObNewRowIterator, common::ObNewRow> ObMockNewRowIterator;
typedef ObMockRowIterator<blocksstable::ObDatumRowIterator, blocksstable::ObDatumRow> ObMockDatumRowIterator;
// init
// parse -> parse->header
@ -270,6 +298,10 @@ public:
static common::ObObjMeta TS_TYPE;
static common::ObObjMeta NU_TYPE;
typedef int (*ObParseDatumFunc)(common::ObIAllocator *,
const common::ObString &,
blocksstable::ObDatumRow &,
int64_t &);
typedef int (*ObParseFunc)(common::ObIAllocator *,
const common::ObString &,
storage::ObStoreRow &,
@ -281,10 +313,19 @@ public:
~ObMockIteratorBuilder() {}
int init(common::ObIAllocator *allocator = NULL, char escape = '\\');
int parse_for_datum(
const common::ObString &str,
ObMockIterator &iter,
uint16_t *col_id_array_list = nullptr);
int parse(
const common::ObString &str,
ObMockIterator &iter,
uint16_t *col_id_array_list = nullptr);
int parse_datum_with_specified_col_ids(
const ObString &str,
ObMockIterator &iter,
uint16_t *col_id_array_list = nullptr,
int64_t *result_col_id_array = nullptr);
int parse_with_specified_col_ids(
const ObString &str,
ObMockIterator &iter,
@ -292,72 +333,164 @@ public:
int64_t *result_col_id_array = nullptr);
private:
static int static_init();
static int parse_varchar(common::ObIAllocator *allocator,
static int prepare_parse_varchar(common::ObIAllocator *allocator,
const common::ObString &word,
const uint16_t count,
int64_t &idx);
static int parse_datum_varchar(common::ObIAllocator *allocator,
const common::ObString &word,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_obj_varchar(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int prepare_parse_lob(common::ObIAllocator *allocator,
const common::ObString &word,
const uint16_t count,
int64_t &idx,
ObLobCommon *&lob_data,
int64_t &val);
static int parse_datum_lob(common::ObIAllocator *allocator,
const common::ObString &word,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_obj_lob(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_lob(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
/*
static int parse_bool(common::ObIAllocator *allocator,
const common::ObString &word, storage::ObStoreRow &row, int64_t &idx);
const common::ObString &word, blocksstable::ObDatumRow &row, int64_t &idx);
*/
static int parse_timestamp(common::ObIAllocator *allocator,
static int prepare_parse_timestamp(common::ObIAllocator *allocator,
const common::ObString &word,
const uint16_t count,
int64_t &idx,
int64_t &usec);
static int parse_datum_timestamp(common::ObIAllocator *allocator,
const common::ObString &word,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_obj_timestamp(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int prepare_parse_int(common::ObIAllocator *allocator,
const common::ObString &word,
const uint16_t count,
int64_t &idx,
int64_t &val);
static int parse_datum_int(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_int(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_bigint(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_number(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_obj_int(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int prepare_parse_bigint(common::ObIAllocator *allocator,
const common::ObString &word,
const uint16_t count,
int64_t &idx,
int64_t &val);
static int parse_datum_bigint(common::ObIAllocator *allocator,
const common::ObString &word,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_obj_bigint(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int prepare_parse_number(common::ObIAllocator *allocator,
const common::ObString &word,
const uint16_t count,
int64_t &idx,
number::ObNumber &nmb);
static int parse_datum_number(common::ObIAllocator *allocator,
const common::ObString &word,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_obj_number(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_dml(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_datum_dml(common::ObIAllocator *allocator,
const common::ObString &word,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_first_dml(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_flag(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int prepare_parse_flag(common::ObIAllocator *allocator,
const common::ObString &word,
int64_t &idx,
int64_t &flag);
static int parse_datum_flag(common::ObIAllocator *allocator,
const common::ObString &word,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_obj_flag(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_base(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_multi_version_row_flag(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_datum_multi_version_row_flag(common::ObIAllocator *allocator,
const common::ObString &word,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_obj_multi_version_row_flag(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_is_get(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_scan_index(common::ObIAllocator *allocator,
static int prepare_parse_scan_index(common::ObIAllocator *allocator,
const common::ObString &word,
int64_t &idx,
int64_t &val);
static int parse_datum_scan_index(common::ObIAllocator *allocator,
const common::ObString &word,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_obj_scan_index(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_datum_trans_id(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
static int parse_trans_id(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
blocksstable::ObDatumRow &row,
int64_t &idx);
static int parse_obj_trans_id(common::ObIAllocator *allocator,
const common::ObString &word,
storage::ObStoreRow &row,
int64_t &idx);
int parse_datum_header(const ObString &str,
int64_t &pos,
ObIArray<ObParseDatumFunc> &header,
int64_t &obj_num,
ObMockIterator &iter);
int parse_header(const common::ObString &str,
int64_t &pos,
common::ObIArray<ObParseFunc> &header,
int64_t &obj_num,
ObMockIterator &iter);
int parse_datum_row(const common::ObString &str,
int64_t &pos,
const common::ObIArray<ObParseDatumFunc> &header,
const uint16_t *col_id_array_list,
blocksstable::ObDatumRow &row);
int parse_row(const common::ObString &str,
int64_t &pos,
const common::ObIArray<ObParseFunc> &header,
@ -388,6 +521,10 @@ private:
static common::hash::ObHashMap<common::ObString, ObParseFunc> str_to_obj_parse_func_;
// hash ObString to row info parse func , such as parse_dml ...
static common::hash::ObHashMap<common::ObString, ObParseFunc> str_to_info_parse_func_;
// hash ObString to obj parse func , such as parse_int ...
static common::hash::ObHashMap<common::ObString, ObParseDatumFunc> str_to_datum_parse_func_;
// hash ObString to row info parse func , such as parse_dml ...
static common::hash::ObHashMap<common::ObString, ObParseDatumFunc> str_to_datum_info_parse_func_;
static common::hash::ObHashMap<common::ObString, common::ObObjMeta*> str_to_obj_type_;
static common::hash::ObHashMap<common::ObString, int64_t> str_to_flag_;
static common::hash::ObHashMap<common::ObString, blocksstable::ObDmlFlag> str_to_dml_;
@ -402,25 +539,25 @@ private:
char escape_;
};
class MockObNewRowIterator: public ObNewRowIterator
{
OB_UNIS_VERSION(1);
public:
MockObNewRowIterator();
~MockObNewRowIterator();
static bool equals(const common::ObNewRow &r1, const common::ObNewRow &r2);
void reset();
int from(const char *cstr, char escape = '\\') { return iter_.from(cstr, escape); }
int from(const common::ObString &str, char escape = '\\') { return iter_.from(str, escape); }
int get_next_row(ObNewRow *&row) { return iter_.get_next_row(row); }
int get_row(const int64_t idex, storage::ObStoreRow *&row) const { return iter_.get_row(idex, row); }
int64_t count() const { return iter_.count(); }
int add_row(storage::ObStoreRow *row) { return iter_.add_row(row); }
// class MockObNewRowIterator: public ObNewRowIterator
// {
// OB_UNIS_VERSION(1);
// public:
// MockObNewRowIterator();
// ~MockObNewRowIterator();
// static bool equals(const common::ObNewRow &r1, const common::ObNewRow &r2);
// void reset();
// int from(const char *cstr, char escape = '\\') { return iter_.from(cstr, escape); }
// int from(const common::ObString &str, char escape = '\\') { return iter_.from(str, escape); }
// int get_next_row(ObNewRow *&row) { return iter_.get_next_row(row); }
// int get_row(const int64_t idex, storage::ObStoreRow *&row) const { return iter_.get_row(idex, row); }
// int64_t count() const { return iter_.count(); }
// int add_row(storage::ObStoreRow *row) { return iter_.add_row(row); }
private:
ObMockIterator iter_;
common::PageArena<char> allocator_;
};
// private:
// ObMockIterator iter_;
// common::PageArena<char> allocator_;
// };
class ObMockDirectReadIterator : public storage::ObStoreRowIterator
{

8
unittest/storage/test_ob_mock_iterator.cpp
View File

@ -429,17 +429,17 @@ TEST_F(TestObMockIteratorBuilder, equals)
TEST_F(TestObMockIteratorBuilder, test_new_row)
{
ObMockNewRowIterator iter1;
ObMockDatumRowIterator iter1;
const char *input1 =
"int var num\n"
"1 2 3\n";
ASSERT_EQ(OB_SUCCESS, iter1.from(input1));
ASSERT_EQ(OB_SUCCESS, iter1.from_for_datum(input1));
ObMockNewRowIterator iter2;
ObMockDatumRowIterator iter2;
const char *input2 =
"int var num\n"
"1 2 4\n";
EXPECT_EQ(OB_SUCCESS, iter2.from(input2));
EXPECT_EQ(OB_SUCCESS, iter2.from_for_datum(input2));
EXPECT_FALSE(iter2.equals(iter1));
}

25
unittest/storage/tx/it/tx_node.cpp
View File

@ -658,12 +658,13 @@ int ObTxNode::write(ObTxDesc &tx,
write_store_ctx));
write_store_ctx.mvcc_acc_ctx_.tx_table_guards_.tx_table_guard_.init(&fake_tx_table_);
ObArenaAllocator allocator;
ObStoreRow row;
ObObj cols[2] = {ObObj(key), ObObj(value)};
row.capacity_ = 2;
row.row_val_.cells_ = cols;
row.row_val_.count_ = 2;
row.flag_ = blocksstable::ObDmlFlag::DF_UPDATE;
ObDatumRow row;
ObStorageDatum cols[2] = {ObStorageDatum(), ObStorageDatum()};
cols[0].set_int(key);
cols[1].set_int(value);
row.count_ = 2;
row.storage_datums_ = cols;
row.row_flag_ = blocksstable::ObDmlFlag::DF_UPDATE;
row.trans_id_.reset();
ObTableIterParam param;
@ -723,11 +724,13 @@ int ObTxNode::write_one_row(ObStoreCtx& write_store_ctx, const int64_t key, cons
const transaction::ObSerializeEncryptMeta *encrypt_meta = NULL;
const int64_t schema_version = 100;
read_info.init(allocator, 2, 1, false, columns_, nullptr/*storage_cols_index*/);
ObStoreRow row;
ObObj cols[2] = {ObObj(key), ObObj(value)};
row.flag_ = blocksstable::ObDmlFlag::DF_UPDATE;
row.row_val_.cells_ = cols;
row.row_val_.count_ = 2;
ObDatumRow row;
ObStorageDatum cols[2] = {ObStorageDatum(), ObStorageDatum()};
cols[0].set_int(key);
cols[1].set_int(value);
row.row_flag_ = blocksstable::ObDmlFlag::DF_UPDATE;
row.storage_datums_ = cols;
row.count_ = 2;
ObTableIterParam param;
ObTableAccessContext context;