From 4edb7b4b37ad607a664ff07d5dd5b989849a8309 Mon Sep 17 00:00:00 2001 From: mjhmllover Date: Tue, 27 Aug 2024 08:31:08 +0000 Subject: [PATCH] [FEAT MERGE] DML stong type refactoring and optimization --- .../test_sstable_row_multi_scanner.cpp | 5 - mittest/mtlenv/storage/test_co_merge.cpp | 39 +- mittest/mtlenv/storage/test_lob_manager.cpp | 4 +- mittest/mtlenv/storage/test_memtable_v2.cpp | 101 +- .../test_multi_version_merge_recycle.cpp | 24 +- .../test_multi_version_sstable_merge.cpp | 81 +- .../test_table_scan_pure_data_table.cpp | 4 +- mittest/mtlenv/storage/test_trans.cpp | 4 +- .../test_callbacks_with_reverse_order.cpp | 2 +- .../table/ob_table_modify_executor.cpp | 4 +- .../ob_table_load_store_trans_px_writer.cpp | 10 +- .../ob_table_load_store_trans_px_writer.h | 2 +- .../table_load/ob_table_load_trans_store.cpp | 25 +- .../table_load/ob_table_load_trans_store.h | 2 +- src/share/datum/ob_datum.h | 10 +- src/sql/das/ob_das_dml_ctx_define.cpp | 90 +- src/sql/das/ob_das_dml_ctx_define.h | 40 +- src/sql/das/ob_das_domain_utils.cpp | 199 ++- src/sql/das/ob_das_domain_utils.h | 6 +- src/sql/das/ob_das_insert_op.cpp | 39 +- src/sql/das/ob_das_insert_op.h | 23 +- src/sql/das/ob_das_update_op.cpp | 21 +- src/sql/das/ob_das_utils.cpp | 47 +- src/sql/das/ob_das_utils.h | 4 +- src/sql/engine/dml/ob_table_insert_up_op.cpp | 4 +- src/sql/engine/dml/ob_table_replace_op.cpp | 4 +- .../expr/ob_expr_pl_get_cursor_attr.cpp | 2 +- src/sql/engine/table/ob_table_scan_op.cpp | 55 +- src/sql/engine/table/ob_table_scan_op.h | 6 +- src/storage/CMakeLists.txt | 3 + src/storage/access/ob_rows_info.cpp | 14 +- src/storage/access/ob_rows_info.h | 6 +- .../access/ob_table_access_context.cpp | 5 +- src/storage/access/ob_table_access_context.h | 3 +- src/storage/blocksstable/ob_datum_row.cpp | 390 +---- src/storage/blocksstable/ob_datum_row.h | 389 +---- .../blocksstable/ob_datum_row_iterator.h | 98 ++ .../blocksstable/ob_datum_row_store.cpp | 242 ++++ src/storage/blocksstable/ob_datum_row_store.h | 92 ++ .../blocksstable/ob_datum_row_utils.cpp | 97 ++ src/storage/blocksstable/ob_datum_row_utils.h | 38 + src/storage/blocksstable/ob_datum_rowkey.cpp | 19 + src/storage/blocksstable/ob_datum_rowkey.h | 7 +- src/storage/blocksstable/ob_row_writer.cpp | 37 +- src/storage/blocksstable/ob_row_writer.h | 6 +- src/storage/blocksstable/ob_storage_datum.cpp | 386 +++++ src/storage/blocksstable/ob_storage_datum.h | 427 ++++++ src/storage/lob/ob_ext_info_callback.cpp | 19 +- src/storage/lob/ob_ext_info_callback.h | 11 +- src/storage/lob/ob_lob_meta.cpp | 4 +- src/storage/lob/ob_lob_meta.h | 4 +- src/storage/lob/ob_lob_persistent_adaptor.cpp | 160 +- src/storage/lob/ob_lob_persistent_adaptor.h | 29 +- .../lob/ob_lob_persistent_iterator.cpp | 24 +- src/storage/lob/ob_lob_persistent_iterator.h | 41 +- src/storage/ls/ob_ls_tablet_service.cpp | 1101 ++++++-------- src/storage/ls/ob_ls_tablet_service.h | 147 +- src/storage/memtable/mvcc/ob_mvcc_ctx.cpp | 5 +- src/storage/memtable/mvcc/ob_mvcc_ctx.h | 3 +- .../memtable/ob_concurrent_control.cpp | 2 +- src/storage/memtable/ob_memtable.cpp | 200 +-- src/storage/memtable/ob_memtable.h | 26 +- src/storage/memtable/ob_memtable_key.cpp | 128 +- src/storage/memtable/ob_memtable_key.h | 53 +- src/storage/ob_dml_running_ctx.cpp | 112 +- src/storage/ob_dml_running_ctx.h | 13 +- src/storage/ob_query_iterator_factory.cpp | 2 +- src/storage/ob_query_iterator_factory.h | 6 +- src/storage/ob_value_row_iterator.cpp | 181 ++- src/storage/ob_value_row_iterator.h | 20 +- src/storage/tablet/ob_tablet.cpp | 31 +- src/storage/tablet/ob_tablet.h | 14 +- .../tablet/ob_tablet_table_store_iterator.cpp | 1 + src/storage/tx_storage/ob_access_service.cpp | 18 +- src/storage/tx_storage/ob_access_service.h | 18 +- .../storage/blocksstable/test_row_reader.cpp | 84 +- unittest/storage/mock_access_service.cpp | 2 +- unittest/storage/mock_access_service.h | 2 +- .../mockcontainer/mock_ob_iterator.cpp | 1283 +++++++++++++---- .../storage/mockcontainer/mock_ob_iterator.h | 253 +++- unittest/storage/test_ob_mock_iterator.cpp | 8 +- unittest/storage/tx/it/tx_node.cpp | 25 +- 82 files changed, 4340 insertions(+), 2806 deletions(-) create mode 100644 src/storage/blocksstable/ob_datum_row_iterator.h create mode 100644 src/storage/blocksstable/ob_datum_row_store.cpp create mode 100644 src/storage/blocksstable/ob_datum_row_store.h create mode 100644 src/storage/blocksstable/ob_datum_row_utils.cpp create mode 100644 src/storage/blocksstable/ob_datum_row_utils.h create mode 100644 src/storage/blocksstable/ob_storage_datum.cpp create mode 100644 src/storage/blocksstable/ob_storage_datum.h diff --git a/mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp b/mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp index b7b803bac..fa958f5d0 100644 --- a/mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_sstable_row_multi_scanner.cpp @@ -769,7 +769,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // first half multi scan, second half multi get ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); @@ -824,7 +823,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // first half multi get, second half multi scan ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); @@ -883,7 +881,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // first one multi get, others multi scan ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); @@ -938,7 +935,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // first one multi scan, others multi get ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); @@ -994,7 +990,6 @@ void TestSSTableRowMultiScanner::test_multi_scan_multi_get_with_scan( // multi scan not exist row STORAGE_LOG(DEBUG, "multi_scan_not_exist_row"); ranges.reuse(); - test_allocator.reuse(); for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) { ObDatumRowkey tmp_rowkey; mget_ranges[i].border_flag_.set_inclusive_start(); diff --git a/mittest/mtlenv/storage/test_co_merge.cpp b/mittest/mtlenv/storage/test_co_merge.cpp index f49df085d..2b50074dd 100644 --- a/mittest/mtlenv/storage/test_co_merge.cpp +++ b/mittest/mtlenv/storage/test_co_merge.cpp @@ -620,7 +620,8 @@ TEST_F(TestCOMerge, test_merge_default_row_store_with_empty_major) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -722,7 +723,8 @@ TEST_F(TestCOMerge, test_merge_default_row_store_with_empty_major) // ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); // ObMockDirectReadIterator sstable_iter; // ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); -// ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); +// bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); +// ASSERT_TRUE(is_equal); // scanner->~ObStoreRowIterator(); // handle1.reset(); // handle2.reset(); @@ -819,7 +821,8 @@ TEST_F(TestCOMerge, test_column_store_merge_with_empty_co_table) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, *cg_read_info)); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); merger.reset(); @@ -946,7 +949,8 @@ TEST_F(TestCOMerge, test_co_merge_with_twice_major) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } handle1.reset(); @@ -1049,7 +1053,8 @@ TEST_F(TestCOMerge, test_co_merge_with_twice_major) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(new_result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1205,7 +1210,8 @@ TEST_F(TestCOMerge, test_merge_range) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1355,7 +1361,8 @@ TEST_F(TestCOMerge, test_merge_range_with_open) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1508,7 +1515,8 @@ TEST_F(TestCOMerge, test_merge_range_with_left_open) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1661,7 +1669,8 @@ TEST_F(TestCOMerge, test_merge_range_with_right_open) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1823,7 +1832,8 @@ TEST_F(TestCOMerge, test_merge_range_left_is_min) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -1988,7 +1998,8 @@ TEST_F(TestCOMerge, test_merge_range_with_right_max) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -2268,7 +2279,8 @@ TEST_F(TestCOMerge, test_merge_range_is_whole_range) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } @@ -2568,7 +2580,8 @@ TEST_F(TestCOMerge, test_rebuild_sstable) ObMockIterator res_iter; res_iter.reset(); ASSERT_EQ(OB_SUCCESS, res_iter.from(result[i])); - ASSERT_TRUE(res_iter.equals(sstable_iter, false/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, false/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); } } diff --git a/mittest/mtlenv/storage/test_lob_manager.cpp b/mittest/mtlenv/storage/test_lob_manager.cpp index d37acf4fa..f8c4b5edf 100644 --- a/mittest/mtlenv/storage/test_lob_manager.cpp +++ b/mittest/mtlenv/storage/test_lob_manager.cpp @@ -328,7 +328,7 @@ void TestLobManager::insert_lob_piece( ASSERT_NE(nullptr, tablet); // insert rows - ObMockNewRowIterator mock_iter; + ObMockDatumRowIterator mock_iter; ObSEArray column_ids; column_ids.push_back(OB_APP_MIN_COLUMN_ID + 0); // pk column_ids.push_back(OB_APP_MIN_COLUMN_ID + 1); // c1 @@ -452,7 +452,7 @@ void TestLobManager::insert_lob_meta( ASSERT_NE(nullptr, tablet); // insert rows - ObMockNewRowIterator mock_iter; + ObMockDatumRowIterator mock_iter; ObSEArray column_ids; for (int i = 0; i < ObLobMetaUtil::LOB_META_COLUMN_CNT; i++) { column_ids.push_back(OB_APP_MIN_COLUMN_ID + i); diff --git a/mittest/mtlenv/storage/test_memtable_v2.cpp b/mittest/mtlenv/storage/test_memtable_v2.cpp index 1c64311f1..3937a4d6d 100644 --- a/mittest/mtlenv/storage/test_memtable_v2.cpp +++ b/mittest/mtlenv/storage/test_memtable_v2.cpp @@ -467,7 +467,7 @@ public: void write_tx(ObStoreCtx *wtx, ObMemtable *memtable, const int64_t snapshot, - const ObStoreRow &write_row, + ObDatumRow &write_row, const int expect_ret = OB_SUCCESS, const int64_t expire_time = 10000000000) { @@ -737,20 +737,23 @@ public: int mock_row(const int64_t key, const int64_t value, ObDatumRowkey &rowkey, - ObStoreRow &row) + ObDatumRow &row) { rowkey_datums_[0].set_int(key); rowkey_datums_[1].set_int(value); rowkey.assign(rowkey_datums_, 1); + ObStorageDatum *datum = new ObStorageDatum[2]; + datum[0].set_int(key); + datum[1].set_int(value); + ObObj *obj = new ObObj[2]; obj[0].set_int(key); obj[1].set_int(value); - row.row_val_.cells_ = obj; - row.row_val_.count_ = 2; - row.row_val_.projector_ = NULL; - row.flag_.set_flag(ObDmlFlag::DF_INSERT); + row.storage_datums_ = datum; + row.count_ = 2; + row.row_flag_.set_flag(ObDmlFlag::DF_INSERT); rowkey.store_rowkey_.assign(obj, 1); return OB_SUCCESS; @@ -758,18 +761,20 @@ public: int mock_delete(const int64_t key, ObDatumRowkey &rowkey, - ObStoreRow &row) + ObDatumRow &row) { rowkey_datums_[0].set_int(key); rowkey.assign(rowkey_datums_, 1); + ObStorageDatum *datum = new ObStorageDatum[1]; + datum[0].set_int(key); + ObObj *obj = new ObObj[1]; obj[0].set_int(key); - row.row_val_.cells_ = obj; - row.row_val_.count_ = 2; - row.row_val_.projector_ = NULL; - row.flag_.set_flag(ObDmlFlag::DF_DELETE); + row.storage_datums_ = datum; + row.count_ = 2; + row.row_flag_.set_flag(ObDmlFlag::DF_DELETE); rowkey.store_rowkey_.assign(obj, 1); return OB_SUCCESS; @@ -1240,7 +1245,7 @@ TEST_F(TestMemtableV2, test_write_read_conflict) TRANS_LOG(INFO, "######## CASE1: write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -1384,7 +1389,7 @@ TEST_F(TestMemtableV2, test_tx_abort) TRANS_LOG(INFO, "######## CASE1: write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -1497,7 +1502,7 @@ TEST_F(TestMemtableV2, test_write_write_conflict) TRANS_LOG(INFO, "######## CASE1: txn1 write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -1542,7 +1547,7 @@ TEST_F(TestMemtableV2, test_write_write_conflict) TRANS_LOG(INFO, "######## CASE2: txn2 write row into memtable, lock for write failed"); ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 3, /*value*/ rowkey2, @@ -1564,7 +1569,7 @@ TEST_F(TestMemtableV2, test_write_write_conflict) TRANS_LOG(INFO, "######## CASE3: txn1 write row into memtable, lock for write succeed"); ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 4, /*value*/ rowkey3, @@ -1713,7 +1718,7 @@ TEST_F(TestMemtableV2, test_lock) TRANS_LOG(INFO, "######## CASE1: txn1 lock row in memtable"); ObDatumRowkey rowkey; - ObStoreRow tmp_row; + ObDatumRow tmp_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -1768,7 +1773,7 @@ TEST_F(TestMemtableV2, test_lock) TRANS_LOG(INFO, "######## CASE3: txn2 write row in memtable with lock for write failed"); ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 3, /*value*/ rowkey2, @@ -1889,7 +1894,7 @@ TEST_F(TestMemtableV2, test_lock) ObTransID write_tx_id3 = ObTransID(3); ObStoreCtx *wtx3 = start_tx(write_tx_id3); ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 4, /*value*/ rowkey3, @@ -1975,7 +1980,7 @@ TEST_F(TestMemtableV2, test_sstable_lock) is_sstable_contains_lock_ = true; ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -2007,8 +2012,8 @@ TEST_F(TestMemtableV2, test_rollback_to) TRANS_LOG(INFO, "######## CASE1: write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; - ObStoreRow write_row2; + ObDatumRow write_row; + ObDatumRow write_row2; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -2070,7 +2075,7 @@ TEST_F(TestMemtableV2, test_rollback_to) wtx_seq_no1 + 1 /*to*/); ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 4, /*value*/ rowkey3, @@ -2099,11 +2104,11 @@ TEST_F(TestMemtableV2, test_replay) TRANS_LOG(INFO, "######## CASE1: txn1 and txn3 write row in lmemtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -2286,9 +2291,9 @@ TEST_F(TestMemtableV2, test_replay) // TRANS_LOG(INFO, "######## CASE1: txn1 write row in lmemtable"); // ObDatumRowkey rowkey; -// ObStoreRow write_row; +// ObDatumRow write_row; // ObDatumRowkey rowkey2; -// ObStoreRow write_row2; +// ObDatumRow write_row2; // EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ // 2, /*value*/ @@ -2421,11 +2426,11 @@ TEST_F(TestMemtableV2, test_compact) TRANS_LOG(INFO, "######## CASE1: txn1 and txn3 write row in lmemtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -2653,11 +2658,11 @@ TEST_F(TestMemtableV2, test_compact_v2) TRANS_LOG(INFO, "######## CASE1: txn1 write two rows and txn3 write a row in memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -2820,13 +2825,13 @@ TEST_F(TestMemtableV2, test_compact_v3) TRANS_LOG(INFO, "######## CASE1: txn1 write two row and txn2 write row in lmemtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; ObDatumRowkey rowkey4; - ObStoreRow write_row4; + ObDatumRow write_row4; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -2958,11 +2963,11 @@ TEST_F(TestMemtableV2, test_dml_flag) TRANS_LOG(INFO, "######## CASE1: txns write and row in lmemtable, test its dml flag"); ObDatumRowkey rowkey; - ObStoreRow write_row1; + ObDatumRow write_row1; ObDatumRowkey rowkey2; - ObStoreRow write_row2; + ObDatumRow write_row2; ObDatumRowkey rowkey3; - ObStoreRow write_row3; + ObDatumRow write_row3; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ @@ -3121,7 +3126,7 @@ TEST_F(TestMemtableV2, test_fast_commit) TRANS_LOG(INFO, "######## CASE1: write row into memtable and fast commit, then check result is ok"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -3241,7 +3246,7 @@ TEST_F(TestMemtableV2, test_fast_commit_with_no_delay_cleanout) TRANS_LOG(INFO, "######## CASE1: write row into memtable and not fast commit, then check result is ok"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -3369,8 +3374,8 @@ TEST_F(TestMemtableV2, test_seq_set_violation) TRANS_LOG(INFO, "######## CASE1: write row into memtable"); ObDatumRowkey rowkey; - ObStoreRow write_row; - ObStoreRow write_row2; + ObDatumRow write_row; + ObDatumRow write_row2; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -3427,7 +3432,7 @@ TEST_F(TestMemtableV2, test_parallel_lock_with_same_txn) TRANS_LOG(INFO, "######## CASE1: lock row into memtable parallelly"); ObDatumRowkey rowkey; - ObStoreRow write_row; + ObDatumRow write_row; EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ 2, /*value*/ rowkey, @@ -3490,14 +3495,14 @@ TEST_F(TestMemtableV2, test_sync_log_fail_on_frozen_memtable) ObStoreCtx *tx_1 = start_tx(txid_1); int i = 1; for (; i <= 10; i++) { - ObStoreRow row1; + ObDatumRow row1; EXPECT_EQ(OB_SUCCESS, mock_row(i, i*2, rowkey, row1)); write_tx(tx_1, memtable, 10000, row1); } ObTransID txid_2 = ObTransID(2); ObStoreCtx *tx_2 = start_tx(txid_2); for (; i <= 20; i++) { - ObStoreRow row1; + ObDatumRow row1; EXPECT_EQ(OB_SUCCESS, mock_row(i, i*4, rowkey, row1)); write_tx(tx_2, memtable, 10000, row1); } diff --git a/mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp b/mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp index d531d5bc4..8304b48f3 100644 --- a/mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp +++ b/mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp @@ -305,7 +305,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycle_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -403,7 +404,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycle_after_reuse) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -499,7 +501,8 @@ TEST_F(TestMultiVersionMergeRecycle, reuse_after_recycle) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -599,7 +602,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycled_micros_after_reuse) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -708,7 +712,8 @@ TEST_F(TestMultiVersionMergeRecycle, rowkeys_across_micros) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -828,7 +833,8 @@ TEST_F(TestMultiVersionMergeRecycle, rowkeys_across_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -925,7 +931,8 @@ TEST_F(TestMultiVersionMergeRecycle, recycle_macro_with_last_row) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -1025,7 +1032,8 @@ TEST_F(TestMultiVersionMergeRecycle, reuse_after_recycle_with_last) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); diff --git a/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp b/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp index 659a45e9e..3a1b47e6f 100644 --- a/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp +++ b/mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp @@ -345,7 +345,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_two_macro_and_second_macro_is_filtere ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -452,7 +453,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_three_macro_inc_merge) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -575,7 +577,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_committed_in_minor) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -667,7 +670,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_in_one_macro_committed_is_last) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -787,7 +791,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_in_one_macro_committed_following_l ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -913,7 +918,8 @@ TEST_F(TestMultiVersionMerge, uncommit_rowkey_in_one_macro_committed_following_s ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1021,7 +1027,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_three_macro_full_merge) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -1175,7 +1182,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1337,7 +1345,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans_can_compact) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1504,7 +1513,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_multi_trans_can_not_compact) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1610,7 +1620,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_macro_reused_with_shadow) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1736,7 +1747,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_macro_reused_without_shadow) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -1820,7 +1832,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_greater_multi_version) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -1931,7 +1944,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_greater_multi_version_and_uncommit ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -2066,7 +2080,8 @@ TEST_F(TestMultiVersionMerge, test_merge_with_ghost_row) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -2151,7 +2166,8 @@ TEST_F(TestMultiVersionMerge, compare_dml_flag) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2250,7 +2266,8 @@ TEST_F(TestMultiVersionMerge, get_last_after_reuse) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2340,7 +2357,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_two_macro_with_commit_scn_less_multi_ ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2461,7 +2479,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_macro_with_last_shadow_version_less_t ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -2562,7 +2581,8 @@ TEST_F(TestMultiVersionMerge, shadow_row_is_last_in_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2684,7 +2704,8 @@ TEST_F(TestMultiVersionMerge, rowkey_cross_macro_without_open_next_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -2792,7 +2813,8 @@ TEST_F(TestMultiVersionMerge, range_cross_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -2933,7 +2955,8 @@ TEST_F(TestMultiVersionMerge, test_merge_base_iter_have_ghost_row) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -3033,7 +3056,8 @@ TEST_F(TestMultiVersionMerge, test_major_range_cross_macro) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); scanner->~ObStoreRowIterator(); handle1.reset(); handle2.reset(); @@ -3150,7 +3174,8 @@ TEST_F(TestMultiVersionMerge, test_trans_cross_macro_with_ghost_row) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -3272,7 +3297,8 @@ TEST_F(TestMultiVersionMerge, test_trans_cross_macro_with_ghost_row2) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); @@ -3387,7 +3413,8 @@ TEST_F(TestMultiVersionMerge, test_running_trans_cross_macro_with_abort_sql_seq) ASSERT_EQ(OB_SUCCESS, res_iter.from(result1)); ObMockDirectReadIterator sstable_iter; ASSERT_EQ(OB_SUCCESS, sstable_iter.init(scanner, allocator_, full_read_info_)); - ASSERT_TRUE(res_iter.equals(sstable_iter, true/*cmp multi version row flag*/)); + bool is_equal = res_iter.equals(sstable_iter, true/*cmp multi version row flag*/); + ASSERT_TRUE(is_equal); ASSERT_EQ(OB_SUCCESS, clear_tx_data()); scanner->~ObStoreRowIterator(); handle1.reset(); diff --git a/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp b/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp index 91cb6d6b1..c03e9d012 100644 --- a/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp +++ b/mittest/mtlenv/storage/test_table_scan_pure_data_table.cpp @@ -97,7 +97,7 @@ void TestTableScanPureDataTable::insert_data_to_tablet(MockObAccessService *acce ASSERT_NE(nullptr, tablet); // insert rows - ObMockNewRowIterator mock_iter; + ObMockDatumRowIterator mock_iter; ObSEArray column_ids; column_ids.push_back(OB_APP_MIN_COLUMN_ID + 0); // pk column_ids.push_back(OB_APP_MIN_COLUMN_ID + 1); // c1 @@ -105,7 +105,7 @@ void TestTableScanPureDataTable::insert_data_to_tablet(MockObAccessService *acce column_ids.push_back(OB_APP_MIN_COLUMN_ID + 3); // c3 column_ids.push_back(OB_APP_MIN_COLUMN_ID + 4); // c4 - ASSERT_EQ(OB_SUCCESS, mock_iter.from(TestDmlCommon::data_row_str)); + ASSERT_EQ(OB_SUCCESS, mock_iter.from_for_datum(TestDmlCommon::data_row_str)); transaction::ObTransService *tx_service = MTL(transaction::ObTransService*); // 1. get tx desc diff --git a/mittest/mtlenv/storage/test_trans.cpp b/mittest/mtlenv/storage/test_trans.cpp index daa2c0142..3b327c3ca 100644 --- a/mittest/mtlenv/storage/test_trans.cpp +++ b/mittest/mtlenv/storage/test_trans.cpp @@ -164,8 +164,8 @@ void TestTrans::create_ls(uint64_t tenant_id, ObLSID &ls_id, ObLS *&ls) void TestTrans::insert_rows(ObLSID &ls_id, ObTabletID &tablet_id, ObTxDesc &tx_desc, ObTxReadSnapshot snapshot, const char* ins_str) { int64_t affected_rows = 0; - ObMockNewRowIterator ins_iter; - ASSERT_EQ(OB_SUCCESS, ins_iter.from(ins_str)); + ObMockDatumRowIterator ins_iter; + ASSERT_EQ(OB_SUCCESS, ins_iter.from_for_datum(ins_str)); ObArenaAllocator allocator; share::schema::ObTableDMLParam table_dml_param(allocator); diff --git a/mittest/simple_server/test_callbacks_with_reverse_order.cpp b/mittest/simple_server/test_callbacks_with_reverse_order.cpp index 8babff814..aef6a21d8 100644 --- a/mittest/simple_server/test_callbacks_with_reverse_order.cpp +++ b/mittest/simple_server/test_callbacks_with_reverse_order.cpp @@ -36,7 +36,7 @@ int ObLSTabletService::insert_tablet_rows( const int64_t row_count, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + ObDatumRow *rows, ObRowsInfo &rows_info) { int ret = OB_SUCCESS; diff --git a/src/observer/table/ob_table_modify_executor.cpp b/src/observer/table/ob_table_modify_executor.cpp index b303f5adc..50610b1cd 100644 --- a/src/observer/table/ob_table_modify_executor.cpp +++ b/src/observer/table/ob_table_modify_executor.cpp @@ -321,11 +321,11 @@ int ObTableApiModifyExecutor::get_next_conflict_rowkey(DASTaskIter &task_iter, bool got_row = false; while (OB_SUCC(ret) && !got_row) { - ObNewRow *dup_row = nullptr; + ObDatumRow *dup_row = nullptr; ObChunkDatumStore::StoredRow *stored_row = nullptr; ObDASWriteBuffer::DmlShadowRow ssr; ObDASInsertOp *ins_op = static_cast(*task_iter); - ObNewRowIterator *conflict_result = ins_op->get_duplicated_result(); + ObDatumRowIterator *conflict_result = ins_op->get_duplicated_result(); const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); if (OB_ISNULL(conflict_result)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp b/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp index 97e83f2eb..a50d1d674 100644 --- a/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp +++ b/src/observer/table_load/ob_table_load_store_trans_px_writer.cpp @@ -186,7 +186,7 @@ int ObTableLoadStoreTransPXWriter::check_status() return ret; } -int ObTableLoadStoreTransPXWriter::write(const ObNewRow &row) +int ObTableLoadStoreTransPXWriter::write(const blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -199,11 +199,13 @@ int ObTableLoadStoreTransPXWriter::write(const ObNewRow &row) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", KR(ret), K(row), K(column_count_)); } else { - ObNewRow new_row; + ObDatumRow new_row; if (is_heap_table_) { - new_row.assign(row.cells_ + 1, row.count_ - 1); + new_row.storage_datums_ = row.storage_datums_ + 1; + new_row.count_ = row.count_ - 1; } else { - new_row.assign(row.cells_, row.count_); + new_row.storage_datums_ = row.storage_datums_; + new_row.count_ = row.count_; } if (OB_FAIL(writer_->px_write(tablet_id_, new_row))) { LOG_WARN("fail to px write", KR(ret), K(row), K(new_row)); diff --git a/src/observer/table_load/ob_table_load_store_trans_px_writer.h b/src/observer/table_load/ob_table_load_store_trans_px_writer.h index 8743d61b2..6590e7075 100644 --- a/src/observer/table_load/ob_table_load_store_trans_px_writer.h +++ b/src/observer/table_load/ob_table_load_store_trans_px_writer.h @@ -38,7 +38,7 @@ public: ObTableLoadTransStoreWriter *writer); int prepare_write(const common::ObTabletID &tablet_id, const common::ObIArray &column_ids); - int write(const common::ObNewRow &row); + int write(const blocksstable::ObDatumRow &row); TO_STRING_KV(KP_(store_ctx), KP_(trans), diff --git a/src/observer/table_load/ob_table_load_trans_store.cpp b/src/observer/table_load/ob_table_load_trans_store.cpp index d41fb2ff8..8fe9f1dde 100644 --- a/src/observer/table_load/ob_table_load_trans_store.cpp +++ b/src/observer/table_load/ob_table_load_trans_store.cpp @@ -317,7 +317,7 @@ int ObTableLoadTransStoreWriter::write(int32_t session_id, return ret; } -int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const ObNewRow &row) +int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -330,22 +330,13 @@ int ObTableLoadTransStoreWriter::px_write(const ObTabletID &tablet_id, const ObN } else { ObTableLoadSequenceNo seq_no(0); // pdml导入的行目前不存在主键冲突，先都用一个默认的seq_no SessionContext &session_ctx = session_ctx_array_[0]; - for (int64_t i = 0; OB_SUCC(ret) && i < table_data_desc_->column_count_; ++i) { - ObStorageDatum &datum = session_ctx.datum_row_.storage_datums_[i]; - const ObObj &obj = row.cells_[i]; - if (OB_FAIL(datum.from_obj_enhance(obj))) { - LOG_WARN("fail to from obj enhance", KR(ret), K(obj)); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(write_row_to_table_store(session_ctx.table_store_, - tablet_id, - seq_no, - session_ctx.datum_row_))) { - LOG_WARN("fail to write row", KR(ret), K(tablet_id)); - } else { - ATOMIC_AAF(&trans_ctx_->ctx_->job_stat_->store_.processed_rows_, 1); - } + if (OB_FAIL(write_row_to_table_store(session_ctx.table_store_, + tablet_id, + seq_no, + row))) { + LOG_WARN("fail to write row", KR(ret), K(tablet_id), K(row)); + } else { + ATOMIC_AAF(&trans_ctx_->ctx_->job_stat_->store_.processed_rows_, 1); } } return ret; diff --git a/src/observer/table_load/ob_table_load_trans_store.h b/src/observer/table_load/ob_table_load_trans_store.h index 338df857b..314920502 100644 --- a/src/observer/table_load/ob_table_load_trans_store.h +++ b/src/observer/table_load/ob_table_load_trans_store.h @@ -74,7 +74,7 @@ public: public: // 只在对应工作线程中调用, 串行执行 int write(int32_t session_id, const table::ObTableLoadTabletObjRowArray &row_array); - int px_write(const ObTabletID &tablet_id, const common::ObNewRow &row); + int px_write(const ObTabletID &tablet_id, const blocksstable::ObDatumRow &row); int flush(int32_t session_id); int clean_up(int32_t session_id); public: diff --git a/src/share/datum/ob_datum.h b/src/share/datum/ob_datum.h index 056c706f2..a1f90f43b 100644 --- a/src/share/datum/ob_datum.h +++ b/src/share/datum/ob_datum.h @@ -128,6 +128,7 @@ struct ObDatumPtr { const char *inner_enumset_; const ObLobCommon *lob_data_; const ObLobLocator *lob_locator_; + const ObMemLobCommon *mem_lob_; const ObObj *extend_obj_; // for extend type const ObDecimalInt *decimal_int_; }; @@ -283,6 +284,10 @@ public: return ObURowIDData(pack_, (const uint8_t *)ptr_); } inline const ObLobLocator &get_lob_locator() const { return *lob_locator_; } + inline void get_mem_lob(ObLobLocatorV2 &lob_locator) const + { + lob_locator.assign_ptr(mem_lob_, len_, has_lob_header()); + } inline const ObLobCommon &get_lob_data() const { return *lob_data_; } inline const ObDecimalInt *get_decimal_int() const { return decimal_int_; } @@ -389,11 +394,6 @@ public: ptr_ = reinterpret_cast(urowid_data.rowid_content_); pack_ = static_cast(urowid_data.rowid_len_); } - inline void set_urowid(const char *ptr, const int64_t size) - { - ptr_ = ptr; - pack_ = static_cast(size); - } inline void set_lob_locator(const ObLobLocator &value) { lob_locator_ = &value; diff --git a/src/sql/das/ob_das_dml_ctx_define.cpp b/src/sql/das/ob_das_dml_ctx_define.cpp index eb7b818de..bd841970c 100644 --- a/src/sql/das/ob_das_dml_ctx_define.cpp +++ b/src/sql/das/ob_das_dml_ctx_define.cpp @@ -19,6 +19,7 @@ #include "sql/engine/dml/ob_dml_service.h" #include "sql/engine/expr/ob_expr_lob_utils.h" #include "storage/access/ob_dml_param.h" +#include "storage/blocksstable/ob_datum_row_utils.h" namespace oceanbase { namespace sql @@ -107,7 +108,7 @@ ObDASDMLIterator::~ObDASDMLIterator() } } -int ObDASDMLIterator::get_next_domain_index_row(ObNewRow *&row) +int ObDASDMLIterator::get_next_domain_index_row(ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(domain_iter_) && OB_FAIL(ObDomainDMLIterator::create_domain_dml_iterator( @@ -121,7 +122,7 @@ int ObDASDMLIterator::get_next_domain_index_row(ObNewRow *&row) return ret; } -int ObDASDMLIterator::get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_count) +int ObDASDMLIterator::get_next_domain_index_rows(ObDatumRow *&rows, int64_t &row_count) { int ret = OB_SUCCESS; if (OB_ISNULL(domain_iter_) && OB_FAIL(ObDomainDMLIterator::create_domain_dml_iterator( @@ -135,25 +136,25 @@ int ObDASDMLIterator::get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_c return ret; } -int ObDASDMLIterator::get_next_row(ObNewRow *&row) +int ObDASDMLIterator::get_next_row(blocksstable::ObDatumRow *&datum_row) { int ret = OB_SUCCESS; - if (OB_ISNULL(cur_row_)) { - if (OB_FAIL(ob_create_row(allocator_, row_projector_->count(), cur_row_))) { - LOG_WARN("create current row failed", K(ret), K(row_projector_)); + if (OB_ISNULL(cur_datum_row_)) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(allocator_, row_projector_->count(), cur_datum_row_))) { + LOG_WARN("create current datum row failed", K(ret), K(row_projector_)); } else if (OB_FAIL(write_buffer_.begin(write_iter_))) { LOG_WARN("begin write iterator failed", K(ret)); } } - if (OB_SUCC(ret) && das_ctdef_->table_param_.get_data_table().is_domain_index()) { - if (OB_FAIL(get_next_domain_index_row(row))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next domain index row", K(ret), K(das_ctdef_->table_param_.get_data_table())); + if (OB_SUCC(ret)) { + if (das_ctdef_->table_param_.get_data_table().is_domain_index()) { + if (OB_FAIL(get_next_domain_index_row(datum_row))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next domain index row", K(ret), K(das_ctdef_->table_param_.get_data_table())); + } } - } - } else { - if (OB_SUCC(ret)) { + } else { const ObChunkDatumStore::StoredRow *sr = nullptr; if (OB_FAIL(write_iter_.get_next_row(sr))) { if (OB_ITER_END != ret) { @@ -163,23 +164,18 @@ int ObDASDMLIterator::get_next_row(ObNewRow *&row) *sr, *row_projector_, allocator_, - *cur_row_))) { + *cur_datum_row_))) { LOG_WARN("project storage row failed", K(ret)); } else { - row = cur_row_; - LOG_TRACE("get next row from dml das iterator", KPC(sr), KPC(row), K(das_ctdef_)); + datum_row = cur_datum_row_; + LOG_TRACE("get next row from dml das iterator", KPC(sr), KPC(datum_row), K(das_ctdef_)); } } } return ret; } -int ObDASDMLIterator::get_next_row() -{ - return OB_NOT_IMPLEMENT; -} - -int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) +int ObDASDMLIterator::get_next_rows(blocksstable::ObDatumRow *&rows, int64_t &row_count) { int ret = OB_SUCCESS; const bool is_domain_index = das_ctdef_->table_param_.get_data_table().is_domain_index(); @@ -193,8 +189,8 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) row_count = 1; } } else { - if (OB_ISNULL(cur_rows_)) { - if (OB_FAIL(ob_create_rows(allocator_, batch_size_, row_projector_->count(), cur_rows_))) { + if (OB_ISNULL(cur_datum_rows_)) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_rows(allocator_, batch_size_, row_projector_->count(), cur_datum_rows_))) { LOG_WARN("Failed to create rows", K(ret), K_(row_projector)); } else if (OB_FAIL(write_buffer_.begin(write_iter_))) { LOG_WARN("Failed to begin write iterator", K(ret)); @@ -215,18 +211,18 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) *sr, *row_projector_, allocator_, - cur_rows_[row_count]))) { + cur_datum_rows_[row_count]))) { LOG_WARN("Failed to project storage row", K(ret)); } else { ++row_count; - LOG_TRACE("Get next rows from dml das iterator", KPC(sr), K(cur_rows_[row_count - 1]), K_(das_ctdef)); + LOG_TRACE("Get next rows from dml das iterator", KPC(sr), K(cur_datum_rows_[row_count - 1]), K_(das_ctdef)); } } if (OB_SUCC(ret) || OB_LIKELY(OB_ITER_END == ret)) { if (0 == row_count) { ret = OB_ITER_END; } else { - rows = cur_rows_; + rows = cur_datum_rows_; ret = OB_SUCCESS; } } @@ -238,8 +234,8 @@ int ObDASDMLIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) int ObDASDMLIterator::rewind(const ObDASDMLBaseCtDef *das_ctdef) { int ret = common::OB_SUCCESS; - cur_row_ = nullptr; - cur_rows_ = nullptr; + cur_datum_row_ = nullptr; + cur_datum_rows_ = nullptr; set_ctdef(das_ctdef); if (OB_NOT_NULL(domain_iter_)) { if (OB_FAIL(domain_iter_->rewind())) { @@ -269,7 +265,7 @@ void ObDASDMLIterator::set_ctdef(const ObDASDMLBaseCtDef *das_ctdef) } } -int ObDASMLogDMLIterator::get_next_row(ObNewRow *&row) +int ObDASMLogDMLIterator::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(row_iter_)) { @@ -368,27 +364,29 @@ int ObDASWriteBuffer::DmlShadowRow::shadow_copy(const ObIArray &exprs, return ret; } -int ObDASWriteBuffer::DmlShadowRow::shadow_copy(const ObNewRow &row) +int ObDASWriteBuffer::DmlShadowRow::shadow_copy(const blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; - if (OB_ISNULL(store_row_) || OB_UNLIKELY(store_row_->cnt_ != row.get_count())) { + if (OB_ISNULL(store_row_) || OB_UNLIKELY(store_row_->cnt_ != row.get_column_count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("NULL datums or count mismatch", K(ret), KPC(store_row_), K(row)); } else { ObDatum *cells = store_row_->cells(); - for (int64_t i = 0; OB_SUCC(ret) && i < row.get_count(); ++i) { - if (lib::is_oracle_mode() && row.get_cell(i).is_lob_locator() && strip_lob_locator_) { + ObObjDatumMapType map_type = OBJ_DATUM_MAPPING_MAX; + for (int64_t i = 0; OB_SUCC(ret) && i < row.get_column_count(); ++i) { + if (lib::is_oracle_mode() && column_types_ != nullptr && column_types_->at(i).is_lob_locator() && strip_lob_locator_) { ObString payload; - if (column_types_ != nullptr && !column_types_->at(i).is_lob()) { + if (!column_types_->at(i).is_lob()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid column type", K(ret), K(column_types_->at(i))); - } else if (OB_FAIL(row.get_cell(i).get_lob_locator()->get_payload(payload))) { + } else if (OB_FAIL(row.storage_datums_[i].get_lob_locator().get_payload(payload))) { LOG_WARN("get lob payload failed", K(ret)); } else { cells[i].set_string(payload); } - } else if (OB_FAIL(cells[i].from_obj(row.get_cell(i)))) { - LOG_WARN("shadow copy obj failed", K(ret), K(i), K(row)); + } else if (FALSE_IT(map_type = ObDatum::get_obj_datum_map_type(column_types_->at(i).get_type()))) { + } else if (OB_FAIL(cells[i].from_storage_datum(row.storage_datums_[i], map_type))) { + LOG_WARN("shadow copy storage datum failed", K(ret), K(i), K(row)); } if (OB_SUCC(ret)) { //add the data length of datum @@ -671,7 +669,7 @@ int ObDASWriteBuffer::begin(NewRowIterator &it, const ObIArray &col_t } } if (OB_SUCC(ret)) { - if (OB_FAIL(ob_create_row(*das_alloc_, col_types.count(), it.cur_new_row_))) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(*das_alloc_, col_types.count(), it.cur_new_row_))) { LOG_WARN("create new row failed", K(ret)); } else { it.col_types_ = &col_types; @@ -683,8 +681,8 @@ int ObDASWriteBuffer::begin(NewRowIterator &it, const ObIArray &col_t int ObDASWriteBuffer::dump_data(const ObDASDMLBaseCtDef &das_base_ctdef) const { int ret = OB_SUCCESS; - ObNewRow *new_row = NULL; - ObNewRow *old_row = NULL; + blocksstable::ObDatumRow *new_row = NULL; + blocksstable::ObDatumRow *old_row = NULL; const ObChunkDatumStore::StoredRow *store_row = NULL; int64_t rownum = 0; ObArenaAllocator tmp_alloc; @@ -705,7 +703,7 @@ int ObDASWriteBuffer::dump_data(const ObDASDMLBaseCtDef &das_base_ctdef) const if (!das_base_ctdef.old_row_projector_.empty()) { // create old row if (OB_ISNULL(old_row) - && OB_FAIL(ob_create_row(tmp_alloc, das_base_ctdef.old_row_projector_.count(), old_row))) { + && OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(tmp_alloc, das_base_ctdef.old_row_projector_.count(), old_row))) { LOG_WARN("create old row buffer failed", K(ret), K(das_base_ctdef.old_row_projector_.count())); } else if (OB_FAIL(ObDASUtils::project_storage_row(das_base_ctdef, *store_row, @@ -720,7 +718,7 @@ int ObDASWriteBuffer::dump_data(const ObDASDMLBaseCtDef &das_base_ctdef) const if (OB_SUCC(ret)) { if (!das_base_ctdef.new_row_projector_.empty()) { if (OB_ISNULL(new_row) - && OB_FAIL(ob_create_row(tmp_alloc, das_base_ctdef.new_row_projector_.count(), new_row))) { + && OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(tmp_alloc, das_base_ctdef.new_row_projector_.count(), new_row))) { LOG_WARN("create new row buffer failed", K(ret), K(das_base_ctdef.new_row_projector_.count())); } else if (OB_FAIL(ObDASUtils::project_storage_row(das_base_ctdef, *store_row, @@ -905,7 +903,7 @@ int ObDASWriteBuffer::Iterator::get_next_row_skip_const(ObEvalCtx &ctx, const Ob return ret; } -int ObDASWriteBuffer::NewRowIterator::get_next_row(ObNewRow *&row) +int ObDASWriteBuffer::NewRowIterator::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; const DmlRow *sr = nullptr; @@ -924,9 +922,7 @@ int ObDASWriteBuffer::NewRowIterator::get_next_row(ObNewRow *&row) if (OB_SUCC(ret) && sr != nullptr) { for (int64_t i = 0; OB_SUCC(ret) && i < sr->cnt_; ++i) { - if (OB_FAIL(sr->cells()[i].to_obj(cur_new_row_->cells_[i], col_types_->at(i)))) { - LOG_WARN("convert datum to ObNewRow failed", K(ret)); - } + cur_new_row_->storage_datums_[i].shallow_copy_from_datum(sr->cells()[i]); } if (OB_SUCC(ret)) { row = cur_new_row_; diff --git a/src/sql/das/ob_das_dml_ctx_define.h b/src/sql/das/ob_das_dml_ctx_define.h index 17e3a5448..d38d79710 100644 --- a/src/sql/das/ob_das_dml_ctx_define.h +++ b/src/sql/das/ob_das_dml_ctx_define.h @@ -22,6 +22,7 @@ #include "sql/engine/ob_operator.h" #include "sql/resolver/dml/ob_hint.h" #include "storage/fts/ob_fts_plugin_helper.h" +#include "storage/blocksstable/ob_datum_row_iterator.h" namespace oceanbase { namespace storage @@ -33,7 +34,8 @@ namespace sql typedef common::ObFixedArray ObjMetaFixedArray; typedef common::ObFixedArray AccuracyFixedArray; static const int64_t SAPTIAL_INDEX_DEFAULT_ROW_COUNT = 32; // 一个wkb生成的cellid数量（设定值） -typedef common::ObSEArray ObDomainIndexRow; +static const int64_t SAPTIAL_INDEX_DEFAULT_COL_COUNT = 2; +typedef common::ObSEArray ObDomainIndexRow; class ObDomainDMLIterator; struct ObDASDMLBaseRtDef; @@ -286,7 +288,7 @@ public: const common::ObIArray &col_types, bool strip_lob_locator); virtual int shadow_copy(const common::ObIArray &exprs, ObEvalCtx &ctx) override; - int shadow_copy(const common::ObNewRow &row); + int shadow_copy(const blocksstable::ObDatumRow &row); // reset && release referenced memory virtual void reset() { @@ -355,13 +357,13 @@ public: { } ~NewRowIterator() { } - int get_next_row(ObNewRow *&row); + int get_next_row(blocksstable::ObDatumRow *&row); inline bool is_inited() { return nullptr != col_types_; } private: DmlRow *cur_row_; ObChunkDatumStore::Iterator *datum_iter_; const common::ObIArray *col_types_; - ObNewRow *cur_new_row_; + blocksstable::ObDatumRow *cur_new_row_; }; OB_UNIS_VERSION(1); @@ -459,7 +461,7 @@ private: uint32_t row_extend_size_; }; -class ObDASDMLIterator : public common::ObNewRowIterator +class ObDASDMLIterator : public blocksstable::ObDatumRowIterator { public: static const int64_t DEFAULT_BATCH_SIZE = 256; @@ -471,8 +473,8 @@ public: das_ctdef_(das_ctdef), row_projector_(nullptr), allocator_(alloc), - cur_row_(nullptr), - cur_rows_(nullptr), + cur_datum_row_(nullptr), + cur_datum_rows_(nullptr), main_ctdef_(das_ctdef), domain_iter_(nullptr) { @@ -480,37 +482,37 @@ public: batch_size_ = MIN(write_buffer_.get_row_cnt(), DEFAULT_BATCH_SIZE); } virtual ~ObDASDMLIterator(); - virtual int get_next_row(common::ObNewRow *&row) override; - virtual int get_next_row() override; - virtual int get_next_rows(ObNewRow *&rows, int64_t &row_count); + virtual int get_next_row(blocksstable::ObDatumRow *&datum_row) override; + virtual int get_next_rows(blocksstable::ObDatumRow *&rows, int64_t &row_count); ObDASWriteBuffer &get_write_buffer() { return write_buffer_; } virtual void reset() override { } int rewind(const ObDASDMLBaseCtDef *das_ctdef); private: void set_ctdef(const ObDASDMLBaseCtDef *das_ctdef); - int get_next_domain_index_row(ObNewRow *&row); - int get_next_domain_index_rows(ObNewRow *&rows, int64_t &row_count); + int get_next_domain_index_row(blocksstable::ObDatumRow *&row); + int get_next_domain_index_rows(blocksstable::ObDatumRow *&rows, int64_t &row_count); private: ObDASWriteBuffer &write_buffer_; const ObDASDMLBaseCtDef *das_ctdef_; const IntFixedArray *row_projector_; ObDASWriteBuffer::Iterator write_iter_; common::ObIAllocator &allocator_; - common::ObNewRow *cur_row_; - common::ObNewRow *cur_rows_; + blocksstable::ObDatumRow *cur_datum_row_; + blocksstable::ObDatumRow *cur_datum_rows_; const ObDASDMLBaseCtDef *main_ctdef_; ObDomainDMLIterator *domain_iter_; int64_t batch_size_; }; -class ObDASMLogDMLIterator : public ObNewRowIterator +class ObDASMLogDMLIterator : public blocksstable::ObDatumRowIterator { public: + // support get next datum row ObDASMLogDMLIterator( const ObTabletID &tablet_id, const storage::ObDMLBaseParam &dml_param, - ObNewRowIterator *iter, + ObDatumRowIterator *iter, ObDASOpType op_type) : tablet_id_(tablet_id), dml_param_(dml_param), @@ -524,14 +526,12 @@ public: } } virtual ~ObDASMLogDMLIterator() {} - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override { return OB_NOT_IMPLEMENT; } - virtual void reset() override {} + virtual int get_next_row(blocksstable::ObDatumRow *&datum_row) override; private: const ObTabletID &tablet_id_; const storage::ObDMLBaseParam &dml_param_; - ObNewRowIterator *row_iter_; + ObDatumRowIterator *row_iter_; ObDASOpType op_type_; bool is_old_row_; }; diff --git a/src/sql/das/ob_das_domain_utils.cpp b/src/sql/das/ob_das_domain_utils.cpp index 80af2f483..a62781e6d 100644 --- a/src/sql/das/ob_das_domain_utils.cpp +++ b/src/sql/das/ob_das_domain_utils.cpp @@ -19,6 +19,7 @@ #include "sql/das/ob_das_utils.h" #include "sql/engine/expr/ob_expr_lob_utils.h" #include "observer/omt/ob_tenant_srs.h" +#include "storage/blocksstable/ob_datum_row_utils.h" using namespace oceanbase::common; @@ -59,10 +60,11 @@ int ObDASDomainUtils::generate_spatial_index_rows( } else { ObS2Adapter s2object(&allocator, srid != 0 ? srs_item->is_geographical_srs() : false); ObSpatialMBR spa_mbr; - ObObj *obj_arr = NULL; ObS2Cellids cellids; - char *mbr = NULL; + char *mbr = nullptr; int64_t mbr_len = 0; + void *rows_buf = nullptr; + blocksstable::ObDatumRow *rows = nullptr; if (OB_FAIL(s2object.init(wkb_str, srs_bound))) { LOG_WARN("Init s2object failed", K(ret)); } else if (OB_FAIL(s2object.get_cellids(cellids, false))) { @@ -81,38 +83,35 @@ int ObDASDomainUtils::generate_spatial_index_rows( LOG_WARN("failed to alloc memory for spatial index row mbr", K(ret)); } else if (OB_FAIL(spa_mbr.to_char(mbr, mbr_len))) { LOG_WARN("failed transform ObSpatialMBR to string", K(ret)); + } else if (OB_ISNULL(rows_buf = reinterpret_cast(allocator.alloc(cellids.size() * sizeof(blocksstable::ObDatumRow))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for spatial index rows buffer", K(ret)); } else { + rows = new (rows_buf) blocksstable::ObDatumRow[cellids.size()]; + int64_t cellid_col_idx = 0; + int64_t mbr_col_idx = 1; for (uint64_t i = 0; OB_SUCC(ret) && i < cellids.size(); i++) { - if (OB_ISNULL(obj_arr = reinterpret_cast(allocator.alloc(sizeof(ObObj) * rowkey_num)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for spatial index row cells", K(ret)); + if (OB_FAIL(rows[i].init(allocator, rowkey_num))) { + LOG_WARN("init datum row failed", K(ret), K(rowkey_num)); } else { // 索引行[cellid_obj][mbr_obj][rowkey_obj] for(uint64_t j = 0; OB_SUCC(ret) && j < rowkey_num; j++) { - obj_arr[j].set_nop_value(); const ObObjMeta &col_type = das_ctdef.column_types_.at(j); const ObAccuracy &col_accuracy = das_ctdef.column_accuracys_.at(j); int64_t projector_idx = row_projector.at(j); - if (OB_FAIL(dml_row.cells()[projector_idx].to_obj(obj_arr[j], col_type))) { - LOG_WARN("stored row to new row obj failed", K(ret), - K(dml_row.cells()[projector_idx]), K(col_type), K(projector_idx), K(j)); - } else if (OB_FAIL(ObDASUtils::reshape_storage_value(col_type, col_accuracy, allocator, obj_arr[j]))) { + if (FALSE_IT(rows[i].storage_datums_[j].shallow_copy_from_datum(dml_row.cells()[projector_idx]))) { + } else if (rows[i].storage_datums_[j].is_null()) { + } else if (OB_FAIL(ObDASUtils::reshape_datum_value(col_type, col_accuracy, true, allocator, rows[i].storage_datums_[j]))) { LOG_WARN("reshape storage value failed", K(ret), K(col_type), K(projector_idx), K(j)); } } if (OB_SUCC(ret)) { - int64_t cellid_col_idx = 0; - int64_t mbr_col_idx = 1; - obj_arr[cellid_col_idx].set_uint64(cellids.at(i)); + rows[i].storage_datums_[cellid_col_idx].set_uint(cellids.at(i)); ObString mbr_val(mbr_len, mbr); - obj_arr[mbr_col_idx].set_varchar(mbr_val); - obj_arr[mbr_col_idx].set_collation_type(CS_TYPE_BINARY); - obj_arr[mbr_col_idx].set_collation_level(CS_LEVEL_IMPLICIT); - ObNewRow row; - row.cells_ = obj_arr; - row.count_ = rowkey_num; - if (OB_FAIL(spat_rows.push_back(row))) { - LOG_WARN("failed to push back spatial index row", K(ret), K(row)); + rows[i].storage_datums_[mbr_col_idx].set_string(mbr_val); + // not set_collation_type(CS_TYPE_BINARY) and set_collation_level(CS_LEVEL_IMPLICIT) + if (OB_FAIL(spat_rows.push_back(&rows[i]))) { + LOG_WARN("failed to push back spatial index row", K(ret), K(rows[i])); } } } @@ -137,7 +136,8 @@ int ObDASDomainUtils::generate_spatial_index_rows( const int64_t ft_word_bkt_cnt = MAX(fulltext.length() / 10, 2); int64_t doc_length = 0; ObFTWordMap ft_word_map; - ObObj *obj_arr = nullptr; + void *rows_buf = nullptr; + blocksstable::ObDatumRow *rows = nullptr; if (OB_ISNULL(helper) || OB_UNLIKELY(!ft_obj_meta.is_valid()) || OB_UNLIKELY(doc_id.length() != sizeof(ObDocId) || doc_id.empty()) ) { @@ -153,47 +153,41 @@ int ObDASDomainUtils::generate_spatial_index_rows( K(ft_obj_meta.get_collation_type()), K(fulltext)); } else if (0 == ft_word_map.size()) { ret = OB_ITER_END; - } else { - const int64_t obj_cnt = ft_word_map.size() * FT_WORD_DOC_COL_CNT; - const int64_t obj_arr_size = sizeof(ObObj) * obj_cnt; - if (OB_ISNULL(obj_arr = reinterpret_cast(allocator.alloc(obj_arr_size)))) { + } else if (OB_ISNULL(rows_buf = reinterpret_cast(allocator.alloc(ft_word_map.size() * sizeof(blocksstable::ObDatumRow))))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate obj array", K(ret), K(obj_arr_size)); - } else { - for (int64_t i = 0; i < obj_cnt; ++i) { - new (obj_arr + i) ObObj(); - } - } + LOG_WARN("failed to alloc memory for full text index rows buffer", K(ret)); + } else { int64_t i = 0; + rows = new (rows_buf) blocksstable::ObDatumRow[ft_word_map.size()]; for (ObFTWordMap::const_iterator iter = ft_word_map.begin(); OB_SUCC(ret) && iter != ft_word_map.end(); ++iter) { - const ObFTWord &ft_word = iter->first; - const int64_t word_cnt = iter->second; - // index row format - // - FTS_INDEX: [WORD], [DOC_ID], [WORD_COUNT] - // - FTS_DOC_WORD: [DOC_ID], [WORD], [WORD_COUNT] - const int64_t word_idx = is_fts_index_aux ? 0 : 1; - const int64_t doc_id_idx = is_fts_index_aux ? 1 : 0; - const int64_t word_cnt_idx = 2; - const int64_t doc_len_idx = 3; - obj_arr[i * FT_WORD_DOC_COL_CNT + word_idx].set_varchar(ft_word.get_word()); - obj_arr[i * FT_WORD_DOC_COL_CNT + word_idx].set_meta_type(ft_obj_meta); - obj_arr[i * FT_WORD_DOC_COL_CNT + doc_id_idx].set_varbinary(doc_id); - obj_arr[i * FT_WORD_DOC_COL_CNT + word_cnt_idx].set_uint64(word_cnt); - obj_arr[i * FT_WORD_DOC_COL_CNT + doc_len_idx].set_uint64(doc_length); - ObNewRow row; - row.cells_ = &(obj_arr[i * FT_WORD_DOC_COL_CNT]); - row.count_ = FT_WORD_DOC_COL_CNT; - if (OB_FAIL(word_rows.push_back(row))) { - LOG_WARN("fail to push back row", K(ret), K(row)); + if (OB_FAIL(rows[i].init(allocator, FT_WORD_DOC_COL_CNT))) { + LOG_WARN("init datum row failed", K(ret), K(FT_WORD_DOC_COL_CNT)); } else { - ObDocId tmp_doc_id; - tmp_doc_id.tablet_id_ = ((const uint64_t *)doc_id.ptr())[0]; - tmp_doc_id.seq_id_ = ((const uint64_t *)doc_id.ptr())[1]; - STORAGE_FTS_LOG(DEBUG, "succeed to add word row", K(ret), K(is_fts_index_aux), "doc_id", tmp_doc_id, - K(ft_word), K(word_cnt), K(i), K(row)); - ++i; + const ObFTWord &ft_word = iter->first; + const int64_t word_cnt = iter->second; + // index row format + // - FTS_INDEX: [WORD], [DOC_ID], [WORD_COUNT] + // - FTS_DOC_WORD: [DOC_ID], [WORD], [WORD_COUNT] + const int64_t word_idx = is_fts_index_aux ? 0 : 1; + const int64_t doc_id_idx = is_fts_index_aux ? 1 : 0; + const int64_t word_cnt_idx = 2; + const int64_t doc_len_idx = 3; + rows[i].storage_datums_[word_idx].set_string(ft_word.get_word()); + rows[i].storage_datums_[doc_id_idx].set_string(doc_id); + rows[i].storage_datums_[word_cnt_idx].set_uint(word_cnt); + rows[i].storage_datums_[doc_len_idx].set_uint(doc_length); + if (OB_FAIL(word_rows.push_back(&rows[i]))) { + LOG_WARN("fail to push back row", K(ret), K(rows[i])); + } else { + ObDocId tmp_doc_id; + tmp_doc_id.tablet_id_ = ((const uint64_t *)doc_id.ptr())[0]; + tmp_doc_id.seq_id_ = ((const uint64_t *)doc_id.ptr())[1]; + STORAGE_FTS_LOG(DEBUG, "succeed to add word row", K(ret), K(is_fts_index_aux), "doc_id", tmp_doc_id, + K(ft_word), K(word_cnt), K(i), K(rows[i])); + ++i; + } } } } @@ -332,65 +326,66 @@ int ObDASDomainUtils::generate_multivalue_index_rows(ObIAllocator &allocator, uint64_t rowkey_column_start = column_num - 1 - data_table_rowkey_cnt; uint64_t rowkey_column_end = column_num - 1; + void *rows_buf = nullptr; + if (OB_FAIL(get_pure_mutivalue_data(json_str, data, data_len, record_num))) { LOG_WARN("failed to parse binary.", K(ret), K(json_str)); } else if (record_num == 0 && (is_unique_index && rowkey_column_start == 1)) { } else if (OB_FAIL(calc_save_rowkey_policy(allocator, das_ctdef, row_projector, dml_row, record_num, is_save_rowkey))) { LOG_WARN("failed to calc store policy.", K(ret), K(data_table_rowkey_cnt)); + } else if (OB_ISNULL(rows_buf = reinterpret_cast(allocator.alloc(record_num * sizeof(blocksstable::ObDatumRow))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for multi value index rows buffer", K(ret)); } else { - ObObj *obj_arr = nullptr; int64_t pos = sizeof(uint32_t); + blocksstable::ObDatumRow *rows = new (rows_buf) blocksstable::ObDatumRow[record_num]; + ObObj obj; for (int i = 0; OB_SUCC(ret) && (i < record_num || !is_none_unique_done) ; ++i) { - if (OB_ISNULL(obj_arr = reinterpret_cast(allocator.alloc(sizeof(ObObj) * column_num)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for multivalue index row cells", K(ret)); - } - for(uint64_t j = 0; OB_SUCC(ret) && j < column_num; j++) { - obj_arr[j].set_nop_value(); - ObObjMeta col_type = das_ctdef.column_types_.at(j); - const ObAccuracy &col_accuracy = das_ctdef.column_accuracys_.at(j); - int64_t projector_idx = row_projector.at(j); + if (OB_FAIL(rows[i].init(allocator, column_num))) { + LOG_WARN("init datum row failed", K(ret), K(column_num)); + } else { + for(uint64_t j = 0; OB_SUCC(ret) && j < column_num; j++) { + ObObjMeta col_type = das_ctdef.column_types_.at(j); + const ObAccuracy &col_accuracy = das_ctdef.column_accuracys_.at(j); + int64_t projector_idx = row_projector.at(j); - if (multivalue_idx == projector_idx) { - if (OB_FAIL(obj_arr[j].deserialize(data, data_len, pos))) { - LOG_WARN("failed to deserialize datum.", K(ret), K(json_str)); - } else { - if (ob_is_number_or_decimal_int_tc(col_type.get_type()) || ob_is_temporal_type(col_type.get_type())) { - col_type.set_collation_level(CS_LEVEL_NUMERIC); + if (multivalue_idx == projector_idx) { + // TODO: change obj to datum when do deserialize@xuanxi + obj.set_nop_value(); + if (OB_FAIL(obj.deserialize(data, data_len, pos))) { + LOG_WARN("failed to deserialize datum", K(ret), K(json_str)); + } else if (OB_FAIL(rows[i].storage_datums_[j].from_obj_enhance(obj))) { + LOG_WARN("failed to convert datum from obj", K(ret), K(obj)); } else { - col_type.set_collation_level(CS_LEVEL_IMPLICIT); + if (ob_is_number_or_decimal_int_tc(col_type.get_type()) || ob_is_temporal_type(col_type.get_type())) { + col_type.set_collation_level(CS_LEVEL_NUMERIC); + } else { + col_type.set_collation_level(CS_LEVEL_IMPLICIT); + } + is_none_unique_done = true; } - - obj_arr[j].set_collation_level(col_type.get_collation_level()); - obj_arr[j].set_collation_type(col_type.get_collation_type()); - obj_arr[j].set_type(col_type.get_type()); - is_none_unique_done = true; + } else if (!is_save_rowkey && (rowkey_column_start >= j && j < rowkey_column_end)) { + rows[i].storage_datums_[j].set_null(); + } else if (multivalue_arr_idx == projector_idx) { + rows[i].storage_datums_[j].set_null(); + } else { + rows[i].storage_datums_[j].shallow_copy_from_datum(dml_row.cells()[projector_idx]); + } + + if (OB_SUCC(ret) && OB_FAIL(ObDASUtils::reshape_datum_value(col_type, col_accuracy, true, allocator, rows[i].storage_datums_[j]))) { + LOG_WARN("reshape storage value failed", K(ret), K(col_type), K(projector_idx), K(j)); } - } else if (!is_save_rowkey && (rowkey_column_start >= j && j < rowkey_column_end)) { - obj_arr[j].set_null(); - } else if (multivalue_arr_idx == projector_idx) { - obj_arr[j].set_null(); - } else if (OB_FAIL(dml_row.cells()[projector_idx].to_obj(obj_arr[j], col_type))) { - LOG_WARN("stored row to new row obj failed", K(ret), - K(dml_row.cells()[projector_idx]), K(col_type), K(projector_idx), K(j)); } - if (OB_SUCC(ret) && OB_FAIL(ObDASUtils::reshape_storage_value(col_type, col_accuracy, allocator, obj_arr[j]))) { - LOG_WARN("reshape storage value failed", K(ret), K(col_type), K(projector_idx), K(j)); - } + if (OB_SUCC(ret)) { + if (OB_FAIL(mvi_rows.push_back(&rows[i]))) { + LOG_WARN("failed to push back spatial index row", K(ret), K(rows[i])); + } + } // end if (OB_SUCC(ret)) } - - if (OB_SUCC(ret)) { - ObNewRow row; - row.cells_ = obj_arr; - row.count_ = column_num; - if (OB_FAIL(mvi_rows.push_back(row))) { - LOG_WARN("failed to push back spatial index row", K(ret), K(row)); - } - } // end if (OB_SUCC(ret)) } } @@ -501,7 +496,7 @@ bool ObDomainDMLIterator::is_same_domain_type(const ObDASDMLBaseCtDef *das_ctdef return is_same_domain_type; } -int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row) +int ObDomainDMLIterator::get_next_domain_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; const ObChunkDatumStore::StoredRow *sr = nullptr; @@ -529,7 +524,7 @@ int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row) } } if (OB_SUCC(ret) && row_idx_ < rows_.count()) { - row = &(rows_[row_idx_]); + row = rows_[row_idx_]; ++row_idx_; got_row = true; } @@ -538,7 +533,7 @@ int ObDomainDMLIterator::get_next_domain_row(ObNewRow *&row) return ret; } -int ObDomainDMLIterator::get_next_domain_rows(ObNewRow *&row, int64_t &row_count) +int ObDomainDMLIterator::get_next_domain_rows(blocksstable::ObDatumRow *&row, int64_t &row_count) { int ret = OB_SUCCESS; const ObChunkDatumStore::StoredRow *sr = nullptr; @@ -572,7 +567,7 @@ int ObDomainDMLIterator::get_next_domain_rows(ObNewRow *&row, int64_t &row_count } } if (OB_SUCC(ret) && row_idx_ < rows_.count()) { - row = &(rows_[row_idx_]); + row = rows_[row_idx_]; row_count = rows_.count() - row_idx_; row_idx_ = rows_.count(); got_row = true; diff --git a/src/sql/das/ob_das_domain_utils.h b/src/sql/das/ob_das_domain_utils.h index 63568f2bc..2ca8c2547 100644 --- a/src/sql/das/ob_das_domain_utils.h +++ b/src/sql/das/ob_das_domain_utils.h @@ -84,7 +84,7 @@ class ObDomainDMLIterator { public: static const int64_t DEFAULT_DOMAIN_ROW_COUNT = 32; - typedef common::ObSEArray ObDomainIndexRow; + typedef common::ObSEArray ObDomainIndexRow; static int create_domain_dml_iterator( common::ObIAllocator &allocator, @@ -104,8 +104,8 @@ public: } void set_ctdef(const ObDASDMLBaseCtDef *das_ctdef, const IntFixedArray *row_projector); void set_row_projector(const IntFixedArray *row_projector) { row_projector_ = row_projector; } - int get_next_domain_row(ObNewRow *&row); - int get_next_domain_rows(ObNewRow *&row, int64_t &row_count); + int get_next_domain_row(blocksstable::ObDatumRow *&row); + int get_next_domain_rows(blocksstable::ObDatumRow *&row, int64_t &row_count); bool is_same_domain_type(const ObDASDMLBaseCtDef *das_ctdef) const; TO_STRING_KV(K_(row_idx), K_(rows), KPC_(row_projector), KPC_(das_ctdef), K_(main_ctdef)); diff --git a/src/sql/das/ob_das_insert_op.cpp b/src/sql/das/ob_das_insert_op.cpp index 688f05b5e..fb7770041 100644 --- a/src/sql/das/ob_das_insert_op.cpp +++ b/src/sql/das/ob_das_insert_op.cpp @@ -149,7 +149,7 @@ int ObDASInsertOp::insert_rows() } int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param, ObAccessService *as, - ObNewRowIterator &dml_iter, + ObDatumRowIterator &dml_iter, ObDASConflictIterator *result_iter, const ObDASInsCtDef *ins_ctdef, ObDASInsRtDef *ins_rtdef, @@ -158,7 +158,7 @@ int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param, common::ObTabletID tablet_id) { int ret = OB_SUCCESS; - ObNewRow *insert_row = NULL; + blocksstable::ObDatumRow *insert_row = NULL; int64_t affected_rows = 0; if (OB_FAIL(ObDMLService::init_dml_param(*ins_ctdef, *ins_rtdef, @@ -170,7 +170,7 @@ int ObDASInsertOp::insert_index_with_fetch(ObDMLBaseParam &dml_param, LOG_WARN("init index dml param failed", K(ret), KPC(ins_ctdef), KPC(ins_rtdef)); } while (OB_SUCC(ret) && OB_SUCC(dml_iter.get_next_row(insert_row))) { - ObNewRowIterator *duplicated_rows = NULL; + blocksstable::ObDatumRowIterator *duplicated_rows = NULL; if (OB_ISNULL(insert_row)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("insert_row is null", K(ret)); @@ -297,7 +297,7 @@ int ObDASInsertOp::insert_row_with_fetch() LOG_WARN("rewind dml iter failed", K(ret)); } else { ObDASMLogDMLIterator mlog_iter(index_tablet_id, dml_param, &dml_iter, DAS_OP_TABLE_INSERT); - ObNewRowIterator *new_iter = nullptr; + ObDatumRowIterator *new_iter = nullptr; if (index_ins_ctdef->table_param_.get_data_table().is_mlog_table() && !index_ins_ctdef->is_access_mlog_as_master_table_) { new_iter = &mlog_iter; @@ -331,7 +331,7 @@ int ObDASInsertOp::store_conflict_row(ObDASInsertResult &ins_result) { int ret = OB_SUCCESS; bool added = false; - ObNewRow *dup_row = nullptr; + ObDatumRow *dup_row = nullptr; ObDASWriteBuffer &result_buffer = ins_result.get_result_buffer(); ObDASWriteBuffer::DmlShadowRow ssr; if (OB_ISNULL(result_)) { @@ -483,10 +483,10 @@ ObDASInsertResult::~ObDASInsertResult() { } -int ObDASInsertResult::get_next_row(ObNewRow *&row) +int ObDASInsertResult::get_next_row(ObDatumRow *&row) { int ret = OB_SUCCESS; - ObNewRow *result_row = NULL; + ObDatumRow *result_row = NULL; if (OB_FAIL(result_newrow_iter_.get_next_row(result_row))) { if (OB_ITER_END != ret) { LOG_WARN("get next row from result iter failed", K(ret)); @@ -497,19 +497,6 @@ int ObDASInsertResult::get_next_row(ObNewRow *&row) return ret; } -int ObDASInsertResult::get_next_rows(int64_t &count, int64_t capacity) -{ - UNUSED(count); - UNUSED(capacity); - return OB_NOT_IMPLEMENT; -} - -int ObDASInsertResult::get_next_row() -{ -int ret = OB_NOT_IMPLEMENT; -return ret; -} - int ObDASInsertResult::link_extra_result(ObDASExtraData &extra_result) { UNUSED(extra_result); @@ -585,11 +572,11 @@ void ObDASConflictIterator::reset() duplicated_iter_list_.reset(); } -int ObDASConflictIterator::get_next_row(common::ObNewRow *&row) +int ObDASConflictIterator::get_next_row(ObDatumRow *&row) { int ret = OB_SUCCESS; bool find_next_iter = false; - ObNewRow *dup_row = NULL; + ObDatumRow *dup_row = NULL; do { if (find_next_iter) { ++curr_iter_; @@ -599,7 +586,7 @@ int ObDASConflictIterator::get_next_row(common::ObNewRow *&row) ret = OB_ITER_END; LOG_DEBUG("fetch conflict row iterator end"); } else { - ObNewRowIterator *dup_row_iter = *curr_iter_; + blocksstable::ObDatumRowIterator *dup_row_iter = *curr_iter_; if (OB_ISNULL(dup_row_iter)) { find_next_iter = true; } else if (OB_FAIL(dup_row_iter->get_next_row(dup_row))) { @@ -624,11 +611,5 @@ int ObDASConflictIterator::get_next_row(common::ObNewRow *&row) return ret; } -int ObDASConflictIterator::get_next_row() -{ - int ret = OB_NOT_IMPLEMENT; - return ret; -} - } // namespace sql } // namespace oceanbase diff --git a/src/sql/das/ob_das_insert_op.h b/src/sql/das/ob_das_insert_op.h index 88a43105a..ab2a30f02 100644 --- a/src/sql/das/ob_das_insert_op.h +++ b/src/sql/das/ob_das_insert_op.h @@ -22,8 +22,8 @@ namespace sql { class ObDASInsertResult; -typedef common::ObList ObDuplicatedIterList; -class ObDASConflictIterator : public common::ObNewRowIterator +typedef common::ObList ObDuplicatedIterList; +class ObDASConflictIterator : public blocksstable::ObDatumRowIterator { public: ObDASConflictIterator(const ObjMetaFixedArray &output_types, @@ -36,9 +36,8 @@ public: ~ObDASConflictIterator() {}; - virtual int get_next_row(common::ObNewRow *&row) override; - virtual int get_next_row() override; - virtual void reset() override; + void reset(); + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; void init_curr_iter() { curr_iter_ = duplicated_iter_list_.begin(); } @@ -78,7 +77,7 @@ public: return insert_buffer_.dump_data(*ins_ctdef_); } - common::ObNewRowIterator *get_duplicated_result() + blocksstable::ObDatumRowIterator *get_duplicated_result() { return result_; } INHERIT_TO_STRING_KV("parent", ObIDASTaskOp, @@ -93,7 +92,7 @@ private: int insert_index_with_fetch(ObDMLBaseParam &dml_param, ObAccessService *as, - common::ObNewRowIterator &dml_iter, + blocksstable::ObDatumRowIterator &dml_iter, ObDASConflictIterator *result_iter, const ObDASInsCtDef *ins_ctdef, ObDASInsRtDef *ins_rtdef, @@ -105,12 +104,12 @@ private: const ObDASInsCtDef *ins_ctdef_; ObDASInsRtDef *ins_rtdef_; ObDASWriteBuffer insert_buffer_; - common::ObNewRowIterator *result_; + blocksstable::ObDatumRowIterator *result_; int64_t affected_rows_; // local execute result, no need to serialize bool is_duplicated_; // local execute result, no need to serialize }; -class ObDASInsertResult : public ObIDASTaskResult, public common::ObNewRowIterator +class ObDASInsertResult : public ObIDASTaskResult, public blocksstable::ObDatumRowIterator { OB_UNIS_VERSION_V(1); public: @@ -118,10 +117,8 @@ public: virtual ~ObDASInsertResult(); virtual int init(const ObIDASTaskOp &op, common::ObIAllocator &alloc) override; virtual int reuse() override; - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override; - virtual int get_next_rows(int64_t &count, int64_t capacity) override; - virtual void reset() override; + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; + virtual void reset(); virtual int link_extra_result(ObDASExtraData &extra_result) override; int init_result_newrow_iter(const ObjMetaFixedArray *output_types); ObDASWriteBuffer &get_result_buffer() { return result_buffer_; } diff --git a/src/sql/das/ob_das_update_op.cpp b/src/sql/das/ob_das_update_op.cpp index 80028ffa6..fdc89e44b 100644 --- a/src/sql/das/ob_das_update_op.cpp +++ b/src/sql/das/ob_das_update_op.cpp @@ -18,6 +18,7 @@ #include "sql/das/ob_das_utils.h" #include "sql/das/ob_das_domain_utils.h" #include "storage/tx_storage/ob_access_service.h" +#include "storage/blocksstable/ob_datum_row_utils.h" #include "sql/engine/expr/ob_expr_lob_utils.h" namespace oceanbase { @@ -43,7 +44,7 @@ using namespace share; namespace sql { -class ObDASUpdIterator : public ObNewRowIterator +class ObDASUpdIterator : public blocksstable::ObDatumRowIterator { public: ObDASUpdIterator(const ObDASUpdCtDef *das_ctdef, @@ -60,10 +61,8 @@ public: { } virtual ~ObDASUpdIterator(); - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override { return OB_NOT_IMPLEMENT; } + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; ObDASWriteBuffer &get_write_buffer() { return write_buffer_; } - virtual void reset() override { } int rewind(const ObDASDMLBaseCtDef *das_ctdef) { int ret = common::OB_SUCCESS; @@ -92,12 +91,12 @@ public: } private: // domain index - int get_next_domain_index_row(ObNewRow *&row); + int get_next_domain_index_row(blocksstable::ObDatumRow *&row); private: const ObDASUpdCtDef *das_ctdef_; ObDASWriteBuffer &write_buffer_; - ObNewRow *old_row_; - ObNewRow *new_row_; + blocksstable::ObDatumRow *old_row_; + blocksstable::ObDatumRow *new_row_; ObDASWriteBuffer::Iterator result_iter_; ObDomainDMLIterator *domain_iter_; bool got_old_row_; @@ -105,7 +104,7 @@ private: common::ObIAllocator &allocator_; }; -int ObDASUpdIterator::get_next_row(ObNewRow *&row) +int ObDASUpdIterator::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; const ObChunkDatumStore::StoredRow *sr = NULL; @@ -119,7 +118,7 @@ int ObDASUpdIterator::get_next_row(ObNewRow *&row) } else if (!got_old_row_) { got_old_row_ = true; if (OB_ISNULL(old_row_)) { - if (OB_FAIL(ob_create_row(allocator_, das_ctdef_->old_row_projector_.count(), old_row_))) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(allocator_, das_ctdef_->old_row_projector_.count(), old_row_))) { LOG_WARN("create row buffer failed", K(ret), K(das_ctdef_->old_row_projector_.count())); } else if (OB_FAIL(write_buffer_.begin(result_iter_))) { LOG_WARN("begin datum result iterator failed", K(ret)); @@ -127,7 +126,7 @@ int ObDASUpdIterator::get_next_row(ObNewRow *&row) } if (OB_SUCC(ret) && OB_ISNULL(new_row_)) { - if (OB_FAIL(ob_create_row(allocator_, das_ctdef_->new_row_projector_.count(), new_row_))) { + if (OB_FAIL(blocksstable::ObDatumRowUtils::ob_create_row(allocator_, das_ctdef_->new_row_projector_.count(), new_row_))) { LOG_WARN("create row buffer failed", K(ret), K(das_ctdef_->new_row_projector_.count())); } } @@ -182,7 +181,7 @@ ObDASUpdIterator::~ObDASUpdIterator() } } -int ObDASUpdIterator::get_next_domain_index_row(ObNewRow *&row) +int ObDASUpdIterator::get_next_domain_index_row(ObDatumRow *&row) { int ret = OB_SUCCESS; if (!iter_has_built_) { diff --git a/src/sql/das/ob_das_utils.cpp b/src/sql/das/ob_das_utils.cpp index b59394bab..66260fd4a 100644 --- a/src/sql/das/ob_das_utils.cpp +++ b/src/sql/das/ob_das_utils.cpp @@ -23,6 +23,7 @@ #include "observer/omt/ob_tenant_srs.h" #include "share/ob_tablet_autoincrement_service.h" #include "storage/access/ob_dml_param.h" +#include "storage/blocksstable/ob_datum_row_utils.h" namespace oceanbase { using namespace common; @@ -203,7 +204,7 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef, const ObDASWriteBuffer::DmlRow &dml_row, const IntFixedArray &row_projector, ObIAllocator &allocator, - ObNewRow &storage_row) + blocksstable::ObDatumRow &storage_row) { int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < row_projector.count(); ++i) { @@ -212,14 +213,17 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef, const ObAccuracy &col_accuracy = dml_ctdef.column_accuracys_.at(i); if (projector_idx < 0) { //this column is not touched by query, only need to be marked as nop - storage_row.cells_[i].set_nop_value(); - } else if (OB_FAIL(dml_row.cells()[projector_idx].to_obj(storage_row.cells_[i], col_type))) { - LOG_WARN("stored row to new row obj failed", K(ret), - K(dml_row.cells()[projector_idx]), K(col_type), K(projector_idx), K(i)); - } else if (OB_FAIL(reshape_storage_value(col_type, col_accuracy, allocator, storage_row.cells_[i]))) { + storage_row.storage_datums_[i].set_nop(); + } else if (FALSE_IT(storage_row.storage_datums_[i].shallow_copy_from_datum(dml_row.cells()[projector_idx]))) { + } else if (storage_row.storage_datums_[i].is_null()) { + //nothing to do + } else if (OB_FAIL(reshape_datum_value(col_type, col_accuracy, true, allocator, storage_row.storage_datums_[i]))) { LOG_WARN("reshape storage value failed", K(ret)); + } else if (col_type.is_lob_storage() && col_type.has_lob_header()) { + storage_row.storage_datums_[i].set_has_lob_header(); } } + //to project shadow rowkey with unique index if (OB_SUCC(ret) && dml_ctdef.spk_cnt_) { bool need_shadow_columns = false; @@ -229,7 +233,7 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef, //need to fill shadow pk with the real pk value bool rowkey_has_null = false; for (int64_t i = 0; !rowkey_has_null && i < index_key_cnt; i++) { - rowkey_has_null = storage_row.cells_[i].is_null(); + rowkey_has_null = storage_row.storage_datums_[i].is_null(); } need_shadow_columns = rowkey_has_null; } else { @@ -237,14 +241,14 @@ int ObDASUtils::project_storage_row(const ObDASDMLBaseCtDef &dml_ctdef, //need to fill shadow pk with the real pk value bool is_rowkey_all_null = true; for (int64_t i = 0; is_rowkey_all_null && i < index_key_cnt; i++) { - is_rowkey_all_null = storage_row.cells_[i].is_null(); + is_rowkey_all_null = storage_row.storage_datums_[i].is_null(); } need_shadow_columns = is_rowkey_all_null; } if (!need_shadow_columns) { for (int64_t i = 0; i < dml_ctdef.spk_cnt_; ++i) { int64_t spk_idx = index_key_cnt + i; - storage_row.cells_[spk_idx].set_null(); + storage_row.storage_datums_[spk_idx].set_null(); } } } @@ -398,7 +402,7 @@ int ObDASUtils::find_child_das_def(const ObDASBaseCtDef *root_ctdef, int ObDASUtils::generate_mlog_row(const ObTabletID &tablet_id, const storage::ObDMLBaseParam &dml_param, - ObNewRow &row, + blocksstable::ObDatumRow &row, ObDASOpType op_type, bool is_old_row) { @@ -432,23 +436,26 @@ int ObDASUtils::generate_mlog_row(const ObTabletID &tablet_id, } } - row.cells_[sequence_col].set_int(ObObjType::ObIntType, static_cast(autoinc_seq)); + row.storage_datums_[sequence_col].reuse(); + row.storage_datums_[dmltype_col].reuse(); + row.storage_datums_[old_new_col].reuse(); + + row.storage_datums_[sequence_col].set_int(static_cast(autoinc_seq)); if (sql::DAS_OP_TABLE_DELETE == op_type) { - row.cells_[dmltype_col].set_varchar("D"); - row.cells_[old_new_col].set_varchar("O"); + row.storage_datums_[dmltype_col].set_string(ObString("D")); + row.storage_datums_[old_new_col].set_string(ObString("O")); } else if (sql::DAS_OP_TABLE_UPDATE == op_type) { - row.cells_[dmltype_col].set_varchar("U"); + row.storage_datums_[dmltype_col].set_string(ObString("U")); if (is_old_row) { - row.cells_[old_new_col].set_varchar("O"); + row.storage_datums_[old_new_col].set_string(ObString("O")); } else { - row.cells_[old_new_col].set_varchar("N"); + row.storage_datums_[old_new_col].set_string(ObString("N")); } } else { - row.cells_[dmltype_col].set_varchar("I"); - row.cells_[old_new_col].set_varchar("N"); + row.storage_datums_[dmltype_col].set_string(ObString("I")); + row.storage_datums_[old_new_col].set_string(ObString("N")); } - row.cells_[dmltype_col].set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); - row.cells_[old_new_col].set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI); + // TODO: if we need to update col_type_ in col_descs with ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI ?@xuanxi } return ret; } diff --git a/src/sql/das/ob_das_utils.h b/src/sql/das/ob_das_utils.h index da50e6a3f..7c76cf4e1 100644 --- a/src/sql/das/ob_das_utils.h +++ b/src/sql/das/ob_das_utils.h @@ -53,7 +53,7 @@ public: const ObDASWriteBuffer::DmlRow &dml_row, const IntFixedArray &row_projector, common::ObIAllocator &allocator, - common::ObNewRow &storage_row); + blocksstable::ObDatumRow &storage_row); static int reshape_storage_value(const common::ObObjMeta &col_type, const common::ObAccuracy &col_accuracy, common::ObIAllocator &allocator, @@ -93,7 +93,7 @@ public: } static int generate_mlog_row(const common::ObTabletID &tablet_id, const storage::ObDMLBaseParam &dml_param, - common::ObNewRow &row, + blocksstable::ObDatumRow &row, ObDASOpType op_type, bool is_old_row); }; diff --git a/src/sql/engine/dml/ob_table_insert_up_op.cpp b/src/sql/engine/dml/ob_table_insert_up_op.cpp index 3cf464d4b..e2ff8d5e5 100644 --- a/src/sql/engine/dml/ob_table_insert_up_op.cpp +++ b/src/sql/engine/dml/ob_table_insert_up_op.cpp @@ -981,11 +981,11 @@ int ObTableInsertUpOp::get_next_conflict_rowkey(DASTaskIter &task_iter) int ret = OB_SUCCESS; bool got_row = false; while (OB_SUCC(ret) && !got_row) { - ObNewRow *dup_row = nullptr; + ObDatumRow *dup_row = nullptr; ObChunkDatumStore::StoredRow *stored_row = nullptr; ObDASWriteBuffer::DmlShadowRow ssr; ObDASInsertOp *ins_op = static_cast(*task_iter); - ObNewRowIterator *conflict_result = ins_op->get_duplicated_result(); + ObDatumRowIterator *conflict_result = ins_op->get_duplicated_result(); const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); // 因为返回的都是主表的主键，主表的主键一定是在存储层有储存的，是不需要再收起来层再做运算的， // 所以这里不需要clear eval flag diff --git a/src/sql/engine/dml/ob_table_replace_op.cpp b/src/sql/engine/dml/ob_table_replace_op.cpp index b0fd727d2..fe824f855 100644 --- a/src/sql/engine/dml/ob_table_replace_op.cpp +++ b/src/sql/engine/dml/ob_table_replace_op.cpp @@ -413,11 +413,11 @@ int ObTableReplaceOp::get_next_conflict_rowkey(DASTaskIter &task_iter) int ret = OB_SUCCESS; bool got_row = false; while (OB_SUCC(ret) && !got_row) { - ObNewRow *dup_row = nullptr; + ObDatumRow *dup_row = nullptr; ObChunkDatumStore::StoredRow *stored_row = nullptr; ObDASWriteBuffer::DmlShadowRow ssr; ObDASInsertOp *ins_op = static_cast(*task_iter); - ObNewRowIterator *conflict_result = ins_op->get_duplicated_result(); + ObDatumRowIterator *conflict_result = ins_op->get_duplicated_result(); const ObDASInsCtDef *ins_ctdef = static_cast(ins_op->get_ctdef()); // 因为返回的都是主表的主键，主表的主键一定是在存储层有储存的，是不需要再收起来层再做运算的， // 所以这里不需要clear eval flag diff --git a/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp b/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp index e5fec97ea..1f7db7f64 100644 --- a/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp +++ b/src/sql/engine/expr/ob_expr_pl_get_cursor_attr.cpp @@ -292,7 +292,7 @@ int ObExprPLGetCursorAttr::calc_pl_get_cursor_attr( } else if (OB_UNLIKELY(rowid.empty())) { expr_datum.set_null(); } else { - expr_datum.set_urowid(rowid.ptr(), rowid.length()); + expr_datum.set_string(rowid.ptr(), rowid.length()); } } break; diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index b8f0eb48c..8d2c5a90f 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -3135,25 +3135,17 @@ int ObTableScanOp::inner_get_next_spatial_index_row() } else if (cellids.size() > SAPTIAL_INDEX_DEFAULT_ROW_COUNT) { ret = OB_ERR_UNEXPECTED; LOG_WARN("cellid over size", K(ret), K(cellids.size())); - } else if (OB_ISNULL(spat_index_.obj_buffer_)) { + } else if (OB_ISNULL(spat_index_.rows_)) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for spatial index row cells", K(ret)); + LOG_WARN("failed to alloc memory for spatial index datum row", K(ret)); } else { - ObObj *obj_arr = reinterpret_cast(spat_index_.obj_buffer_); - uint64_t obj_idx = 0; - for (uint64_t i = 0; OB_SUCC(ret) && i < cellids.size(); i++) { - obj_arr[obj_idx].set_nop_value(); - obj_arr[obj_idx].set_uint64(cellids.at(i)); - obj_arr[obj_idx + 1].set_nop_value(); - obj_arr[obj_idx + 1].set_varchar(mbr_val); - obj_arr[obj_idx + 1].set_collation_type(CS_TYPE_BINARY); - obj_arr[obj_idx + 1].set_collation_level(CS_LEVEL_IMPLICIT); - ObNewRow row; - row.cells_ = &obj_arr[obj_idx]; - row.count_ = 2; - obj_idx += 2; - if (OB_FAIL(spat_index_.spat_rows_->push_back(row))) { - LOG_WARN("failed to push back spatial index row", K(ret), K(row)); + for (uint64_t i = 0, datum_idx = 0; OB_SUCC(ret) && i < cellids.size(); i++) { + spat_index_.rows_[i].reuse(); + spat_index_.rows_[i].storage_datums_[datum_idx].set_uint(cellids.at(i)); + spat_index_.rows_[i].storage_datums_[datum_idx + 1].set_string(mbr_val); + // not set_collation_type(CS_TYPE_BINARY) and set_collation_level(CS_LEVEL_IMPLICIT) + if (OB_FAIL(spat_index_.spat_rows_->push_back(spat_index_.rows_ + i))) { + LOG_WARN("failed to push back spatial index row", K(ret), K(spat_index_.rows_[i])); } } } @@ -3162,10 +3154,10 @@ int ObTableScanOp::inner_get_next_spatial_index_row() } } } - if (OB_SUCC(ret) && !need_ignore_null) { - ObNewRow &row = (*(spat_index_.spat_rows_))[spat_index_.spat_row_index_++]; - ObObj &cellid= row.get_cell(0); - ObObj &mbr = row.get_cell(1); + if (OB_SUCC(ret)) { + ObDatumRow *row = (*(spat_index_.spat_rows_))[spat_index_.spat_row_index_++]; + ObStorageDatum &cellid = row->storage_datums_[0]; + ObStorageDatum &mbr = row->storage_datums_[1]; if (OB_FAIL(fill_generated_cellid_mbr(cellid, mbr))) { LOG_WARN("fill cellid mbr failed", K(ret), K(cellid), K(mbr)); } @@ -3178,19 +3170,24 @@ int ObTableScanOp::init_spatial_index_rows() { int ret = OB_SUCCESS; void *buf = ctx_.get_allocator().alloc(sizeof(ObDomainIndexRow)); + void *row_buf = ctx_.get_allocator().alloc(sizeof(blocksstable::ObDatumRow) * SAPTIAL_INDEX_DEFAULT_ROW_COUNT); void *mbr_buffer = ctx_.get_allocator().alloc(OB_DEFAULT_MBR_SIZE); - void *obj_buf = ctx_.get_allocator().alloc(sizeof(ObObj) * 2 * SAPTIAL_INDEX_DEFAULT_ROW_COUNT); - if (OB_ISNULL(buf) || OB_ISNULL(mbr_buffer) || OB_ISNULL(obj_buf)) { + if (OB_ISNULL(buf) || OB_ISNULL(mbr_buffer) || OB_ISNULL(row_buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate spatial row store failed", K(ret), K(buf), K(mbr_buffer)); } else { spat_index_.spat_rows_ = new(buf) ObDomainIndexRow(); + spat_index_.rows_ = new(row_buf) blocksstable::ObDatumRow[SAPTIAL_INDEX_DEFAULT_ROW_COUNT]; spat_index_.mbr_buffer_ = mbr_buffer; - spat_index_.obj_buffer_ = obj_buf; const ObExprPtrIArray &exprs = MY_SPEC.output_; const uint8_t spatial_expr_cnt = 3; uint8_t cnt = 0; - for (uint32_t i = 0; i < exprs.count() && cnt < spatial_expr_cnt; i++) { + for (uint32_t i = 0; OB_SUCC(ret) && i < SAPTIAL_INDEX_DEFAULT_ROW_COUNT; i++) { + if (OB_FAIL(spat_index_.rows_[i].init(SAPTIAL_INDEX_DEFAULT_COL_COUNT))) { + LOG_WARN("init datum row failed", K(ret)); + } + } + for (uint32_t i = 0; OB_SUCC(ret) && i < exprs.count() && cnt < spatial_expr_cnt; i++) { if (exprs.at(i)->type_ == T_FUN_SYS_SPATIAL_CELLID) { spat_index_.cell_idx_ = i; cnt++; @@ -3202,7 +3199,7 @@ int ObTableScanOp::init_spatial_index_rows() cnt++; } } - if (cnt != spatial_expr_cnt) { + if (OB_FAIL(ret) || cnt != spatial_expr_cnt) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid spatial index exprs", K(ret), K(cnt)); } @@ -3210,7 +3207,7 @@ int ObTableScanOp::init_spatial_index_rows() return ret; } -int ObTableScanOp::fill_generated_cellid_mbr(const ObObj &cellid, const ObObj &mbr) +int ObTableScanOp::fill_generated_cellid_mbr(const ObStorageDatum &cellid, const ObStorageDatum &mbr) { int ret = OB_SUCCESS; const ObExprPtrIArray &exprs = MY_SPEC.output_; @@ -3220,12 +3217,12 @@ int ObTableScanOp::fill_generated_cellid_mbr(const ObObj &cellid, const ObObj &m } else { for (uint8_t i = 0; i < 2 && OB_SUCC(ret); i++) { ObObjDatumMapType type = i == 0 ? OBJ_DATUM_8BYTE_DATA : OBJ_DATUM_STRING; - const ObObj &value = i == 0 ? cellid : mbr; + const ObStorageDatum &value = i == 0 ? cellid : mbr; uint32_t idx = i == 0 ? spat_index_.cell_idx_ : spat_index_.mbr_idx_; ObExpr *expr = exprs.at(idx); ObDatum *datum = &expr->locate_datum_for_write(get_eval_ctx()); ObEvalInfo *eval_info = &expr->get_eval_info(get_eval_ctx()); - if (OB_FAIL(datum->from_obj(value, type))) { + if (OB_FAIL(datum->from_storage_datum(value, type))) { LOG_WARN("fill spatial index row failed", K(ret)); } else { eval_info->evaluated_ = true; diff --git a/src/sql/engine/table/ob_table_scan_op.h b/src/sql/engine/table/ob_table_scan_op.h index a1d814cd3..5ea75621a 100644 --- a/src/sql/engine/table/ob_table_scan_op.h +++ b/src/sql/engine/table/ob_table_scan_op.h @@ -71,18 +71,18 @@ struct ObSpatialIndexCache public: ObSpatialIndexCache() : spat_rows_(nullptr), + rows_(nullptr), spat_row_index_(0), mbr_buffer_(nullptr), - obj_buffer_(nullptr), geo_idx_(0), cell_idx_(0), mbr_idx_(0) {} ~ObSpatialIndexCache() {}; ObDomainIndexRow *spat_rows_; + blocksstable::ObDatumRow *rows_; uint8_t spat_row_index_; void *mbr_buffer_; - void *obj_buffer_; uint32_t geo_idx_; uint32_t cell_idx_; uint32_t mbr_idx_; @@ -509,7 +509,7 @@ protected: ObTableScanStat &scan_stat) const; void set_cache_stat(const ObPlanStat &plan_stat); int inner_get_next_row_implement(); - int fill_generated_cellid_mbr(const ObObj &cellid, const ObObj &mbr); + int fill_generated_cellid_mbr(const ObStorageDatum &cellid, const ObStorageDatum &mbr); int inner_get_next_spatial_index_row(); int init_spatial_index_rows(); void set_real_rescan_cnt(int64_t real_rescan_cnt) { group_rescan_cnt_ = real_rescan_cnt; } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index f54ea23ef..44ae7d35d 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -43,8 +43,11 @@ ob_set_subtarget(ob_storage blocksstable blocksstable/ob_sstable_printer.cpp blocksstable/ob_storage_cache_suite.cpp blocksstable/ob_super_block_buffer_holder.cpp + blocksstable/ob_storage_datum.cpp blocksstable/ob_datum_row.cpp blocksstable/ob_datum_rowkey.cpp + blocksstable/ob_datum_row_store.cpp + blocksstable/ob_datum_row_utils.cpp blocksstable/ob_data_store_desc.cpp blocksstable/ob_table_flag.cpp blocksstable/ob_datum_rowkey_vector.cpp diff --git a/src/storage/access/ob_rows_info.cpp b/src/storage/access/ob_rows_info.cpp index 576a14888..9bab938d4 100644 --- a/src/storage/access/ob_rows_info.cpp +++ b/src/storage/access/ob_rows_info.cpp @@ -14,6 +14,7 @@ #include "storage/ob_storage_struct.h" #include "storage/ob_relative_table.h" #include "storage/ob_storage_schema.h" +#include "storage/blocksstable/ob_datum_row_utils.h" #include "ob_store_row_iterator.h" namespace oceanbase @@ -93,6 +94,7 @@ ObRowsInfo::ObRowsInfo() tablet_id_(), datum_utils_(nullptr), min_key_(), + col_descs_(nullptr), conflict_rowkey_idx_(-1), error_code_(0), delete_count_(0), @@ -124,9 +126,11 @@ void ObRowsInfo::reset() error_code_ = 0; conflict_rowkey_idx_ = -1; is_inited_ = false; + col_descs_ = nullptr; } int ObRowsInfo::init( + const ObColDescIArray &column_descs, const ObRelativeTable &table, ObStoreCtx &store_ctx, const ObITableReadInfo &rowkey_read_info) @@ -139,6 +143,7 @@ int ObRowsInfo::init( } else if (OB_FAIL(exist_helper_.init(table, store_ctx, rowkey_read_info, exist_allocator_, scan_mem_allocator_))) { STORAGE_LOG(WARN, "Failed to init exist helper", K(ret)); } else { + col_descs_ = &column_descs; datum_utils_ = &rowkey_read_info.get_datum_utils(); tablet_id_ = table.get_tablet_id(); rowkey_column_num_ = table.get_rowkey_column_num(); @@ -160,7 +165,7 @@ void ObRowsInfo::reuse() } //not only checking duplicate, but also assign rowkeys -int ObRowsInfo::check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRelativeTable &table) +int ObRowsInfo::check_duplicate(ObDatumRow *rows, const int64_t row_count, ObRelativeTable &table) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -192,10 +197,9 @@ int ObRowsInfo::check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRel } else { ObMarkedRowkeyAndLockState marked_rowkey_and_lock_state; marked_rowkey_and_lock_state.row_idx_ = i; - ObRowkey rowkey(rows_[i].row_val_.cells_, rowkey_column_num_); - if (OB_FAIL(marked_rowkey_and_lock_state.marked_rowkey_.get_rowkey().from_rowkey(rowkey, - key_allocator_))) { - STORAGE_LOG(WARN, "Failed to transfer rowkey", K(ret), K(rowkey)); + ObDatumRowkey &datum_rowkey = marked_rowkey_and_lock_state.marked_rowkey_.get_rowkey(); + if (OB_FAIL(blocksstable::ObDatumRowUtils::prepare_rowkey(rows[i], rowkey_column_num_, *col_descs_, key_allocator_, datum_rowkey))) { + STORAGE_LOG(WARN, "Failed to prepare rowkey", K(ret), K(rowkey_column_num_), K(rows_[i])); } else if (OB_FAIL(rowkeys_.push_back(marked_rowkey_and_lock_state))) { STORAGE_LOG(WARN, "Failed to push back rowkey", K(ret), K(marked_rowkey_and_lock_state)); } diff --git a/src/storage/access/ob_rows_info.h b/src/storage/access/ob_rows_info.h index 604f1fde8..05caf2fc0 100644 --- a/src/storage/access/ob_rows_info.h +++ b/src/storage/access/ob_rows_info.h @@ -90,10 +90,11 @@ public: return delete_count_ == rowkeys_.count(); } int init( + const ObColDescIArray &column_descs, const ObRelativeTable &table, ObStoreCtx &store_ctx, const ObITableReadInfo &rowkey_read_info); - int check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRelativeTable &table); + int check_duplicate(blocksstable::ObDatumRow *rows, const int64_t row_count, ObRelativeTable &table); blocksstable::ObDatumRowkey& get_duplicate_rowkey() { return min_key_; @@ -265,12 +266,13 @@ private: public: ObRowkeyAndLockStates rowkeys_; ObPermutation permutation_; - ObStoreRow *rows_; + blocksstable::ObDatumRow *rows_; ExistHelper exist_helper_; ObTabletID tablet_id_; private: const blocksstable::ObStorageDatumUtils *datum_utils_; blocksstable::ObDatumRowkey min_key_; + const ObColDescIArray *col_descs_; int64_t conflict_rowkey_idx_; int error_code_; int64_t delete_count_; diff --git a/src/storage/access/ob_table_access_context.cpp b/src/storage/access/ob_table_access_context.cpp index eea822cb2..4e68b9c90 100644 --- a/src/storage/access/ob_table_access_context.cpp +++ b/src/storage/access/ob_table_access_context.cpp @@ -266,10 +266,12 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, } return ret; } + int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, ObStoreCtx &ctx, common::ObIAllocator &allocator, - const common::ObVersionRange &trans_version_range) + const common::ObVersionRange &trans_version_range, + CachedIteratorNode *cached_iter_node) { int ret = OB_SUCCESS; if (is_inited_) { @@ -288,6 +290,7 @@ int ObTableAccessContext::init(const common::ObQueryFlag &query_flag, ls_id_ = ctx.ls_id_; tablet_id_ = ctx.tablet_id_; lob_locator_helper_ = nullptr; + cached_iter_node_ = cached_iter_node; if (!micro_block_handle_mgr_.is_valid() && OB_FAIL(micro_block_handle_mgr_.init(enable_limit, table_store_stat_, query_flag_))) { LOG_WARN("Fail to init micro block handle mgr", K(ret)); diff --git a/src/storage/access/ob_table_access_context.h b/src/storage/access/ob_table_access_context.h index 987145afb..93db844a6 100644 --- a/src/storage/access/ob_table_access_context.h +++ b/src/storage/access/ob_table_access_context.h @@ -199,7 +199,8 @@ struct ObTableAccessContext int init(const common::ObQueryFlag &query_flag, ObStoreCtx &ctx, common::ObIAllocator &allocator, - const common::ObVersionRange &trans_version_range); + const common::ObVersionRange &trans_version_range, + CachedIteratorNode *cached_iter_node = nullptr); int alloc_iter_pool(const bool use_column_store); void inc_micro_access_cnt(); int init_scan_allocator(ObTableScanParam &scan_param); diff --git a/src/storage/blocksstable/ob_datum_row.cpp b/src/storage/blocksstable/ob_datum_row.cpp index a71b34464..cab794458 100644 --- a/src/storage/blocksstable/ob_datum_row.cpp +++ b/src/storage/blocksstable/ob_datum_row.cpp @@ -22,73 +22,6 @@ namespace oceanbase using namespace common; namespace blocksstable { -static int nonext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) -{ - int ret = cmp_func.cmp_func_(left, right, cmp_ret); - STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); - return ret; -} - -static int nonext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) -{ - int ret = OB_SUCCESS; - UNUSEDx(left, cmp_func); - if (right.is_max()) { - cmp_ret = -1; - } else if (right.is_min()) { - cmp_ret = 1; - } else { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(right)); - } - STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); - return ret; -} - -static int ext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) -{ - int ret = OB_SUCCESS; - UNUSEDx(right, cmp_func); - if (left.is_max()) { - cmp_ret = 1; - } else if (left.is_min()) { - cmp_ret = -1; - } else { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left)); - } - STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); - return ret; -} - -static int ext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) -{ - int ret = OB_SUCCESS; - UNUSEDx(cmp_func); - int64_t lv = left.is_max() - left.is_min(); - int64_t rv = right.is_max() - right.is_min(); - if (OB_UNLIKELY(0 == lv || 0 == rv)) { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left), K(right)); - } else { - cmp_ret = lv - rv; - } - STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); - - return ret; -} - -typedef int (*ExtSafeCompareFunc)(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret); -static ExtSafeCompareFunc ext_safe_cmp_funcs[2][2] = { - {nonext_nonext_compare, nonext_ext_compare}, - {ext_nonext_compare, ext_ext_compare} -}; - -int ObStorageDatumCmpFunc::compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const -{ - return ext_safe_cmp_funcs[left.is_ext()][right.is_ext()](left, right, cmp_func_, cmp_ret); -} - const char *get_dml_str(ObDmlFlag dml_flag) { @@ -106,86 +39,6 @@ OB_SERIALIZE_MEMBER(ObDmlRowFlag, whole_flag_); OB_SERIALIZE_MEMBER(ObMultiVersionRowFlag, flag_); -/* - *ObStorageDatumBuffer - */ -ObStorageDatumBuffer::ObStorageDatumBuffer(common::ObIAllocator *allocator) - : capacity_(LOCAL_BUFFER_ARRAY), - local_datums_(), - datums_(local_datums_), - allocator_(allocator), - is_inited_(nullptr != allocator) -{} - -ObStorageDatumBuffer::~ObStorageDatumBuffer() -{ - if (datums_ != local_datums_ && nullptr != allocator_) { - allocator_->free(datums_); - } -} - -void ObStorageDatumBuffer::reset() -{ - if (datums_ != local_datums_ && nullptr != allocator_) { - allocator_->free(datums_); - } - allocator_ = nullptr; - datums_ = local_datums_; - capacity_ = LOCAL_BUFFER_ARRAY; - for (int64_t i = 0; i < capacity_; i++) { - datums_[i].reuse(); - } - is_inited_ = false; -} - -int ObStorageDatumBuffer::init(common::ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObStorageDatumBuffer init twice", K(ret), K(*this)); - } else { - OB_ASSERT(datums_ == local_datums_); - allocator_ = &allocator; - is_inited_ = true; - } - - return ret; -} - -int ObStorageDatumBuffer::reserve(const int64_t count, const bool keep_data) -{ - int ret = OB_SUCCESS; - void *buf = nullptr; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - STORAGE_LOG(WARN, "ObStorageDatumBuffer is not inited", K(ret), K(*this)); - } else if (OB_UNLIKELY(count <= 0)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to reserve datum buffer", K(ret), K(count)); - } else if (count <= capacity_){ - } else if (OB_ISNULL(buf = allocator_->alloc(sizeof(ObStorageDatum) * count))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(count)); - } else { - ObStorageDatum *new_datums = new (buf) ObStorageDatum [count]; - if (keep_data) { - for (int64_t i = 0; i < capacity_; i++) { - new_datums[i] = datums_[i]; - } - } - if (nullptr != datums_ && datums_ != local_datums_) { - allocator_->free(datums_); - } - datums_ = new_datums; - capacity_ = count; - } - - return ret; -} - /* *ObConstDatumRow */ @@ -232,8 +85,7 @@ ObDatumRow::ObDatumRow(const uint64_t tenant_id) } ObDatumRow::~ObDatumRow() -{ -} +{} int ObDatumRow::init(ObIAllocator &allocator, const int64_t capacity, char *trans_info_ptr) { @@ -256,7 +108,9 @@ int ObDatumRow::init(ObIAllocator &allocator, const int64_t capacity, char *tran } return ret; + } + int ObDatumRow::init(const int64_t capacity) { int ret = OB_SUCCESS; @@ -442,6 +296,27 @@ int ObDatumRow::is_datums_changed(const ObDatumRow &other, bool &is_changed) con return ret; } +int ObDatumRow::shallow_copy(const ObDatumRow &other) +{ + int ret = OB_SUCCESS; + if (!other.is_valid()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected error for shallow copy invalid datum row", K(ret), K(*this), K(other)); + } else { + trans_info_ = nullptr; + fast_filter_skipped_ = other.fast_filter_skipped_; + storage_datums_ = other.storage_datums_; + snapshot_version_ = other.snapshot_version_; + group_idx_ = other.group_idx_; + scan_index_ = other.scan_index_; + trans_id_ = other.trans_id_; + mvcc_row_flag_ = other.mvcc_row_flag_; + row_flag_ = other.row_flag_; + count_ = other.count_; + } + return ret; +} + OB_DEF_SERIALIZE(ObDatumRow) { int ret = OB_SUCCESS; @@ -503,8 +378,8 @@ DEF_TO_STRING(ObDatumRow) { int64_t pos = 0; J_OBJ_START(); - J_KV(K_(row_flag), K_(trans_id), K_(scan_index), K_(mvcc_row_flag), - K_(snapshot_version), K_(fast_filter_skipped), K_(have_uncommited_row), K_(group_idx), K_(count), K_(datum_buffer)); + J_KV(K_(row_flag), K_(trans_id), K_(scan_index), K_(mvcc_row_flag), K_(snapshot_version), K_(fast_filter_skipped), + K_(have_uncommited_row), K_(group_idx), K_(count), K_(datum_buffer)); if (NULL != buf && buf_len >= 0) { if (NULL != storage_datums_) { J_COMMA(); @@ -615,219 +490,6 @@ int ObNewRowBuilder::build_store_row( return ret; } -/* - *ObStorageDatumUtils - */ -ObStorageDatumUtils::ObStorageDatumUtils() - : rowkey_cnt_(0), - cmp_funcs_(), - hash_funcs_(), - ext_hash_func_(), - is_oracle_mode_(false), - is_inited_(false) -{} - -ObStorageDatumUtils::~ObStorageDatumUtils() -{} - -int ObStorageDatumUtils::transform_multi_version_col_desc(const ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - ObIArray &mv_col_descs) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(schema_rowkey_cnt > col_descs.count())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to transform mv col descs", K(ret), K(schema_rowkey_cnt), K(col_descs)); - } else { - mv_col_descs.reuse(); - for (int64_t i = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; i++) { - if (OB_FAIL(mv_col_descs.push_back(col_descs.at(i)))) { - STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(i)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(storage::ObMultiVersionRowkeyHelpper::add_extra_rowkey_cols(mv_col_descs))) { - STORAGE_LOG(WARN, "Fail to add extra_rowkey_cols", K(ret), K(schema_rowkey_cnt)); - } else { - for (int64_t i = schema_rowkey_cnt; OB_SUCC(ret) && i < col_descs.count(); i++) { - const share::schema::ObColDesc &col_desc = col_descs.at(i); - if (col_desc.col_id_ == common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID - || col_desc.col_id_ == common::OB_HIDDEN_SQL_SEQUENCE_COLUMN_ID) { - continue; - } else if (OB_FAIL(mv_col_descs.push_back(col_desc))) { - STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(col_desc)); - } - } - } - } - - return ret; -} - -int ObStorageDatumUtils::init(const ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - const bool is_oracle_mode, - ObIAllocator &allocator, - const bool is_column_store) -{ - int ret = OB_SUCCESS; - ObSEArray mv_col_descs; - int64_t mv_rowkey_cnt = 0; - int64_t mv_column_cnt = 0; - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this)); - } else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER - || schema_rowkey_cnt > col_descs.count())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(col_descs), K(schema_rowkey_cnt)); - } else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) { - STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret)); - } else if (FALSE_IT(mv_column_cnt = is_column_store ? 0 : storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) { - } else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + mv_column_cnt)) { - } else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, allocator))) { - STORAGE_LOG(WARN, "Failed to reserve cmp func array", K(ret)); - } else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, allocator))) { - STORAGE_LOG(WARN, "Failed to reserve hash func array", K(ret)); - } else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) { - STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt)); - } - - return ret; -} - -int ObStorageDatumUtils::init(const common::ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - const bool is_oracle_mode, - const int64_t arr_buf_len, - char *arr_buf) -{ - int ret = OB_SUCCESS; - ObSEArray mv_col_descs; - int64_t pos = 0; - int64_t mv_rowkey_cnt = 0; - - if (IS_INIT) { - ret = OB_INIT_TWICE; - STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this)); - } else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER - || schema_rowkey_cnt > col_descs.count())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(col_descs), K(schema_rowkey_cnt)); - } else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) { - STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret)); - } else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) { - } else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) { - STORAGE_LOG(WARN, "Failed to init compare function array", K(ret)); - } else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) { - STORAGE_LOG(WARN, "Failed to init hash function array", K(ret)); - } else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) { - STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt)); - } - return ret; -} - -int ObStorageDatumUtils::inner_init( - const common::ObIArray &mv_col_descs, - const int64_t mv_rowkey_col_cnt, - const bool is_oracle_mode) -{ - int ret = OB_SUCCESS; - is_oracle_mode_ = is_oracle_mode; - // support column order index until next task done - // - // we could use the cmp funcs in the basic funcs directlly - bool is_null_last = is_oracle_mode_; - ObCmpFunc cmp_func; - ObHashFunc hash_func; - for (int64_t i = 0; OB_SUCC(ret) && i < mv_rowkey_col_cnt; i++) { - const share::schema::ObColDesc &col_desc = mv_col_descs.at(i); - //TODO @hanhui support desc rowkey - bool is_ascending = true || col_desc.col_order_ == ObOrderType::ASC; - bool has_lob_header = is_lob_storage(col_desc.col_type_.get_type()); - ObPrecision precision = PRECISION_UNKNOWN_YET; - if (col_desc.col_type_.is_decimal_int()) { - precision = col_desc.col_type_.get_stored_precision(); - OB_ASSERT(precision != PRECISION_UNKNOWN_YET); - } - sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(col_desc.col_type_.get_type(), - col_desc.col_type_.get_collation_type(), - col_desc.col_type_.get_scale(), - is_oracle_mode, - has_lob_header, - precision); - if (OB_UNLIKELY(nullptr == basic_funcs - || nullptr == basic_funcs->null_last_cmp_ - || nullptr == basic_funcs->murmur_hash_)) { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected null basic funcs", K(ret), K(col_desc)); - } else { - cmp_func.cmp_func_ = is_null_last ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_; - hash_func.hash_func_ = basic_funcs->murmur_hash_; - if (OB_FAIL(hash_funcs_.push_back(hash_func))) { - STORAGE_LOG(WARN, "Failed to push back hash func", K(ret), K(i), K(col_desc)); - } else if (is_ascending) { - if (OB_FAIL(cmp_funcs_.push_back(ObStorageDatumCmpFunc(cmp_func)))) { - STORAGE_LOG(WARN, "Failed to push back cmp func", K(ret), K(i), K(col_desc)); - } - } else { - ret = OB_ERR_SYS; - STORAGE_LOG(WARN, "Unsupported desc column order", K(ret), K(col_desc), K(i)); - } - } - } - if (OB_SUCC(ret)) { - sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(ObExtendType, CS_TYPE_BINARY); - if (OB_UNLIKELY(nullptr == basic_funcs || nullptr == basic_funcs->murmur_hash_)) { - ret = OB_ERR_SYS; - STORAGE_LOG(ERROR, "Unexpected null basic funcs for extend type", K(ret)); - } else { - ext_hash_func_.hash_func_ = basic_funcs->murmur_hash_; - rowkey_cnt_ = mv_rowkey_col_cnt; - is_inited_ = true; - } - } - - return ret; -} - -int ObStorageDatumUtils::assign(const ObStorageDatumUtils &other_utils, ObIAllocator &allocator) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!other_utils.is_valid())) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to assign datum utils", K(ret), K(other_utils)); - } else { - rowkey_cnt_ = other_utils.get_rowkey_count(); - is_oracle_mode_ = other_utils.is_oracle_mode(); - ext_hash_func_ = other_utils.get_ext_hash_funcs(); - if (OB_FAIL(cmp_funcs_.init_and_assign(other_utils.get_cmp_funcs(), allocator))) { - STORAGE_LOG(WARN, "Failed to assign cmp func array", K(ret)); - } else if (OB_FAIL(hash_funcs_.init_and_assign(other_utils.get_hash_funcs(), allocator))) { - STORAGE_LOG(WARN, "Failed to assign hash func array", K(ret)); - } else { - is_inited_ = true; - } - } - - return ret; -} - -void ObStorageDatumUtils::reset() -{ - rowkey_cnt_ = 0; - cmp_funcs_.reset(); - hash_funcs_.reset(); - ext_hash_func_.hash_func_ = nullptr; - is_inited_ = false; -} - -int64_t ObStorageDatumUtils::get_deep_copy_size() const -{ - return cmp_funcs_.get_deep_copy_size() + hash_funcs_.get_deep_copy_size(); -} - int ObGhostRowUtil::is_ghost_row( const blocksstable::ObMultiVersionRowFlag &flag, bool &is_ghost_row) diff --git a/src/storage/blocksstable/ob_datum_row.h b/src/storage/blocksstable/ob_datum_row.h index 7bb5deda9..23309fa66 100644 --- a/src/storage/blocksstable/ob_datum_row.h +++ b/src/storage/blocksstable/ob_datum_row.h @@ -21,6 +21,7 @@ #include "storage/tx/ob_trans_define.h" #include "common/row/ob_row.h" #include "storage/ob_storage_util.h" +#include "storage/blocksstable/ob_datum_rowkey.h" namespace oceanbase { @@ -38,6 +39,7 @@ namespace blocksstable { struct ObDmlRowFlag; +struct ObDatumRowkey; enum ObDmlFlag { @@ -307,65 +309,6 @@ public: K_(flag)); }; -//TODO optimize number buffer -struct ObStorageDatum : public common::ObDatum -{ - ObStorageDatum() { set_nop(); } - ObStorageDatum(const ObStorageDatum &datum) { reuse(); *this = datum; } - - ~ObStorageDatum() = default; - // ext value section - OB_INLINE void reuse() { ptr_ = buf_; reserved_ = 0; pack_ = 0; } - OB_INLINE void set_ext_value(const int64_t ext_value) - { reuse(); set_ext(); no_cv(extend_obj_)->set_ext(ext_value); } - OB_INLINE void set_nop() { set_ext_value(ObActionFlag::OP_NOP); } - OB_INLINE void set_min() { set_ext_value(common::ObObj::MIN_OBJECT_VALUE); } - OB_INLINE void set_max() { set_ext_value(common::ObObj::MAX_OBJECT_VALUE); } - OB_INLINE bool is_nop_value() const { return is_nop(); } // temp solution - // transfer section - OB_INLINE bool is_local_buf() const { return ptr_ == buf_; } - OB_INLINE int from_buf_enhance(const char *buf, const int64_t buf_len); - OB_INLINE int from_obj_enhance(const common::ObObj &obj); - OB_INLINE int to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const; - OB_INLINE int deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator); - OB_INLINE int deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos); - OB_INLINE void shallow_copy_from_datum(const ObDatum &src); - OB_INLINE int64_t get_deep_copy_size() const; - OB_INLINE ObStorageDatum& operator=(const ObStorageDatum &other); - OB_INLINE int64_t storage_to_string(char *buf, int64_t buf_len, const bool for_dump = false) const; - OB_INLINE bool need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const; - OB_INLINE const char *to_cstring(const bool for_dump = false) const; - //only for unittest - OB_INLINE bool operator==(const ObStorageDatum &other) const; - OB_INLINE bool operator==(const ObObj &other) const; - - //datum 12 byte - int32_t reserved_; - // buf 16 byte - char buf_[common::OBJ_DATUM_NUMBER_RES_SIZE]; -}; - -struct ObStorageDatumBuffer -{ -public: - ObStorageDatumBuffer(common::ObIAllocator *allocator = nullptr); - ~ObStorageDatumBuffer(); - void reset(); - int init(common::ObIAllocator &allocator); - int reserve(const int64_t count, const bool keep_data = false); - OB_INLINE bool is_valid() const { return is_inited_; } - OB_INLINE ObStorageDatum *get_datums() { return datums_; } - OB_INLINE int64_t get_capacity() const { return capacity_; } - TO_STRING_KV(K_(capacity), KP_(datums), KP_(local_datums)); -private: - static const int64_t LOCAL_BUFFER_ARRAY = common::OB_ROW_DEFAULT_COLUMNS_COUNT; - int64_t capacity_; - ObStorageDatum local_datums_[LOCAL_BUFFER_ARRAY]; - ObStorageDatum *datums_; - common::ObIAllocator *allocator_; - bool is_inited_; -}; - struct ObDatumRow { OB_UNIS_VERSION(1); @@ -379,6 +322,7 @@ public: int reserve(const int64_t capacity, const bool keep_data = false); int deep_copy(const ObDatumRow &src, common::ObIAllocator &allocator); int from_store_row(const storage::ObStoreRow &store_row); + int shallow_copy(const ObDatumRow &other); //only for unittest bool operator==(const ObDatumRow &other) const; bool operator==(const common::ObNewRow &other) const; @@ -487,333 +431,6 @@ public: int64_t datum_row_offset_; }; -struct ObStorageDatumCmpFunc -{ -public: - ObStorageDatumCmpFunc(common::ObCmpFunc &cmp_func) : cmp_func_(cmp_func) {} - ObStorageDatumCmpFunc() = default; - ~ObStorageDatumCmpFunc() = default; - int compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const; - OB_INLINE const common::ObCmpFunc &get_cmp_func() const { return cmp_func_; } - TO_STRING_KV(K_(cmp_func)); -private: - common::ObCmpFunc cmp_func_; -}; -typedef storage::ObFixedMetaObjArray ObStoreCmpFuncs; -typedef storage::ObFixedMetaObjArray ObStoreHashFuncs; -struct ObStorageDatumUtils -{ -public: - ObStorageDatumUtils(); - ~ObStorageDatumUtils(); - // init with array memory from allocator - int init(const common::ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - const bool is_oracle_mode, - common::ObIAllocator &allocator, - const bool is_column_store = false); - // init with array memory on fixed size memory buffer - int init(const common::ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - const bool is_oracle_mode, - const int64_t arr_buf_len, - char *arr_buf); - int assign(const ObStorageDatumUtils &other_utils, common::ObIAllocator &allocator); - void reset(); - OB_INLINE bool is_valid() const - { - return is_inited_ && cmp_funcs_.count() >= rowkey_cnt_ && hash_funcs_.count() >= rowkey_cnt_; - } - OB_INLINE bool is_oracle_mode() const { return is_oracle_mode_; } - OB_INLINE int64_t get_rowkey_count() const { return rowkey_cnt_; } - OB_INLINE const ObStoreCmpFuncs &get_cmp_funcs() const { return cmp_funcs_; } - OB_INLINE const ObStoreHashFuncs &get_hash_funcs() const { return hash_funcs_; } - OB_INLINE const common::ObHashFunc &get_ext_hash_funcs() const { return ext_hash_func_; } - int64_t get_deep_copy_size() const; - TO_STRING_KV(K_(is_oracle_mode), K_(rowkey_cnt), K_(is_inited), K_(is_oracle_mode)); -private: - //TODO to be removed by @hanhui - int transform_multi_version_col_desc(const common::ObIArray &col_descs, - const int64_t schema_rowkey_cnt, - common::ObIArray &mv_col_descs); - int inner_init( - const common::ObIArray &mv_col_descs, - const int64_t mv_rowkey_col_cnt, - const bool is_oracle_mode); -private: - int32_t rowkey_cnt_; // multi version rowkey - ObStoreCmpFuncs cmp_funcs_; // multi version rowkey cmp funcs - ObStoreHashFuncs hash_funcs_; // multi version rowkey cmp funcs - common::ObHashFunc ext_hash_func_; - bool is_oracle_mode_; - bool is_inited_; - DISALLOW_COPY_AND_ASSIGN(ObStorageDatumUtils); -}; - - -OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator) -{ - int ret = common::OB_SUCCESS; - - reuse(); - pack_ = src.pack_; - if (is_null()) { - } else if (src.len_ == 0) { - } else if (src.is_local_buf()) { - OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); - MEMCPY(buf_, src.ptr_, src.len_); - ptr_ = buf_; - } else { - char * buf = static_cast(allocator.alloc(src.len_)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - COMMON_LOG(WARN, "allocate memory failed", K(ret), K(src)); - pack_ = 0; - } else { - MEMCPY(buf, src.ptr_, src.len_); - // need set ptr_ after memory copy, if this == &src - ptr_ = buf; - } - } - return ret; -} - -OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos) -{ - int ret = common::OB_SUCCESS; - - reuse(); - pack_ = src.pack_; - if (is_null()) { - } else if (src.len_ == 0) { - } else if (src.is_local_buf()) { - OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); - MEMCPY(buf_, src.ptr_, src.len_); - ptr_ = buf_; - } else if (OB_UNLIKELY(nullptr == buf || buf_len < pos + src.len_)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to deep copy datum", K(ret), K(src), KP(buf), K(buf_len), K(pos)); - pack_ = 0; - } else { - MEMCPY(buf + pos, src.ptr_, src.len_); - // need set ptr_ after memory copy, if this == &src - ptr_ = buf + pos; - pos += src.len_; - } - - return ret; -} - -OB_INLINE void ObStorageDatum::shallow_copy_from_datum(const ObDatum &src) -{ - if (this != &src) { - reuse(); - pack_ = src.pack_; - if (is_null()) { - } else if (src.len_ == 0) { - } else { - ptr_ = src.ptr_; - } - } -} - -OB_INLINE int64_t ObStorageDatum::get_deep_copy_size() const -{ - int64_t deep_copy_len = 0; - if (is_null()) { - } else if (is_local_buf()) { - OB_ASSERT(len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); - } else { - deep_copy_len = len_; - } - return deep_copy_len; -} - -OB_INLINE int ObStorageDatum::from_buf_enhance(const char *buf, const int64_t buf_len) -{ - int ret = common::OB_SUCCESS; - - if (OB_UNLIKELY(nullptr == buf || buf_len < 0 || buf_len > UINT32_MAX)) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument to transfer from buf", K(ret), KP(buf), K(buf_len)); - } else { - reuse(); - len_ = static_cast(buf_len); - if (buf_len > 0) { - ptr_ = buf; - } - } - - - return ret; -} - -OB_INLINE int ObStorageDatum::from_obj_enhance(const common::ObObj &obj) -{ - int ret = common::OB_SUCCESS; - - reuse(); - if (obj.is_ext()) { - set_ext_value(obj.get_ext()); - } else if (OB_FAIL(from_obj(obj))) { - STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(obj)); - } - STORAGE_LOG(DEBUG, "chaser debug from obj", K(obj), K(*this)); - - return ret; -} - - -OB_INLINE int ObStorageDatum::to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const -{ - int ret = common::OB_SUCCESS; - if (is_outrow()) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "lob should not set outrow in datum", K(ret), K(*this), K(obj), K(meta)); - } else if (is_ext()) { - obj.set_ext(get_ext()); - } else if (OB_FAIL(to_obj(obj, meta))) { - STORAGE_LOG(WARN, "Failed to transfer datum to obj", K(ret), K(*this), K(obj), K(meta)); - } - - return ret; -} - -OB_INLINE ObStorageDatum& ObStorageDatum::operator=(const ObStorageDatum &other) -{ - if (&other != this) { - reuse(); - pack_ = other.pack_; - if (is_null()) { - } else if (len_ == 0) { - } else if (other.is_local_buf()) { - OB_ASSERT(other.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); - MEMCPY(buf_, other.ptr_, other.len_); - ptr_ = buf_; - } else { - ptr_ = other.ptr_; - } - } - return *this; -} - -OB_INLINE bool ObStorageDatum::operator==(const ObStorageDatum &other) const -{ - bool bret = true; - if (is_null()) { - bret = other.is_null(); - } else if (is_ext()) { - bret = other.is_ext() && extend_obj_->get_ext() == other.extend_obj_->get_ext(); - } else { - bret = ObDatum::binary_equal(*this, other); - } - if (!bret) { - STORAGE_LOG(DEBUG, "obj and datum no equal", K(other), K(*this)); - } - return bret; - -} - -OB_INLINE bool ObStorageDatum::operator==(const common::ObObj &other) const -{ - - int ret = OB_SUCCESS; - bool bret = true; - ObStorageDatum datum; - if (OB_FAIL(datum.from_obj_enhance(other))) { - STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(other), K(datum)); - } else { - bret = *this == datum; - } - if (!bret) { - STORAGE_LOG(DEBUG, "obj and datum no equal", K(other), K(datum), KPC(this)); - } - return bret; -} - -OB_INLINE int64_t ObStorageDatum::storage_to_string(char *buf, int64_t buf_len, const bool for_dump) const -{ - int64_t pos = 0; - if (is_ext()) { - if (is_nop()) { - J_NOP(); - } else if (is_max()) { - BUF_PRINTF("MAX_OBJ"); - } else if (is_min()) { - BUF_PRINTF("MIN_OBJ"); - } - } else if(!for_dump) { - pos = to_string(buf, buf_len); - } else { - int ret = OB_SUCCESS; - const static int64_t STR_MAX_PRINT_LEN = 128L; - if (null_) { - J_NULL(); - } else { - J_OBJ_START(); - BUF_PRINTF("len: %d, flag: %d, null: %d", len_, flag_, null_); - if (len_ > 0) { - OB_ASSERT(NULL != ptr_); - const int64_t plen = std::min(static_cast(len_), - static_cast(STR_MAX_PRINT_LEN)); - // print hex value - BUF_PRINTF(", hex: "); - if (OB_FAIL(hex_print(ptr_, plen, buf, buf_len, pos))) { - // no logging in to_string function. - } else { - // maybe ObIntTC - if (sizeof(int64_t) == len_) { - BUF_PRINTF(", int: %ld", *int_); - // maybe number with one digit - if (1 == num_->desc_.len_) { - BUF_PRINTF(", num_digit0: %u", num_->digits_[0]); - } - } - // maybe printable C string - int64_t idx = 0; - while (idx < plen && isprint(ptr_[idx])) { - idx++; - } - if (idx >= plen) { - BUF_PRINTF(", cstr: %.*s", static_cast(plen), ptr_); - } - } - } - J_OBJ_END(); - } - } - - return pos; -} - -OB_INLINE const char *ObStorageDatum::to_cstring(const bool for_dump) const -{ - char *buffer = NULL; - int64_t str_len = 0; - CStringBufMgr &mgr = CStringBufMgr::get_thread_local_instance(); - mgr.inc_level(); - const int64_t buf_len = mgr.acquire(buffer); - if (OB_ISNULL(buffer)) { - LIB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "buffer is NULL"); - } else { - str_len = storage_to_string(buffer, buf_len -1, for_dump); - if (str_len >= 0 && str_len < buf_len) { - buffer[str_len] = '\0'; - } else { - buffer[0] = '\0'; - } - mgr.update_position(str_len + 1); - } - mgr.try_clear_list(); - mgr.dec_level(); - return buffer; -} - -OB_INLINE bool ObStorageDatum::need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const -{ - return OBJ_DATUM_STRING == map_type && sizeof(uint64_t) == len_ && is_local_buf(); -} - struct ObGhostRowUtil { public: ObGhostRowUtil() = delete; diff --git a/src/storage/blocksstable/ob_datum_row_iterator.h b/src/storage/blocksstable/ob_datum_row_iterator.h new file mode 100644 index 000000000..95c2ad351 --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_iterator.h @@ -0,0 +1,98 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_DATUM_ROW_ITERATOR_ +#define OCEANBASE_DATUM_ROW_ITERATOR_ + +#include "src/storage/blocksstable/ob_datum_row.h" + +namespace oceanbase +{ +namespace blocksstable +{ + +class ObDatumRowIterator +{ +public: + typedef common::ObReserveArenaAllocator<1024> ObStorageReserveAllocator; +public: + ObDatumRowIterator() {} + virtual ~ObDatumRowIterator() {} + /** + * get the next datum row and move the cursor + * + * @param row [out] + * + * @return OB_ITER_END if end of iteration + */ + virtual int get_next_row(ObDatumRow *&row) = 0; + virtual int get_next_rows(ObDatumRow *&rows, int64_t &row_count) + { + int ret = OB_SUCCESS; + if (OB_FAIL(get_next_row(rows))) { + } else { + row_count = 1; + } + return ret; + } + virtual void reset() {} + TO_STRING_EMPTY(); +}; + +/// wrap one datum row as an iterator +class ObSingleDatumRowIteratorWrapper: public ObDatumRowIterator +{ +public: + ObSingleDatumRowIteratorWrapper(); + ObSingleDatumRowIteratorWrapper(ObDatumRow *row); + virtual ~ObSingleDatumRowIteratorWrapper() {} + + void set_row(ObDatumRow *row) { row_ = row; } + virtual int get_next_row(ObDatumRow *&row); + virtual void reset() { iter_end_ = false; } +private: + // disallow copy + DISALLOW_COPY_AND_ASSIGN(ObSingleDatumRowIteratorWrapper); +private: + // data members + ObDatumRow *row_; + bool iter_end_; +}; + +inline ObSingleDatumRowIteratorWrapper::ObSingleDatumRowIteratorWrapper() + :row_(NULL), + iter_end_(false) +{} + +inline ObSingleDatumRowIteratorWrapper::ObSingleDatumRowIteratorWrapper(ObDatumRow *row) + :row_(row), + iter_end_(false) +{} + +inline int ObSingleDatumRowIteratorWrapper::get_next_row(ObDatumRow *&row) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(row_)) { + ret = OB_NOT_INIT; + } else if (iter_end_) { + ret = OB_ITER_END; + } else { + row = row_; + iter_end_ = true; + } + return ret; +} + +} +} + +#endif //OCEANBASE_DATUM_ROW_ITERATOR_ diff --git a/src/storage/blocksstable/ob_datum_row_store.cpp b/src/storage/blocksstable/ob_datum_row_store.cpp new file mode 100644 index 000000000..c0d2aa45b --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_store.cpp @@ -0,0 +1,242 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX COMMON +#include "ob_datum_row_store.h" +#include "lib/utility/utility.h" +namespace oceanbase +{ +namespace blocksstable +{ + +//////////////////////////////////////////////////////////////// +struct ObDatumRowStore::BlockInfo +{ + explicit BlockInfo(int64_t block_size) + :magic_(0xabcd4444abcd4444), + next_(NULL), + curr_data_pos_(0), + block_size_(block_size) + { + } + OB_INLINE int64_t get_remain_size() const { return block_size_ - curr_data_pos_; } + OB_INLINE int64_t get_remain_size_for_read(int64_t pos) const { return curr_data_pos_ - pos; } + OB_INLINE char *get_buffer() { return data_ + curr_data_pos_; } + OB_INLINE const char *get_buffer_head() const { return data_; } + OB_INLINE void advance(const int64_t length) { curr_data_pos_ += length; } + OB_INLINE BlockInfo *get_next_block() { return next_; } + OB_INLINE const BlockInfo *get_next_block() const { return next_; } + OB_INLINE int64_t get_block_size() const { return block_size_; }; + + int append_row(const ObDatumRow &row, const int64_t length); + friend class ObDatumRowStore::BlockList; +private: +#ifdef __clang__ + int64_t magic_ [[gnu::unused]]; +#else + int64_t magic_; +#endif + BlockInfo *next_; + /** + * cur_data_pos_ must be set when BlockInfo deserialized + */ + int64_t curr_data_pos_; + int64_t block_size_; + char data_[0]; +}; + +int ObDatumRowStore::BlockInfo::append_row(const ObDatumRow &row, const int64_t length) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + if (OB_UNLIKELY(0 > curr_data_pos_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "out of memory range", + K(ret), K_(block_size), K_(curr_data_pos), K(length)); + } else if (OB_FAIL(row.serialize(get_buffer(), get_remain_size(), pos))) { + STORAGE_LOG(WARN, "fail to serialize datum row", K(ret), K(row)); + } else { + advance(pos); + } + return ret; +} + +//////////////////////////////////////////////////////////////// +ObDatumRowStore::BlockList::BlockList() + :first_(NULL), + last_(NULL), + count_(0), + used_mem_size_(0) +{ +}; + +void ObDatumRowStore::BlockList::reset() +{ + first_ = NULL; + last_ = NULL; + count_ = 0; + used_mem_size_ = 0; +} + +int ObDatumRowStore::BlockList::add_last(BlockInfo *block) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(block)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K(block)); + } else { + block->next_ = NULL; + if (OB_ISNULL(last_)) { + if (OB_ISNULL(first_)) { + first_ = block; + last_ = block; + } else { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "invalid block list", K(ret), K_(last), K_(first)); + } + } else { + last_->next_ = block; + last_ = block; + } + } + if (OB_SUCC(ret)) { + used_mem_size_ += block->get_block_size(); + ++count_; + } + return ret; +} + +//////////////////////////////////////////////////////////////// +ObDatumRowStore::Iterator::Iterator(const ObDatumRowStore &row_store) + : row_store_(row_store), + cur_iter_block_(row_store_.blocks_.get_first()), + cur_iter_pos_(0) +{ +} + +int ObDatumRowStore::Iterator::get_next_row(ObDatumRow &row) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + if (OB_UNLIKELY(row.count_ < row_store_.get_col_count())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "column buffer count is not enough", K(ret), K_(row.count), K(row_store_.get_col_count())); + } else if (OB_ISNULL(cur_iter_block_)) { + // the last block + ret = OB_ITER_END; + } else if (OB_FAIL(row.deserialize(cur_iter_block_->get_buffer_head() + cur_iter_pos_, cur_iter_block_->get_remain_size_for_read(cur_iter_pos_), pos))) { + STORAGE_LOG(WARN, "failed to deserialize datum row", K(ret), K(cur_iter_block_->get_block_size()), K(cur_iter_pos_), + K(row.get_serialize_size()), K(pos), K(row)); + } else { + // next + cur_iter_pos_ += pos; + // update current block when current block reach its end + if (cur_iter_block_->get_remain_size_for_read(cur_iter_pos_) <= 0) { + // next block + cur_iter_block_ = cur_iter_block_->get_next_block(); + cur_iter_pos_ = 0; + } + } // end else + return ret; +} + +//////////////////////////////////////////////////////////////// +ObDatumRowStore::ObDatumRowStore() +: inner_alloc_("DatumRowStore", MTL_ID()), + blocks_(), + row_count_(0), + col_count_(0) +{ +} + + +ObDatumRowStore::~ObDatumRowStore() +{ + clear_rows(); +} + +// method for ObAggregateFunction::prepare() +// prepare need to reuse ObDatumRowStore for WRITE, +// it needs to reuse reserved_columns_ which should not be cleared +void ObDatumRowStore::clear_rows() +{ + row_count_ = 0; + col_count_ = 0; + + // free all blocks + BlockInfo *block = blocks_.get_first(); + while (NULL != block) { + BlockInfo *next = block->get_next_block(); + block->~BlockInfo(); + inner_alloc_.free(block); + block = next; + } + blocks_.reset(); +} + +int ObDatumRowStore::new_block(int64_t block_size, ObDatumRowStore::BlockInfo *&block) +{ + int ret = OB_SUCCESS; + // normalize block size + if (block_size > BIG_BLOCK_SIZE) { + block_size = OB_MAX_ROW_LENGTH_IN_MEMTABLE; + } else if (block_size > NORMAL_BLOCK_SIZE) { + block_size = BIG_BLOCK_SIZE; + } else { + block_size = NORMAL_BLOCK_SIZE; + } + // make sure all memory allocated under the right tenant + block = static_cast(inner_alloc_.alloc(block_size)); + if (OB_ISNULL(block)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "failed to alloc block memory", K(ret), K(block_size + sizeof(BlockInfo))); + } else { + block = new(block) BlockInfo(block_size - sizeof(BlockInfo)); + if (OB_FAIL(blocks_.add_last(block))) { + STORAGE_LOG(WARN, "failed to add a new block to block list", K(ret)); + } + } + return ret; +} + +int ObDatumRowStore::add_row(const ObDatumRow &row) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(0 < col_count_) && OB_UNLIKELY(row.count_ != col_count_)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "all rows should have the same columns", K(col_count_), K(row.count_)); + } else { + int64_t length = row.get_serialize_size(); + BlockInfo *block = blocks_.get_last(); + if (OB_ISNULL(block) || block->get_remain_size() < length) { + if (OB_FAIL(new_block(length, block))) { + STORAGE_LOG(WARN, "failed to new block", K(ret), K(length)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(block->append_row(row, length))) { + STORAGE_LOG(WARN, "failed to append row", K(ret), K(row)); + } else { + ++row_count_; + } + } + } + return ret; +} + +ObDatumRowStore::Iterator ObDatumRowStore::begin() const +{ + return Iterator(*this); +} + +} //end common +} //end oceanbase diff --git a/src/storage/blocksstable/ob_datum_row_store.h b/src/storage/blocksstable/ob_datum_row_store.h new file mode 100644 index 000000000..4353301f5 --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_store.h @@ -0,0 +1,92 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_COMMON_OB_DATUM_ROW_STORE_H +#define OCEANBASE_COMMON_OB_DATUM_ROW_STORE_H +#include +#include +#include "common/row/ob_row.h" +#include "common/row/ob_row_iterator.h" +#include "lib/container/ob_fixed_array.h" +#include "lib/string/ob_string.h" +#include "ob_datum_row.h" +#include "storage/ob_i_store.h" + +namespace oceanbase +{ +namespace blocksstable +{ +class ObDatumRowStore +{ +public: + struct BlockInfo; + class Iterator + { + public: + friend class ObDatumRowStore; + int get_next_row(ObDatumRow &row); + private: + explicit Iterator(const ObDatumRowStore &row_store); + protected: + const ObDatumRowStore &row_store_; + const BlockInfo *cur_iter_block_; + int64_t cur_iter_pos_; + }; +public: + ObDatumRowStore(); + ~ObDatumRowStore(); + void clear_rows(); + int add_row(const ObDatumRow &row); + inline int64_t get_row_count() const { return row_count_; } + inline int64_t get_col_count() const { return col_count_; } + Iterator begin() const; + TO_STRING_KV(N_BLOCK_NUM, blocks_.get_block_count(), + N_ROW_COUNT, row_count_, + N_COLUMN_COUNT, col_count_); +private: + DISALLOW_COPY_AND_ASSIGN(ObDatumRowStore); + + static const int64_t BIG_BLOCK_SIZE = OB_MALLOC_BIG_BLOCK_SIZE; + static const int64_t NORMAL_BLOCK_SIZE = OB_MALLOC_NORMAL_BLOCK_SIZE; + + // non-circular doubly linked list + class BlockList + { + public: + BlockList(); + void reset(); + int add_last(BlockInfo *block); + BlockInfo *get_first() { return first_; } + BlockInfo *get_last() { return last_; } + const BlockInfo *get_first() const { return first_; } + const BlockInfo *get_last() const { return last_; } + int64_t get_block_count() const { return count_; } + int64_t get_used_mem_size() const { return used_mem_size_; } + private: + BlockInfo *first_; + BlockInfo *last_; + int64_t count_; + int64_t used_mem_size_; // bytes of all blocks + }; +private: + int new_block(int64_t block_size, BlockInfo *&block); +private: + DefaultPageAllocator inner_alloc_; + BlockList blocks_; + int64_t row_count_; + int64_t col_count_; +}; + +} // end namespace common +} // end namespace oceanbase + +#endif /* OCEANBASE_COMMON_OB_DATUM_ROW_STORE_H */ diff --git a/src/storage/blocksstable/ob_datum_row_utils.cpp b/src/storage/blocksstable/ob_datum_row_utils.cpp new file mode 100644 index 000000000..6d39756a9 --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_utils.cpp @@ -0,0 +1,97 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include "ob_datum_row_utils.h" +namespace oceanbase +{ +namespace blocksstable +{ + +int ObDatumRowUtils::ob_create_row(ObIAllocator &allocator, int64_t col_count, ObDatumRow *&datum_row) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(col_count <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("row_count is invalid", K(ret), K(col_count)); + } else { + void *row_buf = NULL; + if (OB_ISNULL(row_buf = allocator.alloc(sizeof(blocksstable::ObDatumRow)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate row buffer failed", K(ret), K(sizeof(blocksstable::ObDatumRow))); + } else if (FALSE_IT(datum_row = new(row_buf) blocksstable::ObDatumRow())) { + } else if (OB_FAIL(datum_row->init(allocator, col_count))) { + LOG_WARN("fail to init datum row", K(ret), K(col_count)); + } + if (OB_FAIL(ret) && nullptr != datum_row) { + datum_row->~ObDatumRow(); + datum_row = nullptr; + allocator.free(row_buf); + } + } + return ret; +} + +int ObDatumRowUtils::ob_create_rows(ObIAllocator &allocator, int64_t row_count, int64_t col_count, ObDatumRow *&datum_rows) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(col_count <= 0 || row_count <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("col count or row count is invalid", K(ret), K(col_count), K(row_count)); + } else { + void *rows_buf = nullptr; + const size_t rows_buf_len = sizeof(blocksstable::ObDatumRow) * row_count; + if (OB_ISNULL(rows_buf = allocator.alloc(rows_buf_len))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to allocate row buffer", K(ret), K(rows_buf_len)); + } else { + char *row_buf = static_cast(rows_buf); + datum_rows = new(row_buf) blocksstable::ObDatumRow[row_count](); + int64_t i = 0; + for (; OB_SUCC(ret) && i < row_count; ++i) { + if (OB_FAIL(datum_rows[i].init(allocator, col_count))) { + LOG_WARN("fail to init datum row", K(ret), K(col_count), K(datum_rows[i])); + } + } + if (OB_FAIL(ret)) { + // release storage_datums + for (int64_t j = i; j >= 0; --j) { + datum_rows[j].~ObDatumRow(); + } + allocator.free(rows_buf); + } + } + } + return ret; +} + +int ObDatumRowUtils::prepare_rowkey( + const ObDatumRow &datum_row, + const int key_datum_cnt, + const ObColDescIArray &col_descs, + common::ObIAllocator &allocator, + ObDatumRowkey &rowkey) +{ + int ret = OB_SUCCESS; + if (!datum_row.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid datum row", K(ret), K(datum_row)); + } else if (OB_FAIL(rowkey.assign(datum_row.storage_datums_, key_datum_cnt))) { + LOG_WARN("failed to assign datum rowkey", K(ret), K(datum_row), K(key_datum_cnt)); + } else if (OB_FAIL(rowkey.prepare_memtable_readable(col_descs, allocator))) { + LOG_WARN("failed to prepare store rowkey to read memtable", K(ret), K(datum_row), K(rowkey)); + } + return ret; +} + +} // namespace sql +} // namespace oceanbase diff --git a/src/storage/blocksstable/ob_datum_row_utils.h b/src/storage/blocksstable/ob_datum_row_utils.h new file mode 100644 index 000000000..e746602eb --- /dev/null +++ b/src/storage/blocksstable/ob_datum_row_utils.h @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef DEV_SRC_STORAGE_BLOCKSSTABLE_OB_DATUM_ROW_UTILS_H_ +#define DEV_SRC_STORAGE_BLOCKSSTABLE_OB_DATUM_ROW_UTILS_H_ +#include "share/ob_define.h" +#include "ob_datum_row.h" +namespace oceanbase +{ +namespace blocksstable +{ +class ObDatumRowUtils +{ + typedef common::ObIArray ObColDescIArray; +public: + static int ob_create_row(ObIAllocator &allocator, int64_t col_count, ObDatumRow *&datum_row); + static int ob_create_rows(ObIAllocator &allocator, int64_t row_count, int64_t col_count, ObDatumRow *&datum_rows); + // TODO@xuanxi: rewrite it when store rowkey is no longer needed + static int prepare_rowkey( + const ObDatumRow &datum_row, + const int key_datum_cnt, + const ObColDescIArray &col_descs, + common::ObIAllocator &allocator, + ObDatumRowkey &rowkey); +}; + +} // namespace sql +} // namespace oceanbase +#endif /* DEV_SRC_STORAGE_BLOCKSSTABLE_OB_DATUM_ROW_UTILS_H_ */ diff --git a/src/storage/blocksstable/ob_datum_rowkey.cpp b/src/storage/blocksstable/ob_datum_rowkey.cpp index 2b8cfbf60..b66150fbb 100644 --- a/src/storage/blocksstable/ob_datum_rowkey.cpp +++ b/src/storage/blocksstable/ob_datum_rowkey.cpp @@ -616,6 +616,25 @@ int ObDatumRowkeyHelper::convert_store_rowkey(const ObDatumRowkey &datum_rowkey, return ret; } +int ObDatumRowkeyHelper::prepare_datum_rowkey(const ObDatumRow &datum_row, + const int key_datum_cnt, + const ObIArray &col_descs, + ObDatumRowkey &datum_rowkey) +{ + int ret = OB_SUCCESS; + + if (!datum_row.is_valid() || col_descs.count() < datum_row.get_column_count()) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Get invalid datum row", K(ret), K(datum_row), K(col_descs)); + } else if (OB_FAIL(datum_rowkey.assign(datum_row.storage_datums_, key_datum_cnt))) { + STORAGE_LOG(WARN, "Failed to assign datum rowkey", K(ret), K(datum_row), K(key_datum_cnt)); + } else if (OB_FAIL(convert_store_rowkey(datum_rowkey, col_descs, datum_rowkey.store_rowkey_))) { + STORAGE_LOG(WARN, "Failed to convert store rowkeyy", K(ret), K(datum_rowkey)); + } + + return ret; +} + int ObDatumRowkeyHelper::reserve(const int64_t rowkey_cnt) { diff --git a/src/storage/blocksstable/ob_datum_rowkey.h b/src/storage/blocksstable/ob_datum_rowkey.h index b6108c5d1..3164fefdc 100644 --- a/src/storage/blocksstable/ob_datum_rowkey.h +++ b/src/storage/blocksstable/ob_datum_rowkey.h @@ -13,7 +13,7 @@ #ifndef OB_STORAGE_BLOCKSSTABLE_DATUM_ROWKEY_H #define OB_STORAGE_BLOCKSSTABLE_DATUM_ROWKEY_H -#include "ob_datum_row.h" +#include "ob_storage_datum.h" #include "lib/utility/ob_print_kv.h" //to be removed #include "common/rowkey/ob_store_rowkey.h" @@ -27,6 +27,7 @@ struct ObDatumRange; class ObRowkeyVector; struct ObDiscreteDatumRowkey; struct ObCommonDatumRowkey; +struct ObDatumRow; struct ObDatumRowkey { @@ -123,6 +124,10 @@ public: int convert_store_rowkey(const ObDatumRowkey &datum_rowkey, const common::ObIArray &col_descs, common::ObStoreRowkey &rowkey); + int prepare_datum_rowkey(const ObDatumRow &datum_row, + const int key_datum_cnt, + const ObIArray &col_descs, + ObDatumRowkey &datum_rowkey); int reserve(const int64_t rowkey_cnt); OB_INLINE ObStorageDatum *get_datums() { return datum_buffer_.get_datums(); } OB_INLINE int64_t get_capacity() const { return datum_buffer_.get_capacity(); } diff --git a/src/storage/blocksstable/ob_row_writer.cpp b/src/storage/blocksstable/ob_row_writer.cpp index a46a81b4f..e8d96330e 100644 --- a/src/storage/blocksstable/ob_row_writer.cpp +++ b/src/storage/blocksstable/ob_row_writer.cpp @@ -76,17 +76,16 @@ int ObRowWriter::init_common(char *buf, const int64_t buf_size, const int64_t po } int ObRowWriter::check_row_valid( - const ObStoreRow &row, + const ObDatumRow &row, const int64_t rowkey_column_count) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!row.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid row writer input argument", K(row), K(ret)); - } else if (OB_UNLIKELY(rowkey_column_count <= 0 || rowkey_column_count > row.row_val_.count_)) { + LOG_WARN("invalid row writer input argument", K(ret), K(row)); + } else if (OB_UNLIKELY(rowkey_column_count <= 0 || rowkey_column_count > row.count_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid row writer input argument", - K(rowkey_column_count), K(row.row_val_.count_), K(ret)); + LOG_WARN("invalid row writer input argument", K(ret), K(rowkey_column_count), K(row.count_)); } return ret; } @@ -230,24 +229,24 @@ int ObRowWriter::write(const int64_t rowkey_column_cnt, const ObDatumRow &datum_ return ret; } -// when update_idx == nullptr, write full row; else only write rowkey + update cells +// when update_idx == nullptr, write full row; else only write rowkey + update storage_datums int ObRowWriter::write( const int64_t rowkey_column_count, - const storage::ObStoreRow &row, + const ObDatumRow &datum_row, const ObIArray *update_idx, char *&buf, int64_t &len) { int ret = OB_SUCCESS; len = 0; - if (OB_UNLIKELY(nullptr != update_idx && update_idx->count() > row.row_val_.count_)) { + if (OB_UNLIKELY(nullptr != update_idx && update_idx->count() > datum_row.count_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("update idx is invalid", K(ret), KPC(update_idx), K_(row.row_val_.count), K(rowkey_column_count)); + LOG_WARN("update idx is invalid", K(ret), KPC(update_idx), K_(datum_row.count), K(rowkey_column_count)); } else { do { if (OB_FAIL(alloc_buf_and_init(OB_BUF_NOT_ENOUGH == ret))) { LOG_WARN("row writer fail to alloc and init", K(ret)); - } else if (OB_FAIL(inner_write_row(rowkey_column_count, row, update_idx))) { + } else if (OB_FAIL(inner_write_row(rowkey_column_count, datum_row, update_idx))) { if (OB_BUF_NOT_ENOUGH != ret) { LOG_WARN("row writer fail to append row header", K(ret), K(row_buffer_), K(pos_)); } @@ -289,29 +288,29 @@ int ObRowWriter::check_update_idx_array_valid( int ObRowWriter::inner_write_row( const int64_t rowkey_column_count, - const ObStoreRow &row, + const ObDatumRow &datum_row, const ObIArray *update_idx) { int ret = OB_SUCCESS; - if (OB_FAIL(check_row_valid(row, rowkey_column_count))) { + if (OB_FAIL(check_row_valid(datum_row, rowkey_column_count))) { LOG_WARN("row writer fail to init store row", K(ret), K(rowkey_column_count)); } else if (nullptr != update_idx && OB_FAIL(check_update_idx_array_valid(rowkey_column_count, update_idx))) { LOG_WARN("invalid update idx array", K(ret)); } else if (OB_FAIL(append_row_header( - row.flag_.get_serialize_flag(), - row.row_type_flag_.flag_, - row.trans_id_.get_id(), - row.row_val_.count_, + datum_row.row_flag_.get_serialize_flag(), + datum_row.mvcc_row_flag_.flag_, + datum_row.trans_id_.get_id(), + datum_row.count_, rowkey_column_count))) { if (OB_BUF_NOT_ENOUGH != ret) { - LOG_WARN("row writer fail to append row header", K(ret), K(row)); + LOG_WARN("row writer fail to append row header", K(ret), K(datum_row)); } } else { update_idx_array_ = update_idx; rowkey_column_cnt_ = rowkey_column_count; - if (OB_FAIL(inner_write_cells(row.row_val_.cells_, row.row_val_.count_))) { + if (OB_FAIL(inner_write_cells(datum_row.storage_datums_, datum_row.count_))) { if (OB_BUF_NOT_ENOUGH != ret) { - LOG_WARN("failed to write cells", K(ret), K(row)); + LOG_WARN("failed to write cells", K(ret), K(datum_row)); } } } diff --git a/src/storage/blocksstable/ob_row_writer.h b/src/storage/blocksstable/ob_row_writer.h index b3cc98192..773e3f4cb 100644 --- a/src/storage/blocksstable/ob_row_writer.h +++ b/src/storage/blocksstable/ob_row_writer.h @@ -63,7 +63,7 @@ public: int write_rowkey(const common::ObStoreRowkey &rowkey, char *&buf, int64_t &len); int write( const int64_t rowkey_cnt, - const storage::ObStoreRow &row, + const ObDatumRow &datum_row, const ObIArray *update_idx, char *&buf, int64_t &len); @@ -102,7 +102,7 @@ private: }; int inner_write_row( const int64_t rowkey_column_count, - const storage::ObStoreRow &row, + const ObDatumRow &row, const ObIArray *update_idx); OB_INLINE int write_oracle_timestamp(const common::ObOTimestampData &ot_data, const common::ObOTimestampMetaAttrType otmat); int append_column(const common::ObObj &obj); @@ -110,7 +110,7 @@ private: int append_8_bytes_column(const ObStorageDatum &datum); int init_common(char *buf, const int64_t buf_size, const int64_t pos); int check_row_valid( - const storage::ObStoreRow &row, + const ObDatumRow &row, const int64_t rowkey_column_count); int append_row_header( const uint8_t row_flag, diff --git a/src/storage/blocksstable/ob_storage_datum.cpp b/src/storage/blocksstable/ob_storage_datum.cpp new file mode 100644 index 000000000..52e3c513d --- /dev/null +++ b/src/storage/blocksstable/ob_storage_datum.cpp @@ -0,0 +1,386 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include "ob_storage_datum.h" +#include "share/schema/ob_table_param.h" +#include "share/ob_force_print_log.h" +#include "storage/ob_i_store.h" +#include "share/scheduler/ob_tenant_dag_scheduler.h" + +namespace oceanbase +{ +using namespace common; +namespace blocksstable +{ +static int nonext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) +{ + int ret = cmp_func.cmp_func_(left, right, cmp_ret); + STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); + return ret; +} + +static int nonext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) +{ + int ret = OB_SUCCESS; + UNUSEDx(left, cmp_func); + if (right.is_max()) { + cmp_ret = -1; + } else if (right.is_min()) { + cmp_ret = 1; + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(right)); + } + STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); + return ret; +} + +static int ext_nonext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) +{ + int ret = OB_SUCCESS; + UNUSEDx(right, cmp_func); + if (left.is_max()) { + cmp_ret = 1; + } else if (left.is_min()) { + cmp_ret = -1; + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left)); + } + STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); + return ret; +} + +static int ext_ext_compare(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret) +{ + int ret = OB_SUCCESS; + UNUSEDx(cmp_func); + int64_t lv = left.is_max() - left.is_min(); + int64_t rv = right.is_max() - right.is_min(); + if (OB_UNLIKELY(0 == lv || 0 == rv)) { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected datum in rowkey to compare", K(ret), K(left), K(right)); + } else { + cmp_ret = lv - rv; + } + STORAGE_LOG(DEBUG, "chaser debug compare datum", K(ret), K(left), K(right), K(cmp_ret)); + + return ret; +} + +typedef int (*ExtSafeCompareFunc)(const ObStorageDatum &left, const ObStorageDatum &right, const common::ObCmpFunc &cmp_func, int &cmp_ret); +static ExtSafeCompareFunc ext_safe_cmp_funcs[2][2] = { + {nonext_nonext_compare, nonext_ext_compare}, + {ext_nonext_compare, ext_ext_compare} +}; + +int ObStorageDatumCmpFunc::compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const +{ + return ext_safe_cmp_funcs[left.is_ext()][right.is_ext()](left, right, cmp_func_, cmp_ret); +} + +/* + *ObStorageDatumUtils + */ +ObStorageDatumUtils::ObStorageDatumUtils() + : rowkey_cnt_(0), + cmp_funcs_(), + hash_funcs_(), + ext_hash_func_(), + is_oracle_mode_(false), + is_inited_(false) +{} + +ObStorageDatumUtils::~ObStorageDatumUtils() +{} + +int ObStorageDatumUtils::transform_multi_version_col_desc(const ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + ObIArray &mv_col_descs) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(schema_rowkey_cnt > col_descs.count())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to transform mv col descs", K(ret), K(schema_rowkey_cnt), K(col_descs)); + } else { + mv_col_descs.reuse(); + for (int64_t i = 0; OB_SUCC(ret) && i < schema_rowkey_cnt; i++) { + if (OB_FAIL(mv_col_descs.push_back(col_descs.at(i)))) { + STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(i)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(storage::ObMultiVersionRowkeyHelpper::add_extra_rowkey_cols(mv_col_descs))) { + STORAGE_LOG(WARN, "Fail to add extra_rowkey_cols", K(ret), K(schema_rowkey_cnt)); + } else { + for (int64_t i = schema_rowkey_cnt; OB_SUCC(ret) && i < col_descs.count(); i++) { + const share::schema::ObColDesc &col_desc = col_descs.at(i); + if (col_desc.col_id_ == common::OB_HIDDEN_TRANS_VERSION_COLUMN_ID + || col_desc.col_id_ == common::OB_HIDDEN_SQL_SEQUENCE_COLUMN_ID) { + continue; + } else if (OB_FAIL(mv_col_descs.push_back(col_desc))) { + STORAGE_LOG(WARN, "Failed to push back col desc", K(ret), K(col_desc)); + } + } + } + } + + return ret; +} + +int ObStorageDatumUtils::init(const ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + const bool is_oracle_mode, + ObIAllocator &allocator, + const bool is_column_store) +{ + int ret = OB_SUCCESS; + ObSEArray mv_col_descs; + int64_t mv_rowkey_cnt = 0; + int64_t mv_extra_rowkey_cnt = 0; + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this)); + } else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER + || schema_rowkey_cnt > col_descs.count())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(schema_rowkey_cnt), K(col_descs)); + } else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) { + STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret)); + } else if (FALSE_IT(mv_extra_rowkey_cnt = is_column_store ? 0 : storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) { + } else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + mv_extra_rowkey_cnt)) { + } else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, allocator))) { + STORAGE_LOG(WARN, "Failed to reserve cmp func array", K(ret)); + } else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, allocator))) { + STORAGE_LOG(WARN, "Failed to reserve hash func array", K(ret)); + } else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) { + STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt)); + } + + return ret; +} + +int ObStorageDatumUtils::init(const common::ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + const bool is_oracle_mode, + const int64_t arr_buf_len, + char *arr_buf) +{ + int ret = OB_SUCCESS; + ObSEArray mv_col_descs; + int64_t pos = 0; + int64_t mv_rowkey_cnt = 0; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObStorageDatumUtils init twice", K(ret), K(*this)); + } else if (OB_UNLIKELY(schema_rowkey_cnt < 0 || schema_rowkey_cnt > OB_MAX_ROWKEY_COLUMN_NUMBER + || schema_rowkey_cnt > col_descs.count())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to init storage datum utils", K(ret), K(col_descs), K(schema_rowkey_cnt)); + } else if (OB_FAIL(transform_multi_version_col_desc(col_descs, schema_rowkey_cnt, mv_col_descs))) { + STORAGE_LOG(WARN, "Failed to transform multi version col descs", K(ret)); + } else if (FALSE_IT(mv_rowkey_cnt = schema_rowkey_cnt + storage::ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt())) { + } else if (OB_FAIL(cmp_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) { + STORAGE_LOG(WARN, "Failed to init compare function array", K(ret)); + } else if (OB_FAIL(hash_funcs_.init(mv_rowkey_cnt, arr_buf_len, arr_buf, pos))) { + STORAGE_LOG(WARN, "Failed to init hash function array", K(ret)); + } else if (OB_FAIL(inner_init(mv_col_descs, mv_rowkey_cnt, is_oracle_mode))) { + STORAGE_LOG(WARN, "Failed to inner init datum utils", K(ret), K(mv_col_descs), K(mv_rowkey_cnt)); + } + return ret; +} + +int ObStorageDatumUtils::inner_init( + const common::ObIArray &mv_col_descs, + const int64_t mv_rowkey_col_cnt, + const bool is_oracle_mode) +{ + int ret = OB_SUCCESS; + is_oracle_mode_ = is_oracle_mode; + // support column order index until next task done + // + // we could use the cmp funcs in the basic funcs directlly + bool is_null_last = is_oracle_mode_; + ObCmpFunc cmp_func; + ObHashFunc hash_func; + for (int64_t i = 0; OB_SUCC(ret) && i < mv_rowkey_col_cnt; i++) { + const share::schema::ObColDesc &col_desc = mv_col_descs.at(i); + //TODO @hanhui support desc rowkey + bool is_ascending = true || col_desc.col_order_ == ObOrderType::ASC; + bool has_lob_header = is_lob_storage(col_desc.col_type_.get_type()); + ObPrecision precision = PRECISION_UNKNOWN_YET; + if (col_desc.col_type_.is_decimal_int()) { + precision = col_desc.col_type_.get_stored_precision(); + OB_ASSERT(precision != PRECISION_UNKNOWN_YET); + } + sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(col_desc.col_type_.get_type(), + col_desc.col_type_.get_collation_type(), + col_desc.col_type_.get_scale(), + is_oracle_mode, + has_lob_header, + precision); + if (OB_UNLIKELY(nullptr == basic_funcs + || nullptr == basic_funcs->null_last_cmp_ + || nullptr == basic_funcs->murmur_hash_)) { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected null basic funcs", K(ret), K(col_desc)); + } else { + cmp_func.cmp_func_ = is_null_last ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_; + hash_func.hash_func_ = basic_funcs->murmur_hash_; + if (OB_FAIL(hash_funcs_.push_back(hash_func))) { + STORAGE_LOG(WARN, "Failed to push back hash func", K(ret), K(i), K(col_desc)); + } else if (is_ascending) { + if (OB_FAIL(cmp_funcs_.push_back(ObStorageDatumCmpFunc(cmp_func)))) { + STORAGE_LOG(WARN, "Failed to push back cmp func", K(ret), K(i), K(col_desc)); + } + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(WARN, "Unsupported desc column order", K(ret), K(col_desc), K(i)); + } + } + } + if (OB_SUCC(ret)) { + sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(ObExtendType, CS_TYPE_BINARY); + if (OB_UNLIKELY(nullptr == basic_funcs || nullptr == basic_funcs->murmur_hash_)) { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected null basic funcs for extend type", K(ret)); + } else { + ext_hash_func_.hash_func_ = basic_funcs->murmur_hash_; + rowkey_cnt_ = mv_rowkey_col_cnt; + is_inited_ = true; + } + } + + return ret; +} + +int ObStorageDatumUtils::assign(const ObStorageDatumUtils &other_utils, ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(!other_utils.is_valid())) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to assign datum utils", K(ret), K(other_utils)); + } else { + rowkey_cnt_ = other_utils.get_rowkey_count(); + is_oracle_mode_ = other_utils.is_oracle_mode(); + ext_hash_func_ = other_utils.get_ext_hash_funcs(); + if (OB_FAIL(cmp_funcs_.init_and_assign(other_utils.get_cmp_funcs(), allocator))) { + STORAGE_LOG(WARN, "Failed to assign cmp func array", K(ret)); + } else if (OB_FAIL(hash_funcs_.init_and_assign(other_utils.get_hash_funcs(), allocator))) { + STORAGE_LOG(WARN, "Failed to assign hash func array", K(ret)); + } else { + is_inited_ = true; + } + } + + return ret; +} + +void ObStorageDatumUtils::reset() +{ + rowkey_cnt_ = 0; + cmp_funcs_.reset(); + hash_funcs_.reset(); + ext_hash_func_.hash_func_ = nullptr; + is_inited_ = false; +} + +int64_t ObStorageDatumUtils::get_deep_copy_size() const +{ + return cmp_funcs_.get_deep_copy_size() + hash_funcs_.get_deep_copy_size(); +} + +/* + *ObStorageDatumBuffer + */ +ObStorageDatumBuffer::ObStorageDatumBuffer(common::ObIAllocator *allocator) + : capacity_(LOCAL_BUFFER_ARRAY), + local_datums_(), + datums_(local_datums_), + allocator_(allocator), + is_inited_(nullptr != allocator) +{} + +ObStorageDatumBuffer::~ObStorageDatumBuffer() +{ + if (datums_ != local_datums_ && nullptr != allocator_) { + allocator_->free(datums_); + } +} + +void ObStorageDatumBuffer::reset() +{ + if (datums_ != local_datums_ && nullptr != allocator_) { + allocator_->free(datums_); + } + allocator_ = nullptr; + datums_ = local_datums_; + capacity_ = LOCAL_BUFFER_ARRAY; + for (int64_t i = 0; i < capacity_; i++) { + datums_[i].reuse(); + } + is_inited_ = false; +} + +int ObStorageDatumBuffer::init(common::ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + + if (IS_INIT) { + ret = OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObStorageDatumBuffer init twice", K(ret), K(*this)); + } else { + OB_ASSERT(datums_ == local_datums_); + allocator_ = &allocator; + is_inited_ = true; + } + + return ret; +} + +int ObStorageDatumBuffer::reserve(const int64_t count, const bool keep_data) +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "ObStorageDatumBuffer is not inited", K(ret), K(*this)); + } else if (OB_UNLIKELY(count <= 0)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to reserve datum buffer", K(ret), K(count)); + } else if (count <= capacity_){ + } else if (OB_ISNULL(buf = allocator_->alloc(sizeof(ObStorageDatum) * count))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "Failed to alloc memory", K(ret), K(count)); + } else { + ObStorageDatum *new_datums = new (buf) ObStorageDatum [count]; + if (keep_data) { + for (int64_t i = 0; i < capacity_; i++) { + new_datums[i] = datums_[i]; + } + } + if (nullptr != datums_ && datums_ != local_datums_) { + allocator_->free(datums_); + } + datums_ = new_datums; + capacity_ = count; + } + + return ret; +} + +} // namespace blocksstable +} // namespace oceanbase diff --git a/src/storage/blocksstable/ob_storage_datum.h b/src/storage/blocksstable/ob_storage_datum.h new file mode 100644 index 000000000..8dd1477f4 --- /dev/null +++ b/src/storage/blocksstable/ob_storage_datum.h @@ -0,0 +1,427 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_STORAGE_BLOCKSSTABLE_STORAGE_DATUM_H +#define OB_STORAGE_BLOCKSSTABLE_STORAGE_DATUM_H + +#include "common/ob_common_types.h" +#include "common/ob_tablet_id.h" +#include "share/datum/ob_datum.h" +#include "share/datum/ob_datum_funcs.h" +#include "storage/meta_mem/ob_fixed_meta_obj_array.h" +#include "storage/tx/ob_trans_define.h" +#include "common/row/ob_row.h" +#include "storage/ob_storage_util.h" + +namespace oceanbase +{ +namespace share{ +namespace schema +{ +struct ObColDesc; +} +} +namespace storage +{ +struct ObStoreRow; +} +namespace blocksstable +{ + +//TODO optimize number buffer +struct ObStorageDatum : public common::ObDatum +{ + ObStorageDatum() { set_nop(); } + ObStorageDatum(const ObStorageDatum &datum) { reuse(); *this = datum; } + + ~ObStorageDatum() = default; + // ext value section + OB_INLINE void reuse() { ptr_ = buf_; reserved_ = 0; pack_ = 0; } + OB_INLINE void set_ext_value(const int64_t ext_value) + { reuse(); set_ext(); no_cv(extend_obj_)->set_ext(ext_value); } + OB_INLINE void set_nop() { set_ext_value(ObActionFlag::OP_NOP); } + OB_INLINE void set_min() { set_ext_value(common::ObObj::MIN_OBJECT_VALUE); } + OB_INLINE void set_max() { set_ext_value(common::ObObj::MAX_OBJECT_VALUE); } + OB_INLINE bool is_nop_value() const { return is_nop(); } // temp solution + // transfer section + OB_INLINE bool is_local_buf() const { return ptr_ == buf_; } + OB_INLINE int from_buf_enhance(const char *buf, const int64_t buf_len); + OB_INLINE int from_obj_enhance(const common::ObObj &obj); + OB_INLINE int to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const; + OB_INLINE int deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator); + OB_INLINE int deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos); + OB_INLINE void shallow_copy_from_datum(const ObDatum &src); + OB_INLINE int64_t get_deep_copy_size() const; + OB_INLINE ObStorageDatum& operator=(const ObStorageDatum &other); + OB_INLINE int64_t storage_to_string(char *buf, int64_t buf_len, const bool for_dump = false) const; + OB_INLINE bool need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const; + OB_INLINE const char *to_cstring(const bool for_dump = false) const; + //only for unittest + OB_INLINE bool operator==(const ObStorageDatum &other) const; + OB_INLINE bool operator==(const ObObj &other) const; + + //datum 12 byte + int32_t reserved_; + // buf 16 byte + char buf_[common::OBJ_DATUM_NUMBER_RES_SIZE]; +}; + +struct ObStorageDatumCmpFunc +{ +public: + ObStorageDatumCmpFunc(common::ObCmpFunc &cmp_func) : cmp_func_(cmp_func) {} + ObStorageDatumCmpFunc() = default; + ~ObStorageDatumCmpFunc() = default; + int compare(const ObStorageDatum &left, const ObStorageDatum &right, int &cmp_ret) const; + OB_INLINE const common::ObCmpFunc &get_cmp_func() const { return cmp_func_; } + TO_STRING_KV(K_(cmp_func)); +private: + common::ObCmpFunc cmp_func_; +}; +typedef storage::ObFixedMetaObjArray ObStoreCmpFuncs; +typedef storage::ObFixedMetaObjArray ObStoreHashFuncs; +struct ObStorageDatumUtils +{ +public: + ObStorageDatumUtils(); + ~ObStorageDatumUtils(); + // init with array memory from allocator + int init(const common::ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + const bool is_oracle_mode, + common::ObIAllocator &allocator, + const bool is_column_store = false); + // init with array memory on fixed size memory buffer + int init(const common::ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + const bool is_oracle_mode, + const int64_t arr_buf_len, + char *arr_buf); + int assign(const ObStorageDatumUtils &other_utils, common::ObIAllocator &allocator); + void reset(); + OB_INLINE bool is_valid() const + { + return is_inited_ && cmp_funcs_.count() >= rowkey_cnt_ && hash_funcs_.count() >= rowkey_cnt_; + } + OB_INLINE bool is_oracle_mode() const { return is_oracle_mode_; } + OB_INLINE int64_t get_rowkey_count() const { return rowkey_cnt_; } + OB_INLINE const ObStoreCmpFuncs &get_cmp_funcs() const { return cmp_funcs_; } + OB_INLINE const ObStoreHashFuncs &get_hash_funcs() const { return hash_funcs_; } + OB_INLINE const common::ObHashFunc &get_ext_hash_funcs() const { return ext_hash_func_; } + int64_t get_deep_copy_size() const; + TO_STRING_KV(K_(is_oracle_mode), K_(rowkey_cnt), K_(is_inited), K_(is_oracle_mode)); +private: + //TODO to be removed by @hanhui + int transform_multi_version_col_desc(const common::ObIArray &col_descs, + const int64_t schema_rowkey_cnt, + common::ObIArray &mv_col_descs); + int inner_init( + const common::ObIArray &mv_col_descs, + const int64_t mv_rowkey_col_cnt, + const bool is_oracle_mode); +private: + int32_t rowkey_cnt_; // multi version rowkey + ObStoreCmpFuncs cmp_funcs_; // multi version rowkey cmp funcs + ObStoreHashFuncs hash_funcs_; // multi version rowkey cmp funcs + common::ObHashFunc ext_hash_func_; + bool is_oracle_mode_; + bool is_inited_; + DISALLOW_COPY_AND_ASSIGN(ObStorageDatumUtils); +}; + +struct ObStorageDatumBuffer +{ +public: + ObStorageDatumBuffer(common::ObIAllocator *allocator = nullptr); + ~ObStorageDatumBuffer(); + void reset(); + int init(common::ObIAllocator &allocator); + int reserve(const int64_t count, const bool keep_data = false); + OB_INLINE bool is_valid() const { return is_inited_; } + OB_INLINE ObStorageDatum *get_datums() { return datums_; } + OB_INLINE int64_t get_capacity() const { return capacity_; } + TO_STRING_KV(K_(capacity), KP_(datums), KP_(local_datums)); +private: + static const int64_t LOCAL_BUFFER_ARRAY = common::OB_ROW_DEFAULT_COLUMNS_COUNT >> 1; + int64_t capacity_; + ObStorageDatum local_datums_[LOCAL_BUFFER_ARRAY]; + ObStorageDatum *datums_; + common::ObIAllocator *allocator_; + bool is_inited_; +}; + + +OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, common::ObIAllocator &allocator) +{ + int ret = common::OB_SUCCESS; + + reuse(); + pack_ = src.pack_; + if (is_null()) { + } else if (src.len_ == 0) { + } else if (src.is_local_buf()) { + OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); + MEMCPY(buf_, src.ptr_, src.len_); + ptr_ = buf_; + } else { + char * buf = static_cast(allocator.alloc(src.len_)); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + COMMON_LOG(WARN, "allocate memory failed", K(ret), K(src)); + pack_ = 0; + } else { + MEMCPY(buf, src.ptr_, src.len_); + // need set ptr_ after memory copy, if this == &src + ptr_ = buf; + } + } + return ret; +} + +OB_INLINE int ObStorageDatum::deep_copy(const ObStorageDatum &src, char * buf, const int64_t buf_len, int64_t &pos) +{ + int ret = common::OB_SUCCESS; + + reuse(); + pack_ = src.pack_; + if (is_null()) { + } else if (src.len_ == 0) { + } else if (src.is_local_buf()) { + OB_ASSERT(src.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); + MEMCPY(buf_, src.ptr_, src.len_); + ptr_ = buf_; + } else if (OB_UNLIKELY(nullptr == buf || buf_len < pos + src.len_)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to deep copy datum", K(ret), K(src), KP(buf), K(buf_len), K(pos)); + pack_ = 0; + } else { + MEMCPY(buf + pos, src.ptr_, src.len_); + // need set ptr_ after memory copy, if this == &src + ptr_ = buf + pos; + pos += src.len_; + } + + return ret; +} + +OB_INLINE void ObStorageDatum::shallow_copy_from_datum(const ObDatum &src) +{ + if (this != &src) { + reuse(); + pack_ = src.pack_; + if (is_null()) { + } else if (src.len_ == 0) { + } else { + ptr_ = src.ptr_; + } + } +} + +OB_INLINE int64_t ObStorageDatum::get_deep_copy_size() const +{ + int64_t deep_copy_len = 0; + if (is_null()) { + } else if (is_local_buf()) { + OB_ASSERT(len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); + } else { + deep_copy_len = len_; + } + return deep_copy_len; +} + +OB_INLINE int ObStorageDatum::from_buf_enhance(const char *buf, const int64_t buf_len) +{ + int ret = common::OB_SUCCESS; + + if (OB_UNLIKELY(nullptr == buf || buf_len < 0 || buf_len > UINT32_MAX)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to transfer from buf", K(ret), KP(buf), K(buf_len)); + } else { + reuse(); + len_ = static_cast(buf_len); + if (buf_len > 0) { + ptr_ = buf; + } + } + + + return ret; +} + +OB_INLINE int ObStorageDatum::from_obj_enhance(const common::ObObj &obj) +{ + int ret = common::OB_SUCCESS; + + reuse(); + if (obj.is_ext()) { + set_ext_value(obj.get_ext()); + } else if (OB_FAIL(from_obj(obj))) { + STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(obj)); + } + STORAGE_LOG(DEBUG, "chaser debug from obj", K(obj), K(*this)); + + return ret; +} + + +OB_INLINE int ObStorageDatum::to_obj_enhance(common::ObObj &obj, const common::ObObjMeta &meta) const +{ + int ret = common::OB_SUCCESS; + if (is_outrow()) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "lob should not set outrow in datum", K(ret), K(*this), K(obj), K(meta)); + } else if (is_ext()) { + obj.set_ext(get_ext()); + } else if (OB_FAIL(to_obj(obj, meta))) { + STORAGE_LOG(WARN, "Failed to transfer datum to obj", K(ret), K(*this), K(obj), K(meta)); + } + + return ret; +} + +OB_INLINE ObStorageDatum& ObStorageDatum::operator=(const ObStorageDatum &other) +{ + if (&other != this) { + reuse(); + pack_ = other.pack_; + if (is_null()) { + } else if (len_ == 0) { + } else if (other.is_local_buf()) { + OB_ASSERT(other.len_ <= common::OBJ_DATUM_NUMBER_RES_SIZE); + MEMCPY(buf_, other.ptr_, other.len_); + ptr_ = buf_; + } else { + ptr_ = other.ptr_; + } + } + return *this; +} + +OB_INLINE bool ObStorageDatum::operator==(const ObStorageDatum &other) const +{ + bool bret = true; + if (is_null()) { + bret = other.is_null(); + } else if (is_ext()) { + bret = other.is_ext() && extend_obj_->get_ext() == other.extend_obj_->get_ext(); + } else { + bret = ObDatum::binary_equal(*this, other); + } + if (!bret) { + STORAGE_LOG(DEBUG, "datum and datum no equal", K(other), K(*this)); + } + return bret; + +} + +OB_INLINE bool ObStorageDatum::operator==(const common::ObObj &other) const +{ + + int ret = OB_SUCCESS; + bool bret = true; + ObStorageDatum datum; + if (OB_FAIL(datum.from_obj_enhance(other))) { + STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret), K(other), K(datum)); + } else { + bret = *this == datum; + } + if (!bret) { + STORAGE_LOG(DEBUG, "obj and datum no equal", K(other), K(datum), KPC(this)); + } + return bret; +} + +OB_INLINE int64_t ObStorageDatum::storage_to_string(char *buf, int64_t buf_len, const bool for_dump) const +{ + int64_t pos = 0; + if (is_ext()) { + if (is_nop()) { + J_NOP(); + } else if (is_max()) { + BUF_PRINTF("MAX_OBJ"); + } else if (is_min()) { + BUF_PRINTF("MIN_OBJ"); + } + } else if(!for_dump) { + pos = to_string(buf, buf_len); + } else { + int ret = OB_SUCCESS; + const static int64_t STR_MAX_PRINT_LEN = 128L; + if (null_) { + J_NULL(); + } else { + J_OBJ_START(); + BUF_PRINTF("len: %d, flag: %d, null: %d", len_, flag_, null_); + if (len_ > 0) { + OB_ASSERT(NULL != ptr_); + const int64_t plen = std::min(static_cast(len_), + static_cast(STR_MAX_PRINT_LEN)); + // print hex value + BUF_PRINTF(", hex: "); + if (OB_FAIL(hex_print(ptr_, plen, buf, buf_len, pos))) { + // no logging in to_string function. + } else { + // maybe ObIntTC + if (sizeof(int64_t) == len_) { + BUF_PRINTF(", int: %ld", *int_); + // maybe number with one digit + if (1 == num_->desc_.len_) { + BUF_PRINTF(", num_digit0: %u", num_->digits_[0]); + } + } + // maybe printable C string + int64_t idx = 0; + while (idx < plen && isprint(ptr_[idx])) { + idx++; + } + if (idx >= plen) { + BUF_PRINTF(", cstr: %.*s", static_cast(plen), ptr_); + } + } + } + J_OBJ_END(); + } + } + + return pos; +} + +OB_INLINE const char *ObStorageDatum::to_cstring(const bool for_dump) const +{ + char *buffer = NULL; + int64_t str_len = 0; + CStringBufMgr &mgr = CStringBufMgr::get_thread_local_instance(); + mgr.inc_level(); + const int64_t buf_len = mgr.acquire(buffer); + if (OB_ISNULL(buffer)) { + LIB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "buffer is NULL"); + } else { + str_len = storage_to_string(buffer, buf_len -1, for_dump); + if (str_len >= 0 && str_len < buf_len) { + buffer[str_len] = '\0'; + } else { + buffer[0] = '\0'; + } + mgr.update_position(str_len + 1); + } + mgr.try_clear_list(); + mgr.dec_level(); + return buffer; +} + +OB_INLINE bool ObStorageDatum::need_copy_for_encoding_column_with_flat_format(const ObObjDatumMapType map_type) const +{ + return OBJ_DATUM_STRING == map_type && sizeof(uint64_t) == len_ && is_local_buf(); +} +} // namespace blocksstable +} // namespace oceanbase +#endif diff --git a/src/storage/lob/ob_ext_info_callback.cpp b/src/storage/lob/ob_ext_info_callback.cpp index bdd4b12c6..5db3148cb 100644 --- a/src/storage/lob/ob_ext_info_callback.cpp +++ b/src/storage/lob/ob_ext_info_callback.cpp @@ -186,7 +186,7 @@ int ObExtInfoCallback::set( { int ret = OB_SUCCESS; ObLobManager *lob_mngr = MTL(ObLobManager*); - ObDatumRow datum_row; + blocksstable::ObDatumRow datum_row; char *buf = nullptr; int64_t len = 0; @@ -250,7 +250,8 @@ int ObExtInfoCbRegister::register_cb( const blocksstable::ObDmlFlag dml_flag, transaction::ObTxDesc *tx_desc, transaction::ObTxSEQ &parent_seq_no, - ObObj &index_data, + blocksstable::ObStorageDatum &index_data, + ObObjType &type, ObObj &ext_info_data) { int ret = OB_SUCCESS; @@ -269,7 +270,7 @@ int ObExtInfoCbRegister::register_cb( } else if (OB_ISNULL(mvcc_ctx_ = ctx)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("data is empty", K(ret), K(ext_info_data)); - } else if (OB_FAIL(init_header(index_data, ext_info_data))) { + } else if (OB_FAIL(init_header(type))) { LOG_WARN("init_header_ fail", K(ret), K(ext_info_data)); } else if (OB_FAIL(build_data_iter(ext_info_data))) { LOG_WARN("build data iter fail", K(ret)); @@ -306,24 +307,24 @@ int ObExtInfoCbRegister::register_cb( } if (OB_FAIL(ret)) { } else if (OB_FALSE_IT(seq_no_cnt_ = cb_cnt)) { - } else if (OB_FAIL(set_index_data(index_data))) { + } else if (OB_FAIL(set_index_data(index_data, type))) { LOG_WARN("set_index_data fail", K(ret)); } } return ret; } -int ObExtInfoCbRegister::init_header(ObObj& index_data, ObObj &ext_info_data) +int ObExtInfoCbRegister::init_header(ObObjType &type) { int ret = OB_SUCCESS; - header_.type_ = get_type(index_data.get_type()); + header_.type_ = get_type(type); return ret; } -int ObExtInfoCbRegister::set_index_data(ObObj &index_data) +int ObExtInfoCbRegister::set_index_data(blocksstable::ObStorageDatum &index_data, ObObjType &type) { int ret = OB_SUCCESS; - if (is_lob_storage(index_data.get_type())) { + if (is_lob_storage(type)) { if (OB_FAIL(set_outrow_ctx_seq_no(index_data))) { LOG_WARN("set_outrow_ctx_seq_no fail", K(ret)); } @@ -334,7 +335,7 @@ int ObExtInfoCbRegister::set_index_data(ObObj &index_data) return ret; } -int ObExtInfoCbRegister::set_outrow_ctx_seq_no(ObObj& index_data) +int ObExtInfoCbRegister::set_outrow_ctx_seq_no(blocksstable::ObStorageDatum& index_data) { int ret = OB_SUCCESS; ObLobLocatorV2 locator; diff --git a/src/storage/lob/ob_ext_info_callback.h b/src/storage/lob/ob_ext_info_callback.h index b6c32be34..c0857c298 100644 --- a/src/storage/lob/ob_ext_info_callback.h +++ b/src/storage/lob/ob_ext_info_callback.h @@ -153,12 +153,13 @@ public: const blocksstable::ObDmlFlag dml_flag, transaction::ObTxDesc *tx_desc, transaction::ObTxSEQ &parent_seq_no, - ObObj &index_data, + blocksstable::ObStorageDatum &index_data, + ObObjType &type, ObObj &ext_info_data); private: - static ObExtInfoLogType get_type(ObObjType obj_type) + static ObExtInfoLogType get_type(ObObjType &obj_type) { ObExtInfoLogType res = OB_INVALID_EXT_INFO_LOG; switch (obj_type) @@ -174,12 +175,12 @@ private: int build_data_iter(ObObj &ext_info_data); - int set_index_data(ObObj &index_data); - int set_outrow_ctx_seq_no(ObObj& index_data); + int set_index_data(blocksstable::ObStorageDatum &index_data, ObObjType &type); + int set_outrow_ctx_seq_no(blocksstable::ObStorageDatum& index_data); int get_data(ObString &data); - int init_header(ObObj& index_data, ObObj &ext_info_data); + int init_header(ObObjType &type); public: TO_STRING_KV(K(timeout_), K(data_size_), K(seq_no_st_), K(seq_no_cnt_), K(header_writed_)); diff --git a/src/storage/lob/ob_lob_meta.cpp b/src/storage/lob/ob_lob_meta.cpp index 93382f6ba..546f256bd 100644 --- a/src/storage/lob/ob_lob_meta.cpp +++ b/src/storage/lob/ob_lob_meta.cpp @@ -972,7 +972,7 @@ int ObLobMetaManager::write(ObLobAccessParam& param, ObLobMetaInfo& in_row) return ret; } -int ObLobMetaManager::batch_insert(ObLobAccessParam& param, ObNewRowIterator &iter) +int ObLobMetaManager::batch_insert(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter) { int ret = OB_SUCCESS; if (OB_FAIL(persistent_lob_adapter_.write_lob_meta(param, iter))) { @@ -981,7 +981,7 @@ int ObLobMetaManager::batch_insert(ObLobAccessParam& param, ObNewRowIterator &it return ret; } -int ObLobMetaManager::batch_delete(ObLobAccessParam& param, ObNewRowIterator &iter) +int ObLobMetaManager::batch_delete(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter) { int ret = OB_SUCCESS; if (OB_FAIL(persistent_lob_adapter_.erase_lob_meta(param, iter))) { diff --git a/src/storage/lob/ob_lob_meta.h b/src/storage/lob/ob_lob_meta.h index 3d840d07d..707dfd844 100644 --- a/src/storage/lob/ob_lob_meta.h +++ b/src/storage/lob/ob_lob_meta.h @@ -230,8 +230,8 @@ public: ~ObLobMetaManager() {} // write one lob meta row int write(ObLobAccessParam& param, ObLobMetaInfo& in_row); - int batch_insert(ObLobAccessParam& param, ObNewRowIterator &iter); - int batch_delete(ObLobAccessParam& param, ObNewRowIterator &iter); + int batch_insert(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter); + int batch_delete(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &iter); // append int append(ObLobAccessParam& param, ObLobMetaWriteIter& iter); // return ObLobMetaWriteResult diff --git a/src/storage/lob/ob_lob_persistent_adaptor.cpp b/src/storage/lob/ob_lob_persistent_adaptor.cpp index 0fee1d240..e6b8ab4a0 100644 --- a/src/storage/lob/ob_lob_persistent_adaptor.cpp +++ b/src/storage/lob/ob_lob_persistent_adaptor.cpp @@ -418,15 +418,14 @@ int ObPersistentLobApator::erase_lob_piece_tablet(ObLobAccessParam& param, ObLob // construct insert data int64_t affected_rows = 0; - ObObj cell[ObLobPieceUtil::LOB_PIECE_COLUMN_CNT]; char serialize_buf[32] = {0}; // make insert iterator - ObNewRow new_row; + ObDatumRow new_row; + blocksstable::ObSingleDatumRowIteratorWrapper single_iter; - common::ObSingleRowIteratorWrapper single_iter; - single_iter.set_row(&new_row); - - if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, cell, new_row, &single_iter, in_row))) { + if (OB_FAIL(new_row.init(ObLobPieceUtil::LOB_PIECE_COLUMN_CNT))) { + LOG_WARN("failed to init new datum row", K(ret)); + } else if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, new_row, &single_iter, in_row))) { LOG_WARN("failed to set insert piece row.", K(ret), K(in_row)); } else if (OB_FAIL(oas->delete_rows(param.ls_id_, lob_piece_tablet.get_obj()->get_tablet_meta().tablet_id_, @@ -534,15 +533,14 @@ int ObPersistentLobApator::write_lob_piece_tablet(ObLobAccessParam& param, ObLob // construct insert data int64_t affected_rows = 0; - ObObj cell[ObLobPieceUtil::LOB_PIECE_COLUMN_CNT]; char serialize_buf[32] = {0}; // make insert iterator - ObNewRow new_row; + ObDatumRow new_row; + blocksstable::ObSingleDatumRowIteratorWrapper single_iter; - common::ObSingleRowIteratorWrapper single_iter; - single_iter.set_row(&new_row); - - if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, cell, new_row, &single_iter, in_row))) { + if (OB_FAIL(new_row.init(ObLobPieceUtil::LOB_PIECE_COLUMN_CNT))) { + LOG_WARN("failed to init new datum row", K(ret)); + } else if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, new_row, &single_iter, in_row))) { LOG_WARN("failed to set insert piece row.", K(ret), K(in_row)); } else if (OB_FAIL(oas->insert_rows(param.ls_id_, lob_piece_tablet.get_obj()->get_tablet_meta().tablet_id_, @@ -604,15 +602,14 @@ int ObPersistentLobApator::update_lob_piece_tablet(ObLobAccessParam& param, ObLo // construct insert data int64_t affected_rows = 0; - ObObj cell[ObLobPieceUtil::LOB_PIECE_COLUMN_CNT]; char serialize_buf[32] = {0}; // make insert iterator - ObNewRow new_row; + ObDatumRow new_row; + blocksstable::ObSingleDatumRowIteratorWrapper single_iter; - common::ObSingleRowIteratorWrapper single_iter; - single_iter.set_row(&new_row); - - if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, cell, new_row, &single_iter, in_row))) { + if (OB_FAIL(new_row.init(ObLobPieceUtil::LOB_PIECE_COLUMN_CNT))) { + LOG_WARN("failed to init new datum row", K(ret)); + } else if (OB_FAIL(set_lob_piece_row(serialize_buf, 32, new_row, &single_iter, in_row))) { LOG_WARN("failed to set insert piece row.", K(ret), K(in_row)); } else if (OB_FAIL(oas->update_rows(param.ls_id_, lob_piece_tablet.get_obj()->get_tablet_meta().tablet_id_, @@ -824,43 +821,32 @@ int ObPersistentLobApator::inner_get_tablet( } void ObPersistentLobApator::set_lob_meta_row( - ObObj* cell, - ObNewRow& new_row, + blocksstable::ObDatumRow& datum_row, ObLobMetaInfo& in_row) { - for (int64_t i = 0; i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) { - cell[i].reset(); - cell[i].set_nop_value(); - } - cell[ObLobMetaUtil::LOB_ID_COL_ID].set_varchar(reinterpret_cast(&in_row.lob_id_), sizeof(ObLobId)); - cell[ObLobMetaUtil::LOB_ID_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - cell[ObLobMetaUtil::SEQ_ID_COL_ID].set_varchar(in_row.seq_id_); - cell[ObLobMetaUtil::SEQ_ID_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - cell[ObLobMetaUtil::BYTE_LEN_COL_ID].set_uint32(in_row.byte_len_); - cell[ObLobMetaUtil::CHAR_LEN_COL_ID].set_uint32(in_row.char_len_); - cell[ObLobMetaUtil::PIECE_ID_COL_ID].set_uint64(in_row.piece_id_); - - cell[ObLobMetaUtil::LOB_DATA_COL_ID].set_varchar(in_row.lob_data_); - cell[ObLobMetaUtil::LOB_DATA_COL_ID].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - - new_row.assign(cell, ObLobMetaUtil::LOB_META_COLUMN_CNT); + datum_row.reuse(); + datum_row.storage_datums_[ObLobMetaUtil::LOB_ID_COL_ID].set_string(reinterpret_cast(&in_row.lob_id_), sizeof(ObLobId)); + // TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi + datum_row.storage_datums_[ObLobMetaUtil::SEQ_ID_COL_ID].set_string(in_row.seq_id_); + // TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi + datum_row.storage_datums_[ObLobMetaUtil::BYTE_LEN_COL_ID].set_uint32(in_row.byte_len_); + datum_row.storage_datums_[ObLobMetaUtil::CHAR_LEN_COL_ID].set_uint32(in_row.char_len_); + datum_row.storage_datums_[ObLobMetaUtil::PIECE_ID_COL_ID].set_uint(in_row.piece_id_); + datum_row.storage_datums_[ObLobMetaUtil::LOB_DATA_COL_ID].set_string(in_row.lob_data_); + // TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi } int ObPersistentLobApator::set_lob_piece_row( char* buf, size_t buf_len, - ObObj* cell, - ObNewRow& new_row, - common::ObSingleRowIteratorWrapper* new_row_iter, + ObDatumRow& datum_row, + blocksstable::ObSingleDatumRowIteratorWrapper* new_row_iter, ObLobPieceInfo& in_row) { int ret = OB_SUCCESS; - for (int64_t i = 0; i < ObLobPieceUtil::LOB_PIECE_COLUMN_CNT; ++i) { - cell[i].reset(); - cell[i].set_nop_value(); - } - cell[0].set_uint64(in_row.piece_id_); - cell[1].set_uint32(in_row.len_); + datum_row.reuse(); + datum_row.storage_datums_[0].set_uint(in_row.piece_id_); + datum_row.storage_datums_[1].set_uint32(in_row.len_); int64_t pos = 0; if (!in_row.macro_id_.is_valid()) { @@ -868,11 +854,9 @@ int ObPersistentLobApator::set_lob_piece_row( } else if (OB_FAIL(in_row.macro_id_.serialize(buf, buf_len, pos))) { LOG_WARN("failed to serialize macro id", K(ret), K(buf_len), K(pos)); } else { - cell[2].set_varchar(buf, pos); - cell[2].set_collation_type(common::ObCollationType::CS_TYPE_BINARY); - - new_row.assign(cell, ObLobPieceUtil::LOB_PIECE_COLUMN_CNT); - new_row_iter->set_row(&new_row); + datum_row.storage_datums_[2].set_string(buf, pos); + // TODO: if we need set collation type to be common::ObCollationType::CS_TYPE_BINARY@xuanxi + new_row_iter->set_row(&datum_row); } return ret; @@ -928,20 +912,20 @@ int ObPersistentLobApator::do_scan_lob_meta( ObTabletHandle lob_meta_tablet; ObTabletHandle lob_piece_tablet; if (OB_FAIL(get_lob_tablets(param, data_tablet, lob_meta_tablet, lob_piece_tablet))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); + LOG_WARN("failed to get tablets", K(ret), K(param)); } else { ObAccessService *oas = MTL(ObAccessService*); scan_param.tablet_id_ = param.lob_meta_tablet_id_; scan_param.schema_version_ = lob_meta_tablet.get_obj()->get_tablet_meta().max_sync_storage_schema_version_; if (OB_ISNULL(oas)) { ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret)); + LOG_WARN("get access service failed", K(ret)); } else if (OB_FAIL(build_common_scan_param(param, param.has_single_chunk(), ObLobMetaUtil::LOB_META_COLUMN_CNT, scan_param))) { - LOG_WARN("build common scan param failed.", K(ret), K(param)); + LOG_WARN("build common scan param failed", K(ret), K(param)); } else if (OB_FAIL(prepare_table_param(param, scan_param, true/*is_meta*/))) { - LOG_WARN("prepare lob meta table param failed.", K(ret), K(param)); + LOG_WARN("prepare lob meta table param failed", K(ret), K(param)); } else if (OB_FAIL(oas->table_scan(scan_param, meta_iter))) { - LOG_WARN("do table scan falied.", K(ret), K(scan_param), K(param)); + LOG_WARN("do table scan falied", K(ret), K(scan_param), K(param)); } } return ret; @@ -967,7 +951,7 @@ int ObPersistentLobApator::scan_lob_meta( } else if (OB_FAIL(param.get_rowkey_range(rowkey_objs, range))) { LOG_WARN("get_rowkey_range fail", K(ret)); } else if (OB_FAIL(scan_param.key_ranges_.push_back(range))) { - LOG_WARN("failed to push key range.", K(ret), K(scan_param), K(range)); + LOG_WARN("failed to push key range", K(ret), K(scan_param), K(range)); } else if (OB_FAIL(do_scan_lob_meta(param, scan_param, meta_iter))) { LOG_WARN("do_scan_lob_meta fail", K(ret)); } @@ -1004,28 +988,28 @@ int ObPersistentLobApator::set_dml_seq_no(ObLobAccessParam ¶m) LOG_DEBUG("dml lob meta with seq no", K(param.dml_base_param_->spec_seq_no_)); } else { ret = OB_INVALID_ARGUMENT; - LOG_WARN("failed to get seq no from param.", K(ret), K(param)); + LOG_WARN("failed to get seq no from param", K(ret), K(param)); } } else { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid seq no from param.", K(ret), K(param)); + LOG_WARN("invalid seq no from param", K(ret), K(param)); } return ret; } -int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObNewRowIterator& row_iter) +int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObDatumRowIterator& row_iter) { int ret = OB_SUCCESS; int64_t affected_rows = 0; ObAccessService *oas = MTL(ObAccessService*); if (OB_ISNULL(oas)) { ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret), KP(oas)); + LOG_WARN("get access service failed", K(ret), KP(oas)); } else if (OB_ISNULL(param.tx_desc_)) { ret = OB_ERR_NULL_VALUE; - LOG_WARN("get tx desc null.", K(ret), K(param)); + LOG_WARN("get tx desc null", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_tablet_id(param))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); + LOG_WARN("failed to get tablets", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_meta_dml(param))) { LOG_WARN("failed to prepare lob meta dml", K(ret)); } else if (OB_FAIL(oas->delete_rows( @@ -1041,19 +1025,19 @@ int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObNewRowItera return ret; } -int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObNewRowIterator& row_iter) +int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObDatumRowIterator& row_iter) { int ret = OB_SUCCESS; int64_t affected_rows = 0; ObAccessService *oas = MTL(ObAccessService*); if (OB_ISNULL(oas)) { ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret), KP(oas)); + LOG_WARN("get access service failed", K(ret), KP(oas)); } else if (OB_ISNULL(param.tx_desc_)) { ret = OB_ERR_NULL_VALUE; - LOG_WARN("get tx desc null.", K(ret), K(param)); + LOG_WARN("get tx desc null", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_tablet_id(param))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); + LOG_WARN("failed to get tablets", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_meta_dml(param))) { LOG_WARN("failed to prepare lob meta dml", K(ret)); } else if (OB_FAIL(oas->insert_rows( @@ -1069,26 +1053,26 @@ int ObPersistentLobApator::write_lob_meta(ObLobAccessParam& param, ObNewRowItera return ret; } -int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObNewRowIterator &row_iter) +int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObDatumRowIterator &row_iter) { int ret = OB_SUCCESS; int64_t affected_rows = 0; ObAccessService *oas = MTL(ObAccessService*); if (OB_ISNULL(oas)) { ret = OB_ERR_INTERVAL_INVALID; - LOG_WARN("get access service failed.", K(ret), KP(oas)); + LOG_WARN("get access service failed", K(ret), KP(oas)); } else if (OB_ISNULL(param.tx_desc_)) { ret = OB_ERR_NULL_VALUE; - LOG_WARN("get tx desc null.", K(ret), K(param)); + LOG_WARN("get tx desc null", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_tablet_id(param))) { - LOG_WARN("failed to get tablets.", K(ret), K(param)); + LOG_WARN("failed to get tablets", K(ret), K(param)); } else if (OB_FAIL(prepare_lob_meta_dml(param))) { LOG_WARN("failed to prepare lob meta dml", K(ret)); } else { ObSEArray update_column_ids; for (int i = 2; OB_SUCC(ret) && i < ObLobMetaUtil::LOB_META_COLUMN_CNT; ++i) { if (OB_FAIL(update_column_ids.push_back(OB_APP_MIN_COLUMN_ID + i))) { - LOG_WARN("push column ids failed.", K(ret), K(i)); + LOG_WARN("push column ids failed", K(ret), K(i)); } } if (OB_FAIL(ret)) { @@ -1110,11 +1094,12 @@ int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObNewRowIter int ObPersistentLobApator::write_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo& row_info) { int ret = OB_SUCCESS; - ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow new_row; + ObDatumRow new_row; ObLobPersistInsertSingleRowIter single_iter; - set_lob_meta_row(cell, new_row, row_info); - if (OB_FAIL(single_iter.init(¶m, &new_row))) { + if (OB_FAIL(new_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("failed to init datum row", K(ret)); + } else if (FALSE_IT(set_lob_meta_row(new_row, row_info))) { + } else if (OB_FAIL(single_iter.init(¶m, &new_row))) { LOG_WARN("single_iter init fail", K(ret)); } else if (OB_FAIL(write_lob_meta(param, single_iter))) { LOG_WARN("write_lob_meta fail", K(ret)); @@ -1125,11 +1110,12 @@ int ObPersistentLobApator::write_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo& row_info) { int ret = OB_SUCCESS; - ObObj cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow new_row; + ObDatumRow new_row; ObLobPersistDeleteSingleRowIter single_iter; - set_lob_meta_row(cell, new_row, row_info); - if (OB_FAIL(single_iter.init(¶m, &new_row))) { + if (OB_FAIL(new_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("failed to init datum row", K(ret)); + } else if (FALSE_IT(set_lob_meta_row(new_row, row_info))) { + } else if (OB_FAIL(single_iter.init(¶m, &new_row))) { LOG_WARN("single_iter init fail", K(ret)); } else if (OB_FAIL(erase_lob_meta(param, single_iter))) { LOG_WARN("erase_lob_meta fail", K(ret)); @@ -1140,14 +1126,16 @@ int ObPersistentLobApator::erase_lob_meta(ObLobAccessParam ¶m, ObLobMetaInfo int ObPersistentLobApator::update_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& old_row, ObLobMetaInfo& new_row) { int ret = OB_SUCCESS; - ObObj new_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow new_tbl_row; - set_lob_meta_row(new_row_cell, new_tbl_row, new_row); - ObObj old_row_cell[ObLobMetaUtil::LOB_META_COLUMN_CNT]; - ObNewRow old_tbl_row; - set_lob_meta_row(old_row_cell, old_tbl_row, old_row); + ObDatumRow new_datum_row; + ObDatumRow old_datum_row; ObLobPersistUpdateSingleRowIter upd_iter; - if (OB_FAIL(upd_iter.init(¶m, &old_tbl_row, &new_tbl_row))) { + if (OB_FAIL(new_datum_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("failed to init new datum row", K(ret)); + } else if (OB_FAIL(old_datum_row.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("failed to init old datum row", K(ret)); + } else if (FALSE_IT(set_lob_meta_row(new_datum_row, new_row))) { + } else if (FALSE_IT(set_lob_meta_row(old_datum_row, old_row))) { + } else if (OB_FAIL(upd_iter.init(¶m, &old_datum_row, &new_datum_row))) { LOG_WARN("upd_iter init fail", K(ret)); } else if (OB_FAIL(update_lob_meta(param, upd_iter))) { LOG_WARN("update_lob_meta fail", K(ret)); diff --git a/src/storage/lob/ob_lob_persistent_adaptor.h b/src/storage/lob/ob_lob_persistent_adaptor.h index fb41e92bb..1fc4d719d 100644 --- a/src/storage/lob/ob_lob_persistent_adaptor.h +++ b/src/storage/lob/ob_lob_persistent_adaptor.h @@ -17,6 +17,7 @@ #include "storage/blocksstable/ob_macro_block_id.h" #include "ob_i_lob_adaptor.h" #include "common/row/ob_row_iterator.h" +#include "storage/blocksstable/ob_datum_row_iterator.h" namespace oceanbase { @@ -25,22 +26,20 @@ namespace storage class ObStoreCtxGuard; -class ObLobUpdIterator : public ObNewRowIterator +class ObLobUpdIterator : public blocksstable::ObDatumRowIterator { public: - ObLobUpdIterator(ObNewRow *old_row, - ObNewRow *new_row) + ObLobUpdIterator(blocksstable::ObDatumRow *old_row, + blocksstable::ObDatumRow *new_row) : old_row_(old_row), new_row_(new_row), got_old_row_(false), is_iter_end_(false) {} - virtual int get_next_row(ObNewRow *&row) override; - virtual int get_next_row() override { return OB_NOT_IMPLEMENT; } - virtual void reset() override {} + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; private: - ObNewRow *old_row_; - ObNewRow *new_row_; + blocksstable::ObDatumRow *old_row_; + blocksstable::ObDatumRow *new_row_; bool got_old_row_; bool is_iter_end_; }; @@ -94,13 +93,12 @@ public: virtual int update_lob_meta(ObLobAccessParam& param, ObLobMetaInfo& old_row, ObLobMetaInfo& new_row) override; static void set_lob_meta_row( - ObObj* cell, - ObNewRow& new_row, + blocksstable::ObDatumRow& datum_row, ObLobMetaInfo& in_row); - int write_lob_meta(ObLobAccessParam ¶m, ObNewRowIterator& row_iter); - int erase_lob_meta(ObLobAccessParam ¶m, ObNewRowIterator& row_iter); - int update_lob_meta(ObLobAccessParam& param, ObNewRowIterator &row_iter); + int write_lob_meta(ObLobAccessParam ¶m, blocksstable::ObDatumRowIterator& row_iter); + int erase_lob_meta(ObLobAccessParam ¶m, blocksstable::ObDatumRowIterator& row_iter); + int update_lob_meta(ObLobAccessParam& param, blocksstable::ObDatumRowIterator &row_iter); int build_common_scan_param( const ObLobAccessParam ¶m, @@ -159,9 +157,8 @@ private: int set_lob_piece_row( char* buf, size_t buf_len, - ObObj* cell, - ObNewRow& new_row, - common::ObSingleRowIteratorWrapper* new_row_iter, + blocksstable::ObDatumRow& datum_row, + blocksstable::ObSingleDatumRowIteratorWrapper* new_row_iter, ObLobPieceInfo& in_row); int init_table_param(); diff --git a/src/storage/lob/ob_lob_persistent_iterator.cpp b/src/storage/lob/ob_lob_persistent_iterator.cpp index 03fcf147d..bdfc804e5 100644 --- a/src/storage/lob/ob_lob_persistent_iterator.cpp +++ b/src/storage/lob/ob_lob_persistent_iterator.cpp @@ -65,7 +65,7 @@ int ObLobPersistWriteIter::dec_lob_size(ObLobMetaInfo &info) return ret; } -int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, ObNewRow *old_row, ObNewRow *new_row) +int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, blocksstable::ObDatumRow *old_row, blocksstable::ObDatumRow *new_row) { int ret = OB_SUCCESS; if (OB_ISNULL(param) || OB_ISNULL(old_row) || OB_ISNULL(new_row)) { @@ -79,7 +79,7 @@ int ObLobPersistUpdateSingleRowIter::init(ObLobAccessParam *param, ObNewRow *old return ret; } -int ObLobPersistUpdateSingleRowIter::get_next_row(ObNewRow *&row) +int ObLobPersistUpdateSingleRowIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(old_row_) || OB_ISNULL(new_row_)) { @@ -100,7 +100,7 @@ int ObLobPersistUpdateSingleRowIter::get_next_row(ObNewRow *&row) return ret; } -int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row) +int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, blocksstable::ObDatumRow *row) { int ret = OB_SUCCESS; if (OB_ISNULL(param) || OB_ISNULL(row)) { @@ -113,7 +113,7 @@ int ObLobPersistInsertSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row return ret; } -int ObLobPersistInsertSingleRowIter::get_next_row(ObNewRow *&row) +int ObLobPersistInsertSingleRowIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(param_) || OB_ISNULL(row_)) { @@ -130,7 +130,7 @@ int ObLobPersistInsertSingleRowIter::get_next_row(ObNewRow *&row) return ret; } -int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row) +int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, blocksstable::ObDatumRow *row) { int ret = OB_SUCCESS; if (OB_ISNULL(param) || OB_ISNULL(row)) { @@ -143,7 +143,7 @@ int ObLobPersistDeleteSingleRowIter::init(ObLobAccessParam *param, ObNewRow *row return ret; } -int ObLobPersistDeleteSingleRowIter::get_next_row(ObNewRow *&row) +int ObLobPersistDeleteSingleRowIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_ISNULL(param_) || OB_ISNULL(row_)) { @@ -166,6 +166,8 @@ int ObLobPersistInsertIter::init(ObLobAccessParam *param, ObLobMetaWriteIter *me if (OB_ISNULL(param) || OB_ISNULL(meta_iter)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("param or meta_iter is null", K(ret), KP(param), KP(meta_iter)); + } else if (OB_FAIL(new_row_.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("init new datum row failed", K(ret)); } else { param_ = param; meta_iter_ = meta_iter; @@ -173,7 +175,7 @@ int ObLobPersistInsertIter::init(ObLobAccessParam *param, ObLobMetaWriteIter *me return ret; } -int ObLobPersistInsertIter::get_next_row(ObNewRow *&row) +int ObLobPersistInsertIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_FAIL(meta_iter_->get_next_row(result_))) { @@ -188,7 +190,7 @@ int ObLobPersistInsertIter::get_next_row(ObNewRow *&row) } else if (OB_FAIL(inc_lob_size(result_.info_))) { LOG_WARN("inc_lob_size fail", K(ret)); } else { - ObPersistentLobApator::set_lob_meta_row(row_cell_, new_row_, result_.info_); + ObPersistentLobApator::set_lob_meta_row(new_row_, result_.info_); row = &new_row_; } return ret; @@ -200,6 +202,8 @@ int ObLobPersistDeleteIter::init(ObLobAccessParam *param, ObLobMetaScanIter *met if (OB_ISNULL(param) || OB_ISNULL(meta_iter)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("param or meta_iter is null", K(ret), KP(param), KP(meta_iter)); + } else if (OB_FAIL(new_row_.init(ObLobMetaUtil::LOB_META_COLUMN_CNT))) { + LOG_WARN("init new datum row failed", K(ret)); } else { param_ = param; meta_iter_ = meta_iter; @@ -207,7 +211,7 @@ int ObLobPersistDeleteIter::init(ObLobAccessParam *param, ObLobMetaScanIter *met return ret; } -int ObLobPersistDeleteIter::get_next_row(ObNewRow *&row) +int ObLobPersistDeleteIter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_FAIL(meta_iter_->get_next_row(result_))) { @@ -221,7 +225,7 @@ int ObLobPersistDeleteIter::get_next_row(ObNewRow *&row) } else if (OB_FAIL(dec_lob_size(result_.info_))) { LOG_WARN("dec_lob_size fail", K(ret)); } else { - ObPersistentLobApator::set_lob_meta_row(row_cell_, new_row_, result_.info_); + ObPersistentLobApator::set_lob_meta_row(new_row_, result_.info_); row = &new_row_; } return ret; diff --git a/src/storage/lob/ob_lob_persistent_iterator.h b/src/storage/lob/ob_lob_persistent_iterator.h index d3be04c16..19598dfdf 100644 --- a/src/storage/lob/ob_lob_persistent_iterator.h +++ b/src/storage/lob/ob_lob_persistent_iterator.h @@ -12,19 +12,22 @@ #ifndef OCEABASE_STORAGE_OB_LOB_PERSISTENT_ITERATOR_ #define OCEABASE_STORAGE_OB_LOB_PERSISTENT_ITERATOR_ +#include "storage/blocksstable/ob_datum_row_iterator.h" +#include "storage/lob/ob_lob_util.h" +#include "storage/lob/ob_lob_meta.h" namespace oceanbase { namespace storage { -class ObLobPersistWriteIter : public ObNewRowIterator +class ObLobPersistWriteIter : public blocksstable::ObDatumRowIterator { public: ObLobPersistWriteIter(): param_(nullptr) {} virtual ~ObLobPersistWriteIter() {} - virtual int get_next_row() override { return OB_NOT_IMPLEMENT; } + virtual int get_next_row(blocksstable::ObDatumRow *&row) override { return OB_NOT_IMPLEMENT; } protected: int update_seq_no(); @@ -44,10 +47,10 @@ public: row_(nullptr), iter_end_(false) {} - int init(ObLobAccessParam *param, ObNewRow *row); + int init(ObLobAccessParam *param, blocksstable::ObDatumRow *row); virtual ~ObLobPersistInsertSingleRowIter() {} - virtual int get_next_row(ObNewRow *&row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); virtual void reset() { iter_end_ = false; } private: @@ -55,7 +58,7 @@ private: DISALLOW_COPY_AND_ASSIGN(ObLobPersistInsertSingleRowIter); private: // data members - ObNewRow *row_; + blocksstable::ObDatumRow *row_; bool iter_end_; }; @@ -67,10 +70,10 @@ public: row_(nullptr), iter_end_(false) {} - int init(ObLobAccessParam *param, ObNewRow *row); + int init(ObLobAccessParam *param, blocksstable::ObDatumRow *row); virtual ~ObLobPersistDeleteSingleRowIter() {} - virtual int get_next_row(ObNewRow *&row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); virtual void reset() { iter_end_ = false; } private: @@ -78,7 +81,7 @@ private: DISALLOW_COPY_AND_ASSIGN(ObLobPersistDeleteSingleRowIter); private: // data members - ObNewRow *row_; + blocksstable::ObDatumRow *row_; bool iter_end_; }; @@ -94,9 +97,9 @@ public: virtual ~ObLobPersistUpdateSingleRowIter() {} - int init(ObLobAccessParam *param, ObNewRow *old_row, ObNewRow *new_row); + int init(ObLobAccessParam *param, blocksstable::ObDatumRow *old_row, blocksstable::ObDatumRow *new_row); - virtual int get_next_row(ObNewRow *&row) override; + virtual int get_next_row(blocksstable::ObDatumRow *&row) override; virtual void reset() override {} private: @@ -104,8 +107,8 @@ private: DISALLOW_COPY_AND_ASSIGN(ObLobPersistUpdateSingleRowIter); private: - ObNewRow *old_row_; - ObNewRow *new_row_; + blocksstable::ObDatumRow *old_row_; + blocksstable::ObDatumRow *new_row_; bool got_old_row_; bool is_iter_end_; }; @@ -114,10 +117,10 @@ private: class ObLobPersistInsertIter: public ObLobPersistWriteIter { public: - ObLobPersistInsertIter() : meta_iter_(nullptr), new_row_(), row_cell_(), result_() {} + ObLobPersistInsertIter() : meta_iter_(nullptr), new_row_(), result_() {} int init(ObLobAccessParam *param, ObLobMetaWriteIter *meta_iter); virtual ~ObLobPersistInsertIter() {} - virtual int get_next_row(ObNewRow *&row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); virtual void reset() { new_row_.reset(); } private: @@ -126,8 +129,7 @@ private: private: // data members ObLobMetaWriteIter *meta_iter_; - ObNewRow new_row_; - ObObj row_cell_[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + blocksstable::ObDatumRow new_row_; ObLobMetaWriteResult result_; }; @@ -135,10 +137,10 @@ private: class ObLobPersistDeleteIter: public ObLobPersistWriteIter { public: - ObLobPersistDeleteIter() : meta_iter_(nullptr), new_row_(), row_cell_(), result_() {} + ObLobPersistDeleteIter() : meta_iter_(nullptr), new_row_(), result_() {} int init(ObLobAccessParam *param, ObLobMetaScanIter *meta_iter); virtual ~ObLobPersistDeleteIter() {} - virtual int get_next_row(ObNewRow *&row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); virtual void reset() { new_row_.reset(); } @@ -148,8 +150,7 @@ private: private: // data members ObLobMetaScanIter *meta_iter_; - ObNewRow new_row_; - ObObj row_cell_[ObLobMetaUtil::LOB_META_COLUMN_CNT]; + blocksstable::ObDatumRow new_row_; ObLobMetaScanResult result_; }; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 2612b1c53..46f9083b8 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -29,8 +29,11 @@ #include "share/schema/ob_table_dml_param.h" #include "share/schema/ob_tenant_schema_service.h" #include "share/ob_ddl_common.h" +#include "sql/das/ob_das_update_op.h" #include "storage/blocksstable/index_block/ob_index_block_builder.h" #include "storage/blocksstable/ob_sstable_meta.h" +#include "storage/blocksstable/ob_datum_row_store.h" +#include "storage/blocksstable/ob_datum_row_utils.h" #include "storage/ob_dml_running_ctx.h" #include "storage/ob_partition_range_spliter.h" #include "storage/ob_query_iterator_factory.h" @@ -2609,14 +2612,14 @@ int ObLSTabletService::insert_rows( ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; + NG_TRACE(S_insert_rows_begin); int64_t afct_num = 0; int64_t dup_num = 0; - if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); @@ -2647,20 +2650,16 @@ int ObLSTabletService::insert_rows( dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), ObDmlFlag::DF_INSERT); - ObIAllocator &work_allocator = run_ctx.allocator_; - void *ptr = nullptr; - ObStoreRow *tbl_rows = nullptr; int64_t row_count = 0; - int64_t row_buf_cnt = 0; - ObNewRow *rows = nullptr; + ObDatumRow *rows = nullptr; if (OB_FAIL(prepare_dml_running_ctx(&column_ids, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else { ObTabletHandle tmp_handle; SMART_VAR(ObRowsInfo, rows_info) { const ObRelativeTable &data_table = run_ctx.relative_table_; + const ObColDescIArray &col_descs = *(run_ctx.col_descs_); while (OB_SUCC(ret) && OB_SUCC(get_next_rows(row_iter, rows, row_count))) { - ObStoreRow reserved_row; // need to be called just after get_next_row to ensure that previous row's LOB memoroy is valid if get_next_row accesses it dml_param.lob_allocator_.reuse(); // Let ObStorageTableGuard refresh retired memtable, should not hold origin tablet handle @@ -2668,7 +2667,7 @@ int ObLSTabletService::insert_rows( if (tmp_handle.get_obj() != run_ctx.relative_table_.tablet_iter_.get_tablet_handle().get_obj()) { tmp_handle = run_ctx.relative_table_.tablet_iter_.get_tablet_handle(); rows_info.reset(); - if (OB_FAIL(rows_info.init(data_table, ctx, tmp_handle.get_obj()->get_rowkey_read_info()))) { + if (OB_FAIL(rows_info.init(col_descs, data_table, ctx, tmp_handle.get_obj()->get_rowkey_read_info()))) { LOG_WARN("Failed to init rows info", K(ret), K(data_table)); } } @@ -2676,34 +2675,15 @@ int ObLSTabletService::insert_rows( } else if (row_count <= 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("row_count should be greater than 0", K(ret)); - } else if (1 == row_count) { - tbl_rows = &reserved_row; - tbl_rows[0].flag_.set_flag(ObDmlFlag::DF_INSERT); - } else if (row_buf_cnt < row_count) { - if (nullptr != ptr) { - work_allocator.free(ptr); - ptr = nullptr; + } else { + for (int64_t i = 0; i < row_count; i++) { + rows[i].row_flag_.set_flag(ObDmlFlag::DF_INSERT); } - if (OB_ISNULL(ptr = work_allocator.alloc(row_count * sizeof(ObStoreRow)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("fail to allocate memory", K(ret), K(row_count)); - } else { - row_buf_cnt = row_count; - tbl_rows = new (ptr) ObStoreRow[row_count]; - for (int64_t i = 0; i < row_count; i++) { - tbl_rows[i].flag_.set_flag(ObDmlFlag::DF_INSERT); - } - } - } else if (tbl_rows != static_cast(ptr)) { - tbl_rows = static_cast(ptr); } if (OB_FAIL(ret)) { - } else if (OB_ISNULL(tbl_rows)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, tbl_rows is NULL", K(ret), KP(tbl_rows)); } else if (OB_FAIL(insert_rows_to_tablet(tablet_handle, run_ctx, rows, - row_count, rows_info, tbl_rows, afct_num, dup_num))) { + row_count, rows_info, afct_num, dup_num))) { LOG_WARN("insert to each tablets fail", K(ret)); } } @@ -2715,12 +2695,9 @@ int ObLSTabletService::insert_rows( if (OB_ITER_END == ret) { ret = OB_SUCCESS; } - if (nullptr != ptr) { - work_allocator.free(ptr); - } } if (OB_SUCC(ret)) { - LOG_DEBUG("succeeded to insert rows", K(ret)); + LOG_DEBUG("succeeded to insert rows", K(ret), K(afct_num)); affected_rows = afct_num; EVENT_ADD(STORAGE_INSERT_ROW_COUNT, afct_num); } @@ -2735,7 +2712,7 @@ int ObLSTabletService::direct_insert_rows( const int64_t ddl_task_id, const ObTabletID &tablet_id, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -2745,7 +2722,7 @@ int ObLSTabletService::direct_insert_rows( LOG_WARN("fail to get table ctx", KR(ret), K(key)); } else { int64_t row_count = 0; - ObNewRow *rows = nullptr; + ObDatumRow *rows = nullptr; table::ObTableLoadTransId trans_id; trans_id.segment_id_ = px_task_id; trans_id.trans_gid_ = 1; @@ -2765,7 +2742,7 @@ int ObLSTabletService::direct_insert_rows( LOG_WARN("row_count should be greater than 0", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && (i < row_count); ++i) { - const ObNewRow &row = rows[i]; + const ObDatumRow &row = rows[i]; if (OB_FAIL(writer.write(row))) { LOG_WARN("fail to write", KR(ret), K(i), K(row)); } else { @@ -2786,7 +2763,34 @@ int ObLSTabletService::direct_insert_rows( return ret; } -int ObLSTabletService::mock_duplicated_rows_(common::ObNewRowIterator *&duplicated_rows) +int ObLSTabletService::get_storage_row( + const ObDatumRow &sql_row, + const ObIArray &column_ids, + ObSingleRowGetter &row_getter, + ObDMLRunningCtx &run_ctx, + ObDatumRow *&out_row, + bool use_fuse_row_cache) +{ + int ret = OB_SUCCESS; + ObDatumRowkey datum_rowkey; + ObDatumRowkeyHelper rowkey_helper; + ObRelativeTable &data_table = run_ctx.relative_table_; + const ObColDescIArray &column_descs = *run_ctx.col_descs_; + if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(sql_row, data_table.get_rowkey_column_num(), column_descs, datum_rowkey))) { + LOG_WARN("failed to prepare rowkey", K(ret), K(sql_row), K(column_descs)); + } else if (OB_FAIL(init_single_row_getter(row_getter, run_ctx, column_ids, data_table, true))) { + LOG_WARN("failed to init single row getter", K(ret), K(column_ids)); + } else if (OB_FAIL(row_getter.open(datum_rowkey, use_fuse_row_cache))) { + LOG_WARN("failed to open storage row", K(ret), K(datum_rowkey)); + } else if (OB_FAIL(row_getter.get_next_row(out_row))) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to get single storage row", K(ret), K(sql_row)); + } + } + return ret; +} + +int ObLSTabletService::mock_duplicated_rows_(blocksstable::ObDatumRowIterator *&duplicated_rows) { int ret = OB_SUCCESS; ObValueRowIterator *dup_iter = NULL; @@ -2812,10 +2816,10 @@ int ObLSTabletService::insert_row( const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray &duplicated_column_ids, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObInsertFlag flag, int64_t &affected_rows, - common::ObNewRowIterator *&duplicated_rows) + blocksstable::ObDatumRowIterator *&duplicated_rows) { int ret = OB_SUCCESS; const ObTabletID &data_tablet_id = ctx.tablet_id_; @@ -2840,18 +2844,17 @@ int ObLSTabletService::insert_row( ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), ObDmlFlag::DF_INSERT); ObIAllocator &work_allocator = run_ctx.allocator_; - ObStoreRow &tbl_row = run_ctx.tbl_row_; + duplicated_rows = nullptr; const ObRelativeTable &data_table = run_ctx.relative_table_; if (OB_FAIL(prepare_dml_running_ctx(&column_ids, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else { - tbl_row.flag_.set_flag(ObDmlFlag::DF_INSERT); - tbl_row.row_val_ = row; + row.row_flag_.set_flag(ObDmlFlag::DF_INSERT); const bool check_exist = !data_table.is_storage_index_table() || data_table.is_unique_index(); if (OB_FAIL(insert_row_to_tablet(check_exist, tablet_handle, run_ctx, - tbl_row))) { + row))) { if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { int tmp_ret = OB_SUCCESS; // For primary key conflicts caused by concurrent insertions within @@ -2863,7 +2866,7 @@ int ObLSTabletService::insert_row( run_ctx, flag, duplicated_column_ids, - tbl_row.row_val_, + row, duplicated_rows))) { LOG_WARN("failed to get conflict row(s)", K(ret), K(duplicated_column_ids), K(row)); ret = tmp_ret; @@ -2906,7 +2909,7 @@ int ObLSTabletService::update_rows( const ObDMLBaseParam &dml_param, const ObIArray &column_ids, const ObIArray< uint64_t> &updated_column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -2934,14 +2937,13 @@ int ObLSTabletService::update_rows( ObDMLRunningCtx run_ctx(ctx, dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), - ObDmlFlag::DF_UPDATE); - ObIAllocator &work_allocator = run_ctx.allocator_; - ObStoreRow old_tbl_row; - void *old_row_cells = nullptr; - ObStoreRow &new_tbl_row = run_ctx.tbl_row_; + ObDmlFlag::DF_UPDATE, + true /* is_need_row_datum_utils */); + ObDatumRow old_datum_row; + ObDatumRow &new_datum_row = run_ctx.datum_row_; bool rowkey_change = false; UpdateIndexArray update_idx; - ObRowStore row_store; + ObDatumRowStore row_store; bool delay_new = false; bool lob_update = false; const ObRelativeTable &relative_table = run_ctx.relative_table_; @@ -2961,16 +2963,10 @@ int ObLSTabletService::update_rows( LOG_WARN("failed to check rowkey changes", K(ret)); } else { timeguard.click("Check"); - const int64_t num = relative_table.get_column_count(); - if (OB_ISNULL(old_row_cells = work_allocator.alloc( - static_cast(sizeof(common::ObObj) * num)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("failed to malloc temp row cells", K(ret)); + if (OB_FAIL(old_datum_row.init(run_ctx.allocator_, relative_table.get_column_count()))) { + LOG_ERROR("failed to init old datum row", K(ret), K(relative_table.get_column_count())); } else { timeguard.click("AllocOld"); - old_tbl_row.row_val_.cells_ = new (old_row_cells) ObObj[num](); - old_tbl_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); - new_tbl_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); } lob_update = is_lob_update(run_ctx, update_idx); } @@ -2978,9 +2974,9 @@ int ObLSTabletService::update_rows( int64_t cur_time = 0; ObTabletHandle tmp_handle; while (OB_SUCC(ret) - && OB_SUCC(get_next_row_from_iter(row_iter, old_tbl_row, true)) - && OB_SUCC(get_next_row_from_iter(row_iter, new_tbl_row, false))) { - LOG_DEBUG("get_dml_update_row", KP(row_iter), K(old_tbl_row), K(new_tbl_row)); + && OB_SUCC(get_next_row_from_iter(row_iter, old_datum_row, true)) + && OB_SUCC(get_next_row_from_iter(row_iter, new_datum_row, false))) { + LOG_DEBUG("get_dml_update_row", KP(row_iter), K(old_datum_row), K(new_datum_row)); // need to be called just after get_next_row to ensure that previous row's LOB memoroy is valid if get_next_row accesses it dml_param.lob_allocator_.reuse(); // Let ObStorageTableGuard refresh retired memtable, should not hold origin tablet handle @@ -3001,11 +2997,11 @@ int ObLSTabletService::update_rows( update_idx, delay_new, lob_update, - old_tbl_row, - new_tbl_row, + old_datum_row, + new_datum_row, &row_store, duplicate))) { - LOG_WARN("failed to update row to tablets", K(ret), K(old_tbl_row), K(new_tbl_row)); + LOG_WARN("failed to update row to tablets", K(ret), K(old_datum_row), K(new_datum_row)); } else if (duplicate) { dup_num++; } else { @@ -3018,20 +3014,18 @@ int ObLSTabletService::update_rows( } if (OB_SUCC(ret) && row_store.get_row_count() > 0) { - void *ptr1 = nullptr; - const int64_t num = relative_table.get_column_count(); - if (OB_ISNULL(ptr1 = work_allocator.alloc(sizeof(common::ObObj) * num))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("failed to malloc temp row cells", K(ret)); + new_datum_row.storage_datums_ = nullptr; + if (OB_FAIL(new_datum_row.init(run_ctx.allocator_, relative_table.get_column_count()))) { + LOG_ERROR("failed to init new datum row", K(ret), K(relative_table.get_column_count())); } else { timeguard.click("AllocNew"); - new_tbl_row.row_val_.cells_ = new(ptr1) ObObj[num](); - ObRowStore::Iterator row_iter2 = row_store.begin(); + ObDatumRowStore::Iterator row_iter2 = row_store.begin(); ObTabletHandle tmp_handle; - // when total_quantity_log is true, we should iterate new_tbl_row and old_tbl_row, and + const share::schema::ObTableDMLParam *table_param = dml_param.table_param_; + // when total_quantity_log is true, we should iterate new_datum_row and old_datum_row, and // dispose these two rows together, otherwise, when total_quantity_log is false, - // row_iter2 doesn't contain old rows, and old_tbl_row is a dummy param in process_new_row - while (OB_SUCC(ret) && OB_SUCC(row_iter2.get_next_row(new_tbl_row.row_val_))) { + // row_iter2 doesn't contain old rows, and old_datum_row is a dummy param in process_new_row + while (OB_SUCC(ret) && OB_SUCC(row_iter2.get_next_row(new_datum_row))) { // Let ObStorageTableGuard refresh retired memtable, should not hold origin tablet handle // outside the while loop. if (tmp_handle.get_obj() != run_ctx.relative_table_.tablet_iter_.get_tablet_handle().get_obj()) { @@ -3039,38 +3033,37 @@ int ObLSTabletService::update_rows( } int64_t data_tbl_rowkey_len = relative_table.get_rowkey_column_num(); bool tbl_rowkey_change = false; - if (OB_FAIL(row_iter2.get_next_row(old_tbl_row.row_val_))) { - LOG_WARN("fail to get row from row stores", K(ret)); + if (OB_FAIL(row_iter2.get_next_row(old_datum_row))) { + LOG_WARN("fail to get old row from row stores", K(ret)); } else if (FALSE_IT(timeguard.click("GetNext"))) { - } else if (rowkey_change && OB_FAIL(check_rowkey_value_change(old_tbl_row.row_val_, - new_tbl_row.row_val_, + } else if (OB_ISNULL(table_param)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid table param", K(ret)); + } else if (rowkey_change && OB_FAIL(check_rowkey_value_change(old_datum_row, + new_datum_row, data_tbl_rowkey_len, + table_param->get_data_table().get_read_info().get_datum_utils(), tbl_rowkey_change))) { - LOG_WARN("check data table rowkey change failed", K(ret), K(old_tbl_row), - K(new_tbl_row), K(data_tbl_rowkey_len)); + LOG_WARN("check data table rowkey change failed", K(ret), K(old_datum_row), + K(new_datum_row), K(data_tbl_rowkey_len)); } else if (OB_FAIL(process_new_row(tmp_handle, run_ctx, update_idx, - old_tbl_row, - new_tbl_row, + old_datum_row, + new_datum_row, tbl_rowkey_change))) { - LOG_WARN("fail to process new row", K(ret), K(old_tbl_row), K(new_tbl_row)); + LOG_WARN("fail to process new row", K(ret), K(old_datum_row), K(new_datum_row)); } timeguard.click("Process"); } - work_allocator.free(ptr1); - ptr1 = nullptr; - new_tbl_row.row_val_.cells_ = nullptr; + new_datum_row.reset(); } } if (OB_ITER_END == ret) { ret = OB_SUCCESS; } - if (nullptr != old_row_cells) { - work_allocator.free(old_row_cells); - } - old_tbl_row.row_val_.cells_ = nullptr; + old_datum_row.reset(); if (OB_SUCC(ret)) { affected_rows = afct_num; EVENT_ADD(STORAGE_UPDATE_ROW_COUNT, afct_num); @@ -3089,7 +3082,7 @@ int ObLSTabletService::put_rows( ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -3112,16 +3105,12 @@ int ObLSTabletService::put_rows( dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), ObDmlFlag::DF_UPDATE); - ObIAllocator &work_allocator = run_ctx.allocator_; - ObNewRow *row = nullptr; - ObStoreRow &tbl_row = run_ctx.tbl_row_; - const ObRelativeTable &data_table = run_ctx.relative_table_; + ObDatumRow *row = nullptr; if (OB_FAIL(prepare_dml_running_ctx(&column_ids, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else { tablet_handle.reset(); - tbl_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); } int64_t cur_time = 0; @@ -3135,14 +3124,14 @@ int ObLSTabletService::put_rows( tmp_handle = run_ctx.relative_table_.tablet_iter_.get_tablet_handle(); } cur_time = ObClockGenerator::getClock(); - tbl_row.row_val_ = *row; + row->row_flag_ = ObDmlFlag::DF_UPDATE; if (cur_time > dml_param.timeout_) { ret = OB_TIMEOUT; LOG_WARN("query timeout", K(ret), K(cur_time), K(dml_param)); } else if (OB_FAIL(insert_row_to_tablet(false/*check_exist*/, tmp_handle, run_ctx, - tbl_row))) { + *row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("failed to write row", K(ret)); } @@ -3167,7 +3156,7 @@ int ObLSTabletService::delete_rows( ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; @@ -3189,8 +3178,9 @@ int ObLSTabletService::delete_rows( ObDMLRunningCtx run_ctx(ctx, dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), - ObDmlFlag::DF_DELETE); - ObNewRow *row = nullptr; + ObDmlFlag::DF_DELETE, + true /* is_need_row_datum_utils */); + ObDatumRow *row = nullptr; if (OB_FAIL(prepare_dml_running_ctx(&column_ids, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else { @@ -3239,7 +3229,7 @@ int ObLSTabletService::lock_rows( const ObDMLBaseParam &dml_param, const ObLockFlag lock_flag, const bool is_sfu, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { UNUSEDx(lock_flag, is_sfu); @@ -3266,7 +3256,7 @@ int ObLSTabletService::lock_rows( dml_param, ctx.mvcc_acc_ctx_.mem_ctx_->get_query_allocator(), ObDmlFlag::DF_LOCK); - ObNewRow *row = nullptr; + ObDatumRow *row = nullptr; if (OB_FAIL(prepare_dml_running_ctx(nullptr, nullptr, tablet_handle, run_ctx))) { LOG_WARN("failed to prepare dml running ctx", K(ret)); } else if (FALSE_IT(tablet_handle.reset())) { @@ -3278,6 +3268,7 @@ int ObLSTabletService::lock_rows( } else { timeguard.click("GetIds"); run_ctx.column_ids_ = &column_ids; + run_ctx.col_descs_ = &col_desc; ObTabletHandle tmp_handle; while (OB_SUCCESS == ret && OB_SUCC(row_iter->get_next_row(row))) { // Let ObStorageTableGuard refresh retired memtable, should not hold origin tablet handle @@ -3286,19 +3277,20 @@ int ObLSTabletService::lock_rows( tmp_handle = run_ctx.relative_table_.tablet_iter_.get_tablet_handle(); } ObRelativeTable &relative_table = run_ctx.relative_table_; + const ObStorageDatumUtils &datum_utils = dml_param.table_param_->get_data_table().get_read_info().get_datum_utils(); bool is_exists = true; if (ObTimeUtility::current_time() > dml_param.timeout_) { ret = OB_TIMEOUT; int64_t cur_time = ObClockGenerator::getClock(); LOG_WARN("query timeout", K(cur_time), K(dml_param), K(ret)); } else if (GCONF.enable_defensive_check() - && OB_FAIL(check_old_row_legitimacy(tmp_handle, run_ctx, *row))) { + && OB_FAIL(check_old_row_legitimacy(datum_utils.get_cmp_funcs(), tmp_handle, run_ctx, *row))) { LOG_WARN("check row legitimacy failed", K(ret), KPC(row)); } else if (GCONF.enable_defensive_check() - && OB_FAIL(check_new_row_nullable_value(col_desc, relative_table, *row))) { + && OB_FAIL(check_datum_row_nullable_value(col_desc, relative_table, *row))) { LOG_WARN("check lock row nullable failed", K(ret)); } else if (FALSE_IT(timeguard.click("Check"))) { - } else if (OB_FAIL(tmp_handle.get_obj()->lock_row(run_ctx.relative_table_, ctx, *row))) { + } else if (OB_FAIL(tmp_handle.get_obj()->lock_row(run_ctx.relative_table_, ctx, col_desc, *row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("failed to lock row", K(*row), K(ret)); } @@ -3321,7 +3313,7 @@ int ObLSTabletService::lock_row( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, - const ObNewRow &row, + blocksstable::ObDatumRow &row, const ObLockFlag lock_flag, const bool is_sfu) { @@ -3352,7 +3344,7 @@ int ObLSTabletService::lock_row( ret = OB_TIMEOUT; int64_t cur_time = ObClockGenerator::getClock(); LOG_WARN("query timeout", K(cur_time), K(dml_param), K(ret)); - } else if (OB_FAIL(tablet_handle.get_obj()->lock_row(run_ctx.relative_table_, ctx, row))) { + } else if (OB_FAIL(tablet_handle.get_obj()->lock_row(run_ctx.relative_table_, ctx, col_desc, row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("failed to lock row", K(row), K(ret)); } @@ -3655,80 +3647,11 @@ int ObLSTabletService::safe_create_cas_empty_shell( return ret; } -int ObLSTabletService::need_check_old_row_legitimacy(ObDMLRunningCtx &run_ctx, - bool &need_check, - bool &is_udf) -{ - int ret = OB_SUCCESS; - ObRelativeTable &data_table = run_ctx.relative_table_; - need_check = false; - const ObDMLBaseParam &dml_param = run_ctx.dml_param_; - is_udf = false; - // TODO(jingxing): setting this to true - if (OB_FAIL(data_table.has_udf_column(need_check))) { - LOG_WARN("check has udf column failed", K(ret)); - } else if (need_check) { - is_udf = true; - ObTableStoreIterator &table_iter = *data_table.tablet_iter_.table_iter(); - while (OB_SUCC(ret) && !need_check) { - ObITable *table_ptr = nullptr; - if (OB_FAIL(table_iter.get_next(table_ptr))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next table failed", K(ret)); - } - } else if (OB_ISNULL(table_ptr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpected, table ptr must not be nullptr", K(ret)); - } else { - need_check = table_ptr->is_major_sstable(); - } - } - } else if (dml_param.is_batch_stmt_ && !data_table.is_index_table()) { - //batch stmt execution dependency defensive check to check - //if the same row was modified multiple times - need_check = true; - ret = OB_E(EventTable::EN_INS_MULTI_VALUES_BATCH_OPT) OB_SUCCESS; - // no need to check old row, just for bmsql performance optimization - // TODO yuchen.ywc - if (OB_SUCCESS != ret) { - LOG_INFO("error sim when current statement is batch update", K(ret), K(is_udf)); - need_check = false; - ret = OB_SUCCESS; - } - } else if (GCONF.enable_defensive_check()) { - need_check = true; - if (data_table.is_index_table() && !data_table.can_read_index()) { - //index can not be read during building index, so does not check old index row - need_check = false; - } - if (ObDmlFlag::DF_LOCK == run_ctx.dml_flag_) { - need_check = false; - } - } - return ret; -} - -int ObLSTabletService::construct_table_rows( - const ObNewRow *rows, - ObStoreRow *tbl_rows, - int64_t row_count) -{ - int ret = OB_SUCCESS; - if (row_count <= 0) { - ret = OB_ERR_WRONG_VALUE_COUNT_ON_ROW; - LOG_WARN("row count should be bigger than 0", K(row_count), K(ret)); - } else { - for (int64_t i = 0; i < row_count; i++) { - tbl_rows[i].row_val_ = rows[i]; - } - } - return ret; -} - int ObLSTabletService::check_old_row_legitimacy( + const ObStoreCmpFuncs &cmp_funcs, ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - const common::ObNewRow &old_row) + blocksstable::ObDatumRow &old_row) { int ret = OB_SUCCESS; // usage: @@ -3739,56 +3662,41 @@ int ObLSTabletService::check_old_row_legitimacy( ret = OB_ERR_DEFENSIVE_CHECK; } ObRelativeTable &data_table = run_ctx.relative_table_; - ObStoreRowkey rowkey; - bool need_check = false; - bool is_udf = false; - rowkey.assign(old_row.cells_, data_table.get_rowkey_column_num()); if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(rowkey.get_obj_cnt() > old_row.count_) || OB_ISNULL(run_ctx.column_ids_)) { + } else if (OB_UNLIKELY(data_table.get_rowkey_column_num() > old_row.count_) || OB_ISNULL(run_ctx.column_ids_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("old row is invalid", K(ret), K(old_row), K(rowkey.get_obj_cnt()), KP(run_ctx.column_ids_)); - } else if (OB_FAIL(need_check_old_row_legitimacy(run_ctx, need_check, is_udf))) { - LOG_WARN("identify need check old row legitimacy", K(ret)); - } else if (need_check) { + LOG_WARN("old row is invalid", K(ret), K(old_row), K(data_table.get_rowkey_column_num()), KP(run_ctx.column_ids_)); + } else if (run_ctx.is_need_check_old_row_) { //the vertical partition is no longer maintained, //and the defense check skips the vertical partition function - const ObDMLBaseParam &dml_param = run_ctx.dml_param_; ObArenaAllocator scan_allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_TABLE_SCAN_ITER)); - ObIAllocator *allocator = &scan_allocator; - ObSingleRowGetter old_row_getter(*allocator, *data_tablet_handle.get_obj()); - ObNewRow *storage_old_row = nullptr; - //check old row whether different with SQL.old_row, - ObDatumRowkey datum_rowkey; - ObDatumRowkeyHelper rowkey_helper; + ObSingleRowGetter storage_row_getter(scan_allocator, *data_tablet_handle.get_obj()); + ObDatumRow *storage_old_row = nullptr; const ObIArray &column_ids = *run_ctx.column_ids_; + const ObColDescIArray &column_descs = *run_ctx.col_descs_; uint64_t err_col_id = OB_INVALID_ID; - if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(rowkey)); - } else if (OB_FAIL(init_single_row_getter(old_row_getter, run_ctx, column_ids, data_table, true))) { - LOG_WARN("failed to init single row getter", K(ret)); - } else if (OB_FAIL(old_row_getter.open(datum_rowkey, true))) { - LOG_WARN("open old row getter failed", K(ret), K(rowkey)); - } else if (OB_FAIL(old_row_getter.get_next_row(storage_old_row))) { + if (OB_FAIL(get_storage_row(old_row, column_ids, storage_row_getter, run_ctx, storage_old_row, true))) { if (OB_ITER_END == ret) { ret = OB_ERR_DEFENSIVE_CHECK; - FLOG_WARN("old row in storage is not exists", K(ret)); + FLOG_WARN("old row in storage is not exists", K(ret), K(old_row)); } else { - LOG_WARN("get next row from old_row_iter failed", K(ret), KPC(run_ctx.column_ids_), K(old_row)); + LOG_WARN("get next row from old_row_getter failed", K(ret), K(column_ids), K(old_row)); } } else if (OB_ISNULL(storage_old_row)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, storage old row is NULL", K(ret)); - } else if (storage_old_row->get_count() != old_row.get_count()) { + } else if (storage_old_row->count_ != old_row.count_) { ret = OB_ERR_DEFENSIVE_CHECK; FLOG_WARN("storage old row is not matched with sql old row", K(ret)); } else { - for (int64_t i = 0; OB_SUCC(ret) && i < old_row.get_count(); ++i) { - const ObObj &storage_val = storage_old_row->get_cell(i); - const ObObj &sql_val = old_row.get_cell(i); - int cmp = 0; - if (sql_val.is_lob_storage()) { + for (int64_t i = 0; OB_SUCC(ret) && i < old_row.count_; ++i) { + const ObStorageDatum &storage_val = storage_old_row->storage_datums_[i]; + const ObStorageDatum &sql_val = old_row.storage_datums_[i]; + const ObObjMeta &sql_meta = column_descs.at(i).col_type_; + int cmp_ret = 0; + if (sql_meta.is_lob_storage()) { // skip all text and lob - } else if (OB_UNLIKELY(ObLongTextType == storage_val.get_type() && sql_val.is_lob_locator())) { + } else if (sql_meta.is_lob_locator()) { //skip check lob column type when do the normal sql write check } else if (OB_UNLIKELY(storage_val.is_nop_value())) { bool is_nop = false; @@ -3803,16 +3711,16 @@ int ObLSTabletService::check_old_row_legitimacy( } else if (sql_val.is_nop_value()) { //this column is nop val, means that this column does not be touched by DML //just ignore it - } else if (OB_FAIL(storage_val.compare(sql_val, cmp)) || 0 != cmp) { + } else if (OB_FAIL(cmp_funcs.at(i).compare(storage_val, sql_val, cmp_ret)) || 0 != cmp_ret) { ret = OB_ERR_DEFENSIVE_CHECK; err_col_id = column_ids.at(i); FLOG_WARN("storage_val is not equal with sql_val, maybe catch a bug", K(ret), - K(storage_val), K(sql_val), K(cmp), K(column_ids.at(i))); + K(storage_val), K(sql_val), K(column_ids.at(i)), K(cmp_ret)); } } } - if (OB_ERR_DEFENSIVE_CHECK == ret && dml_param.is_batch_stmt_) { + if (OB_ERR_DEFENSIVE_CHECK == ret && run_ctx.dml_param_.is_batch_stmt_) { // 批量删除的时候可能索引表的删除在主表前边，所以所有的表在batch删除的时候出现4377，都可能是重复删导致的 ret = OB_BATCHED_MULTI_STMT_ROLLBACK; LOG_TRACE("batch stmt execution has a correctness error, needs rollback", K(ret), @@ -3836,7 +3744,7 @@ int ObLSTabletService::check_old_row_legitimacy( } else if (OB_TMP_FAIL(check_parts_tx_state_in_transfer_for_4377_(run_ctx.store_ctx_.mvcc_acc_ctx_.tx_desc_))) { ret = tmp_ret; LOG_WARN("check need rollback in transfer for 4377 found exception", K(ret), K(old_row), K(data_table)); - } else if (is_udf) { + } else if (run_ctx.is_udf_) { ret = OB_ERR_INDEX_KEY_NOT_FOUND; LOG_WARN("index key not found on udf column", K(ret), K(old_row)); } else if (data_table.is_index_table() && OB_TMP_FAIL(check_is_gencol_check_failed(data_table, err_col_id, is_virtual_gen_col))) { @@ -3868,6 +3776,7 @@ int ObLSTabletService::check_old_row_legitimacy( } } } + return ret; } @@ -3934,7 +3843,7 @@ int ObLSTabletService::check_is_gencol_check_failed(const ObRelativeTable &data_ int ObLSTabletService::check_new_row_legitimacy( ObDMLRunningCtx &run_ctx, - const common::ObNewRow &new_row) + const blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; ObRelativeTable &data_table = run_ctx.relative_table_; @@ -3942,12 +3851,13 @@ int ObLSTabletService::check_new_row_legitimacy( if (OB_ISNULL(run_ctx.column_ids_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("column ids is nullptr", K(ret)); - } else if (OB_FAIL(check_new_row_nullable_value(*run_ctx.column_ids_, data_table, new_row))) { - LOG_WARN("check new row nullable value failed", K(ret), + } else if (OB_FAIL(check_datum_row_nullable_value(*run_ctx.col_descs_, data_table, datum_row))) { + LOG_WARN("check datum row nullable value failed", K(ret), "dml_param", run_ctx.dml_param_, "dml_type", run_ctx.dml_flag_); - } else if (OB_FAIL(check_new_row_shadow_pk(*run_ctx.column_ids_, data_table, new_row))) { - LOG_WARN("check new row nullable value failed", K(ret), + } else if (OB_FAIL(check_datum_row_shadow_pk(*run_ctx.column_ids_, data_table, datum_row, + run_ctx.dml_param_.table_param_->get_data_table().get_read_info().get_datum_utils()))) { + LOG_WARN("check datum row nullable value failed", K(ret), "dml_param", run_ctx.dml_param_, "dml_type", run_ctx.dml_flag_); } @@ -3957,10 +3867,9 @@ int ObLSTabletService::check_new_row_legitimacy( int ObLSTabletService::insert_rows_to_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - const ObNewRow * const rows, + blocksstable::ObDatumRow *rows, const int64_t row_count, ObRowsInfo &rows_info, - ObStoreRow *tbl_rows, int64_t &afct_num, int64_t &dup_num) { @@ -3973,9 +3882,7 @@ int ObLSTabletService::insert_rows_to_tablet( ret = OB_TIMEOUT; int64_t cur_time = ObClockGenerator::getClock(); LOG_WARN("query timeout", K(cur_time), K(dml_param), K(ret)); - } else if (OB_FAIL(construct_table_rows(rows, tbl_rows, row_count))) { - LOG_WARN("fail to construct table rows", K(ret)); - } else if (OB_FAIL(rows_info.check_duplicate(tbl_rows, row_count, data_table))) { + } else if (OB_FAIL(rows_info.check_duplicate(rows, row_count, data_table))) { if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { char rowkey_buffer[OB_TMP_BUF_SIZE_256]; if (OB_SUCCESS != extract_rowkey(data_table, @@ -3998,9 +3905,9 @@ int ObLSTabletService::insert_rows_to_tablet( } else { LOG_WARN("fail to check duplicate", K(ret)); } - } else if (OB_FAIL(insert_lob_tablet_rows(tablet_handle, run_ctx, tbl_rows, row_count))) { + } else if (OB_FAIL(insert_lob_tablet_rows(tablet_handle, run_ctx, rows, row_count))) { LOG_WARN("failed to insert rows to lob tablet", K(ret)); - } else if (OB_FAIL(insert_tablet_rows(row_count, tablet_handle, run_ctx, tbl_rows, rows_info))) { + } else if (OB_FAIL(insert_tablet_rows(row_count, tablet_handle, run_ctx, rows, rows_info))) { LOG_WARN("failed to insert rows to data tablet", K(ret)); } else { afct_num = afct_num + row_count; @@ -4012,7 +3919,7 @@ int ObLSTabletService::insert_tablet_rows( const int64_t row_count, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + ObDatumRow *rows, ObRowsInfo &rows_info) { int ret = OB_SUCCESS; @@ -4022,9 +3929,9 @@ int ObLSTabletService::insert_tablet_rows( // 1. Defensive checking of new rows. if (GCONF.enable_defensive_check()) { for (int64_t i = 0; OB_SUCC(ret) && i < row_count; i++) { - ObStoreRow &tbl_row = rows[i]; - if (OB_FAIL(check_new_row_legitimacy(run_ctx, tbl_row.row_val_))) { - LOG_WARN("Failed to check new row legitimacy", K(ret), K_(tbl_row.row_val)); + ObDatumRow &datum_row = rows[i]; + if (OB_FAIL(check_new_row_legitimacy(run_ctx, datum_row))) { + LOG_WARN("Failed to check new row legitimacy", K(ret), K(datum_row)); } } } @@ -4082,7 +3989,7 @@ int ObLSTabletService::insert_tablet_rows( int ObLSTabletService::insert_lob_col( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &obj, + blocksstable::ObStorageDatum &datum, ObLobAccessParam *del_param, ObLobCommon *lob_common) { @@ -4092,7 +3999,7 @@ int ObLSTabletService::insert_lob_col( if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]failed to get lob manager handle.", K(ret)); - } else if (!column.col_type_.is_lob_storage() || obj.is_nop_value() || obj.is_null()) { + } else if (!column.col_type_.is_lob_storage() || datum.is_nop_value() || datum.is_null()) { // do nothing } else { // init lob access param @@ -4126,10 +4033,10 @@ int ObLSTabletService::insert_lob_col( lob_param.scan_backward_ = false; lob_param.offset_ = 0; // Notice: currently only inrow data - ObString raw_data = obj.get_string(); + ObString raw_data = datum.get_string(); ObString data; // for not strict sql mode, will insert empty string without lob header - bool has_lob_header = obj.has_lob_header() && raw_data.length() > 0; + bool has_lob_header = datum.has_lob_header() && raw_data.length() > 0; ObLobLocatorV2 loc(raw_data, has_lob_header); if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) { LOG_WARN("set_lob_storage_params fail", K(ret), K(column)); @@ -4137,7 +4044,7 @@ int ObLSTabletService::insert_lob_col( LOG_WARN("[STORAGE_LOB]lob append failed.", K(ret)); } else { ObLobCommon *res_lob_common = lob_param.lob_common_; - obj.set_lob_value(obj.get_type(), res_lob_common, lob_param.handle_size_); + datum.set_lob_data(*res_lob_common, lob_param.handle_size_); LOG_DEBUG("[STORAGE_LOB]write ob lob data.", K(lob_param), KPC(res_lob_common), K(lob_param.handle_size_), K(column.col_type_.get_collation_type())); } @@ -4148,7 +4055,7 @@ int ObLSTabletService::insert_lob_col( int ObLSTabletService::insert_lob_tablet_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, - ObStoreRow &row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; int64_t col_cnt = run_ctx.col_descs_->count(); @@ -4156,9 +4063,9 @@ int ObLSTabletService::insert_lob_tablet_row( if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]failed to get lob manager handle.", K(ret)); - } else if (row.row_val_.count_ != col_cnt) { + } else if (datum_row.count_ != col_cnt) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("[STORAGE_LOB]column count invalid", K(ret), K(col_cnt), K(row.row_val_.count_), KPC(run_ctx.col_descs_)); + LOG_WARN("[STORAGE_LOB]column count invalid", K(ret), K(col_cnt), K(datum_row.count_), KPC(run_ctx.col_descs_)); } else { const int64_t cur_time = ObClockGenerator::getClock(); const int64_t relative_timeout = run_ctx.dml_param_.timeout_ - cur_time; @@ -4169,12 +4076,12 @@ int ObLSTabletService::insert_lob_tablet_row( for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { const ObColDesc &column = run_ctx.col_descs_->at(i); - ObObj &obj = row.row_val_.get_cell(i); - if (obj.is_null() || obj.is_nop_value()) { + ObStorageDatum &datum = datum_row.storage_datums_[i]; + if (datum.is_null() || datum.is_nop_value()) { // do nothing } else if (column.col_type_.is_lob_storage()) { - if (OB_FAIL(insert_lob_col(run_ctx, column, obj, nullptr, nullptr))) { - LOG_WARN("[STORAGE_LOB]failed to insert lob col.", K(ret), K(row), K(i)); + if (OB_FAIL(insert_lob_col(run_ctx, column, datum, nullptr, nullptr))) { + LOG_WARN("[STORAGE_LOB]failed to insert lob col.", K(ret), K(datum_row), K(i)); } } } @@ -4212,7 +4119,7 @@ int update_lob_meta_table_seq_no(ObDMLRunningCtx &run_ctx, int64_t row_count) int ObLSTabletService::insert_lob_tablet_rows( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + blocksstable::ObDatumRow *rows, int64_t row_count) { int ret = OB_SUCCESS; @@ -4300,8 +4207,8 @@ int ObLSTabletService::extract_rowkey( } int ObLSTabletService::get_next_rows( - ObNewRowIterator *row_iter, - ObNewRow *&rows, + blocksstable::ObDatumRowIterator *row_iter, + blocksstable::ObDatumRow *&rows, int64_t &row_count) { return row_iter->get_next_rows(rows, row_count); @@ -4405,9 +4312,10 @@ int ObLSTabletService::check_rowkey_change( } int ObLSTabletService::check_rowkey_value_change( - const common::ObNewRow &old_row, - const common::ObNewRow &new_row, + const ObDatumRow &old_row, + const ObDatumRow &new_row, const int64_t rowkey_len, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils, bool &rowkey_change) { int ret = OB_SUCCESS; @@ -4419,8 +4327,11 @@ int ObLSTabletService::check_rowkey_value_change( K(rowkey_len), K(ret)); } else { rowkey_change = false; - for (int64_t i = 0; !rowkey_change && i < rowkey_len; ++i) { - if (old_row.cells_[i] != new_row.cells_[i]) { + for (int i = 0; OB_SUCC(ret) && !rowkey_change && i < rowkey_len; ++i) { + int cmp_ret = 0; + if (OB_FAIL(rowkey_datum_utils.get_cmp_funcs().at(i).compare(old_row.storage_datums_[i], new_row.storage_datums_[i], cmp_ret))) { + LOG_WARN("compare old datum and new datum failed", K(old_row.storage_datums_[i]), K(new_row.storage_datums_[i])); + } else if (0 != cmp_ret) { rowkey_change = true; } } @@ -4432,9 +4343,9 @@ int ObLSTabletService::check_rowkey_value_change( int ObLSTabletService::process_delta_lob( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &old_obj, + ObStorageDatum &old_datum, ObLobLocatorV2 &delta_lob, - ObObj &obj) + ObStorageDatum &datum) { int ret = OB_SUCCESS; ObLobManager *lob_mngr = MTL(ObLobManager*); @@ -4468,8 +4379,7 @@ int ObLSTabletService::process_delta_lob( ObString old_disk_lob; if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) { LOG_WARN("set_lob_storage_params fail", K(ret), K(column)); - } else if (OB_FAIL(old_obj.get_lob_locatorv2(old_lob))) { - LOG_WARN("get old lob locator failed.", K(ret), K(old_obj)); + } else if (FALSE_IT(old_datum.get_mem_lob(old_lob))) { } else if (!old_lob.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("old lob locator is invalid.", K(ret)); @@ -4487,9 +4397,9 @@ int ObLSTabletService::process_delta_lob( LOG_WARN("failed to process delta lob.", K(ret), K(lob_param), K(delta_lob)); } else { // update obj with new disk locator - obj.set_lob_value(obj.get_type(), lob_param.lob_common_, lob_param.handle_size_); + datum.set_lob_data(*(lob_param.lob_common_), lob_param.handle_size_); if (! lob_param.ext_info_log_.is_null() - && OB_FAIL(register_ext_info_commit_cb(run_ctx, obj, lob_param.ext_info_log_))) { + && OB_FAIL(register_ext_info_commit_cb(run_ctx, datum, column.col_type_.get_type(), lob_param.ext_info_log_))) { LOG_WARN("register_ext_info_commit_cb fail", K(ret), K(lob_param)); } } @@ -4500,7 +4410,8 @@ int ObLSTabletService::process_delta_lob( int ObLSTabletService::register_ext_info_commit_cb( ObDMLRunningCtx &run_ctx, - ObObj &col_data, + ObStorageDatum &col_data, + ObObjType type, ObObj &ext_info_data) { int ret = OB_SUCCESS; @@ -4516,8 +4427,9 @@ int ObLSTabletService::register_ext_info_commit_cb( run_ctx.store_ctx_.mvcc_acc_ctx_.tx_desc_, run_ctx.store_ctx_.mvcc_acc_ctx_.tx_scn_, col_data, + type, ext_info_data))) { - LOG_WARN("register_ext_info_commit_cb fail", K(ret), K(run_ctx.store_ctx_), K(col_data), K(ext_info_data)); + LOG_WARN("register_ext_info_commit_cb fail", K(ret), K(run_ctx.store_ctx_), K(col_data), K(type), K(ext_info_data)); } return ret; } @@ -4548,15 +4460,14 @@ int ObLSTabletService::process_lob_row( ObDMLRunningCtx &run_ctx, const ObIArray &update_idx, bool data_tbl_rowkey_change, - ObStoreRow &old_sql_row, - ObStoreRow &old_row, - ObStoreRow &new_row) + ObDatumRow &old_row, + ObDatumRow &new_row) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(old_row.row_val_.get_count() != new_row.row_val_.get_count())) { + if (OB_UNLIKELY(old_row.count_ != new_row.count_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("[STORAGE_LOB]invalid args", K(old_row), K(new_row), K(ret)); - } else if (OB_UNLIKELY(old_row.row_val_.get_count() != run_ctx.col_descs_->count())) { + } else if (OB_UNLIKELY(old_row.count_ != run_ctx.col_descs_->count())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("[STORAGE_LOB]invalid args", K(old_row), K(new_row), KPC(run_ctx.col_descs_)); } else if (OB_FAIL(update_lob_meta_table_seq_no(run_ctx, 1/*row_count*/))) { @@ -4569,11 +4480,10 @@ int ObLSTabletService::process_lob_row( LOG_WARN("timeout has reached", K(ret), "timeout", run_ctx.dml_param_.timeout_, K(cur_time)); } - for (int64_t i = 0; OB_SUCC(ret) && i < old_row.row_val_.get_count(); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < old_row.count_; ++i) { if (run_ctx.col_descs_->at(i).col_type_.is_lob_storage()) { - ObObj &old_obj = old_row.row_val_.get_cell(i); - ObObj &old_sql_obj = old_sql_row.row_val_.get_cell(i); - ObObj &new_obj = new_row.row_val_.get_cell(i); + ObStorageDatum &old_datum = old_row.storage_datums_[i]; + ObStorageDatum &new_datum = new_row.storage_datums_[i]; bool is_update = false; for (int64_t j = 0; !is_update && j < update_idx.count(); ++j) { if (update_idx.at(j) == i) { @@ -4582,14 +4492,14 @@ int ObLSTabletService::process_lob_row( } if (is_update) { // get new lob locator - ObString new_lob_str = (new_obj.is_null() || new_obj.is_nop_value()) - ? ObString(0, nullptr) : new_obj.get_string(); + ObString new_lob_str = (new_datum.is_null() || new_datum.is_nop_value()) + ? ObString(0, nullptr) : new_datum.get_string(); // for not strict sql mode, will insert empty string without lob header - bool has_lob_header = new_obj.has_lob_header() && new_lob_str.length() > 0; + bool has_lob_header = new_datum.has_lob_header() && new_lob_str.length() > 0; ObLobLocatorV2 new_lob(new_lob_str, has_lob_header); if (OB_FAIL(ret)) { - } else if (new_obj.is_null() || - new_obj.is_nop_value() || + } else if (new_datum.is_null() || + new_datum.is_nop_value() || new_lob.is_full_temp_lob() || new_lob.is_persist_lob() || (new_lob.is_lob_disk_locator() && new_lob.has_inrow_data())) { @@ -4597,47 +4507,42 @@ int ObLSTabletService::process_lob_row( ObLobAccessParam lob_param; if (OB_FAIL(new_lob.get_lob_data_byte_len(lob_param.update_len_))) { LOG_WARN("fail to get new lob byte len", K(ret), K(new_lob), K(i)); - } else if (OB_FAIL(delete_lob_col(run_ctx, run_ctx.col_descs_->at(i), old_obj, old_sql_obj, lob_common, lob_param))) { - LOG_WARN("[STORAGE_LOB]failed to erase old lob col", K(ret), K(old_sql_row), K(old_row), K(i)); - } else if (OB_FAIL(insert_lob_col(run_ctx, run_ctx.col_descs_->at(i), new_obj, &lob_param, lob_common))) { + } else if (OB_FAIL(delete_lob_col(run_ctx, run_ctx.col_descs_->at(i), old_datum, lob_common, lob_param))) { + LOG_WARN("[STORAGE_LOB]failed to erase old lob col", K(ret), K(old_row), K(i)); + } else if (OB_FAIL(insert_lob_col(run_ctx, run_ctx.col_descs_->at(i), new_datum, &lob_param, lob_common))) { LOG_WARN("[STORAGE_LOB]failed to insert new lob col.", K(ret), K(new_row), K(i)); } } else if (new_lob.is_delta_temp_lob()) { - if (OB_FAIL(process_delta_lob(run_ctx, run_ctx.col_descs_->at(i), old_sql_obj, new_lob, new_obj))) { + if (OB_FAIL(process_delta_lob(run_ctx, run_ctx.col_descs_->at(i), old_datum, new_lob, new_datum))) { LOG_WARN("failed to process delta lob.", K(ret), K(i)); } } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected obj for new lob", K(ret), K(i), K(new_obj), K(new_lob)); + LOG_WARN("unexpected obj for new lob", K(ret), K(i), K(new_datum), K(new_lob)); } } else { - if (old_obj.is_null()) { - new_obj.set_null(); - } else if (old_obj.is_nop_value()) { - new_obj.set_nop_value(); - } else if (new_obj.is_nop_value() || new_obj.is_null()) { + if (old_datum.is_null()) { + new_datum.set_null(); + } else if (old_datum.is_nop_value()) { + new_datum.set_nop(); + } else if (new_datum.is_nop_value() || new_datum.is_null()) { // do nothing } else { - if (old_obj.is_null()) { - new_obj.set_null(); - } else if (old_obj.is_nop_value()) { - new_obj.set_nop_value(); - } else { - ObString val_str = old_obj.get_string(); - ObLobCommon *lob_common = reinterpret_cast(val_str.ptr()); - if (!lob_common->in_row_ && data_tbl_rowkey_change) { - ObLobAccessParam lob_param; - if (val_str.length() < ObLobManager::LOB_WITH_OUTROW_CTX_SIZE) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("not enough space for lob header", K(ret), K(val_str), K(i)); - } else if (OB_FAIL(delete_lob_col(run_ctx, run_ctx.col_descs_->at(i), old_obj, old_sql_obj, lob_common, lob_param))) { - LOG_WARN("[STORAGE_LOB]failed to erase old lob col", K(ret), K(old_sql_row), K(old_row), K(i)); - } else if (OB_FAIL(insert_lob_col(run_ctx, run_ctx.col_descs_->at(i), new_obj, nullptr, nullptr))) { // no need del_param - LOG_WARN("[STORAGE_LOB]failed to insert new lob col.", K(ret), K(new_row), K(i)); - } - } else { - new_obj.set_lob_value(new_obj.get_type(), val_str.ptr(), val_str.length()); // remove has lob header flag + ObString val_str = old_datum.get_string(); + ObLobCommon *lob_common = reinterpret_cast(val_str.ptr()); + if (!lob_common->in_row_ && data_tbl_rowkey_change) { + ObLobAccessParam lob_param; + if (val_str.length() < ObLobManager::LOB_WITH_OUTROW_CTX_SIZE) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not enough space for lob header", K(ret), K(val_str), K(i)); + } else if (OB_FAIL(delete_lob_col(run_ctx, run_ctx.col_descs_->at(i), old_datum, lob_common, lob_param))) { + LOG_WARN("[STORAGE_LOB]failed to erase old lob col", K(ret), K(old_row), K(i)); + } else if (OB_FAIL(insert_lob_col(run_ctx, run_ctx.col_descs_->at(i), new_datum, nullptr, nullptr))) { // no need del_param + LOG_WARN("[STORAGE_LOB]failed to insert new lob col.", K(ret), K(new_row), K(i)); } + } else { + new_datum.reuse(); + new_datum.set_string(val_str.ptr(), val_str.length()); // remove has lob header flag } } } @@ -4654,9 +4559,9 @@ int ObLSTabletService::update_row_to_tablet( const ObIArray &update_idx, const bool delay_new, const bool lob_update, - ObStoreRow &old_tbl_row, - ObStoreRow &new_tbl_row, - ObRowStore *row_store, + ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, + ObDatumRowStore *row_store, bool &duplicate) { int ret = OB_SUCCESS; @@ -4665,38 +4570,37 @@ int ObLSTabletService::update_row_to_tablet( bool data_tbl_rowkey_change = false; int64_t data_tbl_rowkey_len = run_ctx.relative_table_.get_rowkey_column_num(); ObSQLMode sql_mode = dml_param.sql_mode_; + const share::schema::ObTableDMLParam *table_param = dml_param.table_param_; duplicate = false; - ObStoreRow old_sql_tbl_row; - old_sql_tbl_row.row_val_ = old_tbl_row.row_val_; - if (OB_UNLIKELY(col_descs.count() != old_tbl_row.row_val_.get_count() - || col_descs.count() != new_tbl_row.row_val_.get_count())) { + if (OB_UNLIKELY(col_descs.count() != old_datum_row.count_ + || col_descs.count() != new_datum_row.count_ || OB_ISNULL(table_param))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(col_descs.count()), - K(old_tbl_row.row_val_), K(new_tbl_row.row_val_)); - } else if (rowkey_change && OB_FAIL(check_rowkey_value_change(old_tbl_row.row_val_, - new_tbl_row.row_val_, + K(old_datum_row), K(new_datum_row)); + } else if (rowkey_change && OB_FAIL(check_rowkey_value_change(old_datum_row, + new_datum_row, data_tbl_rowkey_len, + table_param->get_data_table().get_read_info().get_datum_utils(), data_tbl_rowkey_change))) { LOG_WARN("failed to check data table rowkey change", K(ret), - K(old_tbl_row), K(new_tbl_row), K(data_tbl_rowkey_len)); + K(old_datum_row), K(new_datum_row), K(data_tbl_rowkey_len)); } else if (OB_FAIL(process_old_row(tablet_handle, run_ctx, data_tbl_rowkey_change, lob_update, - old_tbl_row))) { + old_datum_row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("fail to process old row", K(ret), K(*run_ctx.col_descs_), - K(old_tbl_row), K(data_tbl_rowkey_change)); + LOG_WARN("fail to process old row", K(ret), K(col_descs), + K(old_datum_row), K(data_tbl_rowkey_change)); } } else if (OB_FAIL(process_lob_row(tablet_handle, run_ctx, update_idx, data_tbl_rowkey_change, - old_sql_tbl_row, - old_tbl_row, - new_tbl_row))) { - LOG_WARN("failed to process lob col change", K(ret), K(old_tbl_row), K(new_tbl_row)); + old_datum_row, + new_datum_row))) { + LOG_WARN("failed to process lob col change", K(ret), K(old_datum_row), K(new_datum_row)); } else if (delay_new && lib::is_oracle_mode()) { // if total quantity log is needed, we should cache both new row and old row, // and the sequence is new_row1, old_row1, new_row2, old_row2...., @@ -4704,21 +4608,21 @@ int ObLSTabletService::update_row_to_tablet( if (OB_ISNULL(row_store)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("row_store is NULL", K(ret)); - } else if (OB_FAIL(row_store->add_row(new_tbl_row.row_val_))) { - LOG_WARN("failed to store new row", K(new_tbl_row), K(ret)); - } else if (OB_FAIL(row_store->add_row(old_tbl_row.row_val_))) { - LOG_WARN("failed to store old row", K(old_tbl_row), K(ret)); + } else if (OB_FAIL(row_store->add_row(new_datum_row))) { + LOG_WARN("failed to store new row", K(new_datum_row), K(ret)); + } else if (OB_FAIL(row_store->add_row(old_datum_row))) { + LOG_WARN("failed to store old row", K(old_datum_row), K(ret)); } else { - LOG_DEBUG("add row store for delay new", K(old_tbl_row), K(new_tbl_row)); + LOG_DEBUG("add row store for delay new", K(old_datum_row), K(new_datum_row)); } } else if (OB_FAIL(process_new_row(tablet_handle, run_ctx, update_idx, - old_tbl_row, - new_tbl_row, + old_datum_row, + new_datum_row, data_tbl_rowkey_change))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("fail to process new row", K(new_tbl_row), K(ret)); + LOG_WARN("fail to process new row", K(new_datum_row), K(ret)); } } @@ -4730,7 +4634,7 @@ int ObLSTabletService::process_old_row( ObDMLRunningCtx &run_ctx, const bool data_tbl_rowkey_change, const bool lob_update, - ObStoreRow &tbl_row) + ObDatumRow &datum_row) { int ret = OB_SUCCESS; ObStoreCtx &store_ctx = run_ctx.store_ctx_; @@ -4742,68 +4646,66 @@ int ObLSTabletService::process_old_row( } else if (OB_UNLIKELY(!store_ctx.is_valid() || nullptr == run_ctx.col_descs_ || run_ctx.col_descs_->count() <= 0 - || !tbl_row.is_valid())) { + || !datum_row.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(store_ctx), KP(run_ctx.col_descs_), K(tbl_row), K(is_delete_total_quantity_log)); - } else if (OB_FAIL(check_old_row_legitimacy(tablet_handle, run_ctx, tbl_row.row_val_))) { + LOG_WARN("invalid args", K(ret), K(store_ctx), KP(run_ctx.col_descs_), K(datum_row), K(is_delete_total_quantity_log)); + } else if (OB_FAIL(check_old_row_legitimacy(run_ctx.cmp_funcs_, tablet_handle, run_ctx, datum_row))) { if (OB_ERR_DEFENSIVE_CHECK == ret) { - dump_diag_info_for_old_row_loss(relative_table, store_ctx, tbl_row); + dump_diag_info_for_old_row_loss(run_ctx, datum_row); } - LOG_WARN("check old row legitimacy failed", K(tbl_row.row_val_)); - } else if (OB_FAIL(process_old_row_lob_col(tablet_handle, run_ctx, tbl_row))){ - LOG_WARN("failed to process old row lob col", K(ret), K(tbl_row)); + LOG_WARN("check old row legitimacy failed", K(ret), K(datum_row)); + } else if (OB_FAIL(process_old_row_lob_col(tablet_handle, run_ctx, datum_row))){ + LOG_WARN("failed to process old row lob col", K(ret), K(datum_row)); } else { ObColDescIArray &col_descs = const_cast(*run_ctx.col_descs_); const uint64_t &table_id = relative_table.get_table_id(); int64_t rowkey_size = relative_table.get_rowkey_column_num(); - ObStoreRowkey store_rowkey; ObDatumRowkey datum_rowkey; ObDatumRowkeyHelper rowkey_helper(run_ctx.allocator_); - if (OB_UNLIKELY(run_ctx.dml_param_.prelock_)) { bool locked = false; - if (OB_FAIL(store_rowkey.assign(tbl_row.row_val_.cells_, rowkey_size))) { - LOG_WARN("Failed to assign rowkey", K(ret), K(tbl_row), K(rowkey_size)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(store_rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(store_rowkey), K(rowkey_size), K(tbl_row)); + if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(datum_row, rowkey_size, col_descs, datum_rowkey))) { + LOG_WARN("Failed to prepare rowkey", K(ret), K(datum_row), K(rowkey_size), K(datum_rowkey)); } else if (OB_FAIL(check_row_locked_by_myself(tablet_handle, relative_table, store_ctx, datum_rowkey, locked))) { - LOG_WARN("fail to check row locked", K(ret), K(tbl_row)); + LOG_WARN("fail to check row locked", K(ret), K(datum_row), K(datum_rowkey)); } else if (!locked) { ret = OB_ERR_ROW_NOT_LOCKED; - LOG_DEBUG("row has not been locked", K(ret), K(tbl_row)); + LOG_DEBUG("row has not been locked", K(ret), K(datum_row), K(datum_rowkey)); } } if (OB_FAIL(ret)) { } else if (data_tbl_rowkey_change) { - ObStoreRow del_row(tbl_row); - ObStoreRow new_tbl_row; - new_tbl_row.flag_.set_flag(ObDmlFlag::DF_DELETE); - new_tbl_row.row_val_ = tbl_row.row_val_; - del_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); + ObDatumRow del_row; + ObDatumRow new_row; + ObSEArray update_idx; - if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, + if (OB_FAIL(del_row.shallow_copy(datum_row))) { + LOG_WARN("failed to shallow copy datum row", K(ret), K(datum_row), K(del_row)); + } else if (FALSE_IT(del_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE))) { + } else if (OB_FAIL(new_row.shallow_copy(datum_row))) { + LOG_WARN("failed to shallow copy datum row", K(ret), K(datum_row), K(new_row)); + } else if (FALSE_IT(new_row.row_flag_.set_flag(ObDmlFlag::DF_DELETE))) { + } else if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, run_ctx.store_ctx_, col_descs, update_idx, del_row, - new_tbl_row, + new_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("failed to write data tablet row", K(ret), K(del_row), K(new_tbl_row)); + LOG_WARN("failed to write data tablet row", K(ret), K(del_row), K(new_row)); } } } else if (lob_update) { // need to lock main table rows that don't need to be deleted - if (OB_FAIL(store_rowkey.assign(tbl_row.row_val_.cells_, rowkey_size))) { - LOG_WARN("Failed to assign rowkey", K(ret), K(tbl_row), K(rowkey_size)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(store_rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(store_rowkey)); + if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(datum_row, rowkey_size, col_descs, datum_rowkey))) { + LOG_WARN("Failed to prepare rowkey", K(ret), K(datum_row), K(rowkey_size)); } else if (OB_FAIL(tablet_handle.get_obj()->lock_row(relative_table, store_ctx, datum_rowkey))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("lock row failed", K(ret), K(table_id), K(tbl_row), K(rowkey_size)); + LOG_WARN("lock row failed", K(ret), K(table_id), K(datum_row), K(rowkey_size), K(datum_rowkey)); } } - LOG_DEBUG("generate lock node before update lob columns", K(ret), K(table_id), K(tbl_row)); + LOG_DEBUG("generate lock node before update lob columns", K(ret), K(table_id), K(datum_row), K(datum_rowkey)); } } return ret; @@ -4813,28 +4715,28 @@ int ObLSTabletService::process_new_row( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, const common::ObIArray &update_idx, - const ObStoreRow &old_tbl_row, - const ObStoreRow &new_tbl_row, + const ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, const bool rowkey_change) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(update_idx.count() < 0 || !new_tbl_row.is_valid())) { + if (OB_UNLIKELY(update_idx.count() < 0 || !old_datum_row.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(update_idx), K(new_tbl_row), K(rowkey_change)); + LOG_WARN("invalid args", K(ret), K(update_idx), K(old_datum_row), K(rowkey_change)); } else if (GCONF.enable_defensive_check() - && OB_FAIL(check_new_row_legitimacy(run_ctx, new_tbl_row.row_val_))) { - LOG_WARN("check new row legitimacy failed", K(ret), K(new_tbl_row.row_val_)); + && OB_FAIL(check_new_row_legitimacy(run_ctx, old_datum_row))) { + LOG_WARN("check datum row legitimacy failed", K(ret), K(old_datum_row)); } else { // write full column clog needs to construct update_idx and pass to memtable if (OB_FAIL(process_data_table_row(tablet_handle, run_ctx, update_idx, - old_tbl_row, - new_tbl_row, + old_datum_row, + new_datum_row, rowkey_change))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("fail to process data table row", K(ret), - K(update_idx), K(old_tbl_row), K(new_tbl_row), K(rowkey_change)); + K(update_idx), K(old_datum_row), K(new_datum_row), K(rowkey_change)); } } } @@ -4845,8 +4747,8 @@ int ObLSTabletService::process_data_table_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, const ObIArray &update_idx, - const ObStoreRow &old_tbl_row, - const ObStoreRow &new_tbl_row, + const ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, const bool rowkey_change) { int ret = OB_SUCCESS; @@ -4859,34 +4761,34 @@ int ObLSTabletService::process_data_table_row( || nullptr == run_ctx.col_descs_ || run_ctx.col_descs_->count() <= 0 || update_idx.count() < 0 - || (is_update_total_quantity_log && !old_tbl_row.is_valid()) - || !new_tbl_row.is_valid())) { + || (is_update_total_quantity_log && !old_datum_row.is_valid()) + || !new_datum_row.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(ctx), - KP(run_ctx.col_descs_), K(update_idx), K(old_tbl_row), K(new_tbl_row), + KP(run_ctx.col_descs_), K(update_idx), K(old_datum_row), K(new_datum_row), K(is_update_total_quantity_log), K(rowkey_change)); } else { const ObColDescIArray &col_descs = *run_ctx.col_descs_; - ObStoreRow new_row; - new_row.flag_.set_flag(rowkey_change ? ObDmlFlag::DF_INSERT : ObDmlFlag::DF_UPDATE); - new_row.row_val_ = new_tbl_row.row_val_; - + new_datum_row.row_flag_.set_flag(rowkey_change ? ObDmlFlag::DF_INSERT : ObDmlFlag::DF_UPDATE); if (!rowkey_change) { - ObStoreRow old_row; - old_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); - old_row.row_val_ = old_tbl_row.row_val_; - if (!is_update_total_quantity_log) { - // For minimal mode, set pk columns of old_row to nop value, because - // they are already stored in new_row. - const int64_t rowkey_col_cnt = relative_table.get_rowkey_column_num(); - for (int64_t i = 0; i < rowkey_col_cnt; ++i) { - (old_row.row_val_.cells_[i]).set_nop_value(); + ObDatumRow old_row; + if (OB_FAIL(old_row.shallow_copy(old_datum_row))) { + LOG_WARN("failed to shallow copy datum row", K(ret), K(old_datum_row), K(old_row)); + } else { + old_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE); + if (!is_update_total_quantity_log) { + // For minimal mode, set pk columns of old_row to nop value, because + // they are already stored in new_row. + const int64_t rowkey_col_cnt = relative_table.get_rowkey_column_num(); + for (int64_t i = 0; i < rowkey_col_cnt; ++i) { + (old_row.storage_datums_[i]).set_nop(); + } } - } - if (OB_FAIL(data_tablet.get_obj()->update_row(relative_table, - ctx, col_descs, update_idx, old_row, new_row, run_ctx.dml_param_.encrypt_meta_))) { - if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("failed to update to row", K(ret), K(old_row), K(new_row)); + if (OB_FAIL(data_tablet.get_obj()->update_row(relative_table, + ctx, col_descs, update_idx, old_row, new_datum_row, run_ctx.dml_param_.encrypt_meta_))) { + if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { + LOG_WARN("failed to update to row", K(ret), K(old_row), K(new_datum_row)); + } } } } else { @@ -4895,13 +4797,13 @@ int ObLSTabletService::process_data_table_row( ctx, check_exist, col_descs, - new_row, + new_datum_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { char buffer[OB_TMP_BUF_SIZE_256]; - ObStoreRowkey rowkey; - if (OB_SUCCESS != rowkey.assign(new_tbl_row.row_val_.cells_, relative_table.get_rowkey_column_num())) { - LOG_WARN("Failed to assign rowkey", K(new_tbl_row)); + ObDatumRowkey rowkey; + if (OB_SUCCESS != rowkey.assign(new_datum_row.storage_datums_, relative_table.get_rowkey_column_num())) { + LOG_WARN("Failed to assign rowkey", K(new_datum_row)); } else if (OB_SUCCESS != extract_rowkey(relative_table, rowkey, buffer, OB_TMP_BUF_SIZE_256, tz_info)) { LOG_WARN("extract rowkey failed", K(rowkey)); } else { @@ -4911,9 +4813,9 @@ int ObLSTabletService::process_data_table_row( } LOG_USER_ERROR(OB_ERR_PRIMARY_KEY_DUPLICATE, buffer, index_name.length(), index_name.ptr()); } - LOG_WARN("rowkey already exists", K(ret), K(new_tbl_row)); + LOG_WARN("rowkey already exists", K(ret), K(new_datum_row)); } else if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("failed to update to row", K(ret), K(new_row)); + LOG_WARN("failed to update to row", K(ret), K(new_datum_row)); } } } @@ -4921,94 +4823,26 @@ int ObLSTabletService::process_data_table_row( return ret; } -int ObLSTabletService::check_new_row_nullable_value( - const ObIArray &column_ids, - ObRelativeTable &data_table, - const ObNewRow &new_row) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(column_ids.count() != new_row.get_count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("new row is invalid", K(ret), K(new_row.get_count()), K(column_ids.count())); - } - for (int64_t i = 0; OB_SUCC(ret) && i < column_ids.count(); ++i) { - uint64_t column_id = column_ids.at(i); - bool is_nullable = false; - if (OB_UNLIKELY(is_shadow_column(column_id))) { - //the shadow pk is generated internally, - //and the nullable attribute check for it is skipped - } else if (OB_FAIL(data_table.is_column_nullable_for_write(column_id, is_nullable))) { - LOG_WARN("check is_column_nullable_for_write failed", K(ret), K(column_id)); - } else if (new_row.get_cell(i).is_null() && !is_nullable) { - bool is_hidden = false; - bool is_gen_col = false; - bool is_nullable_for_read = false; - if (OB_FAIL(data_table.is_column_nullable_for_read(column_id, is_nullable_for_read))) { - LOG_WARN("check is nullable for read failed", K(ret)); - } else if (is_nullable_for_read) { - LOG_TRACE("Catch a defensive nullable error, but this column is not null novalidate", - K(column_id), K(column_ids), K(new_row), K(data_table)); - } else if (OB_FAIL(data_table.is_hidden_column(column_id, is_hidden))) { - LOG_WARN("get is hidden column failed", K(ret), K(column_id)); - } else if (OB_FAIL(data_table.is_gen_column(column_id, is_gen_col))) { - LOG_WARN("get is gen column failed", K(ret), K(column_id)); - } else if (is_hidden && !is_gen_col) { - ret = OB_BAD_NULL_ERROR; - LOG_WARN("Catch a defensive nullable error, " - "maybe cause by add column not null default null ONLINE", K(ret), - K(column_id), K(column_ids), K(new_row), K(data_table)); - } else { - ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_nullable_value"); - LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); - LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, - "Fatal Error!!! Catch a defensive error!", K(ret), - K(column_id), K(column_ids), K(new_row), K(data_table)); - LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, - OB_ERR_DEFENSIVE_CHECK, - "Fatal Error!!! Catch a defensive error!"); - } - } else if (new_row.get_cell(i).is_number()) { - number::ObNumber num; - if (OB_FAIL(new_row.get_cell(i).get_number(num))) { - LOG_WARN("get number value from object fail", K(ret), K(new_row.get_cell(i))); - } else if (OB_FAIL(num.sanity_check())) { - LOG_WARN("sanity check number failed", K(ret), K(new_row.get_cell(i))); - } - if (OB_SUCCESS != ret) { - ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_nullable_value"); - LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); - LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, - "Fatal Error!!! Catch a defensive error!", K(ret), - K(column_id), K(column_ids), K(new_row), K(data_table)); - LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, - OB_ERR_DEFENSIVE_CHECK, - "Fatal Error!!! Catch a defensive error!"); - } - } - } - return ret; -} - -int ObLSTabletService::check_new_row_nullable_value(const ObIArray &col_descs, +int ObLSTabletService::check_datum_row_nullable_value(const ObIArray &col_descs, ObRelativeTable &relative_table, - const ObNewRow &new_row) + const blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(col_descs.count() > new_row.get_count())) { + if (OB_UNLIKELY(col_descs.count() > datum_row.get_column_count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("new row is invalid", K(ret), K(new_row.get_count()), K(col_descs.count())); + LOG_WARN("new row is invalid", K(ret), K(datum_row.get_column_count()), K(col_descs.count())); } for (int64_t i = 0; OB_SUCC(ret) && i < col_descs.count(); ++i) { uint64_t column_id = col_descs.at(i).col_id_; bool is_nullable = false; - if (OB_UNLIKELY(is_shadow_column(column_id))) { + if (datum_row.storage_datums_[i].is_nop()) { + //nothing + } else if (OB_UNLIKELY(is_shadow_column(column_id))) { //the shadow pk is generated internally, //and the nullable attribute check for it is skipped } else if (OB_FAIL(relative_table.is_column_nullable_for_write(column_id, is_nullable))) { LOG_WARN("check is_column_nullable_for_write failed", K(ret), K(column_id)); - } else if (new_row.get_cell(i).is_null() && !is_nullable) { + } else if (datum_row.storage_datums_[i].is_null() && !is_nullable) { bool is_hidden = false; bool is_gen_col = false; bool is_nullable_for_read = false; @@ -5018,7 +4852,7 @@ int ObLSTabletService::check_new_row_nullable_value(const ObIArray &c //this column is not null novalidate, maybe the null column come from the old data //so output trace log and ignore it LOG_TRACE("Catch a defensive nullable error, but this column is not null novalidate", - K(column_id), K(col_descs), K(new_row), K(relative_table)); + K(column_id), K(col_descs), K(datum_row), K(relative_table)); } else if (OB_FAIL(relative_table.is_hidden_column(column_id, is_hidden))) { LOG_WARN("get is hidden column failed", K(ret), K(column_id)); } else if (OB_FAIL(relative_table.is_gen_column(column_id, is_gen_col))) { @@ -5027,32 +4861,30 @@ int ObLSTabletService::check_new_row_nullable_value(const ObIArray &c ret = OB_BAD_NULL_ERROR; LOG_WARN("Catch a defensive nullable error, " "maybe cause by add column not null default null ONLINE", K(ret), - K(column_id), K(col_descs), K(new_row), K(relative_table)); + K(column_id), K(col_descs), K(datum_row), K(relative_table)); } else { ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_nullable_value"); + ObString func_name = ObString::make_string("check_datum_row_nullable_value"); LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!", K(ret), - K(column_id), K(col_descs), K(new_row), K(relative_table)); + K(column_id), K(col_descs), K(datum_row), K(relative_table)); LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!"); } - } else if (new_row.get_cell(i).is_number()) { - number::ObNumber num; - if (OB_FAIL(new_row.get_cell(i).get_number(num))) { - LOG_WARN("get number value from object fail", K(ret), K(new_row.get_cell(i))); - } else if (OB_FAIL(num.sanity_check())) { - LOG_WARN("sanity check number failed", K(ret), K(new_row.get_cell(i))); + } else if (!datum_row.storage_datums_[i].is_null() && col_descs.at(i).col_type_.is_number()) { + number::ObNumber num(datum_row.storage_datums_[i].get_number()); + if (OB_FAIL(num.sanity_check())) { + LOG_WARN("sanity check number failed", K(ret), K(i), K(datum_row.storage_datums_[i])); } if (OB_SUCCESS != ret) { ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_nullable_value"); + ObString func_name = ObString::make_string("check_datum_row_nullable_value"); LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!", K(ret), - K(column_id), K(col_descs), K(new_row), K(relative_table)); + K(column_id), K(col_descs), K(datum_row), K(relative_table)); LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!"); @@ -5062,10 +4894,11 @@ int ObLSTabletService::check_new_row_nullable_value(const ObIArray &c return ret; } -int ObLSTabletService::check_new_row_shadow_pk( +int ObLSTabletService::check_datum_row_shadow_pk( const ObIArray &column_ids, ObRelativeTable &data_table, - const ObNewRow &new_row) + const blocksstable::ObDatumRow &datum_row, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils) { int ret = OB_SUCCESS; if (data_table.get_shadow_rowkey_column_num() > 0) { @@ -5082,43 +4915,43 @@ int ObLSTabletService::check_new_row_shadow_pk( // mysql兼容：只要unique index key中有null列，则需要填充shadow列 bool rowkey_has_null = false; for (int64_t i = 0; !rowkey_has_null && i < index_col_cnt; i++) { - rowkey_has_null = new_row.get_cell(i).is_null(); + rowkey_has_null = datum_row.storage_datums_[i].is_null(); } need_spk = rowkey_has_null; } else { // oracle兼容：只有unique index key全为null列时，才需要填充shadow列 bool is_rowkey_all_null = true; for (int64_t i = 0; is_rowkey_all_null && i < index_col_cnt; i++) { - is_rowkey_all_null = new_row.get_cell(i).is_null(); + is_rowkey_all_null = datum_row.storage_datums_[i].is_null(); } need_spk = is_rowkey_all_null; } for (int64_t i = index_col_cnt; OB_SUCC(ret) && i < rowkey_cnt; ++i) { uint64_t spk_column_id = column_ids.at(i); uint64_t real_pk_id = spk_column_id - OB_MIN_SHADOW_COLUMN_ID; - const ObObj &spk_value = new_row.get_cell(i); + const ObStorageDatum &spk_value = datum_row.storage_datums_[i]; int64_t pk_idx = OB_INVALID_INDEX; int cmp = 0; if (OB_LIKELY(!need_spk)) { if (!spk_value.is_null()) { ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_shadow_pk"); + ObString func_name = ObString::make_string("check_datum_row_shadow_pk"); LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); LOG_ERROR("Fatal Error!!! Catch a defensive error!", K(ret), - "column_id", column_ids, K(new_row), K(data_table), + "column_id", column_ids, K(datum_row), K(data_table), K(spk_value), K(i), K(spk_column_id), K(real_pk_id)); } } else if (OB_UNLIKELY(!has_exist_in_array(column_ids, real_pk_id, &pk_idx))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("real pk column not exists in column_ids", K(ret), K(column_ids), K(real_pk_id)); - } else if (OB_FAIL(new_row.get_cell(pk_idx).compare(spk_value, cmp)) || 0 != cmp) { + } else if (OB_FAIL(rowkey_datum_utils.get_cmp_funcs().at(i).compare(datum_row.storage_datums_[pk_idx], spk_value, cmp)) || 0 != cmp) { ret = OB_ERR_DEFENSIVE_CHECK; - ObString func_name = ObString::make_string("check_new_row_shadow_pk"); + ObString func_name = ObString::make_string("check_datum_row_shadow_pk"); LOG_USER_ERROR(OB_ERR_DEFENSIVE_CHECK, func_name.length(), func_name.ptr()); LOG_ERROR_RET(OB_ERR_DEFENSIVE_CHECK, "Fatal Error!!! Catch a defensive error!", K(ret), - "column_id", column_ids, K(new_row), K(data_table), - K(spk_value), "pk_value", new_row.get_cell(pk_idx), + "column_id", column_ids, K(datum_row), K(data_table), + K(spk_value), "pk_value", datum_row.storage_datums_[pk_idx], K(pk_idx), K(i), K(spk_column_id), K(real_pk_id)); LOG_DBA_ERROR_V2(OB_STORAGE_DEFENSIVE_CHECK_FAIL, OB_ERR_DEFENSIVE_CHECK, @@ -5163,32 +4996,27 @@ int ObLSTabletService::get_conflict_rows( ObDMLRunningCtx &run_ctx, const ObInsertFlag flag, const common::ObIArray &out_col_ids, - const common::ObNewRow &row, - common::ObNewRowIterator *&duplicated_rows) + blocksstable::ObDatumRow &row, + blocksstable::ObDatumRowIterator *&duplicated_rows) { TRANS_LOG(DEBUG, "get conflict rows", K(flag), K(row), K(lbt())); int ret = OB_SUCCESS; - ObRelativeTable &data_table = run_ctx.relative_table_; ObArenaAllocator scan_allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_TABLE_SCAN_ITER)); - ObIAllocator *allocator = &scan_allocator; ObTablet *data_tablet = tablet_handle.get_obj(); - ObDatumRowkeyHelper rowkey_helper(scan_allocator); - ObDatumRowkey datum_rowkey; - ObStoreRowkey rowkey; - rowkey.assign(row.cells_, data_table.get_rowkey_column_num()); + ObSingleRowGetter storage_row_getter(scan_allocator, *data_tablet); + ObDatumRow *out_row = nullptr; if (OB_ISNULL(data_tablet)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet is null", K(ret), K(tablet_handle)); - } else { - ObSingleRowGetter single_row_getter(*allocator, *data_tablet); - if (OB_FAIL(init_single_row_getter(single_row_getter, run_ctx, out_col_ids, data_table, true))) { - LOG_WARN("failed to init single row getter", K(ret)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(rowkey)); - } else if (OB_FAIL(single_get_row(single_row_getter, datum_rowkey, duplicated_rows))) { - LOG_WARN("failed to single get row", K(ret)); + } else if (OB_FAIL(get_storage_row(row, out_col_ids, storage_row_getter, run_ctx, out_row))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next row from single row getter failed", K(ret)); + } else { + ret = OB_SUCCESS; } + } else if (OB_FAIL(add_duplicate_row(out_row, data_tablet->get_rowkey_read_info().get_datum_utils(), duplicated_rows))) { + LOG_WARN("failed to single get row", K(ret), K(row), K(out_row)); } if (OB_FAIL(ret)) { @@ -5210,33 +5038,25 @@ int ObLSTabletService::init_single_row_getter( { int ret = OB_SUCCESS; - if (OB_FAIL(row_getter.init_dml_access_ctx(run_ctx.store_ctx_, run_ctx.dml_param_, skip_read_lob))) { - LOG_WARN("init dml access ctx failed", K(ret)); - } else if (OB_FAIL(row_getter.init_dml_access_param(relative_table, + if (OB_FAIL(row_getter.init_dml_access_param(relative_table, run_ctx.dml_param_, out_col_ids, skip_read_lob))) { LOG_WARN("init dml access param failed", K(ret)); + } else if (OB_FAIL(row_getter.prepare_cached_iter_node())) { + LOG_WARN("prepare cached iter node failed", K(ret)); + } else if (OB_FAIL(row_getter.init_dml_access_ctx(run_ctx.store_ctx_, run_ctx.dml_param_, skip_read_lob))) { + LOG_WARN("init dml access ctx failed", K(ret)); } return ret; } -int ObLSTabletService::single_get_row( - ObSingleRowGetter &row_getter, - const ObDatumRowkey &rowkey, - ObNewRowIterator *&duplicated_rows) +int ObLSTabletService::add_duplicate_row( + ObDatumRow *storage_row, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils, + blocksstable::ObDatumRowIterator *&duplicated_rows) { int ret = OB_SUCCESS; - ObNewRow *row = nullptr; - - if (OB_FAIL(row_getter.open(rowkey))) { - LOG_WARN("init single row getter failed", K(ret)); - } else if (OB_FAIL(row_getter.get_next_row(row))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next row from single row getter failed", K(ret)); - } else { - ret = OB_SUCCESS; - } - } else if (NULL == duplicated_rows) { + if (NULL == duplicated_rows) { ObValueRowIterator *dup_iter = NULL; if (OB_ISNULL(dup_iter = ObQueryIteratorFactory::get_insert_dup_iter())) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -5248,35 +5068,18 @@ int ObLSTabletService::single_get_row( } } } - if (OB_SUCC(ret) && row != nullptr) { + if (OB_SUCC(ret) && storage_row != nullptr) { ObValueRowIterator *dup_iter = static_cast(duplicated_rows); - if (OB_FAIL(dup_iter->add_row(*row))) { - LOG_WARN("failed to store conflict row", K(ret), K(*row)); + if (OB_FAIL(dup_iter->add_row(*storage_row, rowkey_datum_utils))) { + LOG_WARN("failed to store conflict row", K(ret), K(*storage_row)); } else { - LOG_DEBUG("get conflict row", KPC(row)); + LOG_DEBUG("get conflict row", KPC(storage_row)); } } return ret; } -int ObLSTabletService::convert_row_to_rowkey( - ObSingleRowGetter &index_row_getter, - ObStoreRowkey &rowkey) -{ - int ret = OB_SUCCESS; - ObNewRow *row = nullptr; - if (OB_FAIL(index_row_getter.get_next_row(row))) { - if (OB_ITER_END != ret) { - LOG_WARN("get next row from index row getter failed", K(ret)); - } - } else { - rowkey.assign(row->cells_, row->count_); - } - - return ret; -} - /* this func is an encapsulation of ObNewRowIterator->get_next_row. * 1. need_copy_cells is true, perform a cells copy, but not a deep copy. * memory for store_row.row_val.cells_ has already allocated before @@ -5284,12 +5087,12 @@ int ObLSTabletService::convert_row_to_rowkey( * 2. need_copy_cells is false, just perform an assignment, no any copy behavior, */ int ObLSTabletService::get_next_row_from_iter( - ObNewRowIterator *row_iter, - ObStoreRow &store_row, + blocksstable::ObDatumRowIterator *row_iter, + ObDatumRow &datum_row, const bool need_copy_cells) { int ret = OB_SUCCESS; - ObNewRow *row = nullptr; + ObDatumRow *row = nullptr; if (OB_ISNULL(row_iter)) { ret = OB_INVALID_ARGUMENT; @@ -5300,15 +5103,16 @@ int ObLSTabletService::get_next_row_from_iter( } } else { if (need_copy_cells) { - // in this situation, store_row.row_val has already hold mem for cells_, - // no need to alloc mem here, we copy cells only. - store_row.row_val_.count_ = row->count_; + // in this situation, datum_row has already hold mem for storage_datums_, + // no need to alloc mem here, we copy storage_datums_ only. + datum_row.count_ = row->count_; for (int64_t i = 0; i < row->count_; ++i) { - store_row.row_val_.cells_[i] = row->cells_[i]; + datum_row.storage_datums_[i] = row->storage_datums_[i]; } - } else { - store_row.row_val_ = *row; + } else if (OB_FAIL(datum_row.shallow_copy(*row))) { + LOG_WARN("fail to shallow copy from iterator", K(ret), KPC(row), K(datum_row)); } + datum_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE); } return ret; @@ -5318,7 +5122,7 @@ int ObLSTabletService::insert_row_to_tablet( const bool check_exist, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow &tbl_row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; ObStoreCtx &store_ctx = run_ctx.store_ctx_; @@ -5328,13 +5132,13 @@ int ObLSTabletService::insert_row_to_tablet( || !relative_table.is_valid() || nullptr == run_ctx.col_descs_ || run_ctx.col_descs_->count() <= 0 - || !tbl_row.is_valid())) { + || !datum_row.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(store_ctx), KP(run_ctx.col_descs_), K(tbl_row), K(ret)); + LOG_WARN("invalid args", K(store_ctx), KP(run_ctx.col_descs_), K(datum_row), K(ret)); } else if (GCONF.enable_defensive_check() - && OB_FAIL(check_new_row_legitimacy(run_ctx, tbl_row.row_val_))) { - LOG_WARN("check new row legitimacy failed", K(ret), K(tbl_row.row_val_)); - } else if (OB_FAIL(insert_lob_tablet_row(tablet_handle, run_ctx, tbl_row))) { + && OB_FAIL(check_new_row_legitimacy(run_ctx, datum_row))) { + LOG_WARN("check new row legitimacy failed", K(ret), K(datum_row)); + } else if (OB_FAIL(insert_lob_tablet_row(tablet_handle, run_ctx, datum_row))) { LOG_WARN("failed to write lob tablets rows", K(ret)); } else { const ObColDescIArray &col_descs = *run_ctx.col_descs_; @@ -5343,12 +5147,12 @@ int ObLSTabletService::insert_row_to_tablet( store_ctx, check_exist /*check_exist*/, col_descs, - tbl_row, + datum_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("failed to write table row", K(ret), "table id", relative_table.get_table_id(), - K(col_descs), K(tbl_row)); + K(col_descs), K(datum_row)); } } } @@ -5356,26 +5160,28 @@ int ObLSTabletService::insert_row_to_tablet( return ret; } +// revert start int ObLSTabletService::process_old_row_lob_col( ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow &tbl_row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; + run_ctx.is_old_row_valid_for_lob_ = false; bool has_lob_col = false; bool need_reread = is_sys_table(run_ctx.relative_table_.get_table_id()); int64_t col_cnt = run_ctx.col_descs_->count(); - if (tbl_row.row_val_.count_ != col_cnt) { + if (datum_row.count_ != col_cnt) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("[STORAGE_LOB]Invliad row col cnt", K(ret), K(col_cnt), K(tbl_row)); + LOG_WARN("[STORAGE_LOB]Invliad row col cnt", K(ret), K(col_cnt), K(datum_row)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { const ObColDesc &column = run_ctx.col_descs_->at(i); if (is_lob_storage(column.col_type_.get_type())) { has_lob_col = true; - ObObj &obj = tbl_row.row_val_.cells_[i]; - need_reread = need_reread || (!obj.is_null() && !obj.is_nop_value() && !obj.has_lob_header()); + ObStorageDatum &datum = datum_row.storage_datums_[i]; + need_reread = need_reread || (!datum.is_null() && !datum.is_nop_value() && !datum.has_lob_header()); break; } } @@ -5385,15 +5191,15 @@ int ObLSTabletService::process_old_row_lob_col( for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { const ObColDesc &column = run_ctx.col_descs_->at(i); if (is_lob_storage(column.col_type_.get_type())) { - ObObj &obj = tbl_row.row_val_.cells_[i]; - bool has_lob_header = obj.has_lob_header(); - if (obj.is_null() || obj.is_nop_value()) { + ObStorageDatum &datum = datum_row.storage_datums_[i]; + bool has_lob_header = datum.has_lob_header(); + if (datum.is_null() || datum.is_nop_value()) { // do nothing } else if (!has_lob_header) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("lob should have lob locator here.", K(ret), K(i), K(tbl_row.row_val_.cells_[i])); + LOG_WARN("lob should have lob locator here.", K(ret), K(i), K(datum)); } else { - ObLobLocatorV2 lob(obj.get_string(), has_lob_header); + ObLobLocatorV2 lob(datum.get_string(), has_lob_header); ObString disk_loc; if (!lob.is_valid()) { ret = OB_ERR_UNEXPECTED; @@ -5403,9 +5209,9 @@ int ObLSTabletService::process_old_row_lob_col( } else if (OB_FAIL(lob.get_disk_locator(disk_loc))) { LOG_WARN("failed to get disk lob locator.", K(ret), K(lob)); } else { - obj.set_lob_value(obj.get_type(), disk_loc.ptr(), disk_loc.length()); + datum.set_string(disk_loc.ptr(), disk_loc.length()); if (has_lob_header) { - obj.set_has_lob_header(); + datum.set_has_lob_header(); } run_ctx.is_old_row_valid_for_lob_ = true; } @@ -5413,7 +5219,7 @@ int ObLSTabletService::process_old_row_lob_col( } } } else { - if (OB_FAIL(table_refresh_row(data_tablet_handle, run_ctx, tbl_row.row_val_))) { + if (OB_FAIL(table_refresh_row(data_tablet_handle, run_ctx, datum_row))) { LOG_WARN("[STORAGE_LOB]re-read lob col failed", K(ret)); } } @@ -5424,16 +5230,12 @@ int ObLSTabletService::process_old_row_lob_col( int ObLSTabletService::table_refresh_row( ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - ObNewRow &row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; ObArenaAllocator scan_allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_LOB_ACCESS_BUFFER)); - ObTablet *data_tablet = data_tablet_handle.get_obj(); + ObSingleRowGetter storage_row_getter(scan_allocator, *data_tablet_handle.get_obj()); ObRelativeTable &data_table = run_ctx.relative_table_; - ObStoreRowkey rowkey; - if (OB_FAIL(rowkey.assign(row.cells_, data_table.get_rowkey_column_num()))) { - LOG_WARN("get rowkey col num failed", K(ret)); - } int64_t col_cnt = run_ctx.col_descs_->count(); ObSEArray out_col_ids; @@ -5442,22 +5244,12 @@ int ObLSTabletService::table_refresh_row( LOG_WARN("push col id failed.", K(ret), K(i)); } } - ObDatumRowkey datum_rowkey; - ObDatumRowkeyHelper rowkey_helper(scan_allocator); - ObSingleRowGetter single_row_getter(scan_allocator, *data_tablet); if (OB_FAIL(ret)) { - } else if (OB_FAIL(init_single_row_getter(single_row_getter, - run_ctx, out_col_ids, data_table, true))) { - LOG_WARN("failed to init single row getter", K(ret)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - LOG_WARN("Failed to transfer datum rowkey", K(ret), K(rowkey)); } else { - ObNewRow *new_row = nullptr; - if (OB_FAIL(single_row_getter.open(datum_rowkey))) { - LOG_WARN("init single row getter failed", K(ret)); - } else if (OB_FAIL(single_row_getter.get_next_row(new_row))) { + ObDatumRow *new_row = nullptr; + if (OB_FAIL(get_storage_row(datum_row, out_col_ids, storage_row_getter, run_ctx, new_row))) { if (ret == OB_ITER_END) { - LOG_DEBUG("re-read old row not exist", K(ret), K(row)); + LOG_DEBUG("re-read old row not exist", K(ret), K(datum_row)); ret = OB_SUCCESS; } else { LOG_WARN("get next row from single row getter failed", K(ret)); @@ -5465,18 +5257,16 @@ int ObLSTabletService::table_refresh_row( } else if (OB_ISNULL(new_row)) { ret = OB_ERR_NULL_VALUE; LOG_WARN("get next row from single row null", K(ret)); - } else if (new_row->get_count() != row.get_count()) { + } else if (new_row->count_ != datum_row.count_) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get row from single row col count not equal.", K(ret), K(row.get_count()), K(new_row->get_count())); + LOG_WARN("get row from single row col count not equal.", K(ret), K(datum_row.count_), K(new_row->count_)); } else { - LOG_DEBUG("get new row success.", K(row), KPC(new_row)); + LOG_DEBUG("get new row success.", K(datum_row), KPC(new_row)); // only write cells, not write row - // passing fixed double scale from row to new_row - for (int64_t i = 0; OB_SUCC(ret) && i < new_row->get_count(); ++i) { - if (row.cells_[i].is_fixed_double()) { - new_row->cells_[i].set_scale(row.cells_[i].get_scale()); - } else if (OB_FAIL(ob_write_obj(run_ctx.dml_param_.lob_allocator_, new_row->cells_[i], row.cells_[i]))) { - LOG_WARN("copy ObObj error", K(ret), K(i), K(new_row->cells_[i])); + // TODO：concern about fixed double scale from row to new_row？@xuanxi + for (int64_t i = 0; OB_SUCC(ret) && i < new_row->count_; ++i) { + if (OB_FAIL(datum_row.storage_datums_[i].deep_copy(new_row->storage_datums_[i], run_ctx.dml_param_.lob_allocator_))) { + LOG_WARN("copy storage datum error", K(ret), K(i), K(new_row->storage_datums_[i])); } } if (OB_SUCC(ret)) { @@ -5490,40 +5280,38 @@ int ObLSTabletService::table_refresh_row( int ObLSTabletService::delete_row_in_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - const ObNewRow &row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; const ObDMLBaseParam &dml_param = run_ctx.dml_param_; ObStoreCtx &ctx = run_ctx.store_ctx_; ObRelativeTable &relative_table = run_ctx.relative_table_; - ObStoreRow &tbl_row = run_ctx.tbl_row_; - ObStoreRow new_tbl_row; ObSEArray update_idx; // update_idx is a dummy param here - tbl_row.flag_.set_flag(ObDmlFlag::DF_DELETE); - tbl_row.row_val_ = row; + datum_row.row_flag_.set_flag(ObDmlFlag::DF_DELETE); - if (OB_FAIL(check_old_row_legitimacy(tablet_handle, run_ctx, row))) { + if (OB_FAIL(check_old_row_legitimacy(run_ctx.cmp_funcs_, tablet_handle, run_ctx, datum_row))) { if (OB_ERR_DEFENSIVE_CHECK == ret) { - dump_diag_info_for_old_row_loss(relative_table, ctx, tbl_row); + dump_diag_info_for_old_row_loss(run_ctx, datum_row); } - LOG_WARN("check old row legitimacy failed", K(row)); - } else if (OB_FAIL(process_old_row_lob_col(tablet_handle, run_ctx, tbl_row))) { - LOG_WARN("failed to process old row lob col", K(ret), K(tbl_row)); - } else if (OB_FAIL(delete_lob_tablet_rows(run_ctx, tablet_handle, tbl_row, row))) { - LOG_WARN("failed to delete lob rows.", K(ret), K(tbl_row), K(row)); + LOG_WARN("check old row legitimacy failed", K(ret), K(datum_row)); + } else if (OB_FAIL(process_old_row_lob_col(tablet_handle, run_ctx, datum_row))) { + LOG_WARN("failed to process old row lob col", K(ret), K(datum_row)); + } else if (OB_FAIL(delete_lob_tablet_rows(run_ctx, tablet_handle, datum_row))) { + LOG_WARN("failed to delete lob rows.", K(ret), K(datum_row)); } else { update_idx.reset(); // update_idx is a dummy param here - new_tbl_row.reset(); - new_tbl_row.flag_.set_flag(ObDmlFlag::DF_DELETE); - new_tbl_row.row_val_ = tbl_row.row_val_; - tbl_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); - if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, ctx, - *run_ctx.col_descs_, update_idx, tbl_row, new_tbl_row, dml_param.encrypt_meta_))) { + ObDatumRow new_datum_row; + datum_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE); + if (OB_FAIL(new_datum_row.shallow_copy(datum_row))) { + LOG_WARN("failed to shallow copy datum row", K(ret), K(datum_row), K(new_datum_row)); + } else if (FALSE_IT(new_datum_row.row_flag_.set_flag(ObDmlFlag::DF_DELETE))) { + } else if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, ctx, + *run_ctx.col_descs_, update_idx, datum_row, new_datum_row, dml_param.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { - LOG_WARN("failed to set row", K(ret), K(*run_ctx.col_descs_), K(tbl_row), K(new_tbl_row)); + LOG_WARN("failed to set row", K(ret), K(*run_ctx.col_descs_), K(datum_row), K(new_datum_row)); } } else { - LOG_DEBUG("succeeded to del main table row", K(tbl_row), K(new_tbl_row)); + LOG_DEBUG("succeeded to del main table row", K(datum_row), K(new_datum_row)); } } @@ -5533,8 +5321,7 @@ int ObLSTabletService::delete_row_in_tablet( int ObLSTabletService::delete_lob_col( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &obj, - const ObObj &sql_obj, + blocksstable::ObStorageDatum &datum, ObLobCommon *&lob_common, ObLobAccessParam &lob_param) { @@ -5543,17 +5330,15 @@ int ObLSTabletService::delete_lob_col( if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]get lob manager instance failed.", K(ret)); - } else if (!column.col_type_.is_lob_storage() || obj.is_nop_value() || obj.is_null() || + } else if (!column.col_type_.is_lob_storage() || datum.is_nop_value() || datum.is_null() || !run_ctx.is_old_row_valid_for_lob_) { // do nothing } else { - ObString data = obj.get_string(); - // Notice: Only disk locator here! - ObString sql_data = sql_obj.get_string(); - ObLobLocatorV2 locator(data, obj.has_lob_header()); + ObString data = datum.get_string(); + ObLobLocatorV2 locator(data, datum.has_lob_header()); if (data.length() < sizeof(ObLobCommon)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("[STORAGE_LOB]Invalid Lob data.", K(ret), K(obj), K(data)); + LOG_WARN("[STORAGE_LOB]Invalid Lob data.", K(ret), K(datum), K(data)); } else if (locator.is_inrow()) { // deelete inrow lob no need to use the lob manager } else if (OB_FAIL(set_lob_storage_params(run_ctx, column, lob_param))) { @@ -5599,14 +5384,13 @@ int ObLSTabletService::delete_lob_col( int ObLSTabletService::delete_lob_tablet_rows( ObDMLRunningCtx &run_ctx, ObTabletHandle &data_tablet, - ObStoreRow &tbl_row, - const ObNewRow &row) + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; int64_t col_cnt = run_ctx.col_descs_->count(); - if (tbl_row.row_val_.count_ != col_cnt) { + if (datum_row.count_ != col_cnt) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("[STORAGE_LOB]Invliad row col cnt", K(col_cnt), K(tbl_row)); + LOG_WARN("[STORAGE_LOB]Invliad row col cnt", K(col_cnt), K(datum_row)); } else if (OB_FAIL(update_lob_meta_table_seq_no(run_ctx, 1/*row_count*/))) { LOG_WARN("update_lob_meta_table_seq_no fail", K(ret), K(run_ctx.dml_param_)); } else { @@ -5614,17 +5398,17 @@ int ObLSTabletService::delete_lob_tablet_rows( for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { const ObColDesc &column = run_ctx.col_descs_->at(i); if (column.col_type_.is_lob_storage()) { - ObObj &obj = tbl_row.row_val_.get_cell(i); - const ObObj &sql_obj = row.get_cell(i); + blocksstable::ObStorageDatum &datum = datum_row.storage_datums_[i]; ObLobAccessParam lob_param; - if (OB_FAIL(delete_lob_col(run_ctx, column, obj, sql_obj, lob_common, lob_param))) { - LOG_WARN("[STORAGE_LOB]failed to erase lob col.", K(ret), K(i), K(tbl_row)); + if (OB_FAIL(delete_lob_col(run_ctx, column, datum, lob_common, lob_param))) { + LOG_WARN("[STORAGE_LOB]failed to erase lob col.", K(ret), K(i), K(datum_row)); } } } } return ret; } +// revert end int ObLSTabletService::prepare_scan_table_param( ObTableScanParam ¶m, @@ -5668,16 +5452,17 @@ int ObLSTabletService::prepare_scan_table_param( } void ObLSTabletService::dump_diag_info_for_old_row_loss( - ObRelativeTable &data_table, - ObStoreCtx &store_ctx, - const ObStoreRow &tbl_row) + ObDMLRunningCtx &run_ctx, + blocksstable::ObDatumRow &datum_row) { int ret = OB_SUCCESS; + ObStoreCtx &store_ctx = run_ctx.store_ctx_; + ObRelativeTable &data_table = run_ctx.relative_table_; + ObColDescIArray &col_descs = const_cast(*run_ctx.col_descs_); ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "DumpDIAGInfo")); ObTableAccessParam access_param; ObTableAccessContext access_ctx; ObSEArray out_col_pros; - ObStoreRowkey rowkey; ObDatumRowkey datum_rowkey; ObDatumRowkeyHelper rowkey_helper(allocator); const int64_t schema_rowkey_cnt = data_table.get_rowkey_column_num(); @@ -5704,10 +5489,8 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( } if (OB_FAIL(ret)) { - } else if (OB_FAIL(rowkey.assign(tbl_row.row_val_.cells_, schema_rowkey_cnt))) { - LOG_WARN("Failed to assign rowkey", K(ret)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - STORAGE_LOG(WARN, "Failed to transfer datum rowkey", K(ret), K(rowkey)); + } else if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(datum_row, schema_rowkey_cnt, col_descs, datum_rowkey))) { + LOG_WARN("Failed to prepare rowkey", K(ret), K(datum_row), K(datum_rowkey)); } else if (OB_FAIL(access_ctx.init(query_flag, store_ctx, allocator, trans_version_rang))) { LOG_WARN("Fail to init access ctx", K(ret)); } else { @@ -5724,8 +5507,8 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( ObITable *table = nullptr; const ObDatumRow *row = nullptr; - FLOG_INFO("Try to find the specified rowkey within all the sstable", K(tbl_row), K(table_iter)); - FLOG_INFO("Prepare the diag env to dump the rows", K(store_ctx), K(rowkey), K(datum_rowkey), + FLOG_INFO("Try to find the specified rowkey within all the sstable", K(datum_row), K(table_iter)); + FLOG_INFO("Prepare the diag env to dump the rows", K(store_ctx), K(datum_rowkey), K(access_ctx.trans_version_range_)); table_iter.resume(); @@ -5784,7 +5567,7 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( } if (OB_SUCC(ret)) { - FLOG_INFO("prepare to use single merge to find row", K(rowkey), K(datum_rowkey), K(access_param)); + FLOG_INFO("prepare to use single merge to find row", K(datum_rowkey), K(access_param)); ObSingleMerge *get_merge = nullptr; ObGetTableParam get_table_param; ObDatumRow *row = nullptr; @@ -5804,7 +5587,7 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( while (OB_SUCC(get_merge->get_next_row(row))) { FLOG_INFO("Found one row for the rowkey", KPC(row)); } - FLOG_INFO("Finish to find rowkey with single merge", K(ret), K(rowkey), K(datum_rowkey)); + FLOG_INFO("Finish to find rowkey with single merge", K(ret), K(datum_rowkey)); } if (OB_NOT_NULL(get_merge)) { get_merge->~ObSingleMerge(); diff --git a/src/storage/ls/ob_ls_tablet_service.h b/src/storage/ls/ob_ls_tablet_service.h index 65ca56856..3bbc572c8 100644 --- a/src/storage/ls/ob_ls_tablet_service.h +++ b/src/storage/ls/ob_ls_tablet_service.h @@ -33,6 +33,7 @@ #include "storage/tablet/ob_tablet_persister.h" #include "storage/lob/ob_lob_manager.h" #include "storage/multi_data_source/mds_table_mgr.h" +#include "storage/blocksstable/ob_datum_row_iterator.h" namespace oceanbase { @@ -57,12 +58,18 @@ namespace blocksstable { class ObMigrationSSTableParam; struct ObDatumRowkey; +class ObDatumRowStore; } namespace compaction { class ObTabletMergeCtx; } +namespace sql +{ +class ObDASDMLIterator; +class ObDASUpdIterator; +} namespace storage { @@ -317,7 +324,7 @@ public: ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int insert_row( ObTabletHandle &tablet_handle, @@ -325,31 +332,31 @@ public: const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray &duplicated_column_ids, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObInsertFlag flag, int64_t &affected_rows, - common::ObNewRowIterator *&duplicated_rows); + blocksstable::ObDatumRowIterator *&duplicated_rows); int update_rows( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, const ObIArray< uint64_t> &updated_column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int put_rows( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + ObDatumRowIterator *row_iter, int64_t &affected_rows); // for htable, insert or update int delete_rows( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, const ObIArray &column_ids, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int lock_rows( ObTabletHandle &tablet_handle, @@ -357,13 +364,13 @@ public: const ObDMLBaseParam &dml_param, const ObLockFlag lock_flag, const bool is_sfu, - ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int lock_row( ObTabletHandle &tablet_handle, ObStoreCtx &ctx, const ObDMLBaseParam &dml_param, - const ObNewRow &row, + blocksstable::ObDatumRow &row, const ObLockFlag lock_flag, const bool is_sfu); int get_multi_ranges_cost( @@ -553,7 +560,7 @@ private: int offline_build_tablet_without_memtable_(); int offline_gc_tablet_for_create_or_transfer_in_abort_(); int offline_destroy_memtable_and_mds_table_(); - int mock_duplicated_rows_(common::ObNewRowIterator *&duplicated_rows); + int mock_duplicated_rows_(blocksstable::ObDatumRowIterator *&duplicated_rows); private: static int check_real_leader_for_4377_(const ObLSID ls_id); static int check_need_rollback_in_transfer_for_4377_(const transaction::ObTxDesc *tx_desc, @@ -562,50 +569,42 @@ private: static int build_create_sstable_param_for_migration( const blocksstable::ObMigrationSSTableParam &migrate_sstable_param, ObTabletCreateSSTableParam &create_sstable_param); - static int need_check_old_row_legitimacy( - ObDMLRunningCtx &run_ctx, - bool &need_check, - bool &is_udf); - static int construct_table_rows( - const ObNewRow *rows, - ObStoreRow *tbl_rows, - int64_t row_count); static int check_old_row_legitimacy( + const blocksstable::ObStoreCmpFuncs &cmp_funcs, ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - const common::ObNewRow &old_row); + blocksstable::ObDatumRow &old_row); static int check_new_row_legitimacy( ObDMLRunningCtx &run_ctx, - const common::ObNewRow &new_row); + const blocksstable::ObDatumRow &datum_row); static int insert_rows_to_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - const common::ObNewRow *const rows, + blocksstable::ObDatumRow *rows, const int64_t row_count, ObRowsInfo &rows_info, - storage::ObStoreRow *tbl_rows, int64_t &afct_num, int64_t &dup_num); static int insert_tablet_rows( const int64_t row_count, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + blocksstable::ObDatumRow *rows, ObRowsInfo &rows_info); static int insert_lob_col( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &obj, + blocksstable::ObStorageDatum &datum, ObLobAccessParam *del_param, ObLobCommon *lob_common); static int insert_lob_tablet_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, - ObStoreRow &row); + blocksstable::ObDatumRow &datum_row); static int insert_lob_tablet_rows( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, - ObStoreRow *rows, + blocksstable::ObDatumRow *rows, int64_t row_count); static int extract_rowkey( const ObRelativeTable &table, @@ -620,8 +619,8 @@ private: const int64_t buffer_len, const common::ObTimeZoneInfo *tz_info = nullptr); static int get_next_rows( - ObNewRowIterator *row_iter, - ObNewRow *&rows, + blocksstable::ObDatumRowIterator *row_iter, + blocksstable::ObDatumRow *&rows, int64_t &row_count); static int construct_update_idx( const int64_t schema_rowkey_cnt, @@ -634,19 +633,21 @@ private: bool &rowkey_change, bool &delay_new); static int check_rowkey_value_change( - const common::ObNewRow &old_row, - const common::ObNewRow &new_row, + const ObDatumRow &old_row, + const ObDatumRow &new_row, const int64_t rowkey_len, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils, bool &rowkey_change); static int process_delta_lob( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &old_obj, + ObStorageDatum &old_datum, ObLobLocatorV2 &delta_lob, - ObObj &obj); + ObStorageDatum &datum); static int register_ext_info_commit_cb( ObDMLRunningCtx &run_ctx, - ObObj &col_data, + ObStorageDatum &col_data, + ObObjType type, ObObj &ext_info_data); static int set_lob_storage_params( ObDMLRunningCtx &run_ctx, @@ -658,9 +659,8 @@ private: ObDMLRunningCtx &run_ctx, const ObIArray &update_idx, bool data_tbl_rowkey_change, - ObStoreRow &old_sql_row, - ObStoreRow &old_row, - ObStoreRow &new_row); + ObDatumRow &old_row, + ObDatumRow &new_row); static int update_row_to_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, @@ -668,42 +668,39 @@ private: const ObIArray &update_idx, const bool delay_new, const bool lob_update, - ObStoreRow &old_tbl_row, - ObStoreRow &new_tbl_row, - ObRowStore *row_store, + ObDatumRow &old_row, + ObDatumRow &new_row, + ObDatumRowStore *row_store, bool &duplicate); static int process_old_row( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, const bool data_tbl_rowkey_change, const bool lob_update, - ObStoreRow &tbl_row); + ObDatumRow &datum_row); static int process_new_row( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, const common::ObIArray &update_idx, - const ObStoreRow &old_tbl_row, - const ObStoreRow &new_tbl_row, + const ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, const bool rowkey_change); static int process_data_table_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, const ObIArray &update_idx, - const ObStoreRow &old_tbl_row, - const ObStoreRow &new_tbl_row, + const ObDatumRow &old_datum_row, + ObDatumRow &new_datum_row, const bool rowkey_change); - static int check_new_row_nullable_value( - const ObIArray &column_ids, - ObRelativeTable &data_table, - const ObNewRow &new_row); - static int check_new_row_nullable_value( + static int check_datum_row_nullable_value( const common::ObIArray &col_descs, ObRelativeTable &relative_table, - const common::ObNewRow &new_row); - static int check_new_row_shadow_pk( + const blocksstable::ObDatumRow &datum_row); + static int check_datum_row_shadow_pk( const ObIArray &column_ids, ObRelativeTable &data_table, - const ObNewRow &new_row); + const blocksstable::ObDatumRow &datum_row, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils); static int check_row_locked_by_myself( ObTabletHandle &tablet_handle, ObRelativeTable &relative_table, @@ -715,61 +712,55 @@ private: ObDMLRunningCtx &run_ctx, const ObInsertFlag flag, const common::ObIArray &out_col_ids, - const common::ObNewRow &row, - common::ObNewRowIterator *&duplicated_rows); + blocksstable::ObDatumRow &row, + blocksstable::ObDatumRowIterator *&duplicated_rows); static int init_single_row_getter( ObSingleRowGetter &row_getter, ObDMLRunningCtx &run_ctx, const ObIArray &out_col_ids, ObRelativeTable &relative_table, bool skip_read_lob = false); - static int single_get_row( - ObSingleRowGetter &row_getter, - const blocksstable::ObDatumRowkey &rowkey, - ObNewRowIterator *&duplicated_rows); - static int convert_row_to_rowkey( - ObSingleRowGetter &index_row_getter, - ObStoreRowkey &rowkey); + static int add_duplicate_row( + ObDatumRow *storage_row, + const blocksstable::ObStorageDatumUtils &rowkey_datum_utils, + blocksstable::ObDatumRowIterator *&duplicated_rows); static int get_next_row_from_iter( - ObNewRowIterator *row_iter, - ObStoreRow &store_row, + blocksstable::ObDatumRowIterator *row_iter, + ObDatumRow &datum_row, const bool need_copy_cells); static int insert_row_to_tablet( const bool check_exist, ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow &tbl_row); - static int process_old_row_lob_col( + blocksstable::ObDatumRow &datum_row); + static int process_old_row_lob_col( ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - ObStoreRow &tbl_row); + blocksstable::ObDatumRow &datum_row); static int table_refresh_row( ObTabletHandle &data_tablet_handle, ObDMLRunningCtx &run_ctx, - ObNewRow &row); + blocksstable::ObDatumRow &datum_row); static int delete_row_in_tablet( ObTabletHandle &tablet_handle, ObDMLRunningCtx &run_ctx, - const ObNewRow &row); + blocksstable::ObDatumRow &datum_row); static int delete_lob_col( ObDMLRunningCtx &run_ctx, const ObColDesc &column, - ObObj &obj, - const ObObj &sql_obj, + blocksstable::ObStorageDatum &datum, ObLobCommon *&lob_common, ObLobAccessParam &lob_param); static int delete_lob_tablet_rows( ObDMLRunningCtx &run_ctx, ObTabletHandle &data_tablet, - ObStoreRow &tbl_row, - const ObNewRow &row); + blocksstable::ObDatumRow &datum_row); static int prepare_scan_table_param( ObTableScanParam ¶m, share::schema::ObMultiVersionSchemaService &schema_service); static void dump_diag_info_for_old_row_loss( - ObRelativeTable &data_table, - ObStoreCtx &store_ctx, - const ObStoreRow &tbl_row); + ObDMLRunningCtx &run_ctx, + blocksstable::ObDatumRow &datum_row); int set_allow_to_read_(ObLS *ls); // TODO(chenqingxiang.cqx): remove this int create_empty_shell_tablet( @@ -786,8 +777,14 @@ private: const int64_t ddl_task_id, const common::ObTabletID &tablet_id, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); + static int get_storage_row(const ObDatumRow &sql_row, + const ObIArray &column_ids, + ObSingleRowGetter &row_getter, + ObDMLRunningCtx &run_ctx, + ObDatumRow *&out_row, + bool use_fuse_row_cache = false); static int check_is_gencol_check_failed(const ObRelativeTable &data_table, uint64_t error_col_id, bool &is_virtual_gencol); diff --git a/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp index ff35ce2bd..33b297f5e 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp @@ -269,12 +269,13 @@ int ObIMvccCtx::register_ext_info_commit_cb( const blocksstable::ObDmlFlag dml_flag, transaction::ObTxDesc *tx_desc, transaction::ObTxSEQ &parent_seq_no, - ObObj &index_data, + blocksstable::ObStorageDatum &index_data, + ObObjType &type, ObObj &ext_info_data) { int ret = OB_SUCCESS; storage::ObExtInfoCbRegister cb_register; - if (OB_FAIL(cb_register.register_cb(this, timeout, dml_flag, tx_desc, parent_seq_no, index_data, ext_info_data))) { + if (OB_FAIL(cb_register.register_cb(this, timeout, dml_flag, tx_desc, parent_seq_no, index_data, type, ext_info_data))) { TRANS_LOG(ERROR, "register ext info callback failed", K(ret), K(cb_register), K(*this)); } return ret; diff --git a/src/storage/memtable/mvcc/ob_mvcc_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_ctx.h index b5486a246..c52ce42fe 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_ctx.h @@ -178,7 +178,8 @@ public: const blocksstable::ObDmlFlag dml_flag, transaction::ObTxDesc *tx_desc, transaction::ObTxSEQ &parent_seq_no, - ObObj &index_data, + blocksstable::ObStorageDatum &index_data, + ObObjType &type, ObObj &ext_info_data); public: virtual void reset() diff --git a/src/storage/memtable/ob_concurrent_control.cpp b/src/storage/memtable/ob_concurrent_control.cpp index 2012f0609..1ddd00b5b 100644 --- a/src/storage/memtable/ob_concurrent_control.cpp +++ b/src/storage/memtable/ob_concurrent_control.cpp @@ -68,7 +68,7 @@ int check_sequence_set_violation(const concurrent_control::ObWriteFlag write_fla ret = OB_ERR_PRIMARY_KEY_DUPLICATE; TRANS_LOG(WARN, "pdml duplicate primary key found", K(ret), K(writer_tx_id), K(writer_dml_flag), K(writer_seq_no), - K(locker_tx_id), K(locker_dml_flag), K(locker_seq_no)); + K(locker_tx_id), K(locker_dml_flag), K(locker_seq_no), K(reader_seq_no)); // Case 2.1: For the case of the update in the storage layer, it may be // split into lock and update in a single statement and fail the check, so // we need bypass this case(Currently only the update of the lob will cause diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 837cd1d8b..668857797 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -373,7 +373,7 @@ int ObMemtable::multi_set( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, const common::ObIArray &columns, - const storage::ObStoreRow *rows, + blocksstable::ObDatumRow *rows, const int64_t row_count, const bool check_exist, const share::ObEncryptMeta *encrypt_meta, @@ -381,7 +381,6 @@ int ObMemtable::multi_set( { int ret = OB_SUCCESS; ObMvccWriteGuard guard(ret); - ObMemtableKeyGenerator mtk_generator; if (IS_NOT_INIT) { TRANS_LOG(WARN, "Not inited", K(*this)); ret = OB_NOT_INIT; @@ -389,7 +388,7 @@ int ObMemtable::multi_set( ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "Invalid argument", K(ret), K(param), K(context)); } else if (OB_UNLIKELY(nullptr == context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx() - || param.get_schema_rowkey_count() > columns.count())) { + || param.get_schema_rowkey_count() > columns.count())) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "Invalid param", K(ret), K(param), K(columns.count())); #ifdef OB_BUILD_TDE_SECURITY @@ -400,8 +399,6 @@ int ObMemtable::multi_set( } else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) { TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret)); #endif - } else if (OB_FAIL(mtk_generator.init(rows, row_count, param.get_schema_rowkey_count(), columns))) { - TRANS_LOG(WARN, "fail to generate memtable keys", KPC(encrypt_meta), K(*context.store_ctx_), KR(ret)); } if (OB_FAIL(ret)) { @@ -410,17 +407,22 @@ int ObMemtable::multi_set( } else { lib::CompatModeGuard compat_guard(mode_); if (row_count > 1) { - ret = multi_set_(param, columns, rows, row_count, check_exist, mtk_generator, context, rows_info); + ret = multi_set_(param, columns, rows, row_count, check_exist, context, rows_info); } else { - ret = set_(param, - columns, - rows[0], - nullptr, /*old_row*/ - nullptr, /*update_idx*/ - mtk_generator[0], - check_exist, - context, - nullptr /*mvcc_row*/); + ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns); + if (OB_FAIL(memtable_key_generator.init())) { + TRANS_LOG(WARN, "fail to init memtable key generator", K(ret)); + } else { + ret = set_(param, + columns, + rows[0], + nullptr, /*old_row*/ + nullptr, /*update_idx*/ + check_exist, + context, + memtable_key_generator, + nullptr /*mvcc_row*/); + } } guard.set_memtable(this); } @@ -529,24 +531,23 @@ int ObMemtable::set( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, const common::ObIArray &columns, - const storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, const share::ObEncryptMeta *encrypt_meta, const bool check_exist) { int ret = OB_SUCCESS; ObMvccWriteGuard guard(ret); - ObMemtableKeyGenerator mtk_generator; if (IS_NOT_INIT) { TRANS_LOG(WARN, "not init", K(*this)); ret = OB_NOT_INIT; } else if (!param.is_valid() || !context.is_valid()) { - ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument, ", K(ret), K(param), K(context)); + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid argument, ", K(ret), K(param), K(context)); } else if (NULL == context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx() - || param.get_schema_rowkey_count() > columns.count() - || row.row_val_.count_ < columns.count()) { + || param.get_schema_rowkey_count() > columns.count() + || row.count_ < columns.count()) { TRANS_LOG(WARN, "invalid param", K(param), - K(columns.count()), K(row.row_val_.count_)); + K(columns.count()), K(row.count_)); ret = OB_INVALID_ARGUMENT; #ifdef OB_BUILD_TDE_SECURITY //TODO: table_id is just used as encrypt_index, we may rename it in the future. @@ -556,32 +557,28 @@ int ObMemtable::set( } else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) { TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret)); #endif - } else if (OB_FAIL(mtk_generator.init(&row, 1, param.get_schema_rowkey_count(), columns))) { - TRANS_LOG(WARN, "fail to generate memtable keys", KPC(encrypt_meta), K(*context.store_ctx_), KR(ret)); } if (OB_FAIL(ret)) { } else if (OB_FAIL(guard.write_auth(*context.store_ctx_))) { TRANS_LOG(WARN, "not allow to write", K(*context.store_ctx_)); } else { - lib::CompatModeGuard compat_guard(mode_); - - ret = set_(param, - columns, - row, - NULL, /*old_row*/ - NULL, /*update_idx*/ - mtk_generator[0], - check_exist, - context, - nullptr /*mvcc_row*/); - guard.set_memtable(this); - } - - if (OB_SUCC(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(try_report_dml_stat_(param.table_id_))) { - TRANS_LOG_RET(WARN, tmp_ret, "fail to report dml stat", K_(reported_dml_stat)); + ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns); + if (OB_FAIL(memtable_key_generator.init())) { + TRANS_LOG(WARN, "fail to init memtable key generator", K(ret)); + } else { + lib::CompatModeGuard compat_guard(mode_); + ret = set_(param, + columns, + row, + NULL, /*old_row*/ + NULL, /*update_idx*/ + check_exist, + context, + memtable_key_generator, + nullptr /*mvcc_row*/); + TRANS_LOG(WARN, "[xuanxi] set row", K(ret), K(row), K(check_exist), K(memtable_key_generator.get_memtable_key())); + guard.set_memtable(this); } } return ret; @@ -592,13 +589,12 @@ int ObMemtable::set( storage::ObTableAccessContext &context, const ObIArray &columns, const ObIArray &update_idx, - const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row, + const blocksstable::ObDatumRow &old_row, + blocksstable::ObDatumRow &new_row, const share::ObEncryptMeta *encrypt_meta) { int ret = OB_SUCCESS; ObMvccWriteGuard guard(ret); - ObMemtableKeyGenerator mtk_generator; if (IS_NOT_INIT) { TRANS_LOG(WARN, "not init", K(*this)); ret = OB_NOT_INIT; @@ -606,7 +602,7 @@ int ObMemtable::set( ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid argument, ", K(ret), K(param), K(context)); } else if (NULL == context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx() - || param.get_schema_rowkey_count() > columns.count()) { + || param.get_schema_rowkey_count() > columns.count()) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(ERROR, "invalid param", K(ret), K(param)); #ifdef OB_BUILD_TDE_SECURITY @@ -617,32 +613,28 @@ int ObMemtable::set( } else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) { TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret)); #endif - } else if (OB_FAIL(mtk_generator.init(&new_row, 1, param.get_schema_rowkey_count(), columns))) { - TRANS_LOG(WARN, "fail to generate memtable keys", KPC(encrypt_meta), K(*context.store_ctx_), KR(ret)); } if (OB_FAIL(ret)){ } else if (OB_FAIL(guard.write_auth(*context.store_ctx_))) { TRANS_LOG(WARN, "not allow to write", K(*context.store_ctx_)); } else { - lib::CompatModeGuard compat_guard(mode_); + ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns); + if (OB_FAIL(memtable_key_generator.init())) { + TRANS_LOG(WARN, "fail to init memtable key generator", K(ret)); + } else { + lib::CompatModeGuard compat_guard(mode_); - ret = set_(param, - columns, - new_row, - &old_row, - &update_idx, - mtk_generator[0], - false/*check_exist*/, - context, - nullptr /*mvcc_row*/); - guard.set_memtable(this); - } - - if (OB_SUCC(ret)) { - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(try_report_dml_stat_(param.table_id_))) { - TRANS_LOG_RET(WARN, tmp_ret, "fail to report dml stat", K_(reported_dml_stat)); + ret = set_(param, + columns, + new_row, + &old_row, + &update_idx, + false/*check_exist*/, + context, + memtable_key_generator, + nullptr /*mvcc_row*/); + guard.set_memtable(this); } } return ret; @@ -651,10 +643,13 @@ int ObMemtable::set( int ObMemtable::lock( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const common::ObNewRow &row) + ObColDescArray &col_desc, + blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; ObMvccWriteGuard guard(ret); + ObDatumRowkeyHelper rowkey_helper; + ObDatumRowkey datum_rowkey(row.storage_datums_, param.get_schema_rowkey_count()); ObStoreRowkey tmp_key; ObMemtableKey mtk; ObMvccAccessCtx &acc_ctx = context.store_ctx_->mvcc_acc_ctx_; @@ -670,9 +665,9 @@ int ObMemtable::lock( // actually, there is no circumstance in where locking the index table is need. ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "locking the non-unique local index is not supported", K(ret), K(row), K(param)); - } else if (OB_FAIL(tmp_key.assign(row.cells_, param.get_schema_rowkey_count()))) { - TRANS_LOG(WARN, "Failed to assign rowkey", K(row), K(param)); - } else if (OB_FAIL(mtk.encode(param.get_read_info()->get_columns_desc(), &tmp_key))) { + } else if (OB_FAIL(rowkey_helper.convert_store_rowkey(datum_rowkey, col_desc, tmp_key))) { + LOG_WARN("Failed to convert store rowkey from datum rowkey", K(ret), K(row), K(datum_rowkey)); + } else if (OB_FAIL(mtk.encode(col_desc, &tmp_key))) { TRANS_LOG(WARN, "encode mtk failed", K(ret), K(param)); } else if (acc_ctx.write_flag_.is_check_row_locked()) { if (OB_FAIL(ObRowConflictHandler::check_foreign_key_constraint(param, context, tmp_key))) { @@ -1589,7 +1584,6 @@ void ObMemtable::lock_row_on_frozen_stores_on_failure( int ObMemtable::lock_rows_on_frozen_stores_( const bool check_exist, const storage::ObTableIterParam ¶m, - const ObMemtableKeyGenerator &memtable_keys, storage::ObTableAccessContext &context, ObMvccRowAndWriteResults &mvcc_rows, ObRowsInfo &rows_info) @@ -2499,10 +2493,9 @@ bool ObMemtable::need_for_save(const share::ObEncryptMeta *encrypt_meta) int ObMemtable::multi_set_( const storage::ObTableIterParam ¶m, const common::ObIArray &columns, - const storage::ObStoreRow *rows, + const blocksstable::ObDatumRow *rows, const int64_t row_count, const bool check_exist, - const ObMemtableKeyGenerator &memtable_keys, storage::ObTableAccessContext &context, storage::ObRowsInfo &rows_info) { @@ -2511,8 +2504,13 @@ int ObMemtable::multi_set_( int64_t conflict_idx = -1; int64_t row_size_stat = 0; ObMvccRowAndWriteResults mvcc_rows; + // TODO(xuanxi): remove it later + ObMemtableKeyGenerator::ObMemtableKeyBuffer memtable_key_buffer; + ObMemtableKeyGenerator memtable_key_generator(param.get_schema_rowkey_count(), columns, &memtable_key_buffer); if (OB_FAIL(mvcc_rows.prepare_allocate(row_count))) { TRANS_LOG(WARN, "Failed to prepare allocate mvcc rows", K(ret), K(row_count)); + } else if (OB_FAIL(memtable_key_generator.init())) { + TRANS_LOG(WARN, "fail to init memtable key generator", K(ret)); } // 1. Check write conflict in memtables. @@ -2524,9 +2522,9 @@ int ObMemtable::multi_set_( rows[i], nullptr, /*old_row*/ nullptr, /*update_idx*/ - memtable_keys[i], check_exist, context, + memtable_key_generator, &(mvcc_rows[permutation_idx])))) { if (OB_UNLIKELY(OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret)) { TRANS_LOG(WARN, "Failed to insert new row", K(ret), K(i), K(permutation_idx), K(rows[i])); @@ -2553,7 +2551,7 @@ int ObMemtable::multi_set_( // 2. Check uniqueness constraint and write conflict in sstables. if (OB_FAIL(ret)) { } else if (rows_info.all_rows_found()) { - } else if (OB_FAIL(lock_rows_on_frozen_stores_(check_exist, param, memtable_keys, context, mvcc_rows, rows_info))) { + } else if (OB_FAIL(lock_rows_on_frozen_stores_(check_exist, param, context, mvcc_rows, rows_info))) { TRANS_LOG(WARN, "Failed to lock rows on frozen stores", K(ret)); } else if (rows_info.have_conflict()) { conflict_idx = rows_info.get_conflict_idx(); @@ -2579,9 +2577,9 @@ int ObMemtable::multi_set_( if (param.is_non_unique_local_index_) { // no need to detect deadlock for non-unique local index table } else { - for (int64_t idx = 0; idx < memtable_keys.count(); ++idx) { + for (int64_t idx = 0; idx < memtable_key_buffer.count(); ++idx) { MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), - memtable_keys[idx], + memtable_key_buffer.at(idx), context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); } } @@ -2601,12 +2599,12 @@ int ObMemtable::multi_set_( int ObMemtable::set_( const storage::ObTableIterParam ¶m, const common::ObIArray &columns, - const storage::ObStoreRow &new_row, - const storage::ObStoreRow *old_row, + const blocksstable::ObDatumRow &new_row, + const blocksstable::ObDatumRow *old_row, const common::ObIArray *update_idx, - const ObMemtableKey &mtk, const bool check_exist, storage::ObTableAccessContext &context, + ObMemtableKeyGenerator &memtable_key_generator, ObMvccRowAndWriteResult *mvcc_row) { int ret = OB_SUCCESS; @@ -2645,11 +2643,11 @@ int ObMemtable::set_( TRANS_LOG(WARN, "get write seq failed", K(ret)); } else if (OB_FAIL(row_writer.write(param.get_schema_rowkey_count(), new_row, update_idx, buf, len))) { TRANS_LOG(WARN, "Failed to write new row", K(ret), K(new_row)); - } else if (OB_UNLIKELY(new_row.flag_.is_not_exist())) { + } else if (OB_UNLIKELY(new_row.row_flag_.is_not_exist())) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "Unexpected not exist trans node", K(ret), K(new_row)); } else { - ObMemtableData mtd(new_row.flag_.get_dml_flag(), len, buf); + ObMemtableData mtd(new_row.row_flag_.get_dml_flag(), len, buf); ObTxNodeArg arg( ctx.mvcc_acc_ctx_.tx_id_, /*trans id*/ &mtd, /*memtable_data*/ @@ -2657,24 +2655,27 @@ int ObMemtable::set_( init_timestamp_, /*memstore_version*/ write_seq, /*seq_no*/ write_epoch, /*write_epoch*/ - new_row.row_val_.count_ /*column_cnt*/); - if (OB_FAIL(mvcc_write_(param, + new_row.count_ /*column_cnt*/); + if (OB_FAIL(memtable_key_generator.generate_memtable_key(new_row))) { + TRANS_LOG(WARN, "generate memtable key fail", K(ret), K(new_row)); + } else if (OB_FAIL(mvcc_write_(param, context, - &mtk, + memtable_key_generator.get_memtable_key(), arg, check_exist, is_new_locked, + memtable_key_generator.get_key_buffer(), mvcc_row))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret && OB_ERR_PRIMARY_KEY_DUPLICATE != ret) { - TRANS_LOG(WARN, "mvcc write fail", K(mtk), K(ret)); + TRANS_LOG(WARN, "mvcc write fail", K(memtable_key_generator.get_memtable_key()), K(ret)); } } else { TRANS_LOG(DEBUG, "set end, success", "ret", ret, "tablet_id_", key_.tablet_id_, - "dml_flag", new_row.flag_.get_dml_flag(), + "dml_flag", new_row.row_flag_.get_dml_flag(), "columns", strarray(columns), "old_row", to_cstring(old_row), "new_row", to_cstring(new_row), @@ -2711,9 +2712,8 @@ int ObMemtable::set_( //set_end(ctx.mvcc_acc_ctx_, ret); if (OB_SUCC(ret)) { set_max_data_schema_version(ctx.table_version_); - set_max_column_cnt(new_row.row_val_.count_); + set_max_column_cnt(new_row.count_); } - return ret; } @@ -2754,10 +2754,11 @@ int ObMemtable::lock_( rowkey.get_obj_cnt()); /*column_cnt*/ if (OB_FAIL(mvcc_write_(param, context, - &mtk, + mtk, arg, false, /*check_exist*/ is_new_locked, + nullptr, /*memtable_key_buffer*/ nullptr /*mvcc_row*/))) { } else if (OB_UNLIKELY(!is_new_locked)) { TRANS_LOG(DEBUG, "lock twice, no need to store lock trans node"); @@ -2810,10 +2811,11 @@ int ObMemtable::mvcc_replay_(storage::ObStoreCtx &ctx, int ObMemtable::mvcc_write_( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const ObMemtableKey *key, + const ObMemtableKey &key, const ObTxNodeArg &arg, const bool check_exist, bool &is_new_locked, + ObMemtableKeyGenerator::ObMemtableKeyBuffer *memtable_key_buffer, ObMvccRowAndWriteResult *mvcc_row) { int ret = OB_SUCCESS; @@ -2826,13 +2828,13 @@ int ObMemtable::mvcc_write_( SCN snapshot_version = ctx.mvcc_acc_ctx_.get_snapshot_version(); transaction::ObTxSnapshot &snapshot = ctx.mvcc_acc_ctx_.snapshot_; - if (OB_FAIL(mvcc_engine_.create_kv(key, + if (OB_FAIL(mvcc_engine_.create_kv(&key, // is_insert blocksstable::ObDmlFlag::DF_INSERT == arg.data_->dml_flag_, &stored_key, value, is_new_add))) { - TRANS_LOG(WARN, "create kv failed", K(ret), K(arg), K(*key)); + TRANS_LOG(WARN, "create kv failed", K(ret), K(arg), K(key)); } else if (OB_FAIL(mvcc_engine_.mvcc_write(ctx, snapshot, *value, @@ -2840,29 +2842,29 @@ int ObMemtable::mvcc_write_( res))) { if (OB_TRY_LOCK_ROW_CONFLICT == ret) { ret = post_row_write_conflict_(ctx.mvcc_acc_ctx_, - *key, + key, res.lock_state_, value->get_last_compact_cnt(), value->get_total_trans_node_cnt()); } else if (OB_TRANSACTION_SET_VIOLATION == ret) { - mem_ctx->on_tsc_retry(*key, + mem_ctx->on_tsc_retry(key, snapshot_version, value->get_max_trans_version(), value->get_max_trans_id()); } else if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { - mem_ctx->on_key_duplication_retry(*key); + mem_ctx->on_key_duplication_retry(key); } else { TRANS_LOG(WARN, "mvcc write fail", K(ret)); } } else if (nullptr == mvcc_row && OB_FAIL(lock_row_on_frozen_stores_(param, arg, - key, + &key, check_exist, context, value, res))) { lock_row_on_frozen_stores_on_failure(arg.data_->dml_flag_, - *key, + key, ret, value, context, @@ -2890,6 +2892,8 @@ int ObMemtable::mvcc_write_( (void)mvcc_engine_.mvcc_undo(value); res.is_mvcc_undo_ = true; TRANS_LOG(WARN, "register row commit failed", K(ret)); + } else if (nullptr != memtable_key_buffer && OB_FAIL(memtable_key_buffer->push_back(stored_key))) { + TRANS_LOG(WARN, "push back stored memtable key into buffer failed", K(ret)); } else if (nullptr == mvcc_row && res.has_insert()) { (void)mvcc_engine_.finish_kv(res); /*****[for deadlock]*****/ @@ -2898,7 +2902,7 @@ int ObMemtable::mvcc_write_( // no need to detect deadlock for non-unique local index table } else { MTL(ObLockWaitMgr*)->set_hash_holder(key_.get_tablet_id(), - *key, + key, context.store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_tx_id()); } /***********************/ diff --git a/src/storage/memtable/ob_memtable.h b/src/storage/memtable/ob_memtable.h index dd05d9922..27668e588 100644 --- a/src/storage/memtable/ob_memtable.h +++ b/src/storage/memtable/ob_memtable.h @@ -245,22 +245,22 @@ public: // derived from ObITable const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, const common::ObIArray &columns, // TODO: remove columns - const storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, const share::ObEncryptMeta *encrypt_meta, const bool check_exist); virtual int set( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const common::ObIArray &columns, // TODO: remove columns + const common::ObIArray &columns, const ObIArray &update_idx, - const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row, + const blocksstable::ObDatumRow &old_row, + blocksstable::ObDatumRow &new_row, const share::ObEncryptMeta *encrypt_meta); int multi_set( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, const common::ObIArray &columns, - const storage::ObStoreRow *rows, + blocksstable::ObDatumRow *rows, const int64_t row_count, const bool check_exist, const share::ObEncryptMeta *encrypt_meta, @@ -287,7 +287,8 @@ public: // derived from ObITable virtual int lock( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const common::ObNewRow &row); + ObColDescArray &col_desc, + blocksstable::ObDatumRow &row); virtual int lock( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, @@ -476,10 +477,11 @@ private: int mvcc_write_( const storage::ObTableIterParam ¶m, storage::ObTableAccessContext &context, - const ObMemtableKey *key, + const ObMemtableKey &key, const ObTxNodeArg &arg, const bool check_exist, bool &is_new_locked, + ObMemtableKeyGenerator::ObMemtableKeyBuffer *memtable_key_buffer, ObMvccRowAndWriteResult *mvcc_row = nullptr); int mvcc_replay_(storage::ObStoreCtx &ctx, @@ -513,7 +515,6 @@ private: int lock_rows_on_frozen_stores_( const bool check_exist, const storage::ObTableIterParam ¶m, - const ObMemtableKeyGenerator &memtable_keys, storage::ObTableAccessContext &context, ObMvccRowAndWriteResults &mvcc_rows, ObRowsInfo &rows_info); @@ -544,20 +545,19 @@ private: int set_( const storage::ObTableIterParam ¶m, const common::ObIArray &columns, - const storage::ObStoreRow &new_row, - const storage::ObStoreRow *old_row, + const blocksstable::ObDatumRow &new_row, + const blocksstable::ObDatumRow *old_row, const common::ObIArray *update_idx, - const ObMemtableKey &mtk, const bool check_exist, storage::ObTableAccessContext &context, + ObMemtableKeyGenerator &memtable_key_generator, ObMvccRowAndWriteResult *mvcc_row = nullptr); int multi_set_( const storage::ObTableIterParam ¶m, const common::ObIArray &columns, - const storage::ObStoreRow *rows, + const blocksstable::ObDatumRow *rows, const int64_t row_count, const bool check_exist, - const ObMemtableKeyGenerator &memtable_keys, storage::ObTableAccessContext &context, storage::ObRowsInfo &rows_info); int lock_( diff --git a/src/storage/memtable/ob_memtable_key.cpp b/src/storage/memtable/ob_memtable_key.cpp index 847d70e77..f80bede5c 100644 --- a/src/storage/memtable/ob_memtable_key.cpp +++ b/src/storage/memtable/ob_memtable_key.cpp @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 OceanBase + * Copyright (c) 2024 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: @@ -11,110 +11,56 @@ */ #include "ob_memtable_key.h" -#include "lib/ob_errno.h" -#include "rowkey/ob_rowkey.h" -#include "share/rc/ob_tenant_base.h" -#include "storage/ob_i_store.h" namespace oceanbase { namespace memtable { - -constexpr int64_t ObMemtableKeyGenerator::STACK_BUFFER_SIZE; - -int ObMemtableKeyGenerator::init(const storage::ObStoreRow *rows, - const int64_t row_count, - const int64_t schema_rowkey_count, - const common::ObIArray &columns) +int ObMemtableKeyGenerator::init() { int ret = OB_SUCCESS; - if (size_ != 0) { + if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; + TRANS_LOG(WARN, "init ObMemtableKeyGenerator twice", K(ret)); + } else if (columns_.count() < rowkey_cnt_) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "rowkey number mismatched", K(columns_.count()), K(rowkey_cnt_)); + } else if (OB_FAIL(obj_buf_.init(&allocator_))) { + TRANS_LOG(WARN, "init obj buffer fail", K(ret)); + } else if (OB_FAIL(obj_buf_.reserve(rowkey_cnt_))) { + TRANS_LOG(WARN, "reserve obj buffer fail", K(ret), K(rowkey_cnt_)); } else { - int64_t extra_size = row_count - STACK_BUFFER_SIZE; - if (extra_size > 0) { - if (FALSE_IT(p_extra_store_row_keys_ = - (ObStoreRowkey *)share::mtl_malloc(extra_size * (sizeof(ObStoreRowkey)), "MemTableKey"))) { - } else if (OB_ISNULL(p_extra_store_row_keys_)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "Failed to alloc ObStoreRowkey memory", K(ret), K(row_count), K(schema_rowkey_count), K(columns), KP(MTL_CTX()), - KP(lib::ObMallocAllocator::get_instance()), KP(p_extra_store_row_keys_)); - } else if (FALSE_IT(p_extra_memtable_keys_ = - (ObMemtableKey *)share::mtl_malloc(extra_size * (sizeof(ObMemtableKey)), "MemTableKey"))) { - } else if (OB_ISNULL(p_extra_memtable_keys_)) { - share::mtl_free(p_extra_store_row_keys_); - p_extra_store_row_keys_ = nullptr; - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "Failed to alloc ObMemtableKey memory", K(ret), K(row_count), K(schema_rowkey_count), K(columns), KP(MTL_CTX()), - KP(lib::ObMallocAllocator::get_instance()), KP(p_extra_store_row_keys_)); + is_inited_ = true; + } + return ret; +} + +int ObMemtableKeyGenerator::generate_memtable_key(const blocksstable::ObDatumRow &row) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObMemtableKeyGenerator not inited", K(ret)); + } else if (OB_UNLIKELY(!row.is_valid())) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid argument", K(ret), K(row)); + } else { + ObObj *objs = obj_buf_.get_data(); + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_cnt_; i++) { + if (OB_FAIL(row.storage_datums_[i].to_obj_enhance(objs[i], columns_.at(i).col_type_))) { + TRANS_LOG(WARN, "failed to transfer datum to obj", K(ret), K(i), K(row)); } } - for (int i = 0; i < row_count && OB_SUCC(ret); ++i) { - ObStoreRowkey *p_store_row_key = i < STACK_BUFFER_SIZE ? &store_row_key_buffer_[i] : &p_extra_store_row_keys_[i - STACK_BUFFER_SIZE]; - ObMemtableKey *p_memtable_key = i < STACK_BUFFER_SIZE ? &memtable_key_buffer_[i] : &p_extra_memtable_keys_[i - STACK_BUFFER_SIZE]; - new (p_store_row_key) ObStoreRowkey(); - new (p_memtable_key) ObMemtableKey(); - if (OB_FAIL(p_store_row_key->assign(rows[i].row_val_.cells_, schema_rowkey_count))) { - p_store_row_key->~ObStoreRowkey(); - TRANS_LOG(WARN, "Failed to assign tmp rowkey", K(ret), K(rows[i]), K(row_count), K(schema_rowkey_count)); - } else if (OB_FAIL(p_memtable_key->encode(columns, p_store_row_key))) { - p_memtable_key->~ObMemtableKey(); - p_store_row_key->~ObStoreRowkey(); - TRANS_LOG(WARN, "mtk encode fail", K(ret), K(rows[i]), K(row_count), K(schema_rowkey_count)); - } else { - size_++; + if (OB_SUCC(ret)) { + if (OB_FAIL(store_rowkey_.assign(objs, rowkey_cnt_))) { + TRANS_LOG(WARN, "failed to assign rowkey", K(ret), K(row), K(objs), K(rowkey_cnt_)); + } else if (OB_FAIL(memtable_key_.encode(columns_, &store_rowkey_))) { + TRANS_LOG(WARN, "memtable key encode failed", K(ret), K(row), K(columns_), K(store_rowkey_)); } } - if (OB_FAIL(ret)) { - reset(); - } } return ret; } -ObMemtableKey &ObMemtableKeyGenerator::operator[](int64_t idx) { - ObMemtableKey *element = nullptr; - if (OB_UNLIKELY(idx < 0 || idx >= size_)) { - ob_abort(); - } - if (idx < STACK_BUFFER_SIZE) { - element = &memtable_key_buffer_[idx]; - } else { - element = &p_extra_memtable_keys_[idx - STACK_BUFFER_SIZE]; - } - return *element; -} - -const ObMemtableKey &ObMemtableKeyGenerator::operator[](int64_t idx) const -{ - return const_cast(this)->operator[](idx); -} - -void ObMemtableKeyGenerator::reset() -{ - int64_t idx = size_ - 1; - for(; idx >= STACK_BUFFER_SIZE; --idx) { - p_extra_memtable_keys_[idx - STACK_BUFFER_SIZE].~ObMemtableKey(); - p_extra_store_row_keys_[idx - STACK_BUFFER_SIZE].~ObStoreRowkey(); - } - for(; idx >= 0; --idx) { - memtable_key_buffer_[idx].~ObMemtableKey(); - store_row_key_buffer_[idx].~ObStoreRowkey(); - } - if (OB_UNLIKELY(nullptr != p_extra_memtable_keys_)) { - share::mtl_free(p_extra_memtable_keys_); - } - if (OB_UNLIKELY(nullptr != p_extra_store_row_keys_)) { - share::mtl_free(p_extra_store_row_keys_); - } - new (this) ObMemtableKeyGenerator(); -} - -ObMemtableKeyGenerator::~ObMemtableKeyGenerator() -{ - reset(); -} - -} -} \ No newline at end of file +} // namespace memtable +} // namespace oceanbase \ No newline at end of file diff --git a/src/storage/memtable/ob_memtable_key.h b/src/storage/memtable/ob_memtable_key.h index 22386ee40..882487245 100644 --- a/src/storage/memtable/ob_memtable_key.h +++ b/src/storage/memtable/ob_memtable_key.h @@ -283,7 +283,7 @@ public: private: common::ObStoreRowkey *rowkey_; mutable uint64_t hash_val_; // Perf optimization. - DISALLOW_COPY_AND_ASSIGN(ObMemtableKey); + //DISALLOW_COPY_AND_ASSIGN(ObMemtableKey); }; class ObStoreRowkeyWrapper @@ -308,31 +308,38 @@ public: const common::ObStoreRowkey *rowkey_; }; - -// this is for multi_set pre alloc memory to generate memtable key -class ObMemtableKeyGenerator {// RAII - static constexpr int64_t STACK_BUFFER_SIZE = 32; +// TODO(xuanxi): remove it later +class ObMemtableKeyGenerator { public: - ObMemtableKeyGenerator() : p_extra_store_row_keys_(nullptr), p_extra_memtable_keys_(nullptr), size_(0) {} - ~ObMemtableKeyGenerator(); - int init(const storage::ObStoreRow *rows, - const int64_t row_count, - const int64_t schema_rowkey_count, - const common::ObIArray &columns); - void reset(); - int64_t count() const { return size_; } - ObMemtableKey &operator[](int64_t idx); - const ObMemtableKey &operator[](int64_t idx) const; + using ObMemtableKeyBuffer = common::ObSEArray; +public: + ObMemtableKeyGenerator( + const int64_t rowkey_cnt, + const common::ObIArray &columns, + ObMemtableKeyBuffer *memtable_key_buffer = nullptr) + : allocator_(common::ObMemAttr(MTL_ID(), "ObMemtableKey")), + rowkey_cnt_(rowkey_cnt), + columns_(columns), + memtable_key_buffer_(memtable_key_buffer), + is_inited_(false) + {} + ~ObMemtableKeyGenerator() = default; + int init(); + int generate_memtable_key(const blocksstable::ObDatumRow &datum_row); + ObMemtableKey &get_memtable_key() { return memtable_key_; } + ObMemtableKeyBuffer *get_key_buffer() { return memtable_key_buffer_; } private: - // this is for avoid memory allocation when rows not so much - ObStoreRowkey store_row_key_buffer_[STACK_BUFFER_SIZE]; - ObMemtableKey memtable_key_buffer_[STACK_BUFFER_SIZE]; - ObStoreRowkey *p_extra_store_row_keys_; - ObMemtableKey *p_extra_memtable_keys_; - int64_t size_; + ObArenaAllocator allocator_; + int64_t rowkey_cnt_; + const common::ObIArray &columns_; + storage::ObObjBufArray obj_buf_; + ObStoreRowkey store_rowkey_; + ObMemtableKey memtable_key_; + ObMemtableKeyBuffer *memtable_key_buffer_; + bool is_inited_; }; -} -} +} // namespace memtable +} // namespace oceanbase #endif // OCEANBASE_MEMTABLE_OB_MEMTABLE_KEY2_ diff --git a/src/storage/ob_dml_running_ctx.cpp b/src/storage/ob_dml_running_ctx.cpp index 7e802090a..41b3f56cc 100644 --- a/src/storage/ob_dml_running_ctx.cpp +++ b/src/storage/ob_dml_running_ctx.cpp @@ -35,7 +35,8 @@ ObDMLRunningCtx::ObDMLRunningCtx( ObStoreCtx &store_ctx, const ObDMLBaseParam &dml_param, common::ObIAllocator &allocator, - const blocksstable::ObDmlFlag dml_flag) + const blocksstable::ObDmlFlag dml_flag, + bool is_need_row_datum_utils) : store_ctx_(store_ctx), dml_param_(dml_param), allocator_(allocator), @@ -44,13 +45,21 @@ ObDMLRunningCtx::ObDMLRunningCtx( col_map_(nullptr), col_descs_(nullptr), column_ids_(nullptr), - tbl_row_(), + datum_row_(), + cmp_funcs_(), is_old_row_valid_for_lob_(false), + is_need_check_old_row_(is_need_row_datum_utils), + is_udf_(false), schema_guard_(share::schema::ObSchemaMgrItem::MOD_RELATIVE_TABLE), + is_need_row_datum_utils_(is_need_row_datum_utils), is_inited_(false) { } +ObDMLRunningCtx::~ObDMLRunningCtx() +{ +} + int ObDMLRunningCtx::init( const common::ObIArray *column_ids, const common::ObIArray *upd_col_ids, @@ -89,6 +98,10 @@ int ObDMLRunningCtx::init( LOG_WARN("failed to get relative table", K(ret), K(dml_param_)); } else if (NULL != column_ids && OB_FAIL(prepare_column_info(*column_ids))) { LOG_WARN("fail to get column descriptions and column map", K(ret), K(*column_ids)); + } else if (is_need_check_old_row_ && OB_FAIL(check_need_old_row_legitimacy())) { + LOG_WARN("fail to get flag of checking old row legitimacy", K(ret)); + } else if (is_need_check_old_row_ && OB_FAIL(init_cmp_funcs())) { + LOG_WARN("fail to init compare functions", K(ret)); } else { store_ctx_.mvcc_acc_ctx_.mem_ctx_->set_table_version(dml_param_.schema_version_); store_ctx_.table_version_ = dml_param_.schema_version_; @@ -161,6 +174,101 @@ int ObDMLRunningCtx::prepare_column_info(const common::ObIArray &colum return ret; } +int ObDMLRunningCtx::check_need_old_row_legitimacy() +{ + int ret = OB_SUCCESS; + // TODO(jingxing): setting this to true + if (OB_FAIL(relative_table_.has_udf_column(is_need_check_old_row_))) { + LOG_WARN("check has udf column failed", K(ret)); + } else if (is_need_check_old_row_) { + is_udf_ = true; + ObTableStoreIterator &table_iter = *relative_table_.tablet_iter_.table_iter(); + while (OB_SUCC(ret) && !is_need_check_old_row_) { + ObITable *table_ptr = nullptr; + if (OB_FAIL(table_iter.get_next(table_ptr))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next table failed", K(ret)); + } + } else if (OB_ISNULL(table_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("error unexpected, table ptr must not be nullptr", K(ret)); + } else { + is_need_check_old_row_ = table_ptr->is_major_sstable(); + } + } + } else if (dml_param_.is_batch_stmt_ && !relative_table_.is_index_table()) { + //batch stmt execution dependency defensive check to check + //if the same row was modified multiple times + is_need_check_old_row_ = true; + ret = OB_E(EventTable::EN_INS_MULTI_VALUES_BATCH_OPT) OB_SUCCESS; + // no need to check old row, just for bmsql performance optimization + // TODO yuchen.ywc + if (OB_SUCCESS != ret) { + LOG_INFO("error sim when current statement is batch update", K(ret), K_(is_udf)); + is_need_check_old_row_ = false; + ret = OB_SUCCESS; + } + } else if (GCONF.enable_defensive_check()) { + is_need_check_old_row_ = true; + if (relative_table_.is_index_table() && !relative_table_.can_read_index()) { + //index can not be read during building index, so does not check old index row + is_need_check_old_row_ = false; + } + if (ObDmlFlag::DF_LOCK == dml_flag_) { + is_need_check_old_row_ = false; + } + } + return ret; +} + +int ObDMLRunningCtx::init_cmp_funcs() +{ + int ret = OB_SUCCESS; + const common::ObIArray &col_descs = dml_param_.table_param_->get_col_descs(); + int64_t column_cnt = col_descs.count(); + if (OB_UNLIKELY(column_cnt < 0 || column_cnt > OB_ROW_MAX_COLUMNS_COUNT)) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "Invalid argument to init compare functions", K(ret), K(column_cnt), K(col_descs)); + } else if (OB_FAIL(cmp_funcs_.init(column_cnt, allocator_))) { + STORAGE_LOG(WARN, "Failed to reserve cmp func array", K(ret)); + } else { + bool is_oracle_mode = lib::is_oracle_mode(); + ObCmpFunc cmp_func; + for (int64_t i = 0; OB_SUCC(ret) && i < col_descs.count(); i++) { + const share::schema::ObColDesc &col_desc = col_descs.at(i); + //TODO @hanhui support desc rowkey + bool is_ascending = true || col_desc.col_order_ == ObOrderType::ASC; + bool has_lob_header = is_lob_storage(col_desc.col_type_.get_type()); + ObPrecision precision = PRECISION_UNKNOWN_YET; + if (col_desc.col_type_.is_decimal_int()) { + precision = col_desc.col_type_.get_stored_precision(); + OB_ASSERT(precision != PRECISION_UNKNOWN_YET); + } + sql::ObExprBasicFuncs *basic_funcs = ObDatumFuncs::get_basic_func(col_desc.col_type_.get_type(), + col_desc.col_type_.get_collation_type(), + col_desc.col_type_.get_scale(), + is_oracle_mode, + has_lob_header, + precision); + if (OB_UNLIKELY(nullptr == basic_funcs || nullptr == basic_funcs->null_last_cmp_)) { + ret = OB_ERR_SYS; + STORAGE_LOG(ERROR, "Unexpected null basic funcs", K(ret), K(col_desc)); + } else { + if (is_ascending) { + cmp_func.cmp_func_ = is_oracle_mode ? basic_funcs->null_last_cmp_ : basic_funcs->null_first_cmp_; + if (OB_FAIL(cmp_funcs_.push_back(ObStorageDatumCmpFunc(cmp_func)))) { + STORAGE_LOG(WARN, "Failed to push back cmp func", K(ret), K(i), K(col_desc)); + } + } else { + ret = OB_ERR_SYS; + STORAGE_LOG(WARN, "Unsupported desc column order", K(ret), K(col_desc), K(i)); + } + } + } + } + return ret; +} + int ObDMLRunningCtx::check_schema_version( share::schema::ObMultiVersionSchemaService &schema_service, const uint64_t tenant_id, diff --git a/src/storage/ob_dml_running_ctx.h b/src/storage/ob_dml_running_ctx.h index bac8cf0d2..8af46f265 100644 --- a/src/storage/ob_dml_running_ctx.h +++ b/src/storage/ob_dml_running_ctx.h @@ -47,8 +47,9 @@ public: ObStoreCtx &store_ctx, const ObDMLBaseParam &dml_param, common::ObIAllocator &allocator, - const blocksstable::ObDmlFlag dml_flag); - ~ObDMLRunningCtx() {} + const blocksstable::ObDmlFlag dml_flag, + bool is_need_row_datum_utils = false); + ~ObDMLRunningCtx(); int init( const common::ObIArray *column_ids, @@ -65,6 +66,8 @@ private: const share::schema::ObTableSchemaParam &schema, ObTabletHandle &tablet_handle, const share::SCN &read_snapshot); + int check_need_old_row_legitimacy(); + int init_cmp_funcs(); int check_schema_version(share::schema::ObMultiVersionSchemaService &schema_service, const uint64_t tenant_id, const uint64_t table_id, @@ -86,11 +89,15 @@ public: const share::schema::ColumnMap *col_map_; const ObColDescIArray *col_descs_; const common::ObIArray *column_ids_; - ObStoreRow tbl_row_; + blocksstable::ObDatumRow datum_row_; + blocksstable::ObStoreCmpFuncs cmp_funcs_; bool is_old_row_valid_for_lob_; + bool is_need_check_old_row_; + bool is_udf_; private: share::schema::ObSchemaGetterGuard schema_guard_; + bool is_need_row_datum_utils_; bool is_inited_; }; } // namespace storage diff --git a/src/storage/ob_query_iterator_factory.cpp b/src/storage/ob_query_iterator_factory.cpp index 3aa650133..17f642aff 100644 --- a/src/storage/ob_query_iterator_factory.cpp +++ b/src/storage/ob_query_iterator_factory.cpp @@ -113,7 +113,7 @@ void ObQueryIteratorFactory::free_table_scan_iter(common::ObNewRowIterator *iter } } -void ObQueryIteratorFactory::free_insert_dup_iter(common::ObNewRowIterator *iter) +void ObQueryIteratorFactory::free_insert_dup_iter(blocksstable::ObDatumRowIterator *iter) { if (OB_LIKELY(NULL != iter)) { (void)ATOMIC_FAA(&insert_dup_release_count_, 1); diff --git a/src/storage/ob_query_iterator_factory.h b/src/storage/ob_query_iterator_factory.h index a86302775..708f60f1f 100644 --- a/src/storage/ob_query_iterator_factory.h +++ b/src/storage/ob_query_iterator_factory.h @@ -18,6 +18,10 @@ namespace oceanbase { +namespace blocksstable +{ +class ObDatumRowIterator; +} namespace storage { class ObMultipleScanMerge; @@ -37,7 +41,7 @@ public: static ObColMap *get_col_map(); static void free_table_scan_iter(common::ObNewRowIterator *iter); - static void free_insert_dup_iter(common::ObNewRowIterator *iter); + static void free_insert_dup_iter(blocksstable::ObDatumRowIterator *iter); static void free_merge_iter(ObQueryRowIterator *iter); static void free_col_map(ObColMap *col_map); static void free_work_row(ObStoreRow *row); diff --git a/src/storage/ob_value_row_iterator.cpp b/src/storage/ob_value_row_iterator.cpp index 124f2d377..9cebccf06 100644 --- a/src/storage/ob_value_row_iterator.cpp +++ b/src/storage/ob_value_row_iterator.cpp @@ -25,7 +25,7 @@ using namespace blocksstable; namespace storage { ObValueRowIterator::ObValueRowIterator() - : ObNewRowIterator(), + : ObDatumRowIterator(), is_inited_(false), unique_(false), allocator_("ObValueRowAlloc"), @@ -53,10 +53,9 @@ int ObValueRowIterator::init(bool unique) return ret; } -int ObValueRowIterator::add_row(common::ObNewRow &row) +int ObValueRowIterator::add_row(ObDatumRow &row, const ObStorageDatumUtils &rowkey_datum_utils) { int ret = OB_SUCCESS; - ObNewRow *cur_row = NULL; if (!row.is_valid()) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid row", K(ret), K(row)); @@ -71,29 +70,34 @@ int ObValueRowIterator::add_row(common::ObNewRow &row) //on multiple unique index is small, so there is usually only one row in the value row iterator //so using list traversal to deduplicate unique index is more efficiently //and also saves the CPU overhead that constructs the hash map - ObStoreRowkey rowkey; - ObStoreRowkey tmp_rowkey; - if (OB_FAIL(rowkey.assign(row.cells_, row.count_))) { + ObDatumRowkey rowkey; + ObDatumRowkey tmp_rowkey; + if (OB_FAIL(rowkey.assign(row.storage_datums_, row.count_))) { STORAGE_LOG(WARN, "Failed to assign rowkey", K(ret), K(row)); } for (int64_t i = 0; OB_SUCC(ret) && !exist && i < rows_.count(); ++i) { - if (OB_FAIL(tmp_rowkey.assign(rows_.at(i).cells_, rows_.at(i).count_))) { - STORAGE_LOG(WARN, "Failed to assign rowkey", K(ret), K(i), K(rows_.at(i))); - } else if (OB_UNLIKELY(tmp_rowkey == rowkey)) { - exist = true; + if (OB_FAIL(tmp_rowkey.assign(rows_.at(i)->storage_datums_, rows_.at(i)->count_))) { + STORAGE_LOG(WARN, "Failed to assign rowkey", K(ret), K(i), KPC(rows_.at(i))); + } else if (OB_FAIL(tmp_rowkey.equal(rowkey, rowkey_datum_utils, exist))) { + STORAGE_LOG(WARN, "Failed to compare rowkey", K(ret), K(i), K(tmp_rowkey), K(rowkey)); } } } // store non-exist row if (OB_SUCC(ret)) { if (!exist) { - if (NULL == (cur_row = rows_.alloc_place_holder())) { + ObDatumRow *cur_row = nullptr; + void *buff = nullptr; + if (OB_ISNULL(buff = allocator_.alloc(sizeof(ObDatumRow)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(ERROR, "add row error", K(ret)); - } else if (OB_SUCCESS != (ret = ob_write_row(allocator_, - row, - *cur_row))) { + STORAGE_LOG(WARN, "alloc memory for datum row error", K(ret)); + } else if (FALSE_IT(cur_row = new (buff) ObDatumRow())) { + } else if (OB_FAIL(cur_row->init(allocator_, row.count_))) { + STORAGE_LOG(WARN, "init datum row error", K(ret), K(row), KPC(cur_row)); + } else if (OB_FAIL(cur_row->deep_copy(row, allocator_))) { STORAGE_LOG(WARN, "copy row error", K(ret), K(row)); + } else if (OB_FAIL(rows_.push_back(cur_row))) { + STORAGE_LOG(WARN, "fail to push datum row to iterator array", K(ret), K(cur_row)); } } } @@ -101,27 +105,14 @@ int ObValueRowIterator::add_row(common::ObNewRow &row) return ret; } -int ObValueRowIterator::get_next_row(common::ObNewRow *&row) +int ObValueRowIterator::get_next_row(ObDatumRow *&row) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "ObValueRowIterator is not initialized", K(ret)); } else if (cur_idx_ < rows_.count()) { - row = &rows_.at(cur_idx_++); - } else { - ret = OB_ITER_END; - } - return ret; -} - -int ObValueRowIterator::get_next_rows(ObNewRow *&rows, int64_t &row_count) -{ - int ret = OB_SUCCESS; - if (cur_idx_ < rows_.count()) { - rows = &(rows_.at(cur_idx_)); - row_count = rows_.count() - cur_idx_; - cur_idx_ = rows_.count(); + row = rows_.at(cur_idx_++); } else { ret = OB_ITER_END; } @@ -143,23 +134,23 @@ ObSingleRowGetter::ObSingleRowGetter(ObIAllocator &allocator, ObTablet &tablet) store_ctx_(nullptr), output_projector_(allocator), relative_table_(nullptr), - table_param_(nullptr), allocator_(allocator), - new_row_builder_() + cached_iter_node_(nullptr) { } ObSingleRowGetter::~ObSingleRowGetter() { - if (single_merge_ != nullptr) { - single_merge_->~ObSingleMerge(); - allocator_.free(single_merge_); + if (nullptr != single_merge_) { + if (nullptr == cached_iter_node_) { + single_merge_->~ObSingleMerge(); + allocator_.free(single_merge_); + } single_merge_ = nullptr; } - if (table_param_ != nullptr) { - table_param_->~ObTableParam(); - allocator_.free(table_param_); - table_param_ = nullptr; + if (nullptr != cached_iter_node_) { + ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*); + iter_pool->release(cached_iter_node_); } } @@ -171,6 +162,7 @@ int ObSingleRowGetter::init_dml_access_ctx( int ret = OB_SUCCESS; common::ObQueryFlag query_flag; common::ObVersionRange trans_version_range; + query_flag.set_not_use_bloomfilter_cache(); query_flag.read_latest_ = ObQueryFlag::OBSF_MASK_READ_LATEST; if (skip_read_lob) { query_flag.skip_read_lob_ = ObQueryFlag::OBSF_MASK_SKIP_READ_LOB; @@ -180,7 +172,7 @@ int ObSingleRowGetter::init_dml_access_ctx( trans_version_range.multi_version_start_ = 0; store_ctx_ = &store_ctx; - if (OB_FAIL(access_ctx_.init(query_flag, store_ctx, allocator_, trans_version_range))) { + if (OB_FAIL(access_ctx_.init(query_flag, store_ctx, allocator_, trans_version_range, cached_iter_node_))) { LOG_WARN("failed to init table access ctx", K(ret)); } return ret; @@ -233,18 +225,21 @@ int ObSingleRowGetter::init_dml_access_param(ObRelativeTable &relative_table, return ret; } -int ObSingleRowGetter::create_table_param() +int ObSingleRowGetter::prepare_cached_iter_node() { int ret = OB_SUCCESS; - void *buf = nullptr; - if (table_param_ != nullptr) { - ret = OB_INIT_TWICE; - LOG_WARN("init table param twice", K(ret)); - } else if (OB_ISNULL(buf = allocator_.alloc(sizeof(share::schema::ObTableParam)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate table param failed", K(ret), K(sizeof(share::schema::ObTableParam))); - } else { - table_param_ = new(buf) share::schema::ObTableParam(allocator_); + if (OB_UNLIKELY(nullptr != cached_iter_node_)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected not null cached iter node", K(ret), KP(cached_iter_node_)); + } else if (can_use_global_iter_pool()) { + ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*); + if (OB_FAIL(iter_pool->get(ITER_TYPE, cached_iter_node_))) { + STORAGE_LOG(WARN, "Failed to get from iter pool", K(ret)); + } else if (nullptr != cached_iter_node_) { + access_param_.set_use_global_iter_pool(); + access_param_.iter_param_.set_use_stmt_iter_pool(); + STORAGE_LOG(TRACE, "use global iter pool", K(access_param_)); + } } return ret; } @@ -252,28 +247,18 @@ int ObSingleRowGetter::create_table_param() int ObSingleRowGetter::open(const ObDatumRowkey &rowkey, bool use_fuse_row_cache) { int ret = OB_SUCCESS; - void *buf = nullptr; - const common::ObIArray * col_descs = nullptr; - - if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObSingleMerge)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("Fail to allocate memory for multi get merge ", K(ret)); - } else { - { - ObStorageTableGuard guard(tablet_, *store_ctx_, false); - if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(*relative_table_))) { - STORAGE_LOG(WARN, "fail to protect table", K(ret)); - } + { + ObStorageTableGuard guard(tablet_, *store_ctx_, false); + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(*relative_table_))) { + STORAGE_LOG(WARN, "fail to protect table", K(ret)); } - single_merge_ = new(buf) ObSingleMerge(); } if (OB_SUCC(ret)) { - if (OB_FAIL(single_merge_->init(access_param_, access_ctx_, get_table_param_))) { - STORAGE_LOG(WARN, "Fail to init ObSingleMerge, ", K(ret)); + ACTIVE_GLOBAL_ITERATOR_GUARD(ret, cached_iter_node_); + if (OB_FAIL(init_single_merge())) { + STORAGE_LOG(WARN, "Fail to init ObSingleMerge", K(ret)); } else if (OB_FAIL(single_merge_->open(rowkey))) { - STORAGE_LOG(WARN, "Fail to open iter, ", K(ret)); - } else if (OB_FAIL(new_row_builder_.init(single_merge_->get_out_project_cells(), allocator_))) { - LOG_WARN("Failed to init ObNewRowBuilder", K(ret)); + STORAGE_LOG(WARN, "Fail to open iter", K(ret)); } if (use_fuse_row_cache) { access_ctx_.use_fuse_row_cache_ = true; @@ -282,9 +267,10 @@ int ObSingleRowGetter::open(const ObDatumRowkey &rowkey, bool use_fuse_row_cache return ret; } -int ObSingleRowGetter::get_next_row(ObNewRow *&row) +int ObSingleRowGetter::get_next_row(blocksstable::ObDatumRow *&row) { int ret = OB_SUCCESS; + ACTIVE_GLOBAL_ITERATOR_GUARD(ret, cached_iter_node_); row = nullptr; while (OB_SUCC(ret)) { blocksstable::ObDatumRow *store_row = NULL; @@ -293,9 +279,7 @@ int ObSingleRowGetter::get_next_row(ObNewRow *&row) STORAGE_LOG(WARN, "failed to get next row", K(ret)); } } else if (store_row->row_flag_.is_exist_without_delete()) { - if (OB_FAIL(new_row_builder_.build(*store_row, row))) { - STORAGE_LOG(WARN, "Failed to build new row", K(ret), KPC(store_row)); - } + row = store_row; break; } } @@ -319,5 +303,58 @@ int ObSingleRowGetter::get_next_row(ObNewRow *&row) } return ret; } + +bool ObSingleRowGetter::can_use_global_iter_pool() const +{ + bool use_pool = false; + if (access_param_.iter_param_.tablet_id_.is_inner_tablet()) { + } else if (access_param_.iter_param_.has_lob_column_out_) { + } else { + const int64_t table_cnt = get_table_param_.tablet_iter_.table_iter()->count(); + const int64_t col_cnt = get_table_param_.tablet_iter_.get_tablet()->get_rowkey_read_info().get_schema_column_count(); + ObGlobalIteratorPool *iter_pool = MTL(ObGlobalIteratorPool*); + if (OB_NOT_NULL(iter_pool)) { + use_pool = iter_pool->can_use_iter_pool(table_cnt, col_cnt, ITER_TYPE); + } + } + return use_pool; +} + +int ObSingleRowGetter::init_single_merge() +{ + int ret = OB_SUCCESS; + ObQueryRowIterator *cached_iter = nullptr == cached_iter_node_ ? nullptr : cached_iter_node_->get_iter(); + if (OB_NOT_NULL(cached_iter)) { + if (OB_UNLIKELY(cached_iter->get_type() != ITER_TYPE)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "Unexpected cached iter type", K(ret), K(cached_iter->get_type())); + } else { + single_merge_ = static_cast(cached_iter); + } + } + if (OB_FAIL(ret)) { + } else if (nullptr == single_merge_) { + void *buf = nullptr; + if (OB_ISNULL(buf = access_ctx_.get_long_life_allocator()->alloc(sizeof(ObSingleMerge)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "Fail to allocate memory", K(ret)); + } else { + single_merge_ = new (buf) ObSingleMerge(); + if (OB_FAIL(single_merge_->init(access_param_, access_ctx_, get_table_param_))) { + STORAGE_LOG(WARN, "Failed to init multiple merge", K(ret)); + } + if (OB_FAIL(ret)) { + single_merge_->~ObSingleMerge(); + access_ctx_.get_long_life_allocator()->free(single_merge_); + single_merge_ = nullptr; + } else if (nullptr != cached_iter_node_) { + cached_iter_node_->set_iter(single_merge_); + } + } + } else if (OB_FAIL(single_merge_->switch_table(access_param_, access_ctx_, get_table_param_))) { + STORAGE_LOG(WARN, "Failed to switch table", K(ret), K(access_param_)); + } + return ret; +} } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/ob_value_row_iterator.h b/src/storage/ob_value_row_iterator.h index 64ff40253..903db50f3 100644 --- a/src/storage/ob_value_row_iterator.h +++ b/src/storage/ob_value_row_iterator.h @@ -22,6 +22,7 @@ #include "storage/ob_i_store.h" #include "storage/access/ob_dml_param.h" #include "blocksstable/ob_datum_rowkey.h" +#include "blocksstable/ob_datum_row_iterator.h" namespace oceanbase { @@ -29,17 +30,16 @@ namespace storage { class ObTablet; -class ObValueRowIterator : public common::ObNewRowIterator +class ObValueRowIterator : public blocksstable::ObDatumRowIterator { static const int64_t DEFAULT_ROW_NUM = 2; - typedef common::ObSEArray RowArray; + typedef common::ObSEArray RowArray; public: ObValueRowIterator(); virtual ~ObValueRowIterator(); virtual int init(bool unique); - virtual int get_next_row(common::ObNewRow *&row); - virtual int get_next_rows(common::ObNewRow *&rows, int64_t &row_count); - virtual int add_row(common::ObNewRow &row); + virtual int get_next_row(blocksstable::ObDatumRow *&row); + virtual int add_row(blocksstable::ObDatumRow &row, const blocksstable::ObStorageDatumUtils &rowkey_datum_utils); virtual void reset(); private: bool is_inited_; @@ -55,6 +55,7 @@ class ObSingleMerge; class ObSingleRowGetter { typedef common::ObFixedArray Projector; + const ObQRIterType ITER_TYPE = T_SINGLE_GET; public: ObSingleRowGetter(common::ObIAllocator &allocator, ObTablet &tablet); ~ObSingleRowGetter(); @@ -68,13 +69,15 @@ public: const ObDMLBaseParam &dml_param, const common::ObIArray &out_col_ids, const bool skip_read_lob = false); + int prepare_cached_iter_node(); ObTableAccessParam &get_access_param() { return access_param_; } ObTableAccessContext &get_access_ctx() { return access_ctx_; } void set_relative_table(ObRelativeTable *relative_table) { relative_table_ = relative_table; } int open(const blocksstable::ObDatumRowkey &rowkey, bool use_fuse_row_cache = false); - int get_next_row(common::ObNewRow *&row); + int get_next_row(blocksstable::ObDatumRow *&row); private: - int create_table_param(); + bool can_use_global_iter_pool() const; + int init_single_merge(); private: ObTablet *tablet_; ObSingleMerge *single_merge_; @@ -84,9 +87,8 @@ private: ObTableAccessContext access_ctx_; ObGetTableParam get_table_param_; ObRelativeTable *relative_table_; - share::schema::ObTableParam *table_param_; common::ObIAllocator &allocator_; - blocksstable::ObNewRowBuilder new_row_builder_; + CachedIteratorNode *cached_iter_node_; }; } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 7fbcbf133..7ffa156c8 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -91,6 +91,7 @@ #include "storage/tablet/ob_tablet_mds_table_mini_merger.h" #include "storage/blocksstable/ob_shared_macro_block_manager.h" #include "storage/ob_direct_load_table_guard.h" +#include "storage/blocksstable/ob_datum_row_utils.h" namespace oceanbase { @@ -3639,7 +3640,8 @@ int ObTablet::check_schema_version_for_bounded_staleness_read( int ObTablet::lock_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - const common::ObNewRow &row) + ObColDescArray &col_desc, + blocksstable::ObDatumRow &row) { int ret = OB_SUCCESS; ObStorageTableGuard guard(this, store_ctx, true); @@ -3669,7 +3671,7 @@ int ObTablet::lock_row( LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); } else if (OB_FAIL(prepare_param_ctx(allocator, relative_table, store_ctx, param, context))) { LOG_WARN("prepare param ctx fail, ", K(ret)); - } else if (OB_FAIL(write_memtable->lock(param, context, row))) { + } else if (OB_FAIL(write_memtable->lock(param, context, col_desc, row))) { LOG_WARN("failed to lock write_memtable", K(ret), K(row)); } } @@ -4101,8 +4103,8 @@ int ObTablet::update_row( storage::ObStoreCtx &store_ctx, const common::ObIArray &col_descs, const ObIArray &update_idx, - const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row, + const blocksstable::ObDatumRow &old_row, + blocksstable::ObDatumRow &new_row, const common::ObIArray *encrypt_meta_arr) { int ret = OB_SUCCESS; @@ -4159,7 +4161,7 @@ int ObTablet::update_row( int ObTablet::insert_rows( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - ObStoreRow *rows, + ObDatumRow *rows, ObRowsInfo &rows_info, const bool check_exist, const ObColDescIArray &col_descs, @@ -4209,7 +4211,7 @@ int ObTablet::insert_row_without_rowkey_check( ObStoreCtx &store_ctx, const bool check_exist, const common::ObIArray &col_descs, - const storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, const common::ObIArray *encrypt_meta_arr) { int ret = OB_SUCCESS; @@ -4373,7 +4375,8 @@ int ObTablet::do_rowkeys_exist( int ObTablet::rowkey_exists( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - const common::ObNewRow &row, + const ObColDescIArray &col_descs, + ObDatumRow &row, bool &exists) { int ret = OB_SUCCESS; @@ -4399,24 +4402,22 @@ int ObTablet::rowkey_exists( } if (OB_SUCC(ret)) { - ObStoreRowkey rowkey; ObDatumRowkey datum_rowkey; ObDatumRowkeyHelper rowkey_helper; ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "rowkey_acc_ctx")); ObTableIterParam param; ObTableAccessContext context; - if (OB_FAIL(rowkey.assign(row.cells_, relative_table.get_rowkey_column_num()))) { - LOG_WARN("Failed to assign rowkey", K(ret), K(row)); - } else if (OB_FAIL(rowkey_helper.convert_datum_rowkey(rowkey.get_rowkey(), datum_rowkey))) { - LOG_WARN("Failed to transfer datum rowkey", K(ret), K(rowkey)); + if (OB_FAIL(rowkey_helper.prepare_datum_rowkey(row, relative_table.get_rowkey_column_num(), + col_descs, datum_rowkey))) { + LOG_WARN("Failed to prepare rowkey", K(ret), K(row)); } else if (OB_FAIL(prepare_param_ctx(allocator, relative_table, store_ctx, param, context))) { - LOG_WARN("Failed to prepare param ctx, ", K(ret), K(rowkey)); + LOG_WARN("Failed to prepare param ctx, ", K(ret), K(row)); } else if (OB_FAIL(relative_table.tablet_iter_.get_tablet()->do_rowkey_exists( param, context, datum_rowkey, exists))) { - LOG_WARN("do rowkey exist fail", K(ret), K(rowkey)); + LOG_WARN("do rowkey exist fail", K(ret), K(row), K(datum_rowkey)); } - LOG_DEBUG("chaser debug row", K(ret), K(row), K(rowkey)); + LOG_DEBUG("chaser debug row", K(ret), K(row)); } } return ret; diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 2418aa34c..50e2abad8 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -314,7 +314,7 @@ public: int insert_rows( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - ObStoreRow *rows, + blocksstable::ObDatumRow *rows, ObRowsInfo &rows_info, const bool check_exist, const ObColDescIArray &col_descs, @@ -325,20 +325,21 @@ public: ObStoreCtx &store_ctx, const bool check_exist, const ObColDescIArray &col_descs, - const storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, const common::ObIArray *encrypt_meta_arr); int update_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, const ObColDescIArray &col_descs, const ObIArray &update_idx, - const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row, + const blocksstable::ObDatumRow &old_row, + blocksstable::ObDatumRow &new_row, const common::ObIArray *encrypt_meta_arr); int lock_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - const common::ObNewRow &row); + ObColDescArray &col_desc, + blocksstable::ObDatumRow &row); int lock_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, @@ -399,7 +400,8 @@ public: int rowkey_exists( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, - const common::ObNewRow &row, + const ObColDescIArray &col_descs, + blocksstable::ObDatumRow &row, bool &exists); int rowkeys_exists( ObStoreCtx &store_ctx, diff --git a/src/storage/tablet/ob_tablet_table_store_iterator.cpp b/src/storage/tablet/ob_tablet_table_store_iterator.cpp index fa64bcff8..4f43b26e2 100644 --- a/src/storage/tablet/ob_tablet_table_store_iterator.cpp +++ b/src/storage/tablet/ob_tablet_table_store_iterator.cpp @@ -96,6 +96,7 @@ void ObTableStoreIterator::reset() table_ptr_array_.reset(); sstable_handle_array_.reset(); table_store_handle_.reset(); + if (nullptr != transfer_src_table_store_handle_) { transfer_src_table_store_handle_->~ObStorageMetaHandle(); ob_free(transfer_src_table_store_handle_); diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index be855159c..9fcf90351 100644 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -754,7 +754,7 @@ int ObAccessService::delete_rows( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -808,7 +808,7 @@ int ObAccessService::put_rows( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -861,7 +861,7 @@ int ObAccessService::insert_rows( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -916,10 +916,10 @@ int ObAccessService::insert_row( const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray &duplicated_column_ids, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObInsertFlag flag, int64_t &affected_rows, - common::ObNewRowIterator *&duplicated_rows) + blocksstable::ObDatumRowIterator *&duplicated_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); int ret = OB_SUCCESS; @@ -970,7 +970,7 @@ int ObAccessService::insert_row( return ret; } -int ObAccessService::revert_insert_iter(common::ObNewRowIterator *iter) +int ObAccessService::revert_insert_iter(blocksstable::ObDatumRowIterator *iter) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); int ret = OB_SUCCESS; @@ -987,7 +987,7 @@ int ObAccessService::update_rows( const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray< uint64_t> &updated_column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -1043,7 +1043,7 @@ int ObAccessService::lock_rows( const ObDMLBaseParam &dml_param, const int64_t abs_lock_timeout, const ObLockFlag lock_flag, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); @@ -1098,7 +1098,7 @@ int ObAccessService::lock_row( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const int64_t abs_lock_timeout, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObLockFlag lock_flag) { ACTIVE_SESSION_FLAG_SETTER_GUARD(in_storage_write); diff --git a/src/storage/tx_storage/ob_access_service.h b/src/storage/tx_storage/ob_access_service.h index c7d2f2dd8..541054a44 100644 --- a/src/storage/tx_storage/ob_access_service.h +++ b/src/storage/tx_storage/ob_access_service.h @@ -137,7 +137,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int put_rows( const share::ObLSID &ls_id, @@ -145,7 +145,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int insert_rows( const share::ObLSID &ls_id, @@ -153,7 +153,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int insert_row( const share::ObLSID &ls_id, @@ -162,11 +162,11 @@ public: const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray &duplicated_column_ids, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObInsertFlag flag, int64_t &affected_rows, - common::ObNewRowIterator *&duplicated_rows); - int revert_insert_iter(common::ObNewRowIterator *iter); + blocksstable::ObDatumRowIterator *&duplicated_rows); + int revert_insert_iter(blocksstable::ObDatumRowIterator *iter); int update_rows( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, @@ -174,7 +174,7 @@ public: const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, const common::ObIArray< uint64_t> &updated_column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int lock_rows( const share::ObLSID &ls_id, @@ -183,7 +183,7 @@ public: const ObDMLBaseParam &dml_param, const int64_t abs_lock_timeout, /* -1: undefined, 0: nowait */ const ObLockFlag lock_flag, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); int lock_row( const share::ObLSID &ls_id, @@ -191,7 +191,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const int64_t abs_lock_timeout, - const common::ObNewRow &row, + blocksstable::ObDatumRow &row, const ObLockFlag lock_flag); int estimate_row_count( const ObTableScanParam ¶m, diff --git a/unittest/storage/blocksstable/test_row_reader.cpp b/unittest/storage/blocksstable/test_row_reader.cpp index d47aaa240..6005174a6 100644 --- a/unittest/storage/blocksstable/test_row_reader.cpp +++ b/unittest/storage/blocksstable/test_row_reader.cpp @@ -1326,30 +1326,30 @@ TEST_F(TestNewRowReader, test_write_update_row) const int64_t num = 200; const int64_t write_col_cnt = 80; const int64_t rowkey_cnt = 5; - oceanbase::common::ObObj objs[num]; - ObStoreRow writer_row; - writer_row.row_val_.cells_ = objs; - writer_row.row_val_.count_ = column_num; + blocksstable::ObStorageDatum objs[num]; + ObDatumRow writer_row; + writer_row.storage_datums_ = objs; + writer_row.count_ = column_num; ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(writer_row)); - for (int i = writer_row.row_val_.count_; i < write_col_cnt; ++i) { - writer_row.row_val_.cells_[i] = writer_row.row_val_.cells_[i - writer_row.row_val_.count_]; + for (int i = writer_row.count_; i < write_col_cnt; ++i) { + writer_row.storage_datums_[i] = writer_row.storage_datums_[i - writer_row.count_]; } - writer_row.row_val_.count_ = write_col_cnt; + writer_row.count_ = write_col_cnt; memtable::ObNopBitMap nop_bitmap; bool read_finished = false; - ret = nop_bitmap.init(writer_row.row_val_.count_, rowkey_cnt); + ret = nop_bitmap.init(writer_row.count_, rowkey_cnt); ObDatumRow reader_row; - ASSERT_EQ(OB_SUCCESS, reader_row.init(allocator_, writer_row.row_val_.count_)); - for (int i = 0; i < writer_row.row_val_.count_; ++i) { + ASSERT_EQ(OB_SUCCESS, reader_row.init(allocator_, writer_row.count_)); + for (int i = 0; i < writer_row.count_; ++i) { reader_row.storage_datums_[i].set_nop(); } int64_t array[] = {5, 18, 29, 45, 75, 78}; - writer_row.row_val_.cells_[array[2]].set_int(0); - writer_row.row_val_.cells_[array[4]].set_int(0); + writer_row.storage_datums_[array[2]].set_int(0); + writer_row.storage_datums_[array[4]].set_int(0); build_column_read_info(rowkey_cnt, writer_row); STORAGE_LOG(INFO, "write_row", K(writer_row)); @@ -1363,11 +1363,11 @@ TEST_F(TestNewRowReader, test_write_update_row) int64_t len = 0; char *buf = nullptr; - writer_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); - writer_row.row_val_.cells_[array[2]].set_int(100 * i); - writer_row.row_val_.cells_[array[4]].set_int(100 * i); + writer_row.row_flag_.set_flag(ObDmlFlag::DF_UPDATE); + writer_row.storage_datums_[array[2]].set_int(100 * i); + writer_row.storage_datums_[array[4]].set_int(100 * i); if (i == 4) { - writer_row.flag_.set_flag(ObDmlFlag::DF_INSERT); + writer_row.row_flag_.set_flag(ObDmlFlag::DF_INSERT); ret = row_writer[i].write(rowkey_cnt, writer_row, nullptr, buf, len); } else { ret = row_writer[i].write(rowkey_cnt, writer_row, &update_idx, buf, len); @@ -1387,7 +1387,7 @@ TEST_F(TestNewRowReader, test_write_update_row) STORAGE_LOG(INFO, "chaser check writer row", K(read_info_)); int update_pos = 0; - for (int i = 0; i < writer_row.row_val_.count_; ++i) { + for (int i = 0; i < writer_row.count_; ++i) { bool check_cell_flag = false; if (i < rowkey_cnt) { check_cell_flag = true; @@ -1397,12 +1397,10 @@ TEST_F(TestNewRowReader, test_write_update_row) } if (check_cell_flag) { STORAGE_LOG(INFO, "check", K(i), K(update_pos), K(reader_row.storage_datums_[i])); - if (ObNumberFloatType != writer_row.row_val_.cells_[i].get_type()) { - if (i == array[2] || i == array[4]) { - ASSERT_TRUE(reader_row.storage_datums_[i].get_int() == 0); - } else { - ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.row_val_.cells_[i]); - } + if (i == array[2] || i == array[4]) { + ASSERT_TRUE(reader_row.storage_datums_[i].get_int() == 0); + } else { + ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.storage_datums_[i]); } } else { ASSERT_TRUE(!reader_row.storage_datums_[i].is_nop()); @@ -1416,28 +1414,28 @@ TEST_F(TestNewRowReader, test_write_write_nop_val) const int64_t num = 400; const int64_t write_col_cnt = 302; const int64_t rowkey_cnt = 3; - oceanbase::common::ObObj objs[num]; - ObStoreRow writer_row; - writer_row.row_val_.cells_ = objs; - writer_row.row_val_.count_ = column_num; + blocksstable::ObStorageDatum objs[num]; + ObDatumRow writer_row; + writer_row.storage_datums_ = objs; + writer_row.count_ = column_num; ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(writer_row)); int64_t idx = 0; - writer_row.row_val_.cells_[idx++].set_int(14); - writer_row.row_val_.cells_[idx++].set_int(-1658240586131896801); - writer_row.row_val_.cells_[idx++].set_int(-INT64_MAX); + writer_row.storage_datums_[idx++].set_int(14); + writer_row.storage_datums_[idx++].set_int(-1658240586131896801); + writer_row.storage_datums_[idx++].set_int(-INT64_MAX); for (int i = idx; i < write_col_cnt; ++i) { - writer_row.row_val_.cells_[i].set_nop_value(); + writer_row.storage_datums_[i].set_nop(); } - writer_row.row_val_.count_ = write_col_cnt; + writer_row.count_ = write_col_cnt; memtable::ObNopBitMap nop_bitmap; bool read_finished = false; - ret = nop_bitmap.init(writer_row.row_val_.count_, rowkey_cnt); + ret = nop_bitmap.init(writer_row.count_, rowkey_cnt); ObDatumRow reader_row; ASSERT_EQ(OB_SUCCESS, reader_row.init(allocator_, num)); - reader_row.count_ = writer_row.row_val_.count_; + reader_row.count_ = writer_row.count_; build_column_read_info(rowkey_cnt, writer_row); int64_t len = 0; @@ -1449,11 +1447,9 @@ TEST_F(TestNewRowReader, test_write_write_nop_val) ret = row_reader.read_row(buf, len, &read_info_, reader_row); STORAGE_LOG(INFO, "chaser check writer row", K(read_info_), K(reader_row)); - for (int i = 0; i < writer_row.row_val_.count_; ++i) { + for (int i = 0; i < writer_row.count_; ++i) { STORAGE_LOG(INFO, "check", K(i), K(reader_row.storage_datums_[i])); - if (ObNumberFloatType != writer_row.row_val_.cells_[i].get_type()) { - ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.row_val_.cells_[i]); - } + ASSERT_TRUE(reader_row.storage_datums_[i] == writer_row.storage_datums_[i]); } } @@ -1504,10 +1500,10 @@ TEST_F(TestNewRowReader, test_update_idx_with_rowkey) const int64_t write_col_cnt = 80; const int64_t rowkey_cnt = 3; - oceanbase::common::ObObj objs[num]; - ObStoreRow writer_row; - writer_row.row_val_.cells_ = objs; - writer_row.row_val_.count_ = column_num; + blocksstable::ObStorageDatum objs[num]; + ObDatumRow writer_row; + writer_row.storage_datums_ = objs; + writer_row.count_ = column_num; ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(writer_row)); int64_t update_array[] = {0, 1, 2, 5, 18}; @@ -1515,9 +1511,9 @@ TEST_F(TestNewRowReader, test_update_idx_with_rowkey) build_column_read_info(rowkey_cnt, writer_row); int64_t array_idx = 0; - for (int i = 0; i < writer_row.row_val_.count_; ++i) { + for (int i = 0; i < writer_row.count_; ++i) { if (i != update_array[array_idx]) { - writer_row.row_val_.cells_[i].set_nop_value(); + writer_row.storage_datums_[i].set_nop(); } else { update_idx.push_back(update_array[array_idx]); ++array_idx; diff --git a/unittest/storage/mock_access_service.cpp b/unittest/storage/mock_access_service.cpp index c24caba41..59e2f2875 100644 --- a/unittest/storage/mock_access_service.cpp +++ b/unittest/storage/mock_access_service.cpp @@ -32,7 +32,7 @@ int MockObAccessService::insert_rows( transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows) { int ret = OB_SUCCESS; diff --git a/unittest/storage/mock_access_service.h b/unittest/storage/mock_access_service.h index 741f43030..fd8d750fb 100644 --- a/unittest/storage/mock_access_service.h +++ b/unittest/storage/mock_access_service.h @@ -33,7 +33,7 @@ public: transaction::ObTxDesc &tx_desc, const ObDMLBaseParam &dml_param, const common::ObIArray &column_ids, - common::ObNewRowIterator *row_iter, + blocksstable::ObDatumRowIterator *row_iter, int64_t &affected_rows); public: ObLSTabletService *tablet_service_; // different kinds of mock ls tablet service diff --git a/unittest/storage/mockcontainer/mock_ob_iterator.cpp b/unittest/storage/mockcontainer/mock_ob_iterator.cpp index 9893d1109..7c9051203 100644 --- a/unittest/storage/mockcontainer/mock_ob_iterator.cpp +++ b/unittest/storage/mockcontainer/mock_ob_iterator.cpp @@ -24,8 +24,48 @@ using namespace oceanbase::blocksstable; using namespace oceanbase::common::hash; using namespace oceanbase::share; +int malloc_datum_row( + ObIAllocator &allocator, + const int64_t cell_count, + ObDatumRow *&row) +{ + int ret = common::OB_SUCCESS; + row = NULL; + if (cell_count <= 0) { + ret = common::OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid args", K(cell_count)); + } else { + void *ptr = NULL; + int64_t size = sizeof(ObDatumRow) + sizeof(blocksstable::ObStorageDatum) * cell_count; + if (NULL == (ptr = allocator.alloc(size))) { + ret = common::OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(ERROR, "fail to alloc ObDatumRow", K(size), K(cell_count)); + } else { + void *cell_ptr = static_cast(ptr) + sizeof(ObDatumRow); + row = new (ptr) ObDatumRow; + row->storage_datums_ = new (cell_ptr) blocksstable::ObStorageDatum[cell_count](); + row->row_flag_.set_flag(blocksstable::ObDmlFlag::DF_NOT_EXIST); + row->count_ = cell_count; + } + } + + if (OB_FAIL(ret)) { + row = NULL; + } + return ret; +} + +void free_datum_row(ObIAllocator &allocator, ObDatumRow *&row) +{ + if (nullptr != row) { + allocator.free(row); + row = nullptr; + } +} + ObMockIterator::ObMockIterator(bool reverse) - : rows_(), + : datum_rows_(), + rows_(), cursor_(0), reverse_(reverse), trans_id_(888), @@ -41,7 +81,7 @@ ObMockIterator::~ObMockIterator() void ObMockIterator::setup_start_cursor() { if (reverse_) { - cursor_ = rows_.count() - 1; + cursor_ = rows_.empty() ? (datum_rows_.count() - 1) : (rows_.count() - 1); } else { cursor_ = 0; } @@ -61,10 +101,28 @@ bool ObMockIterator::end_of_row() const if (reverse_) { return cursor_ < 0; } else { - return cursor_ >= rows_.count(); + return cursor_ >= (rows_.empty() ? (datum_rows_.count()) : (rows_.count())); } } +int ObMockIterator::get_next_row(const ObDatumRow *&row) +{ + return get_next_row(const_cast(row)); +} + +int ObMockIterator::get_next_row(ObDatumRow *&row) +{ + int ret = OB_SUCCESS; + if (end_of_row()) { + row = NULL; + ret = OB_ITER_END; + } else { + row = datum_rows_[cursor_]; + advance(); + } + return ret; +} + int ObMockIterator::get_next_row(ObStoreRow *&row) { int ret = OB_SUCCESS; @@ -96,6 +154,32 @@ int ObMockIterator::get_next_row(ObNewRow *&row) return ret; } +int ObMockIterator::get_row(const int64_t idx, const ObDatumRow *&row) const +{ + int ret = OB_SUCCESS; + if (idx < 0 || idx >= rows_.count()) { + STORAGE_LOG(WARN, "failed to idx", K(idx), K(rows_.count())); + row = NULL; + ret = OB_ITER_END; + } else { + row = datum_rows_[idx]; + } + return ret; +} + +int ObMockIterator::get_row(const int64_t idx, ObDatumRow *&row) const +{ + int ret = OB_SUCCESS; + if (idx < 0 || idx >= rows_.count()) { + STORAGE_LOG(WARN, "failed to idx", K(idx), K(rows_.count())); + row = NULL; + ret = OB_ITER_END; + } else { + row = datum_rows_[idx]; + } + return ret; +} + int ObMockIterator::get_row(const int64_t idx, const ObStoreRow *&row) const { int ret = OB_SUCCESS; @@ -135,6 +219,44 @@ int ObMockIterator::get_row(const int64_t idx, ObNewRow *&row) const return ret; } +int ObMockIterator::add_row(blocksstable::ObDatumRow *row) +{ + int ret = OB_SUCCESS; + ObDatumRow *row_clone = NULL; + if (NULL == row) { + ret = OB_ERR_NULL_VALUE; + } else { + ObRowStoreType row_type = FLAT_ROW_STORE; + if (OB_FAIL(malloc_datum_row(allocator_, row->count_, row_clone))) { + STORAGE_LOG(WARN, "failed to malloc datum row", K(ret)); + } else { + row_clone->row_flag_ = row->row_flag_; + if (row->mvcc_row_flag_.is_uncommitted_row()) { // make all row point to the same trans_id + row_clone->trans_id_ = row->trans_id_.is_valid() ? row->trans_id_ : trans_id_; + } + row_clone->count_ = row->count_; + for (int64_t i = 0; OB_SUCC(ret) && i < row->count_; ++i) { + if (OB_SUCCESS != (ret = row_clone->storage_datums_[i].deep_copy(row->storage_datums_[i], allocator_))) { + STORAGE_LOG(WARN, "ob write datum error.", K(ret) ,K(row)); + } + } + + if (OB_SUCC(ret)) { + if (OB_SUCCESS != (ret = datum_rows_.push_back(row_clone))) { + STORAGE_LOG(WARN, "add datum row failed", K(ret), K(row_clone)); + } else { + setup_start_cursor(); + } + } + } + + if (OB_SUCCESS != ret && row_clone != NULL) { + free_datum_row(allocator_, row_clone); + } + } + return ret; +} + // deep copy int ObMockIterator::add_row(ObStoreRow *row) { @@ -187,6 +309,11 @@ void ObMockIterator::reset_iter() void ObMockIterator::reset() { cursor_ = 0; + for (int64_t i = 0; i < datum_rows_.count(); ++i) { + ObDatumRow *row = datum_rows_[i]; + free_datum_row(allocator_, row); + } + datum_rows_.reset(); for (int64_t i = 0; i < rows_.count(); ++i) { ObStoreRow *row = rows_[i]; free_store_row(allocator_, row); @@ -219,6 +346,102 @@ int ObMockIterator::from(const ObString &str, char escape, uint16_t *col_id_arra return ret; } +int ObMockIterator::from_for_datum(const char *cstr, char escape, uint16_t* col_id_array, int64_t *result_col_id_array) +{ + return from_for_datum(ObString::make_string(cstr), escape, col_id_array, result_col_id_array); +} + +int ObMockIterator::from_for_datum(const ObString &str, char escape, uint16_t *col_id_array, int64_t *result_col_id_array) +{ + int ret = OB_SUCCESS; + ObMockIteratorBuilder builder; + ObArenaAllocator buffer("StTemp"); + if (OB_SUCCESS != (ret != builder.init(&buffer, escape))) { + STORAGE_LOG(WARN, "init builder failed"); + } else { + if (OB_ISNULL(result_col_id_array)) { + ret = builder.parse_for_datum(str, *this, col_id_array); + } else { + ret = builder.parse_datum_with_specified_col_ids(str, *this, col_id_array, result_col_id_array); + } + + } + buffer.clear(); + return ret; +} + +bool ObMockIterator::inner_equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2) +{ + bool bool_ret = true; + int64_t idx = 0; + if (!r1.storage_datums_ || !r2.storage_datums_) { + bool_ret = false; + } else if (r1.count_ != r2.count_) { + bool_ret = false; + } else { + for (idx = 0; idx < r1.count_ && bool_ret; idx++) { + if (r1.storage_datums_[idx].is_null() && r2.storage_datums_[idx].is_null()) { + continue; + } else if (r1.storage_datums_[idx].is_null() || r2.storage_datums_[idx].is_null()) { + bool_ret = false; + } else if (r1.storage_datums_[idx] == r2.storage_datums_[idx]) { + continue; + } else { + bool_ret = false; + } + } + } + return bool_ret; +} + +bool ObMockIterator::equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2, + const bool cmp_multi_version_row_flag, const bool cmp_is_get_and_scan_index) +{ + int ret = OB_SUCCESS; + bool bool_ret = false; + STORAGE_LOG(INFO, "compare two rows", K(r1), K(r2)); + if (r1.row_flag_ != r2.row_flag_) { + STORAGE_LOG(WARN, "flag not equals", K(r1), K(r2)); + } else if (!cmp_multi_version_row_flag && (r1.row_flag_.is_not_exist() || r1.row_flag_.is_delete())) { + STORAGE_LOG(DEBUG, "flag is not row exist, return true", K(r1), K(r2)); + bool_ret = true; + } else if (cmp_multi_version_row_flag && r1.mvcc_row_flag_.flag_ != r2.mvcc_row_flag_.flag_) { + STORAGE_LOG(WARN, "row_type_flag not equals", K(r1), K(r2)); + bool_ret = false; + } else if (cmp_is_get_and_scan_index + && (r1.scan_index_ != r2.scan_index_)) { + STORAGE_LOG(WARN, "scan_index not equals", K(r1), K(r2)); + bool_ret = false; + } else { + bool_ret = inner_equals(r1, r2); + STORAGE_LOG(INFO, "compare two rows", K(bool_ret), K(r1), K(r2)); + } + return bool_ret; +} + +bool ObMockIterator::equals(int64_t idx, blocksstable::ObDatumRow &other_row) const +{ + bool bool_ret = false; + int ret = OB_SUCCESS; + const blocksstable::ObDatumRow *this_row = NULL; + if (OB_SUCCESS != get_row(idx, this_row)) { + STORAGE_LOG(WARN, "invalid idx"); + bool_ret = false; + } else { + bool_ret = inner_equals(*this_row, other_row); + if (!bool_ret) { + STORAGE_LOG(WARN, "store row is not equal", + K(idx), KPC(this_row), K(other_row)); + } + } + return bool_ret; +} + +bool ObMockIterator::equals(int64_t idx, const blocksstable::ObDatumRow &other_row) const +{ + return equals(idx, const_cast(other_row)); +} + bool ObMockIterator::equals(const ObNewRow &r1, const ObNewRow &r2) { bool bool_ret = true; @@ -334,6 +557,10 @@ ObHashMap ObMockIteratorBuilder::str_to_obj_parse_func_; ObHashMap ObMockIteratorBuilder::str_to_info_parse_func_; +ObHashMap +ObMockIteratorBuilder::str_to_datum_parse_func_; +ObHashMap +ObMockIteratorBuilder::str_to_datum_info_parse_func_; ObHashMap ObMockIteratorBuilder::str_to_obj_type_; ObHashMap ObMockIteratorBuilder::str_to_flag_; @@ -356,18 +583,46 @@ ObObjMeta ObMockIteratorBuilder::LOB_TYPE; ObObjMeta ObMockIteratorBuilder::TS_TYPE; ObObjMeta ObMockIteratorBuilder::NU_TYPE; -int ObMockIteratorBuilder::parse_varchar(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_varchar(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx) { OB_ASSERT(allocator); int ret = OB_SUCCESS; char *buf = NULL; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else if (NULL == (buf = static_cast(allocator->alloc(word.length())))) { ret = OB_ALLOCATE_MEMORY_FAILED; + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_varchar(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(prepare_parse_varchar(allocator, word, row.count_, idx))) { + STORAGE_LOG(WARN, "failed to parse datum varchar"); + } else { + ObStorageDatum datum_tmp; + datum_tmp.set_string(word); + ret = row.storage_datums_[idx++].deep_copy(datum_tmp, *allocator); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_varchar(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(prepare_parse_varchar(allocator, word, row.row_val_.count_, idx))) { + STORAGE_LOG(WARN, "failed to parse obj varchar"); } else { ObObj obj_tmp; obj_tmp.set_varchar(word); @@ -378,21 +633,22 @@ int ObMockIteratorBuilder::parse_varchar(ObIAllocator *allocator, return ret; } -int ObMockIteratorBuilder::parse_lob(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_lob(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + ObLobCommon *&lob_data, + int64_t &val) { OB_ASSERT(allocator); int ret = OB_SUCCESS; char *buf = NULL; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else if (NULL == (buf = static_cast(allocator->alloc(sizeof(ObLobCommon))))) { ret = OB_ALLOCATE_MEMORY_FAILED; } else { char str[MAX_DATA_LENGTH]; - int64_t val = 0; char *end_ptr = NULL; snprintf(str, MAX_DATA_LENGTH, "%.*s", word.length(), word.ptr()); @@ -416,18 +672,51 @@ int ObMockIteratorBuilder::parse_lob(ObIAllocator *allocator, ret = OB_NUMERIC_OVERFLOW; STORAGE_LOG(WARN, "too big lob index cnt", K(val), K(word)); } else { - ObLobCommon *lob_data = nullptr; - const int64_t block_size = 1 << 20; lob_data = new (buf) ObLobCommon(); - ObObj obj_tmp; - obj_tmp.set_lob_value(ObLongTextType, lob_data, lob_data->get_handle_size(val * block_size)); - obj_tmp.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); - ret = ob_write_obj(*allocator, obj_tmp, row.row_val_.cells_[idx++]); } } return ret; } +int ObMockIteratorBuilder::parse_datum_lob(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + ObLobCommon *lob_data = nullptr; + if (OB_FAIL(prepare_parse_lob(allocator, word, row.count_, idx, lob_data, val))) { + STORAGE_LOG(WARN, "failed to parse datum lob"); + } else { + const int64_t block_size = 1 << 20; + ObStorageDatum datum_tmp; + datum_tmp.set_lob_data(*lob_data, lob_data->get_handle_size(val * block_size)); + ret = row.storage_datums_[idx++].deep_copy(datum_tmp, *allocator); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_lob(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + ObLobCommon *lob_data = nullptr; + if (OB_FAIL(prepare_parse_lob(allocator, word, row.row_val_.count_, idx, lob_data, val))) { + STORAGE_LOG(WARN, "failed to parse obj lob"); + } else { + const int64_t block_size = 1 << 20; + ObObj obj_tmp; + obj_tmp.set_lob_value(ObLongTextType, lob_data, lob_data->get_handle_size(val * block_size)); + obj_tmp.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI); + ret = ob_write_obj(*allocator, obj_tmp, row.row_val_.cells_[idx++]); + } + return ret; +} + /* int ObMockIteratorBuilder::parse_bool(ObIAllocator *allocator, const ObString &word, ObStoreRow &row, int64_t &idx) @@ -460,39 +749,67 @@ int ObMockIteratorBuilder::parse_bool(ObIAllocator *allocator, } */ -int ObMockIteratorBuilder::parse_timestamp(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_timestamp(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &usec) { UNUSED(allocator); int ret = OB_SUCCESS; - int64_t usec = 0; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else if (OB_SUCCESS != (ret = ObTimeUtility2::str_to_usec(word, usec))) { STORAGE_LOG(WARN, "str to microsecond failed", K(word), K(usec), K(ret)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_timestamp(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t usec = 0; + if (OB_FAIL(prepare_parse_timestamp(allocator, word, row.count_, idx, usec))) { + STORAGE_LOG(WARN, "failed to parse datum timestamp"); + } else { + row.storage_datums_[idx++].set_timestamp(usec); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_timestamp(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t usec = 0; + if (OB_FAIL(prepare_parse_timestamp(allocator, word, row.row_val_.count_, idx, usec))) { + STORAGE_LOG(WARN, "failed to parse obj timestamp"); } else { row.row_val_.cells_[idx++].set_timestamp(usec); } return ret; } -int ObMockIteratorBuilder::parse_int(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_int(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &val) { UNUSED(allocator); OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else { char str[MAX_DATA_LENGTH]; - int64_t val = 0; char *end_ptr = NULL; snprintf(str, MAX_DATA_LENGTH, "%.*s", word.length(), word.ptr()); @@ -510,33 +827,59 @@ int ObMockIteratorBuilder::parse_int(ObIAllocator *allocator, || (INT_MAX < val || INT_MIN > val)) { STORAGE_LOG(WARN, "strtol fail", K(val), K(word), K(errno)); ret = OB_NUMERIC_OVERFLOW; - } else { - if (str == end_ptr) { - STORAGE_LOG(WARN, "no digits found"); - ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; - } else { - row.row_val_.cells_[idx++].set_int32(static_cast(val)); - } + } else if (str == end_ptr) { + STORAGE_LOG(WARN, "no digits found"); + ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; } } return ret; } -int ObMockIteratorBuilder::parse_bigint(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_datum_int(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_int(allocator, word, row.count_, idx, val))) { + STORAGE_LOG(WARN, "failed to parse datum int"); + } else { + row.storage_datums_[idx++].set_int32(static_cast(val)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_int(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_int(allocator, word, row.row_val_.count_, idx, val))) { + STORAGE_LOG(WARN, "failed to parse obj int"); + } else { + row.row_val_.cells_[idx++].set_int32(static_cast(val)); + } + return ret; +} + +int ObMockIteratorBuilder::prepare_parse_bigint(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &val) { UNUSED(allocator); OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else { char str[MAX_DATA_LENGTH]; - int64_t val = 0; char *end_ptr = NULL; snprintf(str, MAX_DATA_LENGTH, "%.*s", word.length(), word.ptr()); @@ -553,32 +896,86 @@ int ObMockIteratorBuilder::parse_bigint(ObIAllocator *allocator, || (errno != 0 && val == 0)) { STORAGE_LOG(WARN, "strtoll fail", K(val), K(word), K(errno)); ret = OB_NUMERIC_OVERFLOW; - } else { - if (str == end_ptr) { - STORAGE_LOG(WARN, "no digits found"); - ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; - } else { - row.row_val_.cells_[idx++].set_int(val); - } + } else if (str == end_ptr) { + STORAGE_LOG(WARN, "no digits found"); + ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; } } return ret; } -int ObMockIteratorBuilder::parse_number(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_datum_bigint(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_bigint(allocator, word, row.count_, idx, val))) { + STORAGE_LOG(WARN, "failed to parse datum bigint"); + } else { + row.storage_datums_[idx++].set_int(val); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_bigint(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_bigint(allocator, word, row.row_val_.count_, idx, val))) { + STORAGE_LOG(WARN, "failed to parse obj bigint"); + } else { + row.row_val_.cells_[idx++].set_int(val); + } + return ret; +} + +int ObMockIteratorBuilder::prepare_parse_number(ObIAllocator *allocator, + const ObString &word, + const uint16_t count, + int64_t &idx, + number::ObNumber &nmb) { OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; - number::ObNumber nmb; - if (idx >= row.row_val_.count_) { + if (idx >= count) { ret = OB_ARRAY_OUT_OF_RANGE; } else if (OB_SUCCESS != (ret = nmb.from(word.ptr(), word.length(), *allocator))) { STORAGE_LOG(WARN, "parse number failed", K(ret), K(word)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_number(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + number::ObNumber nmb; + if (OB_FAIL(prepare_parse_number(allocator, word, row.count_, idx, nmb))) { + STORAGE_LOG(WARN, "failed to parse datum int"); + } else { + row.storage_datums_[idx++].set_number(nmb); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_number(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + int ret = OB_SUCCESS; + number::ObNumber nmb; + if (OB_FAIL(prepare_parse_number(allocator, word, row.row_val_.count_, idx, nmb))) { + STORAGE_LOG(WARN, "failed to parse obj int"); } else { row.row_val_.cells_[idx++].set_number(nmb); } @@ -587,7 +984,16 @@ int ObMockIteratorBuilder::parse_number(ObIAllocator *allocator, int ObMockIteratorBuilder::parse_dml(ObIAllocator *allocator, const ObString &word, - ObStoreRow &row, + storage::ObStoreRow &row, + int64_t &idx) +{ + UNUSEDx(allocator, word, row, idx); + return OB_SUCCESS; +} + +int ObMockIteratorBuilder::parse_datum_dml(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, int64_t &idx) { UNUSEDx(allocator, word, row, idx); @@ -596,29 +1002,58 @@ int ObMockIteratorBuilder::parse_dml(ObIAllocator *allocator, int ObMockIteratorBuilder::parse_first_dml(ObIAllocator *allocator, const ObString &word, - ObStoreRow &row, + storage::ObStoreRow &row, int64_t &idx) { UNUSEDx(allocator, word, row, idx); return OB_SUCCESS; } -int ObMockIteratorBuilder::parse_flag(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::prepare_parse_flag(ObIAllocator *allocator, + const ObString &word, + int64_t &idx, + int64_t &flag) { - UNUSED(allocator); - UNUSED(idx); OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; - int64_t flag = 0; if (OB_SUCCESS != str_to_flag_.get_refactored(word, flag)) { STORAGE_LOG(WARN, "failed to parse flag", K(word)); ret = OB_HASH_NOT_EXIST; + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_flag(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + int ret = OB_SUCCESS; + int64_t flag = 0; + if (OB_FAIL(prepare_parse_flag(allocator, word, idx, flag))) { + STORAGE_LOG(WARN, "failed to parse datum row dml flag"); } else { - row.flag_ = (ObDmlFlag)flag; + row.row_flag_.set_flag((ObDmlFlag)flag); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_flag(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + int ret = OB_SUCCESS; + int64_t flag = 0; + if (OB_FAIL(prepare_parse_flag(allocator, word, idx, flag))) { + STORAGE_LOG(WARN, "failed to parse obj row dml flag"); + } else { + row.flag_.set_flag((ObDmlFlag)flag); } return ret; } @@ -659,10 +1094,10 @@ int ObMockIteratorBuilder::parse_is_get(ObIAllocator *allocator, return ret; } -int ObMockIteratorBuilder::parse_trans_id(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_datum_trans_id(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) { UNUSED(allocator); UNUSED(idx); @@ -670,7 +1105,7 @@ int ObMockIteratorBuilder::parse_trans_id(ObIAllocator *allocator, int ret = OB_SUCCESS; if (OB_SUCCESS != str_to_trans_id_.get_refactored(word, row.trans_id_)) { - STORAGE_LOG(WARN, "failed to parse is_get", K(word)); + STORAGE_LOG(WARN, "failed to parse trans_id", K(word)); ret = OB_HASH_NOT_EXIST; } else { STORAGE_LOG(DEBUG, "parse str_to_trans_id_", K(row), K(word), K(row.trans_id_)); @@ -678,17 +1113,35 @@ int ObMockIteratorBuilder::parse_trans_id(ObIAllocator *allocator, return ret; } -int ObMockIteratorBuilder::parse_scan_index(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_obj_trans_id(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); + + int ret = OB_SUCCESS; + if (OB_SUCCESS != str_to_trans_id_.get_refactored(word, row.trans_id_)) { + STORAGE_LOG(WARN, "failed to parse trans_id", K(word)); + ret = OB_HASH_NOT_EXIST; + } else { + STORAGE_LOG(DEBUG, "parse str_to_trans_id_", K(row), K(word), K(row.trans_id_)); + } + return ret; +} + +int ObMockIteratorBuilder::prepare_parse_scan_index(ObIAllocator *allocator, + const ObString &word, + int64_t &idx, + int64_t &val) { UNUSED(allocator); UNUSED(idx); OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); int ret = OB_SUCCESS; char str[MAX_DATA_LENGTH]; - int64_t val = 0; char *end_ptr = NULL; snprintf(str, MAX_DATA_LENGTH, "%.*s", word.length(), word.ptr()); @@ -705,22 +1158,70 @@ int ObMockIteratorBuilder::parse_scan_index(ObIAllocator *allocator, || (errno != 0 && val == 0)) { STORAGE_LOG(WARN, "strtoll fail", K(val), K(word), K(errno)); ret = OB_NUMERIC_OVERFLOW; - } else { - if (str == end_ptr) { - STORAGE_LOG(WARN, "no digits found"); - ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; - } else { - row.scan_index_ = val; - STORAGE_LOG(DEBUG, "parse scan_index", K(row)); - } + } else if (str == end_ptr) { + STORAGE_LOG(WARN, "no digits found"); + ret = OB_ERR_CAST_VARCHAR_TO_NUMBER; } return ret; } -int ObMockIteratorBuilder::parse_multi_version_row_flag(ObIAllocator *allocator, - const ObString &word, - ObStoreRow &row, - int64_t &idx) +int ObMockIteratorBuilder::parse_datum_scan_index(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_scan_index(allocator, word, idx, val))) { + STORAGE_LOG(WARN, "failed to parse datum row scan_index"); + } else { + row.scan_index_ = val; + STORAGE_LOG(DEBUG, "parse scan_index", K(row)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_scan_index(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + int ret = OB_SUCCESS; + int64_t val = 0; + if (OB_FAIL(prepare_parse_scan_index(allocator, word, idx, val))) { + STORAGE_LOG(WARN, "failed to parse obj row scan_index"); + } else { + row.scan_index_ = val; + STORAGE_LOG(DEBUG, "parse scan_index", K(row)); + } + return ret; +} + +int ObMockIteratorBuilder::parse_datum_multi_version_row_flag(ObIAllocator *allocator, + const ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx) +{ + UNUSED(allocator); + UNUSED(idx); + OB_ASSERT(NULL != word.ptr() && 0 <= word.length()); + + int ret = OB_SUCCESS; + if (OB_SUCCESS != str_to_multi_version_row_flag_.get_refactored(word, row.mvcc_row_flag_.flag_)) { + STORAGE_LOG(WARN, "failed to parse multi version row flag for datum row", K(word)); + ret = OB_HASH_NOT_EXIST; + } + return ret; +} + +int ObMockIteratorBuilder::parse_obj_multi_version_row_flag(ObIAllocator *allocator, + const ObString &word, + storage::ObStoreRow &row, + int64_t &idx) { UNUSED(allocator); UNUSED(idx); @@ -728,7 +1229,7 @@ int ObMockIteratorBuilder::parse_multi_version_row_flag(ObIAllocator *allocator, int ret = OB_SUCCESS; if (OB_SUCCESS != str_to_multi_version_row_flag_.get_refactored(word, row.row_type_flag_.flag_)) { - STORAGE_LOG(WARN, "failed to parse multi version row flag", K(word)); + STORAGE_LOG(WARN, "failed to parse multi version row flag for obj row", K(word)); ret = OB_HASH_NOT_EXIST; } return ret; @@ -754,37 +1255,74 @@ int ObMockIteratorBuilder::static_init() trans_id_list_[i - 1] = transaction::ObTransID(i); } + if (OB_SUCCESS != (ret = str_to_datum_parse_func_.create( + cal_next_prime(TYPE_NUM * 10), + ObModIds::OB_HASH_BUCKET))) { + STORAGE_LOG(WARN, "out of memory"); + } else if (OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("int"), + ObMockIteratorBuilder::parse_datum_int) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("bigint"), + ObMockIteratorBuilder::parse_datum_bigint) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("varchar"), + ObMockIteratorBuilder::parse_datum_varchar) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("lob"), + ObMockIteratorBuilder::parse_datum_lob) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("var"), + ObMockIteratorBuilder::parse_datum_varchar) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("ts"), + ObMockIteratorBuilder::parse_datum_timestamp) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("timestamp"), + ObMockIteratorBuilder::parse_datum_timestamp) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("num"), + ObMockIteratorBuilder::parse_datum_number) + || OB_SUCCESS != str_to_datum_parse_func_.set_refactored( + ObString::make_string("number"), + ObMockIteratorBuilder::parse_datum_number)) { + ret = OB_INIT_FAIL; + STORAGE_LOG(WARN, "obj parse func hashtable insert failed"); + } else { + STORAGE_LOG(DEBUG, "init obj parse func hashtable"); + } + if (OB_SUCCESS != (ret = str_to_obj_parse_func_.create( cal_next_prime(TYPE_NUM * 10), ObModIds::OB_HASH_BUCKET))) { STORAGE_LOG(WARN, "out of memory"); } else if (OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("int"), - ObMockIteratorBuilder::parse_int) + ObMockIteratorBuilder::parse_obj_int) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("bigint"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("varchar"), - ObMockIteratorBuilder::parse_varchar) + ObMockIteratorBuilder::parse_obj_varchar) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("lob"), - ObMockIteratorBuilder::parse_lob) + ObMockIteratorBuilder::parse_obj_lob) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("var"), - ObMockIteratorBuilder::parse_varchar) + ObMockIteratorBuilder::parse_obj_varchar) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("ts"), - ObMockIteratorBuilder::parse_timestamp) + ObMockIteratorBuilder::parse_obj_timestamp) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("timestamp"), - ObMockIteratorBuilder::parse_timestamp) + ObMockIteratorBuilder::parse_obj_timestamp) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("num"), - ObMockIteratorBuilder::parse_number) + ObMockIteratorBuilder::parse_obj_number) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("number"), - ObMockIteratorBuilder::parse_number)) { + ObMockIteratorBuilder::parse_obj_number)) { ret = OB_INIT_FAIL; STORAGE_LOG(WARN, "obj parse func hashtable insert failed"); } else { @@ -819,13 +1357,38 @@ int ObMockIteratorBuilder::static_init() STORAGE_LOG(DEBUG, "init obj type hashtable"); } + if (OB_SUCCESS != (ret = str_to_datum_info_parse_func_.create( + cal_next_prime(INFO_NUM), + ObModIds::OB_HASH_BUCKET))) { + STORAGE_LOG(DEBUG, "out of memory"); + } else if (OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("flag"), + ObMockIteratorBuilder::parse_datum_flag) + || OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("dml"), + ObMockIteratorBuilder::parse_datum_dml) + || OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("multi_version_row_flag"), + ObMockIteratorBuilder::parse_datum_multi_version_row_flag) + || OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("scan_index"), + ObMockIteratorBuilder::parse_datum_scan_index) + || OB_SUCCESS != str_to_datum_info_parse_func_.set_refactored( + ObString::make_string("trans_id"), + ObMockIteratorBuilder::parse_datum_trans_id)) { + ret = OB_INIT_FAIL; + STORAGE_LOG(WARN, "info parse func hashtable insert failed"); + } else { + STORAGE_LOG(DEBUG, "init info parse func hashtable"); + } + if (OB_SUCCESS != (ret = str_to_info_parse_func_.create( cal_next_prime(INFO_NUM), ObModIds::OB_HASH_BUCKET))) { STORAGE_LOG(DEBUG, "out of memory"); } else if (OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("flag"), - ObMockIteratorBuilder::parse_flag) + ObMockIteratorBuilder::parse_obj_flag) || OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("dml"), ObMockIteratorBuilder::parse_dml) @@ -834,19 +1397,13 @@ int ObMockIteratorBuilder::static_init() ObMockIteratorBuilder::parse_first_dml) || OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("multi_version_row_flag"), - ObMockIteratorBuilder::parse_multi_version_row_flag) - || OB_SUCCESS != str_to_info_parse_func_.set_refactored( - ObString::make_string("is_get"), - ObMockIteratorBuilder::parse_is_get) + ObMockIteratorBuilder::parse_obj_multi_version_row_flag) || OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("scan_index"), - ObMockIteratorBuilder::parse_scan_index) - || OB_SUCCESS != str_to_info_parse_func_.set_refactored( - ObString::make_string("from_base"), - ObMockIteratorBuilder::parse_base) + ObMockIteratorBuilder::parse_obj_scan_index) || OB_SUCCESS != str_to_info_parse_func_.set_refactored( ObString::make_string("trans_id"), - ObMockIteratorBuilder::parse_trans_id)) { + ObMockIteratorBuilder::parse_obj_trans_id)) { ret = OB_INIT_FAIL; STORAGE_LOG(WARN, "info parse func hashtable insert failed"); } else { @@ -907,36 +1464,6 @@ int ObMockIteratorBuilder::static_init() STORAGE_LOG(DEBUG, "init flag hashtable"); } - if (ret == OB_SUCCESS - && OB_SUCCESS != (ret = str_to_base_.create( - cal_next_prime(BASE_NUM * 2), - ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memory"); - } else if (OB_SUCCESS != str_to_base_.set_refactored( - ObString::make_string("TRUE"), true) - || OB_SUCCESS != str_to_base_.set_refactored( - ObString::make_string("FALSE"), false)) { - ret = OB_INIT_FAIL; - STORAGE_LOG(WARN, "from_base hashtable insert failed"); - } else { - STORAGE_LOG(DEBUG, "init from_base hashtable"); - } - - if (ret == OB_SUCCESS - && OB_SUCCESS != (ret = str_to_is_get_.create( - cal_next_prime(BASE_NUM * 2), - ObModIds::OB_HASH_BUCKET))) { - STORAGE_LOG(WARN, "out of memory"); - } else if (OB_SUCCESS != str_to_is_get_.set_refactored( - ObString::make_string("TRUE"), true) - || OB_SUCCESS != str_to_is_get_.set_refactored( - ObString::make_string("FALSE"), false)) { - ret = OB_INIT_FAIL; - STORAGE_LOG(WARN, "is_get hashtable insert failed"); - } else { - STORAGE_LOG(DEBUG, "init is_get hashtable"); - } - if (ret == OB_SUCCESS && OB_SUCCESS != (ret = str_to_trans_id_.create( cal_next_prime(BASE_NUM * 3), @@ -1006,22 +1533,22 @@ int ObMockIteratorBuilder::static_init() // for sstable row if (OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("table_type"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("start_scn"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("end_scn"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("max_ver"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("upper_ver"), - ObMockIteratorBuilder::parse_bigint) + ObMockIteratorBuilder::parse_obj_bigint) || OB_SUCCESS != str_to_obj_parse_func_.set_refactored( ObString::make_string("row_cnt"), - ObMockIteratorBuilder::parse_bigint)) { + ObMockIteratorBuilder::parse_obj_bigint)) { ret = OB_INIT_FAIL; STORAGE_LOG(WARN, "obj parse func hashtable insert failed"); } else { @@ -1070,6 +1597,65 @@ int ObMockIteratorBuilder::init(ObIAllocator *allocator, char escape) return ret; } +int ObMockIteratorBuilder::parse_for_datum( + const ObString &str, + ObMockIterator &iter, + uint16_t *col_id_array_list) +{ + int ret = OB_SUCCESS; + if (!is_inited_) { + ret = OB_NOT_INIT; + } else { + int64_t pos = 0; + int64_t obj_num = 0; + ObSEArray header; + iter.reset(); + ret = parse_datum_header(str, pos, header, obj_num, iter); + if (OB_ITER_END == ret) { + ret = OB_EMPTY_RESULT; + STORAGE_LOG(WARN, "no data input"); + } else if (OB_FAIL(ret)) { + //OB_ASSERT(false); + // will not happen + STORAGE_LOG(WARN, "parse header error", K(ret)); + } else if (0 == obj_num) { + ret = OB_EMPTY_RESULT; + STORAGE_LOG(WARN, "no data(such as int, var) col", K(str)); + } else if (OB_FAIL(iter.set_column_cnt(obj_num))) { + STORAGE_LOG(WARN, "set column cnt failed", K(ret), K(obj_num)); + } else { + STORAGE_LOG(TRACE, "get header success", K(header.count()), K(obj_num)); + int idx = 0; + while (OB_SUCC(ret)) { + ObDatumRow *row = NULL; + if (OB_FAIL(malloc_datum_row(*allocator_, obj_num, row))) { + STORAGE_LOG(WARN, "failed to malloc store row", K(ret)); + } else { + // set default value + row->row_flag_.set_flag(ObDmlFlag::DF_INSERT); + for (int64_t i = 0; i < obj_num; ++i) { + row->storage_datums_[i].set_nop(); + } + // parse one row + ret = parse_datum_row(str, pos, header, col_id_array_list, *row); + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + STORAGE_LOG(INFO, "parse successfully"); + break; + } else if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "failed to get row", K(ret), K(pos)); + } else { + STORAGE_LOG(WARN, "add row", K(ret), KPC(row)); + iter.add_row(row); + } + } + + } // end of while + } // end of else + } // end of is_inited_'s else + return ret; +} + int ObMockIteratorBuilder::parse( const ObString &str, ObMockIterator &iter, @@ -1128,6 +1714,68 @@ int ObMockIteratorBuilder::parse( return ret; } +int ObMockIteratorBuilder::parse_datum_header(const ObString &str, + int64_t &pos, + ObIArray &header, + int64_t &obj_num, + ObMockIterator &iter) +{ + OB_ASSERT(is_inited_); + + int ret = OB_SUCCESS; + int64_t ext = NOT_EXT; + obj_num = 0; + while (OB_SUCCESS == ret && pos < str.length()) { + char buf[MAX_DATA_LENGTH]; + ObString word; + word.assign_buffer(buf, MAX_DATA_LENGTH); + ret = get_next_word(str, pos, word, ext); + if (OB_FAIL(ret)) { + // will be OB_ITER_END only + STORAGE_LOG(WARN, "fail", K(ret), K(str), K(pos), K(word), K(obj_num)); + } else if (obj_num >= OB_ROW_MAX_COLUMNS_COUNT) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "too many columns", K(ret), K(obj_num)); + } else if (EXT_END == ext) { + // header parse end + break; + } else { + ObParseDatumFunc fp = NULL; + ObObjMeta *type = NULL; + // tolower, as case must be ignore in following hash get function + for (int64_t i = 0; i < word.length(); i++) { + /* + if (word.ptr()[i] >= 'A' && word.ptr()[i] <= 'Z') { + word.ptr()[i] = static_cast(word.ptr()[i] - 'A' + 'a'); + } + */ + word.ptr()[i] = static_cast(tolower(word.ptr()[i])); + } + //STORAGE_LOG(WARN, "chaser debug", K(word), K(ret)); + // match data col, such as var, int, num... + if (OB_SUCCESS == str_to_datum_parse_func_.get_refactored(word, fp) && NULL != fp) { + if (OB_SUCCESS != str_to_obj_type_.get_refactored(word, type)) { + STORAGE_LOG(WARN, "get obj type from hashmap failed", K(ret)); + } else if (NULL == type) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "type is null", K(ret)); + } else { + iter.set_column_type(obj_num, *type); + ret = header.push_back(fp); + ++obj_num; + } + // match row info, such as dml, flag and from_base + } else if (OB_SUCCESS == str_to_datum_info_parse_func_.get_refactored(word, fp) + && NULL != fp) { + ret = header.push_back(fp); + } else { + ret = OB_OBJ_TYPE_ERROR; + } + } + } + return ret; +} + int ObMockIteratorBuilder::parse_header(const ObString &str, int64_t &pos, ObIArray &header, @@ -1190,6 +1838,76 @@ int ObMockIteratorBuilder::parse_header(const ObString &str, return ret; } +int ObMockIteratorBuilder::parse_datum_with_specified_col_ids( + const ObString &str, + ObMockIterator &iter, + uint16_t *col_id_array_list/* = nullptr*/, + int64_t *result_col_id_array/* = nullptr*/) +{ + int ret = OB_SUCCESS; + if (!is_inited_) { + ret = OB_NOT_INIT; + } else { + int64_t pos = 0; + int64_t obj_num = 0; + ObSEArray header; + iter.reset(); + ret = parse_datum_header(str, pos, header, obj_num, iter); + if (OB_ITER_END == ret) { + ret = OB_EMPTY_RESULT; + STORAGE_LOG(WARN, "no data input"); + } else if (OB_FAIL(ret)) { + //OB_ASSERT(false); + // will not happen + STORAGE_LOG(WARN, "parse header error", K(ret)); + } else if (0 == obj_num) { + ret = OB_EMPTY_RESULT; + STORAGE_LOG(WARN, "no data(such as int, var) col"); + } else if (OB_FAIL(iter.set_column_cnt(obj_num))) { + STORAGE_LOG(WARN, "set column cnt failed", K(ret), K(obj_num)); + } else { + STORAGE_LOG(TRACE, "get header success", K(header.count()), K(obj_num)); + bool init_col_id_flag = false; + if (OB_NOT_NULL(col_id_array_list) && OB_NOT_NULL(result_col_id_array)) { + init_col_id_flag = true; + } + int idx = 0; + int col_id_pos = 0; + while (OB_SUCC(ret)) { + ObDatumRow *row = NULL; + if (OB_FAIL(malloc_datum_row(*allocator_, obj_num, row))) { + STORAGE_LOG(WARN, "failed to malloc store row", K(ret)); + } else { + // set default value + row->row_flag_.set_flag(ObDmlFlag::DF_INSERT); + for (int64_t i = 0; i < obj_num; ++i) { + row->storage_datums_[i].set_nop(); + } + // parse one row + ret = parse_datum_row(str, pos, header, nullptr, *row); + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + STORAGE_LOG(INFO, "parse successfully"); + break; + } else if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "failed to get row", K(ret), K(pos)); + } else { + if (init_col_id_flag) { + row->count_ = result_col_id_array[idx]; + col_id_pos += result_col_id_array[idx]; // move forward + STORAGE_LOG(TRACE, "get row successfully", K(idx), K(*row), K(row->trans_id_)); + ++idx; + } + iter.add_row(row); + } + } + + } // end of while + } // end of else + } // end of is_inited_'s else + return ret; +} + int ObMockIteratorBuilder::parse_with_specified_col_ids( const ObString &str, ObMockIterator &iter, @@ -1262,6 +1980,85 @@ int ObMockIteratorBuilder::parse_with_specified_col_ids( return ret; } +int ObMockIteratorBuilder::parse_datum_row(const ObString &str, + int64_t &pos, + const ObIArray &header, + const uint16_t *col_id_array_list, + blocksstable::ObDatumRow &row) +{ + OB_ASSERT(is_inited_); + + int ret = OB_SUCCESS; + int64_t idx = 0; + int64_t ext = NOT_EXT; + int64_t col_id_idx = 0; + // i <= header.count() so row end is included + for (int64_t i = 0; OB_SUCC(ret) && i <= header.count(); ++i) { + char buf[MAX_DATA_LENGTH]; + ObString word; + ObParseDatumFunc fp = NULL; + word.assign_buffer(buf, MAX_DATA_LENGTH); + if (OB_SUCCESS != (ret = get_next_word(str, pos, word, ext))) { + // must be OB_ITER_END only + break; + } else if (EXT_END == ext) { + // check col num if flag is RF_ROW_EXIST + break; + } else if (OB_SUCCESS != (ret = header.at(i, fp))) { + STORAGE_LOG(WARN, "invalid array index"); + } else { + //STORAGE_LOG(DEBUG, "parsing word", K(word)); + switch (ext) { + case EXT_NOP: + row.storage_datums_[idx++].set_nop(); + break; + case EXT_NULL: + row.storage_datums_[idx++].set_null(); + break; + case EXT_MAX: + if (ObMockIteratorBuilder::parse_datum_int != fp && ObMockIteratorBuilder::parse_datum_bigint != fp) { + row.storage_datums_[idx++].set_max(); + } else { + row.storage_datums_[idx++].set_int(INT64_MAX); + } + break; + case EXT_MIN: + if (ObMockIteratorBuilder::parse_datum_int != fp && ObMockIteratorBuilder::parse_datum_bigint != fp) { + row.storage_datums_[idx++].set_min(); + } else { + row.storage_datums_[idx++].set_int(-INT64_MAX); + } + break; + case EXT_MIN_2_TRANS: + if (ObMockIteratorBuilder::parse_datum_int != fp && ObMockIteratorBuilder::parse_datum_bigint != fp) { + row.storage_datums_[idx++].set_min(); + } else { + row.storage_datums_[idx++].set_int(-(INT64_MAX - 7)); + } + break; + case EXT_GHOST: + row.storage_datums_[idx++].set_int(ObGhostRowUtil::GHOST_NUM); + break; + case NOT_EXT: + // use parse func to parse a word + ret = (*fp)(allocator_, word, row, idx); + if (OB_ARRAY_OUT_OF_RANGE == ret) { + STORAGE_LOG(WARN, "data col out of range, you may missing '\\n'?"); + } else if (OB_FAIL(ret)) { + STORAGE_LOG(WARN, "failed to parse word", K(ret), K(word)); + } + break; + default: + OB_ASSERT(false); + } // end of switch + } // end of else + } // end of for + if (OB_SUCC(ret)) { + row.count_ = idx; + } + return ret; +} + int ObMockIteratorBuilder::parse_row(const ObString &str, int64_t &pos, const ObIArray &header, @@ -1302,21 +2099,21 @@ int ObMockIteratorBuilder::parse_row(const ObString &str, row.row_val_.cells_[idx++].set_null(); break; case EXT_MAX: - if (ObMockIteratorBuilder::parse_int != fp && ObMockIteratorBuilder::parse_bigint != fp) { + if (ObMockIteratorBuilder::parse_obj_int != fp && ObMockIteratorBuilder::parse_obj_bigint != fp) { row.row_val_.cells_[idx++].set_max_value(); } else { row.row_val_.cells_[idx++].set_int(INT64_MAX); } break; case EXT_MIN: - if (ObMockIteratorBuilder::parse_int != fp && ObMockIteratorBuilder::parse_bigint != fp) { + if (ObMockIteratorBuilder::parse_obj_int != fp && ObMockIteratorBuilder::parse_obj_bigint != fp) { row.row_val_.cells_[idx++].set_min_value(); } else { row.row_val_.cells_[idx++].set_int(-INT64_MAX); } break; case EXT_MIN_2_TRANS: - if (ObMockIteratorBuilder::parse_int != fp && ObMockIteratorBuilder::parse_bigint != fp) { + if (ObMockIteratorBuilder::parse_obj_int != fp && ObMockIteratorBuilder::parse_obj_bigint != fp) { row.row_val_.cells_[idx++].set_min_value(); } else { row.row_val_.cells_[idx++].set_int(-(INT64_MAX - 7)); @@ -1338,7 +2135,7 @@ int ObMockIteratorBuilder::parse_row(const ObString &str, OB_ASSERT(false); } // end of switch if (OB_SUCC(ret) && row.is_sparse_row_) { - if (EXT_NOP != ext && parse_flag != fp && parse_multi_version_row_flag != fp && parse_trans_id != fp) { + if (EXT_NOP != ext && parse_obj_flag != fp && parse_obj_multi_version_row_flag != fp && parse_obj_trans_id != fp) { row.column_ids_[idx - 1] = col_id_array_list[col_id_idx++]; } } @@ -1487,111 +2284,111 @@ int ObMockIteratorBuilder::handle_escape(const ObString &str, int64_t &pos, char return ret; } -MockObNewRowIterator::MockObNewRowIterator() -{ - allocator_.set_label(ObModIds::OB_TRANS_CHECK); -} +// MockObNewRowIterator::MockObNewRowIterator() +// { +// allocator_.set_label(ObModIds::OB_TRANS_CHECK); +// } -MockObNewRowIterator::~MockObNewRowIterator() -{ - allocator_.free(); -} +// MockObNewRowIterator::~MockObNewRowIterator() +// { +// allocator_.free(); +// } -void MockObNewRowIterator::reset() -{ - allocator_.free(); - return iter_.reset(); -} +// void MockObNewRowIterator::reset() +// { +// allocator_.free(); +// return iter_.reset(); +// } -bool MockObNewRowIterator::equals(const ObNewRow &r1, const ObNewRow &r2) -{ - bool bool_ret = true; - int64_t idx = 0; - if (!r1.cells_ || !r2.cells_) { - bool_ret = false; - } else if (r1.count_ != r2.count_) { - bool_ret = false; - } else { - for (idx = 0; idx < r1.count_ && bool_ret; idx++) { - bool_ret = (r1.cells_[idx] == r2.cells_[idx]); - } - } - return bool_ret; -} +// bool MockObNewRowIterator::equals(const ObNewRow &r1, const ObNewRow &r2) +// { +// bool bool_ret = true; +// int64_t idx = 0; +// if (!r1.cells_ || !r2.cells_) { +// bool_ret = false; +// } else if (r1.count_ != r2.count_) { +// bool_ret = false; +// } else { +// for (idx = 0; idx < r1.count_ && bool_ret; idx++) { +// bool_ret = (r1.cells_[idx] == r2.cells_[idx]); +// } +// } +// return bool_ret; +// } -OB_DEF_SERIALIZE(MockObNewRowIterator) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, iter_.count()))) { - TRANS_LOG(WARN, "MockObNewRowIterator serialize error", K(ret), K(iter_.count())); - } else { - ObStoreRow *row = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < iter_.count(); i++) { - if (OB_SUCCESS != (ret = get_row(i, row))) { - TRANS_LOG(WARN, "MockObNewRowIterator get row error", K(ret), K(i)); - } else if (OB_SUCCESS != (ret = row->serialize(buf, buf_len, pos))) { - TRANS_LOG(WARN, "MockObNewRowIterator serialize ObStoreRow error", K(ret), K(row->row_val_)); - } else { - TRANS_LOG(DEBUG, "MockObNewRowIterator serialize ObStoreRow success", K(row->row_val_)); - } - } - } +// OB_DEF_SERIALIZE(MockObNewRowIterator) +// { +// int ret = OB_SUCCESS; +// if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, iter_.count()))) { +// TRANS_LOG(WARN, "MockObNewRowIterator serialize error", K(ret), K(iter_.count())); +// } else { +// ObStoreRow *row = NULL; +// for (int64_t i = 0; OB_SUCC(ret) && i < iter_.count(); i++) { +// if (OB_SUCCESS != (ret = get_row(i, row))) { +// TRANS_LOG(WARN, "MockObNewRowIterator get row error", K(ret), K(i)); +// } else if (OB_SUCCESS != (ret = row->serialize(buf, buf_len, pos))) { +// TRANS_LOG(WARN, "MockObNewRowIterator serialize ObStoreRow error", K(ret), K(row->row_val_)); +// } else { +// TRANS_LOG(DEBUG, "MockObNewRowIterator serialize ObStoreRow success", K(row->row_val_)); +// } +// } +// } - return ret; -} +// return ret; +// } -OB_DEF_DESERIALIZE(MockObNewRowIterator) -{ - int ret = OB_SUCCESS; - int64_t count = 0; +// OB_DEF_DESERIALIZE(MockObNewRowIterator) +// { +// int ret = OB_SUCCESS; +// int64_t count = 0; - reset(); +// reset(); - if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &count))) { - TRANS_LOG(WARN, "MockObNewRowIterator deserialize row size error", K(ret), K(count)); - } else { - TRANS_LOG(INFO, "MockObNewRowIterator deserialize iter count", K(count)); - for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { - ObStoreRow row; - ObObj *ptr = (ObObj*)allocator_.alloc(sizeof(ObObj) * OB_ROW_MAX_COLUMNS_COUNT); - if (NULL == ptr) { - TRANS_LOG(WARN, "MockObNewRowIterator alloc ObObj error", K(ret)); - ret = OB_ALLOCATE_MEMORY_FAILED; - } else { - row.row_val_.assign(ptr, OB_ROW_MAX_COLUMNS_COUNT); - if (OB_SUCCESS != (ret = row.deserialize(buf, data_len, pos))) { - TRANS_LOG(WARN, "MockObNewRowIterator deserialize ObNewRow error", K(ret)); - } else { - if (OB_SUCCESS != (ret = add_row(&row))) { - TRANS_LOG(WARN, "MockObNewRowIterator deserialize add row error", K(ret), K(row.row_val_)); - } else { - TRANS_LOG(DEBUG, "MockObNewRowIterator deserialize add row success", K(row.row_val_)); - } - } - } - } - } +// if (OB_FAIL(serialization::decode_vi64(buf, data_len, pos, &count))) { +// TRANS_LOG(WARN, "MockObNewRowIterator deserialize row size error", K(ret), K(count)); +// } else { +// TRANS_LOG(INFO, "MockObNewRowIterator deserialize iter count", K(count)); +// for (int64_t i = 0; OB_SUCC(ret) && i < count; i++) { +// ObStoreRow row; +// ObObj *ptr = (ObObj*)allocator_.alloc(sizeof(ObObj) * OB_ROW_MAX_COLUMNS_COUNT); +// if (NULL == ptr) { +// TRANS_LOG(WARN, "MockObNewRowIterator alloc ObObj error", K(ret)); +// ret = OB_ALLOCATE_MEMORY_FAILED; +// } else { +// row.row_val_.assign(ptr, OB_ROW_MAX_COLUMNS_COUNT); +// if (OB_SUCCESS != (ret = row.deserialize(buf, data_len, pos))) { +// TRANS_LOG(WARN, "MockObNewRowIterator deserialize ObNewRow error", K(ret)); +// } else { +// if (OB_SUCCESS != (ret = add_row(&row))) { +// TRANS_LOG(WARN, "MockObNewRowIterator deserialize add row error", K(ret), K(row.row_val_)); +// } else { +// TRANS_LOG(DEBUG, "MockObNewRowIterator deserialize add row success", K(row.row_val_)); +// } +// } +// } +// } +// } - return ret; -} +// return ret; +// } -OB_DEF_SERIALIZE_SIZE(MockObNewRowIterator) -{ - int ret = OB_SUCCESS; - int64_t len = 0; +// OB_DEF_SERIALIZE_SIZE(MockObNewRowIterator) +// { +// int ret = OB_SUCCESS; +// int64_t len = 0; - len += serialization::encoded_length_vi64(iter_.count()); - for (int64_t i = 0; OB_SUCC(ret) && i < iter_.count(); i++) { - ObStoreRow *row = NULL; - if (OB_SUCCESS != (ret = get_row(i, row))) { - TRANS_LOG(WARN, "MockObNewRowIterator get_serialize size get row error", K(ret), K(i)); - } else { - len += row->get_serialize_size(); - } - } +// len += serialization::encoded_length_vi64(iter_.count()); +// for (int64_t i = 0; OB_SUCC(ret) && i < iter_.count(); i++) { +// ObStoreRow *row = NULL; +// if (OB_SUCCESS != (ret = get_row(i, row))) { +// TRANS_LOG(WARN, "MockObNewRowIterator get_serialize size get row error", K(ret), K(i)); +// } else { +// len += row->get_serialize_size(); +// } +// } - return len; -} +// return len; +// } int ObMockDirectReadIterator::init(ObStoreRowIterator *iter, common::ObIAllocator &alloc, diff --git a/unittest/storage/mockcontainer/mock_ob_iterator.h b/unittest/storage/mockcontainer/mock_ob_iterator.h index 00dd49a8a..8f1806ccb 100644 --- a/unittest/storage/mockcontainer/mock_ob_iterator.h +++ b/unittest/storage/mockcontainer/mock_ob_iterator.h @@ -23,6 +23,7 @@ #include "storage/access/ob_store_row_iterator.h" #include "storage/access/ob_table_read_info.h" #include "storage/access/ob_sstable_row_whole_scanner.h" +#include "storage/blocksstable/ob_datum_row_iterator.h" namespace oceanbase { @@ -39,10 +40,16 @@ public: int64_t count() const { return rows_.count(); } // get row at idx, do not move iter + int get_row(const int64_t idx, const blocksstable::ObDatumRow *&row) const; + int get_row(const int64_t idx, blocksstable::ObDatumRow *&row) const; + // get row at idx, do not move iter int get_row(const int64_t idx, const storage::ObStoreRow *&row) const; int get_row(const int64_t idx, storage::ObStoreRow *&row) const; int get_row(const int64_t idx, common::ObNewRow *&row) const; // get row and move iter to the next + int get_next_row(const blocksstable::ObDatumRow *&row); + int get_next_row(blocksstable::ObDatumRow *&row); + // get row and move iter to the next int get_next_row(const storage::ObStoreRow *&row); int get_next_row(storage::ObStoreRow *&row); int get_next_row(common::ObNewRow *&row); @@ -52,6 +59,7 @@ public: // rewind iter void reset_iter(); + int add_row(blocksstable::ObDatumRow *row); int add_row(storage::ObStoreRow *row); int from( const char *cstr, @@ -63,6 +71,24 @@ public: char escape = '\\', uint16_t* col_id_array = nullptr, int64_t *result_col_id_array = nullptr); + int from_for_datum( + const char *cstr, + char escape = '\\', + uint16_t* col_id_array = nullptr, + int64_t *result_col_id_array = nullptr); + int from_for_datum( + const common::ObString &str, + char escape = '\\', + uint16_t* col_id_array = nullptr, + int64_t *result_col_id_array = nullptr); + + static bool inner_equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2); + static bool equals(const blocksstable::ObDatumRow &r1, const blocksstable::ObDatumRow &r2, + const bool cmp_multi_version_row_flag = false, const bool cmp_is_get_and_scan_index = false); + bool equals(int64_t idx, blocksstable::ObDatumRow &row) const; + bool equals(int64_t idx, const blocksstable::ObDatumRow &row) const; + bool equals(blocksstable::ObDatumRow &other_row) const { return equals(0, other_row); } + bool equals(const blocksstable::ObDatumRow &other_row) const { return equals(0, other_row); } static bool equals(const common::ObNewRow &r1, const common::ObNewRow &r2); static bool equals(const storage::ObStoreRow &r1, const storage::ObStoreRow &r2, @@ -97,15 +123,15 @@ public: return ret; } - template + template bool equals(T &other_iter, const bool cmp_multi_version_row_flag = false, const bool cmp_is_get_and_scan_index = false) { bool bool_ret = true; int ret1 = common::OB_SUCCESS; int ret2 = common::OB_SUCCESS; - const storage::ObStoreRow *other_row = NULL; - storage::ObStoreRow *this_row = NULL; + const T_ROW *other_row = NULL; + T_ROW *this_row = NULL; int64_t idx = 0; while (bool_ret) { @@ -155,7 +181,7 @@ private: void setup_start_cursor(); void advance(); bool end_of_row() const; - + common::ObSEArray datum_rows_; common::ObSEArray rows_; int64_t column_cnt_; common::ObObjMeta metas_[common::OB_ROW_MAX_COLUMNS_COUNT]; @@ -177,6 +203,8 @@ public: void reset_iter() { return iter_.reset_iter(); } int from(const char *cstr, char escape = '\\') { return iter_.from(cstr, escape); } int from(const common::ObString &str, char escape = '\\') { return iter_.from(str, escape); } + int from_for_datum(const char *cstr, char escape = '\\') { return iter_.from_for_datum(cstr, escape); } + int from_for_datum(const common::ObString &str, char escape = '\\') { return iter_.from_for_datum(str, escape); } bool equals(ROW_TYPE &row) const { return iter_.equals(row); } bool equals(int64_t idx, ROW_TYPE &row) const { return iter_.equals(idx, row); } // compare to an iterator, rewind the iter before call this function @@ -219,10 +247,10 @@ public: private: ObMockIterator iter_; }; -typedef ObMockRowIterator -ObMockStoreRowIterator; +typedef ObMockRowIterator ObMockStoreRowIterator; typedef ObMockRowIterator ObMockQueryRowIterator; typedef ObMockRowIterator ObMockNewRowIterator; +typedef ObMockRowIterator ObMockDatumRowIterator; // init // parse -> parse->header @@ -270,6 +298,10 @@ public: static common::ObObjMeta TS_TYPE; static common::ObObjMeta NU_TYPE; + typedef int (*ObParseDatumFunc)(common::ObIAllocator *, + const common::ObString &, + blocksstable::ObDatumRow &, + int64_t &); typedef int (*ObParseFunc)(common::ObIAllocator *, const common::ObString &, storage::ObStoreRow &, @@ -281,10 +313,19 @@ public: ~ObMockIteratorBuilder() {} int init(common::ObIAllocator *allocator = NULL, char escape = '\\'); + int parse_for_datum( + const common::ObString &str, + ObMockIterator &iter, + uint16_t *col_id_array_list = nullptr); int parse( const common::ObString &str, ObMockIterator &iter, uint16_t *col_id_array_list = nullptr); + int parse_datum_with_specified_col_ids( + const ObString &str, + ObMockIterator &iter, + uint16_t *col_id_array_list = nullptr, + int64_t *result_col_id_array = nullptr); int parse_with_specified_col_ids( const ObString &str, ObMockIterator &iter, @@ -292,72 +333,164 @@ public: int64_t *result_col_id_array = nullptr); private: static int static_init(); - static int parse_varchar(common::ObIAllocator *allocator, + static int prepare_parse_varchar(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx); + static int parse_datum_varchar(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_varchar(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int prepare_parse_lob(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + ObLobCommon *&lob_data, + int64_t &val); + static int parse_datum_lob(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_lob(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); - static int parse_lob(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); /* static int parse_bool(common::ObIAllocator *allocator, - const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); + const common::ObString &word, blocksstable::ObDatumRow &row, int64_t &idx); */ - static int parse_timestamp(common::ObIAllocator *allocator, + static int prepare_parse_timestamp(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &usec); + static int parse_datum_timestamp(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_timestamp(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int prepare_parse_int(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &val); + static int parse_datum_int(common::ObIAllocator *allocator, const common::ObString &word, - storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, int64_t &idx); - static int parse_int(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); - static int parse_bigint(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); - static int parse_number(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); + static int parse_obj_int(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int prepare_parse_bigint(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + int64_t &val); + static int parse_datum_bigint(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_bigint(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int prepare_parse_number(common::ObIAllocator *allocator, + const common::ObString &word, + const uint16_t count, + int64_t &idx, + number::ObNumber &nmb); + static int parse_datum_number(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_number(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); static int parse_dml(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); + static int parse_datum_dml(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); static int parse_first_dml(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); - static int parse_flag(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); + static int prepare_parse_flag(common::ObIAllocator *allocator, + const common::ObString &word, + int64_t &idx, + int64_t &flag); + static int parse_datum_flag(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_flag(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); static int parse_base(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); - static int parse_multi_version_row_flag(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); + static int parse_datum_multi_version_row_flag(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_multi_version_row_flag(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); static int parse_is_get(common::ObIAllocator *allocator, const common::ObString &word, storage::ObStoreRow &row, int64_t &idx); - static int parse_scan_index(common::ObIAllocator *allocator, + static int prepare_parse_scan_index(common::ObIAllocator *allocator, + const common::ObString &word, + int64_t &idx, + int64_t &val); + static int parse_datum_scan_index(common::ObIAllocator *allocator, + const common::ObString &word, + blocksstable::ObDatumRow &row, + int64_t &idx); + static int parse_obj_scan_index(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + static int parse_datum_trans_id(common::ObIAllocator *allocator, const common::ObString &word, - storage::ObStoreRow &row, - int64_t &idx); - static int parse_trans_id(common::ObIAllocator *allocator, - const common::ObString &word, - storage::ObStoreRow &row, + blocksstable::ObDatumRow &row, int64_t &idx); + static int parse_obj_trans_id(common::ObIAllocator *allocator, + const common::ObString &word, + storage::ObStoreRow &row, + int64_t &idx); + int parse_datum_header(const ObString &str, + int64_t &pos, + ObIArray &header, + int64_t &obj_num, + ObMockIterator &iter); int parse_header(const common::ObString &str, int64_t &pos, common::ObIArray &header, int64_t &obj_num, ObMockIterator &iter); + int parse_datum_row(const common::ObString &str, + int64_t &pos, + const common::ObIArray &header, + const uint16_t *col_id_array_list, + blocksstable::ObDatumRow &row); int parse_row(const common::ObString &str, int64_t &pos, const common::ObIArray &header, @@ -388,6 +521,10 @@ private: static common::hash::ObHashMap str_to_obj_parse_func_; // hash ObString to row info parse func , such as parse_dml ... static common::hash::ObHashMap str_to_info_parse_func_; + // hash ObString to obj parse func , such as parse_int ... + static common::hash::ObHashMap str_to_datum_parse_func_; + // hash ObString to row info parse func , such as parse_dml ... + static common::hash::ObHashMap str_to_datum_info_parse_func_; static common::hash::ObHashMap str_to_obj_type_; static common::hash::ObHashMap str_to_flag_; static common::hash::ObHashMap str_to_dml_; @@ -402,25 +539,25 @@ private: char escape_; }; -class MockObNewRowIterator: public ObNewRowIterator -{ - OB_UNIS_VERSION(1); -public: - MockObNewRowIterator(); - ~MockObNewRowIterator(); - static bool equals(const common::ObNewRow &r1, const common::ObNewRow &r2); - void reset(); - int from(const char *cstr, char escape = '\\') { return iter_.from(cstr, escape); } - int from(const common::ObString &str, char escape = '\\') { return iter_.from(str, escape); } - int get_next_row(ObNewRow *&row) { return iter_.get_next_row(row); } - int get_row(const int64_t idex, storage::ObStoreRow *&row) const { return iter_.get_row(idex, row); } - int64_t count() const { return iter_.count(); } - int add_row(storage::ObStoreRow *row) { return iter_.add_row(row); } +// class MockObNewRowIterator: public ObNewRowIterator +// { +// OB_UNIS_VERSION(1); +// public: +// MockObNewRowIterator(); +// ~MockObNewRowIterator(); +// static bool equals(const common::ObNewRow &r1, const common::ObNewRow &r2); +// void reset(); +// int from(const char *cstr, char escape = '\\') { return iter_.from(cstr, escape); } +// int from(const common::ObString &str, char escape = '\\') { return iter_.from(str, escape); } +// int get_next_row(ObNewRow *&row) { return iter_.get_next_row(row); } +// int get_row(const int64_t idex, storage::ObStoreRow *&row) const { return iter_.get_row(idex, row); } +// int64_t count() const { return iter_.count(); } +// int add_row(storage::ObStoreRow *row) { return iter_.add_row(row); } -private: - ObMockIterator iter_; - common::PageArena allocator_; -}; +// private: +// ObMockIterator iter_; +// common::PageArena allocator_; +// }; class ObMockDirectReadIterator : public storage::ObStoreRowIterator { diff --git a/unittest/storage/test_ob_mock_iterator.cpp b/unittest/storage/test_ob_mock_iterator.cpp index b5a0e1663..f48cc71ff 100644 --- a/unittest/storage/test_ob_mock_iterator.cpp +++ b/unittest/storage/test_ob_mock_iterator.cpp @@ -429,17 +429,17 @@ TEST_F(TestObMockIteratorBuilder, equals) TEST_F(TestObMockIteratorBuilder, test_new_row) { - ObMockNewRowIterator iter1; + ObMockDatumRowIterator iter1; const char *input1 = "int var num\n" "1 2 3\n"; - ASSERT_EQ(OB_SUCCESS, iter1.from(input1)); + ASSERT_EQ(OB_SUCCESS, iter1.from_for_datum(input1)); - ObMockNewRowIterator iter2; + ObMockDatumRowIterator iter2; const char *input2 = "int var num\n" "1 2 4\n"; - EXPECT_EQ(OB_SUCCESS, iter2.from(input2)); + EXPECT_EQ(OB_SUCCESS, iter2.from_for_datum(input2)); EXPECT_FALSE(iter2.equals(iter1)); } diff --git a/unittest/storage/tx/it/tx_node.cpp b/unittest/storage/tx/it/tx_node.cpp index 72c7cb10d..ce128208c 100644 --- a/unittest/storage/tx/it/tx_node.cpp +++ b/unittest/storage/tx/it/tx_node.cpp @@ -658,12 +658,13 @@ int ObTxNode::write(ObTxDesc &tx, write_store_ctx)); write_store_ctx.mvcc_acc_ctx_.tx_table_guards_.tx_table_guard_.init(&fake_tx_table_); ObArenaAllocator allocator; - ObStoreRow row; - ObObj cols[2] = {ObObj(key), ObObj(value)}; - row.capacity_ = 2; - row.row_val_.cells_ = cols; - row.row_val_.count_ = 2; - row.flag_ = blocksstable::ObDmlFlag::DF_UPDATE; + ObDatumRow row; + ObStorageDatum cols[2] = {ObStorageDatum(), ObStorageDatum()}; + cols[0].set_int(key); + cols[1].set_int(value); + row.count_ = 2; + row.storage_datums_ = cols; + row.row_flag_ = blocksstable::ObDmlFlag::DF_UPDATE; row.trans_id_.reset(); ObTableIterParam param; @@ -723,11 +724,13 @@ int ObTxNode::write_one_row(ObStoreCtx& write_store_ctx, const int64_t key, cons const transaction::ObSerializeEncryptMeta *encrypt_meta = NULL; const int64_t schema_version = 100; read_info.init(allocator, 2, 1, false, columns_, nullptr/*storage_cols_index*/); - ObStoreRow row; - ObObj cols[2] = {ObObj(key), ObObj(value)}; - row.flag_ = blocksstable::ObDmlFlag::DF_UPDATE; - row.row_val_.cells_ = cols; - row.row_val_.count_ = 2; + ObDatumRow row; + ObStorageDatum cols[2] = {ObStorageDatum(), ObStorageDatum()}; + cols[0].set_int(key); + cols[1].set_int(value); + row.row_flag_ = blocksstable::ObDmlFlag::DF_UPDATE; + row.storage_datums_ = cols; + row.count_ = 2; ObTableIterParam param; ObTableAccessContext context;