From 82a5c6fe6c65b5e4a787830217df9c766c58adec Mon Sep 17 00:00:00 2001 From: Tsunaou <895254752@qq.com> Date: Wed, 30 Oct 2024 06:44:03 +0000 Subject: [PATCH] [CP] fix create heap table column array and collation type && column checksum error --- deps/oblib/src/common/object/ob_object.cpp | 9 +- deps/oblib/src/common/object/ob_object.h | 3 +- .../src/lib/mysqlclient/ob_mysql_result.h | 4 +- src/rootserver/ob_ddl_service.cpp | 2 +- src/rootserver/ob_tablet_creator.cpp | 21 ++ .../ob_tablet_replica_checksum_operator.cpp | 9 +- .../resolver/dml/ob_default_value_utils.cpp | 10 +- .../compaction/ob_schedule_tablet_func.cpp | 8 + .../compaction/ob_tenant_tablet_scheduler.cpp | 11 +- .../compaction/ob_tenant_tablet_scheduler.h | 2 +- src/storage/ob_storage_schema.cpp | 21 +- src/storage/ob_storage_schema.h | 2 +- unittest/storage/test_compaction_policy.cpp | 298 +++++++++++++++++- 13 files changed, 370 insertions(+), 30 deletions(-) diff --git a/deps/oblib/src/common/object/ob_object.cpp b/deps/oblib/src/common/object/ob_object.cpp index fadb42aa5..8648093ab 100644 --- a/deps/oblib/src/common/object/ob_object.cpp +++ b/deps/oblib/src/common/object/ob_object.cpp @@ -1306,7 +1306,9 @@ bool ObObj::is_zero() const return ret; } -int ObObj::build_not_strict_default_value(int16_t precision) +int ObObj::build_not_strict_default_value( + int16_t precision, + const ObCollationType string_cs_type) { int ret = OB_SUCCESS; const ObObjType &data_type = meta_.get_type(); @@ -1468,6 +1470,11 @@ int ObObj::build_not_strict_default_value(int16_t precision) ret = OB_INVALID_ARGUMENT; _OB_LOG(WARN, "unexpected data type=%u", data_type); } + if (OB_FAIL(ret)) { + } else if (is_string_type()) { + set_collation_level(CS_LEVEL_IMPLICIT); + set_collation_type(string_cs_type); + } return ret; } diff --git a/deps/oblib/src/common/object/ob_object.h b/deps/oblib/src/common/object/ob_object.h index b32a777b9..cb06dd1f6 100644 --- a/deps/oblib/src/common/object/ob_object.h +++ b/deps/oblib/src/common/object/ob_object.h @@ -1361,7 +1361,8 @@ public: explicit ObObj(ObObjType type); inline void reset(); //when in not strict sql mode, build default value refer to data type - int build_not_strict_default_value(int16_t precision); + // @string_cs_type: default value of string type need set collation type, since the space will be trim when compaction, see ObStorageSchema::trim + int build_not_strict_default_value(int16_t precision, const ObCollationType string_cs_type); static ObObj make_min_obj(); static ObObj make_max_obj(); static ObObj make_nop_obj(); diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_result.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_result.h index d647907db..9901eec13 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_result.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_result.h @@ -1132,7 +1132,7 @@ { \ def_obj.set_type(data_type); \ if (is_mysql_mode()) { \ - if (OB_FAIL(def_obj.build_not_strict_default_value(column.get_data_precision()))) { \ + if (OB_FAIL(def_obj.build_not_strict_default_value(column.get_data_precision(), ObCollationType::CS_TYPE_INVALID /*string_cs_type, no need set for json*/))) { \ SQL_LOG(WARN, "failed to build not strict default json value", K(ret)); \ } else { \ res_obj.set_json_value(data_type, def_obj.get_string().ptr(), \ @@ -1275,7 +1275,7 @@ { \ def_obj.set_type(data_type); \ if (is_mysql_mode()) { \ - if (OB_FAIL(def_obj.build_not_strict_default_value(column.get_data_precision()))) { \ + if (OB_FAIL(def_obj.build_not_strict_default_value(column.get_data_precision(), ObCollationType::CS_TYPE_INVALID /*string_cs_type, no need set for json*/))) { \ SQL_LOG(WARN, "failed to build not strict default json value", K(ret)); \ } else { \ res_obj.set_json_value(data_type, def_obj.get_string().ptr(), \ diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 3c7730481..757d1753e 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -8481,7 +8481,7 @@ int ObDDLService::resolve_orig_default_value(ObColumnSchemaV2 &alter_column_sche } else { ObObj default_value; default_value.set_type(alter_column_schema.get_data_type()); - if (OB_FAIL(default_value.build_not_strict_default_value(alter_column_schema.get_accuracy().get_precision()))) { + if (OB_FAIL(default_value.build_not_strict_default_value(alter_column_schema.get_accuracy().get_precision(), alter_column_schema.get_collation_type()))) { LOG_WARN("failed to build not strict default value", K(ret)); } else if (OB_FAIL(alter_column_schema.set_orig_default_value(default_value))) { LOG_WARN("failed to set orig default value", K(ret)); diff --git a/src/rootserver/ob_tablet_creator.cpp b/src/rootserver/ob_tablet_creator.cpp index c7169cfb9..f32689ce9 100644 --- a/src/rootserver/ob_tablet_creator.cpp +++ b/src/rootserver/ob_tablet_creator.cpp @@ -212,6 +212,27 @@ int ObBatchCreateTabletHelper::add_table_schema_( HEAP_VAR(ObTableSchema, table_schema) { if (OB_FAIL(table_schema.assign(const_table_schema))) { LOG_WARN("failed to assign table_schema", KR(ret), K(const_table_schema)); + } else if (table_schema.is_user_table() && table_schema.is_heap_table()) { + /* + * When creating heap table (no explicit primary key), or doing offline ddl to drop primary key, the column array in table_schema here is out of order actually. + * The `__pk_increment` column is pushed back into column array with column_id 1, and in the LAST of column array in table schema. + * Column array in storage schema will be used to construct column group in C-replica, so the `__pk_increment` cg will be the LAST cg. + * However, the table schema read from schema_guard (__all_column table) when doing compaction will sort the column array by column id, + * so the `__pk_increment` cg will be the FIRST cg when compaction, which cause data inconsistency. + * + * So we need to sort column array by column id for heap table when creating tablet. + * + * testcases: + * - tools/deploy/mysql_test/test_suite/column_store_replica/t/drop_heap_table_primary_key.test + * - tools/deploy/mysql_test/test_suite/column_store_replica/t/drop_heap_table_primary_key_oracle.test + * - tools/obtest/t/errsim_storage_compaction/column_store_replica/test_rebuild_heap_table_migrate_major.test + */ + if (OB_FAIL(table_schema.sort_column_array_by_column_id())) { + LOG_WARN("failed to sort column array", K(ret), K(table_schema)); + } + } + + if (OB_FAIL(ret)) { } else if (tenant_data_version < DATA_VERSION_4_2_2_0) { // compatibility with DATA_VERSION_4_2_1. table_schema.reset_partition_schema(); diff --git a/src/share/ob_tablet_replica_checksum_operator.cpp b/src/share/ob_tablet_replica_checksum_operator.cpp index ff1bdd1be..a373b5f29 100644 --- a/src/share/ob_tablet_replica_checksum_operator.cpp +++ b/src/share/ob_tablet_replica_checksum_operator.cpp @@ -360,13 +360,16 @@ int ObTabletReplicaChecksumItem::verify_column_checksum(const ObTabletReplicaChe ret = OB_CHECKSUM_ERROR; } - bool is_large_text_column = false; + bool is_all_large_text_column = false; uint64_t compat_version = 0; if (OB_FAIL(ret) || !is_cs_replica) { } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id_, compat_version))) { LOG_WARN("failed to get min data version", K(ret), K(tenant_id_)); } else if (compat_version >= DATA_VERSION_4_3_4_0) { // should not skip the validation of lob column between cs replica and row replica + if (compaction_scn_ == other.compaction_scn_ && !column_meta_equal) { + ret = OB_CHECKSUM_ERROR; + } } else { ObSEArray column_idxs; for (int64_t idx = 0; OB_SUCC(ret) && idx < column_meta_.column_checksums_.count(); ++idx) { @@ -380,9 +383,9 @@ int ObTabletReplicaChecksumItem::verify_column_checksum(const ObTabletReplicaChe if (FAILEDx(compaction::ObCSReplicaChecksumHelper::check_column_type(tablet_id_, compaction_scn_.get_val_for_tx(), column_idxs, - is_large_text_column))) { + is_all_large_text_column))) { LOG_WARN("failed to check column type for cs replica", K(ret), KPC(this), K(other)); - } else if (is_large_text_column) { + } else if (is_all_large_text_column) { // do nothing } else { ret = OB_CHECKSUM_ERROR; diff --git a/src/sql/resolver/dml/ob_default_value_utils.cpp b/src/sql/resolver/dml/ob_default_value_utils.cpp index 4b2ce325a..f68087dbc 100644 --- a/src/sql/resolver/dml/ob_default_value_utils.cpp +++ b/src/sql/resolver/dml/ob_default_value_utils.cpp @@ -780,11 +780,8 @@ int ObDefaultValueUtils::build_default_expr_not_strict_static( default_value.set_null(); } else { default_value.set_type(column_schema->get_data_type()); - if (OB_FAIL(default_value.build_not_strict_default_value(column_schema->get_accuracy().get_precision()))) { + if (OB_FAIL(default_value.build_not_strict_default_value(column_schema->get_accuracy().get_precision(), column_schema->get_collation_type()))) { LOG_WARN("failed to build not strict default value info", K(column_schema), K(ret)); - } else if (default_value.is_string_type()) { - default_value.set_collation_level(CS_LEVEL_IMPLICIT); - default_value.set_collation_type(column_schema->get_collation_type()); } } if (OB_SUCC(ret)) { @@ -836,11 +833,8 @@ int ObDefaultValueUtils::build_default_expr_not_strict(const ColumnItem *column, default_value.set_null(); } else { default_value.set_type(column->get_column_type()->get_type()); - if (OB_FAIL(default_value.build_not_strict_default_value(column->get_column_type()->get_accuracy().get_precision()))) { + if (OB_FAIL(default_value.build_not_strict_default_value(column->get_column_type()->get_accuracy().get_precision(), column->get_column_type()->get_collation_type()))) { LOG_WARN("failed to build not strict default value info", K(column), K(ret)); - } else if (default_value.is_string_type()) { - default_value.set_collation_level(CS_LEVEL_IMPLICIT); - default_value.set_collation_type(column->get_column_type()->get_collation_type()); } } if (OB_SUCC(ret)) { diff --git a/src/storage/compaction/ob_schedule_tablet_func.cpp b/src/storage/compaction/ob_schedule_tablet_func.cpp index 2041a3e8d..c90dfc062 100644 --- a/src/storage/compaction/ob_schedule_tablet_func.cpp +++ b/src/storage/compaction/ob_schedule_tablet_func.cpp @@ -17,6 +17,7 @@ namespace oceanbase using namespace storage; namespace compaction { +ERRSIM_POINT_DEF(EN_COMPACTION_SKIP_CS_REPLICA_TO_REBUILD); /********************************************ObScheduleTabletFunc impl******************************************/ ObScheduleTabletFunc::ObScheduleTabletFunc( @@ -189,6 +190,13 @@ int ObScheduleTabletFunc::schedule_tablet_execute( FLOG_INFO("ERRSIM EN_MEDIUM_CREATE_DAG", K(ret)); return ret; } + if (OB_SUCC(ret) && ls_status_.get_ls().is_cs_replica()) { + ret = EN_COMPACTION_SKIP_CS_REPLICA_TO_REBUILD; + if (OB_FAIL(ret)) { + LOG_INFO("ERRSIM EN_COMPACTION_SKIP_CS_REPLICA_TO_REBUILD", K(ret)); + return ret; + } + } #endif const ObLSID &ls_id = ls_status_.ls_id_; const ObTabletID &tablet_id = tablet.get_tablet_id(); diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index 8bbce5d30..2585579a3 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -247,7 +247,7 @@ int ObCSReplicaChecksumHelper::check_column_type( const common::ObTabletID &tablet_id, const int64_t compaction_scn, const common::ObIArray &column_idxs, - bool &is_large_text_column) + bool &is_all_large_text_column) { int ret = OB_SUCCESS; share::ObFreezeInfo freeze_info; @@ -257,7 +257,7 @@ int ObCSReplicaChecksumHelper::check_column_type( ObSEArray column_descs; int64_t save_schema_version = 0; const ObTableSchema *table_schema = nullptr; - is_large_text_column = true; + is_all_large_text_column = true; if (OB_UNLIKELY(!tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -292,7 +292,7 @@ int ObCSReplicaChecksumHelper::check_column_type( } else if (OB_FAIL(table_schema->get_multi_version_column_descs(column_descs))) { LOG_WARN("failed to get multi version column descs", K(ret), K(tablet_id), KPC(table_schema)); } else { - for (int64_t idx = 0; is_large_text_column && OB_SUCC(ret) && idx < column_idxs.count(); ++idx) { + for (int64_t idx = 0; is_all_large_text_column && OB_SUCC(ret) && idx < column_idxs.count(); ++idx) { const int64_t cur_col_idx = column_idxs.at(idx); const int64_t column_id = column_descs.at(cur_col_idx).col_id_; const ObColumnSchemaV2 *col_schema = table_schema->get_column_schema(column_id); @@ -300,10 +300,11 @@ int ObCSReplicaChecksumHelper::check_column_type( if (OB_ISNULL(col_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null col schema", K(ret)); - } else { - is_large_text_column = ob_is_large_text(col_schema->get_data_type()); + } else if (ob_is_large_text(col_schema->get_data_type())) { LOG_DEBUG("check column type for cs replica", K(cur_col_idx), KPC(col_schema), "rowkey_cnt", table_schema->get_rowkey_column_num(), KPC(table_schema)); + } else { + is_all_large_text_column = false; } } } diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.h b/src/storage/compaction/ob_tenant_tablet_scheduler.h index 283f303ec..e543b1079 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.h +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.h @@ -135,7 +135,7 @@ public: const common::ObTabletID &tablet_id, const int64_t compaction_scn, const common::ObIArray &column_idxs, - bool &is_large_text_column); + bool &is_all_large_text_column); }; diff --git a/src/storage/ob_storage_schema.cpp b/src/storage/ob_storage_schema.cpp index 08ffabc41..15f9a530d 100644 --- a/src/storage/ob_storage_schema.cpp +++ b/src/storage/ob_storage_schema.cpp @@ -108,8 +108,9 @@ int ObStorageColumnSchema::construct_column_param(share::schema::ObColumnParam & if (OB_FAIL(datum.from_obj(orig_default_value_))) { STORAGE_LOG(WARN, "fail to covent datum from obj", K(ret), K(orig_default_value_)); } else { - (void) ObStorageSchema::trim(orig_default_value_.get_collation_type(), datum); - if (OB_FAIL(datum.to_obj_enhance(obj, orig_default_value_.get_meta()))) { + if (OB_FAIL(ObStorageSchema::trim(orig_default_value_.get_collation_type(), datum))) { + STORAGE_LOG(WARN, "failed to trim datum", K(ret), K_(orig_default_value), K(datum)); + } else if (OB_FAIL(datum.to_obj_enhance(obj, orig_default_value_.get_meta()))) { STORAGE_LOG(WARN, "failed to transfer datum to obj", K(ret), K(datum)); } else if (OB_FAIL(column_param.set_orig_default_value(obj))) { STORAGE_LOG(WARN, "fail to set orig default value", K(ret)); @@ -1926,19 +1927,25 @@ int ObStorageSchema::get_orig_default_row( } else if (OB_FAIL(default_row.storage_datums_[i].from_obj_enhance(col_schema->get_orig_default_value()))) { STORAGE_LOG(WARN, "Failed to transfer obj to datum", K(ret)); } else if (need_trim && col_schema->get_orig_default_value().is_fixed_len_char_type()) { - trim(col_schema->get_orig_default_value().get_collation_type(), default_row.storage_datums_[i]); + if (OB_FAIL(trim(col_schema->get_orig_default_value().get_collation_type(), default_row.storage_datums_[i]))) { + STORAGE_LOG(WARN, "Failed to trim default value", K(ret), KPC(col_schema), K(default_row)); + } } } } return ret; } -void ObStorageSchema::trim(const ObCollationType type, blocksstable::ObStorageDatum &storage_datum) +int ObStorageSchema::trim(const ObCollationType type, blocksstable::ObStorageDatum &storage_datum) { + int ret = OB_SUCCESS; + ObString space_pattern = ObCharsetUtils::get_const_str(type, ' '); + if (OB_UNLIKELY(!ObCharset::is_valid_collation(type) || (0 == space_pattern.length()))) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid collation type", K(ret), K(type), K(space_pattern)); + } else { const char *str = storage_datum.ptr_; int32_t len = storage_datum.len_; - ObString space_pattern = ObCharsetUtils::get_const_str(type, ' '); - for (; len >= space_pattern.length(); len -= space_pattern.length()) { if (0 != MEMCMP(str + len - space_pattern.length(), space_pattern.ptr(), @@ -1947,6 +1954,8 @@ void ObStorageSchema::trim(const ObCollationType type, blocksstable::ObStorageDa } } storage_datum.len_ = len; + } + return ret; } const ObStorageColumnSchema *ObStorageSchema::get_column_schema(const int64_t column_idx) const diff --git a/src/storage/ob_storage_schema.h b/src/storage/ob_storage_schema.h index 88a0de328..6254c38c5 100644 --- a/src/storage/ob_storage_schema.h +++ b/src/storage/ob_storage_schema.h @@ -340,7 +340,7 @@ public: "skip_index_cnt", skip_idx_attr_array_.count(), K_(skip_idx_attr_array), "column_group_cnt", column_group_array_.count(), K_(column_group_array), K_(has_all_column_group)); public: - static void trim(const ObCollationType type, blocksstable::ObStorageDatum &storage_datum); + static int trim(const ObCollationType type, blocksstable::ObStorageDatum &storage_datum); private: int copy_from(const share::schema::ObMergeSchema &input_schema); int deep_copy_str(const ObString &src, ObString &dest); diff --git a/unittest/storage/test_compaction_policy.cpp b/unittest/storage/test_compaction_policy.cpp index c5d4f1731..3763880b0 100644 --- a/unittest/storage/test_compaction_policy.cpp +++ b/unittest/storage/test_compaction_policy.cpp @@ -35,6 +35,8 @@ #include "storage/ob_storage_schema.h" #include "share/scn.h" #include "storage/test_schema_prepare.h" +#include "storage/test_tablet_helper.h" +#include "storage/high_availability/ob_storage_ha_tablet_builder.h" namespace oceanbase { @@ -92,6 +94,25 @@ public: const int64_t max_merged_trans_version, const int64_t upper_trans_version, ObTableHandleV2 &table_handle); + static int mock_column_sstable( + common::ObArenaAllocator &allocator, + const ObITable::TableType &type, + const ObCOSSTableBaseType &co_base_type, + const int64_t base_version, + const int64_t snapshot_version, + const int64_t max_merged_trans_version, + const int64_t upper_trans_version, + const int64_t column_group_cnt, + ObTableHandleV2 &table_handle); + static int mock_co_sstable( + common::ObArenaAllocator &allocator, + const ObCOSSTableBaseType &co_base_type, + const int64_t base_version, + const int64_t snapshot_version, + const int64_t max_merged_trans_version, + const int64_t upper_trans_version, + const int64_t column_group_cnt, + ObTableHandleV2 &table_handle); static int mock_sstable_meta( const int64_t row_count, ObTableHandleV2 &table_handle); @@ -251,7 +272,7 @@ void TestCompactionPolicy::generate_table_key( table_key.reset(); table_key.tablet_id_ = TEST_TABLET_ID; table_key.table_type_ = type; - if (type == ObITable::TableType::MAJOR_SSTABLE) { + if (ObITable::is_major_sstable(type)) { table_key.version_range_.base_version_ = start_scn; table_key.version_range_.snapshot_version_ = end_scn; } else { @@ -315,6 +336,121 @@ int TestCompactionPolicy::mock_sstable( return ret; } +int TestCompactionPolicy::mock_column_sstable( + common::ObArenaAllocator &allocator, + const ObITable::TableType &type, + const ObCOSSTableBaseType &co_base_type, + const int64_t base_version, + const int64_t snapshot_version, + const int64_t max_merged_trans_version, + const int64_t upper_trans_version, + const int64_t column_group_cnt, + ObTableHandleV2 &table_handle) +{ + int ret = OB_SUCCESS; + + ObTableSchema table_schema; + TestSchemaUtils::prepare_data_schema(table_schema); + + ObITable::TableKey table_key; + generate_table_key(type, base_version, snapshot_version, table_key); + + ObTabletID tablet_id; + tablet_id = TEST_TABLET_ID; + ObTabletCreateSSTableParam param; + ObStorageSchema storage_schema; + ObSSTable *sstable = nullptr; + + if (OB_FAIL(storage_schema.init(allocator, table_schema, lib::Worker::CompatMode::MYSQL))) { + LOG_WARN("failed to init storage schema", K(ret)); + } else if (OB_FAIL(ObTabletCreateDeleteHelper::build_create_sstable_param(storage_schema, tablet_id, 100, param))) { + LOG_WARN("failed to build create sstable param", K(ret), K(table_key)); + } else { + param.table_key_ = table_key; + param.max_merged_trans_version_ = max_merged_trans_version; + param.filled_tx_scn_ = table_key.get_end_scn(); + } + + if (OB_FAIL(ret)) { + } else if (ObITable::TableType::COLUMN_ORIENTED_SSTABLE == type) { + param.column_group_cnt_ = column_group_cnt; + param.co_base_type_ = co_base_type; + if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, table_handle))) { + LOG_WARN("failed to create co sstable", K(ret), K(param)); + } + } else if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, table_handle))) { + LOG_WARN("failed to create sstable", K(ret), K(param)); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(table_handle.get_sstable(sstable))) { + LOG_WARN("failed to get table", K(ret), K(table_handle)); + } else if (OB_ISNULL(sstable)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get sstable", K(ret), K(table_handle)); + } else { + sstable->meta_->basic_meta_.max_merged_trans_version_ = max_merged_trans_version; + sstable->meta_->basic_meta_.upper_trans_version_ = upper_trans_version; + sstable->meta_cache_.max_merged_trans_version_ = max_merged_trans_version; + sstable->meta_cache_.upper_trans_version_ = upper_trans_version; + sstable->meta_cache_.nested_size_ = 0; + sstable->meta_cache_.nested_offset_ = 0; + LOG_INFO("Finish mock column sstable", K(ret), KPC(sstable)); + } + return ret; +} + +int TestCompactionPolicy::mock_co_sstable( + common::ObArenaAllocator &allocator, + const ObCOSSTableBaseType &co_base_type, + const int64_t base_version, + const int64_t snapshot_version, + const int64_t max_merged_trans_version, + const int64_t upper_trans_version, + const int64_t column_group_cnt, + ObTableHandleV2 &table_handle) +{ + int ret = OB_SUCCESS; + ObITable *co_table = nullptr; + ObCOSSTableV2 *co_sstable = nullptr; + ObSEArray cg_handles; + ObSEArray cg_sstables; + if (OB_FAIL(TestCompactionPolicy::mock_column_sstable(allocator, ObITable::COLUMN_ORIENTED_SSTABLE, ObCOSSTableBaseType::ROWKEY_CG_TYPE, + base_version, snapshot_version, max_merged_trans_version, upper_trans_version, column_group_cnt, table_handle))) { + LOG_WARN("failed to mock co sstable", K(ret)); + } else if (OB_ISNULL(co_table = table_handle.get_table()) || !co_table->is_co_sstable() || OB_ISNULL(co_sstable = static_cast(co_table))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get co sstable", K(ret), KPC(co_table)); + } else { + const int64_t normal_cg_cnt = column_group_cnt - 1; + for (int64_t idx = 0; OB_SUCC(ret) && idx < normal_cg_cnt; idx++) { + ObITable *cg_sstable = nullptr; + if (OB_FAIL(cg_handles.push_back(ObTableHandleV2()))) { + LOG_WARN("failed to push back cg handle", K(ret)); + } else if (OB_FAIL(TestCompactionPolicy::mock_column_sstable(allocator, ObITable::NORMAL_COLUMN_GROUP_SSTABLE, ObCOSSTableBaseType::MAX_TYPE, + base_version, snapshot_version, max_merged_trans_version, upper_trans_version, column_group_cnt, cg_handles[idx]))) { + LOG_WARN("failed to mock cg sstable", K(ret)); + } else if (OB_ISNULL(cg_sstable = cg_handles[idx].get_table()) || !cg_sstable->is_cg_sstable()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get cg sstable", K(ret), K(cg_handles[idx]), KPC(cg_sstable)); + } else if (FALSE_IT(cg_sstable->key_.column_group_idx_ = idx + 1)) { + } else if (OB_FAIL(cg_sstables.push_back(cg_sstable))) { + LOG_WARN("failed to push back cg sstable", K(ret), KP(cg_sstable)); + } else { + LOG_INFO("Finish mock cg sstable", K(ret), KPC(cg_sstable)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(co_sstable->fill_cg_sstables(cg_sstables))) { + LOG_WARN("failed to fill cg sstables", K(ret), KPC(co_sstable), K(cg_sstables)); + } else { + LOG_INFO("Finish mock co sstable", K(ret), KPC(co_sstable)); + } + } + return ret; +} + int TestCompactionPolicy::mock_sstable_meta( const int64_t row_count, ObTableHandleV2 &table_handle) @@ -1270,6 +1406,166 @@ TEST_F(TestCompactionPolicy, check_minor_merge_policy_with_large_minor3) ASSERT_EQ(350, result.scn_range_.end_scn_.get_val_for_tx()); } +TEST_F(TestCompactionPolicy, test_co_convert_replace_old_major) +{ + int ret = OB_SUCCESS; + const char *key_data = + "table_type start_scn end_scn max_ver upper_ver\n" + "10 0 1 1 1 \n" + "11 1 80 80 120 \n" + "11 80 150 150 500 \n" + "0 150 200 180 180 \n" + "0 200 0 0 0 \n"; + + ret = prepare_tablet(key_data, 150, 150); + ASSERT_EQ(OB_SUCCESS, ret); + ObTabletTableStore &table_store = *tablet_handle_.get_obj()->table_store_addr_.get_ptr(); + LOG_INFO("[CS-Replica] show table store", K(ret), K(table_store), K(ObPrintTableStore(table_store))); + + ObSEArray major_tables; + ObSEArray new_sstables; + ObTableHandleV2 co_table_handle; + ret = mock_co_sstable(allocator_, ObCOSSTableBaseType::ROWKEY_CG_TYPE, 0 /*base_version*/, 1 /*snapshot_version*/, + 1 /*max_merged_trans_version*/, 1 /*upper_trans_version*/, 4 /*column_group_cnt*/, co_table_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ObITable *co_sstable = co_table_handle.get_table(); + ASSERT_NE(nullptr, co_sstable); + ASSERT_EQ(OB_SUCCESS, new_sstables.push_back(co_sstable)); + ASSERT_EQ(OB_SUCCESS, table_store.major_tables_.replace_twin_majors_and_build_new(new_sstables, major_tables)); +} + +TEST_F(TestCompactionPolicy, test_co_convert_replace_old_major_rebuild) +{ + int ret = OB_SUCCESS; + const char *key_data = + "table_type start_scn end_scn max_ver upper_ver\n" + "11 1 80 80 120 \n" + "11 80 150 150 500 \n" + "0 150 200 180 180 \n" + "0 200 0 0 0 \n"; + + ret = prepare_tablet(key_data, 150, 150); + ASSERT_EQ(OB_SUCCESS, ret); + ObTabletTableStore &table_store = *tablet_handle_.get_obj()->table_store_addr_.get_ptr(); + LOG_INFO("[CS-Replica] show table store", K(ret), K(table_store), K(ObPrintTableStore(table_store))); + + ObSEArray hybrid_major_tables; + ObTableHandleV2 co_table_handle0; + ret = mock_co_sstable(allocator_, ObCOSSTableBaseType::ROWKEY_CG_TYPE, 0 /*base_version*/, 1 /*snapshot_version*/, + 1 /*max_merged_trans_version*/, 1 /*upper_trans_version*/, 4 /*column_group_cnt*/, co_table_handle0); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, co_table_handle0.get_table()); + ASSERT_EQ(OB_SUCCESS, hybrid_major_tables.push_back(co_table_handle0.get_table())); + + ObTableHandleV2 table_handle0; + ret = mock_sstable(allocator_, ObITable::MAJOR_SSTABLE, 0 /*base_version*/, 100 /*snapshot_version*/, + 100 /*max_merged_trans_version*/, 100 /*upper_trans_version*/, table_handle0); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, table_handle0.get_table()); + ASSERT_EQ(OB_SUCCESS, hybrid_major_tables.push_back(table_handle0.get_table())); + + ObTableHandleV2 table_handle1; + ret = mock_sstable(allocator_, ObITable::MAJOR_SSTABLE, 0 /*base_version*/, 200 /*snapshot_version*/, + 200 /*max_merged_trans_version*/, 200 /*upper_trans_version*/, table_handle1); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, table_handle1.get_table()); + ASSERT_EQ(OB_SUCCESS, hybrid_major_tables.push_back(table_handle1.get_table())); + + ObSSTableArray mock_old_table_store_majors; + ASSERT_EQ(OB_SUCCESS, mock_old_table_store_majors.init(allocator_, hybrid_major_tables, 0)); + + ObSEArray major_tables; + ObSEArray new_sstables; + ObTableHandleV2 co_table_handle; + ret = mock_co_sstable(allocator_, ObCOSSTableBaseType::ROWKEY_CG_TYPE, 0 /*base_version*/, 200 /*snapshot_version*/, + 200 /*max_merged_trans_version*/, 200 /*upper_trans_version*/, 4 /*column_group_cnt*/, co_table_handle); + ASSERT_EQ(OB_SUCCESS, ret); + ObITable *co_sstable = co_table_handle.get_table(); + ASSERT_NE(nullptr, co_sstable); + ASSERT_EQ(OB_SUCCESS, new_sstables.push_back(co_sstable)); + ASSERT_EQ(OB_SUCCESS, mock_old_table_store_majors.replace_twin_majors_and_build_new(new_sstables, major_tables)); + table_store.major_tables_.reset(); + ASSERT_EQ(OB_SUCCESS, table_store.major_tables_.init(allocator_, major_tables, 0 /*start_pos*/)); + LOG_INFO("[CS-Replica] after replace co major", K(ret), K(table_store), K(ObPrintTableStore(table_store))); +} + +TEST_F(TestCompactionPolicy, test_build_tablet_for_hybrid_store) +{ + int ret = OB_SUCCESS; + ObSEArray hybrid_major_tables; + ObTablesHandleArray hybrid_major_handle_array; + ObTableHandleV2 co_table_handle0; + ret = mock_co_sstable(allocator_, ObCOSSTableBaseType::ROWKEY_CG_TYPE, 0 /*base_version*/, 1 /*snapshot_version*/, + 1 /*max_merged_trans_version*/, 1 /*upper_trans_version*/, 4 /*column_group_cnt*/, co_table_handle0); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, co_table_handle0.get_table()); + ASSERT_EQ(OB_SUCCESS, hybrid_major_tables.push_back(co_table_handle0.get_table())); + + ObTableHandleV2 table_handle0; + ret = mock_sstable(allocator_, ObITable::MAJOR_SSTABLE, 0 /*base_version*/, 100 /*snapshot_version*/, + 100 /*max_merged_trans_version*/, 100 /*upper_trans_version*/, table_handle0); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, table_handle0.get_table()); + ASSERT_EQ(OB_SUCCESS, hybrid_major_tables.push_back(table_handle0.get_table())); + + ObTableHandleV2 table_handle1; + ret = mock_sstable(allocator_, ObITable::MAJOR_SSTABLE, 0 /*base_version*/, 200 /*snapshot_version*/, + 200 /*max_merged_trans_version*/, 200 /*upper_trans_version*/, table_handle1); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, table_handle1.get_table()); + ASSERT_EQ(OB_SUCCESS, hybrid_major_tables.push_back(table_handle1.get_table())); + + ObTableHandleV2 co_table_handle1; + ret = mock_co_sstable(allocator_, ObCOSSTableBaseType::ROWKEY_CG_TYPE, 0 /*base_version*/, 300 /*snapshot_version*/, + 300 /*max_merged_trans_version*/, 300 /*upper_trans_version*/, 4 /*column_group_cnt*/, co_table_handle1); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, co_table_handle1.get_table()); + ASSERT_EQ(OB_SUCCESS, hybrid_major_tables.push_back(co_table_handle1.get_table())); + + ObTableHandleV2 table_handle2; + ret = mock_sstable(allocator_, ObITable::MAJOR_SSTABLE, 0 /*base_version*/, 400 /*snapshot_version*/, + 400 /*max_merged_trans_version*/, 400 /*upper_trans_version*/, table_handle2); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_NE(nullptr, table_handle2.get_table()); + ASSERT_EQ(OB_SUCCESS, hybrid_major_tables.push_back(table_handle2.get_table())); + + ASSERT_EQ(OB_SUCCESS, hybrid_major_handle_array.add_table(co_table_handle0)); + ASSERT_EQ(OB_SUCCESS, hybrid_major_handle_array.add_table(table_handle0)); + ASSERT_EQ(OB_SUCCESS, hybrid_major_handle_array.add_table(table_handle1)); + ASSERT_EQ(OB_SUCCESS, hybrid_major_handle_array.add_table(co_table_handle1)); + ASSERT_EQ(OB_SUCCESS, hybrid_major_handle_array.add_table(table_handle2)); + + ObLSID ls_id = ObLSID(TEST_LS_ID); + ObTabletID tablet_id = ObTabletID(TEST_TABLET_ID + 1); + ObLSHandle ls_handle; + ASSERT_NE(nullptr, MTL(ObLSService *)); + ASSERT_EQ(OB_SUCCESS, MTL(ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD)); + ASSERT_TRUE(ls_handle.is_valid()); + ObLS *ls = ls_handle.get_ls(); + ASSERT_NE(nullptr, ls); + + ObTableSchema table_schema; + TestSchemaUtils::prepare_data_schema(table_schema); + ObStorageSchema storage_schema; + ASSERT_EQ(OB_SUCCESS, storage_schema.init(allocator_, table_schema, lib::Worker::CompatMode::MYSQL)); + ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema, allocator_)); + + ObStorageHATabletBuilderUtil::BatchBuildTabletTablesExtraParam extra_batch_param; + ret = ObStorageHATabletBuilderUtil::build_tablet_for_row_store_(ls, tablet_id, hybrid_major_handle_array, storage_schema, extra_batch_param); + ASSERT_EQ(OB_SUCCESS, ret); + + ObTabletHandle tablet_handle; + ASSERT_EQ(OB_SUCCESS, ls->get_tablet(tablet_id, tablet_handle)); + ASSERT_TRUE(tablet_handle.is_valid()); + ObTablet *tablet = tablet_handle.get_obj(); + ASSERT_NE(nullptr, tablet); + ObTabletMemberWrapper table_store_wrapper; + ASSERT_EQ(OB_SUCCESS, tablet->fetch_table_store(table_store_wrapper)); + ASSERT_TRUE(table_store_wrapper.is_valid()); + const ObTabletTableStore &table_store = *table_store_wrapper.get_member(); + LOG_INFO("[CS-Replica] show hybrid table store", K(ret), K(table_store), K(ObPrintTableStore(table_store))); +} + } //unittest } //oceanbase