diff --git a/src/storage/backup/ob_backup_data_struct.cpp b/src/storage/backup/ob_backup_data_struct.cpp index ecb38789d..8908121a0 100644 --- a/src/storage/backup/ob_backup_data_struct.cpp +++ b/src/storage/backup/ob_backup_data_struct.cpp @@ -778,12 +778,6 @@ void ObBackupTabletMeta::reset() } /* ObBackupSSTableMeta */ -OB_SERIALIZE_MEMBER(ObBackupSSTableMeta, tablet_id_, sstable_meta_, logic_id_list_, - entry_block_addr_for_other_block_, // FARM COMPAT WHITELIST - total_other_block_count_, // FARM COMPAT WHITELIST - is_major_compaction_mview_dep_ - ); - ObBackupSSTableMeta::ObBackupSSTableMeta() : tablet_id_(), sstable_meta_(), logic_id_list_(), entry_block_addr_for_other_block_(), total_other_block_count_(0), is_major_compaction_mview_dep_(false) {} @@ -822,6 +816,137 @@ int ObBackupSSTableMeta::assign(const ObBackupSSTableMeta &backup_sstable_meta) return ret; } +int ObBackupSSTableMeta::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + OB_UNIS_ENCODE(UNIS_VERSION); + if (OB_SUCC(ret)) { + int64_t size_nbytes = serialization::OB_SERIALIZE_SIZE_NEED_BYTES; + int64_t pos_bak = (pos += size_nbytes); + if (OB_SUCC(ret)) { + if (OB_FAIL(serialize_(buf, buf_len, pos))) { + LOG_WARN("fail to serialize_", KR(ret)); + } + } + int64_t serial_size = pos - pos_bak; + int64_t tmp_pos = 0; + CHECK_SERIALIZE_SIZE(ObBackupSSTableMeta, serial_size); + if (OB_SUCC(ret)) { + ret = serialization::encode_fixed_bytes_i64(buf + pos_bak - size_nbytes, size_nbytes, tmp_pos, serial_size); + } + } + return ret; +} + +int ObBackupSSTableMeta::serialize_(char *buf, const int64_t buf_len, int64_t &pos) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(tablet_id_.serialize(buf, buf_len, pos))) { + LOG_WARN("fail to serialize", KR(ret), KP(buf), K(buf_len), K(pos)); + } else if (OB_FAIL(sstable_meta_.serialize(buf, buf_len, pos))) { + LOG_WARN("fail to serialize", KR(ret), KP(buf), K(buf_len), K(pos)); + } else if (OB_FAIL(logic_id_list_.serialize(buf, buf_len, pos))) { + LOG_WARN("fail to serialize", KR(ret), KP(buf), K(buf_len), K(pos)); + } else if (OB_FAIL(entry_block_addr_for_other_block_.serialize(buf, buf_len, pos))) { + LOG_WARN("fail to serialize", KR(ret), KP(buf), K(buf_len), K(pos)); + } else if (OB_FAIL(serialization::encode(buf, buf_len, pos, total_other_block_count_))) { + LOG_WARN("fail to encode", KR(ret), KP(buf), K(buf_len), K(pos)); + } else if (OB_FAIL(serialization::encode(buf, buf_len, pos, is_major_compaction_mview_dep_))) { + LOG_WARN("fail to encode", KR(ret), KP(buf), K(buf_len), K(pos)); + } + return ret; +} + +int ObBackupSSTableMeta::deserialize(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + int64_t version = 0; + int64_t len = 0; + if (OB_SUCC(ret)) { + OB_UNIS_DECODE(version); + OB_UNIS_DECODE(len); + if (OB_BACKUP_SSTABLE_META_V1 != version && OB_BACKUP_SSTABLE_META_V2 != version) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("version not match", K(ret), K(version)); + } + } + if (OB_SUCC(ret)) { + int64_t pos_orig = pos; + pos = 0; + if (OB_BACKUP_SSTABLE_META_V1 == version) { + if (OB_FAIL(deserialize_v1(buf + pos_orig, len, pos))) { + LOG_WARN("fail to deserialize_v1", KR(ret), K(len), K(pos)); + } + } else if (OB_BACKUP_SSTABLE_META_V2 == version) { + if (OB_FAIL(deserialize_v2(buf + pos_orig, len, pos))) { + LOG_WARN("fail to deserialize_v2", KR(ret), K(len), K(pos)); + } + } + + pos = pos_orig + len; + } + return ret; +} + +int ObBackupSSTableMeta::deserialize_v2(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + if (pos < data_len && OB_FAIL(tablet_id_.deserialize(buf, data_len, pos))) { + LOG_WARN("fail to deserialize", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(sstable_meta_.deserialize(buf, data_len, pos))) { + LOG_WARN("fail to deserialize", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(logic_id_list_.deserialize(buf, data_len, pos))) { + LOG_WARN("fail to deserialize", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(entry_block_addr_for_other_block_.deserialize(buf, data_len, pos))) { + LOG_WARN("fail to deserialize", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(serialization::decode(buf, data_len, pos, total_other_block_count_))) { + LOG_WARN("fail to decode", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(serialization::decode(buf, data_len, pos, is_major_compaction_mview_dep_))) { + LOG_WARN("fail to decode", KR(ret), KP(buf), K(data_len), K(pos)); + } + return ret; +} + +int ObBackupSSTableMeta::deserialize_v1(const char *buf, const int64_t data_len, int64_t &pos) +{ + int ret = OB_SUCCESS; + ObBackupPhysicalID physical_id; + entry_block_addr_for_other_block_.reset(); + if (pos < data_len && OB_FAIL(tablet_id_.deserialize(buf, data_len, pos))) { + LOG_WARN("fail to deserialize", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(sstable_meta_.deserialize(buf, data_len, pos))) { + LOG_WARN("fail to deserialize", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(logic_id_list_.deserialize(buf, data_len, pos))) { + LOG_WARN("fail to deserialize", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(physical_id.deserialize(buf, data_len, pos))) { + LOG_WARN("fail to deserialize", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(serialization::decode(buf, data_len, pos, total_other_block_count_))) { + LOG_WARN("fail to decode", KR(ret), KP(buf), K(data_len), K(pos)); + } else if (pos < data_len && OB_FAIL(serialization::decode(buf, data_len, pos, is_major_compaction_mview_dep_))) { + LOG_WARN("fail to decode", KR(ret), KP(buf), K(data_len), K(pos)); + } + return ret; +} + +int64_t ObBackupSSTableMeta::get_serialize_size(void) const +{ + int64_t len = get_serialize_size_(); + OB_UNIS_ADD_LEN(UNIS_VERSION); + len += NS_::OB_SERIALIZE_SIZE_NEED_BYTES; + return len; +} + +int64_t ObBackupSSTableMeta::get_serialize_size_(void) const +{ + int64_t len = serialization::encoded_length(tablet_id_); + len += serialization::encoded_length(sstable_meta_); + len += serialization::encoded_length(logic_id_list_); + len += serialization::encoded_length(entry_block_addr_for_other_block_); + len += serialization::encoded_length(total_other_block_count_); + len += serialization::encoded_length(is_major_compaction_mview_dep_); + return len; +} + /* ObBackupMacroBlockIDPair */ OB_SERIALIZE_MEMBER(ObBackupMacroBlockIDPair, diff --git a/src/storage/backup/ob_backup_data_struct.h b/src/storage/backup/ob_backup_data_struct.h index 87b0cf334..4f11371b1 100644 --- a/src/storage/backup/ob_backup_data_struct.h +++ b/src/storage/backup/ob_backup_data_struct.h @@ -565,7 +565,9 @@ public: }; struct ObBackupSSTableMeta { - OB_UNIS_VERSION(1); + static const int64_t OB_BACKUP_SSTABLE_META_V1 = 1; + static const int64_t OB_BACKUP_SSTABLE_META_V2 = 2; + OB_UNIS_VERSION(OB_BACKUP_SSTABLE_META_V2); public: ObBackupSSTableMeta(); @@ -583,6 +585,10 @@ public: ObBackupLinkedBlockAddr entry_block_addr_for_other_block_; int64_t total_other_block_count_; bool is_major_compaction_mview_dep_; + +private: + int deserialize_v1(const char *buf, const int64_t len, int64_t &pos); + int deserialize_v2(const char *buf, const int64_t len, int64_t &pos); }; struct ObBackupMacroBlockIDPair final { diff --git a/unittest/storage/CMakeLists.txt b/unittest/storage/CMakeLists.txt index dbad7abec..7591dc2f1 100644 --- a/unittest/storage/CMakeLists.txt +++ b/unittest/storage/CMakeLists.txt @@ -114,6 +114,7 @@ storage_unittest(test_backup_macro_block_index_merger backup/test_backup_macro_b storage_unittest(test_backup_device_wrapper backup/test_backup_device_wrapper.cpp) storage_unittest(test_backup_sync_io_mock_async_io backup/test_backup_sync_io_mock_async_io.cpp) storage_unittest(test_backup_device_macro_block_id backup/test_backup_device_macro_block_id.cpp) +storage_unittest(test_backup_compatible backup/test_backup_compatible.cpp) #storage_unittest(test_create_tablet_clog tx_storage/test_create_tablet_clog.cpp) storage_unittest(test_safe_destroy_handler tx_storage/test_safe_destroy_handler.cpp) diff --git a/unittest/storage/backup/test_backup_compatible.cpp b/unittest/storage/backup/test_backup_compatible.cpp new file mode 100644 index 000000000..b359383de --- /dev/null +++ b/unittest/storage/backup/test_backup_compatible.cpp @@ -0,0 +1,192 @@ + +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include +#include "test_backup.h" +#include "storage/backup/ob_backup_data_struct.h" +#include "storage/blocksstable/ob_logic_macro_id.h" +#define private public +#define protected public + +using namespace oceanbase; +using namespace oceanbase::storage; +using namespace oceanbase::common; +using namespace oceanbase::share; +using namespace oceanbase::backup; + +namespace oceanbase { +namespace backup { + +struct CompatBackupSSTableMeta { + OB_UNIS_VERSION(1); + +public: + CompatBackupSSTableMeta(); + virtual ~CompatBackupSSTableMeta() = default; + bool is_valid() const; + void reset(); + int assign(const CompatBackupSSTableMeta &backup_sstable_meta); + + TO_STRING_KV(K_(tablet_id), K_(sstable_meta), K_(logic_id_list), + K_(entry_block_addr_for_other_block), K_(total_other_block_count), + K_(is_major_compaction_mview_dep)); + + common::ObTabletID tablet_id_; + blocksstable::ObMigrationSSTableParam sstable_meta_; + common::ObSArray logic_id_list_; + ObBackupPhysicalID entry_block_addr_for_other_block_; + int64_t total_other_block_count_; + bool is_major_compaction_mview_dep_; +}; + +OB_SERIALIZE_MEMBER(CompatBackupSSTableMeta, + tablet_id_, + sstable_meta_, + logic_id_list_, + entry_block_addr_for_other_block_, + total_other_block_count_, + is_major_compaction_mview_dep_); + +CompatBackupSSTableMeta::CompatBackupSSTableMeta() : tablet_id_(), sstable_meta_(), logic_id_list_(), + entry_block_addr_for_other_block_(), total_other_block_count_(0), is_major_compaction_mview_dep_(false) +{} + +bool CompatBackupSSTableMeta::is_valid() const +{ + return tablet_id_.is_valid() && sstable_meta_.is_valid(); +} + +void CompatBackupSSTableMeta::reset() +{ + tablet_id_.reset(); + sstable_meta_.reset(); + logic_id_list_.reset(); + entry_block_addr_for_other_block_.reset(); + total_other_block_count_ = 0; + is_major_compaction_mview_dep_ = false; +} + +int CompatBackupSSTableMeta::assign(const CompatBackupSSTableMeta &backup_sstable_meta) +{ + int ret = OB_SUCCESS; + if (!backup_sstable_meta.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("backup sstable meta is invalid", K(ret), K(backup_sstable_meta)); + } else if (OB_FAIL(sstable_meta_.assign(backup_sstable_meta.sstable_meta_))) { + LOG_WARN("failed to assign sstable meta", K(ret), K(backup_sstable_meta)); + } else if (OB_FAIL(logic_id_list_.assign(backup_sstable_meta.logic_id_list_))) { + LOG_WARN("failed to assign logic id list", K(ret), K(backup_sstable_meta)); + } else { + entry_block_addr_for_other_block_ = backup_sstable_meta.entry_block_addr_for_other_block_; + total_other_block_count_ = backup_sstable_meta.total_other_block_count_; + tablet_id_ = backup_sstable_meta.tablet_id_; + is_major_compaction_mview_dep_ = backup_sstable_meta.is_major_compaction_mview_dep_; + } + return ret; +} + +void prepare_migration_sstable_param(ObMigrationSSTableParam &mig_param) +{ + storage::ObITable::TableKey table_key; + table_key.table_type_ = storage::ObITable::TableType::MAJOR_SSTABLE; + table_key.tablet_id_ = 20221106; + table_key.version_range_.base_version_ = 1; + table_key.version_range_.snapshot_version_ = 11; + + blocksstable::ObSSTableMeta sstable_meta; + ObMetaDiskAddr addr; + addr.set_none_addr(); + sstable_meta.basic_meta_.row_count_ = 111; + sstable_meta.basic_meta_.occupy_size_ = 222; + sstable_meta.basic_meta_.data_checksum_ = 333; + sstable_meta.basic_meta_.index_type_ = 0; + sstable_meta.basic_meta_.rowkey_column_count_ = 1; + sstable_meta.basic_meta_.column_cnt_ = 1; + sstable_meta.basic_meta_.index_macro_block_count_ = 3; + sstable_meta.basic_meta_.use_old_macro_block_count_ = 1; + sstable_meta.basic_meta_.index_macro_block_count_ = 2; + sstable_meta.basic_meta_.sstable_format_version_ = 0; + sstable_meta.basic_meta_.schema_version_ = 3; + sstable_meta.basic_meta_.create_snapshot_version_ = 0; + sstable_meta.basic_meta_.progressive_merge_round_ = 0; + sstable_meta.basic_meta_.progressive_merge_step_ = 0; + sstable_meta.basic_meta_.upper_trans_version_ = 0; + sstable_meta.basic_meta_.max_merged_trans_version_ = 0; + sstable_meta.basic_meta_.ddl_scn_.set_min(); + sstable_meta.basic_meta_.filled_tx_scn_.set_min(); + sstable_meta.basic_meta_.contain_uncommitted_row_ = 0; + sstable_meta.data_root_info_.addr_ = addr; + sstable_meta.basic_meta_.root_row_store_type_ = ObRowStoreType::ENCODING_ROW_STORE; + sstable_meta.basic_meta_.latest_row_store_type_ = ObRowStoreType::ENCODING_ROW_STORE; + sstable_meta.basic_meta_.data_index_tree_height_ = 0; + sstable_meta.macro_info_.macro_meta_info_.addr_ = addr; + ASSERT_TRUE(sstable_meta.check_meta()); + sstable_meta.is_inited_ = true; + ASSERT_TRUE(!mig_param.is_valid()); + mig_param.reset(); + ASSERT_TRUE(!mig_param.is_valid()); + mig_param.basic_meta_ = sstable_meta.get_basic_meta(); + for (int64_t i = 0; i < sstable_meta.get_col_checksum_cnt(); ++i) { + ASSERT_EQ(OB_SUCCESS, mig_param.column_checksums_.push_back(sstable_meta.get_col_checksum()[i])); + } + mig_param.table_key_ = table_key; + char str[15] ="test serialize"; + mig_param.root_block_addr_.set_mem_addr(0, 15); + mig_param.root_block_buf_ = str; + blocksstable::MacroBlockId block_id(4096,0,0); + mig_param.data_block_macro_meta_addr_.set_block_addr(block_id, 1024, 2048, ObMetaDiskAddr::DiskType::BLOCK); + ASSERT_EQ(nullptr, mig_param.data_block_macro_meta_buf_); + mig_param.is_meta_root_ = true; + +} + +TEST(TestBackupCompatible, test_backup_compatible) +{ + CompatBackupSSTableMeta compat_sstable_meta; + compat_sstable_meta.tablet_id_ = ObTabletID(1); + prepare_migration_sstable_param(compat_sstable_meta.sstable_meta_); + compat_sstable_meta.entry_block_addr_for_other_block_ = ObBackupPhysicalID::get_default(); + compat_sstable_meta.total_other_block_count_ = 0; + compat_sstable_meta.is_major_compaction_mview_dep_ = false; + + int64_t pos = 0; + const int64_t buf_len = compat_sstable_meta.get_serialize_size(); + char *buf = new char [buf_len]; + ASSERT_EQ(OB_SUCCESS, compat_sstable_meta.serialize(buf, buf_len, pos)); + + CompatBackupSSTableMeta tmp_compat_sstable_meta; + pos = 0; + ASSERT_EQ(OB_SUCCESS, tmp_compat_sstable_meta.deserialize(buf, buf_len, pos)); + + ObBackupSSTableMeta backup_sstable_meta; + pos = 0; + ASSERT_EQ(OB_SUCCESS, backup_sstable_meta.deserialize(buf, buf_len, pos)); + + ASSERT_EQ(compat_sstable_meta.tablet_id_, backup_sstable_meta.tablet_id_); + ASSERT_EQ(compat_sstable_meta.sstable_meta_.basic_meta_, backup_sstable_meta.sstable_meta_.basic_meta_); + ASSERT_EQ(compat_sstable_meta.total_other_block_count_, backup_sstable_meta.total_other_block_count_); + ASSERT_EQ(compat_sstable_meta.is_major_compaction_mview_dep_, backup_sstable_meta.is_major_compaction_mview_dep_); +} + +} // namespace backup +} // namespace oceanbase + +int main(int argc, char **argv) +{ + system("rm -f test_backup_compatible.log*"); + OB_LOGGER.set_file_name("test_backup_compatible.log", true); + OB_LOGGER.set_log_level("info"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}