From 7068d87d0490a1d431e7c9fc8a3b3d0bf961e905 Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 8 Feb 2024 16:29:26 +0000 Subject: [PATCH] fix macro block id array use searray operator= --- .../ob_index_block_data_prepare.h | 8 +- mittest/mtlenv/storage/test_co_merge.cpp | 4 +- .../ob_shared_macro_block_manager.cpp | 56 +-- .../compaction/ob_tablet_merge_info.cpp | 92 +---- src/storage/ddl/ob_ddl_merge_task.cpp | 179 ++------- .../ob_physical_copy_task.cpp | 83 +---- .../tablet/ob_tablet_create_sstable_param.cpp | 351 ++++++++++++++++++ .../tablet/ob_tablet_create_sstable_param.h | 44 +++ .../ob_multi_version_sstable_test.h | 4 +- 9 files changed, 439 insertions(+), 382 deletions(-) diff --git a/mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h b/mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h index d01e20784..e5debf109 100644 --- a/mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h +++ b/mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h @@ -579,8 +579,8 @@ void TestIndexBlockDataPrepare::close_builder_and_prepare_sstable(const int64_t param.nested_offset_ = res.nested_offset_; param.nested_size_ = res.nested_size_; param.compressor_type_ = ObCompressorType::NONE_COMPRESSOR; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; + ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_)); + ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_)); param.ddl_scn_.set_min(); param.filled_tx_scn_.set_min(); param.contain_uncommitted_row_ = false; @@ -1036,13 +1036,13 @@ void TestIndexBlockDataPrepare::prepare_partial_sstable(const int64_t column_cnt param.nested_offset_ = res.nested_offset_; param.nested_size_ = res.nested_size_; param.compressor_type_ = ObCompressorType::NONE_COMPRESSOR; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; param.ddl_scn_.convert_from_ts(ObTimeUtility::current_time()); param.filled_tx_scn_.set_min(); param.contain_uncommitted_row_ = false; param.encrypt_id_ = res.encrypt_id_; param.master_key_id_ = res.master_key_id_; + ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_)); + ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_)); if (param.table_key_.is_co_sstable() && param.column_group_cnt_ <= 1) { param.column_group_cnt_ = column_cnt + 2; /* set column group_cnt to avoid return err, cnt is calculated as each + all + default*/ } diff --git a/mittest/mtlenv/storage/test_co_merge.cpp b/mittest/mtlenv/storage/test_co_merge.cpp index d10e5684b..41ebb1dc2 100644 --- a/mittest/mtlenv/storage/test_co_merge.cpp +++ b/mittest/mtlenv/storage/test_co_merge.cpp @@ -138,8 +138,8 @@ void close_builder_and_prepare_sstable( param.master_key_id_ = res.master_key_id_; param.nested_size_ = res.nested_size_; param.nested_offset_ = res.nested_offset_; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; + ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_)); + ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_)); param.nested_size_ = res.nested_size_; param.nested_offset_ = res.nested_offset_; if (is_major_merge_type(data_store_desc.get_merge_type())) { diff --git a/src/storage/blocksstable/ob_shared_macro_block_manager.cpp b/src/storage/blocksstable/ob_shared_macro_block_manager.cpp index ce6339e8e..9860c8d8c 100644 --- a/src/storage/blocksstable/ob_shared_macro_block_manager.cpp +++ b/src/storage/blocksstable/ob_shared_macro_block_manager.cpp @@ -717,58 +717,10 @@ int ObSharedMacroBlockMgr::create_new_sstable( ObSSTableMetaHandle meta_handle; if (OB_FAIL(old_table.get_meta(meta_handle))) { LOG_WARN("get meta handle fail", K(ret), K(old_table)); - } else { - const ObSSTableBasicMeta &basic_meta = meta_handle.get_sstable_meta().get_basic_meta(); - param.filled_tx_scn_ = basic_meta.filled_tx_scn_; - param.ddl_scn_ = basic_meta.ddl_scn_; - param.table_key_ = old_table.get_key(); - param.sstable_logic_seq_ = meta_handle.get_sstable_meta().get_sstable_seq(); - param.table_mode_ = basic_meta.table_mode_; - param.index_type_ = static_cast(basic_meta.index_type_); - param.schema_version_ = basic_meta.schema_version_; - param.create_snapshot_version_ = basic_meta.create_snapshot_version_; - param.progressive_merge_round_ = basic_meta.progressive_merge_round_; - param.progressive_merge_step_ = basic_meta.progressive_merge_step_; - param.rowkey_column_cnt_ = basic_meta.rowkey_column_count_; - param.recycle_version_ = basic_meta.recycle_version_; - param.latest_row_store_type_ = basic_meta.latest_row_store_type_; - param.is_ready_for_read_ = true; - - ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, - param.root_block_addr_, param.root_block_data_); - ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, - param.data_block_macro_meta_addr_, param.data_block_macro_meta_); - param.root_row_store_type_ = res.root_row_store_type_; - param.data_index_tree_height_ = res.root_desc_.height_; - param.index_blocks_cnt_ = res.index_blocks_cnt_; - param.data_blocks_cnt_ = res.data_blocks_cnt_; - param.micro_block_cnt_ = res.micro_block_cnt_; - param.use_old_macro_block_count_ = res.use_old_macro_block_count_; - param.row_count_ = res.row_count_; - param.column_cnt_ = res.data_column_cnt_; - param.data_checksum_ = res.data_checksum_; - param.occupy_size_ = res.occupy_size_; - param.original_size_ = res.original_size_; - param.max_merged_trans_version_ = res.max_merged_trans_version_; - param.contain_uncommitted_row_ = res.contain_uncommitted_row_; - param.compressor_type_ = res.compressor_type_; - param.encrypt_id_ = res.encrypt_id_; - param.master_key_id_ = res.master_key_id_; - param.data_block_ids_ = res.data_block_ids_; - param.is_meta_root_ = res.data_root_desc_.is_meta_root_; - param.nested_offset_ = block_info.nested_offset_; - param.nested_size_ = block_info.nested_size_; - param.other_block_ids_ = res.other_block_ids_; - MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - - if (param.table_key_.is_major_sstable()) { - if (OB_FAIL(param.column_checksums_.assign(res.data_column_checksums_))) { - LOG_WARN("fail to fill column checksum", K(ret), K(res)); - } - } - } - if (OB_FAIL(ret)) { - // do nothing + } else if (OB_FAIL(param.init_for_small_sstable(res, old_table.get_key(), + meta_handle.get_sstable_meta(), block_info))) { + LOG_WARN("fail to init create sstable param for small sst", + K(ret), K(res), K(old_table.get_key()), K(meta_handle.get_sstable_meta()), K(block_info)); } else if (OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(param)); diff --git a/src/storage/compaction/ob_tablet_merge_info.cpp b/src/storage/compaction/ob_tablet_merge_info.cpp index a112fd412..91446b497 100644 --- a/src/storage/compaction/ob_tablet_merge_info.cpp +++ b/src/storage/compaction/ob_tablet_merge_info.cpp @@ -131,101 +131,17 @@ int ObTabletMergeInfo::build_create_sstable_param(const ObBasicTabletMergeCtx &c const bool is_main_table) { int ret = OB_SUCCESS; - const ObStaticMergeParam &static_param = ctx.static_param_; if (OB_UNLIKELY(!ctx.is_valid() || !res.is_valid() || (nullptr != cg_schema && (!cg_schema->is_valid() || column_group_idx < 0)))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid merge ctx", K(ret), K(ctx), K(res), K(column_group_idx), KPC(cg_schema)); } else if (OB_UNLIKELY(nullptr != cg_schema && cg_schema->column_cnt_ != res.data_column_cnt_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table column cnt is unexpected mismatched!", K(ret), KPC(cg_schema), K(res)); - } else if (OB_FAIL(static_param.schema_->get_stored_column_count_in_sstable(param.full_column_cnt_))) { - LOG_WARN("fail to get_stored_column_count_in_sstable", K(ret), KPC(cg_schema), K(res)); - } else { - ObITable::TableKey table_key; - table_key.table_type_ = ctx.get_merged_table_type(cg_schema, is_main_table); - table_key.tablet_id_ = ctx.get_tablet_id(); - table_key.column_group_idx_ = (nullptr == cg_schema) ? 0 : column_group_idx; - if (is_major_or_meta_merge_type(static_param.get_merge_type())) { - table_key.version_range_.snapshot_version_ = static_param.version_range_.snapshot_version_; - } else { - table_key.scn_range_ = static_param.scn_range_; - } - param.table_key_ = table_key; - - if (ObITable::TableType::COLUMN_ORIENTED_SSTABLE == table_key.table_type_ || - ObITable::TableType::COLUMN_ORIENTED_META_SSTABLE == table_key.table_type_) { - param.co_base_type_ = cg_schema->is_all_column_group() - ? ObCOSSTableBaseType::ALL_CG_TYPE - : ObCOSSTableBaseType::ROWKEY_CG_TYPE; - } - - param.sstable_logic_seq_ = static_param.sstable_logic_seq_; - param.filled_tx_scn_ = ctx.get_merge_scn(); - - param.table_mode_ = ctx.get_schema()->get_table_mode_struct(); - param.index_type_ = ctx.get_schema()->get_index_type(); - if (nullptr != cg_schema && !cg_schema->is_rowkey_column_group() && !cg_schema->is_all_column_group()) { - param.rowkey_column_cnt_ = 0; - } else { - param.column_group_cnt_ = static_param.schema_->get_column_group_count(); - param.rowkey_column_cnt_ = static_param.schema_->get_rowkey_column_num() - + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - } - param.latest_row_store_type_ = ctx.get_schema()->get_row_store_type(); - if (is_minor_merge_type(static_param.get_merge_type())) { - param.recycle_version_ = static_param.version_range_.base_version_; - } else { - param.recycle_version_ = 0; - } - param.schema_version_ = static_param.schema_version_; - param.create_snapshot_version_ = static_param.create_snapshot_version_; - param.progressive_merge_round_ = static_param.progressive_merge_round_; - param.progressive_merge_step_ = std::min( - static_param.progressive_merge_num_, static_param.progressive_merge_step_ + 1); - - ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, - param.root_block_addr_, param.root_block_data_); - ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, - param.data_block_macro_meta_addr_, param.data_block_macro_meta_); - param.is_meta_root_ = res.data_root_desc_.is_meta_root_; - param.root_row_store_type_ = res.root_row_store_type_; - param.data_index_tree_height_ = res.root_desc_.height_; - param.index_blocks_cnt_ = res.index_blocks_cnt_; - param.data_blocks_cnt_ = res.data_blocks_cnt_; - param.is_empty_co_table_ = is_main_table ? (0 == param.data_blocks_cnt_) : false; - param.micro_block_cnt_ = res.micro_block_cnt_; - param.use_old_macro_block_count_ = res.use_old_macro_block_count_; - param.row_count_ = res.row_count_; - param.column_cnt_ = res.data_column_cnt_; - param.data_checksum_ = res.data_checksum_; - param.occupy_size_ = res.occupy_size_; - param.original_size_ = res.original_size_; - if ((0 == res.row_count_ && 0 == res.max_merged_trans_version_) - || (nullptr != cg_schema && !cg_schema->has_multi_version_column())) { - // empty mini table merged forcely - param.max_merged_trans_version_ = static_param.version_range_.snapshot_version_; - } else { - param.max_merged_trans_version_ = res.max_merged_trans_version_; - } - param.contain_uncommitted_row_ = res.contain_uncommitted_row_; - param.compressor_type_ = res.compressor_type_; - param.encrypt_id_ = res.encrypt_id_; - param.master_key_id_ = res.master_key_id_; - param.nested_size_ = res.nested_size_; - param.nested_offset_ = res.nested_offset_; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; - param.ddl_scn_.set_min(); - MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - if (is_major_or_meta_merge_type(static_param.get_merge_type())) { - if (OB_FAIL(param.column_checksums_.assign(res.data_column_checksums_))) { - LOG_WARN("fail to fill column checksum", K(ret), K(res)); - } - } - - if (OB_SUCC(ret) && ctx.get_tablet_id().is_ls_tx_data_tablet()) { + } else if (OB_FAIL(param.init_for_merge(ctx, res, cg_schema, column_group_idx, is_main_table))) { + LOG_WARN("fail to init create sstable param for merge", + K(ret), K(ctx), K(res), KPC(cg_schema), K(column_group_idx), K(is_main_table)); + } else if (ctx.get_tablet_id().is_ls_tx_data_tablet()) { ret = record_start_tx_scn_for_tx_data(ctx, param); - } } return ret; } diff --git a/src/storage/ddl/ob_ddl_merge_task.cpp b/src/storage/ddl/ob_ddl_merge_task.cpp index 1bcaaf446..3a0ab8c85 100644 --- a/src/storage/ddl/ob_ddl_merge_task.cpp +++ b/src/storage/ddl/ob_ddl_merge_task.cpp @@ -555,9 +555,9 @@ int ObTabletDDLUtil::create_ddl_sstable(ObTablet &tablet, ObIndexBlockRebuilder index_block_rebuilder; ObWholeDataStoreDesc data_desc(true/*is_ddl*/); int64_t macro_block_column_count = 0; - if (OB_UNLIKELY(!ddl_param.is_valid())) { + if (OB_UNLIKELY(!ddl_param.is_valid() || OB_ISNULL(storage_schema))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ddl_param)); + LOG_WARN("invalid argument", K(ret), K(ddl_param), KP(storage_schema)); } else if (OB_FAIL(ObTabletDDLUtil::prepare_index_data_desc( tablet, ddl_param.table_key_.is_column_store_sstable() ? ddl_param.table_key_.get_column_group_id() : -1/*negative value means row store*/, @@ -613,162 +613,31 @@ int ObTabletDDLUtil::create_ddl_sstable( ObTableHandleV2 &sstable_handle) { int ret = OB_SUCCESS; - SMART_VAR(ObSSTableMergeRes, res) { - if (OB_UNLIKELY(nullptr == sstable_index_builder || !ddl_param.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KP(sstable_index_builder), K(ddl_param)); + if (OB_UNLIKELY(nullptr == sstable_index_builder || !ddl_param.is_valid() || OB_ISNULL(storage_schema))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), KP(sstable_index_builder), K(ddl_param), KP(storage_schema)); + } else { + const int64_t create_schema_version_on_tablet = tablet.get_tablet_meta().create_schema_version_; + ObTabletCreateSSTableParam param; + if (OB_FAIL(param.init_for_ddl(sstable_index_builder, ddl_param, first_ddl_sstable, + *storage_schema, macro_block_column_count, create_schema_version_on_tablet))) { + LOG_WARN("fail to init param for ddl", + K(ret), K(macro_block_column_count), K(create_schema_version_on_tablet), + KPC(sstable_index_builder), K(ddl_param), + KPC(first_ddl_sstable), KPC(storage_schema)); + } else if (ddl_param.table_key_.is_co_sstable()) { + if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, sstable_handle))) { + LOG_WARN("create sstable failed", K(ret), K(param)); + } } else { - int64_t column_count = 0; - int64_t full_column_cnt = 0; // only used for co sstable - share::schema::ObTableMode table_mode = storage_schema->get_table_mode_struct(); - share::schema::ObIndexType index_type = storage_schema->get_index_type(); - int64_t rowkey_column_cnt = storage_schema->get_rowkey_column_num() + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - common::ObRowStoreType row_store_type = storage_schema->get_row_store_type(); - if (nullptr != first_ddl_sstable) { - ObSSTableMetaHandle meta_handle; - if (OB_FAIL(first_ddl_sstable->get_meta(meta_handle))) { - LOG_WARN("get sstable meta handle fail", K(ret), KPC(first_ddl_sstable)); - } else { - column_count = meta_handle.get_sstable_meta().get_column_count(); - table_mode = meta_handle.get_sstable_meta().get_basic_meta().table_mode_; - index_type = static_cast(meta_handle.get_sstable_meta().get_basic_meta().index_type_); - rowkey_column_cnt = meta_handle.get_sstable_meta().get_basic_meta().rowkey_column_count_; - row_store_type = meta_handle.get_sstable_meta().get_basic_meta().latest_row_store_type_; - if (first_ddl_sstable->is_co_sstable()) { - const ObCOSSTableV2 *first_co_sstable = static_cast(first_ddl_sstable); - if (OB_ISNULL((first_co_sstable))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("first co sstable is null", K(ret), KP(first_co_sstable), KPC(first_ddl_sstable)); - } else { - full_column_cnt = first_co_sstable->get_cs_meta().full_column_cnt_; - } - } - } - } else if (ddl_param.table_key_.is_column_store_sstable()) { - if (ddl_param.table_key_.is_normal_cg_sstable()) { - rowkey_column_cnt = 0; - column_count = 1; - } else { // co sstable with all cg or rowkey cg - const ObIArray &cg_schemas = storage_schema->get_column_groups(); - const int64_t cg_idx = ddl_param.table_key_.get_column_group_id(); - if (cg_idx < 0 || cg_idx >= cg_schemas.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected column group index", K(ret), K(cg_idx)); - } else if (OB_FAIL(storage_schema->get_stored_column_count_in_sstable(full_column_cnt))) { // set full_column_cnt in first ddl sstable - LOG_WARN("fail to get stored column count in sstable", K(ret)); - } else if (cg_schemas.at(cg_idx).is_rowkey_column_group()) { - column_count = rowkey_column_cnt; - } else { - column_count = full_column_cnt; - if (macro_block_column_count > 0 && macro_block_column_count < column_count) { - LOG_INFO("use macro block column count", K(ddl_param), K(macro_block_column_count), K(column_count)); - column_count = macro_block_column_count; - full_column_cnt = macro_block_column_count; - } - } - } - } else { // row store sstable - if (OB_FAIL(storage_schema->get_stored_column_count_in_sstable(column_count))) { - LOG_WARN("fail to get stored column count in sstable", K(ret)); - } else if (macro_block_column_count > 0 && macro_block_column_count < column_count) { - LOG_INFO("use macro block column count", K(ddl_param), K(macro_block_column_count), K(column_count)); - column_count = macro_block_column_count; - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(sstable_index_builder->close(res))) { - LOG_WARN("close sstable index builder close failed", K(ret)); - } else if (ddl_param.table_key_.is_normal_cg_sstable() // index builder of cg sstable cannot get trans_version from row, manually set it - && FALSE_IT(res.max_merged_trans_version_ = ddl_param.snapshot_version_)) { - } else if (OB_UNLIKELY((ddl_param.table_key_.is_major_sstable() || - ddl_param.table_key_.is_ddl_sstable()) && - res.row_count_ > 0 && - res.max_merged_trans_version_ != ddl_param.snapshot_version_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("max_merged_trans_version_ in res is different from ddl snapshot version", K(ret), - K(res), K(ddl_param)); - } else { - const int64_t create_schema_version_on_tablet = tablet.get_tablet_meta().create_schema_version_; - ObTabletCreateSSTableParam param; - param.table_key_ = ddl_param.table_key_; - param.table_mode_ = table_mode; - param.index_type_ = index_type; - param.rowkey_column_cnt_ = rowkey_column_cnt; - param.schema_version_ = create_schema_version_on_tablet; - param.latest_row_store_type_ = row_store_type; - param.create_snapshot_version_ = ddl_param.snapshot_version_; - param.ddl_scn_ = ddl_param.start_scn_; - ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, - param.root_block_addr_, param.root_block_data_); - ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, - param.data_block_macro_meta_addr_, param.data_block_macro_meta_); - param.is_meta_root_ = res.data_root_desc_.is_meta_root_; - param.root_row_store_type_ = res.root_row_store_type_; - param.data_index_tree_height_ = res.root_desc_.height_; - param.index_blocks_cnt_ = res.index_blocks_cnt_; - param.data_blocks_cnt_ = res.data_blocks_cnt_; - param.micro_block_cnt_ = res.micro_block_cnt_; - param.use_old_macro_block_count_ = res.use_old_macro_block_count_; - param.row_count_ = res.row_count_; - param.column_cnt_ = column_count; - param.full_column_cnt_ = full_column_cnt; - param.data_checksum_ = res.data_checksum_; - param.occupy_size_ = res.occupy_size_; - param.original_size_ = res.original_size_; - param.max_merged_trans_version_ = ddl_param.snapshot_version_; - param.contain_uncommitted_row_ = res.contain_uncommitted_row_; - param.compressor_type_ = res.compressor_type_; - param.encrypt_id_ = res.encrypt_id_; - param.master_key_id_ = res.master_key_id_; - param.nested_size_ = res.nested_size_; - param.nested_offset_ = res.nested_offset_; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; - MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - if (ddl_param.table_key_.is_co_sstable()) { - param.column_group_cnt_ = storage_schema->get_column_group_count(); - // only set true when build empty major sstable. ddl co sstable must set false and fill cg sstables - param.is_empty_co_table_ = ddl_param.table_key_.is_major_sstable() && 0 == param.data_blocks_cnt_; - const int64_t base_cg_idx = ddl_param.table_key_.get_column_group_id(); - if (base_cg_idx < 0 || base_cg_idx >= storage_schema->get_column_group_count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid column group index", K(ret), K(ddl_param.table_key_)); - } else { - const ObStorageColumnGroupSchema &base_cg_schema = storage_schema->get_column_groups().at(base_cg_idx); - if (base_cg_schema.is_all_column_group()) { - param.co_base_type_ = ObCOSSTableBaseType::ALL_CG_TYPE; - } else if (base_cg_schema.is_rowkey_column_group()) { - param.co_base_type_ = ObCOSSTableBaseType::ROWKEY_CG_TYPE; - } else { - ret = OB_ERR_SYS; - LOG_WARN("unknown type of base cg schema", K(ret), K(base_cg_idx)); - } - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(param.column_checksums_.assign(res.data_column_checksums_))) { - LOG_WARN("fail to fill column checksum for empty major", K(ret), K(param)); - } else if (OB_UNLIKELY(param.column_checksums_.count() != column_count)) { - // we have corrected the col_default_checksum_array_ in prepare_index_data_desc - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected column checksums", K(ret), K(column_count), K(param)); - } else { - if (ddl_param.table_key_.is_co_sstable()) { - if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, sstable_handle))) { - LOG_WARN("create sstable failed", K(ret), K(param)); - } - } else { - if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, sstable_handle))) { - LOG_WARN("create sstable failed", K(ret), K(param)); - } - } - if (OB_SUCC(ret)) { - LOG_INFO("create ddl sstable success", K(ddl_param), K(sstable_handle), - "create_schema_version", create_schema_version_on_tablet); - } - } + if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, sstable_handle))) { + LOG_WARN("create sstable failed", K(ret), K(param)); } } + if (OB_SUCC(ret)) { + LOG_INFO("create ddl sstable success", K(ddl_param), K(sstable_handle), + "create_schema_version", create_schema_version_on_tablet); + } } return ret; } diff --git a/src/storage/high_availability/ob_physical_copy_task.cpp b/src/storage/high_availability/ob_physical_copy_task.cpp index 46cc747b1..ef2c7c56d 100644 --- a/src/storage/high_availability/ob_physical_copy_task.cpp +++ b/src/storage/high_availability/ob_physical_copy_task.cpp @@ -1060,7 +1060,6 @@ int ObSSTableCopyFinishTask::build_create_sstable_param_( const blocksstable::ObSSTableMergeRes &res, ObTabletCreateSSTableParam ¶m) { - //TODO(lingchuan) this param maker ObTablet class will be better to be safeguard int ret = OB_SUCCESS; if (!is_inited_) { ret = OB_INVALID_ARGUMENT; @@ -1068,48 +1067,8 @@ int ObSSTableCopyFinishTask::build_create_sstable_param_( } else if (OB_UNLIKELY(OB_ISNULL(tablet) || !res.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("build create sstable param get invalid argument", K(ret), KP(tablet), K(res)); - } else { - param.table_key_ = sstable_param_->table_key_; - param.sstable_logic_seq_ = sstable_param_->basic_meta_.sstable_logic_seq_; - param.schema_version_ = sstable_param_->basic_meta_.schema_version_; - param.table_mode_ = sstable_param_->basic_meta_.table_mode_; - param.index_type_ = static_cast(sstable_param_->basic_meta_.index_type_); - param.create_snapshot_version_ = sstable_param_->basic_meta_.create_snapshot_version_; - param.progressive_merge_round_ = sstable_param_->basic_meta_.progressive_merge_round_; - param.progressive_merge_step_ = sstable_param_->basic_meta_.progressive_merge_step_; - param.latest_row_store_type_ = sstable_param_->basic_meta_.latest_row_store_type_; - - ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, - param.root_block_addr_, param.root_block_data_); - ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, - param.data_block_macro_meta_addr_, param.data_block_macro_meta_); - param.is_meta_root_ = res.data_root_desc_.is_meta_root_; - param.root_row_store_type_ = res.root_row_store_type_; - param.data_index_tree_height_ = res.root_desc_.height_; - param.index_blocks_cnt_ = res.index_blocks_cnt_; - param.data_blocks_cnt_ = res.data_blocks_cnt_; - param.micro_block_cnt_ = res.micro_block_cnt_; - param.use_old_macro_block_count_ = res.use_old_macro_block_count_; - param.row_count_ = res.row_count_; - param.column_cnt_ = res.data_column_cnt_; - param.data_checksum_ = res.data_checksum_; - param.occupy_size_ = res.occupy_size_; - param.original_size_ = res.original_size_; - param.max_merged_trans_version_ = res.max_merged_trans_version_; - param.contain_uncommitted_row_ = res.contain_uncommitted_row_; - param.compressor_type_ = res.compressor_type_; - param.encrypt_id_ = res.encrypt_id_; - param.master_key_id_ = res.master_key_id_; - param.nested_size_ = res.nested_size_; - param.nested_offset_ = res.nested_offset_; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; - param.rowkey_column_cnt_ = sstable_param_->basic_meta_.rowkey_column_count_; - param.ddl_scn_ = sstable_param_->basic_meta_.ddl_scn_; - MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - if (OB_FAIL(param.column_checksums_.assign(sstable_param_->column_checksums_))) { - LOG_WARN("fail to fill column checksum", K(ret), KPC_(sstable_param)); - } + } else if (OB_FAIL(param.init_for_ha(*sstable_param_, res))) { + LOG_WARN("fail to init create sstable param", K(ret), KPC(sstable_param_), K(res)); } return ret; } @@ -1117,7 +1076,6 @@ int ObSSTableCopyFinishTask::build_create_sstable_param_( int ObSSTableCopyFinishTask::build_create_sstable_param_( ObTabletCreateSSTableParam ¶m) { - //TODO(lingchuan) this param maker ObTablet class will be better to be safeguard //using sstable meta to create sstable int ret = OB_SUCCESS; if (!is_inited_) { @@ -1126,41 +1084,8 @@ int ObSSTableCopyFinishTask::build_create_sstable_param_( } else if (0 != sstable_param_->basic_meta_.data_macro_block_count_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("sstable param has data macro block, can not build sstable from basic meta", K(ret), KPC(sstable_param_)); - } else { - param.table_key_ = sstable_param_->table_key_; - param.sstable_logic_seq_ = sstable_param_->basic_meta_.sstable_logic_seq_; - param.schema_version_ = sstable_param_->basic_meta_.schema_version_; - param.create_snapshot_version_ = sstable_param_->basic_meta_.create_snapshot_version_; - param.table_mode_ = sstable_param_->basic_meta_.table_mode_; - param.index_type_ = static_cast(sstable_param_->basic_meta_.index_type_); - param.progressive_merge_round_ = sstable_param_->basic_meta_.progressive_merge_round_; - param.progressive_merge_step_ = sstable_param_->basic_meta_.progressive_merge_step_; - param.is_ready_for_read_ = true; - param.root_row_store_type_ = sstable_param_->basic_meta_.root_row_store_type_; - param.latest_row_store_type_ = sstable_param_->basic_meta_.latest_row_store_type_; - param.index_blocks_cnt_ = sstable_param_->basic_meta_.index_macro_block_count_; - param.data_blocks_cnt_ = sstable_param_->basic_meta_.data_macro_block_count_; - param.micro_block_cnt_ = sstable_param_->basic_meta_.data_micro_block_count_; - param.use_old_macro_block_count_ = sstable_param_->basic_meta_.use_old_macro_block_count_; - param.row_count_ = sstable_param_->basic_meta_.row_count_; - param.column_cnt_ = sstable_param_->basic_meta_.column_cnt_; - param.data_checksum_ = sstable_param_->basic_meta_.data_checksum_; - param.occupy_size_ = sstable_param_->basic_meta_.occupy_size_; - param.original_size_ = sstable_param_->basic_meta_.original_size_; - param.max_merged_trans_version_ = sstable_param_->basic_meta_.max_merged_trans_version_; - param.ddl_scn_ = sstable_param_->basic_meta_.ddl_scn_; - param.filled_tx_scn_ = sstable_param_->basic_meta_.filled_tx_scn_; - param.contain_uncommitted_row_ = sstable_param_->basic_meta_.contain_uncommitted_row_; - param.compressor_type_ = sstable_param_->basic_meta_.compressor_type_; - param.encrypt_id_ = sstable_param_->basic_meta_.encrypt_id_; - param.master_key_id_ = sstable_param_->basic_meta_.master_key_id_; - param.root_block_addr_.set_none_addr(); - param.data_block_macro_meta_addr_.set_none_addr(); - param.rowkey_column_cnt_ = sstable_param_->basic_meta_.rowkey_column_count_; - MEMCPY(param.encrypt_key_, sstable_param_->basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - if (OB_FAIL(param.column_checksums_.assign(sstable_param_->column_checksums_))) { - LOG_WARN("fail to assign column checksums", K(ret), KPC(sstable_param_)); - } + } else if (OB_FAIL(param.init_for_ha(*sstable_param_))) { + LOG_WARN("fail to init create sstable param", K(ret), KPC(sstable_param_)); } return ret; } diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.cpp b/src/storage/tablet/ob_tablet_create_sstable_param.cpp index 219817231..aa1dd9646 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.cpp +++ b/src/storage/tablet/ob_tablet_create_sstable_param.cpp @@ -19,7 +19,11 @@ #include "storage/ob_sstable_struct.h" #include "storage/blocksstable/ob_macro_block_struct.h" #include "storage/blocksstable/ob_block_sstable_struct.h" +#include "storage/blocksstable/index_block/ob_index_block_builder.h" #include "storage/column_store/ob_column_oriented_sstable.h" +#include "storage/blocksstable/ob_shared_macro_block_manager.h" +#include "storage/compaction/ob_basic_tablet_merge_ctx.h" +#include "storage/ddl/ob_direct_load_struct.h" namespace oceanbase { @@ -134,5 +138,352 @@ bool ObTabletCreateSSTableParam::is_block_meta_valid(const storage::ObMetaDiskAd return addr.is_valid() && (!addr.is_memory() || (data.is_valid() && data.size_ == addr.size())); } +int ObTabletCreateSSTableParam::inner_init_with_merge_res(const blocksstable::ObSSTableMergeRes &res) +{ + int ret = OB_SUCCESS; + ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, + root_block_addr_, root_block_data_); + ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, + data_block_macro_meta_addr_, data_block_macro_meta_); + root_row_store_type_ = res.root_row_store_type_; + data_index_tree_height_ = res.root_desc_.height_; + index_blocks_cnt_ = res.index_blocks_cnt_; + data_blocks_cnt_ = res.data_blocks_cnt_; + micro_block_cnt_ = res.micro_block_cnt_; + use_old_macro_block_count_ = res.use_old_macro_block_count_; + row_count_ = res.row_count_; + data_checksum_ = res.data_checksum_; + occupy_size_ = res.occupy_size_; + original_size_ = res.original_size_; + contain_uncommitted_row_ = res.contain_uncommitted_row_; + compressor_type_ = res.compressor_type_; + encrypt_id_ = res.encrypt_id_; + master_key_id_ = res.master_key_id_; + is_meta_root_ = res.data_root_desc_.is_meta_root_; + STATIC_ASSERT(ARRAYSIZEOF(encrypt_key_) == share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH, + "ObTabletCreateSSTableParam encrypt_key_ array size mismatch OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH"); + STATIC_ASSERT(ARRAYSIZEOF(res.encrypt_key_) == share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH, + "ObSSTableMergeRes encrypt_key_ array size mismatch OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH"); + MEMCPY(encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); + + if (OB_FAIL(data_block_ids_.assign(res.data_block_ids_))) { + LOG_WARN("fail to fill data block ids", K(ret), K(res.data_block_ids_)); + } else if (OB_FAIL(other_block_ids_.assign(res.other_block_ids_))) { + LOG_WARN("fail to fill other block ids", K(ret), K(res.other_block_ids_)); + } + return ret; +} + +int ObTabletCreateSSTableParam::init_for_small_sstable(const blocksstable::ObSSTableMergeRes &res, + const ObITable::TableKey &table_key, + const blocksstable::ObSSTableMeta &sstable_meta, + const blocksstable::ObBlockInfo &block_info) +{ + int ret = OB_SUCCESS; + const blocksstable::ObSSTableBasicMeta &basic_meta = sstable_meta.get_basic_meta(); + filled_tx_scn_ = basic_meta.filled_tx_scn_; + ddl_scn_ = basic_meta.ddl_scn_; + table_key_ = table_key; + sstable_logic_seq_ = sstable_meta.get_sstable_seq(); + table_mode_ = basic_meta.table_mode_; + index_type_ = static_cast(basic_meta.index_type_); + schema_version_ = basic_meta.schema_version_; + create_snapshot_version_ = basic_meta.create_snapshot_version_; + progressive_merge_round_ = basic_meta.progressive_merge_round_; + progressive_merge_step_ = basic_meta.progressive_merge_step_; + rowkey_column_cnt_ = basic_meta.rowkey_column_count_; + recycle_version_ = basic_meta.recycle_version_; + latest_row_store_type_ = basic_meta.latest_row_store_type_; + is_ready_for_read_ = true; + column_cnt_ = res.data_column_cnt_; + max_merged_trans_version_ = res.max_merged_trans_version_; + nested_offset_ = block_info.nested_offset_; + nested_size_ = block_info.nested_size_; + if (OB_FAIL(inner_init_with_merge_res(res))) { + LOG_WARN("fail to inner init with merge res", K(ret), K(res)); + } else if (table_key_.is_major_sstable()) { + if (OB_FAIL(column_checksums_.assign(res.data_column_checksums_))) { + LOG_WARN("fail to fill column checksum", K(ret), K(res.data_column_checksums_)); + } + } + + return ret; +} + +int ObTabletCreateSSTableParam::init_for_merge(const compaction::ObBasicTabletMergeCtx &ctx, + const blocksstable::ObSSTableMergeRes &res, + const ObStorageColumnGroupSchema *cg_schema, + const int64_t column_group_idx, + const bool is_main_table) +{ + int ret = OB_SUCCESS; + const compaction::ObStaticMergeParam &static_param = ctx.static_param_; + if (OB_FAIL(static_param.schema_->get_stored_column_count_in_sstable(full_column_cnt_))) { + LOG_WARN("fail to get_stored_column_count_in_sstable", K(ret), KPC(cg_schema), K(res)); + } else { + ObITable::TableKey table_key; + table_key.table_type_ = ctx.get_merged_table_type(cg_schema, is_main_table); + table_key.tablet_id_ = ctx.get_tablet_id(); + table_key.column_group_idx_ = (nullptr == cg_schema) ? 0 : column_group_idx; + if (is_major_or_meta_merge_type(static_param.get_merge_type())) { + table_key.version_range_.snapshot_version_ = static_param.version_range_.snapshot_version_; + } else { + table_key.scn_range_ = static_param.scn_range_; + } + table_key_ = table_key; + + if (ObITable::TableType::COLUMN_ORIENTED_SSTABLE == table_key.table_type_ || + ObITable::TableType::COLUMN_ORIENTED_META_SSTABLE == table_key.table_type_) { + co_base_type_ = cg_schema->is_all_column_group() + ? ObCOSSTableBaseType::ALL_CG_TYPE + : ObCOSSTableBaseType::ROWKEY_CG_TYPE; + } + + sstable_logic_seq_ = static_param.sstable_logic_seq_; + filled_tx_scn_ = ctx.get_merge_scn(); + + table_mode_ = ctx.get_schema()->get_table_mode_struct(); + index_type_ = ctx.get_schema()->get_index_type(); + if (nullptr != cg_schema && !cg_schema->is_rowkey_column_group() && !cg_schema->is_all_column_group()) { + rowkey_column_cnt_ = 0; + } else { + column_group_cnt_ = static_param.schema_->get_column_group_count(); + rowkey_column_cnt_ = static_param.schema_->get_rowkey_column_num() + + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + } + latest_row_store_type_ = ctx.get_schema()->get_row_store_type(); + if (is_minor_merge_type(static_param.get_merge_type())) { + recycle_version_ = static_param.version_range_.base_version_; + } else { + recycle_version_ = 0; + } + schema_version_ = static_param.schema_version_; + create_snapshot_version_ = static_param.create_snapshot_version_; + progressive_merge_round_ = static_param.progressive_merge_round_; + progressive_merge_step_ = std::min( + static_param.progressive_merge_num_, static_param.progressive_merge_step_ + 1); + is_empty_co_table_ = is_main_table ? (0 == res.data_blocks_cnt_) : false; + column_cnt_ = res.data_column_cnt_; + if ((0 == res.row_count_ && 0 == res.max_merged_trans_version_) + || (nullptr != cg_schema && !cg_schema->has_multi_version_column())) { + // empty mini table merged forcely + max_merged_trans_version_ = static_param.version_range_.snapshot_version_; + } else { + max_merged_trans_version_ = res.max_merged_trans_version_; + } + nested_size_ = res.nested_size_; + nested_offset_ = res.nested_offset_; + ddl_scn_.set_min(); + + if (OB_FAIL(inner_init_with_merge_res(res))) { + LOG_WARN("fail to init with merge res", K(ret), K(res.data_block_ids_)); + } else if (is_major_or_meta_merge_type(static_param.get_merge_type())) { + if (OB_FAIL(column_checksums_.assign(res.data_column_checksums_))) { + LOG_WARN("fail to fill column checksum", K(ret), K(res.data_column_checksums_)); + } + } + } + return ret; +} + +int ObTabletCreateSSTableParam::init_for_ddl(blocksstable::ObSSTableIndexBuilder *sstable_index_builder, + const ObTabletDDLParam &ddl_param, + const blocksstable::ObSSTable *first_ddl_sstable, + const ObStorageSchema &storage_schema, + const int64_t macro_block_column_count, + const int64_t create_schema_version_on_tablet) +{ + int ret = OB_SUCCESS; + SMART_VAR(blocksstable::ObSSTableMergeRes, res) { + int64_t column_count = 0; + int64_t full_column_cnt = 0; // only used for co sstable + share::schema::ObTableMode table_mode = storage_schema.get_table_mode_struct(); + share::schema::ObIndexType index_type = storage_schema.get_index_type(); + int64_t rowkey_column_cnt = storage_schema.get_rowkey_column_num() + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + common::ObRowStoreType row_store_type = storage_schema.get_row_store_type(); + if (nullptr != first_ddl_sstable) { + blocksstable::ObSSTableMetaHandle meta_handle; + if (OB_FAIL(first_ddl_sstable->get_meta(meta_handle))) { + LOG_WARN("get sstable meta handle fail", K(ret), KPC(first_ddl_sstable)); + } else { + column_count = meta_handle.get_sstable_meta().get_column_count(); + table_mode = meta_handle.get_sstable_meta().get_basic_meta().table_mode_; + index_type = static_cast(meta_handle.get_sstable_meta().get_basic_meta().index_type_); + rowkey_column_cnt = meta_handle.get_sstable_meta().get_basic_meta().rowkey_column_count_; + row_store_type = meta_handle.get_sstable_meta().get_basic_meta().latest_row_store_type_; + if (first_ddl_sstable->is_co_sstable()) { + const ObCOSSTableV2 *first_co_sstable = static_cast(first_ddl_sstable); + if (OB_ISNULL((first_co_sstable))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("first co sstable is null", K(ret), KP(first_co_sstable), KPC(first_ddl_sstable)); + } else { + full_column_cnt = first_co_sstable->get_cs_meta().full_column_cnt_; + } + } + } + } else if (ddl_param.table_key_.is_column_store_sstable()) { + if (ddl_param.table_key_.is_normal_cg_sstable()) { + rowkey_column_cnt = 0; + column_count = 1; + } else { // co sstable with all cg or rowkey cg + const ObIArray &cg_schemas = storage_schema.get_column_groups(); + const int64_t cg_idx = ddl_param.table_key_.get_column_group_id(); + if (cg_idx < 0 || cg_idx >= cg_schemas.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column group index", K(ret), K(cg_idx)); + } else if (OB_FAIL(storage_schema.get_stored_column_count_in_sstable(full_column_cnt))) { // set full_column_cnt in first ddl sstable + LOG_WARN("fail to get stored column count in sstable", K(ret)); + } else if (cg_schemas.at(cg_idx).is_rowkey_column_group()) { + column_count = rowkey_column_cnt; + } else { + column_count = full_column_cnt; + if (macro_block_column_count > 0 && macro_block_column_count < column_count) { + LOG_INFO("use macro block column count", K(ddl_param), K(macro_block_column_count), K(column_count)); + column_count = macro_block_column_count; + full_column_cnt = macro_block_column_count; + } + } + } + } else { // row store sstable + if (OB_FAIL(storage_schema.get_stored_column_count_in_sstable(column_count))) { + LOG_WARN("fail to get stored column count in sstable", K(ret)); + } else if (macro_block_column_count > 0 && macro_block_column_count < column_count) { + LOG_INFO("use macro block column count", K(ddl_param), K(macro_block_column_count), K(column_count)); + column_count = macro_block_column_count; + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(sstable_index_builder->close(res))) { + LOG_WARN("close sstable index builder close failed", K(ret)); + } else if (ddl_param.table_key_.is_normal_cg_sstable() // index builder of cg sstable cannot get trans_version from row, manually set it + && FALSE_IT(res.max_merged_trans_version_ = ddl_param.snapshot_version_)) { + } else if (OB_UNLIKELY((ddl_param.table_key_.is_major_sstable() || + ddl_param.table_key_.is_ddl_sstable()) && + res.row_count_ > 0 && + res.max_merged_trans_version_ != ddl_param.snapshot_version_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("max_merged_trans_version_ in res is different from ddl snapshot version", K(ret), + K(res), K(ddl_param)); + } else { + table_key_ = ddl_param.table_key_; + table_mode_ = table_mode; + index_type_ = index_type; + rowkey_column_cnt_ = rowkey_column_cnt; + schema_version_ = create_schema_version_on_tablet; + latest_row_store_type_ = row_store_type; + create_snapshot_version_ = ddl_param.snapshot_version_; + ddl_scn_ = ddl_param.start_scn_; + column_cnt_ = column_count; + full_column_cnt_ = full_column_cnt; + max_merged_trans_version_ = ddl_param.snapshot_version_; + nested_size_ = res.nested_size_; + nested_offset_ = res.nested_offset_; + + if (OB_FAIL(inner_init_with_merge_res(res))) { + LOG_WARN("fail to inner init with merge res", K(ret), K(res)); + } else if (ddl_param.table_key_.is_co_sstable()) { + column_group_cnt_ = storage_schema.get_column_group_count(); + // only set true when build empty major sstable. ddl co sstable must set false and fill cg sstables + is_empty_co_table_ = ddl_param.table_key_.is_major_sstable() && 0 == data_blocks_cnt_; + const int64_t base_cg_idx = ddl_param.table_key_.get_column_group_id(); + if (base_cg_idx < 0 || base_cg_idx >= storage_schema.get_column_group_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid column group index", K(ret), K(ddl_param.table_key_)); + } else { + const ObStorageColumnGroupSchema &base_cg_schema = storage_schema.get_column_groups().at(base_cg_idx); + if (base_cg_schema.is_all_column_group()) { + co_base_type_ = ObCOSSTableBaseType::ALL_CG_TYPE; + } else if (base_cg_schema.is_rowkey_column_group()) { + co_base_type_ = ObCOSSTableBaseType::ROWKEY_CG_TYPE; + } else { + ret = OB_ERR_SYS; + LOG_WARN("unknown type of base cg schema", K(ret), K(base_cg_idx)); + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(column_checksums_.assign(res.data_column_checksums_))) { + LOG_WARN("fail to fill column checksum for empty major", K(ret), K(res.data_column_checksums_)); + } else if (OB_UNLIKELY(column_checksums_.count() != column_count)) { + // we have corrected the col_default_checksum_array_ in prepare_index_data_desc + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column checksums", K(ret), K(column_count), KPC(this)); + } + } + } + return ret; +} + +int ObTabletCreateSSTableParam::init_for_ha( + const blocksstable::ObMigrationSSTableParam &sstable_param, + const blocksstable::ObSSTableMergeRes &res) +{ + int ret = OB_SUCCESS; + + table_key_ = sstable_param.table_key_; + sstable_logic_seq_ = sstable_param.basic_meta_.sstable_logic_seq_; + schema_version_ = sstable_param.basic_meta_.schema_version_; + table_mode_ = sstable_param.basic_meta_.table_mode_; + index_type_ = static_cast(sstable_param.basic_meta_.index_type_); + create_snapshot_version_ = sstable_param.basic_meta_.create_snapshot_version_; + progressive_merge_round_ = sstable_param.basic_meta_.progressive_merge_round_; + progressive_merge_step_ = sstable_param.basic_meta_.progressive_merge_step_; + latest_row_store_type_ = sstable_param.basic_meta_.latest_row_store_type_; + column_cnt_ = res.data_column_cnt_; + nested_size_ = res.nested_size_; + nested_offset_ = res.nested_offset_; + max_merged_trans_version_ = res.max_merged_trans_version_; + rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_; + ddl_scn_ = sstable_param.basic_meta_.ddl_scn_; + if (OB_FAIL(inner_init_with_merge_res(res))) { + LOG_WARN("fail to inner init with merge res", K(ret), K(res)); + } else if (OB_FAIL(column_checksums_.assign(sstable_param.column_checksums_))) { + LOG_WARN("fail to fill column checksum", K(ret), K(sstable_param)); + } + + return ret; +} + +int ObTabletCreateSSTableParam::init_for_ha(const blocksstable::ObMigrationSSTableParam &sstable_param) +{ + int ret = OB_SUCCESS; + table_key_ = sstable_param.table_key_; + sstable_logic_seq_ = sstable_param.basic_meta_.sstable_logic_seq_; + schema_version_ = sstable_param.basic_meta_.schema_version_; + create_snapshot_version_ = sstable_param.basic_meta_.create_snapshot_version_; + table_mode_ = sstable_param.basic_meta_.table_mode_; + index_type_ = static_cast(sstable_param.basic_meta_.index_type_); + progressive_merge_round_ = sstable_param.basic_meta_.progressive_merge_round_; + progressive_merge_step_ = sstable_param.basic_meta_.progressive_merge_step_; + is_ready_for_read_ = true; + root_row_store_type_ = sstable_param.basic_meta_.root_row_store_type_; + latest_row_store_type_ = sstable_param.basic_meta_.latest_row_store_type_; + index_blocks_cnt_ = sstable_param.basic_meta_.index_macro_block_count_; + data_blocks_cnt_ = sstable_param.basic_meta_.data_macro_block_count_; + micro_block_cnt_ = sstable_param.basic_meta_.data_micro_block_count_; + use_old_macro_block_count_ = sstable_param.basic_meta_.use_old_macro_block_count_; + row_count_ = sstable_param.basic_meta_.row_count_; + column_cnt_ = sstable_param.basic_meta_.column_cnt_; + data_checksum_ = sstable_param.basic_meta_.data_checksum_; + occupy_size_ = sstable_param.basic_meta_.occupy_size_; + original_size_ = sstable_param.basic_meta_.original_size_; + max_merged_trans_version_ = sstable_param.basic_meta_.max_merged_trans_version_; + ddl_scn_ = sstable_param.basic_meta_.ddl_scn_; + filled_tx_scn_ = sstable_param.basic_meta_.filled_tx_scn_; + contain_uncommitted_row_ = sstable_param.basic_meta_.contain_uncommitted_row_; + compressor_type_ = sstable_param.basic_meta_.compressor_type_; + encrypt_id_ = sstable_param.basic_meta_.encrypt_id_; + master_key_id_ = sstable_param.basic_meta_.master_key_id_; + root_block_addr_.set_none_addr(); + data_block_macro_meta_addr_.set_none_addr(); + rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_; + MEMCPY(encrypt_key_, sstable_param.basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); + if (OB_FAIL(column_checksums_.assign(sstable_param.column_checksums_))) { + LOG_WARN("fail to assign column checksums", K(ret), K(sstable_param)); + } + + return ret; +} + } // namespace storage } // namespace oceanbase diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.h b/src/storage/tablet/ob_tablet_create_sstable_param.h index b3950b0d1..452eb4a89 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.h +++ b/src/storage/tablet/ob_tablet_create_sstable_param.h @@ -23,8 +23,23 @@ namespace oceanbase { +namespace blocksstable +{ +struct ObSSTableMergeRes; +struct ObBlockInfo; +class ObSSTableMeta; +class ObMigrationSSTableParam; +class ObSSTable; +class ObSSTableIndexBuilder; +} +namespace compaction +{ +struct ObBasicTabletMergeCtx; +} namespace storage { +struct ObTabletDDLParam; + struct ObTabletCreateSSTableParam final { public: @@ -37,6 +52,34 @@ public: bool is_block_meta_valid(const ObMetaDiskAddr &addr, const blocksstable::ObMicroBlockData &data) const; + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_small_sstable(const blocksstable::ObSSTableMergeRes &res, + const ObITable::TableKey &table_key, + const blocksstable::ObSSTableMeta &sstable_meta, + const blocksstable::ObBlockInfo &block_info); + + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_merge(const compaction::ObBasicTabletMergeCtx &ctx, + const blocksstable::ObSSTableMergeRes &res, + const ObStorageColumnGroupSchema *cg_schema, + const int64_t column_group_idx, + const bool is_main_table); + + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_ddl(blocksstable::ObSSTableIndexBuilder *sstable_index_builder, + const ObTabletDDLParam &ddl_param, + const blocksstable::ObSSTable *first_ddl_sstable, + const ObStorageSchema &storage_schema, + const int64_t macro_block_column_count, + const int64_t create_schema_version_on_tablet); + + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_ha(const blocksstable::ObMigrationSSTableParam &migration_param, + const blocksstable::ObSSTableMergeRes &res); + + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_ha(const blocksstable::ObMigrationSSTableParam &migration_param); + TO_STRING_KV(K_(table_key), K_(sstable_logic_seq), K_(schema_version), @@ -81,6 +124,7 @@ public: KPHEX_(encrypt_key, sizeof(encrypt_key_))); private: static const int64_t DEFAULT_MACRO_BLOCK_CNT = 64; + int inner_init_with_merge_res(const blocksstable::ObSSTableMergeRes &res); public: ObITable::TableKey table_key_; int16_t sstable_logic_seq_; diff --git a/unittest/storage/blocksstable/ob_multi_version_sstable_test.h b/unittest/storage/blocksstable/ob_multi_version_sstable_test.h index aa4af5df8..206537323 100644 --- a/unittest/storage/blocksstable/ob_multi_version_sstable_test.h +++ b/unittest/storage/blocksstable/ob_multi_version_sstable_test.h @@ -501,8 +501,8 @@ void ObMultiVersionSSTableTest::prepare_data_end( ObTabletCreateSSTableParam param; table_key_.table_type_ = table_type; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; + ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_)); + ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_)); param.table_key_ = table_key_; param.schema_version_ = SCHEMA_VERSION; param.create_snapshot_version_ = 0;