/** * Copyright (c) 2021 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: * http://license.coscl.org.cn/MulanPubL-2.0 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PubL v2 for more details. */ #define USING_LOG_PREFIX STORAGE_COMPACTION #include "storage/ddl/ob_ddl_merge_task.h" #include "logservice/palf/scn.h" #include "share/ob_ddl_checksum.h" #include "share/ob_get_compat_mode.h" #include "share/schema/ob_table_schema.h" #include "share/schema/ob_multi_version_schema_service.h" #include "storage/blocksstable/ob_index_block_builder.h" #include "storage/blocksstable/ob_sstable_sec_meta_iterator.h" #include "storage/ddl/ob_tablet_ddl_kv_mgr.h" #include "storage/ls/ob_ls.h" #include "storage/meta_mem/ob_tablet_handle.h" #include "storage/tablet/ob_tablet_create_delete_helper.h" #include "storage/tablet/ob_tablet_create_sstable_param.h" #include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tx_storage/ob_ls_handle.h" #include "share/schema/ob_multi_version_schema_service.h" using namespace oceanbase::observer; using namespace oceanbase::share::schema; using namespace oceanbase::share; using namespace oceanbase::common; using namespace oceanbase::blocksstable; namespace oceanbase { namespace storage { /****************** ObDDLTableMergeDag *****************/ ObDDLTableMergeDag::ObDDLTableMergeDag() : ObIDag(ObDagType::DAG_TYPE_DDL_KV_MERGE), is_inited_(false), ddl_param_(), compat_mode_(lib::Worker::CompatMode::INVALID) { } ObDDLTableMergeDag::~ObDDLTableMergeDag() { } int ObDDLTableMergeDag::init_by_param(const share::ObIDagInitParam *param) { int ret = OB_SUCCESS; ObLSHandle ls_handle; ObLS *ls = nullptr; ObTablet *tablet = nullptr; ObTabletHandle tablet_handle; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K(ddl_param_)); } else if (OB_ISNULL(param) || !param->is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), KP(param)); } else { ddl_param_ = *static_cast(param); if (OB_FAIL(MTL(ObLSService *)->get_ls(ddl_param_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("failed to get log stream", K(ret), K(ddl_param_)); } else if (OB_ISNULL(ls = ls_handle.get_ls())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls is unexpected null", K(ret)); } else if (OB_FAIL(ls->get_tablet_svr()->get_tablet(ddl_param_.tablet_id_, tablet_handle, ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { LOG_WARN("failed to get tablet", K(ret), K(ddl_param_)); } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet is null", K(ret), K(ddl_param_)); } else { const ObStorageSchema &storage_schema = tablet->get_storage_schema(); compat_mode_ = static_cast(storage_schema.compat_mode_); is_inited_ = true; } } return ret; } int ObDDLTableMergeDag::create_first_task() { int ret = OB_SUCCESS; ObLSService *ls_service = MTL(ObLSService *); ObLSHandle ls_handle; ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; ObDDLKVsHandle ddl_kvs_handle; ObDDLTableMergeTask *merge_task = nullptr; if (OB_FAIL(ls_service->get_ls(ddl_param_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("get ls failed", K(ret), K(ddl_param_)); } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(ddl_param_.tablet_id_, tablet_handle, ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { LOG_WARN("get tablet failed", K(ret), K(ddl_param_)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { LOG_WARN("get ddl kv mgr failed", K(ret), K(ddl_param_)); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->freeze_ddl_kv())) { LOG_WARN("ddl kv manager try freeze failed", K(ret), K(ddl_param_)); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_ddl_kvs(true/*frozen_only*/, ddl_kvs_handle))) { LOG_WARN("get freezed ddl kv failed", K(ret), K(ddl_param_)); } else if (OB_FAIL(alloc_task(merge_task))) { LOG_WARN("Fail to alloc task", K(ret), K(ddl_param_)); } else if (OB_FAIL(merge_task->init(ddl_param_))) { LOG_WARN("failed to init ddl table merge task", K(ret), K(*this)); } else if (OB_FAIL(add_task(*merge_task))) { LOG_WARN("Fail to add task", K(ret), K(ddl_param_)); } else { // use chain task to ensure log ts continuious in table store ObDDLTableDumpTask *last_task = nullptr; for (int64_t i = 0; OB_SUCC(ret) && i < ddl_kvs_handle.get_count(); ++i) { ObDDLKV *ddl_kv = nullptr; ObDDLTableDumpTask *task = nullptr; if (OB_FAIL(ddl_kvs_handle.get_ddl_kv(i, ddl_kv))) { LOG_WARN("get ddl kv failed", K(ret), K(i)); } else if (OB_FAIL(alloc_task(task))) { LOG_WARN("Fail to alloc task", K(ret)); } else if (OB_FAIL(task->init(ddl_param_.ls_id_, ddl_param_.tablet_id_, ddl_kv->get_freeze_scn()))) { LOG_WARN("failed to init ddl dump task", K(ret), K(ddl_param_), K(ddl_kv->get_freeze_scn())); } else if (OB_FAIL(add_task(*task))) { LOG_WARN("Fail to add task", K(ret), K(ddl_param_)); } else { if (nullptr != last_task && OB_FAIL(last_task->add_child(*task))) { LOG_WARN("add child task failed", K(ret), K(ddl_param_)); } last_task = task; } } if (OB_SUCC(ret)) { if (nullptr != last_task && OB_FAIL(last_task->add_child(*merge_task))) { LOG_WARN("add child merge task failed", K(ret), K(ddl_param_)); } } } return ret; } bool ObDDLTableMergeDag::operator == (const ObIDag &other) const { bool is_same = true; if (this == &other) { } else if (get_type() != other.get_type()) { is_same = false; } else { const ObDDLTableMergeDag &other_dag = static_cast (other); // each tablet has max 1 dag in running, so that the compaction task is unique and no need to consider concurrency is_same = ddl_param_.tablet_id_ == other_dag.ddl_param_.tablet_id_ && ddl_param_.ls_id_ == other_dag.ddl_param_.ls_id_; } return is_same; } int64_t ObDDLTableMergeDag::hash() const { return ddl_param_.tablet_id_.hash(); } int ObDDLTableMergeDag::fill_comment(char *buf, const int64_t buf_len) const { int ret = OB_SUCCESS; if (OB_FAIL(databuff_printf(buf, buf_len, "ddl table merge task, logstream_id=%ld tablet_id=%ld rec_scn=%lu", ddl_param_.ls_id_.id(), ddl_param_.tablet_id_.id(), ddl_param_.rec_scn_.get_val_for_inner_table_field()))) { LOG_WARN("fill comment for ddl table merge dag failed", K(ret), K(ddl_param_)); } return ret; } int ObDDLTableMergeDag::fill_dag_key(char *buf, const int64_t buf_len) const { int ret = OB_SUCCESS; if (OB_FAIL(databuff_printf(buf, buf_len, "ddl table merge task: logstream_id=%ld tablet_id=%ld rec_scn=%lu", ddl_param_.ls_id_.id(), ddl_param_.tablet_id_.id(), ddl_param_.rec_scn_.get_val_for_inner_table_field()))) { LOG_WARN("fill dag key for ddl table merge dag failed", K(ret), K(ddl_param_)); } return ret; } /****************** ObDDLTableDumpTask *****************/ ObDDLTableDumpTask::ObDDLTableDumpTask() : ObITask(ObITaskType::TASK_TYPE_DDL_KV_DUMP), is_inited_(false), ls_id_(), tablet_id_(), freeze_scn_() { } ObDDLTableDumpTask::~ObDDLTableDumpTask() { } int ObDDLTableDumpTask::init(const share::ObLSID &ls_id, const ObTabletID &tablet_id, const palf::SCN &freeze_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K(tablet_id_)); } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || !freeze_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(freeze_scn)); } else { ls_id_ = ls_id; tablet_id_ = tablet_id; freeze_scn_ = freeze_scn; is_inited_ = true; } return ret; } int ObDDLTableDumpTask::process() { int ret = OB_SUCCESS; LOG_INFO("ddl dump task start process", K(*this)); ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; ObLSHandle ls_handle; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); } else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id_, tablet_handle, ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { LOG_WARN("failed to get tablet", K(ret), K(tablet_id_)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { LOG_WARN("get ddl kv mgr failed", K(ret)); } else { ObDDLKVHandle ddl_kv_handle; ObDDLKV *ddl_kv = nullptr; ObTablesHandleArray ddl_sstable_handles; bool need_compact = false; ObArray freeze_log_ts_array; ObArray candidate_sstables; if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_freezed_ddl_kv(freeze_scn_, ddl_kv_handle))) { LOG_WARN("get ddl kv handle failed", K(ret), K(freeze_scn_)); } else if (OB_FAIL(ddl_kv_handle.get_ddl_kv(ddl_kv))) { LOG_WARN("get ddl kv failed", K(ret)); } else if (OB_FAIL(ddl_kv->close())) { LOG_WARN("close ddl kv failed", K(ret)); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->release_ddl_kvs(freeze_scn_))) { LOG_WARN("release ddl kv failed", K(ret), K(freeze_scn_)); } } return ret; } ObDDLTableMergeTask::ObDDLTableMergeTask() : ObITask(ObITaskType::TASK_TYPE_DDL_KV_MERGE), is_inited_(false), merge_param_() { } ObDDLTableMergeTask::~ObDDLTableMergeTask() { } int ObDDLTableMergeTask::init(const ObDDLTableMergeDagParam &ddl_dag_param) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret), K(merge_param_)); } else if (OB_UNLIKELY(!ddl_dag_param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ddl_dag_param)); } else { merge_param_ = ddl_dag_param; is_inited_ = true; } return ret; } int ObDDLTableMergeTask::process() { int ret = OB_SUCCESS; const int64_t MAX_DDL_SSTABLE = 128; LOG_INFO("ddl merge task start process", K(*this)); ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; ObLSHandle ls_handle; ObTablesHandleArray ddl_sstable_handles; const uint64_t tenant_id = MTL_ID(); ObSSTable *sstable = nullptr; bool skip_major_process = false; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); } else if (OB_FAIL(MTL(ObLSService *)->get_ls(merge_param_.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("failed to get log stream", K(ret), K(merge_param_)); } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(merge_param_.tablet_id_, tablet_handle, ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { LOG_WARN("failed to get tablet", K(ret), K(merge_param_)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { LOG_WARN("get ddl kv mgr failed", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_sstable_handles(ddl_sstable_handles))) { LOG_WARN("get ddl sstable handles failed", K(ret)); } else if (ddl_sstable_handles.get_count() >= MAX_DDL_SSTABLE || merge_param_.is_commit_) { DEBUG_SYNC(BEFORE_DDL_TABLE_MERGE_TASK); ObTabletDDLParam ddl_param; ObTableHandleV2 table_handle; bool is_data_complete = false; bool is_major_sstable_exist = false; if (OB_FAIL(ObTabletDDLUtil::check_if_major_sstable_exist(merge_param_.ls_id_, merge_param_.tablet_id_, is_major_sstable_exist))) { LOG_WARN("check if major sstable exist failed", K(ret)); } else if (is_major_sstable_exist) { LOG_INFO("major sstable has been created before", K(merge_param_), K(ddl_param.table_key_)); // TODO(cangdi): after compacting ddl sstable, a major compaction occurs, which may lead to checksum error sstable = static_cast(tablet_handle.get_obj()->get_table_store().get_major_sstables().get_boundary_table(true/*last*/)); } else if (tablet_handle.get_obj()->get_tablet_meta().table_store_flag_.with_major_sstable()) { skip_major_process = true; LOG_INFO("tablet me says with major but no major, meaning its a migrated deleted tablet, skip"); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_ddl_param(ddl_param))) { LOG_WARN("get tablet ddl param failed", K(ret)); } else if (merge_param_.is_commit_ && OB_FAIL(check_data_integrity(ddl_sstable_handles, ddl_param.start_scn_, merge_param_.rec_scn_, is_data_complete))) { LOG_WARN("check ddl sstable integrity failed", K(ret), K(ddl_sstable_handles), K(ddl_param), K(merge_param_)); } else if (merge_param_.is_commit_ && !is_data_complete) { ret = OB_NOT_ENOUGH_STORE; LOG_WARN("current ddl sstables not contain all data", K(ddl_sstable_handles), K(ddl_param), K(merge_param_)); } else if (FALSE_IT(ddl_param.table_key_.table_type_ = merge_param_.is_commit_ ? ObITable::TableType::MAJOR_SSTABLE : ObITable::TableType::KV_DUMP_SSTABLE)) { } else if (OB_FAIL(ObTabletDDLUtil::compact_ddl_sstable(ddl_sstable_handles.get_tables(), tablet_handle.get_obj()->get_index_read_info(), ddl_param, table_handle))) { LOG_WARN("compact sstables failed", K(ret)); } else { sstable = static_cast(table_handle.get_table()); } if (OB_SUCC(ret) && merge_param_.rec_scn_.is_valid()) { // when the ddl dag is self scheduled when ddl kv is full, the rec_scn is invalid // but no worry, the formmer ddl dump task will also release ddl kvs if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->release_ddl_kvs(merge_param_.rec_scn_))) { LOG_WARN("release ddl kv failed", K(ret)); } } if (OB_SUCC(ret) && merge_param_.is_commit_) { if (skip_major_process) { } else if (OB_ISNULL(sstable)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ddl major sstable is null", K(ret), K(ddl_param)); } else if (merge_param_.table_id_ > 0 && merge_param_.execution_id_ > 0 && OB_FAIL(ObTabletDDLUtil::report_ddl_checksum(merge_param_.ls_id_, merge_param_.tablet_id_, merge_param_.table_id_, merge_param_.execution_id_, merge_param_.ddl_task_id_, sstable->get_meta().get_col_checksum()))) { LOG_WARN("report ddl column checksum failed", K(ret), K(merge_param_)); } else if (OB_FAIL(GCTX.ob_service_->submit_tablet_checksums_task(tenant_id, merge_param_.ls_id_, merge_param_.tablet_id_))) { LOG_WARN("fail to submit tablet checksums task", K(ret), K(tenant_id), K(merge_param_)); } else if (OB_FAIL(GCTX.ob_service_->submit_tablet_update_task(tenant_id, merge_param_.ls_id_, merge_param_.tablet_id_))) { LOG_WARN("fail to submit tablet update task", K(ret), K(tenant_id), K(merge_param_)); } if (OB_FAIL(ret)) { } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->set_commit_success())) { LOG_WARN("set is commit success failed", K(ret)); } else { LOG_INFO("commit ddl sstable succ", K(ddl_param), K(merge_param_)); } } } return ret; } int ObDDLTableMergeTask::check_data_integrity(const ObTablesHandleArray &ddl_sstables, const palf::SCN &start_scn, const palf::SCN &prepare_scn, bool &is_data_complete) { int ret = OB_SUCCESS; is_data_complete = false; if (OB_UNLIKELY(ddl_sstables.empty() || !start_scn.is_valid() || !prepare_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ddl_sstables.get_count()), K(start_scn), K(prepare_scn)); } else { const ObSSTable *first_ddl_sstable = static_cast(ddl_sstables.get_tables().at(0)); const ObSSTable *last_ddl_sstable = static_cast(ddl_sstables.get_tables().at(ddl_sstables.get_count() - 1)); if (first_ddl_sstable->get_start_scn() != start_scn) { LOG_INFO("start log ts not match", K(first_ddl_sstable->get_key()), K(start_scn)); } else if (last_ddl_sstable->get_end_scn() != prepare_scn) { LOG_INFO("prepare log ts not match", K(last_ddl_sstable->get_key()), K(prepare_scn)); } else { is_data_complete = true; palf::SCN last_end_scn = first_ddl_sstable->get_end_scn(); for (int64_t i = 1; OB_SUCC(ret) && i < ddl_sstables.get_count(); ++i) { const ObSSTable *cur_ddl_sstable = static_cast(ddl_sstables.get_tables().at(i)); if (cur_ddl_sstable->get_start_scn() != last_end_scn) { is_data_complete = false; LOG_INFO("ddl sstable not continue", K(i), K(cur_ddl_sstable->get_key()), K(last_end_scn)); break; } else { last_end_scn = cur_ddl_sstable->get_end_scn(); } } } } return ret; } ObTabletDDLParam::ObTabletDDLParam() : tenant_id_(0), ls_id_(), table_key_(), start_scn_(), snapshot_version_(0), cluster_version_(0) { } ObTabletDDLParam::~ObTabletDDLParam() { } bool ObTabletDDLParam::is_valid() const { return tenant_id_ > 0 && tenant_id_ != OB_INVALID_ID && ls_id_.is_valid() && table_key_.is_valid() && start_scn_.is_valid() && snapshot_version_ > 0 && cluster_version_ >= 0; } int ObTabletDDLUtil::prepare_index_data_desc(const share::ObLSID &ls_id, const ObTabletID &tablet_id, const int64_t snapshot_version, const int64_t cluster_version, ObDataStoreDesc &data_desc) { int ret = OB_SUCCESS; data_desc.reset(); ObLSService *ls_service = MTL(ObLSService *); ObLSHandle ls_handle; ObTabletHandle tablet_handle; if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || snapshot_version <= 0 || cluster_version < 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(cluster_version)); } else if (OB_ISNULL(ls_service)) { ret = OB_ERR_SYS; LOG_WARN("ls service is null", K(ret), K(ls_id)); } else if (OB_FAIL(ls_service->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("get ls failed", K(ret), K(ls_id)); } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle, ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { LOG_WARN("get tablet failed", K(ret)); } else if (OB_FAIL(data_desc.init(tablet_handle.get_obj()->get_storage_schema(), ls_id, tablet_id, MAJOR_MERGE, snapshot_version, cluster_version))) { LOG_WARN("init data store desc failed", K(ret), K(tablet_id)); } else { data_desc.is_ddl_ = true; data_desc.row_column_count_ = data_desc.rowkey_column_count_ + 1; // prepare col_desc_array for index block and macro meta block data_desc.col_desc_array_.reset(); if (OB_FAIL(data_desc.col_desc_array_.init(data_desc.row_column_count_))) { LOG_WARN("failed to reserve column desc array", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->get_storage_schema().get_rowkey_column_ids(data_desc.col_desc_array_))) { LOG_WARN("failed to get rowkey column ids", K(ret)); } else if (OB_FAIL(storage::ObMultiVersionRowkeyHelpper::add_extra_rowkey_cols(data_desc.col_desc_array_))) { LOG_WARN("failed to add extra rowkey cols", K(ret)); } else { // column desc for the column of index info ObObjMeta meta; meta.set_varchar(); meta.set_collation_type(CS_TYPE_BINARY); share::schema::ObColDesc col; col.col_id_ = static_cast(data_desc.row_column_count_ + OB_APP_MIN_COLUMN_ID); col.col_type_ = meta; col.col_order_ = DESC; if (OB_FAIL(data_desc.col_desc_array_.push_back(col))) { LOG_WARN("failed to push back last col for index", K(ret), K(col)); } } } return ret; } int ObTabletDDLUtil::prepare_index_builder(const ObTabletDDLParam &ddl_param, ObIAllocator &allocator, ObSSTableIndexBuilder *&sstable_index_builder, ObIndexBlockRebuilder *&index_block_rebuilder) { int ret = OB_SUCCESS; sstable_index_builder = nullptr; index_block_rebuilder = nullptr; ObDataStoreDesc data_desc; void *buf = nullptr; if (OB_FAIL(prepare_index_data_desc(ddl_param.ls_id_, ddl_param.table_key_.get_tablet_id(), ddl_param.table_key_.version_range_.snapshot_version_, ddl_param.cluster_version_, data_desc))) { LOG_WARN("prepare data desc failed", K(ret), K(ddl_param)); } else if (OB_ISNULL(buf = allocator.alloc(sizeof(ObSSTableIndexBuilder)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret)); } else if (FALSE_IT(sstable_index_builder = new (buf) ObSSTableIndexBuilder)) { } else if (OB_FAIL(sstable_index_builder->init(data_desc))) { LOG_WARN("init sstable index builder failed", K(ret)); } else if (OB_ISNULL(buf = allocator.alloc(sizeof(ObIndexBlockRebuilder)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret)); } else if (FALSE_IT(index_block_rebuilder = new (buf) ObIndexBlockRebuilder)) { } else if (OB_FAIL(index_block_rebuilder->init(*sstable_index_builder))) { LOG_WARN("fail to alloc index builder", K(ret)); } if (OB_FAIL(ret)) { if (nullptr != index_block_rebuilder) { index_block_rebuilder->~ObIndexBlockRebuilder(); allocator.free(index_block_rebuilder); index_block_rebuilder = nullptr; } if (nullptr != sstable_index_builder) { sstable_index_builder->~ObSSTableIndexBuilder(); allocator.free(sstable_index_builder); sstable_index_builder = nullptr; } } return ret; } int ObTabletDDLUtil::create_ddl_sstable(ObSSTableIndexBuilder *sstable_index_builder, const ObTabletDDLParam &ddl_param, ObTableHandleV2 &table_handle) { int ret = OB_SUCCESS; table_handle.reset(); ObLSService *ls_service = MTL(ObLSService *); ObLSHandle ls_handle; ObTabletHandle tablet_handle; SMART_VAR(ObSSTableMergeRes, res) { if (OB_UNLIKELY(nullptr == sstable_index_builder || !ddl_param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KP(sstable_index_builder), K(ddl_param)); } else if (OB_ISNULL(ls_service)) { ret = OB_ERR_SYS; LOG_WARN("ls service is null", K(ret), K(ddl_param)); } else if (OB_FAIL(ls_service->get_ls(ddl_param.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("get ls failed", K(ret), K(ddl_param)); } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(ddl_param.table_key_.tablet_id_, tablet_handle, ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { LOG_WARN("get tablet failed", K(ret), K(ddl_param)); } else { const ObStorageSchema &storage_schema = tablet_handle.get_obj()->get_storage_schema(); int64_t column_count = 0; if (OB_FAIL(storage_schema.get_stored_column_count_in_sstable(column_count))) { LOG_WARN("fail to get stored column count in sstable", K(ret)); } else if (OB_FAIL(sstable_index_builder->close(column_count, res))) { LOG_WARN("close sstable index builder close failed", K(ret)); } else { ObTabletCreateSSTableParam param; param.table_key_ = ddl_param.table_key_; param.table_mode_ = storage_schema.get_table_mode_struct(); param.index_type_ = storage_schema.get_index_type(); param.rowkey_column_cnt_ = storage_schema.get_rowkey_column_num() + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); param.schema_version_ = storage_schema.get_schema_version(); param.create_snapshot_version_ = ddl_param.snapshot_version_; param.ddl_scn_ = ddl_param.start_scn_; ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, param.root_block_addr_, param.root_block_data_); ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, param.data_block_macro_meta_addr_, param.data_block_macro_meta_); param.root_row_store_type_ = res.root_desc_.row_type_; param.data_index_tree_height_ = res.root_desc_.height_; param.index_blocks_cnt_ = res.index_blocks_cnt_; param.data_blocks_cnt_ = res.data_blocks_cnt_; param.micro_block_cnt_ = res.micro_block_cnt_; param.use_old_macro_block_count_ = res.use_old_macro_block_count_; param.row_count_ = res.row_count_; param.column_cnt_ = column_count; param.data_checksum_ = res.data_checksum_; param.occupy_size_ = res.occupy_size_; param.original_size_ = res.original_size_; param.max_merged_trans_version_ = res.max_merged_trans_version_; param.contain_uncommitted_row_ = res.contain_uncommitted_row_; param.compressor_type_ = res.compressor_type_; param.encrypt_id_ = res.encrypt_id_; param.master_key_id_ = res.master_key_id_; param.data_block_ids_ = res.data_block_ids_; param.other_block_ids_ = res.other_block_ids_; MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); if (OB_FAIL(res.fill_column_checksum(&storage_schema, param.column_checksums_))) { LOG_WARN("fail to fill column checksum for empty major", K(ret), K(param)); } else if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, table_handle))) { LOG_WARN("create sstable failed", K(ret), K(param)); } else { const int64_t rebuild_seq = ls_handle.get_ls()->get_rebuild_seq(); ObTabletHandle new_tablet_handle; ObUpdateTableStoreParam param(table_handle, tablet_handle.get_obj()->get_snapshot_version(), ddl_param.table_key_.is_major_sstable() ? false: true, // keep_old_ddl_sstable &storage_schema, rebuild_seq, ddl_param.table_key_.is_major_sstable() ? true : false, // update_with_major_flag ddl_param.table_key_.is_major_sstable() ? true : false); // need report checksum if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(ddl_param.table_key_.get_tablet_id(), param, new_tablet_handle))) { LOG_WARN("failed to update tablet table store", K(ret), K(ddl_param.table_key_), K(param)); } else { LOG_INFO("create ddl sstable success", K(ddl_param), K(param)); } } } } } return ret; } int ObTabletDDLUtil::compact_ddl_sstable(const ObIArray &ddl_sstables, const ObTableReadInfo &read_info, const ObTabletDDLParam &ddl_param, ObTableHandleV2 &table_handle) { int ret = OB_SUCCESS; table_handle.reset(); ObArenaAllocator arena; ObSSTableIndexBuilder *sstable_index_builder = nullptr; ObIndexBlockRebuilder *index_block_rebuilder = nullptr; if (OB_UNLIKELY(!ddl_param.is_valid() || ddl_sstables.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ddl_param), K(ddl_sstables.count())); } else if (OB_FAIL(prepare_index_builder(ddl_param, arena, sstable_index_builder, index_block_rebuilder))) { LOG_WARN("prepare sstable index builder failed", K(ret)); } else { // campact SMART_VAR(ObSSTableSecMetaIterator, meta_iter) { ObDatumRange query_range; query_range.set_whole_range(); ObDataMacroBlockMeta data_macro_meta; for (int64_t i = 0; OB_SUCC(ret) && i < ddl_sstables.count(); ++i) { const ObSSTable *cur_sstable = static_cast(ddl_sstables.at(i)); if (OB_FAIL(meta_iter.open(query_range, ObMacroBlockMetaType::DATA_BLOCK_META, *cur_sstable, read_info, arena))) { LOG_WARN("sstable secondary meta iterator open failed", K(ret)); } else { while (OB_SUCC(ret)) { if (OB_FAIL(meta_iter.get_next(data_macro_meta))) { if (OB_ITER_END != ret) { LOG_WARN("get data macro meta failed", K(ret)); } } else if (OB_FAIL(index_block_rebuilder->append_macro_row(data_macro_meta))) { LOG_WARN("append macro row failed", K(ret)); } } if (OB_ITER_END == ret) { ret = OB_SUCCESS; } #ifdef ERRSIM if (OB_SUCC(ret) && ddl_param.table_key_.is_major_sstable()) { ret = E(EventTable::EN_DDL_COMPACT_FAIL) OB_SUCCESS; if (OB_FAIL(ret)) { LOG_WARN("errsim compact ddl sstable failed", KR(ret)); } } #endif } if (OB_SUCC(ret)) { meta_iter.reset(); } } } } // close if (OB_SUCC(ret)) { if (OB_FAIL(index_block_rebuilder->close())) { LOG_WARN("index block rebuilder close failed", K(ret)); } else if (OB_FAIL(create_ddl_sstable(sstable_index_builder, ddl_param, table_handle))) { LOG_WARN("create ddl sstable failed", K(ret)); } else { LOG_INFO("compact ddl sstable success", K(ddl_param)); } } if (nullptr != index_block_rebuilder) { index_block_rebuilder->~ObIndexBlockRebuilder(); arena.free(index_block_rebuilder); index_block_rebuilder = nullptr; } if (nullptr != sstable_index_builder) { sstable_index_builder->~ObSSTableIndexBuilder(); arena.free(sstable_index_builder); sstable_index_builder = nullptr; } return ret; } int ObTabletDDLUtil::report_ddl_checksum(const share::ObLSID &ls_id, const ObTabletID &tablet_id, const uint64_t table_id, const int64_t execution_id, const int64_t ddl_task_id, const ObIArray &column_checksums) { int ret = OB_SUCCESS; ObMySQLProxy *sql_proxy = GCTX.sql_proxy_; ObMultiVersionSchemaService *schema_service = GCTX.schema_service_; ObSchemaGetterGuard schema_guard; const ObTableSchema *table_schema = nullptr; const uint64_t tenant_id = MTL_ID(); if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || OB_INVALID_ID == ddl_task_id || !is_valid_id(table_id) || 0 == table_id || execution_id <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(table_id), K(execution_id)); } else if (!is_valid_tenant_id(tenant_id) || OB_ISNULL(sql_proxy) || OB_ISNULL(schema_service)) { ret = OB_ERR_SYS; LOG_WARN("ls service or sql proxy is null", K(ret), K(tenant_id), KP(sql_proxy), KP(schema_service)); } else if (OB_FAIL(schema_service->get_tenant_schema_guard(tenant_id, schema_guard))) { LOG_WARN("get tenant schema guard failed", K(ret), K(tenant_id)); } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table_schema))) { LOG_WARN("get table schema failed", K(ret), K(tenant_id), K(table_id)); } else if (OB_ISNULL(table_schema)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid table schema", K(ret), K(tenant_id), K(table_id)); } else { ObArray column_ids; ObArray ddl_checksum_items; if (OB_FAIL(table_schema->get_multi_version_column_descs(column_ids))) { LOG_WARN("fail to get column ids", K(ret), K(ls_id), K(tablet_id)); } else if (OB_UNLIKELY(column_checksums.count() != column_ids.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect error, column checksums count didn't equal to column ids count", K(ret), K(ls_id), K(tablet_id), K(column_checksums.count()), K(column_ids.count())); } for (int64_t i = 0; OB_SUCC(ret) && i < column_checksums.count(); ++i) { share::ObDDLChecksumItem item; item.execution_id_ = execution_id; item.tenant_id_ = tenant_id; item.table_id_ = table_id; item.ddl_task_id_ = ddl_task_id; item.column_id_ = column_ids.at(i).col_id_; item.task_id_ = tablet_id.id(); item.checksum_ = column_checksums.at(i); #ifdef ERRSIM if (OB_SUCC(ret)) { ret = E(EventTable::EN_HIDDEN_CHECKSUM_DDL_TASK) OB_SUCCESS; // set the checksum of the second column inconsistent with the report checksum of data table. (report_ddl_column_checksum()) if (OB_FAIL(ret) && 17 == item.column_id_) { item.checksum_ = i + 100; } } #endif if (item.column_id_ >= OB_MIN_SHADOW_COLUMN_ID || item.column_id_ == OB_HIDDEN_TRANS_VERSION_COLUMN_ID || item.column_id_ == OB_HIDDEN_SQL_SEQUENCE_COLUMN_ID) { continue; } else if (OB_FAIL(ddl_checksum_items.push_back(item))) { LOG_WARN("push back column checksum item failed", K(ret)); } } #ifdef ERRSIM if (OB_SUCC(ret)) { ret = E(EventTable::EN_DDL_REPORT_CHECKSUM_FAIL) OB_SUCCESS; if (OB_FAIL(ret)) { LOG_WARN("errsim report checksum failed", KR(ret)); } } #endif if (OB_FAIL(ret)) { } else if (OB_FAIL(ObDDLChecksumOperator::update_checksum(ddl_checksum_items, *sql_proxy))) { LOG_WARN("fail to update checksum", K(ret), K(ls_id), K(tablet_id), K(table_id), K(ddl_checksum_items)); } else { LOG_INFO("report ddl checkum success", K(ls_id), K(tablet_id), K(table_id), K(execution_id)); } } return ret; } int ObTabletDDLUtil::check_if_major_sstable_exist(const share::ObLSID &ls_id, const ObTabletID &tablet_id, bool &is_major_sstable_exist) { int ret = OB_SUCCESS; ObLSHandle ls_handle; ObTabletHandle tablet_handle; ObSSTable *latest_major_sstable = nullptr; if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id)); } else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("failed to get log stream", K(ret), K(ls_id)); } else if (OB_FAIL(ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle, ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { LOG_WARN("get tablet handle failed", K(ret), K(ls_id), K(tablet_id)); } else if (OB_UNLIKELY(nullptr == tablet_handle.get_obj())) { ret = OB_ERR_SYS; LOG_WARN("tablet handle is null", K(ret), K(ls_id), K(tablet_id)); } else { latest_major_sstable = static_cast( tablet_handle.get_obj()->get_table_store().get_major_sstables().get_boundary_table(true/*last*/)); is_major_sstable_exist = nullptr != latest_major_sstable; } return ret; } } // namespace storage } // namespace oceanbase