From b3984bd166f73f9f75064624e632eb7aa59a5df7 Mon Sep 17 00:00:00 2001 From: xianyu-w <707512433@qq.com> Date: Thu, 5 Dec 2024 08:14:44 +0000 Subject: [PATCH] Fix the bug of slow partition information retrieval --- src/objit/src/ob_llvm_di_helper.cpp | 18 ++-- src/share/schema/ob_table_schema.cpp | 156 +++++++++++++++++++++++++++ src/share/schema/ob_table_schema.h | 10 ++ src/sql/optimizer/ob_join_order.cpp | 109 ++++++++++++------- src/sql/optimizer/ob_join_order.h | 7 +- 5 files changed, 254 insertions(+), 46 deletions(-) diff --git a/src/objit/src/ob_llvm_di_helper.cpp b/src/objit/src/ob_llvm_di_helper.cpp index f17b4c502..f87def7e4 100644 --- a/src/objit/src/ob_llvm_di_helper.cpp +++ b/src/objit/src/ob_llvm_di_helper.cpp @@ -37,9 +37,9 @@ uint64_t ObLLVMDIType::get_size_bits() } uint64_t ObLLVMDIType::get_align_bits() -{ - return OB_ISNULL(v_) ? 0 : v_->getAlignInBits(); -} +{ + return OB_ISNULL(v_) ? 0 : v_->getAlignInBits(); +} int ObLLVMDIHelper::init(core::JitContext *jc) { @@ -158,9 +158,9 @@ int ObLLVMDIHelper::create_local_variable(const ObString &name, uint32_t arg_no, || OB_ISNULL(sp = jc_->sp_) || OB_ISNULL(type.get_v())) { ret = OB_NOT_INIT; LOG_WARN("jc or file or sp or type is NULL", K(name), K(ret)); - } else { - var_ptr = (arg_no > 0) ? - jc_->dbuilder_.createParameterVariable(sp, StringRef(name.ptr(), name.length()), + } else { + var_ptr = (arg_no > 0) ? + jc_->dbuilder_.createParameterVariable(sp, StringRef(name.ptr(), name.length()), arg_no, file, line, type.get_v(), true) : jc_->dbuilder_.createAutoVariable(sp, StringRef(name.ptr(), name.length()), file, line, type.get_v(), true); @@ -224,9 +224,9 @@ int ObLLVMDIHelper::create_pointer_type(ObLLVMDIType &pointee_type, if (OB_ISNULL(jc_)) { ret = OB_NOT_INIT; LOG_WARN("jc is NULL", K(ret)); - } else if (OB_ISNULL(ptr_type = jc_->dbuilder_.createPointerType(pte_type, 64, 64))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to create pointer type", K(ret)); + } else if (OB_ISNULL(ptr_type = jc_->dbuilder_.createPointerType(pte_type, 64, 64))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to create pointer type", K(ret)); } else { pointer_type.set_v(ptr_type); } diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index 8bfa8374e..b9992d84b 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -1302,6 +1302,162 @@ int ObSimpleTableSchemaV2::get_part_id_by_tablet(const ObTabletID &tablet_id, in return ret; } +int ObSimpleTableSchemaV2::get_all_first_level_part_ids(ObIArray &first_level_part_ids) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(PARTITION_LEVEL_TWO != get_part_level()) || + OB_ISNULL(partition_array_) || partition_num_ <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("part_array is null or is empty", + K(ret), KP_(partition_array), K_(partition_num), K(get_part_level())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < partition_num_; i++) { + if (OB_ISNULL(partition_array_[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("partition is null", K(ret)); + } else if (OB_FAIL(first_level_part_ids.push_back(partition_array_[i]->get_part_id()))) { + LOG_WARN("failed to push back", K(ret)); + } + } + return ret; +} + +int ObSimpleTableSchemaV2::get_part_ids_by_subpart_ids(const ObIArray &subpart_ids, + ObIArray &part_ids, + int64_t &subpart_cnt_in_parts) const +{ + int ret = OB_SUCCESS; + subpart_cnt_in_parts = 0; + ObHashSet subpart_hashset; + ObPartition **part_array = NULL; + int64_t part_num = get_partition_num(); + part_ids.reuse(); + if (OB_UNLIKELY(PARTITION_LEVEL_TWO != part_level_) || + OB_ISNULL(part_array = get_part_array())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid part", KR(ret), KPC(this)); + } else if (OB_FAIL(subpart_hashset.create(get_all_part_num()))) { + LOG_WARN("create hashset failed", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < subpart_ids.count(); i ++) { + if (OB_FAIL(subpart_hashset.set_refactored(subpart_ids.at(i)))) { + LOG_WARN("failed to set refactored", K(ret)); + } + } + for (int64_t i = 0; i < part_num && OB_SUCC(ret); ++i) { + if (OB_ISNULL(part_array[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("NULL ptr", K(i), KPC(this), KR(ret)); + } else { + ObSubPartition **subpart_array = part_array[i]->get_subpart_array(); + int64_t subpart_num = part_array[i]->get_subpartition_num(); + if (OB_ISNULL(subpart_array)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("subpart array is null", KPC(this), KR(ret)); + } else { + bool found = false; + for (int64_t j = 0; j < subpart_num && !found && OB_SUCC(ret); ++j) { + if (OB_ISNULL(subpart_array[j])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("NULL ptr", KPC(this), KR(ret)); + } else { + int tmp_ret = subpart_hashset.exist_refactored((subpart_array[j]->get_sub_part_id())); + if (OB_HASH_EXIST == tmp_ret) { + if (OB_FAIL(part_ids.push_back(part_array[i]->get_part_id()))) { + LOG_WARN("failed to push back", K(ret)); + } else { + found = true; + subpart_cnt_in_parts += subpart_num; + } + } else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != tmp_ret)) { + ret = tmp_ret; + LOG_WARN("fail to check subpartids exist", K(ret)); + } + } + } + } + } + } + if (subpart_hashset.created()) { + int tmp_ret = subpart_hashset.destroy(); + if (OB_SUCC(ret) && OB_FAIL(tmp_ret)) { + LOG_WARN("failed to destory hashset", K(ret)); + } + } + return ret; +} + +int ObSimpleTableSchemaV2::get_part_idx_by_part_id(const ObIArray &part_ids, + ObIArray &part_idx, + ObIArray &subpart_idx) const +{ + int ret = OB_SUCCESS; + ObHashSet id_hashset; + ObPartition **part_array = NULL; + int64_t part_num = get_partition_num(); + part_idx.reuse(); + subpart_idx.reuse(); + if (OB_ISNULL(part_array = get_part_array()) || + OB_UNLIKELY(PARTITION_LEVEL_ZERO == part_level_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid part", KR(ret), KPC(this)); + } else if (OB_FAIL(id_hashset.create(get_all_part_num()))) { + LOG_WARN("create hashset failed", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < part_ids.count(); i ++) { + if (OB_FAIL(id_hashset.set_refactored(part_ids.at(i)))) { + LOG_WARN("failed to set refactored", K(ret)); + } + } + for (int64_t i = 0; i < part_num && OB_SUCC(ret); ++i) { + if (OB_ISNULL(part_array[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("NULL ptr", K(i), KPC(this), KR(ret)); + } else if (PARTITION_LEVEL_ONE == part_level_) { + int tmp_ret = id_hashset.exist_refactored(part_array[i]->get_part_id()); + if (OB_HASH_EXIST == tmp_ret) { + OZ(part_idx.push_back(i)); + OZ(subpart_idx.push_back(OB_INVALID_ID)); + } else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != tmp_ret)) { + ret = tmp_ret; + LOG_WARN("fail to check part id exist", K(ret)); + } + } else if (PARTITION_LEVEL_TWO == part_level_) { + ObSubPartition **subpart_array = part_array[i]->get_subpart_array(); + int64_t subpart_num = part_array[i]->get_subpartition_num(); + if (OB_ISNULL(subpart_array)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("subpart array is null", KPC(this), KR(ret)); + } + for (int64_t j = 0; j < subpart_num && OB_SUCC(ret); ++j) { + if (OB_ISNULL(subpart_array[j])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("NULL ptr", KPC(this), KR(ret)); + } else { + int tmp_ret = id_hashset.exist_refactored(subpart_array[j]->get_sub_part_id()); + if (OB_HASH_EXIST == tmp_ret) { + OZ(part_idx.push_back(i)); + OZ(subpart_idx.push_back(j)); + } else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != tmp_ret)) { + ret = tmp_ret; + LOG_WARN("fail to check part id exist", K(ret)); + } + } + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("4.0 not support part type", KR(ret), KPC(this)); + } + } + if (id_hashset.created()) { + int tmp_ret = id_hashset.destroy(); + if (OB_SUCC(ret) && OB_FAIL(tmp_ret)) { + LOG_WARN("failed to destory hashset", K(ret)); + } + } + return ret; +} + int ObSimpleTableSchemaV2::get_part_id_and_tablet_id_by_idx(const int64_t part_idx, const int64_t subpart_idx, ObObjectID &object_id, diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 3bd07a489..fbc299766 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -895,6 +895,16 @@ public: const ObTabletID &tablet_id, int64_t &part_id, int64_t &subpart_id) const; + int get_all_first_level_part_ids( + ObIArray &first_level_part_ids) const; + int get_part_ids_by_subpart_ids( + const ObIArray &subpart_ids, + ObIArray &part_ids, + int64_t &subpart_cnt_in_parts) const; + int get_part_idx_by_part_id( + const ObIArray &part_ids, + ObIArray &part_idx, + ObIArray &subpart_idx) const; int get_hidden_part_id_by_tablet_id( const ObTabletID &tablet_id, int64_t &part_id) const; diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 32c8d3e4b..57f0c0d9f 100755 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -14259,7 +14259,7 @@ int ObJoinOrder::create_and_add_nl_path(const Path *left_path, int ObJoinOrder::get_used_stat_partitions(const uint64_t ref_table_id, const share::schema::ObTableSchema &schema, - const ObIArray &all_used_part_ids, + ObIArray &all_used_part_ids, ObIArray &table_stat_part_ids, double &table_stat_scale_ratio, ObIArray *hist_stat_part_ids/* = NULL*/) @@ -14275,6 +14275,7 @@ int ObJoinOrder::get_used_stat_partitions(const uint64_t ref_table_id, ObArray table_id; ObArray part_ids; ObArray subpart_ids; + ObArray new_used_part_ids; ObSchemaGetterGuard *schema_guard = NULL; common::ObOptStatManager *opt_stat_manager = NULL; int64_t subpart_cnt_in_parts = 0; @@ -14345,6 +14346,9 @@ int ObJoinOrder::get_used_stat_partitions(const uint64_t ref_table_id, LOG_WARN("unexpected null", K(stat_parts)); } else if (!stat_condition[i] || stat_parts[i]->empty()) { // do nothing + } else if (new_used_part_ids.empty() && scale_ratios[i] == 1.0 && + OB_FAIL(new_used_part_ids.assign(*stat_parts[i]))) { + LOG_WARN("failed to assign", K(ret)); } else if (!table_stat_part_ids.empty() && NULL != hist_stat_part_ids && partition_limit < stat_parts[i]->count()) { // partitions are too many to use hist, do nothing @@ -14373,6 +14377,10 @@ int ObJoinOrder::get_used_stat_partitions(const uint64_t ref_table_id, (NULL == hist_stat_part_ids || !hist_stat_part_ids->empty()); } } + if (OB_SUCC(ret) && !new_used_part_ids.empty() && + OB_FAIL(all_used_part_ids.assign(new_used_part_ids))) { + LOG_WARN("failed to assign", K(ret)); + } } LOG_TRACE("get table statistics partition infos", @@ -14395,7 +14403,7 @@ int ObJoinOrder::get_partition_infos(const uint64_t ref_table_id, part_ids.reuse(); subpart_ids.reuse(); subpart_cnt_in_parts = 0; - if (FAILEDx(table_id.push_back(schema.is_partitioned_table() ? -1 : ref_table_id))) { + if (OB_FAIL(table_id.push_back(schema.is_partitioned_table() ? -1 : ref_table_id))) { LOG_WARN("failed to push back", K(ret)); } else if (!schema.is_partitioned_table()) { // do nothing @@ -14406,24 +14414,18 @@ int ObJoinOrder::get_partition_infos(const uint64_t ref_table_id, } else if (PARTITION_LEVEL_TWO == schema.get_part_level()) { if (OB_FAIL(subpart_ids.assign(all_used_part_id))) { LOG_WARN("failed to assign", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < all_used_part_id.count(); ++i) { - int64_t part_id = OB_INVALID_ID; - int64_t subpart_id = OB_INVALID_ID; - common::ObTabletID tablet_id; - ObArray subpart_ids_in_part; - if (OB_FAIL(schema.get_tablet_id_by_object_id(all_used_part_id.at(i), tablet_id))) { - LOG_WARN("failed to get tablet id", K(ret)); - } else if (OB_FAIL(schema.get_part_id_by_tablet(tablet_id, part_id, subpart_id))) { - LOG_WARN("failed to get part id by tablet", K(ret), K(tablet_id)); - } else if (!ObOptimizerUtil::find_item(part_ids, part_id)) { - if (OB_FAIL(part_ids.push_back(part_id))) { - LOG_WARN("failed to push back part id", K(ret)); - } else if (OB_FAIL(schema.get_subpart_ids(part_id, subpart_ids_in_part))) { - LOG_WARN("failed to get subpart ids", K(ret)); - } else { - subpart_cnt_in_parts += subpart_ids_in_part.count(); - } + } else if (all_used_part_id.count() == schema.get_all_part_num()) { + // full table + if (OB_FAIL(schema.get_all_first_level_part_ids(part_ids))) { + LOG_WARN("failed to get part ids", K(ret)); + } else { + subpart_cnt_in_parts = all_used_part_id.count(); + } + } else { + if (OB_FAIL(schema.get_part_ids_by_subpart_ids(subpart_ids, + part_ids, + subpart_cnt_in_parts))) { + LOG_WARN("failed to get part id", K(ret)); } } } else { @@ -14575,6 +14577,52 @@ int ObJoinOrder::init_est_sel_info_for_access_path(const uint64_t table_id, return ret; } +int ObJoinOrder::get_index_partition_ids(const ObCandiTabletLocIArray &part_loc_info_array, + const share::schema::ObTableSchema &table_schema, + const share::schema::ObTableSchema &index_schema, + ObIArray &index_part_ids) +{ + int ret = OB_SUCCESS; + if (part_loc_info_array.count() == table_schema.get_all_part_num()) { + // full table + ObArray dummy; + ObArray tmp_part_ids; + if (OB_FAIL(index_schema.get_all_tablet_and_object_ids(dummy, tmp_part_ids))) { + LOG_WARN("failed to get part ids", K(ret)); + } else if (OB_FAIL(append(index_part_ids, tmp_part_ids))) { + LOG_WARN("failed to append", K(ret)); + } + } else { + ObArray table_part_ids; + ObArray part_idx; + ObArray subpart_idx; + for (int64_t i = 0; OB_SUCC(ret) && i < part_loc_info_array.count(); ++i) { + const ObOptTabletLoc &part_loc = part_loc_info_array.at(i).get_partition_location(); + if (OB_FAIL(table_part_ids.push_back(part_loc.get_partition_id()))) { + LOG_WARN("failed to push back partition id", K(ret)); + } + } + if (FAILEDx(table_schema.get_part_idx_by_part_id(table_part_ids, part_idx, subpart_idx))) { + LOG_WARN("failed to get part idx", K(ret)); + } else if (OB_UNLIKELY(part_idx.count() != subpart_idx.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected part idx", K(part_idx), K(subpart_idx)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < part_idx.count(); ++i) { + ObObjectID object_id = -1; + ObObjectID dummy1; + ObTabletID dummy2; + if (OB_FAIL(index_schema.get_part_id_and_tablet_id_by_idx( + part_idx.at(i), subpart_idx.at(i), object_id, dummy1, dummy2))) { + LOG_WARN("failed to get part id", K(ret)); + } else if (OB_FAIL(index_part_ids.push_back(static_cast(object_id)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + } + return ret; +} + int ObJoinOrder::init_est_info_for_index(const uint64_t table_id, const uint64_t index_id, ObIndexMetaInfo &index_meta_info, @@ -14611,22 +14659,11 @@ int ObJoinOrder::init_est_info_for_index(const uint64_t table_id, if (OB_FAIL(all_used_part_id.push_back(index_id))) { LOG_WARN("failed to push back partition id", K(ret)); } - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < part_loc_info_array.count(); ++i) { - const ObOptTabletLoc &part_loc = part_loc_info_array.at(i).get_partition_location(); - int64_t part_idx = -1; - int64_t subpart_idx = -1; - ObObjectID object_id = -1; - ObObjectID dummy1; - ObTabletID dummy2; - if (OB_FAIL(table_schema.get_part_idx_by_tablet(part_loc.get_tablet_id(), part_idx, subpart_idx))) { - LOG_WARN("failed to get part idx", K(ret), K(part_loc.get_tablet_id())); - } else if (OB_FAIL(index_schema.get_part_id_and_tablet_id_by_idx(part_idx, subpart_idx, object_id, dummy1, dummy2))) { - LOG_WARN("failed to get part id", K(ret)); - } else if (OB_FAIL(all_used_part_id.push_back(static_cast(object_id)))) { - LOG_WARN("failed to push back", K(ret)); - } - } + } else if (OB_FAIL(get_index_partition_ids(part_loc_info_array, + table_schema, + index_schema, + all_used_part_id))) { + LOG_WARN("failed to get index partition ids", K(ret)); } if (OB_SUCC(ret)) { const int64_t origin_part_cnt = all_used_part_id.count(); diff --git a/src/sql/optimizer/ob_join_order.h b/src/sql/optimizer/ob_join_order.h index db17066a6..bfd4f1dab 100755 --- a/src/sql/optimizer/ob_join_order.h +++ b/src/sql/optimizer/ob_join_order.h @@ -1669,7 +1669,7 @@ struct NullAwareAntiJoinInfo { int get_used_stat_partitions(const uint64_t ref_table_id, const share::schema::ObTableSchema &table_schema, - const ObIArray &all_used_part_ids, + ObIArray &all_used_part_ids, ObIArray &table_stat_part_ids, double &table_stat_scale_ratio, ObIArray *hist_stat_part_ids = NULL); @@ -1682,6 +1682,11 @@ struct NullAwareAntiJoinInfo { ObIArray &subpart_ids, int64_t &subpart_cnt_in_parts); + int get_index_partition_ids(const ObCandiTabletLocIArray &part_loc_info_array, + const share::schema::ObTableSchema &table_schema, + const share::schema::ObTableSchema &index_schema, + ObIArray &index_part_ids); + int init_est_info_for_index(const uint64_t table_id, const uint64_t index_id, ObIndexMetaInfo &meta_info,