hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix the bug of slow partition information retrieval

Browse Source
This commit is contained in:
xianyu-w 2024-12-05 08:14:44 +00:00 committed by ob-robot
parent 7fda92db95
commit b3984bd166
5 changed files with 254 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/objit/src/ob_llvm_di_helper.cpp
View File

@ -37,9 +37,9 @@ uint64_t ObLLVMDIType::get_size_bits()
}
uint64_t ObLLVMDIType::get_align_bits()
{
return OB_ISNULL(v_) ? 0 : v_->getAlignInBits();
}
{
return OB_ISNULL(v_) ? 0 : v_->getAlignInBits();
}
int ObLLVMDIHelper::init(core::JitContext *jc)
{
@ -158,9 +158,9 @@ int ObLLVMDIHelper::create_local_variable(const ObString &name, uint32_t arg_no,
|| OB_ISNULL(sp = jc_->sp_) || OB_ISNULL(type.get_v())) {
ret = OB_NOT_INIT;
LOG_WARN("jc or file or sp or type is NULL", K(name), K(ret));
} else {
var_ptr = (arg_no > 0) ?
jc_->dbuilder_.createParameterVariable(sp, StringRef(name.ptr(), name.length()),
} else {
var_ptr = (arg_no > 0) ?
jc_->dbuilder_.createParameterVariable(sp, StringRef(name.ptr(), name.length()),
arg_no, file, line, type.get_v(), true) :
jc_->dbuilder_.createAutoVariable(sp, StringRef(name.ptr(), name.length()),
file, line, type.get_v(), true);
@ -224,9 +224,9 @@ int ObLLVMDIHelper::create_pointer_type(ObLLVMDIType &pointee_type,
if (OB_ISNULL(jc_)) {
ret = OB_NOT_INIT;
LOG_WARN("jc is NULL", K(ret));
} else if (OB_ISNULL(ptr_type = jc_->dbuilder_.createPointerType(pte_type, 64, 64))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to create pointer type", K(ret));
} else if (OB_ISNULL(ptr_type = jc_->dbuilder_.createPointerType(pte_type, 64, 64))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to create pointer type", K(ret));
} else {
pointer_type.set_v(ptr_type);
}

156
src/share/schema/ob_table_schema.cpp
View File

@ -1302,6 +1302,162 @@ int ObSimpleTableSchemaV2::get_part_id_by_tablet(const ObTabletID &tablet_id, in
return ret;
}
int ObSimpleTableSchemaV2::get_all_first_level_part_ids(ObIArray<int64_t> &first_level_part_ids) const
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(PARTITION_LEVEL_TWO != get_part_level()) ||
OB_ISNULL(partition_array_) || partition_num_ <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("part_array is null or is empty",
K(ret), KP_(partition_array), K_(partition_num), K(get_part_level()));
}
for (int64_t i = 0; OB_SUCC(ret) && i < partition_num_; i++) {
if (OB_ISNULL(partition_array_[i])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("partition is null", K(ret));
} else if (OB_FAIL(first_level_part_ids.push_back(partition_array_[i]->get_part_id()))) {
LOG_WARN("failed to push back", K(ret));
}
}
return ret;
}
int ObSimpleTableSchemaV2::get_part_ids_by_subpart_ids(const ObIArray<int64_t> &subpart_ids,
ObIArray<int64_t> &part_ids,
int64_t &subpart_cnt_in_parts) const
{
int ret = OB_SUCCESS;
subpart_cnt_in_parts = 0;
ObHashSet<int64_t> subpart_hashset;
ObPartition **part_array = NULL;
int64_t part_num = get_partition_num();
part_ids.reuse();
if (OB_UNLIKELY(PARTITION_LEVEL_TWO != part_level_) ||
OB_ISNULL(part_array = get_part_array())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid part", KR(ret), KPC(this));
} else if (OB_FAIL(subpart_hashset.create(get_all_part_num()))) {
LOG_WARN("create hashset failed", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < subpart_ids.count(); i ++) {
if (OB_FAIL(subpart_hashset.set_refactored(subpart_ids.at(i)))) {
LOG_WARN("failed to set refactored", K(ret));
}
}
for (int64_t i = 0; i < part_num && OB_SUCC(ret); ++i) {
if (OB_ISNULL(part_array[i])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL ptr", K(i), KPC(this), KR(ret));
} else {
ObSubPartition **subpart_array = part_array[i]->get_subpart_array();
int64_t subpart_num = part_array[i]->get_subpartition_num();
if (OB_ISNULL(subpart_array)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("subpart array is null", KPC(this), KR(ret));
} else {
bool found = false;
for (int64_t j = 0; j < subpart_num && !found && OB_SUCC(ret); ++j) {
if (OB_ISNULL(subpart_array[j])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL ptr", KPC(this), KR(ret));
} else {
int tmp_ret = subpart_hashset.exist_refactored((subpart_array[j]->get_sub_part_id()));
if (OB_HASH_EXIST == tmp_ret) {
if (OB_FAIL(part_ids.push_back(part_array[i]->get_part_id()))) {
LOG_WARN("failed to push back", K(ret));
} else {
found = true;
subpart_cnt_in_parts += subpart_num;
}
} else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != tmp_ret)) {
ret = tmp_ret;
LOG_WARN("fail to check subpartids exist", K(ret));
}
}
}
}
}
}
if (subpart_hashset.created()) {
int tmp_ret = subpart_hashset.destroy();
if (OB_SUCC(ret) && OB_FAIL(tmp_ret)) {
LOG_WARN("failed to destory hashset", K(ret));
}
}
return ret;
}
int ObSimpleTableSchemaV2::get_part_idx_by_part_id(const ObIArray<int64_t> &part_ids,
ObIArray<int64_t> &part_idx,
ObIArray<int64_t> &subpart_idx) const
{
int ret = OB_SUCCESS;
ObHashSet<int64_t> id_hashset;
ObPartition **part_array = NULL;
int64_t part_num = get_partition_num();
part_idx.reuse();
subpart_idx.reuse();
if (OB_ISNULL(part_array = get_part_array()) ||
OB_UNLIKELY(PARTITION_LEVEL_ZERO == part_level_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid part", KR(ret), KPC(this));
} else if (OB_FAIL(id_hashset.create(get_all_part_num()))) {
LOG_WARN("create hashset failed", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < part_ids.count(); i ++) {
if (OB_FAIL(id_hashset.set_refactored(part_ids.at(i)))) {
LOG_WARN("failed to set refactored", K(ret));
}
}
for (int64_t i = 0; i < part_num && OB_SUCC(ret); ++i) {
if (OB_ISNULL(part_array[i])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL ptr", K(i), KPC(this), KR(ret));
} else if (PARTITION_LEVEL_ONE == part_level_) {
int tmp_ret = id_hashset.exist_refactored(part_array[i]->get_part_id());
if (OB_HASH_EXIST == tmp_ret) {
OZ(part_idx.push_back(i));
OZ(subpart_idx.push_back(OB_INVALID_ID));
} else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != tmp_ret)) {
ret = tmp_ret;
LOG_WARN("fail to check part id exist", K(ret));
}
} else if (PARTITION_LEVEL_TWO == part_level_) {
ObSubPartition **subpart_array = part_array[i]->get_subpart_array();
int64_t subpart_num = part_array[i]->get_subpartition_num();
if (OB_ISNULL(subpart_array)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("subpart array is null", KPC(this), KR(ret));
}
for (int64_t j = 0; j < subpart_num && OB_SUCC(ret); ++j) {
if (OB_ISNULL(subpart_array[j])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL ptr", KPC(this), KR(ret));
} else {
int tmp_ret = id_hashset.exist_refactored(subpart_array[j]->get_sub_part_id());
if (OB_HASH_EXIST == tmp_ret) {
OZ(part_idx.push_back(i));
OZ(subpart_idx.push_back(j));
} else if (OB_UNLIKELY(OB_HASH_NOT_EXIST != tmp_ret)) {
ret = tmp_ret;
LOG_WARN("fail to check part id exist", K(ret));
}
}
}
} else {
ret = OB_NOT_SUPPORTED;
LOG_WARN("4.0 not support part type", KR(ret), KPC(this));
}
}
if (id_hashset.created()) {
int tmp_ret = id_hashset.destroy();
if (OB_SUCC(ret) && OB_FAIL(tmp_ret)) {
LOG_WARN("failed to destory hashset", K(ret));
}
}
return ret;
}
int ObSimpleTableSchemaV2::get_part_id_and_tablet_id_by_idx(const int64_t part_idx,
const int64_t subpart_idx,
ObObjectID &object_id,

10
src/share/schema/ob_table_schema.h
View File

@ -895,6 +895,16 @@ public:
const ObTabletID &tablet_id,
int64_t &part_id,
int64_t &subpart_id) const;
int get_all_first_level_part_ids(
ObIArray<int64_t> &first_level_part_ids) const;
int get_part_ids_by_subpart_ids(
const ObIArray<int64_t> &subpart_ids,
ObIArray<int64_t> &part_ids,
int64_t &subpart_cnt_in_parts) const;
int get_part_idx_by_part_id(
const ObIArray<int64_t> &part_ids,
ObIArray<int64_t> &part_idx,
ObIArray<int64_t> &subpart_idx) const;
int get_hidden_part_id_by_tablet_id(
const ObTabletID &tablet_id,
int64_t &part_id) const;

109
src/sql/optimizer/ob_join_order.cpp
View File

@ -14259,7 +14259,7 @@ int ObJoinOrder::create_and_add_nl_path(const Path *left_path,
int ObJoinOrder::get_used_stat_partitions(const uint64_t ref_table_id,
const share::schema::ObTableSchema &schema,
const ObIArray<int64_t> &all_used_part_ids,
ObIArray<int64_t> &all_used_part_ids,
ObIArray<int64_t> &table_stat_part_ids,
double &table_stat_scale_ratio,
ObIArray<int64_t> *hist_stat_part_ids/* = NULL*/)
@ -14275,6 +14275,7 @@ int ObJoinOrder::get_used_stat_partitions(const uint64_t ref_table_id,
ObArray<int64_t> table_id;
ObArray<int64_t> part_ids;
ObArray<int64_t> subpart_ids;
ObArray<int64_t> new_used_part_ids;
ObSchemaGetterGuard *schema_guard = NULL;
common::ObOptStatManager *opt_stat_manager = NULL;
int64_t subpart_cnt_in_parts = 0;
@ -14345,6 +14346,9 @@ int ObJoinOrder::get_used_stat_partitions(const uint64_t ref_table_id,
LOG_WARN("unexpected null", K(stat_parts));
} else if (!stat_condition[i] || stat_parts[i]->empty()) {
// do nothing
} else if (new_used_part_ids.empty() && scale_ratios[i] == 1.0 &&
OB_FAIL(new_used_part_ids.assign(*stat_parts[i]))) {
LOG_WARN("failed to assign", K(ret));
} else if (!table_stat_part_ids.empty() && NULL != hist_stat_part_ids &&
partition_limit < stat_parts[i]->count()) {
// partitions are too many to use hist, do nothing
@ -14373,6 +14377,10 @@ int ObJoinOrder::get_used_stat_partitions(const uint64_t ref_table_id,
(NULL == hist_stat_part_ids || !hist_stat_part_ids->empty());
}
}
if (OB_SUCC(ret) && !new_used_part_ids.empty() &&
OB_FAIL(all_used_part_ids.assign(new_used_part_ids))) {
LOG_WARN("failed to assign", K(ret));
}
}
LOG_TRACE("get table statistics partition infos",
@ -14395,7 +14403,7 @@ int ObJoinOrder::get_partition_infos(const uint64_t ref_table_id,
part_ids.reuse();
subpart_ids.reuse();
subpart_cnt_in_parts = 0;
if (FAILEDx(table_id.push_back(schema.is_partitioned_table() ? -1 : ref_table_id))) {
if (OB_FAIL(table_id.push_back(schema.is_partitioned_table() ? -1 : ref_table_id))) {
LOG_WARN("failed to push back", K(ret));
} else if (!schema.is_partitioned_table()) {
// do nothing
@ -14406,24 +14414,18 @@ int ObJoinOrder::get_partition_infos(const uint64_t ref_table_id,
} else if (PARTITION_LEVEL_TWO == schema.get_part_level()) {
if (OB_FAIL(subpart_ids.assign(all_used_part_id))) {
LOG_WARN("failed to assign", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < all_used_part_id.count(); ++i) {
int64_t part_id = OB_INVALID_ID;
int64_t subpart_id = OB_INVALID_ID;
common::ObTabletID tablet_id;
ObArray<int64_t> subpart_ids_in_part;
if (OB_FAIL(schema.get_tablet_id_by_object_id(all_used_part_id.at(i), tablet_id))) {
LOG_WARN("failed to get tablet id", K(ret));
} else if (OB_FAIL(schema.get_part_id_by_tablet(tablet_id, part_id, subpart_id))) {
LOG_WARN("failed to get part id by tablet", K(ret), K(tablet_id));
} else if (!ObOptimizerUtil::find_item(part_ids, part_id)) {
if (OB_FAIL(part_ids.push_back(part_id))) {
LOG_WARN("failed to push back part id", K(ret));
} else if (OB_FAIL(schema.get_subpart_ids(part_id, subpart_ids_in_part))) {
LOG_WARN("failed to get subpart ids", K(ret));
} else {
subpart_cnt_in_parts += subpart_ids_in_part.count();
}
} else if (all_used_part_id.count() == schema.get_all_part_num()) {
// full table
if (OB_FAIL(schema.get_all_first_level_part_ids(part_ids))) {
LOG_WARN("failed to get part ids", K(ret));
} else {
subpart_cnt_in_parts = all_used_part_id.count();
}
} else {
if (OB_FAIL(schema.get_part_ids_by_subpart_ids(subpart_ids,
part_ids,
subpart_cnt_in_parts))) {
LOG_WARN("failed to get part id", K(ret));
}
}
} else {
@ -14575,6 +14577,52 @@ int ObJoinOrder::init_est_sel_info_for_access_path(const uint64_t table_id,
return ret;
}
int ObJoinOrder::get_index_partition_ids(const ObCandiTabletLocIArray &part_loc_info_array,
const share::schema::ObTableSchema &table_schema,
const share::schema::ObTableSchema &index_schema,
ObIArray<int64_t> &index_part_ids)
{
int ret = OB_SUCCESS;
if (part_loc_info_array.count() == table_schema.get_all_part_num()) {
// full table
ObArray<ObTabletID> dummy;
ObArray<ObObjectID> tmp_part_ids;
if (OB_FAIL(index_schema.get_all_tablet_and_object_ids(dummy, tmp_part_ids))) {
LOG_WARN("failed to get part ids", K(ret));
} else if (OB_FAIL(append(index_part_ids, tmp_part_ids))) {
LOG_WARN("failed to append", K(ret));
}
} else {
ObArray<int64_t> table_part_ids;
ObArray<int64_t> part_idx;
ObArray<int64_t> subpart_idx;
for (int64_t i = 0; OB_SUCC(ret) && i < part_loc_info_array.count(); ++i) {
const ObOptTabletLoc &part_loc = part_loc_info_array.at(i).get_partition_location();
if (OB_FAIL(table_part_ids.push_back(part_loc.get_partition_id()))) {
LOG_WARN("failed to push back partition id", K(ret));
}
}
if (FAILEDx(table_schema.get_part_idx_by_part_id(table_part_ids, part_idx, subpart_idx))) {
LOG_WARN("failed to get part idx", K(ret));
} else if (OB_UNLIKELY(part_idx.count() != subpart_idx.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected part idx", K(part_idx), K(subpart_idx));
}
for (int64_t i = 0; OB_SUCC(ret) && i < part_idx.count(); ++i) {
ObObjectID object_id = -1;
ObObjectID dummy1;
ObTabletID dummy2;
if (OB_FAIL(index_schema.get_part_id_and_tablet_id_by_idx(
part_idx.at(i), subpart_idx.at(i), object_id, dummy1, dummy2))) {
LOG_WARN("failed to get part id", K(ret));
} else if (OB_FAIL(index_part_ids.push_back(static_cast<int64_t>(object_id)))) {
LOG_WARN("failed to push back", K(ret));
}
}
}
return ret;
}
int ObJoinOrder::init_est_info_for_index(const uint64_t table_id,
const uint64_t index_id,
ObIndexMetaInfo &index_meta_info,
@ -14611,22 +14659,11 @@ int ObJoinOrder::init_est_info_for_index(const uint64_t table_id,
if (OB_FAIL(all_used_part_id.push_back(index_id))) {
LOG_WARN("failed to push back partition id", K(ret));
}
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < part_loc_info_array.count(); ++i) {
const ObOptTabletLoc &part_loc = part_loc_info_array.at(i).get_partition_location();
int64_t part_idx = -1;
int64_t subpart_idx = -1;
ObObjectID object_id = -1;
ObObjectID dummy1;
ObTabletID dummy2;
if (OB_FAIL(table_schema.get_part_idx_by_tablet(part_loc.get_tablet_id(), part_idx, subpart_idx))) {
LOG_WARN("failed to get part idx", K(ret), K(part_loc.get_tablet_id()));
} else if (OB_FAIL(index_schema.get_part_id_and_tablet_id_by_idx(part_idx, subpart_idx, object_id, dummy1, dummy2))) {
LOG_WARN("failed to get part id", K(ret));
} else if (OB_FAIL(all_used_part_id.push_back(static_cast<int64_t>(object_id)))) {
LOG_WARN("failed to push back", K(ret));
}
}
} else if (OB_FAIL(get_index_partition_ids(part_loc_info_array,
table_schema,
index_schema,
all_used_part_id))) {
LOG_WARN("failed to get index partition ids", K(ret));
}
if (OB_SUCC(ret)) {
const int64_t origin_part_cnt = all_used_part_id.count();

7
src/sql/optimizer/ob_join_order.h
View File

@ -1669,7 +1669,7 @@ struct NullAwareAntiJoinInfo {
int get_used_stat_partitions(const uint64_t ref_table_id,
const share::schema::ObTableSchema &table_schema,
const ObIArray<int64_t> &all_used_part_ids,
ObIArray<int64_t> &all_used_part_ids,
ObIArray<int64_t> &table_stat_part_ids,
double &table_stat_scale_ratio,
ObIArray<int64_t> *hist_stat_part_ids = NULL);
@ -1682,6 +1682,11 @@ struct NullAwareAntiJoinInfo {
ObIArray<int64_t> &subpart_ids,
int64_t &subpart_cnt_in_parts);
int get_index_partition_ids(const ObCandiTabletLocIArray &part_loc_info_array,
const share::schema::ObTableSchema &table_schema,
const share::schema::ObTableSchema &index_schema,
ObIArray<int64_t> &index_part_ids);
int init_est_info_for_index(const uint64_t table_id,
const uint64_t index_id,
ObIndexMetaInfo &meta_info,