diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index 864f094e8..e6ac48b45 100644 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -2597,7 +2597,8 @@ typedef enum ObItemType T_RB_ITERATE_EXPRESSION = 4737, T_MODULE_DATA = 4738, T_MODULE_NAME = 4739, - + T_UNION_MERGE_HINT = 4740, + T_UNION_MERGE_LIST = 4741, T_MAX //Attention: add a new type before T_MAX } ObItemType; diff --git a/src/share/system_variable/ob_system_variable_init.cpp b/src/share/system_variable/ob_system_variable_init.cpp index f773db4da..8bcdad6cb 100644 --- a/src/share/system_variable/ob_system_variable_init.cpp +++ b/src/share/system_variable/ob_system_variable_init.cpp @@ -3308,7 +3308,7 @@ static struct VarsInit{ }(); [&] (){ - ObSysVars[234].default_value_ = "4.3.3.0" ; + ObSysVars[234].default_value_ = "4.3.4.0" ; ObSysVars[234].info_ = "enabling a series of optimizer features based on an OceanBase release number" ; ObSysVars[234].name_ = "optimizer_features_enable" ; ObSysVars[234].data_type_ = ObVarcharType ; diff --git a/src/share/system_variable/ob_system_variable_init.json b/src/share/system_variable/ob_system_variable_init.json index 292f2920e..b1e7dbca4 100644 --- a/src/share/system_variable/ob_system_variable_init.json +++ b/src/share/system_variable/ob_system_variable_init.json @@ -3349,7 +3349,7 @@ "optimizer_features_enable": { "id": 10150, "name": "optimizer_features_enable", - "default_value": "4.3.3.0", + "default_value": "4.3.4.0", "base_value": "", "data_type": "varchar", "info": "enabling a series of optimizer features based on an OceanBase release number", diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 439985433..ef4835d24 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -80,6 +80,7 @@ ob_set_subtarget(ob_sql das das/iter/ob_das_text_retrieval_merge_iter.cpp das/iter/ob_das_iter_utils.cpp das/iter/ob_das_vid_merge_iter.cpp + das/iter/ob_das_index_merge_iter.cpp ) ob_set_subtarget(ob_sql dtl diff --git a/src/sql/code_generator/ob_tsc_cg_service.cpp b/src/sql/code_generator/ob_tsc_cg_service.cpp index fd238b7ff..7eb7cef1c 100644 --- a/src/sql/code_generator/ob_tsc_cg_service.cpp +++ b/src/sql/code_generator/ob_tsc_cg_service.cpp @@ -46,6 +46,9 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc } tsc_ctdef.scan_flags_ = query_flag; + if (op.use_index_merge()) { + tsc_ctdef.use_index_merge_ = true; + } if (OB_SUCC(ret) && op.get_table_type() == share::schema::EXTERNAL_TABLE) { const ObTableSchema *table_schema = nullptr; ObSqlSchemaGuard *schema_guard = cg_.opt_ctx_->get_sql_schema_guard(); @@ -197,6 +200,15 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc } } + if (OB_SUCC(ret) && op.use_index_merge()) { + ObDASIndexMergeCtDef *index_merge_ctdef = nullptr; + if (OB_FAIL(generate_index_merge_ctdef(op, tsc_ctdef, root_ctdef))) { + LOG_WARN("failed to generate index merge ctdef", K(ret)); + } else { + need_attach = true; + } + } + ObDASVIdMergeCtDef *vid_merge_ctdef = nullptr; if (OB_SUCC(ret) && op.get_index_back()) { ObDASTableLookupCtDef *lookup_ctdef = nullptr; @@ -499,9 +511,27 @@ int ObTscCgService::generate_tsc_filter(const ObLogTableScan &op, ObTableScanSpe ObArray lookup_pushdown_filters; ObDASScanCtDef &scan_ctdef = spec.tsc_ctdef_.scan_ctdef_; ObDASScanCtDef *lookup_ctdef = spec.tsc_ctdef_.get_lookup_ctdef(); - if (OB_FAIL(op.extract_pushdown_filters(nonpushdown_filters, - scan_pushdown_filters, - lookup_pushdown_filters))) { + if (op.use_index_merge()) { + // full filters is used for final check when in index merge + // we need to pushdown full filters to lookup as much as possible to avoid + // the transmission of large results during DAS remote execution + // all index table scan filters are generated in @generate_das_scan_ctdef() + const ObIArray &full_filters = op.get_full_filters(); + if (OB_FAIL(op.extract_nonpushdown_filters(full_filters, + nonpushdown_filters, + lookup_pushdown_filters))) { + LOG_WARN("failed to extract lookup pushdown filters", K(ret)); + } else if (lookup_ctdef != nullptr && OB_FAIL(generate_pd_storage_flag(op.get_plan(), + op.get_ref_table_id(), + op.get_access_exprs(), + op.get_type(), + op.get_index_back() && op.get_is_index_global(), + lookup_ctdef->pd_expr_spec_))) { + LOG_WARN("generate pd storage flag for lookup ctdef failed", K(ret)); + } + } else if (OB_FAIL(op.extract_pushdown_filters(nonpushdown_filters, + scan_pushdown_filters, + lookup_pushdown_filters))) { LOG_WARN("extract pushdown filters failed", K(ret)); } else if (op.get_contains_fake_cte()) { // do nothing @@ -628,6 +658,7 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op, { int ret = OB_SUCCESS; const ObTableID &scan_table_id = scan_ctdef.ref_table_id_; + const bool use_index_merge = scan_ctdef.is_index_merge_; if ((op.is_text_retrieval_scan() && scan_table_id != op.get_ref_table_id()) || (op.is_multivalue_index_scan() && scan_table_id == op.get_doc_id_index_table_id())) { // non main table scan in text retrieval @@ -644,23 +675,29 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op, if (OB_FAIL(extract_rowkey_vid_access_columns(op, scan_ctdef, access_exprs))) { LOG_WARN("fail to extract rowkey doc access columns", K(ret)); } - } else if (op.get_index_back() && scan_table_id == op.get_real_index_table_id()) { + } else if (op.get_index_back() && (scan_table_id != op.get_real_ref_table_id() || use_index_merge)) { //this das scan is index scan and will lookup the data table later //index scan + lookup data table: the index scan only need access //range condition columns + index filter columns + the data table rowkeys - const ObIArray &range_conditions = op.get_range_conditions(); + const ObIArray &range_conditions = use_index_merge ? + op.get_index_range_conds(scan_ctdef.index_merge_idx_) : op.get_range_conditions(); if (OB_FAIL(ObRawExprUtils::extract_column_exprs(range_conditions, access_exprs))) { LOG_WARN("extract column exprs failed", K(ret)); } + //store index filter columns if (OB_SUCC(ret)) { + ObArray filter_columns; ObArray nonpushdown_filters; ObArray scan_pushdown_filters; ObArray lookup_pushdown_filters; - ObArray filter_columns; // the column in scan pushdown filters - if (OB_FAIL(const_cast(op).extract_pushdown_filters(nonpushdown_filters, - scan_pushdown_filters, - lookup_pushdown_filters))) { + if (use_index_merge && + OB_FAIL(scan_pushdown_filters.assign(op.get_index_filters(scan_ctdef.index_merge_idx_)))) { + LOG_WARN("failed to assign index merge filters", K(ret)); + } else if (!use_index_merge && + OB_FAIL(const_cast(op).extract_pushdown_filters(nonpushdown_filters, + scan_pushdown_filters, + lookup_pushdown_filters))) { LOG_WARN("extract pushdown filters failed", K(ret)); } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(scan_pushdown_filters, filter_columns))) { @@ -669,6 +706,7 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op, LOG_WARN("append filter column to access exprs failed", K(ret)); } } + //store data table rowkeys if (OB_SUCC(ret)) { if (OB_FAIL(append_array_no_dup(access_exprs, op.get_rowkey_exprs()))) { @@ -679,6 +717,22 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op, } } else if (OB_FAIL(access_exprs.assign(op.get_access_exprs()))) { LOG_WARN("assign access exprs failed", K(ret)); + } else if (op.use_index_merge()) { + // add lookup pushdown exprs when use index merge + ObArray nonpushdown_filters; + ObArray lookup_pushdown_filters; + ObArray filter_columns; + const ObIArray &full_filters = op.get_full_filters(); + if (OB_FAIL(op.extract_nonpushdown_filters(full_filters, + nonpushdown_filters, + lookup_pushdown_filters))) { + LOG_WARN("failed to extract lookup pushdown filters", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(lookup_pushdown_filters, + filter_columns))) { + LOG_WARN("failed to extract column exprs", K(ret)); + } else if (OB_FAIL(append_array_no_dup(access_exprs, filter_columns))) { + LOG_WARN("failed to append filter columns", K(ret)); + } } // store group_id_expr when use group id if (OB_SUCC(ret) && op.use_group_id()) { @@ -731,6 +785,8 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op, LOG_WARN("failed to remove generated column exprs", K(ret)); } } + LOG_TRACE("extract das access exprs", K(scan_table_id), K(op.get_real_ref_table_id()), + K(op.get_index_back()), K(scan_ctdef.is_index_merge_), K(access_exprs)); return ret; } @@ -1028,6 +1084,41 @@ int ObTscCgService::generate_das_scan_ctdef(const ObLogTableScan &op, LOG_WARN("failed to init result outputs", K(ret)); } } + //8. generate rowkey exprs and index pushdown filters when use index merge + if (OB_SUCC(ret) && scan_ctdef.is_index_merge_) { + ObArray rowkey_exprs; + ObArray scan_pushdown_filters; + if (OB_FAIL(rowkey_exprs.assign(op.get_rowkey_exprs()))) { + LOG_WARN("failed to assign rowkey exprs", K(ret)); + } else if (!op.get_is_index_global() && OB_FAIL(mapping_oracle_real_agent_virtual_exprs(op, rowkey_exprs))) { + LOG_WARN("failed to mapping oracle real virtual exprs", K(ret)); + } else if (OB_FAIL(cg_.generate_rt_exprs(rowkey_exprs, scan_ctdef.rowkey_exprs_))) { + LOG_WARN("failed to generate main table rowkey exprs", K(ret)); + } else if (OB_FAIL(op.get_index_filters(scan_ctdef.index_merge_idx_, scan_pushdown_filters))) { + LOG_WARN("failed to get index filters", K(ret)); + } else if (!scan_pushdown_filters.empty()) { + if (OB_FAIL(generate_pd_storage_flag(op.get_plan(), + op.get_ref_table_id(), + op.get_access_exprs(), + op.get_type(), + op.get_index_back() && op.get_is_index_global(), + scan_ctdef.pd_expr_spec_))) { + LOG_WARN("failed to generate pd storage flag for index scan ctdef", K(scan_ctdef.ref_table_id_), K(ret)); + } else if (OB_FAIL(cg_.generate_rt_exprs(scan_pushdown_filters, scan_ctdef.pd_expr_spec_.pushdown_filters_))) { + LOG_WARN("failed to generate index scan pushdown filter", K(scan_ctdef.ref_table_id_), K(ret)); + } else if (scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_filter_pushdown()) { + ObPushdownFilterConstructor filter_constructor( + &cg_.phy_plan_->get_allocator(), cg_, &op, + scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_use_column_store()); + if (OB_FAIL(filter_constructor.apply( + scan_pushdown_filters, scan_ctdef.pd_expr_spec_.pd_storage_filters_.get_pushdown_filter()))) { + LOG_WARN("failed to apply filter constructor", K(ret)); + } + LOG_TRACE("index merge pushdown filters", K(scan_ctdef.ref_table_id_), K(scan_pushdown_filters)); + } + } + } + return ret; } @@ -1039,7 +1130,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op, int ret = OB_SUCCESS; ObArray das_output_cols; const ObTableID &table_id = scan_ctdef.ref_table_id_; - + const bool use_index_merge = scan_ctdef.is_index_merge_; if ((op.is_text_retrieval_scan() && table_id != op.get_ref_table_id()) || (op.is_multivalue_index_scan() && table_id == op.get_doc_id_index_table_id())) { // non main table scan in text retrieval @@ -1057,7 +1148,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op, if (OB_FAIL(extract_rowkey_vid_output_columns_ids(index_schema, op, scan_ctdef, output_cids))) { LOG_WARN("fail to extract rowkey doc output columns ids", K(ret)); } - } else if ((op.get_index_back() || op.is_multivalue_index_scan()) && op.get_real_index_table_id() == table_id) { + } else if ((op.get_index_back() || op.is_multivalue_index_scan()) && (op.get_real_ref_table_id() != table_id || use_index_merge)) { //this situation is index lookup, and the index table scan is being processed //the output column id of index lookup is the rowkey of the data table const ObTableSchema *table_schema = nullptr; @@ -1148,6 +1239,9 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op, } } } + + LOG_TRACE("extract das output column ids", K(ret), K(table_id), K(op.get_ref_table_id()), + K(op.get_index_back()), K(scan_ctdef.is_index_merge_), K(output_cids)); return ret; } int ObTscCgService::extract_das_column_ids(const ObIArray &column_exprs, @@ -1553,6 +1647,111 @@ int ObTscCgService::generate_text_ir_ctdef(const ObLogTableScan &op, return ret; } +int ObTscCgService::generate_index_merge_ctdef(const ObLogTableScan &op, + ObTableScanCtDef &tsc_ctdef, + ObDASBaseCtDef *&root_ctdef) +{ + int ret = OB_SUCCESS; + const IndexMergePath *path = nullptr; + common::ObIAllocator &ctdef_alloc = cg_.phy_plan_->get_allocator(); + if (OB_ISNULL(op.get_access_path())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ret)); + } else { + OB_ASSERT(op.use_index_merge()); + path = static_cast(op.get_access_path()); + ObIndexMergeNode *root = path->root_; + if (OB_FAIL(generate_index_merge_node_ctdef(op, root, ctdef_alloc, root_ctdef))) { + LOG_WARN("failed to generate index merge ctdef", K(root_ctdef)); + } + } + return ret; +} + +int ObTscCgService::generate_index_merge_node_ctdef(const ObLogTableScan &op, + ObIndexMergeNode *node, + common::ObIAllocator &alloc, + ObDASBaseCtDef *&node_ctdef) +{ + int ret = OB_SUCCESS; + bool has_rowscn = false; + if (OB_ISNULL(node) || !node->is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", KPC(node), K(ret)); + } else if (node->is_leaf_node_) { + ObDASScanCtDef *scan_ctdef = nullptr; + if (OB_ISNULL(node->ap_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null access path", K(ret)); + } else if (OB_FAIL(ObDASTaskFactory::alloc_das_ctdef(DAS_OP_TABLE_SCAN, alloc, scan_ctdef))) { + LOG_WARN("failed to allocate scan ctdef", K(ret)); + } else if (FALSE_IT(scan_ctdef->ref_table_id_ = node->index_tid_)) { + } else if (FALSE_IT(scan_ctdef->index_merge_idx_ = node->idx_)) { + } else if (FALSE_IT(scan_ctdef->is_index_merge_ = true)) { + } else if (OB_FAIL(generate_das_scan_ctdef(op, *scan_ctdef, has_rowscn))) { + LOG_WARN("failed to generate das scan ctdef", KPC(scan_ctdef), K(ret)); + } else if (OB_NOT_NULL(node->ap_->pre_query_range_) && + OB_FAIL(scan_ctdef->pre_query_range_.deep_copy(*node->ap_->pre_query_range_))) { + LOG_WARN("failed to deep copy pre query range", K(ret)); + } else if (!node->is_ror_) { + // for non-ROR situations, we need to insert a sort iter + ObDASSortCtDef *sort_ctdef = nullptr; + ObSEArray order_items; + ObArray rowkey_exprs; + if (OB_FAIL(rowkey_exprs.assign(op.get_rowkey_exprs()))) { + LOG_WARN("failed to assign rowkey exprs", K(ret)); + } else if (!op.get_is_index_global() && OB_FAIL(mapping_oracle_real_agent_virtual_exprs(op, rowkey_exprs))) { + LOG_WARN("failed to mapping oracle real virtual exprs", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_exprs.count(); i++) { + if (OB_FAIL(order_items.push_back(OrderItem(rowkey_exprs.at(i), op.get_scan_direction())))) { + LOG_WARN("failed to push back order item", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(ObDASTaskFactory::alloc_das_ctdef(DAS_OP_SORT, alloc, sort_ctdef))) { + LOG_WARN("failed to allocate sort ctdef", K(ret)); + } else if (OB_FAIL(generate_das_sort_ctdef(order_items, + false, + nullptr, + nullptr, + scan_ctdef, + sort_ctdef))) { + LOG_WARN("failed to generate das sort ctdef", K(ret)); + } else { + node_ctdef = sort_ctdef; + } + } + } + } else { + node_ctdef = scan_ctdef; + } + } else { + ObDASIndexMergeCtDef *merge_ctdef = nullptr; + ObDASBaseCtDef *left_ctdef = nullptr; + ObDASBaseCtDef *right_ctdef = nullptr; + if (OB_FAIL(ObDASTaskFactory::alloc_das_ctdef(DAS_OP_INDEX_MERGE, alloc, merge_ctdef))) { + LOG_WARN("failed to allocate index merge ctdef", K(ret)); + } else if (OB_ISNULL(merge_ctdef->children_ = OB_NEW_ARRAY(ObDASBaseCtDef*, &alloc, 2))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate index merge ctdef children", K(ret)); + } else if (OB_FAIL(generate_index_merge_node_ctdef(op, node->left_node_, alloc, left_ctdef))) { + LOG_WARN("failed to generate left node ctdef", K(node->left_node_), K(ret)); + } else if (OB_FAIL(generate_index_merge_node_ctdef(op, node->right_node_, alloc, right_ctdef))) { + LOG_WARN("failed to generate right node ctdef", K(node->right_node_), K(ret)); + } else { + merge_ctdef->children_[0] = left_ctdef; + merge_ctdef->children_[1] = right_ctdef; + merge_ctdef->children_cnt_ = 2; + merge_ctdef->merge_type_ = node->merge_type_; + merge_ctdef->is_left_child_leaf_node_ = (left_ctdef->op_type_ != DAS_OP_INDEX_MERGE); + merge_ctdef->is_reverse_ = is_descending_direction(op.get_scan_direction()); + node_ctdef = merge_ctdef; + } + } + return ret; +} + int ObTscCgService::append_fts_relavence_project_col( ObDASIRAuxLookupCtDef *aux_lookup_ctdef, ObDASIRScanCtDef *ir_scan_ctdef) @@ -2502,7 +2701,7 @@ int ObTscCgService::generate_das_sort_ctdef( if (OB_FAIL(cg_.generate_rt_expr(*order_item.expr_, expr))) { LOG_WARN("failed to generate rt expr", K(ret)); } else { - ObSortFieldCollation field_collation(field_idx++, + ObSortFieldCollation field_collation(field_idx, expr->datum_meta_.cs_type_, order_item.is_ascending(), (order_item.is_null_first() ^ order_item.is_ascending()) ? NULL_LAST : NULL_FIRST); @@ -2538,9 +2737,13 @@ int ObTscCgService::generate_das_sort_ctdef( } else if (ObDASTaskFactory::is_attached(child_ctdef->op_type_) && OB_FAIL(append_array_no_dup(result_output, static_cast(child_ctdef)->result_output_))) { LOG_WARN("failed to append child result output", K(ret)); + } else if (child_ctdef->op_type_ == DAS_OP_TABLE_SCAN + && OB_FAIL(append_array_no_dup(result_output, static_cast(child_ctdef)->result_output_))) { + LOG_WARN("failed to append child result output", K(ret)); } else if (OB_FAIL(sort_ctdef->result_output_.assign(result_output))) { LOG_WARN("failed to assign result output", K(ret)); } + LOG_TRACE("generate sort ctdef finished", K(sort_keys), K(sort_ctdef->sort_exprs_), K(result_output), K(ret)); return ret; } diff --git a/src/sql/code_generator/ob_tsc_cg_service.h b/src/sql/code_generator/ob_tsc_cg_service.h index aa7cf022d..3c14960cd 100644 --- a/src/sql/code_generator/ob_tsc_cg_service.h +++ b/src/sql/code_generator/ob_tsc_cg_service.h @@ -136,6 +136,11 @@ private: ObDASSortCtDef *&sort_ctdef); int mapping_oracle_real_agent_virtual_exprs(const ObLogTableScan &op, common::ObIArray &access_exprs); + int generate_index_merge_ctdef(const ObLogTableScan &op, ObTableScanCtDef &tsc_ctdef, ObDASBaseCtDef *&root_ctdef); + int generate_index_merge_node_ctdef(const ObLogTableScan &op, + ObIndexMergeNode *node, + common::ObIAllocator &alloc, + ObDASBaseCtDef *&node_ctdef); private: ObStaticEngineCG &cg_; diff --git a/src/sql/das/iter/ob_das_index_merge_iter.cpp b/src/sql/das/iter/ob_das_index_merge_iter.cpp new file mode 100644 index 000000000..0a153fdb1 --- /dev/null +++ b/src/sql/das/iter/ob_das_index_merge_iter.cpp @@ -0,0 +1,931 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_DAS +#include "sql/das/iter/ob_das_index_merge_iter.h" +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +int ObDASIndexMergeIter::RowStore::init(common::ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(store_rows_ = + static_cast(allocator.alloc(max_size_ * sizeof(LastDASStoreRow))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K_(max_size), K(ret)); + } else { + for (int64_t i = 0; i < max_size_; i++) { + new (store_rows_ + i) LastDASStoreRow(allocator); + store_rows_[i].reuse_ = true; + } + } + + return ret; +} + +int ObDASIndexMergeIter::RowStore::save(bool is_vectorized, int64_t size) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(size > max_size_) || OB_ISNULL(store_rows_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error for save store rows", K(size), K_(max_size), K_(store_rows), K(ret)); + } else { + if (is_vectorized) { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_); + batch_info_guard.set_batch_size(size); + for (int64_t i = 0; OB_SUCC(ret) && i < size; i++) { + batch_info_guard.set_batch_idx(i); + if (OB_FAIL(store_rows_[i].save_store_row(*exprs_, *eval_ctx_))) { + LOG_WARN("index merge iter failed to store rows", K(ret)); + } + } + } else if (OB_FAIL(store_rows_[0].save_store_row(*exprs_, *eval_ctx_))) { + LOG_WARN("index merge iter failed to store rows", K(ret)); + } + } + if (OB_SUCC(ret)) { + cur_idx_ = 0; + saved_size_ = size; + } + + return ret; +} + +int ObDASIndexMergeIter::RowStore::to_expr(bool is_vectorized, int64_t size) +{ + int ret = OB_SUCCESS; + if (is_vectorized) { + if (cur_idx_ + size > saved_size_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument, exceeds saved size", K_(cur_idx), K(size), K_(saved_size), K(ret)); + } else { + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_); + batch_info_guard.set_batch_size(size); + for (int64_t i = 0; OB_SUCC(ret) && i < size; i++) { + batch_info_guard.set_batch_idx(i); + OZ(store_rows_[cur_idx_ + i].store_row_->to_expr(*exprs_, *eval_ctx_)); + } + cur_idx_ += size; + } + } else { + OZ(store_rows_[cur_idx_].store_row_->to_expr(*exprs_, *eval_ctx_)); + cur_idx_++; + } + + return ret; +} + +const ObDatum *ObDASIndexMergeIter::RowStore::cur_datums() +{ + OB_ASSERT(cur_idx_ < saved_size_); + return store_rows_ != nullptr ? store_rows_[cur_idx_].store_row_->cells() : nullptr; +} + +void ObDASIndexMergeIter::RowStore::reuse() +{ + cur_idx_ = OB_INVALID_INDEX; + saved_size_ = 0; +} + +void ObDASIndexMergeIter::RowStore::reset() +{ + if (OB_NOT_NULL(store_rows_)) { + for (int64_t i = 0; i < max_size_; i++) { + store_rows_[i].~LastDASStoreRow(); + } + store_rows_ = nullptr; + } + exprs_ = nullptr; + eval_ctx_ = nullptr; + max_size_ = 1; + saved_size_ = 0; + cur_idx_ = OB_INVALID_INDEX; +} + +int ObDASIndexMergeIter::inner_init(ObDASIterParam ¶m) +{ + int ret = OB_SUCCESS; + if (ObDASIterType::DAS_ITER_INDEX_MERGE != param.type_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("inner init das iter with bad param type", K(param)); + } else { + ObDASIndexMergeIterParam &index_merge_param = static_cast(param); + merge_type_ = index_merge_param.merge_type_; + left_iter_ = index_merge_param.left_iter_; + right_iter_ = index_merge_param.right_iter_; + rowkey_exprs_ = index_merge_param.rowkey_exprs_; + left_output_ = index_merge_param.left_output_; + right_output_ = index_merge_param.right_output_; + get_next_row_ = (merge_type_ == INDEX_MERGE_UNION) ? + &ObDASIndexMergeIter::union_get_next_row : &ObDASIndexMergeIter::intersect_get_next_row; + get_next_rows_ = (merge_type_ == INDEX_MERGE_UNION) ? + &ObDASIndexMergeIter::union_get_next_rows : &ObDASIndexMergeIter::intersect_get_next_rows; + merge_ctdef_ = index_merge_param.ctdef_; + merge_rtdef_ = index_merge_param.rtdef_; + tx_desc_ = index_merge_param.tx_desc_; + snapshot_ = index_merge_param.snapshot_; + is_reverse_ = index_merge_param.is_reverse_; + if (merge_ctdef_->is_left_child_leaf_node_) { + OB_ASSERT(merge_rtdef_->children_[0]->op_type_ == DAS_OP_TABLE_SCAN || + (merge_rtdef_->children_[0]->op_type_ == DAS_OP_SORT && + merge_rtdef_->children_[0]->children_[0]->op_type_ == DAS_OP_TABLE_SCAN)); + left_scan_rtdef_ = merge_rtdef_->children_[0]->op_type_ == DAS_OP_TABLE_SCAN ? + static_cast(merge_rtdef_->children_[0]) : + static_cast(merge_rtdef_->children_[0]->children_[0]); + OB_ASSERT(left_scan_rtdef_ != nullptr); + left_scan_ctdef_ = static_cast(left_scan_rtdef_->ctdef_); + } + OB_ASSERT(merge_rtdef_->children_[1]->op_type_ == DAS_OP_TABLE_SCAN || + (merge_rtdef_->children_[1]->op_type_ == DAS_OP_SORT && + merge_rtdef_->children_[1]->children_[0]->op_type_ == DAS_OP_TABLE_SCAN)); + right_scan_rtdef_ = merge_rtdef_->children_[1]->op_type_ == DAS_OP_TABLE_SCAN ? + static_cast(merge_rtdef_->children_[1]) : + static_cast(merge_rtdef_->children_[1]->children_[0]); + OB_ASSERT(right_scan_rtdef_ != nullptr); + right_scan_ctdef_ = static_cast(right_scan_rtdef_->ctdef_); + + lib::ContextParam context_param; + context_param.set_mem_attr(MTL_ID(), "DASIndexMerge", ObCtxIds::DEFAULT_CTX_ID) + .set_properties(lib::USE_TL_PAGE_OPTIONAL); + if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_ctx_, context_param))) { + LOG_WARN("failed to create index merge memctx", K(ret)); + } else { + const common::ObIArray *left_exprs = (merge_type_ == INDEX_MERGE_UNION) ? + rowkey_exprs_ : left_output_; + const common::ObIArray *right_exprs = (merge_type_ == INDEX_MERGE_UNION) ? + rowkey_exprs_ : right_output_; + common::ObArenaAllocator &alloc = mem_ctx_->get_arena_allocator(); + if (OB_ISNULL(left_row_store_ = OB_NEWx(RowStore, &alloc, left_exprs, eval_ctx_, max_size_)) || + OB_ISNULL(right_row_store_ = OB_NEWx(RowStore, &alloc, right_exprs, eval_ctx_, max_size_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate row store", K(max_size_), K(ret)); + } else if (OB_FAIL(left_row_store_->init(alloc)) || OB_FAIL(right_row_store_->init(alloc))) { + LOG_WARN("failed to init row store", K(ret)); + } + } + } + return ret; +} + +int ObDASIndexMergeIter::init_scan_param(const share::ObLSID &ls_id, + const common::ObTabletID &tablet_id, + const sql::ObDASScanCtDef *ctdef, + sql::ObDASScanRtDef *rtdef, + ObTableScanParam &scan_param) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), KPC(ctdef), KPC(rtdef), K(ls_id), K(tablet_id)); + } else { + uint64_t tenant_id = MTL_ID(); + scan_param.tenant_id_ = tenant_id; + scan_param.key_ranges_.set_attr(ObMemAttr(tenant_id, "ScanParamKR")); + scan_param.ss_key_ranges_.set_attr(ObMemAttr(tenant_id, "ScanParamSSKR")); + scan_param.tx_lock_timeout_ = rtdef->tx_lock_timeout_; + scan_param.index_id_ = ctdef->ref_table_id_; + scan_param.is_get_ = false; // scan + scan_param.is_for_foreign_check_ = false; + scan_param.timeout_ = rtdef->timeout_ts_; + scan_param.scan_flag_ = rtdef->scan_flag_; + scan_param.reserved_cell_count_ = ctdef->access_column_ids_.count(); + scan_param.allocator_ = &rtdef->stmt_allocator_; + scan_param.scan_allocator_ = &rtdef->scan_allocator_; + scan_param.sql_mode_ = rtdef->sql_mode_; + scan_param.frozen_version_ = rtdef->frozen_version_; + scan_param.force_refresh_lc_ = rtdef->force_refresh_lc_; + scan_param.output_exprs_ = &(ctdef->pd_expr_spec_.access_exprs_); + scan_param.calc_exprs_ = &(ctdef->pd_expr_spec_.calc_exprs_); + scan_param.aggregate_exprs_ = &(ctdef->pd_expr_spec_.pd_storage_aggregate_output_); + scan_param.table_param_ = &(ctdef->table_param_); + scan_param.op_ = rtdef->p_pd_expr_op_; + scan_param.row2exprs_projector_ = rtdef->p_row2exprs_projector_; + scan_param.schema_version_ = ctdef->schema_version_; + scan_param.tenant_schema_version_ = rtdef->tenant_schema_version_; + scan_param.limit_param_ = rtdef->limit_param_; + scan_param.need_scn_ = rtdef->need_scn_; + scan_param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_; + scan_param.fb_snapshot_ = rtdef->fb_snapshot_; + scan_param.fb_read_tx_uncommitted_ = rtdef->fb_read_tx_uncommitted_; + scan_param.ls_id_ = ls_id; + scan_param.tablet_id_ = tablet_id; + if (!ctdef->pd_expr_spec_.pushdown_filters_.empty()) { + scan_param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_; + } + scan_param.pd_storage_filters_ = rtdef->p_pd_expr_op_->pd_storage_filters_; + if (OB_NOT_NULL(tx_desc_)) { + scan_param.tx_id_ = tx_desc_->get_tx_id(); + } else { + scan_param.tx_id_.reset(); + } + + if (OB_NOT_NULL(snapshot_)) { + if (OB_FAIL(scan_param.snapshot_.assign(*snapshot_))) { + LOG_WARN("assign snapshot fail", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("null snapshot", K(ret), KPC_(snapshot)); + } + + if (FAILEDx(scan_param.column_ids_.assign(ctdef->access_column_ids_))) { + LOG_WARN("failed to init column ids", K(ret)); + } else if (OB_FAIL(prepare_scan_ranges(scan_param, rtdef))) { + LOG_WARN("failed to prepare scan ranges", K(ret)); + } + } + return ret; +} + +int ObDASIndexMergeIter::prepare_scan_ranges(ObTableScanParam &scan_param, const ObDASScanRtDef *rtdef) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(rtdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr scan rtdef", K(ret)); + } else if (OB_FAIL(scan_param.key_ranges_.assign(rtdef->key_ranges_))) { + LOG_WARN("failed to assign key ranges", K(ret)); + } else if (OB_FAIL(scan_param.ss_key_ranges_.assign(rtdef->ss_key_ranges_))) { + LOG_WARN("failed to assign ss key ranges", K(ret)); + } else if (OB_FAIL(scan_param.mbr_filters_.assign(rtdef->mbr_filters_))) { + LOG_WARN("failed to assign mbr filters", K(ret)); + } + + LOG_TRACE("index merge iter prepare scan ranges", K(scan_param), KPC(rtdef), K(ret)); + return ret; +} + +void ObDASIndexMergeIter::reset_datum_ptr(const common::ObIArray *exprs, int64_t size) +{ + if (OB_NOT_NULL(exprs) && size > 0) { + for (int64_t i = 0; i < exprs->count(); i++) { + ObExpr *expr = exprs->at(i); + if (OB_NOT_NULL(expr)) { + expr->locate_datums_for_update(*eval_ctx_, size); + ObEvalInfo &info = expr->get_eval_info(*eval_ctx_); + info.point_to_frame_ = true; + } + } + } +} + +int ObDASIndexMergeIter::do_table_scan() +{ + int ret = OB_SUCCESS; + if (merge_ctdef_->is_left_child_leaf_node_) { + OB_ASSERT(left_scan_ctdef_ != nullptr && left_scan_rtdef_ != nullptr); + if (OB_FAIL(init_scan_param(ls_id_, left_tablet_id_, left_scan_ctdef_, left_scan_rtdef_, left_scan_param_))) { + LOG_WARN("failed to init left scan param", K(ret)); + } + } + + if (OB_SUCC(ret)) { + OB_ASSERT(right_scan_ctdef_ != nullptr && right_scan_rtdef_ != nullptr); + if (OB_FAIL(init_scan_param(ls_id_, right_tablet_id_, right_scan_ctdef_, right_scan_rtdef_, right_scan_param_))) { + LOG_WARN("failed to init right scan param", K(ret)); + } else if (OB_FAIL(left_iter_->do_table_scan())) { + LOG_WARN("left iter failed to do table scan", K(ret)); + } else if (OB_FAIL(right_iter_->do_table_scan())) { + LOG_WARN("right iter failed to do table scan", K(ret)); + } + } + + return ret; +} + +int ObDASIndexMergeIter::rescan() +{ + int ret = OB_SUCCESS; + if (merge_ctdef_->is_left_child_leaf_node_) { + left_scan_param_.tablet_id_ = left_tablet_id_; + left_scan_param_.ls_id_ = ls_id_; + if (OB_FAIL(prepare_scan_ranges(left_scan_param_, left_scan_rtdef_))) { + LOG_WARN("failed to prepare left rescan ranges", K(ret)); + } + } + right_scan_param_.tablet_id_ = right_tablet_id_; + right_scan_param_.ls_id_ = ls_id_; + if (FAILEDx(prepare_scan_ranges(right_scan_param_, right_scan_rtdef_))) { + LOG_WARN("failed to prepare right rescan ranges", K(ret)); + } else if (OB_FAIL(left_iter_->rescan())) { + LOG_WARN("left iter failed to rescan", K(ret)); + } else if (OB_FAIL(right_iter_->rescan())) { + LOG_WARN("left iter failed to rescan", K(ret)); + } + return ret; +} + +void ObDASIndexMergeIter::clear_evaluated_flag() +{ + if (OB_NOT_NULL(left_iter_)) { + left_iter_->clear_evaluated_flag(); + } + if (OB_NOT_NULL(right_iter_)) { + right_iter_->clear_evaluated_flag(); + } +} + +int ObDASIndexMergeIter::set_ls_tablet_ids(const ObLSID &ls_id, const ObDASRelatedTabletID &related_tablet_ids) +{ + int ret = OB_SUCCESS; + ls_id_ = ls_id; + if (merge_ctdef_->is_left_child_leaf_node_) { + OB_ASSERT(left_scan_ctdef_ != nullptr); + left_tablet_id_ = related_tablet_ids.index_merge_tablet_ids_.at(left_scan_ctdef_->index_merge_idx_); + } + OB_ASSERT(right_scan_ctdef_ != nullptr); + right_tablet_id_ = related_tablet_ids.index_merge_tablet_ids_.at(right_scan_ctdef_->index_merge_idx_); + return ret; +} + +int ObDASIndexMergeIter::inner_reuse() +{ + int ret = OB_SUCCESS; + if (merge_ctdef_->is_left_child_leaf_node_) { + const ObTabletID &old_left_tablet_id = left_scan_param_.tablet_id_; + left_scan_param_.need_switch_param_ = left_scan_param_.need_switch_param_ || + (old_left_tablet_id.is_valid() && old_left_tablet_id != left_tablet_id_); + left_scan_param_.key_ranges_.reuse(); + left_scan_param_.ss_key_ranges_.reuse(); + left_scan_param_.mbr_filters_.reuse(); + } + const ObTabletID &old_right_tablet_id = right_scan_param_.tablet_id_; + right_scan_param_.need_switch_param_ = right_scan_param_.need_switch_param_ || + (old_right_tablet_id.is_valid() && old_right_tablet_id != right_tablet_id_); + right_scan_param_.key_ranges_.reuse(); + right_scan_param_.ss_key_ranges_.reuse(); + right_scan_param_.mbr_filters_.reuse(); + + if (OB_FAIL(left_iter_->reuse())) { + LOG_WARN("index merge iter failed to reuse left iter", K(ret)); + } else if (OB_FAIL(right_iter_->reuse())) { + LOG_WARN("index merge iter failed to reuse right iter", K(ret)); + } + if (OB_NOT_NULL(left_row_store_)) { + left_row_store_->reuse(); + } + if (OB_NOT_NULL(right_row_store_)) { + right_row_store_->reuse(); + } + left_iter_end_ = false; + right_iter_end_ = false; + state_ = FILL_LEFT_ROW; + + return ret; +} + +int ObDASIndexMergeIter::inner_release() +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(left_row_store_)) { + left_row_store_->reset(); + left_row_store_ = nullptr; + } + if (OB_NOT_NULL(right_row_store_)) { + right_row_store_->reset(); + right_row_store_ = nullptr; + } + left_iter_end_ = false; + right_iter_end_ = false; + left_scan_param_.destroy_schema_guard(); + left_scan_param_.snapshot_.reset(); + left_scan_param_.destroy(); + right_scan_param_.destroy_schema_guard(); + right_scan_param_.snapshot_.reset(); + right_scan_param_.destroy(); + if (OB_NOT_NULL(mem_ctx_)) { + mem_ctx_->reset_remain_one_page(); + DESTROY_CONTEXT(mem_ctx_); + mem_ctx_ = nullptr; + } + return ret; +} + +int ObDASIndexMergeIter::inner_get_next_row() +{ + int ret = OB_SUCCESS; + clear_evaluated_flag(); + if (OB_FAIL((this->*get_next_row_)())) { + if (ret != OB_ITER_END) { + LOG_WARN("index merge iter failed to get next row", K(ret)); + } + } + return ret; +} + +int ObDASIndexMergeIter::inner_get_next_rows(int64_t &count, int64_t capacity) +{ + int ret = OB_SUCCESS; + clear_evaluated_flag(); + if (OB_FAIL((this->*get_next_rows_)(count, capacity))) { + if (ret != OB_ITER_END) { + LOG_WARN("index merge iter failed to get next rows", K(ret)); + } + } + LOG_TRACE("[DAS ITER] index merge iter get next rows", K(count), K(capacity), K(ret)); + const ObBitVector *skip = nullptr; + PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_, count, skip); + + return ret; +} + +int ObDASIndexMergeIter::compare(int &cmp_ret) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(left_row_store_) || + OB_ISNULL(right_row_store_) || + !left_row_store_->have_data() || + !right_row_store_->have_data()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KPC_(left_row_store), KPC_(right_row_store)); + } else { + const ObDatum *left_datums = left_row_store_->cur_datums(); + const ObDatum *right_datums = right_row_store_->cur_datums(); + if (OB_ISNULL(left_datums) || OB_ISNULL(right_datums)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(left_datums), K(right_datums)); + } + bool end_compare = false; + ObObj left_obj; + ObObj right_obj; + for (int64_t i = 0; !end_compare && OB_SUCC(ret) && i < rowkey_exprs_->count(); i++) { + const ObExpr *expr = rowkey_exprs_->at(i); + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ret)); + } else if (OB_FAIL(left_datums[i].to_obj(left_obj, expr->obj_meta_, expr->obj_datum_map_))) { + LOG_WARN("failed to convert left datum to obj", K(i), KPC(expr), K(ret)); + } else if (OB_FAIL(right_datums[i].to_obj(right_obj, expr->obj_meta_, expr->obj_datum_map_))) { + LOG_WARN("failed to convert right datum to obj", K(i), KPC(expr), K(ret)); + } else if (OB_FAIL(left_obj.check_collation_free_and_compare(right_obj, cmp_ret))) { + LOG_WARN("failed to compare cur obj with output obj", K(ret)); + } else if (cmp_ret != 0) { + end_compare = true; + cmp_ret = OB_UNLIKELY(is_reverse_) ? -cmp_ret : cmp_ret; + } + } + } + return ret; +} + +int ObDASIndexMergeIter::intersect_get_next_row() +{ + int ret = OB_SUCCESS; + bool got_next_row = false; + do { + switch(state_) { + case FILL_LEFT_ROW: { + while (OB_SUCC(ret) && !left_iter_end_ && !left_row_store_->have_data()) { + if (OB_FAIL(left_iter_->get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next row from left iter", K(ret)); + } else { + left_iter_end_ = true; + ret = OB_SUCCESS; + } + } else if (OB_FAIL(left_row_store_->save(false, 1))) { + LOG_WARN("failed to save left row", K(ret)); + } + } + + if (OB_SUCC(ret)) { + state_ = left_iter_end_ ? FINISHED : FILL_RIGHT_ROW; + } + break; + } + + case FILL_RIGHT_ROW: { + while (OB_SUCC(ret) && !right_iter_end_ && !right_row_store_->have_data()) { + if (OB_FAIL(right_iter_->get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next row from right iter", K(ret)); + } else { + right_iter_end_ = true; + ret = OB_SUCCESS; + } + } else if (OB_FAIL(right_row_store_->save(false, 1))) { + LOG_WARN("failed to save right row", K(ret)); + } + } + + if (OB_SUCC(ret)) { + state_ = right_iter_end_ ? FINISHED : MERGE_AND_OUTPUT; + } + break; + } + + case MERGE_AND_OUTPUT: { + if (OB_SUCC(ret)) { + int cmp_ret = 0; + if (!left_row_store_->have_data() || !left_row_store_->have_data()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected no data in left_row or right_row", K_(left_row_store), K_(right_row_store)); + } else if (OB_FAIL(compare(cmp_ret))) { + LOG_WARN("failed to compare left row and right row", K(ret)); + } else if (cmp_ret > 0) { + state_ = FILL_RIGHT_ROW; + } else if (cmp_ret < 0) { + state_ = FILL_LEFT_ROW; + } else { + if (OB_FAIL(left_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to convert left store row to expr", K(ret)); + } else if (OB_FAIL(right_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to convert right store row to expr", K(ret)); + } else { + state_ = FILL_LEFT_ROW; + got_next_row = true; + } + } + } + break; + } + + case FINISHED: { + ret = OB_ITER_END; + break; + } + + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index merge state", K_(state)); + } + } + } while (!got_next_row && OB_SUCC(ret)); + + return ret; +} + +int ObDASIndexMergeIter::intersect_get_next_rows(int64_t &count, int64_t capacity) +{ + int ret = OB_SUCCESS; + bool got_next_rows = false; + int64_t left_count = 0; + int64_t right_count = 0; + do { + switch(state_) { + case FILL_LEFT_ROW: { + while (OB_SUCC(ret) && !left_iter_end_ && !left_row_store_->have_data()) { + if (OB_FAIL(left_iter_->get_next_rows(left_count, capacity))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next rows from left iter", K(ret)); + } else { + ret = OB_SUCCESS; + if (left_count == 0) { + left_iter_end_ = true; + } + } + } + if (OB_SUCC(ret) && !left_iter_end_ && OB_FAIL(left_row_store_->save(true, left_count))) { + LOG_WARN("failed to save left rows", K(left_count), K(ret)); + } + } + + if (OB_SUCC(ret)) { + state_ = left_iter_end_ ? FINISHED : FILL_RIGHT_ROW; + } + break; + } + + case FILL_RIGHT_ROW: { + while (OB_SUCC(ret) && !right_iter_end_ && !right_row_store_->have_data()) { + if (OB_FAIL(right_iter_->get_next_rows(right_count, capacity))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next rows from right iter", K(ret)); + } else { + ret = OB_SUCCESS; + if (right_count == 0) { + right_iter_end_ = true; + } + } + } + if (OB_SUCC(ret) && !right_iter_end_ && OB_FAIL(right_row_store_->save(true, right_count))) { + LOG_WARN("failed to save right rows", K(right_count), K(ret)); + } + } + + if (OB_SUCC(ret)) { + state_ = right_iter_end_ ? FINISHED : MERGE_AND_OUTPUT; + } + break; + } + + case MERGE_AND_OUTPUT: { + if (OB_SUCC(ret)) { + int cmp_ret = 0; + if (!left_row_store_->have_data()) { + state_ = FILL_LEFT_ROW; + } else if (!right_row_store_->have_data()) { + state_ = FILL_RIGHT_ROW; + } else if (OB_FAIL(compare(cmp_ret))) { + LOG_WARN("failed to compare left row and right row", K(ret)); + } else if (cmp_ret > 0) { + state_ = FILL_RIGHT_ROW; + } else if (cmp_ret < 0) { + state_ = FILL_LEFT_ROW; + } else { + if (OB_FAIL(left_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to convert left store row to expr", K(ret)); + } else if (OB_FAIL(right_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to convert right store row to expr", K(ret)); + } else { + count = 1; + got_next_rows = true; + } + } + } + break; + } + + case FINISHED: { + ret = OB_ITER_END; + break; + } + + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index merge state", K_(state)); + } + } + } while (!got_next_rows && OB_SUCC(ret)); + + return ret; +} + +int ObDASIndexMergeIter::union_get_next_row() +{ + int ret = OB_SUCCESS; + bool got_next_row = false; + do { + switch(state_) { + case FILL_LEFT_ROW: { + while (OB_SUCC(ret) && !left_iter_end_ && !left_row_store_->have_data()) { + if (OB_FAIL(left_iter_->get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next row from left iter", K(ret)); + } else { + left_iter_end_ = true; + ret = OB_SUCCESS; + } + } else if (OB_FAIL(left_row_store_->save(false, 1))) { + LOG_WARN("failed to save left row", K(ret)); + } + } + + if (OB_SUCC(ret)) { + state_ = FILL_RIGHT_ROW; + } + break; + } + + case FILL_RIGHT_ROW: { + while (OB_SUCC(ret) && !right_iter_end_ && !right_row_store_->have_data()) { + if (OB_FAIL(right_iter_->get_next_row())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next row from right iter", K(ret)); + } else { + right_iter_end_ = true; + ret = OB_SUCCESS; + } + } else if (OB_FAIL(right_row_store_->save(false, 1))) { + LOG_WARN("failed to save right row", K(ret)); + } + } + + if (OB_SUCC(ret)) { + state_ = MERGE_AND_OUTPUT; + } + break; + } + + case MERGE_AND_OUTPUT: { + if (OB_SUCC(ret)) { + if (left_iter_end_ && right_iter_end_) { + state_ = FINISHED; + } else if (left_iter_end_) { + if (right_row_store_->have_data()) { + if (OB_FAIL(right_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to convert right store row to expr", K(ret)); + } else { + got_next_row = true; + state_ = FILL_RIGHT_ROW; + } + } else { + state_ = FILL_RIGHT_ROW; + } + } else if (right_iter_end_) { + if (left_row_store_->have_data()) { + if (OB_FAIL(left_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to convert left store row to expr", K(ret)); + } else { + got_next_row = true; + state_ = FILL_LEFT_ROW; + } + } else { + state_ = FILL_LEFT_ROW; + } + } else { + int cmp_ret = 0; + if (OB_FAIL(compare(cmp_ret))) { + LOG_WARN("failed to compare left row and right row", K(ret)); + } else if (cmp_ret < 0) { + if (OB_FAIL(left_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to convert left store row to expr", K(ret)); + } else { + got_next_row = true; + state_ = FILL_LEFT_ROW; + } + } else if (cmp_ret > 0) { + if (OB_FAIL(right_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to convert right store row to expr", K(ret)); + } else { + got_next_row = true; + state_ = FILL_RIGHT_ROW; + } + } else { + if (OB_FAIL(left_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to convert store row to expr", K(ret)); + } else if (OB_FAIL(right_row_store_->to_expr(false, 1))) { + LOG_WARN("failed to "); + } else { + got_next_row = true; + state_ = FILL_LEFT_ROW; + } + } + } + } + + break; + } + + case FINISHED: { + ret = OB_ITER_END; + break; + } + + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index merge state", K_(state)); + } + } + } while (!got_next_row && OB_SUCC(ret)); + + return ret; +} + +int ObDASIndexMergeIter::union_get_next_rows(int64_t &count, int64_t capacity) +{ + int ret = OB_SUCCESS; + bool got_next_rows = false; + int64_t left_count = 0; + int64_t right_count = 0; + do { + switch(state_) { + case FILL_LEFT_ROW: { + while (OB_SUCC(ret) && !left_iter_end_ && !left_row_store_->have_data()) { + if (OB_FAIL(left_iter_->get_next_rows(left_count, capacity))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next rows from left iter", K(ret)); + } else { + ret = OB_SUCCESS; + if (left_count == 0) { + left_iter_end_ = true; + } + } + } + if (OB_SUCC(ret) && !left_iter_end_ && OB_FAIL(left_row_store_->save(true, left_count))) { + LOG_WARN("failed to save left rows", K(left_count), K(ret)); + } + if (left_iter_->get_type() == DAS_ITER_SORT) { + reset_datum_ptr(left_output_, left_count); + } + } + + if (OB_SUCC(ret)) { + state_ = FILL_RIGHT_ROW; + } + break; + } + + case FILL_RIGHT_ROW: { + while (OB_SUCC(ret) && !right_iter_end_ && !right_row_store_->have_data()) { + if (OB_FAIL(right_iter_->get_next_rows(right_count, capacity))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next rows from right iter", K(ret)); + } else { + ret = OB_SUCCESS; + if (right_count == 0) { + right_iter_end_ = true; + } + } + } + if (OB_SUCC(ret) && !right_iter_end_ && OB_FAIL(right_row_store_->save(true, right_count))) { + LOG_WARN("failed to save right rows", K(right_count), K(ret)); + } + if (right_iter_->get_type() == DAS_ITER_SORT) { + reset_datum_ptr(right_output_, right_count); + } + } + + if (OB_SUCC(ret)) { + state_ = MERGE_AND_OUTPUT; + } + break; + } + + case MERGE_AND_OUTPUT: { + if (OB_SUCC(ret)) { + if (left_iter_end_ && right_iter_end_) { + state_ = FINISHED; + } else if (left_iter_end_) { + if (right_row_store_->have_data()) { + int64_t right_rows_cnt = right_row_store_->rows_cnt(); + if (OB_FAIL(right_row_store_->to_expr(true, right_rows_cnt))) { + LOG_WARN("failed to convert right store rows to expr", K(right_rows_cnt), K(ret)); + } else { + count = right_rows_cnt; + got_next_rows = true; + state_ = FILL_RIGHT_ROW; + } + } else { + state_ = FILL_RIGHT_ROW; + } + } else if (right_iter_end_) { + if (left_row_store_->have_data()) { + int64_t left_rows_cnt = left_row_store_->rows_cnt(); + if (OB_FAIL(left_row_store_->to_expr(true, left_rows_cnt))) { + LOG_WARN("failed to convert left store rows to expr", K(left_rows_cnt), K(ret)); + } else { + count = left_rows_cnt; + got_next_rows = true; + state_ = FILL_LEFT_ROW; + } + } else { + state_ = FILL_LEFT_ROW; + } + } else { + int cmp_ret = 0; + if (OB_FAIL(compare(cmp_ret))) { + LOG_WARN("failed to compare left row and right row", K(ret)); + } else if (cmp_ret < 0) { + if (OB_FAIL(left_row_store_->to_expr(true, 1))) { + LOG_WARN("failed to convert left store row to expr", K(ret)); + } else { + got_next_rows = true; + state_ = FILL_LEFT_ROW; + } + } else if (cmp_ret > 0) { + if (OB_FAIL(right_row_store_->to_expr(true, 1))) { + LOG_WARN("failed to convert right store row to expr", K(ret)); + } else { + got_next_rows = true; + state_ = FILL_RIGHT_ROW; + } + } else { + if (OB_FAIL(left_row_store_->to_expr(true, 1))) { + LOG_WARN("failed to convert left store row to expr", K(ret)); + } else if (OB_FAIL(right_row_store_->to_expr(true, 1))) { + LOG_WARN("failed to convert right store row to expr", K(ret)); + } else { + got_next_rows = true; + state_ = FILL_LEFT_ROW; + } + } + if (OB_SUCC(ret) && got_next_rows) { + count = 1; + } + } + } + + break; + } + + case FINISHED: { + ret = OB_ITER_END; + break; + } + + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index merge state", K_(state)); + } + } + } while (!got_next_rows && OB_SUCC(ret)); + + return ret; +} + + +} // namespace sql +} // namespace oceanbase diff --git a/src/sql/das/iter/ob_das_index_merge_iter.h b/src/sql/das/iter/ob_das_index_merge_iter.h new file mode 100644 index 000000000..3f7e40633 --- /dev/null +++ b/src/sql/das/iter/ob_das_index_merge_iter.h @@ -0,0 +1,228 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_INDEX_MERGE_ITER_H_ +#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_INDEX_MERGE_ITER_H_ + +#include "sql/das/iter/ob_das_iter.h" +#include "sql/optimizer/ob_join_order.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ + +struct ObDASIndexMergeCtDef; +struct ObDASIndexMergeRtDef; + +struct ObDASIndexMergeIterParam : public ObDASIterParam +{ +public: + ObDASIndexMergeIterParam() + : ObDASIterParam(DAS_ITER_INDEX_MERGE), + merge_type_(INDEX_MERGE_INVALID), + rowkey_exprs_(nullptr), + left_iter_(nullptr), + left_output_(nullptr), + right_iter_(nullptr), + right_output_(nullptr), + ctdef_(nullptr), + rtdef_(nullptr), + tx_desc_(nullptr), + snapshot_(nullptr), + is_reverse_(false) + {} + + ObIndexMergeType merge_type_; + const ExprFixedArray *rowkey_exprs_; + ObDASIter *left_iter_; + const common::ObIArray *left_output_; + ObDASIter *right_iter_; + const common::ObIArray *right_output_; + const ObDASIndexMergeCtDef *ctdef_; + ObDASIndexMergeRtDef *rtdef_; + transaction::ObTxDesc *tx_desc_; + transaction::ObTxReadSnapshot *snapshot_; + bool is_reverse_; + + virtual bool is_valid() const + { + return rowkey_exprs_ != nullptr && + left_iter_ != nullptr && + left_output_ != nullptr && + right_iter_ != nullptr && + right_output_ != nullptr && + ctdef_ != nullptr && + rtdef_ != nullptr && + merge_type_ != INDEX_MERGE_INVALID && + ObDASIterParam::is_valid(); + } +}; + +class ObDASIndexMergeIter : public ObDASIter +{ + +public: + struct RowStore + { + public: + RowStore() + : exprs_(nullptr), + eval_ctx_(nullptr), + max_size_(1), + saved_size_(0), + cur_idx_(OB_INVALID_INDEX), + store_rows_(nullptr) + {} + RowStore(const common::ObIArray *exprs, + ObEvalCtx *eval_ctx, + int64_t max_size) + : exprs_(exprs), + eval_ctx_(eval_ctx), + max_size_(max_size), + saved_size_(0), + cur_idx_(OB_INVALID_INDEX), + store_rows_(nullptr) + {} + + int init(common::ObIAllocator &allocator); + int save(bool is_vectorized, int64_t size); + int to_expr(bool is_vectorized, int64_t size); + bool have_data() const { return cur_idx_ != OB_INVALID_INDEX && cur_idx_ < saved_size_; } + int64_t rows_cnt() const { return have_data() ? saved_size_ - cur_idx_ : 0; } + const ObDatum *cur_datums(); + void reuse(); + void reset(); + TO_STRING_KV(K_(exprs), + K_(saved_size), + K_(cur_idx)); + + public: + typedef ObChunkDatumStore::LastStoredRow LastDASStoreRow; + const common::ObIArray *exprs_; + ObEvalCtx *eval_ctx_; + int64_t max_size_; + int64_t saved_size_; + int64_t cur_idx_; + LastDASStoreRow *store_rows_; + }; + +public: + ObDASIndexMergeIter() + : ObDASIter(ObDASIterType::DAS_ITER_INDEX_MERGE), + merge_type_(INDEX_MERGE_INVALID), + state_(FILL_LEFT_ROW), + left_iter_(nullptr), + right_iter_(nullptr), + left_row_store_(nullptr), + right_row_store_(nullptr), + left_scan_param_(), + right_scan_param_(), + left_iter_end_(false), + right_iter_end_(false), + rowkey_exprs_(nullptr), + left_output_(nullptr), + right_output_(nullptr), + mem_ctx_(), + get_next_row_(nullptr), + get_next_rows_(nullptr), + ls_id_(), + left_tablet_id_(), + right_tablet_id_(), + merge_ctdef_(nullptr), + merge_rtdef_(nullptr), + left_scan_ctdef_(nullptr), + left_scan_rtdef_(nullptr), + right_scan_ctdef_(nullptr), + right_scan_rtdef_(nullptr), + tx_desc_(nullptr), + snapshot_(nullptr), + is_reverse_(false) + {} + + virtual ~ObDASIndexMergeIter() {} + +public: + virtual int do_table_scan() override; + virtual int rescan() override; + virtual void clear_evaluated_flag() override; + int set_ls_tablet_ids(const ObLSID &ls_id, const ObDASRelatedTabletID &related_tablet_ids); + ObTableScanParam &get_left_param() { return left_scan_param_; } + ObTableScanParam &get_right_param() { return right_scan_param_; } + +protected: + virtual int inner_init(ObDASIterParam ¶m) override; + virtual int inner_reuse() override; + virtual int inner_release() override; + virtual int inner_get_next_row() override; + virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override; + +private: + int compare(int &cmp_ret); + int intersect_get_next_row(); + int intersect_get_next_rows(int64_t &count, int64_t capacity); + int union_get_next_row(); + int union_get_next_rows(int64_t &count, int64_t capacity); + int init_scan_param(const share::ObLSID &ls_id, + const common::ObTabletID &tablet_id, + const sql::ObDASScanCtDef *ctdef, + sql::ObDASScanRtDef *rtdef, + ObTableScanParam &scan_param); + int prepare_scan_ranges(ObTableScanParam &scan_param, const ObDASScanRtDef *rtdef); + void reset_datum_ptr(const common::ObIArray *exprs, int64_t size); + +private: + enum MergeState : uint32_t + { + FILL_LEFT_ROW, + FILL_RIGHT_ROW, + MERGE_AND_OUTPUT, + FINISHED + }; + + ObIndexMergeType merge_type_; + MergeState state_; + ObDASIter *left_iter_; + ObDASIter *right_iter_; + RowStore *left_row_store_; + RowStore *right_row_store_; + ObTableScanParam left_scan_param_; + ObTableScanParam right_scan_param_; + bool left_iter_end_; + bool right_iter_end_; + const ExprFixedArray *rowkey_exprs_; + const common::ObIArray *left_output_; + const common::ObIArray *right_output_; + lib::MemoryContext mem_ctx_; + int (ObDASIndexMergeIter::*get_next_row_)(); + int (ObDASIndexMergeIter::*get_next_rows_)(int64_t&, int64_t); + ObLSID ls_id_; + ObTabletID left_tablet_id_; + ObTabletID right_tablet_id_; + const ObDASIndexMergeCtDef *merge_ctdef_; + ObDASIndexMergeRtDef *merge_rtdef_; + // nullptr if left child is not a leaf node + const ObDASScanCtDef *left_scan_ctdef_; + ObDASScanRtDef *left_scan_rtdef_; + const ObDASScanCtDef *right_scan_ctdef_; + ObDASScanRtDef *right_scan_rtdef_; + transaction::ObTxDesc *tx_desc_; + transaction::ObTxReadSnapshot *snapshot_; + bool is_reverse_; +}; + +} // namespace sql +} // namespace oceanbase + + +#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_INDEX_MERGE_ITER_H_ */ diff --git a/src/sql/das/iter/ob_das_iter_define.h b/src/sql/das/iter/ob_das_iter_define.h index db1487664..e6914fb10 100644 --- a/src/sql/das/iter/ob_das_iter_define.h +++ b/src/sql/das/iter/ob_das_iter_define.h @@ -69,6 +69,10 @@ enum ObDASIterTreeType : uint32_t struct ObDASRelatedTabletID { public: + ObDASRelatedTabletID(common::ObIAllocator &alloc) + : index_merge_tablet_ids_(alloc) + { reset(); } + common::ObTabletID lookup_tablet_id_; common::ObTabletID aux_lookup_tablet_id_; common::ObTabletID rowkey_vid_tablet_id_; @@ -78,6 +82,11 @@ public: common::ObTabletID fwd_idx_tablet_id_; common::ObTabletID doc_id_idx_tablet_id_; /* used by fulltext index */ + + /* used by index merge */ + common::ObFixedArray index_merge_tablet_ids_; + /* used by index merge */ + void reset() { lookup_tablet_id_.reset(); @@ -86,6 +95,7 @@ public: inv_idx_tablet_id_.reset(); fwd_idx_tablet_id_.reset(); doc_id_idx_tablet_id_.reset(); + index_merge_tablet_ids_.reset(); } }; diff --git a/src/sql/das/iter/ob_das_iter_utils.cpp b/src/sql/das/iter/ob_das_iter_utils.cpp index 22eb401c6..3a6b01311 100644 --- a/src/sql/das/iter/ob_das_iter_utils.cpp +++ b/src/sql/das/iter/ob_das_iter_utils.cpp @@ -50,6 +50,10 @@ int ObDASIterUtils::create_das_scan_iter_tree(ObDASIterTreeType tree_type, ret = create_text_retrieval_tree(scan_param, alloc, attach_ctdef, attach_rtdef, related_tablet_ids, trans_desc, snapshot, iter_tree); break; } + case ITER_TREE_INDEX_MERGE: { + ret = create_index_merge_iter_tree(scan_param, alloc, attach_ctdef, attach_rtdef, related_tablet_ids, trans_desc, snapshot, iter_tree); + break; + } default: { ret = OB_ERR_UNEXPECTED; } @@ -58,7 +62,7 @@ int ObDASIterUtils::create_das_scan_iter_tree(ObDASIterTreeType tree_type, LOG_WARN("failed to create das scan iter tree", K(ret)); } - LOG_DEBUG("create das scan iter tree", K(tree_type), K(ret)); + LOG_TRACE("create das scan iter tree", K(tree_type), K(ret)); return ret; } @@ -214,6 +218,71 @@ int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_ } return ret; } + +int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctdef, + const ObDASRelatedTabletID &related_tablet_ids, + const ObLSID &ls_id, + ObDASIter *root_iter) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(attach_ctdef) || OB_ISNULL(root_iter)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ret), KP(attach_ctdef), KP(root_iter)); + } else { + bool need_set_child = false; + const ObDASIterType &iter_type = root_iter->get_type(); + switch (attach_ctdef->op_type_) { + case ObDASOpType::DAS_OP_TABLE_LOOKUP: { + if (OB_UNLIKELY(iter_type != ObDASIterType::DAS_ITER_LOCAL_LOOKUP)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); + } else { + ObDASLocalLookupIter *lookup_iter = static_cast(root_iter); + lookup_iter->set_ls_id(ls_id); + lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); + need_set_child = true; + } + break; + } + case ObDASOpType::DAS_OP_INDEX_MERGE: { + if (OB_UNLIKELY(iter_type != ObDASIterType::DAS_ITER_INDEX_MERGE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type)); + } else { + ObDASIndexMergeIter *merge_iter = static_cast(root_iter); + if (OB_FAIL(merge_iter->set_ls_tablet_ids(ls_id, related_tablet_ids))) { + LOG_WARN("failed to set related tablet ids", K(ret)); + } + need_set_child = true; + } + break; + } + default: { + need_set_child = false; + break; + } + } + + if (OB_FAIL(ret)) { + } else if (!need_set_child) { + } else if (OB_UNLIKELY(attach_ctdef->children_cnt_ != root_iter->get_children_cnt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected iter children count not equal to ctdef children count", + K(attach_ctdef->children_cnt_), K(root_iter->get_children_cnt()), K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < attach_ctdef->children_cnt_; ++i) { + if (OB_FAIL(set_index_merge_related_ids(attach_ctdef->children_[i], + related_tablet_ids, + ls_id, + root_iter->get_children()[i]))) { + LOG_WARN("failed to set index merge related ids", K(ret)); + } + } + } + } + return ret; +} + /***************** PUBLIC END *****************/ int ObDASIterUtils::create_partition_scan_tree(storage::ObTableScanParam &scan_param, @@ -230,6 +299,10 @@ int ObDASIterUtils::create_partition_scan_tree(storage::ObTableScanParam &scan_p int ret = OB_SUCCESS; ObDASScanIterParam param; param.scan_ctdef_ = scan_ctdef; + param.max_size_ = scan_rtdef->eval_ctx_->is_vectorized() ? scan_rtdef->eval_ctx_->max_batch_size_ : 1; + param.eval_ctx_ = scan_rtdef->eval_ctx_; + param.exec_ctx_ = &scan_rtdef->eval_ctx_->exec_ctx_; + param.output_ = &scan_ctdef->result_output_; ObDASScanIter *scan_iter = nullptr; if (OB_FAIL(create_das_iter(alloc, param, scan_iter))) { LOG_WARN("failed to create das scan iter", K(ret)); @@ -268,8 +341,16 @@ int ObDASIterUtils::create_local_lookup_tree(ObTableScanParam &scan_param, ObDASLocalLookupIter *lookup_iter = nullptr; ObDASScanIterParam index_table_param; index_table_param.scan_ctdef_ = scan_ctdef; + index_table_param.max_size_ = scan_rtdef->eval_ctx_->is_vectorized() ? scan_rtdef->eval_ctx_->max_batch_size_ : 1; + index_table_param.eval_ctx_ = scan_rtdef->eval_ctx_; + index_table_param.exec_ctx_ = &scan_rtdef->eval_ctx_->exec_ctx_; + index_table_param.output_ = &scan_ctdef->result_output_; ObDASScanIterParam data_table_param; data_table_param.scan_ctdef_ = lookup_ctdef; + data_table_param.max_size_ = lookup_rtdef->eval_ctx_->is_vectorized() ? lookup_rtdef->eval_ctx_->max_batch_size_ : 1; + data_table_param.eval_ctx_ = lookup_rtdef->eval_ctx_; + data_table_param.exec_ctx_ = &lookup_rtdef->eval_ctx_->exec_ctx_; + data_table_param.output_ = &lookup_ctdef->result_output_; if (OB_FAIL(create_das_iter(alloc, index_table_param, index_table_iter))) { LOG_WARN("failed to create index table iter", K(ret)); } else if (OB_FAIL(create_das_iter(alloc, data_table_param, data_table_iter))) { @@ -456,6 +537,10 @@ int ObDASIterUtils::create_text_retrieval_sub_tree(const ObLSID &ls_id, merge_iter_param.snapshot_ = snapshot; if (ir_scan_ctdef->need_do_total_doc_cnt()) { doc_cnt_agg_param.scan_ctdef_ = ir_scan_ctdef->get_doc_id_idx_agg_ctdef(); + doc_cnt_agg_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_; + doc_cnt_agg_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_; + doc_cnt_agg_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_; + doc_cnt_agg_param.output_ = &ir_scan_ctdef->get_doc_id_idx_agg_ctdef()->result_output_; if (OB_FAIL(create_das_iter(alloc, doc_cnt_agg_param, doc_cnt_agg_iter))) { LOG_WARN("failed to create doc cnt agg scan iter", K(ret)); } else { @@ -484,12 +569,24 @@ int ObDASIterUtils::create_text_retrieval_sub_tree(const ObLSID &ls_id, ObDASScanIterParam inv_idx_scan_iter_param; ObDASScanIter *inv_idx_scan_iter = nullptr; inv_idx_scan_iter_param.scan_ctdef_ = ir_scan_ctdef->get_inv_idx_scan_ctdef(); + inv_idx_scan_iter_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_; + inv_idx_scan_iter_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_; + inv_idx_scan_iter_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_; + inv_idx_scan_iter_param.output_ = &ir_scan_ctdef->get_inv_idx_scan_ctdef()->result_output_; ObDASScanIterParam inv_idx_agg_iter_param; ObDASScanIter *inv_idx_agg_iter = nullptr; inv_idx_agg_iter_param.scan_ctdef_ = ir_scan_ctdef->get_inv_idx_agg_ctdef(); + inv_idx_agg_iter_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_; + inv_idx_agg_iter_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_; + inv_idx_agg_iter_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_; + inv_idx_agg_iter_param.output_ = &ir_scan_ctdef->get_inv_idx_agg_ctdef()->result_output_; ObDASScanIterParam fwd_idx_iter_param; ObDASScanIter *fwd_idx_iter = nullptr; fwd_idx_iter_param.scan_ctdef_ = ir_scan_ctdef->get_fwd_idx_agg_ctdef(); + fwd_idx_iter_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_; + fwd_idx_iter_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_; + fwd_idx_iter_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_; + fwd_idx_iter_param.output_ = &ir_scan_ctdef->get_fwd_idx_agg_ctdef()->result_output_; if (OB_FAIL(create_das_iter(alloc, inv_idx_scan_iter_param, inv_idx_scan_iter))) { LOG_WARN("failed to create inv idx iter", K(ret)); } else if (ir_scan_ctdef->need_inv_idx_agg() @@ -569,16 +666,21 @@ int ObDASIterUtils::create_vid_scan_sub_tree( ObDASIter *&iter_tree) { int ret = OB_SUCCESS; - if (OB_ISNULL(merge_ctdef) || OB_UNLIKELY(2 != merge_ctdef->children_cnt_) + if (OB_ISNULL(merge_ctdef) || OB_ISNULL(merge_rtdef) || OB_UNLIKELY(2 != merge_ctdef->children_cnt_) || OB_ISNULL(iter_tree) || OB_UNLIKELY(ObDASIterType::DAS_ITER_SCAN != iter_tree->get_type())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid arguments", K(ret), KPC(merge_ctdef), KPC(iter_tree)); + LOG_WARN("invalid arguments", K(ret), KPC(merge_ctdef), KPC(merge_rtdef), KPC(iter_tree)); } else { ObDASVIdMergeIterParam vid_merge_param; ObDASVIdMergeIter *vid_merge_iter = nullptr; ObDASScanIterParam rowkey_vid_param; ObDASScanIter *rowkey_vid_iter = nullptr; rowkey_vid_param.scan_ctdef_ = static_cast(merge_ctdef->children_[1]); + rowkey_vid_param.max_size_ = merge_rtdef->eval_ctx_->is_vectorized() ? + merge_rtdef->eval_ctx_->max_batch_size_ : 1; + rowkey_vid_param.eval_ctx_ = merge_rtdef->eval_ctx_; + rowkey_vid_param.exec_ctx_ = &merge_rtdef->eval_ctx_->exec_ctx_; + rowkey_vid_param.output_ = &rowkey_vid_param.scan_ctdef_->result_output_; if (OB_FAIL(create_das_iter(alloc, rowkey_vid_param, rowkey_vid_iter))) { LOG_WARN("fail to create das scan iter", K(ret), K(rowkey_vid_param)); } else { @@ -634,7 +736,10 @@ int ObDASIterUtils::create_domain_lookup_sub_tree(ObTableScanParam &scan_param, ObDASScanIter *doc_id_table_iter = nullptr; ObDASScanIterParam doc_id_table_param; doc_id_table_param.scan_ctdef_ = aux_lookup_ctdef->get_lookup_scan_ctdef(); - + doc_id_table_param.max_size_ = 1; + doc_id_table_param.eval_ctx_ = aux_lookup_rtdef->eval_ctx_; + doc_id_table_param.exec_ctx_ = &aux_lookup_rtdef->eval_ctx_->exec_ctx_; + doc_id_table_param.output_ = &aux_lookup_ctdef->result_output_; if (OB_FAIL(create_das_iter(alloc, doc_id_table_param, doc_id_table_iter))) { LOG_WARN("failed to create doc id table iter", K(ret)); } else { @@ -674,6 +779,10 @@ int ObDASIterUtils::create_domain_lookup_sub_tree(ObTableScanParam &scan_param, if (OB_SUCC(ret)) { ObDASScanIterParam data_table_param; data_table_param.scan_ctdef_ = table_lookup_ctdef->get_lookup_scan_ctdef(); + data_table_param.max_size_ = 1; + data_table_param.eval_ctx_ = table_lookup_rtdef->eval_ctx_; + data_table_param.exec_ctx_ = &table_lookup_rtdef->eval_ctx_->exec_ctx_; + data_table_param.output_ = &table_lookup_ctdef->result_output_; if (OB_FAIL(create_das_iter(alloc, data_table_param, data_table_iter))) { LOG_WARN("failed to create data table lookup scan iter", K(ret)); } else { @@ -898,6 +1007,215 @@ int ObDASIterUtils::create_global_lookup_iter_tree(const ObTableScanCtDef &tsc_c return ret; } +int ObDASIterUtils::create_index_merge_iter_tree(ObTableScanParam &scan_param, + common::ObIAllocator &alloc, + const ObDASBaseCtDef *attach_ctdef, + ObDASBaseRtDef *attach_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *tx_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&iter_tree) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(attach_ctdef) || OB_ISNULL(attach_rtdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ret)); + } else { + const bool need_lookup = attach_ctdef->op_type_ == DAS_OP_TABLE_LOOKUP; + const ObDASBaseCtDef *index_merge_ctdef = need_lookup ? attach_ctdef->children_[0] : attach_ctdef; + ObDASBaseRtDef *index_merge_rtdef = need_lookup ? attach_rtdef->children_[0] : attach_rtdef; + ObDASIter *index_merge_root = nullptr; + if (OB_FAIL(create_index_merge_sub_tree(scan_param.ls_id_, + alloc, + index_merge_ctdef, + index_merge_rtdef, + related_tablet_ids, + tx_desc, + snapshot, + index_merge_root))) { + LOG_WARN("failed to create index merge iter tree", K(ret)); + } else if (need_lookup) { + ObDASLocalLookupIter *lookup_iter = nullptr; + ObDASScanIter *data_table_iter = nullptr; + const ObDASScanCtDef *lookup_ctdef = static_cast(attach_ctdef->children_[1]); + ObDASScanRtDef *lookup_rtdef = static_cast(attach_rtdef->children_[1]); + ObDASScanIterParam data_table_param; + data_table_param.scan_ctdef_ = lookup_ctdef; + data_table_param.max_size_ = lookup_rtdef->eval_ctx_->is_vectorized() ? lookup_rtdef->eval_ctx_->max_batch_size_ : 1; + data_table_param.eval_ctx_ = lookup_rtdef->eval_ctx_; + data_table_param.exec_ctx_ = &lookup_rtdef->eval_ctx_->exec_ctx_; + data_table_param.output_ = &lookup_ctdef->result_output_; + if (OB_FAIL(create_das_iter(alloc, data_table_param, data_table_iter))) { + LOG_WARN("failed to create data table iter", K(ret)); + } else { + ObDASLocalLookupIterParam lookup_param; + lookup_param.max_size_ = lookup_rtdef->eval_ctx_->is_vectorized() ? lookup_rtdef->eval_ctx_->max_batch_size_ : 1; + lookup_param.eval_ctx_ = lookup_rtdef->eval_ctx_; + lookup_param.exec_ctx_ = &lookup_rtdef->eval_ctx_->exec_ctx_; + lookup_param.output_ = &lookup_ctdef->result_output_; + lookup_param.default_batch_row_count_ = 1000; // hard code 1000 for local lookup + lookup_param.index_ctdef_ = index_merge_ctdef; + lookup_param.index_rtdef_ = index_merge_rtdef; + lookup_param.lookup_ctdef_ = lookup_ctdef; + lookup_param.lookup_rtdef_ = lookup_rtdef; + lookup_param.index_table_iter_ = index_merge_root; + lookup_param.data_table_iter_ = data_table_iter; + lookup_param.trans_desc_ = tx_desc; + lookup_param.snapshot_ = snapshot; + lookup_param.rowkey_exprs_ = &lookup_ctdef->rowkey_exprs_; + if (OB_FAIL(create_das_iter(alloc, lookup_param, lookup_iter))) { + LOG_WARN("failed to create local lookup iter", K(ret)); + } else if (OB_FAIL(create_iter_children_array(2, alloc, lookup_iter))) { + LOG_WARN("failed to create iter children array", K(ret)); + } else { + lookup_iter->get_children()[0] = index_merge_root; + lookup_iter->get_children()[1] = data_table_iter; + data_table_iter->set_scan_param(lookup_iter->get_lookup_param()); + lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_); + lookup_iter->set_ls_id(scan_param.ls_id_); + iter_tree = lookup_iter; + } + } + } else { + iter_tree = index_merge_root; + } + } + + return ret; +} + +int ObDASIterUtils::create_index_merge_sub_tree(const ObLSID &ls_id, + common::ObIAllocator &alloc, + const ObDASBaseCtDef *ctdef, + ObDASBaseRtDef *rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *tx_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&iter) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ctdef), K(rtdef)); + } else if (ctdef->op_type_ == DAS_OP_TABLE_SCAN) { + ObDASScanIterParam scan_param; + scan_param.scan_ctdef_ = static_cast(ctdef); + scan_param.max_size_ = rtdef->eval_ctx_->is_vectorized() ? rtdef->eval_ctx_->max_batch_size_ : 1; + scan_param.eval_ctx_ = rtdef->eval_ctx_; + scan_param.exec_ctx_ = &rtdef->eval_ctx_->exec_ctx_; + scan_param.output_ = &scan_param.scan_ctdef_->result_output_; + ObDASScanIter *scan_iter = nullptr; + if (OB_FAIL(create_das_iter(alloc, scan_param, scan_iter))) { + LOG_WARN("failed to create das scan iter", K(ret)); + } else { + iter = scan_iter; + } + } else if (ctdef->op_type_ == DAS_OP_SORT) { + const ObDASSortCtDef *sort_ctdef = static_cast(ctdef); + ObDASSortRtDef *sort_rtdef = static_cast(rtdef); + OB_ASSERT(ctdef->children_ != nullptr && + ctdef->children_cnt_ == 1 && + ctdef->children_[0]->op_type_ == DAS_OP_TABLE_SCAN); + const ObDASScanCtDef *scan_ctdef = static_cast(ctdef->children_[0]); + ObDASScanIterParam child_scan_param; + child_scan_param.scan_ctdef_ = scan_ctdef; + child_scan_param.max_size_ = rtdef->eval_ctx_->is_vectorized() ? rtdef->eval_ctx_->max_batch_size_ : 1; + child_scan_param.eval_ctx_ = rtdef->eval_ctx_; + child_scan_param.exec_ctx_ = &rtdef->eval_ctx_->exec_ctx_; + child_scan_param.output_ = &scan_ctdef->result_output_; + ObDASScanIter *child_scan_iter = nullptr; + ObDASIter *sort_iter = nullptr; + if (OB_FAIL(create_das_iter(alloc, child_scan_param, child_scan_iter))) { + LOG_WARN("failed to create das scan iter", K(ret)); + } else if (OB_FAIL(create_sort_sub_tree(alloc, sort_ctdef, sort_rtdef, child_scan_iter, sort_iter))) { + LOG_WARN("failed to create das sort iter", K(ret)); + } else { + iter = sort_iter; + } + } else if (ctdef->op_type_ == DAS_OP_INDEX_MERGE) { + const ObDASIndexMergeCtDef *merge_ctdef = static_cast(ctdef); + ObDASIndexMergeRtDef *merge_rtdef = static_cast(rtdef); + const ObDASBaseCtDef *left_ctdef = merge_ctdef->children_[0]; + const ObDASBaseCtDef *right_ctdef = merge_ctdef->children_[1]; + const ObDASScanCtDef *right_scan_ctdef = nullptr; + OB_ASSERT(right_ctdef->op_type_ == DAS_OP_TABLE_SCAN || right_ctdef->op_type_ == DAS_OP_SORT); + if (right_ctdef->op_type_ == DAS_OP_TABLE_SCAN) { + right_scan_ctdef = static_cast(right_ctdef); + } else { + OB_ASSERT(right_ctdef->children_ != nullptr && + right_ctdef->children_cnt_ == 1 && + right_ctdef->children_[0]->op_type_ == DAS_OP_TABLE_SCAN); + right_scan_ctdef = static_cast(right_ctdef->children_[0]); + } + OB_ASSERT(right_scan_ctdef != nullptr); + ObDASIter *left_iter = nullptr; + ObDASIter *right_iter = nullptr; + if (OB_FAIL(create_index_merge_sub_tree(ls_id, + alloc, + merge_ctdef->children_[0], + merge_rtdef->children_[0], + related_tablet_ids, + tx_desc, + snapshot, + left_iter))) { + LOG_WARN("failed to create index merge sub tree", K(ret)); + } else if (OB_FAIL(create_index_merge_sub_tree(ls_id, + alloc, + merge_ctdef->children_[1], + merge_rtdef->children_[1], + related_tablet_ids, + tx_desc, + snapshot, + right_iter))) { + LOG_WARN("failed to create index merge sub tree", K(ret)); + } else { + ObDASIndexMergeIterParam merge_param; + ObDASIndexMergeIter *merge_iter = nullptr; + merge_param.max_size_ = merge_rtdef->eval_ctx_->is_vectorized() ? + merge_rtdef->eval_ctx_->max_batch_size_ : 1; + merge_param.eval_ctx_ = merge_rtdef->eval_ctx_; + merge_param.exec_ctx_ = &merge_rtdef->eval_ctx_->exec_ctx_; + merge_param.output_ = &right_scan_ctdef->rowkey_exprs_; + merge_param.merge_type_ = merge_ctdef->merge_type_; + merge_param.rowkey_exprs_ = &right_scan_ctdef->rowkey_exprs_; + merge_param.left_iter_ = left_iter; + merge_param.left_output_ = left_iter->get_output(); + merge_param.right_iter_ = right_iter; + merge_param.right_output_ = right_iter->get_output(); + merge_param.ctdef_ = merge_ctdef; + merge_param.rtdef_ = merge_rtdef; + merge_param.tx_desc_ = tx_desc; + merge_param.snapshot_ = snapshot; + merge_param.is_reverse_ = merge_ctdef->is_reverse_; + if (OB_FAIL(create_das_iter(alloc, merge_param, merge_iter))) { + LOG_WARN("failed to create index merge iter", K(merge_param)); + } else if (OB_FAIL(merge_iter->set_ls_tablet_ids(ls_id, related_tablet_ids))) { + LOG_WARN("failed to set ls tablet ids", K(ret)); + } else if (OB_FAIL(create_iter_children_array(2, alloc, merge_iter))) { + LOG_WARN("failed to create iter children array", K(ret)); + } else { + merge_iter->get_children()[0] = left_iter; + merge_iter->get_children()[1] = right_iter; + if (merge_ctdef->is_left_child_leaf_node_) { + OB_ASSERT(left_iter->get_type() == DAS_ITER_SCAN || left_iter->get_type() == DAS_ITER_SORT); + ObDASScanIter *left_scan_iter = (left_iter->get_type() == DAS_ITER_SCAN) ? + static_cast(left_iter) : static_cast(left_iter->get_children()[0]); + OB_ASSERT(left_scan_iter != nullptr); + left_scan_iter->set_scan_param(merge_iter->get_left_param()); + } + OB_ASSERT(right_iter->get_type() == DAS_ITER_SCAN || right_iter->get_type() == DAS_ITER_SORT); + ObDASScanIter *right_scan_iter = (right_iter->get_type() == DAS_ITER_SCAN) ? + static_cast(right_iter) : static_cast(right_iter->get_children()[0]); + OB_ASSERT(right_scan_iter != nullptr); + right_scan_iter->set_scan_param(merge_iter->get_right_param()); + iter = merge_iter; + } + } + } + + return ret; +} + int ObDASIterUtils::create_iter_children_array(const int64_t children_cnt, common::ObIAllocator &alloc, ObDASIter *iter) diff --git a/src/sql/das/iter/ob_das_iter_utils.h b/src/sql/das/iter/ob_das_iter_utils.h index b157817ff..d1bc8688f 100644 --- a/src/sql/das/iter/ob_das_iter_utils.h +++ b/src/sql/das/iter/ob_das_iter_utils.h @@ -23,6 +23,7 @@ #include "sql/das/iter/ob_das_text_retrieval_iter.h" #include "sql/das/iter/ob_das_text_retrieval_merge_iter.h" #include "sql/das/iter/ob_das_vid_merge_iter.h" +#include "sql/das/iter/ob_das_index_merge_iter.h" #include "sql/engine/table/ob_table_scan_op.h" namespace oceanbase @@ -73,6 +74,11 @@ public: const ObLSID &ls_id, ObDASIter *root_iter); + static int set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctdef, + const ObDASRelatedTabletID &related_tablet_ids, + const ObLSID &ls_id, + ObDASIter *root_iter); + private: static int create_partition_scan_tree(ObTableScanParam &scan_param, common::ObIAllocator &alloc, @@ -166,6 +172,24 @@ private: ObDASMergeIter *&scan_iter, ObDASIter *&iter_tree); + static int create_index_merge_iter_tree(ObTableScanParam &scan_param, + common::ObIAllocator &alloc, + const ObDASBaseCtDef *attach_ctdef, + ObDASBaseRtDef *attach_rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *tx_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&iter_tree); + + static int create_index_merge_sub_tree(const ObLSID &ls_id, + common::ObIAllocator &alloc, + const ObDASBaseCtDef *ctdef, + ObDASBaseRtDef *rtdef, + const ObDASRelatedTabletID &related_tablet_ids, + transaction::ObTxDesc *tx_desc, + transaction::ObTxReadSnapshot *snapshot, + ObDASIter *&iter); + static int create_iter_children_array(const int64_t children_cnt, common::ObIAllocator &alloc, ObDASIter *iter); diff --git a/src/sql/das/iter/ob_das_lookup_iter.cpp b/src/sql/das/iter/ob_das_lookup_iter.cpp index 887e0232d..ac0444599 100644 --- a/src/sql/das/iter/ob_das_lookup_iter.cpp +++ b/src/sql/das/iter/ob_das_lookup_iter.cpp @@ -104,7 +104,7 @@ int ObDASLookupIter::inner_get_next_row() while (OB_SUCC(ret) && !index_end_ && lookup_rowkey_cnt_ < default_row_batch_cnt) { index_table_iter_->clear_evaluated_flag(); if (OB_FAIL(index_table_iter_->get_next_row())) { - if(OB_UNLIKELY(OB_ITER_END != ret)) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("failed to get next row from index table", K(ret)); } else { index_end_ = true; diff --git a/src/sql/das/iter/ob_das_merge_iter.cpp b/src/sql/das/iter/ob_das_merge_iter.cpp index 90068cfdb..f2781e21f 100644 --- a/src/sql/das/iter/ob_das_merge_iter.cpp +++ b/src/sql/das/iter/ob_das_merge_iter.cpp @@ -127,18 +127,18 @@ void MergeStoreRows::reuse() void MergeStoreRows::reset() { - exprs_ = nullptr; - eval_ctx_ = nullptr; - group_id_idx_ = OB_INVALID_INDEX; - max_size_ = 1; - saved_size_ = 0; - cur_idx_ = OB_INVALID_INDEX; if (OB_NOT_NULL(store_rows_)) { for (int64_t i = 0; i < max_size_; i++) { store_rows_[i].~LastDASStoreRow(); } store_rows_ = nullptr; } + exprs_ = nullptr; + eval_ctx_ = nullptr; + group_id_idx_ = OB_INVALID_INDEX; + max_size_ = 1; + saved_size_ = 0; + cur_idx_ = OB_INVALID_INDEX; } int ObDASMergeIter::set_merge_status(MergeType merge_type) @@ -268,7 +268,7 @@ int ObDASMergeIter::do_table_scan() LOG_WARN("failed to push back das task ptr", K(ret)); } } // for end - LOG_DEBUG("[DAS ITER] do table scan", K(ref_table_id_), K(das_tasks_arr_.count())); + LOG_DEBUG("[DAS ITER] merge iter do table scan", K(ref_table_id_), K(das_tasks_arr_.count())); } return ret; } @@ -382,7 +382,7 @@ int ObDASMergeIter::inner_get_next_rows(int64_t &count, int64_t capacity) LOG_WARN("das merge iter failed to get next rows", K(ret)); } } - LOG_DEBUG("das merge iter get next rows end", K(count), K(merge_type_), K(merge_state_arr_), K(ret)); + LOG_DEBUG("[DAS ITER] merge iter get next rows end", K(count), K(merge_type_), K(merge_state_arr_), K(ret)); const ObBitVector *skip = nullptr; PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_, count, skip); return ret; @@ -811,7 +811,7 @@ int ObDASMergeIter::compare(int64_t cur_idx, int64_t &output_idx) } } } - LOG_DEBUG("das merge iter compare finished", K(cur_idx), K(output_idx), K(used_for_keep_order_)); + LOG_DEBUG("[DAS ITER] merge iter compare finished", K(cur_idx), K(output_idx), K(used_for_keep_order_)); return ret; } diff --git a/src/sql/das/iter/ob_das_merge_iter.h b/src/sql/das/iter/ob_das_merge_iter.h index 04df78554..a7148777c 100644 --- a/src/sql/das/iter/ob_das_merge_iter.h +++ b/src/sql/das/iter/ob_das_merge_iter.h @@ -27,7 +27,18 @@ struct ObDASMergeIterParam : public ObDASIterParam { public: ObDASMergeIterParam() - : ObDASIterParam(ObDASIterType::DAS_ITER_MERGE) + : ObDASIterParam(ObDASIterType::DAS_ITER_MERGE), + eval_infos_(nullptr), + need_update_partition_id_(false), + pdml_partition_id_(nullptr), + partition_id_calc_type_(0), + should_scan_index_(false), + ref_table_id_(), + is_vectorized_(false), + frame_info_(nullptr), + execute_das_directly_(false), + enable_rich_format_(false), + used_for_keep_order_(false) {} ObFixedArray *eval_infos_; bool need_update_partition_id_; diff --git a/src/sql/das/iter/ob_das_scan_iter.cpp b/src/sql/das/iter/ob_das_scan_iter.cpp index 5d388d40e..88f130b17 100644 --- a/src/sql/das/iter/ob_das_scan_iter.cpp +++ b/src/sql/das/iter/ob_das_scan_iter.cpp @@ -83,7 +83,7 @@ int ObDASScanIter::do_table_scan() LOG_WARN("fail to scan table", KPC_(scan_param), K(ret)); } } - LOG_DEBUG("das scan iter do table scan", KPC_(scan_param), K(ret)); + LOG_DEBUG("[DAS ITER] scan iter do table scan", KPC_(scan_param), K(ret)); return ret; } @@ -100,7 +100,7 @@ int ObDASScanIter::rescan() // reset need_switch_param_ after real rescan. scan_param_->need_switch_param_ = false; } - LOG_DEBUG("das scan iter rescan", KPC_(scan_param), K(ret)); + LOG_DEBUG("[DAS ITER] das scan iter rescan", KPC_(scan_param), K(ret)); return ret; } @@ -130,6 +130,9 @@ int ObDASScanIter::inner_get_next_rows(int64_t &count, int64_t capacity) LOG_WARN("failed to get next row", K(ret)); } } + LOG_TRACE("[DAS ITER] scan iter get next rows", K(count), K(capacity), KPC_(scan_param), K(ret)); + const ObBitVector *skip = nullptr; + PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_, count, skip); return ret; } diff --git a/src/sql/das/iter/ob_das_scan_iter.h b/src/sql/das/iter/ob_das_scan_iter.h index c3053a41f..839a8e01c 100644 --- a/src/sql/das/iter/ob_das_scan_iter.h +++ b/src/sql/das/iter/ob_das_scan_iter.h @@ -31,7 +31,7 @@ public: const ObDASScanCtDef *scan_ctdef_; virtual bool is_valid() const override { - return nullptr != scan_ctdef_; + return nullptr != scan_ctdef_ && ObDASIterParam::is_valid(); } }; diff --git a/src/sql/das/iter/ob_das_sort_iter.cpp b/src/sql/das/iter/ob_das_sort_iter.cpp index df34d33f9..33ff714a1 100644 --- a/src/sql/das/iter/ob_das_sort_iter.cpp +++ b/src/sql/das/iter/ob_das_sort_iter.cpp @@ -197,21 +197,39 @@ int ObDASSortIter::inner_get_next_rows(int64_t &count, int64_t capacity) } else if (!sort_finished_ && OB_FAIL(do_sort(true))) { LOG_WARN("failed to do sort", K(ret)); } else { - bool got_rows = false; - // TODO: @bingfan use vectorized interface instead. - while (OB_SUCC(ret) && !got_rows) { - if (OB_FAIL(sort_impl_.get_next_row(sort_row_))) { + if (input_row_cnt_ == 0 && limit_param_.limit_ > 0 && limit_param_.offset_ > 0) { + int64_t need_offset_count = limit_param_.offset_; + while (OB_SUCC(ret) && need_offset_count > 0) { + int64_t got_count = 0; + if (OB_FAIL(sort_impl_.get_next_batch(sort_row_, need_offset_count, got_count))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("failed to get next rows from child iter", K(ret)); + } + } + input_row_cnt_ += got_count; + need_offset_count = need_offset_count - got_count; + got_count = 0; + } + if (OB_SUCC(ret)) { + if (OB_LIKELY(need_offset_count == 0) && OB_LIKELY(input_row_cnt_ == limit_param_.offset_)) { + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected need offset count", K(ret), K(need_offset_count), K(input_row_cnt_), K(limit_param_.offset_)); + } + } + } + if (OB_SUCC(ret)) { + int64_t min_capacity = limit_param_.limit_ > 0 ? OB_MIN(capacity, limit_param_.limit_ - output_row_cnt_) : capacity; + if (OB_FAIL(sort_impl_.get_next_batch(sort_row_, min_capacity, count))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("failed to get next row from sort impl", K(ret)); - } - } else { - ++input_row_cnt_; - if (input_row_cnt_ > limit_param_.offset_) { - got_rows = true; - count = 1; - ++output_row_cnt_; + LOG_WARN("failed to get next rows from child iter", K(ret)); } } + output_row_cnt_ += count; + if (OB_FAIL(ret)) { + } else if (limit_param_.limit_ > 0 && (output_row_cnt_ >= limit_param_.limit_)) { + ret = OB_ITER_END; + } } } return ret; diff --git a/src/sql/das/ob_das_attach_define.cpp b/src/sql/das/ob_das_attach_define.cpp index 11bc9853f..e083f993f 100644 --- a/src/sql/das/ob_das_attach_define.cpp +++ b/src/sql/das/ob_das_attach_define.cpp @@ -71,6 +71,22 @@ OB_SERIALIZE_MEMBER((ObDASVIdMergeCtDef, ObDASAttachCtDef)); OB_SERIALIZE_MEMBER((ObDASVIdMergeRtDef, ObDASAttachRtDef)); +const ObDASBaseCtDef *ObDASIndexMergeCtDef::get_left_ctdef() const +{ + OB_ASSERT(2 == children_cnt_ && children_ != nullptr); + return children_[0]; +} + +const ObDASBaseCtDef *ObDASIndexMergeCtDef::get_right_ctdef() const +{ + OB_ASSERT(2 == children_cnt_ && children_ != nullptr); + return children_[1]; +} + +OB_SERIALIZE_MEMBER((ObDASIndexMergeCtDef, ObDASAttachCtDef), merge_type_, is_left_child_leaf_node_, is_reverse_); + +OB_SERIALIZE_MEMBER((ObDASIndexMergeRtDef, ObDASAttachRtDef)); + OB_DEF_SERIALIZE(ObDASAttachSpec) { int ret = OB_SUCCESS; diff --git a/src/sql/das/ob_das_attach_define.h b/src/sql/das/ob_das_attach_define.h index fa85b9524..f28ea2c19 100644 --- a/src/sql/das/ob_das_attach_define.h +++ b/src/sql/das/ob_das_attach_define.h @@ -19,6 +19,7 @@ #include "share/ob_define.h" #include "sql/engine/expr/ob_expr.h" #include "sql/engine/sort/ob_sort_basic_info.h" +#include "sql/optimizer/ob_join_order.h" namespace oceanbase { @@ -144,6 +145,36 @@ public: INHERIT_TO_STRING_KV("ObDASVIdMergeRtDef", ObDASAttachRtDef, KP(this)); }; +struct ObDASIndexMergeCtDef : ObDASAttachCtDef +{ + OB_UNIS_VERSION(1); +public: + ObDASIndexMergeCtDef(common::ObIAllocator &alloc) + : ObDASAttachCtDef(alloc, DAS_OP_INDEX_MERGE), + merge_type_(INDEX_MERGE_INVALID), + is_left_child_leaf_node_(false), + is_reverse_(false) + {} + + virtual ~ObDASIndexMergeCtDef() {} + const ObDASBaseCtDef *get_left_ctdef() const; + const ObDASBaseCtDef *get_right_ctdef() const; +public: + ObIndexMergeType merge_type_; + bool is_left_child_leaf_node_; + bool is_reverse_; +}; + +struct ObDASIndexMergeRtDef : ObDASAttachRtDef +{ + OB_UNIS_VERSION(1); +public: + ObDASIndexMergeRtDef() + : ObDASAttachRtDef(DAS_OP_INDEX_MERGE) {} + + virtual ~ObDASIndexMergeRtDef() {} +}; + struct ObDASAttachSpec { OB_UNIS_VERSION(1); @@ -162,6 +193,12 @@ public: const ObDASTableLocMeta *get_attach_loc_meta(int64_t table_location_id, int64_t ref_table_id) const; int set_calc_exprs(const ExprFixedArray &calc_exprs, const int64_t max_batch_size); + const ExprFixedArray &get_result_output() const + { + OB_ASSERT(attach_ctdef_ != nullptr); + return static_cast(attach_ctdef_)->result_output_; + } + TO_STRING_KV(K_(attach_loc_metas), K_(attach_ctdef)); private: diff --git a/src/sql/das/ob_das_def_reg.h b/src/sql/das/ob_das_def_reg.h index 279a737f9..e4acdec46 100644 --- a/src/sql/das/ob_das_def_reg.h +++ b/src/sql/das/ob_das_def_reg.h @@ -147,6 +147,10 @@ struct ObDASVIdMergeRtDef; REGISTER_DAS_ATTACH_OP(DAS_OP_VID_MERGE, ObDASVIdMergeCtDef, ObDASVIdMergeRtDef); +struct ObDASIndexMergeCtDef; +struct ObDASIndexMergeRtDef; +REGISTER_DAS_ATTACH_OP(DAS_OP_INDEX_MERGE, ObDASIndexMergeCtDef, ObDASIndexMergeRtDef); + #undef REGISTER_DAS_ATTACH_OP } // namespace sql } // namespace oceanbase diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index 683ac31ca..f3e0f5fec 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -23,6 +23,7 @@ #include "storage/concurrency_control/ob_data_validation_service.h" #include "sql/das/iter/ob_das_iter_utils.h" + namespace oceanbase { namespace common @@ -45,7 +46,7 @@ using namespace storage; using namespace transaction; namespace sql { -OB_SERIALIZE_MEMBER(ObDASScanCtDef, +OB_SERIALIZE_MEMBER(ObDASScanCtDef, // FARM COMPAT WHITELIST ref_table_id_, access_column_ids_, schema_version_, @@ -68,7 +69,9 @@ OB_SERIALIZE_MEMBER(ObDASScanCtDef, doc_id_idx_, vec_vid_idx_, multivalue_idx_, - multivalue_type_); + multivalue_type_, + index_merge_idx_, // FARM COMPAT WHITELIST + flags_); // FARM COMPAT WHITELIST OB_DEF_SERIALIZE(ObDASScanRtDef) { @@ -87,7 +90,9 @@ OB_DEF_SERIALIZE(ObDASScanRtDef) pd_storage_flag_, need_check_output_datum_, is_for_foreign_check_, - fb_read_tx_uncommitted_); + fb_read_tx_uncommitted_, + key_ranges_, + ss_key_ranges_); return ret; } @@ -108,7 +113,9 @@ OB_DEF_DESERIALIZE(ObDASScanRtDef) pd_storage_flag_, need_check_output_datum_, is_for_foreign_check_, - fb_read_tx_uncommitted_); + fb_read_tx_uncommitted_, + key_ranges_, + ss_key_ranges_); if (OB_SUCC(ret)) { (void)ObSQLUtils::adjust_time_by_ntp_offset(timeout_ts_); } @@ -132,7 +139,9 @@ OB_DEF_SERIALIZE_SIZE(ObDASScanRtDef) pd_storage_flag_, need_check_output_datum_, is_for_foreign_check_, - fb_read_tx_uncommitted_); + fb_read_tx_uncommitted_, + key_ranges_, + ss_key_ranges_); return len; } @@ -179,6 +188,7 @@ ObDASScanOp::ObDASScanOp(ObIAllocator &op_alloc) scan_rtdef_(nullptr), result_(nullptr), remain_row_cnt_(0), + tablet_ids_(op_alloc), retry_alloc_(nullptr), ir_param_() { @@ -351,13 +361,33 @@ ObDASIterTreeType ObDASScanOp::get_iter_tree_type() const tree_type = ObDASIterTreeType::ITER_TREE_GIS_LOOKUP; } else if (is_multivalue_index || is_vector_index) { tree_type = ObDASIterTreeType::ITER_TREE_DOMAIN_LOOKUP; + } else if (OB_UNLIKELY(is_index_merge(attach_ctdef_))) { + tree_type = ObDASIterTreeType::ITER_TREE_INDEX_MERGE; } else { tree_type = OB_ISNULL(get_lookup_ctdef()) ? ObDASIterTreeType::ITER_TREE_PARTITION_SCAN : ObDASIterTreeType::ITER_TREE_LOCAL_LOOKUP; } + LOG_TRACE("get iter tree type", K(tree_type), K(scan_param_), KPC(attach_ctdef_)); return tree_type; } +bool ObDASScanOp::is_index_merge(const ObDASBaseCtDef *attach_ctdef) const +{ + bool bret = false; + if (attach_ctdef != nullptr) { + if (attach_ctdef->op_type_ == DAS_OP_INDEX_MERGE) { + bret = true; + } else if (attach_ctdef->op_type_ == DAS_OP_TABLE_LOOKUP) { + const ObDASBaseCtDef *rowkey_scan_ctdef = + static_cast(attach_ctdef)->get_rowkey_scan_ctdef(); + if (rowkey_scan_ctdef != nullptr) { + bret = rowkey_scan_ctdef->op_type_ == DAS_OP_INDEX_MERGE; + } + } + } + return bret; +} + int ObDASScanOp::init_related_tablet_ids(ObDASRelatedTabletID &related_tablet_ids) { int ret = OB_SUCCESS; @@ -373,6 +403,8 @@ int ObDASScanOp::init_related_tablet_ids(ObDASRelatedTabletID &related_tablet_id related_tablet_ids.fwd_idx_tablet_id_, related_tablet_ids.doc_id_idx_tablet_id_))) { LOG_WARN("failed to get text ir tablet ids", K(ret)); + } else if (OB_FAIL(get_index_merge_tablet_ids(related_tablet_ids.index_merge_tablet_ids_))) { + LOG_WARN("failed to get index merge tablet ids", K(ret)); } return ret; } @@ -468,7 +500,7 @@ int ObDASScanOp::open_op() LOG_WARN("init scan param failed", K(ret)); } else if (FALSE_IT(tree_type = get_iter_tree_type())) { } else if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type - || ITER_TREE_TEXT_RETRIEVAL == tree_type) { + || ITER_TREE_TEXT_RETRIEVAL == tree_type || ITER_TREE_INDEX_MERGE == tree_type) { ObDASIter *result = nullptr; if (OB_FAIL(init_related_tablet_ids(tablet_ids_))) { LOG_WARN("failed to init related tablet ids", K(ret)); @@ -516,7 +548,7 @@ int ObDASScanOp::release_op() int ret = OB_SUCCESS; ObDASIterTreeType tree_type = get_iter_tree_type(); if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type - || ITER_TREE_TEXT_RETRIEVAL == tree_type) { + || ITER_TREE_TEXT_RETRIEVAL == tree_type || ITER_TREE_INDEX_MERGE == tree_type) { if (OB_NOT_NULL(result_)) { ObDASIter *result = static_cast(result_); if (OB_FAIL(result->release())) { @@ -744,42 +776,47 @@ void ObDASScanOp::reset_access_datums_ptr(int64_t capacity) if (scan_rtdef_->p_pd_expr_op_->is_vectorized()) { int64_t reset_batch_size = capacity > 0 ? capacity : scan_rtdef_->eval_ctx_->max_batch_size_; reset_batch_size = min(reset_batch_size, scan_rtdef_->eval_ctx_->max_batch_size_); - FOREACH_CNT(e, scan_ctdef_->pd_expr_spec_.access_exprs_) { - (*e)->locate_datums_for_update(*scan_rtdef_->eval_ctx_, reset_batch_size); - ObEvalInfo &info = (*e)->get_eval_info(*scan_rtdef_->eval_ctx_); - info.point_to_frame_ = true; - } - FOREACH_CNT(e, scan_ctdef_->pd_expr_spec_.pd_storage_aggregate_output_) { - (*e)->locate_datums_for_update(*scan_rtdef_->eval_ctx_, scan_rtdef_->eval_ctx_->max_batch_size_); - ObEvalInfo &info = (*e)->get_eval_info(*scan_rtdef_->eval_ctx_); - info.point_to_frame_ = true; - } - - FOREACH_CNT(e, scan_ctdef_->pd_expr_spec_.ext_file_column_exprs_) { - (*e)->locate_datums_for_update(*scan_rtdef_->eval_ctx_, scan_rtdef_->eval_ctx_->max_batch_size_); - ObEvalInfo &info = (*e)->get_eval_info(*scan_rtdef_->eval_ctx_); - info.point_to_frame_ = true; - } - if (OB_NOT_NULL(scan_ctdef_->trans_info_expr_)) { - ObExpr *trans_expr = scan_ctdef_->trans_info_expr_; - trans_expr->locate_datums_for_update(*scan_rtdef_->eval_ctx_, reset_batch_size); - ObEvalInfo &info = trans_expr->get_eval_info(*scan_rtdef_->eval_ctx_); - info.point_to_frame_ = true; + if (attach_rtdef_ != nullptr) { + reset_access_datums_ptr(attach_rtdef_, reset_batch_size); + } else { + reset_access_datums_ptr(scan_rtdef_, reset_batch_size); + if (get_lookup_rtdef() != nullptr) { + reset_access_datums_ptr(get_lookup_rtdef(), reset_batch_size); + } } } - if (get_lookup_rtdef() != nullptr && get_lookup_rtdef()->p_pd_expr_op_->is_vectorized()) { - int64_t reset_batch_size = capacity > 0 ? capacity : scan_rtdef_->eval_ctx_->max_batch_size_; - reset_batch_size = min(reset_batch_size, scan_rtdef_->eval_ctx_->max_batch_size_); - FOREACH_CNT(e, get_lookup_ctdef()->pd_expr_spec_.access_exprs_) { - (*e)->locate_datums_for_update(*get_lookup_rtdef()->eval_ctx_, reset_batch_size); - ObEvalInfo &info = (*e)->get_eval_info(*get_lookup_rtdef()->eval_ctx_); - info.point_to_frame_ = true; - } - if (OB_NOT_NULL(get_lookup_ctdef()->trans_info_expr_)) { - ObExpr *trans_expr = get_lookup_ctdef()->trans_info_expr_; - trans_expr->locate_datums_for_update(*get_lookup_rtdef()->eval_ctx_, reset_batch_size); - ObEvalInfo &info = trans_expr->get_eval_info(*scan_rtdef_->eval_ctx_); - info.point_to_frame_ = true; +} + +void ObDASScanOp::reset_access_datums_ptr(ObDASBaseRtDef *rtdef, int64_t capacity) +{ + if (rtdef != nullptr) { + if (rtdef->op_type_== DAS_OP_TABLE_SCAN) { + const ObDASScanCtDef *scan_ctdef = static_cast(rtdef->ctdef_); + FOREACH_CNT(e, scan_ctdef->pd_expr_spec_.access_exprs_) { + (*e)->locate_datums_for_update(*rtdef->eval_ctx_, capacity); + ObEvalInfo &info = (*e)->get_eval_info(*rtdef->eval_ctx_); + info.point_to_frame_ = true; + } + FOREACH_CNT(e, scan_ctdef->pd_expr_spec_.pd_storage_aggregate_output_) { + (*e)->locate_datums_for_update(*rtdef->eval_ctx_, capacity); + ObEvalInfo &info = (*e)->get_eval_info(*rtdef->eval_ctx_); + info.point_to_frame_ = true; + } + FOREACH_CNT(e, scan_ctdef->pd_expr_spec_.ext_file_column_exprs_) { + (*e)->locate_datums_for_update(*rtdef->eval_ctx_, capacity); + ObEvalInfo &info = (*e)->get_eval_info(*rtdef->eval_ctx_); + info.point_to_frame_ = true; + } + if (scan_ctdef->trans_info_expr_ != nullptr) { + ObExpr *trans_expr = scan_ctdef->trans_info_expr_; + trans_expr->locate_datums_for_update(*rtdef->eval_ctx_, capacity); + ObEvalInfo &info = trans_expr->get_eval_info(*rtdef->eval_ctx_); + info.point_to_frame_ = true; + } + } else { + for (int64_t i = 0; i < rtdef->children_cnt_; i++) { + reset_access_datums_ptr(rtdef->children_[i], capacity); + } } } } @@ -952,7 +989,7 @@ int ObDASScanOp::rescan() "range_pos", scan_param_.range_array_pos_); ObDASIterTreeType tree_type = get_iter_tree_type(); if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type - || ITER_TREE_TEXT_RETRIEVAL == tree_type) { + || ITER_TREE_TEXT_RETRIEVAL == tree_type || ITER_TREE_INDEX_MERGE == tree_type) { if (OB_ISNULL(result_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr das iter tree", K(ret)); @@ -996,7 +1033,7 @@ int ObDASScanOp::reuse_iter() ObITabletScan &tsc_service = get_tsc_service(); ObLocalIndexLookupOp *lookup_op = get_lookup_op(); if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type - || ITER_TREE_TEXT_RETRIEVAL == tree_type) { + || ITER_TREE_TEXT_RETRIEVAL == tree_type || ITER_TREE_INDEX_MERGE == tree_type) { if (OB_ISNULL(result_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected nullptr das iter tree", K(ret)); @@ -1027,6 +1064,14 @@ int ObDASScanOp::reuse_iter() } break; } + case ITER_TREE_INDEX_MERGE: { + ObDASIter *result_iter = static_cast(result_); + if (OB_FAIL(ObDASIterUtils::set_index_merge_related_ids( + attach_ctdef_, tablet_ids_, ls_id_, result_iter))) { + LOG_WARN("failed to set index merge related ids", K(ret)); + } + break; + } default: { ret = OB_ERR_UNEXPECTED; } @@ -1315,6 +1360,30 @@ int ObDASScanOp::get_text_ir_tablet_ids( return ret; } +int ObDASScanOp::get_index_merge_tablet_ids(common::ObIArray &index_merge_tablet_ids) +{ + int ret = OB_SUCCESS; + int64_t N = related_tablet_ids_.count(); + if (OB_UNLIKELY(related_ctdefs_.count() != related_tablet_ids_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected related scan array not match", K(ret), K_(related_ctdefs), K_(related_tablet_ids)); + } else if (OB_FAIL(index_merge_tablet_ids.prepare_allocate(N - 1))) { // need to remove main table + LOG_WARN("failed to prepare allocate index merge tablet ids", K(N), K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) { + const ObDASScanCtDef *ctdef = static_cast(related_ctdefs_.at(i)); + if (OB_ISNULL(ctdef)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KPC(ctdef), K(related_tablet_ids_.count()), K(ret)); + } else if (ctdef->is_index_merge_) { + OB_ASSERT(ctdef->index_merge_idx_ >= 0 && ctdef->index_merge_idx_ < N - 1); + index_merge_tablet_ids.at(ctdef->index_merge_idx_) = related_tablet_ids_.at(i); + } + } + } + return ret; +} + OB_SERIALIZE_MEMBER((ObDASScanOp, ObIDASTaskOp), scan_param_.key_ranges_, scan_ctdef_, diff --git a/src/sql/das/ob_das_scan_op.h b/src/sql/das/ob_das_scan_op.h index acb2b6815..7f78d2dc2 100644 --- a/src/sql/das/ob_das_scan_op.h +++ b/src/sql/das/ob_das_scan_op.h @@ -18,6 +18,7 @@ #include "sql/engine/table/ob_index_lookup_op_impl.h" #include "sql/das/ob_group_scan_iter.h" #include "sql/das/iter/ob_das_iter.h" +#include "sql/rewrite/ob_query_range.h" namespace oceanbase { @@ -54,7 +55,10 @@ public: doc_id_idx_(-1), vec_vid_idx_(-1), multivalue_idx_(-1), - multivalue_type_(0) + multivalue_type_(0), + index_merge_idx_(OB_INVALID_ID), + pre_query_range_(), + flags_(0) { } //in das scan op, column described with column expr virtual bool has_expr() const override { return true; } @@ -91,7 +95,10 @@ public: K_(ir_scan_type), K_(rowkey_exprs), K_(table_scan_opt), - K_(vec_vid_idx)); + K_(vec_vid_idx), + K_(index_merge_idx), + K_(pre_query_range), + K_(is_index_merge)); common::ObTableID ref_table_id_; UIntFixedArray access_column_ids_; int64_t schema_version_; @@ -118,6 +125,15 @@ public: int64_t vec_vid_idx_; int64_t multivalue_idx_; int32_t multivalue_type_; + int64_t index_merge_idx_; // idx of the index scan node in index merge tree + ObQueryRange pre_query_range_; + union { + uint64_t flags_; + struct { + uint64_t is_index_merge_ : 1; // whether used for index merge + uint64_t reserved_ : 63; + }; + }; }; struct ObDASScanRtDef : ObDASBaseRtDef @@ -144,7 +160,10 @@ public: stmt_allocator_("StmtScanAlloc"), scan_allocator_("TableScanAlloc"), sample_info_(nullptr), - is_for_foreign_check_(false) + is_for_foreign_check_(false), + key_ranges_(), + ss_key_ranges_(), + mbr_filters_() { } virtual ~ObDASScanRtDef(); bool enable_rich_format() const { return scan_flag_.enable_rich_format_; } @@ -159,7 +178,10 @@ public: K_(timeout_ts), K_(tx_lock_timeout), K_(sql_mode), - K_(scan_flag)); + K_(scan_flag), + K_(key_ranges), + K_(ss_key_ranges), + K_(mbr_filters)); int init_pd_op(ObExecContext &exec_ctx, const ObDASScanCtDef &scan_ctdef); storage::ObRow2ExprsProjector *p_row2exprs_projector_; ObPushdownOperator *p_pd_expr_op_; @@ -180,6 +202,9 @@ public: common::ObWrapperAllocatorWithAttr scan_allocator_; const common::SampleInfo *sample_info_; //Block(Row)SampleScan, only support local das scan bool is_for_foreign_check_; + common::ObSEArray key_ranges_; + common::ObSEArray ss_key_ranges_; + common::ObSEArray mbr_filters_; private: union { storage::ObRow2ExprsProjector row2exprs_projector_; @@ -240,6 +265,7 @@ public: int rescan(); int reuse_iter(); void reset_access_datums_ptr(int64_t capacity = 0); + void reset_access_datums_ptr(ObDASBaseRtDef *rtdef, int64_t capacity); ObLocalIndexLookupOp *get_lookup_op(); bool is_contain_trans_info() {return NULL != scan_ctdef_->trans_info_expr_; } int do_table_scan(); @@ -254,6 +280,7 @@ public: common::ObTabletID &index_id_tid, common::ObTabletID &snapshot_tid, common::ObTabletID &com_aux_vec_tid); + int get_index_merge_tablet_ids(common::ObIArray &index_merge_tablet_ids); bool enable_rich_format() const { return scan_rtdef_->enable_rich_format(); } INHERIT_TO_STRING_KV("parent", ObIDASTaskOp, KPC_(scan_ctdef), @@ -267,6 +294,7 @@ protected: common::ObNewRowIterator *get_storage_scan_iter(); common::ObNewRowIterator *get_output_result_iter() { return result_; } ObDASIterTreeType get_iter_tree_type() const; + bool is_index_merge(const ObDASBaseCtDef *attach_ctdef) const; public: ObSEArray trans_info_array_; protected: diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index 792dccfcc..c5e8d8012 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -942,7 +942,9 @@ int ObTableScanOp::prepare_all_das_tasks() if (need_perform_real_batch_rescan()) { grp_guard.switch_group_rescan_param(i); } - if (OB_FAIL(prepare_single_scan_range(i))) { + if (MY_CTDEF.use_index_merge_ && OB_FAIL(prepare_index_merge_scan_range(i))) { + LOG_WARN("failed to prepare index merge scan range", K(ret)); + } else if (!MY_CTDEF.use_index_merge_ && OB_FAIL(prepare_single_scan_range(i))) { LOG_WARN("prepare single scan range failed", K(ret)); } else if (OB_FAIL(prepare_das_task())) { LOG_WARN("prepare das task failed", K(ret)); @@ -1172,7 +1174,11 @@ int ObTableScanOp::prepare_scan_range() { int ret = OB_SUCCESS; if (!need_perform_real_batch_rescan()) { - ret = prepare_single_scan_range(); + if (MY_CTDEF.use_index_merge_ && OB_FAIL(prepare_index_merge_scan_range())) { + LOG_WARN("failed to prepare index merge range", K(ret)); + } else if (!MY_CTDEF.use_index_merge_ && OB_FAIL(prepare_single_scan_range())) { + LOG_WARN("failed to prepare single scan range", K(ret)); + } } else { ret = prepare_batch_scan_range(); } @@ -1371,6 +1377,123 @@ int ObTableScanOp::prepare_single_scan_range(int64_t group_idx) return ret; } +// for index merge, disable equal range optimization +int ObTableScanOp::prepare_index_merge_scan_range(int64_t group_idx) +{ + int ret = OB_SUCCESS; + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); + ObIAllocator &range_allocator = (table_rescan_allocator_ != nullptr ? + *table_rescan_allocator_ : ctx_.get_allocator()); + ObDASBaseRtDef *attach_rtdef = tsc_rtdef_.attach_rtinfo_->attach_rtdef_; + if (OB_ISNULL(attach_rtdef)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KPC(attach_rtdef), K(ret)); + } else { + ObDASBaseRtDef *index_merge_rtdef = attach_rtdef->op_type_ == DAS_OP_TABLE_LOOKUP ? + attach_rtdef->children_[0] : attach_rtdef; + if (OB_FAIL(prepare_range_for_each_index(group_idx, range_allocator, index_merge_rtdef))) { + LOG_WARN("failed to prepare range for each index", KPC(index_merge_rtdef), K(ret)); + } + } + return ret; +} + +int ObTableScanOp::prepare_range_for_each_index(int64_t group_idx, + ObIAllocator &allocator, + ObDASBaseRtDef *rtdef) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(rtdef)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KPC(rtdef), K(ret)); + } else if (rtdef->op_type_ == DAS_OP_TABLE_SCAN) { + ObQueryRangeArray key_ranges; + ObQueryRangeArray ss_key_ranges; + ObDASScanRtDef *scan_rtdef = static_cast(rtdef); + const ObDASScanCtDef *scan_ctdef = static_cast(rtdef->ctdef_); + const ObQueryRange &pre_query_range = scan_ctdef->pre_query_range_; + scan_rtdef->key_ranges_.reuse(); + scan_rtdef->ss_key_ranges_.reuse(); + scan_rtdef->mbr_filters_.reuse(); + if (OB_UNLIKELY(!pre_query_range.has_range())) { + // virtual table, do nothing + } else if (pre_query_range.is_contain_geo_filters() && + OB_FAIL(ObSQLUtils::extract_geo_query_range( + pre_query_range, + allocator, + ctx_, + key_ranges, + scan_rtdef->mbr_filters_, + ObBasicSessionInfo::create_dtc_params(ctx_.get_my_session())))) { + LOG_WARN("failed to extract pre query ranges", K(ret)); + } else if (!pre_query_range.is_contain_geo_filters() && + OB_FAIL(ObSQLUtils::extract_pre_query_range( + pre_query_range, + allocator, + ctx_, + key_ranges, + ObBasicSessionInfo::create_dtc_params(ctx_.get_my_session())))) { + LOG_WARN("failed to extract pre query ranges", K(ret)); + } else if (OB_FAIL(pre_query_range.get_ss_tablet_ranges(allocator, + ctx_, + ss_key_ranges, + ObBasicSessionInfo::create_dtc_params(ctx_.get_my_session())))) { + LOG_WARN("failed to final extract index skip query range", K(ret)); + } else if (!ss_key_ranges.empty()) { + // index skip scan, ranges from extract_pre_query_range/get_ss_tablet_ranges, + // prefix range and postfix range is single range + if (1 != ss_key_ranges.count() || 1 != key_ranges.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index skip scan range", K(ret), K(key_ranges), K(ss_key_ranges)); + } else { + key_ranges.at(0)->table_id_ = scan_ctdef->ref_table_id_; + key_ranges.at(0)->group_idx_ = group_idx; + ss_key_ranges.at(0)->table_id_ = scan_ctdef->ref_table_id_; + ss_key_ranges.at(0)->group_idx_ = group_idx; + if (OB_FAIL(scan_rtdef->key_ranges_.push_back(*key_ranges.at(0))) || + OB_FAIL(scan_rtdef->ss_key_ranges_.push_back(*ss_key_ranges.at(0)))) { + LOG_WARN("failed to push back ss key range", KPC(scan_rtdef), K(ret)); + } + } + } else { + ObNewRange whole_range; + ObNewRange *key_range = nullptr; + whole_range.set_whole_range(); + whole_range.table_id_ = scan_ctdef->ref_table_id_; + whole_range.group_idx_ = group_idx; + for (int64_t i = 0; OB_SUCC(ret) && i < key_ranges.count(); ++i) { + key_range = key_ranges.at(i); + key_range->table_id_ = scan_ctdef->ref_table_id_; + key_range->group_idx_ = group_idx; + if (OB_FAIL(scan_rtdef->key_ranges_.push_back(*key_range)) || + OB_FAIL(scan_rtdef->ss_key_ranges_.push_back(whole_range))) { + LOG_WARN("failed to push back key range", KPC(scan_rtdef), K(ret)); + } + } + } + if (OB_SUCC(ret) && MY_SPEC.is_vt_mapping_) { + OZ(vt_result_converter_->convert_key_ranges(scan_rtdef->key_ranges_)); + } + } else if (rtdef->op_type_ == DAS_OP_SORT) { + OB_ASSERT(rtdef->children_ != nullptr && rtdef->children_cnt_ == 1); + if (OB_FAIL(prepare_range_for_each_index(group_idx, allocator, rtdef->children_[0]))) { + LOG_WARN("failed to prepare scan range for child", K(ret)); + } + } else if (rtdef->op_type_ == DAS_OP_INDEX_MERGE) { + OB_ASSERT(rtdef->children_ != nullptr && rtdef->children_cnt_ == 2); + if (OB_FAIL(prepare_range_for_each_index(group_idx, allocator, rtdef->children_[0]))) { + LOG_WARN("failed to prepare scan range for left tree", K(ret)); + } else if (OB_FAIL(prepare_range_for_each_index(group_idx, allocator, rtdef->children_[1]))) { + LOG_WARN("failed to prepare scan range for right tree", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected rtdef type", K(rtdef), K(ret)); + } + + return ret; +} + int ObTableScanOp::single_equal_scan_check_type(const ParamStore ¶m_store, bool& is_same_type) { int ret = OB_SUCCESS; diff --git a/src/sql/engine/table/ob_table_scan_op.h b/src/sql/engine/table/ob_table_scan_op.h index 757426e54..004ef73b2 100644 --- a/src/sql/engine/table/ob_table_scan_op.h +++ b/src/sql/engine/table/ob_table_scan_op.h @@ -137,6 +137,7 @@ struct GroupRescanParamInfo common::ObObjParam cur_param_; //current param in param store, used to restore paramstore state after the completion of group rescan. }; typedef common::ObFixedArray GroupRescanParamArray; + struct ObTableScanCtDef { OB_UNIS_VERSION(1); @@ -158,7 +159,9 @@ public: { } const ExprFixedArray &get_das_output_exprs() const { - return lookup_ctdef_ != nullptr ? lookup_ctdef_->result_output_ : scan_ctdef_.result_output_; + return attach_spec_.attach_ctdef_ != nullptr ? attach_spec_.get_result_output() + : lookup_ctdef_ != nullptr ? lookup_ctdef_->result_output_ + : scan_ctdef_.result_output_; } const UIntFixedArray &get_full_acccess_cids() const { @@ -179,7 +182,9 @@ public: KPC_(das_dppr_tbl), KPC_(calc_part_id_expr), K_(global_index_rowkey_exprs), - K_(attach_spec)); + K_(attach_spec), + K_(is_das_keep_order), + K_(use_index_merge)); //the query range of index scan/table scan ObQueryRange pre_query_range_; FlashBackItem flashback_item_; @@ -211,7 +216,8 @@ public: uint64_t flags_; struct { uint64_t is_das_keep_order_ : 1; // whether das need keep ordering - uint64_t reserved_ : 63; + uint64_t use_index_merge_ : 1; // whether use index merge + uint64_t reserved_ : 62; }; }; }; @@ -485,7 +491,8 @@ protected: int single_equal_scan_check_type(const ParamStore ¶m_store, bool& is_same_type); bool need_extract_range() const { return MY_SPEC.tsc_ctdef_.pre_query_range_.has_range(); } int prepare_single_scan_range(int64_t group_idx = 0); - + int prepare_index_merge_scan_range(int64_t group_idx = 0); + int prepare_range_for_each_index(int64_t group_idx, ObIAllocator &allocator, ObDASBaseRtDef *rtdef); int reuse_table_rescan_allocator(); int local_iter_rescan(); diff --git a/src/sql/optimizer/ob_index_info_cache.cpp b/src/sql/optimizer/ob_index_info_cache.cpp index 34c07cfd4..f59e7cf86 100644 --- a/src/sql/optimizer/ob_index_info_cache.cpp +++ b/src/sql/optimizer/ob_index_info_cache.cpp @@ -30,7 +30,8 @@ ObIndexInfoCache::~ObIndexInfoCache() int ObIndexInfoCache::get_index_info_entry(const uint64_t table_id, const uint64_t index_id, - IndexInfoEntry *&entry) const + IndexInfoEntry *&entry, + int64_t *idx) const { int ret = OB_SUCCESS; entry = NULL; @@ -45,6 +46,9 @@ int ObIndexInfoCache::get_index_info_entry(const uint64_t table_id, LOG_WARN("entry should not be null", K(ret)); } else if (index_entrys_[i]->get_index_id() == index_id) { entry = index_entrys_[i]; + if (idx != nullptr) { + *idx = i; + } break; } } @@ -106,9 +110,17 @@ int ObIndexInfoCache::get_access_path_ordering(const uint64_t table_id, int ObIndexInfoCache::add_index_info_entry(IndexInfoEntry *entry) { int ret = OB_SUCCESS; + IndexInfoEntry *old_entry; + int64_t idx = OB_INVALID_INDEX; if (OB_ISNULL(entry)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("entry should not be null", K(ret)); + } else if (OB_FAIL(get_index_info_entry(table_id_, entry->get_index_id(), old_entry, &idx))) { + LOG_WARN("failed to get index info entry", KPC(entry), K(ret)); + } else if (old_entry != nullptr) { + // update index info entry + old_entry->~IndexInfoEntry(); + index_entrys_[idx] = entry; } else if (entry_count_ >= common::OB_MAX_INDEX_PER_TABLE + 1) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid entry count", K(ret), K_(entry_count), diff --git a/src/sql/optimizer/ob_index_info_cache.h b/src/sql/optimizer/ob_index_info_cache.h index 661ce1888..dbf08f8cc 100644 --- a/src/sql/optimizer/ob_index_info_cache.h +++ b/src/sql/optimizer/ob_index_info_cache.h @@ -223,12 +223,14 @@ public: int get_query_range(const uint64_t table_id, const uint64_t index_id, const QueryRangeInfo *&query_range_info) const; + int get_access_path_ordering(const uint64_t table_id, const uint64_t index_id, const OrderingInfo *&ordering_info) const; int get_index_info_entry(const uint64_t table_id, const uint64_t index_id, - IndexInfoEntry *&entry) const; + IndexInfoEntry *&entry, + int64_t *idx = nullptr) const; void set_table_id(const uint64_t table_id) { table_id_ = table_id; } void set_base_table_id(const uint64_t base_table_id) { base_table_id_ = base_table_id; } uint64_t get_table_id() const { return table_id_; } diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 3cbd37b32..e5440c092 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -1782,6 +1782,8 @@ int ObJoinOrder::create_one_access_path(const uint64_t table_id, ap->est_cost_info_.use_column_store_ = use_column_store; ap->contain_das_op_ = ap->use_das_; + ap->is_ror_ = (ref_id == index_id) ? true + : range_info.get_equal_prefix_count() >= range_info.get_index_column_count(); if (OB_FAIL(init_sample_info_for_access_path(ap, table_id, table_item))) { LOG_WARN("failed to init sample info", K(ret)); } else if (OB_FAIL(add_access_filters(ap, @@ -2742,6 +2744,9 @@ int ObJoinOrder::create_access_paths(const uint64_t table_id, const ParamStore *params = NULL; bool is_valid = true; ObSQLSessionInfo *session_info = NULL; + bool use_index_merge = false; + ObArray index_merge_list; + ObRawExpr* index_merge_root = NULL; if (OB_ISNULL(get_plan()) || OB_ISNULL(stmt = get_plan()->get_stmt()) || OB_ISNULL(opt_ctx = &get_plan()->get_optimizer_context()) || @@ -2756,6 +2761,23 @@ int ObJoinOrder::create_access_paths(const uint64_t table_id, } else if (OB_FAIL(get_generated_col_index_qual(table_id, helper.filters_, helper))) { LOG_WARN("get prefix index qual failed"); + } else if (OB_FAIL(check_can_use_index_merge(table_id, + ref_table_id, + helper, + use_index_merge, + index_merge_list, + index_merge_root))) { + LOG_WARN("failed to check can use index merge", K(ret)); + } else if (OB_UNLIKELY(use_index_merge)) { + if (OB_FAIL(create_index_merge_path(table_id, + ref_table_id, + helper, + access_paths, + index_info_cache, + index_merge_list, + index_merge_root))) { + LOG_WARN("failed to create index merge path", K(index_merge_list), K(ret)); + } } else if (OB_FAIL(get_valid_index_ids(table_id, ref_table_id, candi_index_ids))) { @@ -2771,21 +2793,16 @@ int ObJoinOrder::create_access_paths(const uint64_t table_id, valid_index_ids, helper))) { LOG_WARN("failed to add table by heuristics", K(ret)); - } else if (!valid_index_ids.empty()) { - LOG_TRACE("table added using heuristics", K(table_id)); - } else if (OB_FAIL(skyline_prunning_index(table_id, - ref_table_id, - stmt, - true, - index_info_cache, - candi_index_ids, - valid_index_ids, - helper.filters_))) { + } else if (valid_index_ids.empty() && OB_FAIL(skyline_prunning_index(table_id, + ref_table_id, + stmt, + true, + index_info_cache, + candi_index_ids, + valid_index_ids, + helper.filters_))) { LOG_WARN("failed to pruning_index", K(table_id), K(ref_table_id), K(ret)); } else { - LOG_TRACE("table added using skyline", K(table_id), K(valid_index_ids)); - } - if (OB_SUCC(ret)) { helper.table_opt_info_->optimization_method_ = OptimizationMethod::COST_BASED; for (int64_t i = 0; OB_SUCC(ret) && i < valid_index_ids.count(); ++i) { bool is_create_basic_path = false; @@ -2850,38 +2867,267 @@ int ObJoinOrder::create_access_paths(const uint64_t table_id, OB_FAIL(access_paths.push_back(das_column_store_access_path))) { LOG_WARN("failed to push back access path", K(ret)); } else if (is_create_basic_path && - use_row_store && - OB_FAIL(create_one_access_path(table_id, - ref_table_id, - valid_index_ids.at(i), - index_info_cache, - helper, - basic_row_store_access_path, - false, - false, - use_skip_scan))) { + use_row_store && + OB_FAIL(create_one_access_path(table_id, + ref_table_id, + valid_index_ids.at(i), + index_info_cache, + helper, + basic_row_store_access_path, + false, + false, + use_skip_scan))) { LOG_WARN("failed to make index path", "index_table_id", valid_index_ids.at(i), K(ret)); } else if(OB_NOT_NULL(basic_row_store_access_path) && OB_FAIL(access_paths.push_back(basic_row_store_access_path))) { LOG_WARN("failed to push back access path", K(ret)); } else if (is_create_basic_path && - use_column_store && - OB_FAIL(create_one_access_path(table_id, - ref_table_id, - valid_index_ids.at(i), - index_info_cache, - helper, - basic_column_store_access_path, - false, - true, - use_skip_scan))) { + use_column_store && + OB_FAIL(create_one_access_path(table_id, + ref_table_id, + valid_index_ids.at(i), + index_info_cache, + helper, + basic_column_store_access_path, + false, + true, + use_skip_scan))) { LOG_WARN("failed to make index path", "index_table_id", valid_index_ids.at(i), K(ret)); - } else if( OB_NOT_NULL(basic_column_store_access_path) && - OB_FAIL(access_paths.push_back(basic_column_store_access_path))) { + } else if(OB_NOT_NULL(basic_column_store_access_path) && + OB_FAIL(access_paths.push_back(basic_column_store_access_path))) { LOG_WARN("failed to push back access path", K(ret)); } } } + + return ret; +} + +int ObJoinOrder::check_can_use_index_merge(const uint64_t table_id, + const uint64_t ref_table_id, + PathHelper &helper, + bool &use_index_merge, + ObIArray &index_merge_list, + ObRawExpr *&index_merge_root) +{ + int ret = OB_SUCCESS; + use_index_merge = false; + index_merge_list.reuse(); + index_merge_root = nullptr; + bool contains_invalid_index = false; + bool is_all_local_index = false; + bool is_all_global_index = false; + ObArray> merge_index_column_ids; + const LogTableHint *log_table_hint = nullptr; + const ObUnionMergeHint *union_merge_hint = nullptr; + ObSqlSchemaGuard *schema_guard = nullptr; + const ObDMLStmt *stmt = nullptr; + ObQueryCtx *query_ctx = nullptr; + ObRawExpr *root = nullptr; + OPT_TRACE_TITLE("BEGIN CHECK USE INDEX MERGE"); + OPT_TRACE("table_id: ", table_id, "ref_table_id: ", ref_table_id); + OPT_TRACE("full query filters: ", helper.filters_); + if (OB_ISNULL(get_plan()) || + OB_ISNULL(stmt = get_plan()->get_stmt()) || + OB_ISNULL(schema_guard = get_plan()->get_optimizer_context().get_sql_schema_guard()) || + OB_ISNULL(query_ctx = stmt->get_query_ctx())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get unexpected null", K(get_plan()), K(stmt), K(schema_guard), K(query_ctx), K(ret)); + } else if (query_ctx->optimizer_features_enable_version_ < COMPAT_VERSION_4_3_4) { + LOG_TRACE("failed to index merge due to compat version < 4.3.4"); + } else if (is_virtual_table(ref_table_id)) { + LOG_TRACE("failed to index merge due to virtual table"); + } else if (FALSE_IT(log_table_hint = get_plan()->get_log_plan_hint().get_log_table_hint(table_id))) { + } else if (log_table_hint == nullptr || log_table_hint->merge_index_list_.count() < 2) { + OPT_TRACE("failed to index merge due to no valid index merge hint", log_table_hint); + LOG_TRACE("failed to index merge due to no valid index merge hint", K(log_table_hint)); + } else if (OB_FAIL(check_index_merge_list(table_id, + ref_table_id, + log_table_hint->merge_index_list_, + contains_invalid_index, + is_all_local_index, + is_all_global_index, + merge_index_column_ids))) { + LOG_WARN("failed to check index merge list", K(ret)); + } else if (contains_invalid_index) { + OPT_TRACE("failed to index merge due to contains invalid index"); + LOG_TRACE("failed to index merge due to contains invalid index", + K(log_table_hint->merge_index_list_)); + } else if (!is_all_local_index) { + OPT_TRACE("failed to index merge due to not all local index"); + LOG_TRACE("failed to index merge due to not all local index", + K(log_table_hint->merge_index_list_)); + } else { + // only Disjunctive Normal Form (DNF) and Conjunctive Normal Form (CNF) support index merge. + // for DNF, the number of predicate segments connected in the DNF must match the number of indexes + // specified in the hint. each part of predicate segment will be processed by the corresponding index. + // for CNF, we try to find the first disjunctive subformula which satisfies the index merge + // requirement and perform index merge with the subformula. + OPT_TRACE("got index merge list", log_table_hint->merge_index_list_); + int64_t index_cnt = log_table_hint->merge_index_list_.count(); + bool found_valid_disjunctive_form = false; + for (int64_t i = 0; OB_SUCC(ret) && !found_valid_disjunctive_form && i < helper.filters_.count(); ++i) { + if (OB_ISNULL(root = helper.filters_.at(i))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected null filter", K(ret)); + } else if (!IndexMergePath::is_disjunctive_clause(root)) { + OPT_TRACE("filter failed to index merge due to not match disjunctive clause", root); + LOG_TRACE("filter failed to index merge due to not match disjunctive clause", K(root)); + } else if (root->get_param_count() != index_cnt) { + OPT_TRACE("filter failed to index merge due to filter and index count not matched", + root, "filter count", root->get_param_count(), "index count", index_cnt); + LOG_TRACE("filter failed to index merge due to not match disjunctive clause", K(root)); + } else { + // check whether each filter overlap by the corresponding index columns + bool index_overlap_filter = true; + for (int64_t i = 0; OB_SUCC(ret) && i < index_cnt && index_overlap_filter; ++i) { + ObRawExpr *param_expr = root->get_param_expr(i); + ObArray column_ids; + if (OB_ISNULL(param_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null filter", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_column_ids(param_expr, column_ids))) { + LOG_WARN("failed to extract column ids", KPC(param_expr), K(ret)); + } else if (!ObOptimizerUtil::overlap(column_ids, merge_index_column_ids.at(i))) { + index_overlap_filter = false; + OPT_TRACE("filter failed to index merge due to not overlap with index", root, + "filter", param_expr, "index", log_table_hint->merge_index_list_.at(i), "filter column ids", + column_ids, "index column ids", merge_index_column_ids.at(i)); + LOG_TRACE("filter failed to index merge due to not overlap with index", KPC(param_expr), K(column_ids), + K(merge_index_column_ids.at(i))); + } + } + if (OB_SUCC(ret) && index_overlap_filter) { + if (OB_FAIL(index_merge_list.assign(log_table_hint->merge_index_list_))) { + LOG_WARN("failed to assign index merge list", K(ret)); + } else { + found_valid_disjunctive_form = true; + use_index_merge = true; + index_merge_root = root; + } + } + } + } + } + + OPT_TRACE("index merge list", index_merge_list, "use_index_merge", use_index_merge, + "index_merge_root", index_merge_root); + OPT_TRACE_TITLE("END CHECK USE INDEX MERGE"); + LOG_TRACE("check can use index merge finished", K(table_id), K(ref_table_id), + K(use_index_merge), K(index_merge_list), K(index_merge_root)); + return ret; +} + +int ObJoinOrder::create_index_merge_path(const uint64_t table_id, + const uint64_t ref_table_id, + PathHelper &helper, + ObIArray &access_paths, + ObIndexInfoCache &index_info_cache, + const ObIArray &index_merge_list, + ObRawExpr *index_merge_root) +{ + int ret = OB_SUCCESS; + const ObDMLStmt *stmt = nullptr; + ObSqlSchemaGuard *schema_guard = nullptr; + IndexMergePath *index_merge_path = nullptr; + ObArray backup_filters; // backup helper filters + if (OB_ISNULL(get_plan()) || + OB_ISNULL(stmt = get_plan()->get_stmt()) || + OB_ISNULL(schema_guard = get_plan()->get_optimizer_context().get_sql_schema_guard()) || + index_merge_list.count() < 2 || + OB_ISNULL(index_merge_root) || + OB_ISNULL(allocator_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected null", K(get_plan()), K(stmt), K(schema_guard), K(index_merge_list), + K(index_merge_root), K(allocator_), K(ret)); + } else if (OB_FAIL(backup_filters.assign(helper.filters_))) { + LOG_WARN("failed to assign filters array", K(ret)); + } else if (OB_ISNULL(index_merge_path = OB_NEWx(IndexMergePath, allocator_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to new a index merge path", K(ret)); + } else { + // simply use das for all global index, otherwise not use das. + // since px is not supported, use das for all situations now. + bool use_das = true; + bool use_column_store = false; + OptSkipScanState use_skip_scan = OptSkipScanState::SS_UNSET; + index_info_cache.set_table_id(table_id); + index_info_cache.set_base_table_id(ref_table_id); + ObArray index_name_list; + + OB_ASSERT(index_merge_list.count() == index_merge_root->get_param_count()); + /* create access path for each index scan and combine them to genrate a index merge path */ + for (int64_t i = 0; OB_SUCC(ret) && i < index_merge_list.count(); i++) { + AccessPath *ap = nullptr; + IndexInfoEntry *index_info_entry = nullptr; + const ObTableSchema *index_schema = nullptr; + ObString index_name; + uint64_t index_id = index_merge_list.at(i); + // replace helper.filters_ with the corresponding filters of the current index table + // to extract the query range and create index table access path + helper.filters_.at(0) = index_merge_root->get_param_expr(i); + if (OB_FAIL(fill_index_info_entry(table_id, ref_table_id, index_id, index_info_entry, helper))) { + LOG_WARN("failed to fill index info entry", K(ret)); + } else if (OB_ISNULL(index_info_entry)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to fill index info entry", K(ret)); + } else if (FALSE_IT(index_info_entry->set_is_index_back(true))) { + // set force index back when use index merge + } else if (OB_FAIL(index_info_cache.add_index_info_entry(index_info_entry))) { + LOG_WARN("failed to add index info entry", K(ret)); + } else if (OB_FAIL(create_one_access_path(table_id, + ref_table_id, + index_id, + index_info_cache, + helper, + ap, + use_das, + use_column_store, + use_skip_scan))) { + LOG_WARN("failed to create one access path", K(table_id), K(ref_table_id), K(index_id)); + } else if (OB_FAIL(index_merge_path->add_index_scan_node(INDEX_MERGE_UNION, ap, allocator_))) { + LOG_WARN("failed to add index scan node", K(index_merge_path), K(ap)); + } else if (ref_table_id == index_id) { + index_name = ObIndexHint::PRIMARY_KEY; + } else if (OB_FAIL(schema_guard->get_table_schema(table_id, index_id, stmt, index_schema))) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("failed to get index table schema", K(index_id), K(ret)); + } else if (OB_FAIL(index_schema->get_index_name(index_name))) { + LOG_WARN("failed to get index name", K(index_id), K(ret)); + } + if (OB_SUCC(ret) && OB_FAIL(index_name_list.push_back(index_name))) { + LOG_WARN("failed to push back index name", K(ret)); + } + } // for end + + /* adjust index merge path ordering and scan direction */ + if (OB_SUCC(ret)) { + ObArray rowkey_exprs; + ObOrderDirection direction = default_asc_direction(); + if (!index_merge_path->is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid index merge path", KPC(index_merge_path), K(ret)); + } else if (OB_FAIL(index_merge_path->filters_.assign(backup_filters))) { + LOG_WARN("failed to assign filters", K(ret)); + } else if (OB_FAIL(get_plan()->get_rowkey_exprs(table_id, ref_table_id, rowkey_exprs))) { + LOG_WARN("failed to get rowkey exprs", K(ret)); + } else if (FALSE_IT(index_merge_path->ordering_.reuse())) { + } else if (OB_FAIL(get_index_scan_direction(rowkey_exprs, stmt, get_plan()->get_equal_sets(), direction))) { + LOG_WARN("failed to get index scan direction", K(ret)); + } else if (OB_FAIL(index_merge_path->set_scan_direction(direction))) { + LOG_WARN("failed to set scan direction for index merge path", K(ret), KPC(index_merge_path)); + } else if (OB_FAIL(ObOptimizerUtil::make_sort_keys(rowkey_exprs, + index_merge_path->order_direction_, + index_merge_path->ordering_))) { + LOG_WARN("failed to make rowkey ordering for index merge path", K(ret)); + } else if (OB_FAIL(index_merge_path->index_name_list_.assign(index_name_list))) { + LOG_WARN("failed to assign index name list", K(ret)); + } else if (FALSE_IT(index_merge_path->force_used_by_hint_ = true)) { + } else if (OB_FAIL(access_paths.push_back(index_merge_path))) { + LOG_WARN("failed to push back index merge path", K(ret)); + } + } + } return ret; } @@ -6161,6 +6407,206 @@ int AccessPath::compute_valid_inner_path() return ret; } +bool ObIndexMergeNode::is_valid() const +{ + return is_leaf_node_ ? + (ap_ != NULL && left_node_ == NULL && right_node_ == NULL) : + (merge_type_ != INDEX_MERGE_INVALID && + left_node_ != NULL && + right_node_ != NULL && + left_node_->is_valid() && + right_node_->is_valid()); +} + +int ObIndexMergeNode::set_scan_direction(const ObOrderDirection &direction) +{ + int ret = OB_SUCCESS; + if (is_leaf_node_) { + if (OB_ISNULL(ap_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(ap_), K(ret)); + } else { + ap_->order_direction_ = direction; + } + } else if (OB_ISNULL(left_node_) || OB_ISNULL(right_node_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(left_node_), K(right_node_), K(ret)); + } else if (OB_FAIL(SMART_CALL(left_node_->set_scan_direction(direction))) || + OB_FAIL(SMART_CALL(right_node_->set_scan_direction(direction)))) { + LOG_WARN("failed to set scan direction", K(ret)); + } + return ret; +} + +int IndexMergePath::init(ObIndexMergeNode *node, common::ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + const AccessPath *ap = nullptr; + if (OB_ISNULL(node) || OB_ISNULL(allocator) || OB_ISNULL(ap = node->ap_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KPC(node), K(allocator), K(ret)); + } else if (OB_FAIL(assign(*ap, allocator))) { + LOG_WARN("failed to assign access path", K(ret)); + } else { + root_ = node; + is_index_merge_ = true; + } + return ret; +} + +int IndexMergePath::add_index_scan_node(const ObIndexMergeType merge_type, + AccessPath *ap, + common::ObIAllocator *allocator) +{ + int ret = OB_SUCCESS; + ObIndexMergeNode *node = nullptr; + if (OB_ISNULL(ap) || OB_ISNULL(allocator) || merge_type == INDEX_MERGE_INVALID) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(merge_type), K(ap), K(allocator), K(ret)); + } else if (OB_ISNULL(node = OB_NEWx(ObIndexMergeNode, allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to create a index merge node", K(ret)); + } else if (OB_FAIL(add_var_to_array_no_dup(index_table_ids_, ap->index_id_))) { + LOG_WARN("failed to push back index table id", K(ret)); + } else { + node->ap_ = ap; + node->is_leaf_node_ = true; + node->index_tid_ = ap->index_id_; + node->idx_ = index_cnt_; + node->is_ror_ = ap->is_ror_; + + if (OB_ISNULL(root_)) { + // first add node, init index merge path with first index access path + if (OB_FAIL(init(node, allocator))) { + LOG_WARN("failed to init index merge path", K(ret)); + } + } else { + ObIndexMergeNode *new_root = nullptr; + if (OB_ISNULL(new_root = OB_NEWx(ObIndexMergeNode, allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to create a index merge node", K(ret)); + } else { + new_root->merge_type_ = merge_type; + new_root->left_node_ = root_; + new_root->right_node_ = node; + new_root->is_leaf_node_ = false; + root_ = new_root; + } + } + } + + if (OB_SUCC(ret)) { + index_cnt_++; + } + return ret; +} + +int IndexMergePath::set_scan_direction(const ObOrderDirection &direction) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(root_) || !root_->is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KPC(root_), K(ret)); + } else if (OB_FAIL(root_->set_scan_direction(direction))) { + LOG_WARN("failed to set scan direction for index merge path", K(ret)); + } else { + order_direction_ = direction; + } + return ret; +} + +bool IndexMergePath::is_valid() const +{ + return root_ == nullptr ? false : root_->is_valid(); +} + +bool IndexMergePath::is_disjunctive_clause(const ObRawExpr *root) +{ + bool bret = false; + if (OB_NOT_NULL(root)) { + if (T_OP_OR == root->get_expr_type()) { + bret = true; + for (int64_t i = 0; bret && i < root->get_param_count(); i++) { + const ObRawExpr *expr = root->get_param_expr(i); + if (expr->has_flag(IS_SIMPLE_COND) || expr->has_flag(IS_RANGE_COND)) { + } else if (T_OP_AND == expr->get_expr_type()) { + bret = bret && is_simple_conjunctive_clause(expr); + } else { + bret = false; + } + } + } + } + return bret; +} + +bool IndexMergePath::is_simple_conjunctive_clause(const ObRawExpr *root) +{ + bool bret = false; + if (OB_NOT_NULL(root)) { + if (T_OP_AND == root->get_expr_type()) { + bret = true; + for (int64_t i = 0; bret && i < root->get_param_count(); i++) { + const ObRawExpr *expr = root->get_param_expr(i); + if (expr->has_flag(IS_SIMPLE_COND) || expr->has_flag(IS_RANGE_COND)) { + } else { + bret = false; + } + } + } + } + return bret; +} + +int ObJoinOrder::check_index_merge_list(const uint64_t table_id, + const uint64_t ref_table_id, + const common::ObIArray &index_list, + bool &contains_invalid_index, + bool &is_all_local_index, + bool &is_all_global_index, + common::ObIArray> &merge_index_column_ids) +{ + int ret = OB_SUCCESS; + contains_invalid_index = false; + is_all_local_index = true; + is_all_global_index = true; + merge_index_column_ids.reuse(); + const ObDMLStmt *stmt = nullptr; + ObSqlSchemaGuard *schema_guard = nullptr; + const ObTableSchema *index_schema = nullptr; + if (OB_ISNULL(get_plan()) || + OB_ISNULL(stmt = get_plan()->get_stmt()) || + OB_ISNULL(schema_guard = get_plan()->get_optimizer_context().get_sql_schema_guard())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr", K(get_plan()), K(stmt), K(schema_guard), K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && !contains_invalid_index && i < index_list.count(); i++) { + const ObTableID &index_id = index_list.at(i); + ObArray index_column_ids; + bool is_global_index = false; + if (OB_UNLIKELY(index_id == 0)) { + // invalid index id, mark as invalid index merge path + contains_invalid_index = true; + } else if (OB_FAIL(schema_guard->get_table_schema(index_id, index_schema))) { + LOG_WARN("falied to get table schema", K(index_schema)); + } else if (OB_ISNULL(index_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null index schema", K(index_id), K(ret)); + } else if (FALSE_IT(is_global_index = index_schema->is_global_index_table())) { + } else if (OB_FAIL(index_schema->get_column_ids(index_column_ids))) { + LOG_WARN("failed to get column ids", K(index_id), K(ret)); + } else if (OB_FAIL(merge_index_column_ids.push_back(index_column_ids))) { + LOG_WARN("failed to push back index column ids", K(ret)); + } else if (OB_UNLIKELY(ref_table_id == index_id)) { + // skip index type check when use primary key + } else { + is_all_local_index &= !is_global_index; + is_all_global_index &= is_global_index; + } + } + return ret; +} + int FunctionTablePath::assign(const FunctionTablePath &other, common::ObIAllocator *allocator) { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_join_order.h b/src/sql/optimizer/ob_join_order.h index eec9044e5..0191a6278 100644 --- a/src/sql/optimizer/ob_join_order.h +++ b/src/sql/optimizer/ob_join_order.h @@ -539,7 +539,9 @@ struct EstimateCostInfo { for_update_(false), use_skip_scan_(OptSkipScanState::SS_UNSET), use_column_store_(false), - is_valid_inner_path_(false) + is_valid_inner_path_(false), + is_index_merge_(false), + is_ror_(false) { } virtual ~AccessPath() { @@ -636,7 +638,9 @@ struct EstimateCostInfo { K_(use_das), K_(use_skip_scan), K_(use_column_store), - K_(is_valid_inner_path)); + K_(is_valid_inner_path), + K_(is_index_merge), + K_(is_ror)); public: //member variables uint64_t table_id_; @@ -661,10 +665,87 @@ struct EstimateCostInfo { bool use_column_store_; // mark this access path is inner path and contribute query range bool is_valid_inner_path_; + bool is_index_merge_; // whether used for index merge + bool is_ror_; // indicate whether result from index table scan is ordered by primary key private: DISALLOW_COPY_AND_ASSIGN(AccessPath); }; + enum ObIndexMergeType : uint32_t + { + INDEX_MERGE_INVALID = 0, + INDEX_MERGE_UNION, + INDEX_MERGE_INTERSECT, + INDEX_HASH_UNON, + INDEX_HASH_INTERSECT + }; + + struct ObIndexMergeNode + { + public: + ObIndexMergeNode() + : merge_type_(INDEX_MERGE_INVALID), + left_node_(nullptr), + right_node_(nullptr), + ap_(nullptr), + is_leaf_node_(false), + index_tid_(OB_INVALID_ID), + idx_(OB_INVALID_ID), + is_ror_(false) + {} + + bool is_valid() const; + int set_scan_direction(const ObOrderDirection &direction); + TO_STRING_KV(K_(merge_type), + K_(left_node), + K_(right_node), + K_(is_leaf_node), + K_(index_tid), + K_(is_ror)); + + public: + ObIndexMergeType merge_type_; + ObIndexMergeNode *left_node_; + ObIndexMergeNode *right_node_; + + /*** for leaf node, which is, the index scan node ***/ + AccessPath *ap_; + bool is_leaf_node_; + uint64_t index_tid_; // the table id of index table + int64_t idx_; // a unique identifier for each index node + bool is_ror_; + }; + + class IndexMergePath : public AccessPath + { + public: + IndexMergePath() + : AccessPath(OB_INVALID_ID, OB_INVALID_ID, OB_INVALID_ID, nullptr, NULLS_FIRST_ASC), + root_(nullptr), + index_cnt_(0), + filters_(), + force_used_by_hint_(false) + {} + + int init(ObIndexMergeNode *node, common::ObIAllocator *allocator); + int add_index_scan_node(const ObIndexMergeType merge_type, AccessPath *ap, common::ObIAllocator *allocator); + bool is_valid() const; + int set_scan_direction(const ObOrderDirection &direction); + static bool is_disjunctive_clause(const ObRawExpr *root); + static bool is_simple_conjunctive_clause(const ObRawExpr *root); + + public: + ObIndexMergeNode *root_; + int64_t index_cnt_; + common::ObSEArray index_table_ids_; + common::ObSEArray index_name_list_; + common::ObSEArray filters_; + bool force_used_by_hint_; + + private: + DISALLOW_COPY_AND_ASSIGN(IndexMergePath); + }; + class JoinPath : public Path { public: @@ -1523,6 +1604,29 @@ struct NullAwareAntiJoinInfo { bool use_column_store, OptSkipScanState use_skip_scan); + int check_can_use_index_merge(const uint64_t table_id, + const uint64_t ref_table_id, + PathHelper &helper, + bool &use_index_merge, + ObIArray &index_merge_list, + ObRawExpr *&index_merge_root); + + int check_index_merge_list(const uint64_t table_id, + const uint64_t ref_table_id, + const ObIArray &index_merge_list, + bool &contains_invalid_index, + bool &is_all_local_index, + bool &is_all_global_index, + common::ObIArray> &merge_index_column_ids); + + int create_index_merge_path(const uint64_t table_id, + const uint64_t ref_table_id, + PathHelper &helper, + ObIArray &access_paths, + ObIndexInfoCache &index_info_cache, + const ObIArray &index_merge_list, + ObRawExpr *index_merge_root); + int init_sample_info_for_access_path(AccessPath *ap, const uint64_t table_id, const TableItem *table_item); diff --git a/src/sql/optimizer/ob_log_join.cpp b/src/sql/optimizer/ob_log_join.cpp index 707ab3184..ef1ccc195 100644 --- a/src/sql/optimizer/ob_log_join.cpp +++ b/src/sql/optimizer/ob_log_join.cpp @@ -1417,6 +1417,8 @@ int ObLogJoin::check_if_disable_batch(ObLogicalOperator* root, bool &can_use_bat can_use_batch_nlj = false; } else if (ts->get_scan_direction() != default_asc_direction() && ts->get_scan_direction() != ObOrderDirection::UNORDERED) { can_use_batch_nlj = false; + } else if (ts->use_index_merge()) { + can_use_batch_nlj = false; } else { SMART_VAR(ObTablePartitionInfo, tmp_info) { ObTablePartitionInfo *tmp_info_ptr = &tmp_info; diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 4d3950410..333c908c5 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -2889,8 +2889,10 @@ int ObLogPlan::allocate_access_path(AccessPath *ap, } else { LOG_DEBUG("handle text ir expr in plan", K(ret), K(non_match_filters), K(match_filters)); } - } else if (OB_FAIL(scan->set_table_scan_filters(ap->filter_))) { - LOG_WARN("failed to set filters", K(ret)); + } else if (scan->use_index_merge() && OB_FAIL(scan->set_index_merge_scan_filters(ap))) { + LOG_WARN("failed to set index merge filters", K(ret)); + } else if (!scan->use_index_merge() && OB_FAIL(scan->set_table_scan_filters(ap->filter_))) { + LOG_WARN("failed to set table scan filters", K(ret)); } else if (OB_FAIL(append(scan->get_pushdown_filter_exprs(), ap->pushdown_filters_))) { LOG_WARN("failed to append pushdown filters", K(ret)); } else if (ap->est_cost_info_.index_meta_info_.is_multivalue_index_ && @@ -11276,6 +11278,18 @@ int ObLogPlan::collect_location_related_info(ObLogicalOperator &op) } } + if (OB_SUCC(ret) && tsc_op.use_index_merge()) { + ObArray index_tids; + if (OB_FAIL(tsc_op.get_index_tids(index_tids))) { + LOG_WARN("failed to get index tids", K(ret)); + } else if (OB_FAIL(append_array_no_dup(rel_info.related_ids_, index_tids))) { + LOG_WARN("failed to append index merge table ids", K(index_tids), K(ret)); + } else if (OB_FAIL(add_var_to_array_no_dup(rel_info.related_ids_, tsc_op.get_real_ref_table_id()))) { + LOG_WARN("failed to append main table id", K(ret)); + } + } + LOG_TRACE("collect location related info", K(rel_info)); + if (OB_SUCC(ret) && OB_FAIL(optimizer_context_.get_loc_rel_infos().push_back(rel_info))) { LOG_WARN("store location related info failed", K(ret)); } diff --git a/src/sql/optimizer/ob_log_subplan_filter.cpp b/src/sql/optimizer/ob_log_subplan_filter.cpp index eaffed1b7..81712722c 100644 --- a/src/sql/optimizer/ob_log_subplan_filter.cpp +++ b/src/sql/optimizer/ob_log_subplan_filter.cpp @@ -456,6 +456,8 @@ int ObLogSubPlanFilter::check_if_match_das_group_rescan(ObLogicalOperator *root, LOG_WARN("unexpected null", K(ret)); } else if (!tsc->use_das()) { group_rescan = false; + } else if (tsc->use_index_merge()) { + group_rescan = false; } if (OB_SUCC(ret) && group_rescan) { group_rescan = !(is_virtual_table(ap->ref_table_id_) diff --git a/src/sql/optimizer/ob_log_table_scan.cpp b/src/sql/optimizer/ob_log_table_scan.cpp index 0f158b84e..e2e42a82a 100644 --- a/src/sql/optimizer/ob_log_table_scan.cpp +++ b/src/sql/optimizer/ob_log_table_scan.cpp @@ -58,6 +58,8 @@ const char *ObLogTableScan::get_name() const name = use_das() ? "DISTRIBUTED TABLE SKIP SCAN" : "TABLE SKIP SCAN"; } else if (EXTERNAL_TABLE == get_table_type()) { name = "EXTERNAL TABLE SCAN"; + } else if (use_index_merge()) { + name = use_das() ? "DISTRIBUTED INDEX MERGE SCAN" : "INDEX MERGE SCAN"; } else if (use_das()) { if (is_get) { name = "DISTRIBUTED TABLE GET"; @@ -212,6 +214,8 @@ int ObLogTableScan::get_op_exprs(ObIArray &all_exprs) LOG_WARN("failed to analyze filter monotonicity", K(ret)); } else if (OB_FAIL(get_filter_assist_exprs(all_exprs))) { LOG_WARN("failed to get filter assist expr", K(ret)); + } else if (use_index_merge() && OB_FAIL(append_array_no_dup(all_exprs, full_filters_))) { + LOG_WARN("failed to append index merge full filters", K(ret)); } else if (OB_FAIL(ObLogicalOperator::get_op_exprs(all_exprs))) { LOG_WARN("failed to get exprs", K(ret)); } else { /*do nothing*/ } @@ -517,6 +521,15 @@ int ObLogTableScan::generate_access_exprs() } } + if (OB_SUCC(ret) && use_index_merge() && !full_filters_.empty()) { + ObArray column_exprs; + if (OB_FAIL(ObRawExprUtils::extract_column_exprs(full_filters_, column_exprs))) { + LOG_WARN("failed to extract column exprs", K(full_filters_), K(ret)); + } else if (OB_FAIL(append_array_no_dup(access_exprs_, column_exprs))) { + LOG_WARN("failed to append array no dup", K(column_exprs), K(ret)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_pseudo_column_like_exprs().count(); i++) { ObRawExpr *expr = stmt->get_pseudo_column_like_exprs().at(i); if (OB_ISNULL(expr)) { @@ -624,10 +637,43 @@ int ObLogTableScan::replace_index_back_pushdown_filters(ObRawExprReplacer &repla LOG_WARN("failed to replace non pushdown expr", K(ret)); } else if (OB_FAIL(replace_exprs_action(replacer, lookup_pushdown_filters))) { LOG_WARN("failed to replace lookup pushdown expr", K(ret)); + } else if (OB_UNLIKELY(use_index_merge())) { + // when use index merge, we need to replace the filter exprs related to virtual generated columns + // for those main table participates as a branch of merge. + IndexMergePath *path = static_cast(access_path_); + if (OB_ISNULL(path)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null index merge path", K(ret)); + } else if (OB_FAIL(replace_index_merge_pushdown_filters(path->root_, replacer))) { + LOG_WARN("failed to replace index merge pushdown filters", K(ret)); + } } else { /* do nothing */ } return ret; } +int ObLogTableScan::replace_index_merge_pushdown_filters(ObIndexMergeNode *node, ObRawExprReplacer &replacer) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(node) || !node->is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid index merge node", KPC(node), K(ret)); + } else if (node->is_leaf_node_) { + if (OB_UNLIKELY(node->index_tid_ == ref_table_id_)) { + // main table participates as a branch of merge + ObArray scan_pushdown_filters; + if (OB_FAIL(get_index_filters(node->idx_, scan_pushdown_filters))) { + LOG_WARN("failed to get index scan pushdown filters", K(node->index_tid_), K(ret)); + } else if (OB_FAIL(replace_exprs_action(replacer, scan_pushdown_filters))) { + LOG_WARN("failed to replace index scan pushdown filters", K(ret)); + } + } + } else if (OB_FAIL(SMART_CALL(replace_index_merge_pushdown_filters(node->left_node_, replacer))) || + OB_FAIL(SMART_CALL(replace_index_merge_pushdown_filters(node->right_node_, replacer)))) { + LOG_WARN("failed to set index merge pushdown filters", K(ref_table_id_), K(ret)); + } + return ret; +} + int ObLogTableScan::has_nonpushdown_filter(bool &has_npd_filter) { int ret = OB_SUCCESS; @@ -711,6 +757,44 @@ int ObLogTableScan::extract_pushdown_filters(ObIArray &nonpushdown_f return ret; } +int ObLogTableScan::extract_nonpushdown_filters(const ObIArray &filters, + ObIArray &nonpushdown_filters, + ObIArray &pushdown_filters) const +{ + int ret = OB_SUCCESS; + if (get_contains_fake_cte() || + is_virtual_table(get_ref_table_id()) || + EXTERNAL_TABLE == get_table_type()) { + // all filters can not push down to storage + if (OB_FAIL(nonpushdown_filters.assign(filters))) { + LOG_WARN("failed to assign full filter to non-pushdown filters", K(ret)); + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < filters.count(); ++i) { + if (use_batch() && filters.at(i)->has_flag(CNT_DYNAMIC_PARAM)) { + //In Batch table scan the dynamic param filter do not push down to storage + if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) { + LOG_WARN("failed to push dynamic filter to non-pushdown filters", K(ret), K(i)); + } + } else if (filters.at(i)->has_flag(CNT_PL_UDF) || + filters.at(i)->has_flag(CNT_OBJ_ACCESS_EXPR)) { + //User Define Function/obj access expr filter do not push down to storage + if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) { + LOG_WARN("failed to push UDF/obj access filter to non-pushdown filters", K(ret), K(i)); + } + } else if (filters.at(i)->has_flag(CNT_DYNAMIC_USER_VARIABLE) || + filters.at(i)->has_flag(CNT_ASSIGN_EXPR)) { + if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) { + LOG_WARN("failed to push variable assign filter to non-pushdown filters", K(ret), K(i)); + } + } else if (OB_FAIL(pushdown_filters.push_back(filters.at(i)))) { + LOG_WARN("failed to push back pushdown filter", K(ret)); + } + } + } + return ret; +} + int ObLogTableScan::extract_virtual_gen_access_exprs( ObIArray &access_exprs, uint64_t scan_table_id) @@ -1060,7 +1144,10 @@ int ObLogTableScan::index_back_check() { int ret = OB_SUCCESS; bool column_found = true; - if (!is_index_scan()) { + if (use_index_merge()) { + // force set index back when index merge + column_found = false; + } else if (!is_index_scan()) { column_found = true; } else { for (int64_t i = 0; OB_SUCC(ret) && column_found && i < access_exprs_.count(); ++i) { @@ -1142,6 +1229,121 @@ int ObLogTableScan::set_table_scan_filters(const common::ObIArray & return ret; } +int ObLogTableScan::set_index_merge_scan_filters(const AccessPath *path) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(path) || !path->is_index_merge_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KPC(path)); + } else { + const IndexMergePath *root_path = static_cast(path); + if (OB_FAIL(full_filters_.assign(root_path->filters_))) { + LOG_WARN("failed to assign filters array", K(ret)); + } else if (OB_FAIL(index_range_conds_.prepare_allocate(root_path->index_cnt_)) || + OB_FAIL(index_filters_.prepare_allocate(root_path->index_cnt_))) { + LOG_WARN("failed to prepare allocate index range filters", K(ret)); + } else if (OB_FAIL(set_index_table_scan_filters(root_path->root_))) { + LOG_WARN("failed to set index table scan filters", K(ret)); + } + } + LOG_TRACE("index merge set range conds and filters", K(index_range_conds_), K(index_filters_)); + return ret; +} + +int ObLogTableScan::set_index_table_scan_filters(ObIndexMergeNode *node) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(node) || !node->is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KPC(node)); + } else if (node->is_leaf_node_) { + bool is_get = false; + ObOptimizerContext *opt_ctx = nullptr; + ObSqlSchemaGuard *schema_guard = nullptr; + const share::schema::ObTableSchema *index_schema = nullptr; + AccessPath *ap = nullptr; + /* + * virtual table may have hash index, + * for hash index, if it is a get, we should still extract the range condition + */ + if (OB_ISNULL(get_plan()) + || OB_ISNULL(opt_ctx = &get_plan()->get_optimizer_context()) + || OB_ISNULL(schema_guard = opt_ctx->get_sql_schema_guard()) + || OB_ISNULL(ap = node->ap_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected nullptr", K(get_plan()), K(opt_ctx), K(schema_guard), K(ap), K(ret)); + } else if (get_contains_fake_cte()) { + // do nothing + } else if (OB_FAIL(schema_guard->get_table_schema(table_id_, ap->index_id_, get_stmt(), index_schema))) { + LOG_WARN("failed to get table schema", K(ret)); + } else if (OB_ISNULL(index_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (OB_FAIL(is_table_get(is_get))) { + LOG_WARN("failed to check is table get", K(ret)); + } else if ((index_schema->is_ordered() || is_get) && NULL != ap->pre_query_range_) { + const ObIArray &range_exprs = ap->pre_query_range_->get_range_exprs(); + ObArray scan_pushdown_filters; + ObArray filter_before_index_back; + ObArray index_column_ids; + // extract the filters can be pushed down to index table scan + for (ObTableSchema::const_column_iterator iter = index_schema->column_begin(); + OB_SUCC(ret) && iter != index_schema->column_end(); ++iter) { + if (OB_ISNULL(iter)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr column iter", K(iter), K(ret)); + } else { + const ObColumnSchemaV2 *column_schema = *iter; + if (OB_ISNULL(column_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr column schema", K(ret)); + } else if (OB_FAIL(index_column_ids.push_back(column_schema->get_column_id()))) { + LOG_WARN("failed to push back column id", K(ret)); + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObOptimizerUtil::check_filter_before_indexback(ap->filter_, + index_column_ids, + filter_before_index_back))) { + LOG_WARN("failed to check filter before index back", K(ap->filter_), K(ret)); + } else { + OB_ASSERT(ap->filter_.count() == filter_before_index_back.count()); + for (int64_t i = 0; OB_SUCC(ret) && i < ap->filter_.count(); i++) { + if (filter_before_index_back.at(i) && OB_FAIL(scan_pushdown_filters.push_back(ap->filter_.at(i)))) { + LOG_WARN("failed to push back filter", K(ret)); + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < scan_pushdown_filters.count(); i++) { + bool found_expr = false; + for (int64_t j = 0; OB_SUCC(ret) && !found_expr && j < range_exprs.count(); j++) { + if (scan_pushdown_filters.at(i) == range_exprs.at(j)) { + //有重复表达式，忽略掉 + found_expr = true; + } else { /* Do nothing */ } + } + // for virtual table, even if we extract query range, we need to maintain the condition into the filter + if (OB_SUCC(ret) && (!found_expr || (is_virtual_table(ref_table_id_)))) { + if (OB_FAIL(index_filters_.at(node->idx_).push_back(scan_pushdown_filters.at(i)))) { + LOG_WARN("add filter expr failed", KPC(node), K(ret)); + } else { /* Do nothing */ } + } + if (OB_SUCC(ret) && found_expr) { + if (OB_FAIL(index_range_conds_.at(node->idx_).push_back(scan_pushdown_filters.at(i)))) { + LOG_WARN("failed to push back expr", K(ret)); + } else { /*do nothing*/} + } + } //end for + } + } else if (OB_FAIL(SMART_CALL(set_index_table_scan_filters(node->left_node_))) || + OB_FAIL(SMART_CALL(set_index_table_scan_filters(node->right_node_)))) { + LOG_WARN("failed to set index table filters", K(ret)); + } + + return ret; +} + int ObLogTableScan::pick_out_query_range_exprs() { int ret = OB_SUCCESS; @@ -1421,6 +1623,14 @@ int ObLogTableScan::get_plan_item_info(PlanText &plan_text, LOG_WARN("BUF_PRINTF fails", K(ret)); } else { /* Do nothing */ } + if (OB_SUCC(ret) && use_index_merge()) { + if (OB_FAIL(BUF_PRINTF(", "))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF("use_index_merge=true"))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else { /* Do nothing */ } + } + if (OB_SUCC(ret) && is_tsc_with_vid_) { if (is_tsc_with_vid_ && OB_FAIL(BUF_PRINTF(", "))) { LOG_WARN("BUF_PRINTF fails", K(ret)); @@ -1524,6 +1734,34 @@ int ObLogTableScan::get_plan_object_info(PlanText &plan_text, BEGIN_BUF_PRINT; if (OB_FAIL(BUF_PRINTF("%.*s", name.length(), name.ptr()))) { LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (use_index_merge()) { + if (OB_FAIL(BUF_PRINTF("%s", LEFT_BRACKET))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else { + ObArray index_name_list; + if (OB_FAIL(get_index_name_list(index_name_list))) { + LOG_WARN("failed to get index name list", K(ret)); + } else { + int64_t N = index_name_list.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < N - 1; i++) { + if (OB_FAIL(BUF_PRINTF("%.*s", index_name_list.at(i).length(), index_name_list.at(i).ptr()))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF(","))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(BUF_PRINTF("%.*s", index_name_list.at(N-1).length(), index_name_list.at(N-1).ptr()))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (is_descending_direction(get_scan_direction()) && + OB_FAIL(BUF_PRINTF("%s", COMMA_REVERSE))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF("%s", RIGHT_BRACKET))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } + } + } + } } else if (is_index_scan()) { if (OB_FAIL(BUF_PRINTF("%s", LEFT_BRACKET))) { LOG_WARN("BUF_PRINTF fails", K(ret)); @@ -1837,65 +2075,95 @@ int ObLogTableScan::print_range_annotation(char *buf, ExplainType type) { int ret = OB_SUCCESS; - ObArray range_key; - for (int64_t i = 0; OB_SUCC(ret) && i < range_columns_.count(); ++i) { - if (OB_ISNULL(range_columns_.at(i).expr_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Expr in column item should not be NULL", K(i), K(ret)); - } else if (OB_FAIL(range_key.push_back(range_columns_.at(i).expr_))) { - LOG_WARN("Fail to add range expr", K(i), K(ret)); - } else { /* Do nothing */ } - } - if (OB_SUCC(ret)) { - EXPLAIN_PRINT_EXPRS(range_key, type); + if (use_index_merge()) { + ObArray index_name_list; + if (OB_FAIL(get_index_name_list(index_name_list))) { + LOG_WARN("failed to get index name list", K(ret)); + } else { + OB_ASSERT(index_name_list.count() == index_range_conds_.count()); + for (int64_t i = 0; OB_SUCC(ret) && i < index_range_conds_.count(); ++i) { + const ObString &index_name = index_name_list.at(i); + const ObIArray &range_cond = index_range_conds_.at(i); + const ObIArray &filter = index_filters_.at(i); + if (OB_FAIL(BUF_PRINTF("index_name: %.*s, ", index_name.length(), index_name.ptr()))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } + EXPLAIN_PRINT_EXPRS(range_cond, type); + if (OB_SUCC(ret)) { + if (OB_FAIL(BUF_PRINTF(", "))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else { /* Do nothing */ } + } + EXPLAIN_PRINT_EXPRS(filter, type); + if (OB_SUCC(ret)) { + if (OB_FAIL(BUF_PRINTF("\n "))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else { /* Do nothing */ } + } + } + } + } else { + ObArray range_key; + for (int64_t i = 0; OB_SUCC(ret) && i < range_columns_.count(); ++i) { + if (OB_ISNULL(range_columns_.at(i).expr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Expr in column item should not be NULL", K(i), K(ret)); + } else if (OB_FAIL(range_key.push_back(range_columns_.at(i).expr_))) { + LOG_WARN("Fail to add range expr", K(i), K(ret)); + } else { /* Do nothing */ } + } if (OB_SUCC(ret)) { - if (OB_FAIL(BUF_PRINTF(", "))) { + EXPLAIN_PRINT_EXPRS(range_key, type); + if (OB_SUCC(ret)) { + if (OB_FAIL(BUF_PRINTF(", "))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF("range"))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else { /* Do nothing */ } + } else { /* Do nothing */ } + } else { /* Do nothing */ } + + //When range is empty. Range is always true. + if (OB_SUCC(ret) && 0 >= ranges_.count()) { + if (OB_FAIL(BUF_PRINTF("("))) { LOG_WARN("BUF_PRINTF fails", K(ret)); - } else if (OB_FAIL(BUF_PRINTF("range"))) { + } else if (OB_FAIL(BUF_PRINTF("MIN"))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF(" ; "))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF("MAX"))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF(")"))) { LOG_WARN("BUF_PRINTF fails", K(ret)); } else { /* Do nothing */ } } else { /* Do nothing */ } - } else { /* Do nothing */ } - //When range is empty. Range is always true. - if (OB_SUCC(ret) && 0 >= ranges_.count()) { - if (OB_FAIL(BUF_PRINTF("("))) { - LOG_WARN("BUF_PRINTF fails", K(ret)); - } else if (OB_FAIL(BUF_PRINTF("MIN"))) { - LOG_WARN("BUF_PRINTF fails", K(ret)); - } else if (OB_FAIL(BUF_PRINTF(" ; "))) { - LOG_WARN("BUF_PRINTF fails", K(ret)); - } else if (OB_FAIL(BUF_PRINTF("MAX"))) { - LOG_WARN("BUF_PRINTF fails", K(ret)); - } else if (OB_FAIL(BUF_PRINTF(")"))) { - LOG_WARN("BUF_PRINTF fails", K(ret)); - } else { /* Do nothing */ } - } else { /* Do nothing */ } - if (OB_SUCC(ret)) { - ret = print_ranges(buf, buf_len, pos, ranges_); - } + if (OB_SUCC(ret)) { + ret = print_ranges(buf, buf_len, pos, ranges_); + } - if (OB_SUCC(ret) && is_skip_scan()) { - int64_t skip_scan_offset = get_pre_query_range()->get_skip_scan_offset(); - if (OB_FAIL(BUF_PRINTF("\n prefix_columns_cnt = %ld , skip_scan_range", skip_scan_offset))) { - LOG_WARN("BUF_PRINTF fails", K(ret)); - } else if (ss_ranges_.empty() && OB_FAIL(BUF_PRINTF("(MIN ; MAX)"))) { - LOG_WARN("BUF_PRINTF fails", K(ret)); - } else if (OB_FAIL(print_ranges(buf, buf_len, pos, ss_ranges_))) { - LOG_WARN("failed to print index skip ranges", K(ret)); - } else { /* Do nothing */ } - } - - if (OB_SUCC(ret)) { - if (!range_conds_.empty()) { - //print range condition - const ObIArray &range_cond = range_conds_; - if (OB_FAIL(BUF_PRINTF(", "))) { + if (OB_SUCC(ret) && is_skip_scan()) { + int64_t skip_scan_offset = get_pre_query_range()->get_skip_scan_offset(); + if (OB_FAIL(BUF_PRINTF("\n prefix_columns_cnt = %ld , skip_scan_range", skip_scan_offset))) { LOG_WARN("BUF_PRINTF fails", K(ret)); - } else if (OB_FAIL(BUF_PRINTF("\n "))) { + } else if (ss_ranges_.empty() && OB_FAIL(BUF_PRINTF("(MIN ; MAX)"))) { LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(print_ranges(buf, buf_len, pos, ss_ranges_))) { + LOG_WARN("failed to print index skip ranges", K(ret)); + } else { /* Do nothing */ } + } + + if (OB_SUCC(ret)) { + if (!range_conds_.empty()) { + //print range condition + const ObIArray &range_cond = range_conds_; + if (OB_FAIL(BUF_PRINTF(", "))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } else if (OB_FAIL(BUF_PRINTF("\n "))) { + LOG_WARN("BUF_PRINTF fails", K(ret)); + } + EXPLAIN_PRINT_EXPRS(range_cond, type); } - EXPLAIN_PRINT_EXPRS(range_cond, type); } } @@ -2019,7 +2287,8 @@ int ObLogTableScan::print_outline_data(PlanText &plan_text) } if (OB_FAIL(index_hint.print_hint(plan_text))) { LOG_WARN("failed to print index hint", K(ret)); - } else if (use_das()) { + } + if (OB_SUCC(ret) && use_das()) { ObIndexHint use_das_hint(T_USE_DAS_HINT); use_das_hint.set_qb_name(qb_name); use_das_hint.get_table().set_table(*table_item); @@ -2035,8 +2304,18 @@ int ObLogTableScan::print_outline_data(PlanText &plan_text) LOG_WARN("failed to print use column store hint", K(ret)); } } + if (OB_SUCC(ret) && use_index_merge()) { + ObUnionMergeHint union_merge_hint; + union_merge_hint.set_qb_name(qb_name); + union_merge_hint.get_table().set_table(*table_item); + ObIArray &index_name_list = union_merge_hint.get_index_name_list(); + if (OB_FAIL(get_index_name_list(index_name_list))) { + LOG_WARN("failed to get index name list", K(ret)); + } else if (OB_FAIL(union_merge_hint.print_hint(plan_text))) { + LOG_WARN("failed to print index merge hint", K(ret)); + } + } } - return ret; } @@ -2074,6 +2353,10 @@ int ObLogTableScan::print_used_hint(PlanText &plan_text) && use_column_store() == table_hint->use_column_store_hint_->is_enable_hint() && OB_FAIL(table_hint->use_column_store_hint_->print_hint(plan_text))) { LOG_WARN("failed to print use column_store hint", K(ret)); + } else if (NULL != table_hint->union_merge_hint_ + && use_index_merge_by_hint() == table_hint->union_merge_hint_->is_enable_hint() + && OB_FAIL(table_hint->union_merge_hint_->print_hint(plan_text))) { + LOG_WARN("failed to print use union merge hint", K(ret)); } else if (table_hint->index_list_.empty()) { /*do nothing*/ } else if (OB_UNLIKELY(table_hint->index_list_.count() != table_hint->index_hints_.count())) { @@ -3170,6 +3453,82 @@ int ObLogTableScan::get_filter_assist_exprs(ObIArray &assist_exprs) return ret; } +bool ObLogTableScan::use_index_merge() const +{ + bool bret = false; + if (OB_NOT_NULL(access_path_)) { + bret = access_path_->is_index_merge_; + } + return bret; +} + +bool ObLogTableScan::use_index_merge_by_hint() const +{ + bool bret = false; + if (use_index_merge()) { + bret = static_cast(access_path_)->force_used_by_hint_; + } + return bret; +} + +int ObLogTableScan::get_index_range_conds(int64_t idx, ObIArray &index_range_conds) const +{ + int ret = OB_SUCCESS; + index_range_conds.reuse(); + if (OB_UNLIKELY(idx < 0 || idx >= index_range_conds_.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid idx of index range conds", K(idx), K(index_range_conds_.count())); + } else if (OB_FAIL(index_range_conds.assign(index_range_conds_.at(idx)))) { + LOG_WARN("failed to assign index range conds", K(ret)); + } + return ret; +} + +int ObLogTableScan::get_index_filters(int64_t idx, ObIArray &index_filters) const +{ + int ret = OB_SUCCESS; + index_filters.reuse(); + if (OB_UNLIKELY(idx < 0 || idx >= index_filters_.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid idx of index filters", K(idx), K(index_filters_.count())); + } else if (OB_FAIL(index_filters.assign(index_filters_.at(idx)))) { + LOG_WARN("failed to assign index filters", K(ret)); + } + return ret; +} + +int ObLogTableScan::get_index_tids(ObIArray &index_tids) const +{ + int ret = OB_SUCCESS; + index_tids.reuse(); + if (OB_ISNULL(access_path_) || !access_path_->is_index_merge_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected null index merge path", KPC(access_path_)); + } else { + IndexMergePath *index_merge_path = static_cast(access_path_); + if (OB_FAIL(index_tids.assign(index_merge_path->index_table_ids_))) { + LOG_WARN("failed to assign index merge table ids", K(ret)); + } + } + return ret; +} + +int ObLogTableScan::get_index_name_list(ObIArray &index_name_list) const +{ + int ret = OB_SUCCESS; + index_name_list.reuse(); + if (OB_ISNULL(access_path_) || !access_path_->is_index_merge_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected null index merge path", KPC(access_path_)); + } else { + IndexMergePath *index_merge_path = static_cast(access_path_); + if (OB_FAIL(index_name_list.assign(index_merge_path->index_name_list_))) { + LOG_WARN("failed to assign index merge table names", K(ret)); + } + } + return ret; +} + int ObLogTableScan::check_das_need_scan_with_vid() { int ret = OB_SUCCESS; diff --git a/src/sql/optimizer/ob_log_table_scan.h b/src/sql/optimizer/ob_log_table_scan.h index 48fe36eba..986b9f859 100644 --- a/src/sql/optimizer/ob_log_table_scan.h +++ b/src/sql/optimizer/ob_log_table_scan.h @@ -11,7 +11,7 @@ */ #ifndef OCEANBASE_SQL_OB_LOG_TABLE_SCAN_H -#define OCEANBASE_SQL_OB_LOG_TABLE_SCAN_H 1 +#define OCEANBASE_SQL_OB_LOG_TABLE_SCAN_H #include "sql/optimizer/ob_logical_operator.h" #include "sql/optimizer/ob_log_operator_factory.h" #include "sql/optimizer/ob_join_order.h" @@ -565,6 +565,9 @@ public: bool is_need_feedback() const; int set_table_scan_filters(const common::ObIArray &filters); + // for index merge, we need to set range conds and filters for each index scan + int set_index_merge_scan_filters(const AccessPath *path); + int set_index_table_scan_filters(ObIndexMergeNode *node); inline common::ObIArray &get_range_conditions() { return range_conds_; } const common::ObIArray &get_range_conditions() const { return range_conds_; } inline void set_diverse_path_count(int64_t count) { diverse_path_count_ = count; } @@ -622,8 +625,12 @@ public: ObIArray &scan_pushdown_filters, ObIArray &lookup_pushdown_filters, bool ignore_pd_filter = false) const; + int extract_nonpushdown_filters(const ObIArray &filters, + ObIArray &nonpushdown_filters, + ObIArray &pushdown_filters) const; int has_nonpushdown_filter(bool &has_npd_filter); int replace_index_back_pushdown_filters(ObRawExprReplacer &replacer); + int replace_index_merge_pushdown_filters(ObIndexMergeNode *node, ObRawExprReplacer &replacer); int extract_virtual_gen_access_exprs(ObIArray &access_exprs, uint64_t scan_table_id); int adjust_print_access_info(ObIArray &access_exprs); @@ -664,6 +671,17 @@ public: const ObColumnRefRawExpr *col_expr, PushdownFilterMonotonicity &mono, ObIArray &assist_exprs) const; + + bool use_index_merge() const; + const ObIArray &get_full_filters() const { return full_filters_; } + const ObIArray &get_index_range_conds(int64_t idx) const { return index_range_conds_.at(idx); } + const ObIArray &get_index_filters(int64_t idx) const { return index_filters_.at(idx); } + int get_index_range_conds(int64_t idx, ObIArray &index_range_conds) const; + int get_index_filters(int64_t idx, ObIArray &index_filters) const; + int get_index_tids(ObIArray &index_tids) const; + int get_index_name_list(ObIArray &index_name_list) const; + bool use_index_merge_by_hint() const; + private: // member functions //called when index_back_ set int pick_out_query_range_exprs(); @@ -717,6 +735,29 @@ protected: // memeber variables const common::ObIArray *part_ids_; common::ObSEArray range_conds_; + // for index merge, we need to prepare range conds and filters for each index scan, and we need + // to store full query filters for final check. + // for example, consider following query: + // create table t1(c1 int primary key, c2 int, c3 int, c4 int); + // create index c2 on t1(c2) local; + // create index c3 on t1(c3) local; + // create index c4 on t1(c4) local; + // select /*+union_merge(t1 c2 c3 c4)*/ * from t1 where c1=1 or c2=1 or c4<1; + // when we choose index merge plan, range conds and filters need to be prepared, thus: + // --------------------------------------------------------- + // | index table | range conds | filters | + // --------------------------------------------------------- + // | c2 | NULL | c1 = 1 | + // | c3 | NULL | NULL | + // | c4 | c4 < 1 | NULL | + // --------------------------------------------------------- + // NOTE: only filters before index back can be pushed down to index scan. + // and full filters 'c1=1 or c2=1 or c4<1' will be used after lookup for final check. + typedef common::ObSEArray ExprSEArray; + common::ObSEArray index_range_conds_; + common::ObSEArray index_filters_; + ExprSEArray full_filters_; + // index primary key columns. // indicates use which columns to extract query range common::ObSEArray range_columns_; diff --git a/src/sql/parser/sql_parser_mysql_mode.l b/src/sql/parser/sql_parser_mysql_mode.l index 835e601ca..69c2eae52 100644 --- a/src/sql/parser/sql_parser_mysql_mode.l +++ b/src/sql/parser/sql_parser_mysql_mode.l @@ -993,6 +993,7 @@ Timestamp{whitespace}?\"[^\"]*\" { } INDEX { return INDEX_HINT; } NO_INDEX { return NO_INDEX_HINT; } +UNION_MERGE { return UNION_MERGE_HINT; } USE_DAS { return USE_DAS_HINT; } NO_USE_DAS { return NO_USE_DAS_HINT; } INDEX_SS { return INDEX_SS_HINT; } diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index 228b5a644..b35dcca84 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -181,7 +181,7 @@ COALESCE_AGGR NO_COALESCE_AGGR WITH_PULLUP WO_PULLUP MV_REWRITE NO_MV_REWRITE DECORRELATE NO_DECORRELATE // optimize hint -INDEX_HINT FULL_HINT NO_INDEX_HINT USE_DAS_HINT NO_USE_DAS_HINT +INDEX_HINT FULL_HINT NO_INDEX_HINT USE_DAS_HINT NO_USE_DAS_HINT UNION_MERGE_HINT INDEX_SS_HINT INDEX_SS_ASC_HINT INDEX_SS_DESC_HINT USE_COLUMN_STORE_HINT NO_USE_COLUMN_STORE_HINT LEADING_HINT ORDERED @@ -427,7 +427,7 @@ END_P SET_VAR DELIMITER %type relation_factor_in_hint relation_factor_in_hint_list relation_factor_in_pq_hint opt_relation_factor_in_hint_list relation_factor_in_use_join_hint_list relation_factor_in_mv_hint_list opt_relation_factor_in_mv_hint_list %type relation_factor_in_leading_hint_list joined_table tbl_name table_subquery table_subquery_alias %type relation_factor_with_star relation_with_star_list opt_with_star -%type index_hint_type key_or_index index_hint_scope index_element index_list opt_index_list opt_index_prefix +%type index_hint_type key_or_index index_hint_scope index_element index_list opt_index_list opt_index_prefix union_merge_list %type add_key_or_index_opt add_key_or_index add_unique_key_opt add_unique_key add_constraint_uniq_key_opt add_constraint_uniq_key add_constraint_pri_key_opt add_constraint_pri_key add_primary_key_opt add_primary_key add_spatial_index_opt add_spatial_index %type index_hint_definition index_hint_list %type intnum_list @@ -11506,6 +11506,18 @@ qb_name_list: } ; +union_merge_list: +NAME_OB +{ + $$ = $1; +} +| union_merge_list opt_comma NAME_OB +{ + (void) $2; + malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $3); +} +; + optimize_hint: INDEX_HINT '(' qb_name_option relation_factor_in_hint NAME_OB opt_index_prefix ')' { @@ -11515,6 +11527,12 @@ INDEX_HINT '(' qb_name_option relation_factor_in_hint NAME_OB opt_index_prefix ' { malloc_non_terminal_node($$, result->malloc_pool_, T_NO_INDEX_HINT, 3, $3, $4, $5); } +| UNION_MERGE_HINT '(' qb_name_option relation_factor_in_hint union_merge_list ')' +{ + ParseNode *index_list = NULL; + merge_nodes(index_list, result, T_UNION_MERGE_LIST, $5); + malloc_non_terminal_node($$, result->malloc_pool_, T_UNION_MERGE_HINT, 3, $3, $4, index_list); +} | FULL_HINT '(' qb_name_option relation_factor_in_hint ')' { malloc_non_terminal_node($$, result->malloc_pool_, T_FULL_HINT, 2, $3, $4); diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index d39997f0b..685af2741 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -14621,6 +14621,12 @@ int ObDMLResolver::resolve_optimize_hint(const ParseNode &hint_node, } break; } + case T_UNION_MERGE_HINT: { + if (OB_FAIL(resolve_union_merge_hint(hint_node, opt_hint))) { + LOG_WARN("failed to resolve union merge hint", K(ret)); + } + break; + } case T_ORDERED: case T_LEADING: { if (OB_FAIL(resolve_join_order_hint(hint_node, opt_hint))) { @@ -14871,6 +14877,50 @@ int ObDMLResolver::resolve_index_hint(const TableItem &table, const ParseNode &i return ret; } +int ObDMLResolver::resolve_union_merge_hint(const ParseNode &hint_node, + ObOptHint *&opt_hint) +{ + int ret = OB_SUCCESS; + opt_hint = NULL; + ObUnionMergeHint *union_merge_hint = NULL; + ParseNode *table_node = NULL; + ParseNode *index_list_node = NULL; + ObString qb_name; + if (OB_UNLIKELY(3 != hint_node.num_child_) + || OB_ISNULL(table_node = hint_node.children_[1])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected index merge hint", K(ret), K(hint_node.type_), K(hint_node.num_child_), + K(table_node)); + } else if (OB_FAIL(ObQueryHint::create_hint(allocator_, hint_node.type_, union_merge_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_FAIL(resolve_qb_name_node(hint_node.children_[0], qb_name))) { + LOG_WARN("Failed to resolve qb name node", K(ret)); + } else if (OB_FAIL(resolve_table_relation_in_hint(*table_node, union_merge_hint->get_table()))) { + LOG_WARN("Resolve table relation fail", K(ret)); + } else if (OB_ISNULL(index_list_node = hint_node.children_[2])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected union merge hint", K(ret)); + } else { + const ParseNode *index_node = NULL; + ObString index_name; + for (int32_t i = 0; OB_SUCC(ret) && i < index_list_node->num_child_; i++) { + if (OB_ISNULL(index_node = index_list_node->children_[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected nullptr index node", K(ret)); + } else { + index_name.assign_ptr(index_node->str_value_, static_cast(index_node->str_len_)); + if (OB_FAIL(union_merge_hint->get_index_name_list().push_back(index_name))) { + LOG_WARN("failed to push back index name", K(index_name), K(union_merge_hint->get_index_name_list())); + } + } + } + union_merge_hint->set_qb_name(qb_name); + opt_hint = union_merge_hint; + } + LOG_TRACE("resolve union merge hint finished", KPC(union_merge_hint)); + return ret; +} + int ObDMLResolver::resolve_join_order_hint(const ParseNode &hint_node, ObOptHint *&opt_hint) { diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index 144aa526f..227b35737 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -942,6 +942,8 @@ private: ObOptHint *&opt_hint); int resolve_index_hint(const TableItem &table, // resolved mysql mode index hint after table const ParseNode &index_hint_node); + int resolve_union_merge_hint(const ParseNode &hint_node, + ObOptHint *&opt_hint); int resolve_table_parallel_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_join_order_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_join_hint(const ParseNode &join_node, ObIArray &join_hints); diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index 13a5cdf5b..b15fc04f1 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -1261,6 +1261,7 @@ const char* ObHint::get_hint_name(ObItemType type, bool is_enable_hint /* defaul case T_FULL_HINT: return "FULL"; case T_NO_INDEX_HINT: return "NO_INDEX"; case T_USE_DAS_HINT: return is_enable_hint ? "USE_DAS" : "NO_USE_DAS"; + case T_UNION_MERGE_HINT: return "UNION_MERGE"; case T_USE_COLUMN_STORE_HINT: return is_enable_hint ? "USE_COLUMN_TABLE" : "NO_USE_COLUMN_TABLE"; case T_INDEX_SS_HINT: return "INDEX_SS"; case T_INDEX_SS_ASC_HINT: return "INDEX_SS_ASC"; @@ -1364,6 +1365,7 @@ int ObHint::deep_copy_hint_contain_table(ObIAllocator *allocator, ObHint *&hint) case HINT_JOIN_FILTER: DEEP_COPY_NORMAL_HINT(ObJoinFilterHint); break; case HINT_WIN_MAGIC: DEEP_COPY_NORMAL_HINT(ObWinMagicHint); break; case HINT_COALESCE_AGGR: DEEP_COPY_NORMAL_HINT(ObCoalesceAggrHint); break; + case HINT_UNION_MERGE: DEEP_COPY_NORMAL_HINT(ObUnionMergeHint); break; default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected hint type to deep copy", K(ret), K(hint_class_)); @@ -2276,6 +2278,37 @@ int ObIndexHint::print_hint_desc(PlanText &plan_text) const return ret; } +int ObUnionMergeHint::assign(const ObUnionMergeHint &other) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(table_.assign(other.table_))) { + LOG_WARN("failed to assign table", K(ret)); + } else if (OB_FAIL(index_name_list_.assign(other.index_name_list_))) { + LOG_WARN("failed to assign index name list", K(ret)); + } else if (OB_FAIL(ObOptHint::assign(other))) { + LOG_WARN("fail to assign hint", K(ret)); + } + return ret; +} + +int ObUnionMergeHint::print_hint_desc(PlanText &plan_text) const +{ + int ret = OB_SUCCESS; + char *buf = plan_text.buf_; + int64_t &buf_len = plan_text.buf_len_; + int64_t &pos = plan_text.pos_; + if (OB_FAIL(table_.print_table_in_hint(plan_text))) { + LOG_WARN("fail to print table in hint", K(ret)); + } else { + for (int64_t i = 0; i < index_name_list_.count(); i++) { + if (OB_FAIL(BUF_PRINTF(" \"%.*s\"", index_name_list_.at(i).length(), index_name_list_.at(i).ptr()))) { + LOG_WARN("fail to print index name", K(ret)); + } + } + } + return ret; +} + int ObJoinHint::assign(const ObJoinHint &other) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index be0595ecd..9cb74f215 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -269,7 +269,8 @@ struct ObGlobalHint { #define COMPAT_VERSION_4_3_1 (oceanbase::common::cal_version(4, 3, 1, 0)) #define COMPAT_VERSION_4_3_2 (oceanbase::common::cal_version(4, 3, 2, 0)) #define COMPAT_VERSION_4_3_3 (oceanbase::common::cal_version(4, 3, 3, 0)) -#define LASTED_COMPAT_VERSION COMPAT_VERSION_4_3_3 +#define COMPAT_VERSION_4_3_4 (oceanbase::common::cal_version(4, 3, 4, 0)) +#define LASTED_COMPAT_VERSION COMPAT_VERSION_4_3_4 static bool is_valid_opt_features_version(uint64_t version) { return COMPAT_VERSION_4_0 <= version && LASTED_COMPAT_VERSION >= version; } @@ -572,7 +573,8 @@ public: HINT_PQ_SET, HINT_JOIN_FILTER, HINT_TABLE_DYNAMIC_SAMPLING, - HINT_PQ + HINT_PQ, + HINT_UNION_MERGE }; static const int64_t MAX_EXPR_STR_LENGTH_IN_HINT = 1024; @@ -649,6 +651,7 @@ public: bool is_coalesce_aggr_hint() const {return HINT_COALESCE_AGGR == hint_class_; } bool is_trans_added() const { return is_trans_added_; } bool set_trans_added(bool is_trans_added) { return is_trans_added_ = is_trans_added; } + bool is_union_merge_hint() const { return T_UNION_MERGE_HINT == hint_type_; } VIRTUAL_TO_STRING_KV("hint_type", get_type_name(hint_type_), K_(hint_class), K_(qb_name), @@ -1031,6 +1034,28 @@ private: int64_t index_prefix_; }; +class ObUnionMergeHint : public ObOptHint +{ +public: + ObUnionMergeHint(ObItemType hint_type = T_UNION_MERGE_HINT) + : ObOptHint(hint_type) + { + set_hint_class(HINT_UNION_MERGE); + } + int assign(const ObUnionMergeHint &other); + virtual ~ObUnionMergeHint() {} + virtual int get_all_table_in_hint(ObIArray &all_tables) override { return all_tables.push_back(&table_); } + virtual int print_hint_desc(PlanText &plan_text) const override; + ObTableInHint &get_table() { return table_; } + const ObTableInHint &get_table() const { return table_; } + common::ObIArray &get_index_name_list() { return index_name_list_; } + const common::ObIArray &get_index_name_list() const { return index_name_list_; } + INHERIT_TO_STRING_KV("ObHint", ObHint, K_(table), K_(index_name_list)); +private: + ObTableInHint table_; + common::ObSEArray index_name_list_; +}; + class ObTableParallelHint : public ObOptHint { public: diff --git a/src/sql/resolver/dml/ob_sql_hint.cpp b/src/sql/resolver/dml/ob_sql_hint.cpp index eea39c459..f9e5dc1d9 100644 --- a/src/sql/resolver/dml/ob_sql_hint.cpp +++ b/src/sql/resolver/dml/ob_sql_hint.cpp @@ -1345,7 +1345,8 @@ int ObStmtHint::merge_hint(ObHint &hint, || hint.is_join_filter_hint() || hint.is_table_parallel_hint() || hint.is_table_dynamic_sampling_hint() - || hint.is_pq_subquery_hint()) { + || hint.is_pq_subquery_hint() + || hint.is_union_merge_hint()) { if (OB_FAIL(add_var_to_array_no_dup(other_opt_hints_, &hint))) { LOG_WARN("failed to add var to array", K(ret)); } @@ -1501,6 +1502,10 @@ int ObLogPlanHint::init_other_opt_hints(ObSqlSchemaGuard &schema_guard, if (OB_FAIL(normal_hints_.push_back(hint))) { LOG_WARN("failed to push back", K(ret)); } + } else if (hint->is_union_merge_hint()) { + if (OB_FAIL(add_union_merge_hint(stmt, query_hint, *static_cast(hint)))) { + LOG_WARN("failed to add union merge hint", K(ret)); + } } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected hint type in other_opt_hints_", K(ret), K(*hint)); @@ -1566,6 +1571,25 @@ int ObLogPlanHint::add_index_hint(const ObDMLStmt &stmt, return ret; } +int ObLogPlanHint::add_union_merge_hint(const ObDMLStmt &stmt, + const ObQueryHint &query_hint, + const ObUnionMergeHint &union_merge_hint) +{ + int ret = OB_SUCCESS; + LogTableHint *log_table_hint = NULL; + if (OB_FAIL(get_log_table_hint_for_update(stmt, query_hint, union_merge_hint.get_table(), + true, log_table_hint))) { + LOG_WARN("failed to get log table hint by hint", K(ret)); + } else if (NULL == log_table_hint) { + /* do nothing */ + } else if (T_UNION_MERGE_HINT == union_merge_hint.get_hint_type()) { + if (NULL == log_table_hint->union_merge_hint_) { + log_table_hint->union_merge_hint_ = &union_merge_hint; + } + } + return ret; +} + int ObLogPlanHint::add_table_parallel_hint(const ObDMLStmt &stmt, const ObQueryHint &query_hint, const ObTableParallelHint &table_parallel_hint) @@ -1859,6 +1883,13 @@ int ObLogPlanHint::check_use_das(uint64_t table_id, bool &force_das, bool &force return ret; } +const ObUnionMergeHint *ObLogPlanHint::get_union_merge_hint(uint64_t table_id) const +{ + int ret = OB_SUCCESS; + const LogTableHint *log_table_hint = get_log_table_hint(table_id); + return NULL == log_table_hint ? NULL : log_table_hint->union_merge_hint_; +} + int ObLogPlanHint::check_use_column_store(uint64_t table_id, bool &force_column_store, bool &force_no_column_store) const { int ret = OB_SUCCESS; @@ -2387,6 +2418,7 @@ int LogTableHint::assign(const LogTableHint &other) parallel_hint_ = other.parallel_hint_; use_das_hint_ = other.use_das_hint_; use_column_store_hint_ = other.use_column_store_hint_; + union_merge_hint_ = other.union_merge_hint_; if (OB_FAIL(index_list_.assign(other.index_list_))) { LOG_WARN("failed to assign index list", K(ret)); } else if (OB_FAIL(index_hints_.assign(other.index_hints_))) { @@ -2407,7 +2439,7 @@ int LogTableHint::init_index_hints(ObSqlSchemaGuard &schema_guard) if (OB_ISNULL(table_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected log index hint", K(ret), K(this)); - } else if (index_hints_.empty()) { + } else if (index_hints_.empty() && union_merge_hint_ == nullptr) { /* do nothing */ } else if (OB_FAIL(schema_guard.get_can_read_index_array(table_->ref_id_, tids, @@ -2420,6 +2452,9 @@ int LogTableHint::init_index_hints(ObSqlSchemaGuard &schema_guard) } else if (table_index_count > OB_MAX_INDEX_PER_TABLE) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Table index count is bigger than OB_MAX_INDEX_PER_TABLE", K(ret), K(table_index_count)); + } else if (union_merge_hint_ != nullptr && + OB_FAIL(merge_index_list_.prepare_allocate(union_merge_hint_->get_index_name_list().count()))) { + LOG_WARN("failed to prepare allocate merge index list", KPC(union_merge_hint_), K(ret)); } else { LOG_TRACE("get readable index", K(table_index_count)); const share::schema::ObTableSchema *index_schema = NULL; @@ -2488,6 +2523,19 @@ int LogTableHint::init_index_hints(ObSqlSchemaGuard &schema_guard) LOG_WARN("fail to push back", K(ret), K(hint_pos)); } } + + if (OB_SUCC(ret) && union_merge_hint_ != nullptr) { + for (int64_t i = 0; OB_SUCC(ret) && i < union_merge_hint_->get_index_name_list().count(); ++i) { + if (0 != union_merge_hint_->get_index_name_list().at(i).case_compare(index_name)) { + /* do nothing */ + } else if (OB_UNLIKELY(i < 0 || i >= merge_index_list_.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(i), K(merge_index_list_.count()), K(ret)); + } else { + merge_index_list_.at(i) = index_id; + } + } + } } } if (OB_SUCC(ret)) { diff --git a/src/sql/resolver/dml/ob_sql_hint.h b/src/sql/resolver/dml/ob_sql_hint.h index e40271aef..29f1e7726 100644 --- a/src/sql/resolver/dml/ob_sql_hint.h +++ b/src/sql/resolver/dml/ob_sql_hint.h @@ -290,12 +290,14 @@ struct LogTableHint parallel_hint_(NULL), use_das_hint_(NULL), use_column_store_hint_(NULL), + union_merge_hint_(NULL), dynamic_sampling_hint_(NULL), is_ds_hint_conflict_(false) {} LogTableHint(const TableItem *table) : table_(table), parallel_hint_(NULL), use_das_hint_(NULL), use_column_store_hint_(NULL), + union_merge_hint_(NULL), dynamic_sampling_hint_(NULL), is_ds_hint_conflict_(false) {} int assign(const LogTableHint &other); @@ -305,7 +307,8 @@ struct LogTableHint bool is_valid() const { return !index_list_.empty() || NULL != parallel_hint_ || NULL != use_das_hint_ || !join_filter_hints_.empty() || dynamic_sampling_hint_ != NULL - || NULL != use_column_store_hint_; } + || NULL != use_column_store_hint_ + || NULL != union_merge_hint_; } int get_join_filter_hint(const ObRelIds &left_tables, bool part_join_filter, const ObJoinFilterHint *&hint) const; @@ -319,7 +322,7 @@ struct LogTableHint int get_index_prefix(const uint64_t index_id, int64_t &index_prefix) const; TO_STRING_KV(K_(table), K_(index_list), K_(index_hints), - K_(parallel_hint), K_(use_das_hint), + K_(parallel_hint), K_(use_das_hint), K_(union_merge_hint), K_(join_filter_hints), K_(left_tables), KPC(dynamic_sampling_hint_), K(is_ds_hint_conflict_)); @@ -329,6 +332,8 @@ struct LogTableHint const ObTableParallelHint *parallel_hint_; const ObIndexHint *use_das_hint_; const ObIndexHint *use_column_store_hint_; + const ObUnionMergeHint *union_merge_hint_; + common::ObSEArray merge_index_list_; ObSEArray join_filter_hints_; ObSEArray left_tables_; // left table relids in join filter hint const ObTableDynamicSamplingHint *dynamic_sampling_hint_; @@ -426,6 +431,9 @@ struct ObLogPlanHint int add_index_hint(const ObDMLStmt &stmt, const ObQueryHint &query_hint, const ObIndexHint &index_hint); + int add_union_merge_hint(const ObDMLStmt &stmt, + const ObQueryHint &query_hint, + const ObUnionMergeHint &union_merge_hint); int add_join_hint(const ObDMLStmt &stmt, const ObQueryHint &query_hint, const ObJoinHint &join_hint); @@ -451,6 +459,7 @@ struct ObLogPlanHint bool config_disable, JoinFilterPushdownHintInfo& info) const; int check_use_das(uint64_t table_id, bool &force_das, bool &force_no_das) const; + const ObUnionMergeHint *get_union_merge_hint(uint64_t table_id) const; int check_use_column_store(uint64_t table_id, bool &force_column_store, bool &force_no_column_store) const; int check_use_skip_scan(uint64_t table_id, uint64_t index_id, bool &force_skip_scan,