hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] INDEX MERGE PHASE I

Browse Source
This commit is contained in:
pe-99y 2024-09-25 04:55:20 +00:00 committed by ob-robot
parent 2f5b00730d
commit 02dbf9bcbd
41 changed files with 3405 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/objit/include/objit/common/ob_item_type.h
View File

@ -2597,7 +2597,8 @@ typedef enum ObItemType
T_RB_ITERATE_EXPRESSION = 4737,
T_MODULE_DATA = 4738,
T_MODULE_NAME = 4739,
T_UNION_MERGE_HINT = 4740,
T_UNION_MERGE_LIST = 4741,
T_MAX //Attention: add a new type before T_MAX
} ObItemType;

2
src/share/system_variable/ob_system_variable_init.cpp
View File

@ -3308,7 +3308,7 @@ static struct VarsInit{
}();
[&] (){
ObSysVars[234].default_value_ = "4.3.3.0" ;
ObSysVars[234].default_value_ = "4.3.4.0" ;
ObSysVars[234].info_ = "enabling a series of optimizer features based on an OceanBase release number" ;
ObSysVars[234].name_ = "optimizer_features_enable" ;
ObSysVars[234].data_type_ = ObVarcharType ;

2
src/share/system_variable/ob_system_variable_init.json
View File

@ -3349,7 +3349,7 @@
"optimizer_features_enable": {
"id": 10150,
"name": "optimizer_features_enable",
"default_value": "4.3.3.0",
"default_value": "4.3.4.0",
"base_value": "",
"data_type": "varchar",
"info": "enabling a series of optimizer features based on an OceanBase release number",

1
src/sql/CMakeLists.txt
View File

@ -80,6 +80,7 @@ ob_set_subtarget(ob_sql das
das/iter/ob_das_text_retrieval_merge_iter.cpp
das/iter/ob_das_iter_utils.cpp
das/iter/ob_das_vid_merge_iter.cpp
das/iter/ob_das_index_merge_iter.cpp
)
ob_set_subtarget(ob_sql dtl

227
src/sql/code_generator/ob_tsc_cg_service.cpp
View File

@ -46,6 +46,9 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc
}
tsc_ctdef.scan_flags_ = query_flag;
if (op.use_index_merge()) {
tsc_ctdef.use_index_merge_ = true;
}
if (OB_SUCC(ret) && op.get_table_type() == share::schema::EXTERNAL_TABLE) {
const ObTableSchema *table_schema = nullptr;
ObSqlSchemaGuard *schema_guard = cg_.opt_ctx_->get_sql_schema_guard();
@ -197,6 +200,15 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc
}
}
if (OB_SUCC(ret) && op.use_index_merge()) {
ObDASIndexMergeCtDef *index_merge_ctdef = nullptr;
if (OB_FAIL(generate_index_merge_ctdef(op, tsc_ctdef, root_ctdef))) {
LOG_WARN("failed to generate index merge ctdef", K(ret));
} else {
need_attach = true;
}
}
ObDASVIdMergeCtDef *vid_merge_ctdef = nullptr;
if (OB_SUCC(ret) && op.get_index_back()) {
ObDASTableLookupCtDef *lookup_ctdef = nullptr;
@ -499,9 +511,27 @@ int ObTscCgService::generate_tsc_filter(const ObLogTableScan &op, ObTableScanSpe
ObArray<ObRawExpr *> lookup_pushdown_filters;
ObDASScanCtDef &scan_ctdef = spec.tsc_ctdef_.scan_ctdef_;
ObDASScanCtDef *lookup_ctdef = spec.tsc_ctdef_.get_lookup_ctdef();
if (OB_FAIL(op.extract_pushdown_filters(nonpushdown_filters,
scan_pushdown_filters,
lookup_pushdown_filters))) {
if (op.use_index_merge()) {
// full filters is used for final check when in index merge
// we need to pushdown full filters to lookup as much as possible to avoid
// the transmission of large results during DAS remote execution
// all index table scan filters are generated in @generate_das_scan_ctdef()
const ObIArray<ObRawExpr*> &full_filters = op.get_full_filters();
if (OB_FAIL(op.extract_nonpushdown_filters(full_filters,
nonpushdown_filters,
lookup_pushdown_filters))) {
LOG_WARN("failed to extract lookup pushdown filters", K(ret));
} else if (lookup_ctdef != nullptr && OB_FAIL(generate_pd_storage_flag(op.get_plan(),
op.get_ref_table_id(),
op.get_access_exprs(),
op.get_type(),
op.get_index_back() && op.get_is_index_global(),
lookup_ctdef->pd_expr_spec_))) {
LOG_WARN("generate pd storage flag for lookup ctdef failed", K(ret));
}
} else if (OB_FAIL(op.extract_pushdown_filters(nonpushdown_filters,
scan_pushdown_filters,
lookup_pushdown_filters))) {
LOG_WARN("extract pushdown filters failed", K(ret));
} else if (op.get_contains_fake_cte()) {
// do nothing
@ -628,6 +658,7 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op,
{
int ret = OB_SUCCESS;
const ObTableID &scan_table_id = scan_ctdef.ref_table_id_;
const bool use_index_merge = scan_ctdef.is_index_merge_;
if ((op.is_text_retrieval_scan() && scan_table_id != op.get_ref_table_id())
|| (op.is_multivalue_index_scan() && scan_table_id == op.get_doc_id_index_table_id())) {
// non main table scan in text retrieval
@ -644,23 +675,29 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op,
if (OB_FAIL(extract_rowkey_vid_access_columns(op, scan_ctdef, access_exprs))) {
LOG_WARN("fail to extract rowkey doc access columns", K(ret));
}
} else if (op.get_index_back() && scan_table_id == op.get_real_index_table_id()) {
} else if (op.get_index_back() && (scan_table_id != op.get_real_ref_table_id() || use_index_merge)) {
//this das scan is index scan and will lookup the data table later
//index scan + lookup data table: the index scan only need access
//range condition columns + index filter columns + the data table rowkeys
const ObIArray<ObRawExpr*> &range_conditions = op.get_range_conditions();
const ObIArray<ObRawExpr*> &range_conditions = use_index_merge ?
op.get_index_range_conds(scan_ctdef.index_merge_idx_) : op.get_range_conditions();
if (OB_FAIL(ObRawExprUtils::extract_column_exprs(range_conditions, access_exprs))) {
LOG_WARN("extract column exprs failed", K(ret));
}
//store index filter columns
if (OB_SUCC(ret)) {
ObArray<ObRawExpr *> filter_columns;
ObArray<ObRawExpr *> nonpushdown_filters;
ObArray<ObRawExpr *> scan_pushdown_filters;
ObArray<ObRawExpr *> lookup_pushdown_filters;
ObArray<ObRawExpr *> filter_columns; // the column in scan pushdown filters
if (OB_FAIL(const_cast<ObLogTableScan &>(op).extract_pushdown_filters(nonpushdown_filters,
scan_pushdown_filters,
lookup_pushdown_filters))) {
if (use_index_merge &&
OB_FAIL(scan_pushdown_filters.assign(op.get_index_filters(scan_ctdef.index_merge_idx_)))) {
LOG_WARN("failed to assign index merge filters", K(ret));
} else if (!use_index_merge &&
OB_FAIL(const_cast<ObLogTableScan &>(op).extract_pushdown_filters(nonpushdown_filters,
scan_pushdown_filters,
lookup_pushdown_filters))) {
LOG_WARN("extract pushdown filters failed", K(ret));
} else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(scan_pushdown_filters,
filter_columns))) {
@ -669,6 +706,7 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op,
LOG_WARN("append filter column to access exprs failed", K(ret));
}
}
//store data table rowkeys
if (OB_SUCC(ret)) {
if (OB_FAIL(append_array_no_dup(access_exprs, op.get_rowkey_exprs()))) {
@ -679,6 +717,22 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op,
}
} else if (OB_FAIL(access_exprs.assign(op.get_access_exprs()))) {
LOG_WARN("assign access exprs failed", K(ret));
} else if (op.use_index_merge()) {
// add lookup pushdown exprs when use index merge
ObArray<ObRawExpr*> nonpushdown_filters;
ObArray<ObRawExpr*> lookup_pushdown_filters;
ObArray<ObRawExpr*> filter_columns;
const ObIArray<ObRawExpr*> &full_filters = op.get_full_filters();
if (OB_FAIL(op.extract_nonpushdown_filters(full_filters,
nonpushdown_filters,
lookup_pushdown_filters))) {
LOG_WARN("failed to extract lookup pushdown filters", K(ret));
} else if (OB_FAIL(ObRawExprUtils::extract_column_exprs(lookup_pushdown_filters,
filter_columns))) {
LOG_WARN("failed to extract column exprs", K(ret));
} else if (OB_FAIL(append_array_no_dup(access_exprs, filter_columns))) {
LOG_WARN("failed to append filter columns", K(ret));
}
}
// store group_id_expr when use group id
if (OB_SUCC(ret) && op.use_group_id()) {
@ -731,6 +785,8 @@ int ObTscCgService::extract_das_access_exprs(const ObLogTableScan &op,
LOG_WARN("failed to remove generated column exprs", K(ret));
}
}
LOG_TRACE("extract das access exprs", K(scan_table_id), K(op.get_real_ref_table_id()),
K(op.get_index_back()), K(scan_ctdef.is_index_merge_), K(access_exprs));
return ret;
}
@ -1028,6 +1084,41 @@ int ObTscCgService::generate_das_scan_ctdef(const ObLogTableScan &op,
LOG_WARN("failed to init result outputs", K(ret));
}
}
//8. generate rowkey exprs and index pushdown filters when use index merge
if (OB_SUCC(ret) && scan_ctdef.is_index_merge_) {
ObArray<ObRawExpr*> rowkey_exprs;
ObArray<ObRawExpr*> scan_pushdown_filters;
if (OB_FAIL(rowkey_exprs.assign(op.get_rowkey_exprs()))) {
LOG_WARN("failed to assign rowkey exprs", K(ret));
} else if (!op.get_is_index_global() && OB_FAIL(mapping_oracle_real_agent_virtual_exprs(op, rowkey_exprs))) {
LOG_WARN("failed to mapping oracle real virtual exprs", K(ret));
} else if (OB_FAIL(cg_.generate_rt_exprs(rowkey_exprs, scan_ctdef.rowkey_exprs_))) {
LOG_WARN("failed to generate main table rowkey exprs", K(ret));
} else if (OB_FAIL(op.get_index_filters(scan_ctdef.index_merge_idx_, scan_pushdown_filters))) {
LOG_WARN("failed to get index filters", K(ret));
} else if (!scan_pushdown_filters.empty()) {
if (OB_FAIL(generate_pd_storage_flag(op.get_plan(),
op.get_ref_table_id(),
op.get_access_exprs(),
op.get_type(),
op.get_index_back() && op.get_is_index_global(),
scan_ctdef.pd_expr_spec_))) {
LOG_WARN("failed to generate pd storage flag for index scan ctdef", K(scan_ctdef.ref_table_id_), K(ret));
} else if (OB_FAIL(cg_.generate_rt_exprs(scan_pushdown_filters, scan_ctdef.pd_expr_spec_.pushdown_filters_))) {
LOG_WARN("failed to generate index scan pushdown filter", K(scan_ctdef.ref_table_id_), K(ret));
} else if (scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_filter_pushdown()) {
ObPushdownFilterConstructor filter_constructor(
&cg_.phy_plan_->get_allocator(), cg_, &op,
scan_ctdef.pd_expr_spec_.pd_storage_flag_.is_use_column_store());
if (OB_FAIL(filter_constructor.apply(
scan_pushdown_filters, scan_ctdef.pd_expr_spec_.pd_storage_filters_.get_pushdown_filter()))) {
LOG_WARN("failed to apply filter constructor", K(ret));
}
LOG_TRACE("index merge pushdown filters", K(scan_ctdef.ref_table_id_), K(scan_pushdown_filters));
}
}
}
return ret;
}
@ -1039,7 +1130,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op,
int ret = OB_SUCCESS;
ObArray<ObRawExpr*> das_output_cols;
const ObTableID &table_id = scan_ctdef.ref_table_id_;
const bool use_index_merge = scan_ctdef.is_index_merge_;
if ((op.is_text_retrieval_scan() && table_id != op.get_ref_table_id()) ||
(op.is_multivalue_index_scan() && table_id == op.get_doc_id_index_table_id())) {
// non main table scan in text retrieval
@ -1057,7 +1148,7 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op,
if (OB_FAIL(extract_rowkey_vid_output_columns_ids(index_schema, op, scan_ctdef, output_cids))) {
LOG_WARN("fail to extract rowkey doc output columns ids", K(ret));
}
} else if ((op.get_index_back() || op.is_multivalue_index_scan()) && op.get_real_index_table_id() == table_id) {
} else if ((op.get_index_back() || op.is_multivalue_index_scan()) && (op.get_real_ref_table_id() != table_id || use_index_merge)) {
//this situation is index lookup, and the index table scan is being processed
//the output column id of index lookup is the rowkey of the data table
const ObTableSchema *table_schema = nullptr;
@ -1148,6 +1239,9 @@ int ObTscCgService::extract_das_output_column_ids(const ObLogTableScan &op,
}
}
}
LOG_TRACE("extract das output column ids", K(ret), K(table_id), K(op.get_ref_table_id()),
K(op.get_index_back()), K(scan_ctdef.is_index_merge_), K(output_cids));
return ret;
}
int ObTscCgService::extract_das_column_ids(const ObIArray<ObRawExpr*> &column_exprs,
@ -1553,6 +1647,111 @@ int ObTscCgService::generate_text_ir_ctdef(const ObLogTableScan &op,
return ret;
}
int ObTscCgService::generate_index_merge_ctdef(const ObLogTableScan &op,
ObTableScanCtDef &tsc_ctdef,
ObDASBaseCtDef *&root_ctdef)
{
int ret = OB_SUCCESS;
const IndexMergePath *path = nullptr;
common::ObIAllocator &ctdef_alloc = cg_.phy_plan_->get_allocator();
if (OB_ISNULL(op.get_access_path())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret));
} else {
OB_ASSERT(op.use_index_merge());
path = static_cast<const IndexMergePath*>(op.get_access_path());
ObIndexMergeNode *root = path->root_;
if (OB_FAIL(generate_index_merge_node_ctdef(op, root, ctdef_alloc, root_ctdef))) {
LOG_WARN("failed to generate index merge ctdef", K(root_ctdef));
}
}
return ret;
}
int ObTscCgService::generate_index_merge_node_ctdef(const ObLogTableScan &op,
ObIndexMergeNode *node,
common::ObIAllocator &alloc,
ObDASBaseCtDef *&node_ctdef)
{
int ret = OB_SUCCESS;
bool has_rowscn = false;
if (OB_ISNULL(node) || !node->is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", KPC(node), K(ret));
} else if (node->is_leaf_node_) {
ObDASScanCtDef *scan_ctdef = nullptr;
if (OB_ISNULL(node->ap_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null access path", K(ret));
} else if (OB_FAIL(ObDASTaskFactory::alloc_das_ctdef(DAS_OP_TABLE_SCAN, alloc, scan_ctdef))) {
LOG_WARN("failed to allocate scan ctdef", K(ret));
} else if (FALSE_IT(scan_ctdef->ref_table_id_ = node->index_tid_)) {
} else if (FALSE_IT(scan_ctdef->index_merge_idx_ = node->idx_)) {
} else if (FALSE_IT(scan_ctdef->is_index_merge_ = true)) {
} else if (OB_FAIL(generate_das_scan_ctdef(op, *scan_ctdef, has_rowscn))) {
LOG_WARN("failed to generate das scan ctdef", KPC(scan_ctdef), K(ret));
} else if (OB_NOT_NULL(node->ap_->pre_query_range_) &&
OB_FAIL(scan_ctdef->pre_query_range_.deep_copy(*node->ap_->pre_query_range_))) {
LOG_WARN("failed to deep copy pre query range", K(ret));
} else if (!node->is_ror_) {
// for non-ROR situations, we need to insert a sort iter
ObDASSortCtDef *sort_ctdef = nullptr;
ObSEArray<OrderItem, 2> order_items;
ObArray<ObRawExpr*> rowkey_exprs;
if (OB_FAIL(rowkey_exprs.assign(op.get_rowkey_exprs()))) {
LOG_WARN("failed to assign rowkey exprs", K(ret));
} else if (!op.get_is_index_global() && OB_FAIL(mapping_oracle_real_agent_virtual_exprs(op, rowkey_exprs))) {
LOG_WARN("failed to mapping oracle real virtual exprs", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < rowkey_exprs.count(); i++) {
if (OB_FAIL(order_items.push_back(OrderItem(rowkey_exprs.at(i), op.get_scan_direction())))) {
LOG_WARN("failed to push back order item", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ObDASTaskFactory::alloc_das_ctdef(DAS_OP_SORT, alloc, sort_ctdef))) {
LOG_WARN("failed to allocate sort ctdef", K(ret));
} else if (OB_FAIL(generate_das_sort_ctdef(order_items,
false,
nullptr,
nullptr,
scan_ctdef,
sort_ctdef))) {
LOG_WARN("failed to generate das sort ctdef", K(ret));
} else {
node_ctdef = sort_ctdef;
}
}
}
} else {
node_ctdef = scan_ctdef;
}
} else {
ObDASIndexMergeCtDef *merge_ctdef = nullptr;
ObDASBaseCtDef *left_ctdef = nullptr;
ObDASBaseCtDef *right_ctdef = nullptr;
if (OB_FAIL(ObDASTaskFactory::alloc_das_ctdef(DAS_OP_INDEX_MERGE, alloc, merge_ctdef))) {
LOG_WARN("failed to allocate index merge ctdef", K(ret));
} else if (OB_ISNULL(merge_ctdef->children_ = OB_NEW_ARRAY(ObDASBaseCtDef*, &alloc, 2))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate index merge ctdef children", K(ret));
} else if (OB_FAIL(generate_index_merge_node_ctdef(op, node->left_node_, alloc, left_ctdef))) {
LOG_WARN("failed to generate left node ctdef", K(node->left_node_), K(ret));
} else if (OB_FAIL(generate_index_merge_node_ctdef(op, node->right_node_, alloc, right_ctdef))) {
LOG_WARN("failed to generate right node ctdef", K(node->right_node_), K(ret));
} else {
merge_ctdef->children_[0] = left_ctdef;
merge_ctdef->children_[1] = right_ctdef;
merge_ctdef->children_cnt_ = 2;
merge_ctdef->merge_type_ = node->merge_type_;
merge_ctdef->is_left_child_leaf_node_ = (left_ctdef->op_type_ != DAS_OP_INDEX_MERGE);
merge_ctdef->is_reverse_ = is_descending_direction(op.get_scan_direction());
node_ctdef = merge_ctdef;
}
}
return ret;
}
int ObTscCgService::append_fts_relavence_project_col(
ObDASIRAuxLookupCtDef *aux_lookup_ctdef,
ObDASIRScanCtDef *ir_scan_ctdef)
@ -2502,7 +2701,7 @@ int ObTscCgService::generate_das_sort_ctdef(
if (OB_FAIL(cg_.generate_rt_expr(*order_item.expr_, expr))) {
LOG_WARN("failed to generate rt expr", K(ret));
} else {
ObSortFieldCollation field_collation(field_idx++,
ObSortFieldCollation field_collation(field_idx,
expr->datum_meta_.cs_type_,
order_item.is_ascending(),
(order_item.is_null_first() ^ order_item.is_ascending()) ? NULL_LAST : NULL_FIRST);
@ -2538,9 +2737,13 @@ int ObTscCgService::generate_das_sort_ctdef(
} else if (ObDASTaskFactory::is_attached(child_ctdef->op_type_)
&& OB_FAIL(append_array_no_dup(result_output, static_cast<ObDASAttachCtDef *>(child_ctdef)->result_output_))) {
LOG_WARN("failed to append child result output", K(ret));
} else if (child_ctdef->op_type_ == DAS_OP_TABLE_SCAN
&& OB_FAIL(append_array_no_dup(result_output, static_cast<ObDASScanCtDef *>(child_ctdef)->result_output_))) {
LOG_WARN("failed to append child result output", K(ret));
} else if (OB_FAIL(sort_ctdef->result_output_.assign(result_output))) {
LOG_WARN("failed to assign result output", K(ret));
}
LOG_TRACE("generate sort ctdef finished", K(sort_keys), K(sort_ctdef->sort_exprs_), K(result_output), K(ret));
return ret;
}

5
src/sql/code_generator/ob_tsc_cg_service.h
View File

@ -136,6 +136,11 @@ private:
ObDASSortCtDef *&sort_ctdef);
int mapping_oracle_real_agent_virtual_exprs(const ObLogTableScan &op,
common::ObIArray<ObRawExpr*> &access_exprs);
int generate_index_merge_ctdef(const ObLogTableScan &op, ObTableScanCtDef &tsc_ctdef, ObDASBaseCtDef *&root_ctdef);
int generate_index_merge_node_ctdef(const ObLogTableScan &op,
ObIndexMergeNode *node,
common::ObIAllocator &alloc,
ObDASBaseCtDef *&node_ctdef);
private:
ObStaticEngineCG &cg_;

931
src/sql/das/iter/ob_das_index_merge_iter.cpp Normal file
View File

@ -0,0 +1,931 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_DAS
#include "sql/das/iter/ob_das_index_merge_iter.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
int ObDASIndexMergeIter::RowStore::init(common::ObIAllocator &allocator)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(store_rows_ =
static_cast<LastDASStoreRow*>(allocator.alloc(max_size_ * sizeof(LastDASStoreRow))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory", K_(max_size), K(ret));
} else {
for (int64_t i = 0; i < max_size_; i++) {
new (store_rows_ + i) LastDASStoreRow(allocator);
store_rows_[i].reuse_ = true;
}
}
return ret;
}
int ObDASIndexMergeIter::RowStore::save(bool is_vectorized, int64_t size)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(size > max_size_) || OB_ISNULL(store_rows_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error for save store rows", K(size), K_(max_size), K_(store_rows), K(ret));
} else {
if (is_vectorized) {
ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_);
batch_info_guard.set_batch_size(size);
for (int64_t i = 0; OB_SUCC(ret) && i < size; i++) {
batch_info_guard.set_batch_idx(i);
if (OB_FAIL(store_rows_[i].save_store_row(*exprs_, *eval_ctx_))) {
LOG_WARN("index merge iter failed to store rows", K(ret));
}
}
} else if (OB_FAIL(store_rows_[0].save_store_row(*exprs_, *eval_ctx_))) {
LOG_WARN("index merge iter failed to store rows", K(ret));
}
}
if (OB_SUCC(ret)) {
cur_idx_ = 0;
saved_size_ = size;
}
return ret;
}
int ObDASIndexMergeIter::RowStore::to_expr(bool is_vectorized, int64_t size)
{
int ret = OB_SUCCESS;
if (is_vectorized) {
if (cur_idx_ + size > saved_size_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument, exceeds saved size", K_(cur_idx), K(size), K_(saved_size), K(ret));
} else {
ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_);
batch_info_guard.set_batch_size(size);
for (int64_t i = 0; OB_SUCC(ret) && i < size; i++) {
batch_info_guard.set_batch_idx(i);
OZ(store_rows_[cur_idx_ + i].store_row_->to_expr<true>(*exprs_, *eval_ctx_));
}
cur_idx_ += size;
}
} else {
OZ(store_rows_[cur_idx_].store_row_->to_expr<false>(*exprs_, *eval_ctx_));
cur_idx_++;
}
return ret;
}
const ObDatum *ObDASIndexMergeIter::RowStore::cur_datums()
{
OB_ASSERT(cur_idx_ < saved_size_);
return store_rows_ != nullptr ? store_rows_[cur_idx_].store_row_->cells() : nullptr;
}
void ObDASIndexMergeIter::RowStore::reuse()
{
cur_idx_ = OB_INVALID_INDEX;
saved_size_ = 0;
}
void ObDASIndexMergeIter::RowStore::reset()
{
if (OB_NOT_NULL(store_rows_)) {
for (int64_t i = 0; i < max_size_; i++) {
store_rows_[i].~LastDASStoreRow();
}
store_rows_ = nullptr;
}
exprs_ = nullptr;
eval_ctx_ = nullptr;
max_size_ = 1;
saved_size_ = 0;
cur_idx_ = OB_INVALID_INDEX;
}
int ObDASIndexMergeIter::inner_init(ObDASIterParam &param)
{
int ret = OB_SUCCESS;
if (ObDASIterType::DAS_ITER_INDEX_MERGE != param.type_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("inner init das iter with bad param type", K(param));
} else {
ObDASIndexMergeIterParam &index_merge_param = static_cast<ObDASIndexMergeIterParam&>(param);
merge_type_ = index_merge_param.merge_type_;
left_iter_ = index_merge_param.left_iter_;
right_iter_ = index_merge_param.right_iter_;
rowkey_exprs_ = index_merge_param.rowkey_exprs_;
left_output_ = index_merge_param.left_output_;
right_output_ = index_merge_param.right_output_;
get_next_row_ = (merge_type_ == INDEX_MERGE_UNION) ?
&ObDASIndexMergeIter::union_get_next_row : &ObDASIndexMergeIter::intersect_get_next_row;
get_next_rows_ = (merge_type_ == INDEX_MERGE_UNION) ?
&ObDASIndexMergeIter::union_get_next_rows : &ObDASIndexMergeIter::intersect_get_next_rows;
merge_ctdef_ = index_merge_param.ctdef_;
merge_rtdef_ = index_merge_param.rtdef_;
tx_desc_ = index_merge_param.tx_desc_;
snapshot_ = index_merge_param.snapshot_;
is_reverse_ = index_merge_param.is_reverse_;
if (merge_ctdef_->is_left_child_leaf_node_) {
OB_ASSERT(merge_rtdef_->children_[0]->op_type_ == DAS_OP_TABLE_SCAN ||
(merge_rtdef_->children_[0]->op_type_ == DAS_OP_SORT &&
merge_rtdef_->children_[0]->children_[0]->op_type_ == DAS_OP_TABLE_SCAN));
left_scan_rtdef_ = merge_rtdef_->children_[0]->op_type_ == DAS_OP_TABLE_SCAN ?
static_cast<ObDASScanRtDef*>(merge_rtdef_->children_[0]) :
static_cast<ObDASScanRtDef*>(merge_rtdef_->children_[0]->children_[0]);
OB_ASSERT(left_scan_rtdef_ != nullptr);
left_scan_ctdef_ = static_cast<const ObDASScanCtDef*>(left_scan_rtdef_->ctdef_);
}
OB_ASSERT(merge_rtdef_->children_[1]->op_type_ == DAS_OP_TABLE_SCAN ||
(merge_rtdef_->children_[1]->op_type_ == DAS_OP_SORT &&
merge_rtdef_->children_[1]->children_[0]->op_type_ == DAS_OP_TABLE_SCAN));
right_scan_rtdef_ = merge_rtdef_->children_[1]->op_type_ == DAS_OP_TABLE_SCAN ?
static_cast<ObDASScanRtDef*>(merge_rtdef_->children_[1]) :
static_cast<ObDASScanRtDef*>(merge_rtdef_->children_[1]->children_[0]);
OB_ASSERT(right_scan_rtdef_ != nullptr);
right_scan_ctdef_ = static_cast<const ObDASScanCtDef*>(right_scan_rtdef_->ctdef_);
lib::ContextParam context_param;
context_param.set_mem_attr(MTL_ID(), "DASIndexMerge", ObCtxIds::DEFAULT_CTX_ID)
.set_properties(lib::USE_TL_PAGE_OPTIONAL);
if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_ctx_, context_param))) {
LOG_WARN("failed to create index merge memctx", K(ret));
} else {
const common::ObIArray<ObExpr*> *left_exprs = (merge_type_ == INDEX_MERGE_UNION) ?
rowkey_exprs_ : left_output_;
const common::ObIArray<ObExpr*> *right_exprs = (merge_type_ == INDEX_MERGE_UNION) ?
rowkey_exprs_ : right_output_;
common::ObArenaAllocator &alloc = mem_ctx_->get_arena_allocator();
if (OB_ISNULL(left_row_store_ = OB_NEWx(RowStore, &alloc, left_exprs, eval_ctx_, max_size_)) ||
OB_ISNULL(right_row_store_ = OB_NEWx(RowStore, &alloc, right_exprs, eval_ctx_, max_size_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate row store", K(max_size_), K(ret));
} else if (OB_FAIL(left_row_store_->init(alloc)) || OB_FAIL(right_row_store_->init(alloc))) {
LOG_WARN("failed to init row store", K(ret));
}
}
}
return ret;
}
int ObDASIndexMergeIter::init_scan_param(const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const sql::ObDASScanCtDef *ctdef,
sql::ObDASScanRtDef *rtdef,
ObTableScanParam &scan_param)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KPC(ctdef), KPC(rtdef), K(ls_id), K(tablet_id));
} else {
uint64_t tenant_id = MTL_ID();
scan_param.tenant_id_ = tenant_id;
scan_param.key_ranges_.set_attr(ObMemAttr(tenant_id, "ScanParamKR"));
scan_param.ss_key_ranges_.set_attr(ObMemAttr(tenant_id, "ScanParamSSKR"));
scan_param.tx_lock_timeout_ = rtdef->tx_lock_timeout_;
scan_param.index_id_ = ctdef->ref_table_id_;
scan_param.is_get_ = false; // scan
scan_param.is_for_foreign_check_ = false;
scan_param.timeout_ = rtdef->timeout_ts_;
scan_param.scan_flag_ = rtdef->scan_flag_;
scan_param.reserved_cell_count_ = ctdef->access_column_ids_.count();
scan_param.allocator_ = &rtdef->stmt_allocator_;
scan_param.scan_allocator_ = &rtdef->scan_allocator_;
scan_param.sql_mode_ = rtdef->sql_mode_;
scan_param.frozen_version_ = rtdef->frozen_version_;
scan_param.force_refresh_lc_ = rtdef->force_refresh_lc_;
scan_param.output_exprs_ = &(ctdef->pd_expr_spec_.access_exprs_);
scan_param.calc_exprs_ = &(ctdef->pd_expr_spec_.calc_exprs_);
scan_param.aggregate_exprs_ = &(ctdef->pd_expr_spec_.pd_storage_aggregate_output_);
scan_param.table_param_ = &(ctdef->table_param_);
scan_param.op_ = rtdef->p_pd_expr_op_;
scan_param.row2exprs_projector_ = rtdef->p_row2exprs_projector_;
scan_param.schema_version_ = ctdef->schema_version_;
scan_param.tenant_schema_version_ = rtdef->tenant_schema_version_;
scan_param.limit_param_ = rtdef->limit_param_;
scan_param.need_scn_ = rtdef->need_scn_;
scan_param.pd_storage_flag_ = ctdef->pd_expr_spec_.pd_storage_flag_.pd_flag_;
scan_param.fb_snapshot_ = rtdef->fb_snapshot_;
scan_param.fb_read_tx_uncommitted_ = rtdef->fb_read_tx_uncommitted_;
scan_param.ls_id_ = ls_id;
scan_param.tablet_id_ = tablet_id;
if (!ctdef->pd_expr_spec_.pushdown_filters_.empty()) {
scan_param.op_filters_ = &ctdef->pd_expr_spec_.pushdown_filters_;
}
scan_param.pd_storage_filters_ = rtdef->p_pd_expr_op_->pd_storage_filters_;
if (OB_NOT_NULL(tx_desc_)) {
scan_param.tx_id_ = tx_desc_->get_tx_id();
} else {
scan_param.tx_id_.reset();
}
if (OB_NOT_NULL(snapshot_)) {
if (OB_FAIL(scan_param.snapshot_.assign(*snapshot_))) {
LOG_WARN("assign snapshot fail", K(ret));
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("null snapshot", K(ret), KPC_(snapshot));
}
if (FAILEDx(scan_param.column_ids_.assign(ctdef->access_column_ids_))) {
LOG_WARN("failed to init column ids", K(ret));
} else if (OB_FAIL(prepare_scan_ranges(scan_param, rtdef))) {
LOG_WARN("failed to prepare scan ranges", K(ret));
}
}
return ret;
}
int ObDASIndexMergeIter::prepare_scan_ranges(ObTableScanParam &scan_param, const ObDASScanRtDef *rtdef)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(rtdef)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr scan rtdef", K(ret));
} else if (OB_FAIL(scan_param.key_ranges_.assign(rtdef->key_ranges_))) {
LOG_WARN("failed to assign key ranges", K(ret));
} else if (OB_FAIL(scan_param.ss_key_ranges_.assign(rtdef->ss_key_ranges_))) {
LOG_WARN("failed to assign ss key ranges", K(ret));
} else if (OB_FAIL(scan_param.mbr_filters_.assign(rtdef->mbr_filters_))) {
LOG_WARN("failed to assign mbr filters", K(ret));
}
LOG_TRACE("index merge iter prepare scan ranges", K(scan_param), KPC(rtdef), K(ret));
return ret;
}
void ObDASIndexMergeIter::reset_datum_ptr(const common::ObIArray<ObExpr*> *exprs, int64_t size)
{
if (OB_NOT_NULL(exprs) && size > 0) {
for (int64_t i = 0; i < exprs->count(); i++) {
ObExpr *expr = exprs->at(i);
if (OB_NOT_NULL(expr)) {
expr->locate_datums_for_update(*eval_ctx_, size);
ObEvalInfo &info = expr->get_eval_info(*eval_ctx_);
info.point_to_frame_ = true;
}
}
}
}
int ObDASIndexMergeIter::do_table_scan()
{
int ret = OB_SUCCESS;
if (merge_ctdef_->is_left_child_leaf_node_) {
OB_ASSERT(left_scan_ctdef_ != nullptr && left_scan_rtdef_ != nullptr);
if (OB_FAIL(init_scan_param(ls_id_, left_tablet_id_, left_scan_ctdef_, left_scan_rtdef_, left_scan_param_))) {
LOG_WARN("failed to init left scan param", K(ret));
}
}
if (OB_SUCC(ret)) {
OB_ASSERT(right_scan_ctdef_ != nullptr && right_scan_rtdef_ != nullptr);
if (OB_FAIL(init_scan_param(ls_id_, right_tablet_id_, right_scan_ctdef_, right_scan_rtdef_, right_scan_param_))) {
LOG_WARN("failed to init right scan param", K(ret));
} else if (OB_FAIL(left_iter_->do_table_scan())) {
LOG_WARN("left iter failed to do table scan", K(ret));
} else if (OB_FAIL(right_iter_->do_table_scan())) {
LOG_WARN("right iter failed to do table scan", K(ret));
}
}
return ret;
}
int ObDASIndexMergeIter::rescan()
{
int ret = OB_SUCCESS;
if (merge_ctdef_->is_left_child_leaf_node_) {
left_scan_param_.tablet_id_ = left_tablet_id_;
left_scan_param_.ls_id_ = ls_id_;
if (OB_FAIL(prepare_scan_ranges(left_scan_param_, left_scan_rtdef_))) {
LOG_WARN("failed to prepare left rescan ranges", K(ret));
}
}
right_scan_param_.tablet_id_ = right_tablet_id_;
right_scan_param_.ls_id_ = ls_id_;
if (FAILEDx(prepare_scan_ranges(right_scan_param_, right_scan_rtdef_))) {
LOG_WARN("failed to prepare right rescan ranges", K(ret));
} else if (OB_FAIL(left_iter_->rescan())) {
LOG_WARN("left iter failed to rescan", K(ret));
} else if (OB_FAIL(right_iter_->rescan())) {
LOG_WARN("left iter failed to rescan", K(ret));
}
return ret;
}
void ObDASIndexMergeIter::clear_evaluated_flag()
{
if (OB_NOT_NULL(left_iter_)) {
left_iter_->clear_evaluated_flag();
}
if (OB_NOT_NULL(right_iter_)) {
right_iter_->clear_evaluated_flag();
}
}
int ObDASIndexMergeIter::set_ls_tablet_ids(const ObLSID &ls_id, const ObDASRelatedTabletID &related_tablet_ids)
{
int ret = OB_SUCCESS;
ls_id_ = ls_id;
if (merge_ctdef_->is_left_child_leaf_node_) {
OB_ASSERT(left_scan_ctdef_ != nullptr);
left_tablet_id_ = related_tablet_ids.index_merge_tablet_ids_.at(left_scan_ctdef_->index_merge_idx_);
}
OB_ASSERT(right_scan_ctdef_ != nullptr);
right_tablet_id_ = related_tablet_ids.index_merge_tablet_ids_.at(right_scan_ctdef_->index_merge_idx_);
return ret;
}
int ObDASIndexMergeIter::inner_reuse()
{
int ret = OB_SUCCESS;
if (merge_ctdef_->is_left_child_leaf_node_) {
const ObTabletID &old_left_tablet_id = left_scan_param_.tablet_id_;
left_scan_param_.need_switch_param_ = left_scan_param_.need_switch_param_ ||
(old_left_tablet_id.is_valid() && old_left_tablet_id != left_tablet_id_);
left_scan_param_.key_ranges_.reuse();
left_scan_param_.ss_key_ranges_.reuse();
left_scan_param_.mbr_filters_.reuse();
}
const ObTabletID &old_right_tablet_id = right_scan_param_.tablet_id_;
right_scan_param_.need_switch_param_ = right_scan_param_.need_switch_param_ ||
(old_right_tablet_id.is_valid() && old_right_tablet_id != right_tablet_id_);
right_scan_param_.key_ranges_.reuse();
right_scan_param_.ss_key_ranges_.reuse();
right_scan_param_.mbr_filters_.reuse();
if (OB_FAIL(left_iter_->reuse())) {
LOG_WARN("index merge iter failed to reuse left iter", K(ret));
} else if (OB_FAIL(right_iter_->reuse())) {
LOG_WARN("index merge iter failed to reuse right iter", K(ret));
}
if (OB_NOT_NULL(left_row_store_)) {
left_row_store_->reuse();
}
if (OB_NOT_NULL(right_row_store_)) {
right_row_store_->reuse();
}
left_iter_end_ = false;
right_iter_end_ = false;
state_ = FILL_LEFT_ROW;
return ret;
}
int ObDASIndexMergeIter::inner_release()
{
int ret = OB_SUCCESS;
if (OB_NOT_NULL(left_row_store_)) {
left_row_store_->reset();
left_row_store_ = nullptr;
}
if (OB_NOT_NULL(right_row_store_)) {
right_row_store_->reset();
right_row_store_ = nullptr;
}
left_iter_end_ = false;
right_iter_end_ = false;
left_scan_param_.destroy_schema_guard();
left_scan_param_.snapshot_.reset();
left_scan_param_.destroy();
right_scan_param_.destroy_schema_guard();
right_scan_param_.snapshot_.reset();
right_scan_param_.destroy();
if (OB_NOT_NULL(mem_ctx_)) {
mem_ctx_->reset_remain_one_page();
DESTROY_CONTEXT(mem_ctx_);
mem_ctx_ = nullptr;
}
return ret;
}
int ObDASIndexMergeIter::inner_get_next_row()
{
int ret = OB_SUCCESS;
clear_evaluated_flag();
if (OB_FAIL((this->*get_next_row_)())) {
if (ret != OB_ITER_END) {
LOG_WARN("index merge iter failed to get next row", K(ret));
}
}
return ret;
}
int ObDASIndexMergeIter::inner_get_next_rows(int64_t &count, int64_t capacity)
{
int ret = OB_SUCCESS;
clear_evaluated_flag();
if (OB_FAIL((this->*get_next_rows_)(count, capacity))) {
if (ret != OB_ITER_END) {
LOG_WARN("index merge iter failed to get next rows", K(ret));
}
}
LOG_TRACE("[DAS ITER] index merge iter get next rows", K(count), K(capacity), K(ret));
const ObBitVector *skip = nullptr;
PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_, count, skip);
return ret;
}
int ObDASIndexMergeIter::compare(int &cmp_ret)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(left_row_store_) ||
OB_ISNULL(right_row_store_) ||
!left_row_store_->have_data() ||
!right_row_store_->have_data()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KPC_(left_row_store), KPC_(right_row_store));
} else {
const ObDatum *left_datums = left_row_store_->cur_datums();
const ObDatum *right_datums = right_row_store_->cur_datums();
if (OB_ISNULL(left_datums) || OB_ISNULL(right_datums)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(left_datums), K(right_datums));
}
bool end_compare = false;
ObObj left_obj;
ObObj right_obj;
for (int64_t i = 0; !end_compare && OB_SUCC(ret) && i < rowkey_exprs_->count(); i++) {
const ObExpr *expr = rowkey_exprs_->at(i);
if (OB_ISNULL(expr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret));
} else if (OB_FAIL(left_datums[i].to_obj(left_obj, expr->obj_meta_, expr->obj_datum_map_))) {
LOG_WARN("failed to convert left datum to obj", K(i), KPC(expr), K(ret));
} else if (OB_FAIL(right_datums[i].to_obj(right_obj, expr->obj_meta_, expr->obj_datum_map_))) {
LOG_WARN("failed to convert right datum to obj", K(i), KPC(expr), K(ret));
} else if (OB_FAIL(left_obj.check_collation_free_and_compare(right_obj, cmp_ret))) {
LOG_WARN("failed to compare cur obj with output obj", K(ret));
} else if (cmp_ret != 0) {
end_compare = true;
cmp_ret = OB_UNLIKELY(is_reverse_) ? -cmp_ret : cmp_ret;
}
}
}
return ret;
}
int ObDASIndexMergeIter::intersect_get_next_row()
{
int ret = OB_SUCCESS;
bool got_next_row = false;
do {
switch(state_) {
case FILL_LEFT_ROW: {
while (OB_SUCC(ret) && !left_iter_end_ && !left_row_store_->have_data()) {
if (OB_FAIL(left_iter_->get_next_row())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from left iter", K(ret));
} else {
left_iter_end_ = true;
ret = OB_SUCCESS;
}
} else if (OB_FAIL(left_row_store_->save(false, 1))) {
LOG_WARN("failed to save left row", K(ret));
}
}
if (OB_SUCC(ret)) {
state_ = left_iter_end_ ? FINISHED : FILL_RIGHT_ROW;
}
break;
}
case FILL_RIGHT_ROW: {
while (OB_SUCC(ret) && !right_iter_end_ && !right_row_store_->have_data()) {
if (OB_FAIL(right_iter_->get_next_row())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from right iter", K(ret));
} else {
right_iter_end_ = true;
ret = OB_SUCCESS;
}
} else if (OB_FAIL(right_row_store_->save(false, 1))) {
LOG_WARN("failed to save right row", K(ret));
}
}
if (OB_SUCC(ret)) {
state_ = right_iter_end_ ? FINISHED : MERGE_AND_OUTPUT;
}
break;
}
case MERGE_AND_OUTPUT: {
if (OB_SUCC(ret)) {
int cmp_ret = 0;
if (!left_row_store_->have_data() || !left_row_store_->have_data()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected no data in left_row or right_row", K_(left_row_store), K_(right_row_store));
} else if (OB_FAIL(compare(cmp_ret))) {
LOG_WARN("failed to compare left row and right row", K(ret));
} else if (cmp_ret > 0) {
state_ = FILL_RIGHT_ROW;
} else if (cmp_ret < 0) {
state_ = FILL_LEFT_ROW;
} else {
if (OB_FAIL(left_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to convert left store row to expr", K(ret));
} else if (OB_FAIL(right_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to convert right store row to expr", K(ret));
} else {
state_ = FILL_LEFT_ROW;
got_next_row = true;
}
}
}
break;
}
case FINISHED: {
ret = OB_ITER_END;
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index merge state", K_(state));
}
}
} while (!got_next_row && OB_SUCC(ret));
return ret;
}
int ObDASIndexMergeIter::intersect_get_next_rows(int64_t &count, int64_t capacity)
{
int ret = OB_SUCCESS;
bool got_next_rows = false;
int64_t left_count = 0;
int64_t right_count = 0;
do {
switch(state_) {
case FILL_LEFT_ROW: {
while (OB_SUCC(ret) && !left_iter_end_ && !left_row_store_->have_data()) {
if (OB_FAIL(left_iter_->get_next_rows(left_count, capacity))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next rows from left iter", K(ret));
} else {
ret = OB_SUCCESS;
if (left_count == 0) {
left_iter_end_ = true;
}
}
}
if (OB_SUCC(ret) && !left_iter_end_ && OB_FAIL(left_row_store_->save(true, left_count))) {
LOG_WARN("failed to save left rows", K(left_count), K(ret));
}
}
if (OB_SUCC(ret)) {
state_ = left_iter_end_ ? FINISHED : FILL_RIGHT_ROW;
}
break;
}
case FILL_RIGHT_ROW: {
while (OB_SUCC(ret) && !right_iter_end_ && !right_row_store_->have_data()) {
if (OB_FAIL(right_iter_->get_next_rows(right_count, capacity))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next rows from right iter", K(ret));
} else {
ret = OB_SUCCESS;
if (right_count == 0) {
right_iter_end_ = true;
}
}
}
if (OB_SUCC(ret) && !right_iter_end_ && OB_FAIL(right_row_store_->save(true, right_count))) {
LOG_WARN("failed to save right rows", K(right_count), K(ret));
}
}
if (OB_SUCC(ret)) {
state_ = right_iter_end_ ? FINISHED : MERGE_AND_OUTPUT;
}
break;
}
case MERGE_AND_OUTPUT: {
if (OB_SUCC(ret)) {
int cmp_ret = 0;
if (!left_row_store_->have_data()) {
state_ = FILL_LEFT_ROW;
} else if (!right_row_store_->have_data()) {
state_ = FILL_RIGHT_ROW;
} else if (OB_FAIL(compare(cmp_ret))) {
LOG_WARN("failed to compare left row and right row", K(ret));
} else if (cmp_ret > 0) {
state_ = FILL_RIGHT_ROW;
} else if (cmp_ret < 0) {
state_ = FILL_LEFT_ROW;
} else {
if (OB_FAIL(left_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to convert left store row to expr", K(ret));
} else if (OB_FAIL(right_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to convert right store row to expr", K(ret));
} else {
count = 1;
got_next_rows = true;
}
}
}
break;
}
case FINISHED: {
ret = OB_ITER_END;
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index merge state", K_(state));
}
}
} while (!got_next_rows && OB_SUCC(ret));
return ret;
}
int ObDASIndexMergeIter::union_get_next_row()
{
int ret = OB_SUCCESS;
bool got_next_row = false;
do {
switch(state_) {
case FILL_LEFT_ROW: {
while (OB_SUCC(ret) && !left_iter_end_ && !left_row_store_->have_data()) {
if (OB_FAIL(left_iter_->get_next_row())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from left iter", K(ret));
} else {
left_iter_end_ = true;
ret = OB_SUCCESS;
}
} else if (OB_FAIL(left_row_store_->save(false, 1))) {
LOG_WARN("failed to save left row", K(ret));
}
}
if (OB_SUCC(ret)) {
state_ = FILL_RIGHT_ROW;
}
break;
}
case FILL_RIGHT_ROW: {
while (OB_SUCC(ret) && !right_iter_end_ && !right_row_store_->have_data()) {
if (OB_FAIL(right_iter_->get_next_row())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from right iter", K(ret));
} else {
right_iter_end_ = true;
ret = OB_SUCCESS;
}
} else if (OB_FAIL(right_row_store_->save(false, 1))) {
LOG_WARN("failed to save right row", K(ret));
}
}
if (OB_SUCC(ret)) {
state_ = MERGE_AND_OUTPUT;
}
break;
}
case MERGE_AND_OUTPUT: {
if (OB_SUCC(ret)) {
if (left_iter_end_ && right_iter_end_) {
state_ = FINISHED;
} else if (left_iter_end_) {
if (right_row_store_->have_data()) {
if (OB_FAIL(right_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to convert right store row to expr", K(ret));
} else {
got_next_row = true;
state_ = FILL_RIGHT_ROW;
}
} else {
state_ = FILL_RIGHT_ROW;
}
} else if (right_iter_end_) {
if (left_row_store_->have_data()) {
if (OB_FAIL(left_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to convert left store row to expr", K(ret));
} else {
got_next_row = true;
state_ = FILL_LEFT_ROW;
}
} else {
state_ = FILL_LEFT_ROW;
}
} else {
int cmp_ret = 0;
if (OB_FAIL(compare(cmp_ret))) {
LOG_WARN("failed to compare left row and right row", K(ret));
} else if (cmp_ret < 0) {
if (OB_FAIL(left_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to convert left store row to expr", K(ret));
} else {
got_next_row = true;
state_ = FILL_LEFT_ROW;
}
} else if (cmp_ret > 0) {
if (OB_FAIL(right_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to convert right store row to expr", K(ret));
} else {
got_next_row = true;
state_ = FILL_RIGHT_ROW;
}
} else {
if (OB_FAIL(left_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to convert store row to expr", K(ret));
} else if (OB_FAIL(right_row_store_->to_expr(false, 1))) {
LOG_WARN("failed to ");
} else {
got_next_row = true;
state_ = FILL_LEFT_ROW;
}
}
}
}
break;
}
case FINISHED: {
ret = OB_ITER_END;
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index merge state", K_(state));
}
}
} while (!got_next_row && OB_SUCC(ret));
return ret;
}
int ObDASIndexMergeIter::union_get_next_rows(int64_t &count, int64_t capacity)
{
int ret = OB_SUCCESS;
bool got_next_rows = false;
int64_t left_count = 0;
int64_t right_count = 0;
do {
switch(state_) {
case FILL_LEFT_ROW: {
while (OB_SUCC(ret) && !left_iter_end_ && !left_row_store_->have_data()) {
if (OB_FAIL(left_iter_->get_next_rows(left_count, capacity))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next rows from left iter", K(ret));
} else {
ret = OB_SUCCESS;
if (left_count == 0) {
left_iter_end_ = true;
}
}
}
if (OB_SUCC(ret) && !left_iter_end_ && OB_FAIL(left_row_store_->save(true, left_count))) {
LOG_WARN("failed to save left rows", K(left_count), K(ret));
}
if (left_iter_->get_type() == DAS_ITER_SORT) {
reset_datum_ptr(left_output_, left_count);
}
}
if (OB_SUCC(ret)) {
state_ = FILL_RIGHT_ROW;
}
break;
}
case FILL_RIGHT_ROW: {
while (OB_SUCC(ret) && !right_iter_end_ && !right_row_store_->have_data()) {
if (OB_FAIL(right_iter_->get_next_rows(right_count, capacity))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next rows from right iter", K(ret));
} else {
ret = OB_SUCCESS;
if (right_count == 0) {
right_iter_end_ = true;
}
}
}
if (OB_SUCC(ret) && !right_iter_end_ && OB_FAIL(right_row_store_->save(true, right_count))) {
LOG_WARN("failed to save right rows", K(right_count), K(ret));
}
if (right_iter_->get_type() == DAS_ITER_SORT) {
reset_datum_ptr(right_output_, right_count);
}
}
if (OB_SUCC(ret)) {
state_ = MERGE_AND_OUTPUT;
}
break;
}
case MERGE_AND_OUTPUT: {
if (OB_SUCC(ret)) {
if (left_iter_end_ && right_iter_end_) {
state_ = FINISHED;
} else if (left_iter_end_) {
if (right_row_store_->have_data()) {
int64_t right_rows_cnt = right_row_store_->rows_cnt();
if (OB_FAIL(right_row_store_->to_expr(true, right_rows_cnt))) {
LOG_WARN("failed to convert right store rows to expr", K(right_rows_cnt), K(ret));
} else {
count = right_rows_cnt;
got_next_rows = true;
state_ = FILL_RIGHT_ROW;
}
} else {
state_ = FILL_RIGHT_ROW;
}
} else if (right_iter_end_) {
if (left_row_store_->have_data()) {
int64_t left_rows_cnt = left_row_store_->rows_cnt();
if (OB_FAIL(left_row_store_->to_expr(true, left_rows_cnt))) {
LOG_WARN("failed to convert left store rows to expr", K(left_rows_cnt), K(ret));
} else {
count = left_rows_cnt;
got_next_rows = true;
state_ = FILL_LEFT_ROW;
}
} else {
state_ = FILL_LEFT_ROW;
}
} else {
int cmp_ret = 0;
if (OB_FAIL(compare(cmp_ret))) {
LOG_WARN("failed to compare left row and right row", K(ret));
} else if (cmp_ret < 0) {
if (OB_FAIL(left_row_store_->to_expr(true, 1))) {
LOG_WARN("failed to convert left store row to expr", K(ret));
} else {
got_next_rows = true;
state_ = FILL_LEFT_ROW;
}
} else if (cmp_ret > 0) {
if (OB_FAIL(right_row_store_->to_expr(true, 1))) {
LOG_WARN("failed to convert right store row to expr", K(ret));
} else {
got_next_rows = true;
state_ = FILL_RIGHT_ROW;
}
} else {
if (OB_FAIL(left_row_store_->to_expr(true, 1))) {
LOG_WARN("failed to convert left store row to expr", K(ret));
} else if (OB_FAIL(right_row_store_->to_expr(true, 1))) {
LOG_WARN("failed to convert right store row to expr", K(ret));
} else {
got_next_rows = true;
state_ = FILL_LEFT_ROW;
}
}
if (OB_SUCC(ret) && got_next_rows) {
count = 1;
}
}
}
break;
}
case FINISHED: {
ret = OB_ITER_END;
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index merge state", K_(state));
}
}
} while (!got_next_rows && OB_SUCC(ret));
return ret;
}
} // namespace sql
} // namespace oceanbase

228
src/sql/das/iter/ob_das_index_merge_iter.h Normal file
View File

@ -0,0 +1,228 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OBDEV_SRC_SQL_DAS_ITER_OB_DAS_INDEX_MERGE_ITER_H_
#define OBDEV_SRC_SQL_DAS_ITER_OB_DAS_INDEX_MERGE_ITER_H_
#include "sql/das/iter/ob_das_iter.h"
#include "sql/optimizer/ob_join_order.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
struct ObDASIndexMergeCtDef;
struct ObDASIndexMergeRtDef;
struct ObDASIndexMergeIterParam : public ObDASIterParam
{
public:
ObDASIndexMergeIterParam()
: ObDASIterParam(DAS_ITER_INDEX_MERGE),
merge_type_(INDEX_MERGE_INVALID),
rowkey_exprs_(nullptr),
left_iter_(nullptr),
left_output_(nullptr),
right_iter_(nullptr),
right_output_(nullptr),
ctdef_(nullptr),
rtdef_(nullptr),
tx_desc_(nullptr),
snapshot_(nullptr),
is_reverse_(false)
{}
ObIndexMergeType merge_type_;
const ExprFixedArray *rowkey_exprs_;
ObDASIter *left_iter_;
const common::ObIArray<ObExpr*> *left_output_;
ObDASIter *right_iter_;
const common::ObIArray<ObExpr*> *right_output_;
const ObDASIndexMergeCtDef *ctdef_;
ObDASIndexMergeRtDef *rtdef_;
transaction::ObTxDesc *tx_desc_;
transaction::ObTxReadSnapshot *snapshot_;
bool is_reverse_;
virtual bool is_valid() const
{
return rowkey_exprs_ != nullptr &&
left_iter_ != nullptr &&
left_output_ != nullptr &&
right_iter_ != nullptr &&
right_output_ != nullptr &&
ctdef_ != nullptr &&
rtdef_ != nullptr &&
merge_type_ != INDEX_MERGE_INVALID &&
ObDASIterParam::is_valid();
}
};
class ObDASIndexMergeIter : public ObDASIter
{
public:
struct RowStore
{
public:
RowStore()
: exprs_(nullptr),
eval_ctx_(nullptr),
max_size_(1),
saved_size_(0),
cur_idx_(OB_INVALID_INDEX),
store_rows_(nullptr)
{}
RowStore(const common::ObIArray<ObExpr*> *exprs,
ObEvalCtx *eval_ctx,
int64_t max_size)
: exprs_(exprs),
eval_ctx_(eval_ctx),
max_size_(max_size),
saved_size_(0),
cur_idx_(OB_INVALID_INDEX),
store_rows_(nullptr)
{}
int init(common::ObIAllocator &allocator);
int save(bool is_vectorized, int64_t size);
int to_expr(bool is_vectorized, int64_t size);
bool have_data() const { return cur_idx_ != OB_INVALID_INDEX && cur_idx_ < saved_size_; }
int64_t rows_cnt() const { return have_data() ? saved_size_ - cur_idx_ : 0; }
const ObDatum *cur_datums();
void reuse();
void reset();
TO_STRING_KV(K_(exprs),
K_(saved_size),
K_(cur_idx));
public:
typedef ObChunkDatumStore::LastStoredRow LastDASStoreRow;
const common::ObIArray<ObExpr*> *exprs_;
ObEvalCtx *eval_ctx_;
int64_t max_size_;
int64_t saved_size_;
int64_t cur_idx_;
LastDASStoreRow *store_rows_;
};
public:
ObDASIndexMergeIter()
: ObDASIter(ObDASIterType::DAS_ITER_INDEX_MERGE),
merge_type_(INDEX_MERGE_INVALID),
state_(FILL_LEFT_ROW),
left_iter_(nullptr),
right_iter_(nullptr),
left_row_store_(nullptr),
right_row_store_(nullptr),
left_scan_param_(),
right_scan_param_(),
left_iter_end_(false),
right_iter_end_(false),
rowkey_exprs_(nullptr),
left_output_(nullptr),
right_output_(nullptr),
mem_ctx_(),
get_next_row_(nullptr),
get_next_rows_(nullptr),
ls_id_(),
left_tablet_id_(),
right_tablet_id_(),
merge_ctdef_(nullptr),
merge_rtdef_(nullptr),
left_scan_ctdef_(nullptr),
left_scan_rtdef_(nullptr),
right_scan_ctdef_(nullptr),
right_scan_rtdef_(nullptr),
tx_desc_(nullptr),
snapshot_(nullptr),
is_reverse_(false)
{}
virtual ~ObDASIndexMergeIter() {}
public:
virtual int do_table_scan() override;
virtual int rescan() override;
virtual void clear_evaluated_flag() override;
int set_ls_tablet_ids(const ObLSID &ls_id, const ObDASRelatedTabletID &related_tablet_ids);
ObTableScanParam &get_left_param() { return left_scan_param_; }
ObTableScanParam &get_right_param() { return right_scan_param_; }
protected:
virtual int inner_init(ObDASIterParam &param) override;
virtual int inner_reuse() override;
virtual int inner_release() override;
virtual int inner_get_next_row() override;
virtual int inner_get_next_rows(int64_t &count, int64_t capacity) override;
private:
int compare(int &cmp_ret);
int intersect_get_next_row();
int intersect_get_next_rows(int64_t &count, int64_t capacity);
int union_get_next_row();
int union_get_next_rows(int64_t &count, int64_t capacity);
int init_scan_param(const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const sql::ObDASScanCtDef *ctdef,
sql::ObDASScanRtDef *rtdef,
ObTableScanParam &scan_param);
int prepare_scan_ranges(ObTableScanParam &scan_param, const ObDASScanRtDef *rtdef);
void reset_datum_ptr(const common::ObIArray<ObExpr*> *exprs, int64_t size);
private:
enum MergeState : uint32_t
{
FILL_LEFT_ROW,
FILL_RIGHT_ROW,
MERGE_AND_OUTPUT,
FINISHED
};
ObIndexMergeType merge_type_;
MergeState state_;
ObDASIter *left_iter_;
ObDASIter *right_iter_;
RowStore *left_row_store_;
RowStore *right_row_store_;
ObTableScanParam left_scan_param_;
ObTableScanParam right_scan_param_;
bool left_iter_end_;
bool right_iter_end_;
const ExprFixedArray *rowkey_exprs_;
const common::ObIArray<ObExpr*> *left_output_;
const common::ObIArray<ObExpr*> *right_output_;
lib::MemoryContext mem_ctx_;
int (ObDASIndexMergeIter::*get_next_row_)();
int (ObDASIndexMergeIter::*get_next_rows_)(int64_t&, int64_t);
ObLSID ls_id_;
ObTabletID left_tablet_id_;
ObTabletID right_tablet_id_;
const ObDASIndexMergeCtDef *merge_ctdef_;
ObDASIndexMergeRtDef *merge_rtdef_;
// nullptr if left child is not a leaf node
const ObDASScanCtDef *left_scan_ctdef_;
ObDASScanRtDef *left_scan_rtdef_;
const ObDASScanCtDef *right_scan_ctdef_;
ObDASScanRtDef *right_scan_rtdef_;
transaction::ObTxDesc *tx_desc_;
transaction::ObTxReadSnapshot *snapshot_;
bool is_reverse_;
};
} // namespace sql
} // namespace oceanbase
#endif /* OBDEV_SRC_SQL_DAS_ITER_OB_DAS_INDEX_MERGE_ITER_H_ */

10
src/sql/das/iter/ob_das_iter_define.h
View File

@ -69,6 +69,10 @@ enum ObDASIterTreeType : uint32_t
struct ObDASRelatedTabletID
{
public:
ObDASRelatedTabletID(common::ObIAllocator &alloc)
: index_merge_tablet_ids_(alloc)
{ reset(); }
common::ObTabletID lookup_tablet_id_;
common::ObTabletID aux_lookup_tablet_id_;
common::ObTabletID rowkey_vid_tablet_id_;
@ -78,6 +82,11 @@ public:
common::ObTabletID fwd_idx_tablet_id_;
common::ObTabletID doc_id_idx_tablet_id_;
/* used by fulltext index */
/* used by index merge */
common::ObFixedArray<common::ObTabletID, ObIAllocator> index_merge_tablet_ids_;
/* used by index merge */
void reset()
{
lookup_tablet_id_.reset();
@ -86,6 +95,7 @@ public:
inv_idx_tablet_id_.reset();
fwd_idx_tablet_id_.reset();
doc_id_idx_tablet_id_.reset();
index_merge_tablet_ids_.reset();
}
};

326
src/sql/das/iter/ob_das_iter_utils.cpp
View File

@ -50,6 +50,10 @@ int ObDASIterUtils::create_das_scan_iter_tree(ObDASIterTreeType tree_type,
ret = create_text_retrieval_tree(scan_param, alloc, attach_ctdef, attach_rtdef, related_tablet_ids, trans_desc, snapshot, iter_tree);
break;
}
case ITER_TREE_INDEX_MERGE: {
ret = create_index_merge_iter_tree(scan_param, alloc, attach_ctdef, attach_rtdef, related_tablet_ids, trans_desc, snapshot, iter_tree);
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
}
@ -58,7 +62,7 @@ int ObDASIterUtils::create_das_scan_iter_tree(ObDASIterTreeType tree_type,
LOG_WARN("failed to create das scan iter tree", K(ret));
}
LOG_DEBUG("create das scan iter tree", K(tree_type), K(ret));
LOG_TRACE("create das scan iter tree", K(tree_type), K(ret));
return ret;
}
@ -214,6 +218,71 @@ int ObDASIterUtils::set_text_retrieval_related_ids(const ObDASBaseCtDef *attach_
}
return ret;
}
int ObDASIterUtils::set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctdef,
const ObDASRelatedTabletID &related_tablet_ids,
const ObLSID &ls_id,
ObDASIter *root_iter)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(attach_ctdef) || OB_ISNULL(root_iter)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret), KP(attach_ctdef), KP(root_iter));
} else {
bool need_set_child = false;
const ObDASIterType &iter_type = root_iter->get_type();
switch (attach_ctdef->op_type_) {
case ObDASOpType::DAS_OP_TABLE_LOOKUP: {
if (OB_UNLIKELY(iter_type != ObDASIterType::DAS_ITER_LOCAL_LOOKUP)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type));
} else {
ObDASLocalLookupIter *lookup_iter = static_cast<ObDASLocalLookupIter *>(root_iter);
lookup_iter->set_ls_id(ls_id);
lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_);
need_set_child = true;
}
break;
}
case ObDASOpType::DAS_OP_INDEX_MERGE: {
if (OB_UNLIKELY(iter_type != ObDASIterType::DAS_ITER_INDEX_MERGE)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("iter type not match with ctdef", K(ret), K(attach_ctdef->op_type_), K(iter_type));
} else {
ObDASIndexMergeIter *merge_iter = static_cast<ObDASIndexMergeIter *>(root_iter);
if (OB_FAIL(merge_iter->set_ls_tablet_ids(ls_id, related_tablet_ids))) {
LOG_WARN("failed to set related tablet ids", K(ret));
}
need_set_child = true;
}
break;
}
default: {
need_set_child = false;
break;
}
}
if (OB_FAIL(ret)) {
} else if (!need_set_child) {
} else if (OB_UNLIKELY(attach_ctdef->children_cnt_ != root_iter->get_children_cnt())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected iter children count not equal to ctdef children count",
K(attach_ctdef->children_cnt_), K(root_iter->get_children_cnt()), K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < attach_ctdef->children_cnt_; ++i) {
if (OB_FAIL(set_index_merge_related_ids(attach_ctdef->children_[i],
related_tablet_ids,
ls_id,
root_iter->get_children()[i]))) {
LOG_WARN("failed to set index merge related ids", K(ret));
}
}
}
}
return ret;
}
/***************** PUBLIC END *****************/
int ObDASIterUtils::create_partition_scan_tree(storage::ObTableScanParam &scan_param,
@ -230,6 +299,10 @@ int ObDASIterUtils::create_partition_scan_tree(storage::ObTableScanParam &scan_p
int ret = OB_SUCCESS;
ObDASScanIterParam param;
param.scan_ctdef_ = scan_ctdef;
param.max_size_ = scan_rtdef->eval_ctx_->is_vectorized() ? scan_rtdef->eval_ctx_->max_batch_size_ : 1;
param.eval_ctx_ = scan_rtdef->eval_ctx_;
param.exec_ctx_ = &scan_rtdef->eval_ctx_->exec_ctx_;
param.output_ = &scan_ctdef->result_output_;
ObDASScanIter *scan_iter = nullptr;
if (OB_FAIL(create_das_iter(alloc, param, scan_iter))) {
LOG_WARN("failed to create das scan iter", K(ret));
@ -268,8 +341,16 @@ int ObDASIterUtils::create_local_lookup_tree(ObTableScanParam &scan_param,
ObDASLocalLookupIter *lookup_iter = nullptr;
ObDASScanIterParam index_table_param;
index_table_param.scan_ctdef_ = scan_ctdef;
index_table_param.max_size_ = scan_rtdef->eval_ctx_->is_vectorized() ? scan_rtdef->eval_ctx_->max_batch_size_ : 1;
index_table_param.eval_ctx_ = scan_rtdef->eval_ctx_;
index_table_param.exec_ctx_ = &scan_rtdef->eval_ctx_->exec_ctx_;
index_table_param.output_ = &scan_ctdef->result_output_;
ObDASScanIterParam data_table_param;
data_table_param.scan_ctdef_ = lookup_ctdef;
data_table_param.max_size_ = lookup_rtdef->eval_ctx_->is_vectorized() ? lookup_rtdef->eval_ctx_->max_batch_size_ : 1;
data_table_param.eval_ctx_ = lookup_rtdef->eval_ctx_;
data_table_param.exec_ctx_ = &lookup_rtdef->eval_ctx_->exec_ctx_;
data_table_param.output_ = &lookup_ctdef->result_output_;
if (OB_FAIL(create_das_iter(alloc, index_table_param, index_table_iter))) {
LOG_WARN("failed to create index table iter", K(ret));
} else if (OB_FAIL(create_das_iter(alloc, data_table_param, data_table_iter))) {
@ -456,6 +537,10 @@ int ObDASIterUtils::create_text_retrieval_sub_tree(const ObLSID &ls_id,
merge_iter_param.snapshot_ = snapshot;
if (ir_scan_ctdef->need_do_total_doc_cnt()) {
doc_cnt_agg_param.scan_ctdef_ = ir_scan_ctdef->get_doc_id_idx_agg_ctdef();
doc_cnt_agg_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_;
doc_cnt_agg_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_;
doc_cnt_agg_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_;
doc_cnt_agg_param.output_ = &ir_scan_ctdef->get_doc_id_idx_agg_ctdef()->result_output_;
if (OB_FAIL(create_das_iter(alloc, doc_cnt_agg_param, doc_cnt_agg_iter))) {
LOG_WARN("failed to create doc cnt agg scan iter", K(ret));
} else {
@ -484,12 +569,24 @@ int ObDASIterUtils::create_text_retrieval_sub_tree(const ObLSID &ls_id,
ObDASScanIterParam inv_idx_scan_iter_param;
ObDASScanIter *inv_idx_scan_iter = nullptr;
inv_idx_scan_iter_param.scan_ctdef_ = ir_scan_ctdef->get_inv_idx_scan_ctdef();
inv_idx_scan_iter_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_;
inv_idx_scan_iter_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_;
inv_idx_scan_iter_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_;
inv_idx_scan_iter_param.output_ = &ir_scan_ctdef->get_inv_idx_scan_ctdef()->result_output_;
ObDASScanIterParam inv_idx_agg_iter_param;
ObDASScanIter *inv_idx_agg_iter = nullptr;
inv_idx_agg_iter_param.scan_ctdef_ = ir_scan_ctdef->get_inv_idx_agg_ctdef();
inv_idx_agg_iter_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_;
inv_idx_agg_iter_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_;
inv_idx_agg_iter_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_;
inv_idx_agg_iter_param.output_ = &ir_scan_ctdef->get_inv_idx_agg_ctdef()->result_output_;
ObDASScanIterParam fwd_idx_iter_param;
ObDASScanIter *fwd_idx_iter = nullptr;
fwd_idx_iter_param.scan_ctdef_ = ir_scan_ctdef->get_fwd_idx_agg_ctdef();
fwd_idx_iter_param.max_size_ = ir_scan_rtdef->eval_ctx_->max_batch_size_;
fwd_idx_iter_param.eval_ctx_ = ir_scan_rtdef->eval_ctx_;
fwd_idx_iter_param.exec_ctx_ = &ir_scan_rtdef->eval_ctx_->exec_ctx_;
fwd_idx_iter_param.output_ = &ir_scan_ctdef->get_fwd_idx_agg_ctdef()->result_output_;
if (OB_FAIL(create_das_iter(alloc, inv_idx_scan_iter_param, inv_idx_scan_iter))) {
LOG_WARN("failed to create inv idx iter", K(ret));
} else if (ir_scan_ctdef->need_inv_idx_agg()
@ -569,16 +666,21 @@ int ObDASIterUtils::create_vid_scan_sub_tree(
ObDASIter *&iter_tree)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(merge_ctdef) || OB_UNLIKELY(2 != merge_ctdef->children_cnt_)
if (OB_ISNULL(merge_ctdef) || OB_ISNULL(merge_rtdef) || OB_UNLIKELY(2 != merge_ctdef->children_cnt_)
|| OB_ISNULL(iter_tree) || OB_UNLIKELY(ObDASIterType::DAS_ITER_SCAN != iter_tree->get_type())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), KPC(merge_ctdef), KPC(iter_tree));
LOG_WARN("invalid arguments", K(ret), KPC(merge_ctdef), KPC(merge_rtdef), KPC(iter_tree));
} else {
ObDASVIdMergeIterParam vid_merge_param;
ObDASVIdMergeIter *vid_merge_iter = nullptr;
ObDASScanIterParam rowkey_vid_param;
ObDASScanIter *rowkey_vid_iter = nullptr;
rowkey_vid_param.scan_ctdef_ = static_cast<ObDASScanCtDef *>(merge_ctdef->children_[1]);
rowkey_vid_param.max_size_ = merge_rtdef->eval_ctx_->is_vectorized() ?
merge_rtdef->eval_ctx_->max_batch_size_ : 1;
rowkey_vid_param.eval_ctx_ = merge_rtdef->eval_ctx_;
rowkey_vid_param.exec_ctx_ = &merge_rtdef->eval_ctx_->exec_ctx_;
rowkey_vid_param.output_ = &rowkey_vid_param.scan_ctdef_->result_output_;
if (OB_FAIL(create_das_iter(alloc, rowkey_vid_param, rowkey_vid_iter))) {
LOG_WARN("fail to create das scan iter", K(ret), K(rowkey_vid_param));
} else {
@ -634,7 +736,10 @@ int ObDASIterUtils::create_domain_lookup_sub_tree(ObTableScanParam &scan_param,
ObDASScanIter *doc_id_table_iter = nullptr;
ObDASScanIterParam doc_id_table_param;
doc_id_table_param.scan_ctdef_ = aux_lookup_ctdef->get_lookup_scan_ctdef();
doc_id_table_param.max_size_ = 1;
doc_id_table_param.eval_ctx_ = aux_lookup_rtdef->eval_ctx_;
doc_id_table_param.exec_ctx_ = &aux_lookup_rtdef->eval_ctx_->exec_ctx_;
doc_id_table_param.output_ = &aux_lookup_ctdef->result_output_;
if (OB_FAIL(create_das_iter(alloc, doc_id_table_param, doc_id_table_iter))) {
LOG_WARN("failed to create doc id table iter", K(ret));
} else {
@ -674,6 +779,10 @@ int ObDASIterUtils::create_domain_lookup_sub_tree(ObTableScanParam &scan_param,
if (OB_SUCC(ret)) {
ObDASScanIterParam data_table_param;
data_table_param.scan_ctdef_ = table_lookup_ctdef->get_lookup_scan_ctdef();
data_table_param.max_size_ = 1;
data_table_param.eval_ctx_ = table_lookup_rtdef->eval_ctx_;
data_table_param.exec_ctx_ = &table_lookup_rtdef->eval_ctx_->exec_ctx_;
data_table_param.output_ = &table_lookup_ctdef->result_output_;
if (OB_FAIL(create_das_iter(alloc, data_table_param, data_table_iter))) {
LOG_WARN("failed to create data table lookup scan iter", K(ret));
} else {
@ -898,6 +1007,215 @@ int ObDASIterUtils::create_global_lookup_iter_tree(const ObTableScanCtDef &tsc_c
return ret;
}
int ObDASIterUtils::create_index_merge_iter_tree(ObTableScanParam &scan_param,
common::ObIAllocator &alloc,
const ObDASBaseCtDef *attach_ctdef,
ObDASBaseRtDef *attach_rtdef,
const ObDASRelatedTabletID &related_tablet_ids,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
ObDASIter *&iter_tree)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(attach_ctdef) || OB_ISNULL(attach_rtdef)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ret));
} else {
const bool need_lookup = attach_ctdef->op_type_ == DAS_OP_TABLE_LOOKUP;
const ObDASBaseCtDef *index_merge_ctdef = need_lookup ? attach_ctdef->children_[0] : attach_ctdef;
ObDASBaseRtDef *index_merge_rtdef = need_lookup ? attach_rtdef->children_[0] : attach_rtdef;
ObDASIter *index_merge_root = nullptr;
if (OB_FAIL(create_index_merge_sub_tree(scan_param.ls_id_,
alloc,
index_merge_ctdef,
index_merge_rtdef,
related_tablet_ids,
tx_desc,
snapshot,
index_merge_root))) {
LOG_WARN("failed to create index merge iter tree", K(ret));
} else if (need_lookup) {
ObDASLocalLookupIter *lookup_iter = nullptr;
ObDASScanIter *data_table_iter = nullptr;
const ObDASScanCtDef *lookup_ctdef = static_cast<const ObDASScanCtDef*>(attach_ctdef->children_[1]);
ObDASScanRtDef *lookup_rtdef = static_cast<ObDASScanRtDef*>(attach_rtdef->children_[1]);
ObDASScanIterParam data_table_param;
data_table_param.scan_ctdef_ = lookup_ctdef;
data_table_param.max_size_ = lookup_rtdef->eval_ctx_->is_vectorized() ? lookup_rtdef->eval_ctx_->max_batch_size_ : 1;
data_table_param.eval_ctx_ = lookup_rtdef->eval_ctx_;
data_table_param.exec_ctx_ = &lookup_rtdef->eval_ctx_->exec_ctx_;
data_table_param.output_ = &lookup_ctdef->result_output_;
if (OB_FAIL(create_das_iter(alloc, data_table_param, data_table_iter))) {
LOG_WARN("failed to create data table iter", K(ret));
} else {
ObDASLocalLookupIterParam lookup_param;
lookup_param.max_size_ = lookup_rtdef->eval_ctx_->is_vectorized() ? lookup_rtdef->eval_ctx_->max_batch_size_ : 1;
lookup_param.eval_ctx_ = lookup_rtdef->eval_ctx_;
lookup_param.exec_ctx_ = &lookup_rtdef->eval_ctx_->exec_ctx_;
lookup_param.output_ = &lookup_ctdef->result_output_;
lookup_param.default_batch_row_count_ = 1000; // hard code 1000 for local lookup
lookup_param.index_ctdef_ = index_merge_ctdef;
lookup_param.index_rtdef_ = index_merge_rtdef;
lookup_param.lookup_ctdef_ = lookup_ctdef;
lookup_param.lookup_rtdef_ = lookup_rtdef;
lookup_param.index_table_iter_ = index_merge_root;
lookup_param.data_table_iter_ = data_table_iter;
lookup_param.trans_desc_ = tx_desc;
lookup_param.snapshot_ = snapshot;
lookup_param.rowkey_exprs_ = &lookup_ctdef->rowkey_exprs_;
if (OB_FAIL(create_das_iter(alloc, lookup_param, lookup_iter))) {
LOG_WARN("failed to create local lookup iter", K(ret));
} else if (OB_FAIL(create_iter_children_array(2, alloc, lookup_iter))) {
LOG_WARN("failed to create iter children array", K(ret));
} else {
lookup_iter->get_children()[0] = index_merge_root;
lookup_iter->get_children()[1] = data_table_iter;
data_table_iter->set_scan_param(lookup_iter->get_lookup_param());
lookup_iter->set_tablet_id(related_tablet_ids.lookup_tablet_id_);
lookup_iter->set_ls_id(scan_param.ls_id_);
iter_tree = lookup_iter;
}
}
} else {
iter_tree = index_merge_root;
}
}
return ret;
}
int ObDASIterUtils::create_index_merge_sub_tree(const ObLSID &ls_id,
common::ObIAllocator &alloc,
const ObDASBaseCtDef *ctdef,
ObDASBaseRtDef *rtdef,
const ObDASRelatedTabletID &related_tablet_ids,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
ObDASIter *&iter)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(ctdef) || OB_ISNULL(rtdef)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ctdef), K(rtdef));
} else if (ctdef->op_type_ == DAS_OP_TABLE_SCAN) {
ObDASScanIterParam scan_param;
scan_param.scan_ctdef_ = static_cast<const ObDASScanCtDef*>(ctdef);
scan_param.max_size_ = rtdef->eval_ctx_->is_vectorized() ? rtdef->eval_ctx_->max_batch_size_ : 1;
scan_param.eval_ctx_ = rtdef->eval_ctx_;
scan_param.exec_ctx_ = &rtdef->eval_ctx_->exec_ctx_;
scan_param.output_ = &scan_param.scan_ctdef_->result_output_;
ObDASScanIter *scan_iter = nullptr;
if (OB_FAIL(create_das_iter(alloc, scan_param, scan_iter))) {
LOG_WARN("failed to create das scan iter", K(ret));
} else {
iter = scan_iter;
}
} else if (ctdef->op_type_ == DAS_OP_SORT) {
const ObDASSortCtDef *sort_ctdef = static_cast<const ObDASSortCtDef*>(ctdef);
ObDASSortRtDef *sort_rtdef = static_cast<ObDASSortRtDef*>(rtdef);
OB_ASSERT(ctdef->children_ != nullptr &&
ctdef->children_cnt_ == 1 &&
ctdef->children_[0]->op_type_ == DAS_OP_TABLE_SCAN);
const ObDASScanCtDef *scan_ctdef = static_cast<ObDASScanCtDef*>(ctdef->children_[0]);
ObDASScanIterParam child_scan_param;
child_scan_param.scan_ctdef_ = scan_ctdef;
child_scan_param.max_size_ = rtdef->eval_ctx_->is_vectorized() ? rtdef->eval_ctx_->max_batch_size_ : 1;
child_scan_param.eval_ctx_ = rtdef->eval_ctx_;
child_scan_param.exec_ctx_ = &rtdef->eval_ctx_->exec_ctx_;
child_scan_param.output_ = &scan_ctdef->result_output_;
ObDASScanIter *child_scan_iter = nullptr;
ObDASIter *sort_iter = nullptr;
if (OB_FAIL(create_das_iter(alloc, child_scan_param, child_scan_iter))) {
LOG_WARN("failed to create das scan iter", K(ret));
} else if (OB_FAIL(create_sort_sub_tree(alloc, sort_ctdef, sort_rtdef, child_scan_iter, sort_iter))) {
LOG_WARN("failed to create das sort iter", K(ret));
} else {
iter = sort_iter;
}
} else if (ctdef->op_type_ == DAS_OP_INDEX_MERGE) {
const ObDASIndexMergeCtDef *merge_ctdef = static_cast<const ObDASIndexMergeCtDef*>(ctdef);
ObDASIndexMergeRtDef *merge_rtdef = static_cast<ObDASIndexMergeRtDef*>(rtdef);
const ObDASBaseCtDef *left_ctdef = merge_ctdef->children_[0];
const ObDASBaseCtDef *right_ctdef = merge_ctdef->children_[1];
const ObDASScanCtDef *right_scan_ctdef = nullptr;
OB_ASSERT(right_ctdef->op_type_ == DAS_OP_TABLE_SCAN || right_ctdef->op_type_ == DAS_OP_SORT);
if (right_ctdef->op_type_ == DAS_OP_TABLE_SCAN) {
right_scan_ctdef = static_cast<const ObDASScanCtDef*>(right_ctdef);
} else {
OB_ASSERT(right_ctdef->children_ != nullptr &&
right_ctdef->children_cnt_ == 1 &&
right_ctdef->children_[0]->op_type_ == DAS_OP_TABLE_SCAN);
right_scan_ctdef = static_cast<const ObDASScanCtDef*>(right_ctdef->children_[0]);
}
OB_ASSERT(right_scan_ctdef != nullptr);
ObDASIter *left_iter = nullptr;
ObDASIter *right_iter = nullptr;
if (OB_FAIL(create_index_merge_sub_tree(ls_id,
alloc,
merge_ctdef->children_[0],
merge_rtdef->children_[0],
related_tablet_ids,
tx_desc,
snapshot,
left_iter))) {
LOG_WARN("failed to create index merge sub tree", K(ret));
} else if (OB_FAIL(create_index_merge_sub_tree(ls_id,
alloc,
merge_ctdef->children_[1],
merge_rtdef->children_[1],
related_tablet_ids,
tx_desc,
snapshot,
right_iter))) {
LOG_WARN("failed to create index merge sub tree", K(ret));
} else {
ObDASIndexMergeIterParam merge_param;
ObDASIndexMergeIter *merge_iter = nullptr;
merge_param.max_size_ = merge_rtdef->eval_ctx_->is_vectorized() ?
merge_rtdef->eval_ctx_->max_batch_size_ : 1;
merge_param.eval_ctx_ = merge_rtdef->eval_ctx_;
merge_param.exec_ctx_ = &merge_rtdef->eval_ctx_->exec_ctx_;
merge_param.output_ = &right_scan_ctdef->rowkey_exprs_;
merge_param.merge_type_ = merge_ctdef->merge_type_;
merge_param.rowkey_exprs_ = &right_scan_ctdef->rowkey_exprs_;
merge_param.left_iter_ = left_iter;
merge_param.left_output_ = left_iter->get_output();
merge_param.right_iter_ = right_iter;
merge_param.right_output_ = right_iter->get_output();
merge_param.ctdef_ = merge_ctdef;
merge_param.rtdef_ = merge_rtdef;
merge_param.tx_desc_ = tx_desc;
merge_param.snapshot_ = snapshot;
merge_param.is_reverse_ = merge_ctdef->is_reverse_;
if (OB_FAIL(create_das_iter(alloc, merge_param, merge_iter))) {
LOG_WARN("failed to create index merge iter", K(merge_param));
} else if (OB_FAIL(merge_iter->set_ls_tablet_ids(ls_id, related_tablet_ids))) {
LOG_WARN("failed to set ls tablet ids", K(ret));
} else if (OB_FAIL(create_iter_children_array(2, alloc, merge_iter))) {
LOG_WARN("failed to create iter children array", K(ret));
} else {
merge_iter->get_children()[0] = left_iter;
merge_iter->get_children()[1] = right_iter;
if (merge_ctdef->is_left_child_leaf_node_) {
OB_ASSERT(left_iter->get_type() == DAS_ITER_SCAN || left_iter->get_type() == DAS_ITER_SORT);
ObDASScanIter *left_scan_iter = (left_iter->get_type() == DAS_ITER_SCAN) ?
static_cast<ObDASScanIter*>(left_iter) : static_cast<ObDASScanIter*>(left_iter->get_children()[0]);
OB_ASSERT(left_scan_iter != nullptr);
left_scan_iter->set_scan_param(merge_iter->get_left_param());
}
OB_ASSERT(right_iter->get_type() == DAS_ITER_SCAN || right_iter->get_type() == DAS_ITER_SORT);
ObDASScanIter *right_scan_iter = (right_iter->get_type() == DAS_ITER_SCAN) ?
static_cast<ObDASScanIter*>(right_iter) : static_cast<ObDASScanIter*>(right_iter->get_children()[0]);
OB_ASSERT(right_scan_iter != nullptr);
right_scan_iter->set_scan_param(merge_iter->get_right_param());
iter = merge_iter;
}
}
}
return ret;
}
int ObDASIterUtils::create_iter_children_array(const int64_t children_cnt,
common::ObIAllocator &alloc,
ObDASIter *iter)

24
src/sql/das/iter/ob_das_iter_utils.h
View File

@ -23,6 +23,7 @@
#include "sql/das/iter/ob_das_text_retrieval_iter.h"
#include "sql/das/iter/ob_das_text_retrieval_merge_iter.h"
#include "sql/das/iter/ob_das_vid_merge_iter.h"
#include "sql/das/iter/ob_das_index_merge_iter.h"
#include "sql/engine/table/ob_table_scan_op.h"
namespace oceanbase
@ -73,6 +74,11 @@ public:
const ObLSID &ls_id,
ObDASIter *root_iter);
static int set_index_merge_related_ids(const ObDASBaseCtDef *attach_ctdef,
const ObDASRelatedTabletID &related_tablet_ids,
const ObLSID &ls_id,
ObDASIter *root_iter);
private:
static int create_partition_scan_tree(ObTableScanParam &scan_param,
common::ObIAllocator &alloc,
@ -166,6 +172,24 @@ private:
ObDASMergeIter *&scan_iter,
ObDASIter *&iter_tree);
static int create_index_merge_iter_tree(ObTableScanParam &scan_param,
common::ObIAllocator &alloc,
const ObDASBaseCtDef *attach_ctdef,
ObDASBaseRtDef *attach_rtdef,
const ObDASRelatedTabletID &related_tablet_ids,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
ObDASIter *&iter_tree);
static int create_index_merge_sub_tree(const ObLSID &ls_id,
common::ObIAllocator &alloc,
const ObDASBaseCtDef *ctdef,
ObDASBaseRtDef *rtdef,
const ObDASRelatedTabletID &related_tablet_ids,
transaction::ObTxDesc *tx_desc,
transaction::ObTxReadSnapshot *snapshot,
ObDASIter *&iter);
static int create_iter_children_array(const int64_t children_cnt,
common::ObIAllocator &alloc,
ObDASIter *iter);

2
src/sql/das/iter/ob_das_lookup_iter.cpp
View File

@ -104,7 +104,7 @@ int ObDASLookupIter::inner_get_next_row()
while (OB_SUCC(ret) && !index_end_ && lookup_rowkey_cnt_ < default_row_batch_cnt) {
index_table_iter_->clear_evaluated_flag();
if (OB_FAIL(index_table_iter_->get_next_row())) {
if(OB_UNLIKELY(OB_ITER_END != ret)) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from index table", K(ret));
} else {
index_end_ = true;

18
src/sql/das/iter/ob_das_merge_iter.cpp
View File

@ -127,18 +127,18 @@ void MergeStoreRows::reuse()
void MergeStoreRows::reset()
{
exprs_ = nullptr;
eval_ctx_ = nullptr;
group_id_idx_ = OB_INVALID_INDEX;
max_size_ = 1;
saved_size_ = 0;
cur_idx_ = OB_INVALID_INDEX;
if (OB_NOT_NULL(store_rows_)) {
for (int64_t i = 0; i < max_size_; i++) {
store_rows_[i].~LastDASStoreRow();
}
store_rows_ = nullptr;
}
exprs_ = nullptr;
eval_ctx_ = nullptr;
group_id_idx_ = OB_INVALID_INDEX;
max_size_ = 1;
saved_size_ = 0;
cur_idx_ = OB_INVALID_INDEX;
}
int ObDASMergeIter::set_merge_status(MergeType merge_type)
@ -268,7 +268,7 @@ int ObDASMergeIter::do_table_scan()
LOG_WARN("failed to push back das task ptr", K(ret));
}
} // for end
LOG_DEBUG("[DAS ITER] do table scan", K(ref_table_id_), K(das_tasks_arr_.count()));
LOG_DEBUG("[DAS ITER] merge iter do table scan", K(ref_table_id_), K(das_tasks_arr_.count()));
}
return ret;
}
@ -382,7 +382,7 @@ int ObDASMergeIter::inner_get_next_rows(int64_t &count, int64_t capacity)
LOG_WARN("das merge iter failed to get next rows", K(ret));
}
}
LOG_DEBUG("das merge iter get next rows end", K(count), K(merge_type_), K(merge_state_arr_), K(ret));
LOG_DEBUG("[DAS ITER] merge iter get next rows end", K(count), K(merge_type_), K(merge_state_arr_), K(ret));
const ObBitVector *skip = nullptr;
PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_, count, skip);
return ret;
@ -811,7 +811,7 @@ int ObDASMergeIter::compare(int64_t cur_idx, int64_t &output_idx)
}
}
}
LOG_DEBUG("das merge iter compare finished", K(cur_idx), K(output_idx), K(used_for_keep_order_));
LOG_DEBUG("[DAS ITER] merge iter compare finished", K(cur_idx), K(output_idx), K(used_for_keep_order_));
return ret;
}

13
src/sql/das/iter/ob_das_merge_iter.h
View File

@ -27,7 +27,18 @@ struct ObDASMergeIterParam : public ObDASIterParam
{
public:
ObDASMergeIterParam()
: ObDASIterParam(ObDASIterType::DAS_ITER_MERGE)
: ObDASIterParam(ObDASIterType::DAS_ITER_MERGE),
eval_infos_(nullptr),
need_update_partition_id_(false),
pdml_partition_id_(nullptr),
partition_id_calc_type_(0),
should_scan_index_(false),
ref_table_id_(),
is_vectorized_(false),
frame_info_(nullptr),
execute_das_directly_(false),
enable_rich_format_(false),
used_for_keep_order_(false)
{}
ObFixedArray<ObEvalInfo*, ObIAllocator> *eval_infos_;
bool need_update_partition_id_;

7
src/sql/das/iter/ob_das_scan_iter.cpp
View File

@ -83,7 +83,7 @@ int ObDASScanIter::do_table_scan()
LOG_WARN("fail to scan table", KPC_(scan_param), K(ret));
}
}
LOG_DEBUG("das scan iter do table scan", KPC_(scan_param), K(ret));
LOG_DEBUG("[DAS ITER] scan iter do table scan", KPC_(scan_param), K(ret));
return ret;
}
@ -100,7 +100,7 @@ int ObDASScanIter::rescan()
// reset need_switch_param_ after real rescan.
scan_param_->need_switch_param_ = false;
}
LOG_DEBUG("das scan iter rescan", KPC_(scan_param), K(ret));
LOG_DEBUG("[DAS ITER] das scan iter rescan", KPC_(scan_param), K(ret));
return ret;
}
@ -130,6 +130,9 @@ int ObDASScanIter::inner_get_next_rows(int64_t &count, int64_t capacity)
LOG_WARN("failed to get next row", K(ret));
}
}
LOG_TRACE("[DAS ITER] scan iter get next rows", K(count), K(capacity), KPC_(scan_param), K(ret));
const ObBitVector *skip = nullptr;
PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_, count, skip);
return ret;
}

2
src/sql/das/iter/ob_das_scan_iter.h
View File

@ -31,7 +31,7 @@ public:
const ObDASScanCtDef *scan_ctdef_;
virtual bool is_valid() const override
{
return nullptr != scan_ctdef_;
return nullptr != scan_ctdef_ && ObDASIterParam::is_valid();
}
};

42
src/sql/das/iter/ob_das_sort_iter.cpp
View File

@ -197,21 +197,39 @@ int ObDASSortIter::inner_get_next_rows(int64_t &count, int64_t capacity)
} else if (!sort_finished_ && OB_FAIL(do_sort(true))) {
LOG_WARN("failed to do sort", K(ret));
} else {
bool got_rows = false;
// TODO: @bingfan use vectorized interface instead.
while (OB_SUCC(ret) && !got_rows) {
if (OB_FAIL(sort_impl_.get_next_row(sort_row_))) {
if (input_row_cnt_ == 0 && limit_param_.limit_ > 0 && limit_param_.offset_ > 0) {
int64_t need_offset_count = limit_param_.offset_;
while (OB_SUCC(ret) && need_offset_count > 0) {
int64_t got_count = 0;
if (OB_FAIL(sort_impl_.get_next_batch(sort_row_, need_offset_count, got_count))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next rows from child iter", K(ret));
}
}
input_row_cnt_ += got_count;
need_offset_count = need_offset_count - got_count;
got_count = 0;
}
if (OB_SUCC(ret)) {
if (OB_LIKELY(need_offset_count == 0) && OB_LIKELY(input_row_cnt_ == limit_param_.offset_)) {
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected need offset count", K(ret), K(need_offset_count), K(input_row_cnt_), K(limit_param_.offset_));
}
}
}
if (OB_SUCC(ret)) {
int64_t min_capacity = limit_param_.limit_ > 0 ? OB_MIN(capacity, limit_param_.limit_ - output_row_cnt_) : capacity;
if (OB_FAIL(sort_impl_.get_next_batch(sort_row_, min_capacity, count))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("failed to get next row from sort impl", K(ret));
}
} else {
++input_row_cnt_;
if (input_row_cnt_ > limit_param_.offset_) {
got_rows = true;
count = 1;
++output_row_cnt_;
LOG_WARN("failed to get next rows from child iter", K(ret));
}
}
output_row_cnt_ += count;
if (OB_FAIL(ret)) {
} else if (limit_param_.limit_ > 0 && (output_row_cnt_ >= limit_param_.limit_)) {
ret = OB_ITER_END;
}
}
}
return ret;

16
src/sql/das/ob_das_attach_define.cpp
View File

@ -71,6 +71,22 @@ OB_SERIALIZE_MEMBER((ObDASVIdMergeCtDef, ObDASAttachCtDef));
OB_SERIALIZE_MEMBER((ObDASVIdMergeRtDef, ObDASAttachRtDef));
const ObDASBaseCtDef *ObDASIndexMergeCtDef::get_left_ctdef() const
{
OB_ASSERT(2 == children_cnt_ && children_ != nullptr);
return children_[0];
}
const ObDASBaseCtDef *ObDASIndexMergeCtDef::get_right_ctdef() const
{
OB_ASSERT(2 == children_cnt_ && children_ != nullptr);
return children_[1];
}
OB_SERIALIZE_MEMBER((ObDASIndexMergeCtDef, ObDASAttachCtDef), merge_type_, is_left_child_leaf_node_, is_reverse_);
OB_SERIALIZE_MEMBER((ObDASIndexMergeRtDef, ObDASAttachRtDef));
OB_DEF_SERIALIZE(ObDASAttachSpec)
{
int ret = OB_SUCCESS;

37
src/sql/das/ob_das_attach_define.h
View File

@ -19,6 +19,7 @@
#include "share/ob_define.h"
#include "sql/engine/expr/ob_expr.h"
#include "sql/engine/sort/ob_sort_basic_info.h"
#include "sql/optimizer/ob_join_order.h"
namespace oceanbase
{
@ -144,6 +145,36 @@ public:
INHERIT_TO_STRING_KV("ObDASVIdMergeRtDef", ObDASAttachRtDef, KP(this));
};
struct ObDASIndexMergeCtDef : ObDASAttachCtDef
{
OB_UNIS_VERSION(1);
public:
ObDASIndexMergeCtDef(common::ObIAllocator &alloc)
: ObDASAttachCtDef(alloc, DAS_OP_INDEX_MERGE),
merge_type_(INDEX_MERGE_INVALID),
is_left_child_leaf_node_(false),
is_reverse_(false)
{}
virtual ~ObDASIndexMergeCtDef() {}
const ObDASBaseCtDef *get_left_ctdef() const;
const ObDASBaseCtDef *get_right_ctdef() const;
public:
ObIndexMergeType merge_type_;
bool is_left_child_leaf_node_;
bool is_reverse_;
};
struct ObDASIndexMergeRtDef : ObDASAttachRtDef
{
OB_UNIS_VERSION(1);
public:
ObDASIndexMergeRtDef()
: ObDASAttachRtDef(DAS_OP_INDEX_MERGE) {}
virtual ~ObDASIndexMergeRtDef() {}
};
struct ObDASAttachSpec
{
OB_UNIS_VERSION(1);
@ -162,6 +193,12 @@ public:
const ObDASTableLocMeta *get_attach_loc_meta(int64_t table_location_id, int64_t ref_table_id) const;
int set_calc_exprs(const ExprFixedArray &calc_exprs, const int64_t max_batch_size);
const ExprFixedArray &get_result_output() const
{
OB_ASSERT(attach_ctdef_ != nullptr);
return static_cast<const ObDASAttachCtDef*>(attach_ctdef_)->result_output_;
}
TO_STRING_KV(K_(attach_loc_metas),
K_(attach_ctdef));
private:

4
src/sql/das/ob_das_def_reg.h
View File

@ -147,6 +147,10 @@ struct ObDASVIdMergeRtDef;
REGISTER_DAS_ATTACH_OP(DAS_OP_VID_MERGE, ObDASVIdMergeCtDef, ObDASVIdMergeRtDef);
struct ObDASIndexMergeCtDef;
struct ObDASIndexMergeRtDef;
REGISTER_DAS_ATTACH_OP(DAS_OP_INDEX_MERGE, ObDASIndexMergeCtDef, ObDASIndexMergeRtDef);
#undef REGISTER_DAS_ATTACH_OP
} // namespace sql
} // namespace oceanbase

155
src/sql/das/ob_das_scan_op.cpp
View File

@ -23,6 +23,7 @@
#include "storage/concurrency_control/ob_data_validation_service.h"
#include "sql/das/iter/ob_das_iter_utils.h"
namespace oceanbase
{
namespace common
@ -45,7 +46,7 @@ using namespace storage;
using namespace transaction;
namespace sql
{
OB_SERIALIZE_MEMBER(ObDASScanCtDef,
OB_SERIALIZE_MEMBER(ObDASScanCtDef, // FARM COMPAT WHITELIST
ref_table_id_,
access_column_ids_,
schema_version_,
@ -68,7 +69,9 @@ OB_SERIALIZE_MEMBER(ObDASScanCtDef,
doc_id_idx_,
vec_vid_idx_,
multivalue_idx_,
multivalue_type_);
multivalue_type_,
index_merge_idx_, // FARM COMPAT WHITELIST
flags_); // FARM COMPAT WHITELIST
OB_DEF_SERIALIZE(ObDASScanRtDef)
{
@ -87,7 +90,9 @@ OB_DEF_SERIALIZE(ObDASScanRtDef)
pd_storage_flag_,
need_check_output_datum_,
is_for_foreign_check_,
fb_read_tx_uncommitted_);
fb_read_tx_uncommitted_,
key_ranges_,
ss_key_ranges_);
return ret;
}
@ -108,7 +113,9 @@ OB_DEF_DESERIALIZE(ObDASScanRtDef)
pd_storage_flag_,
need_check_output_datum_,
is_for_foreign_check_,
fb_read_tx_uncommitted_);
fb_read_tx_uncommitted_,
key_ranges_,
ss_key_ranges_);
if (OB_SUCC(ret)) {
(void)ObSQLUtils::adjust_time_by_ntp_offset(timeout_ts_);
}
@ -132,7 +139,9 @@ OB_DEF_SERIALIZE_SIZE(ObDASScanRtDef)
pd_storage_flag_,
need_check_output_datum_,
is_for_foreign_check_,
fb_read_tx_uncommitted_);
fb_read_tx_uncommitted_,
key_ranges_,
ss_key_ranges_);
return len;
}
@ -179,6 +188,7 @@ ObDASScanOp::ObDASScanOp(ObIAllocator &op_alloc)
scan_rtdef_(nullptr),
result_(nullptr),
remain_row_cnt_(0),
tablet_ids_(op_alloc),
retry_alloc_(nullptr),
ir_param_()
{
@ -351,13 +361,33 @@ ObDASIterTreeType ObDASScanOp::get_iter_tree_type() const
tree_type = ObDASIterTreeType::ITER_TREE_GIS_LOOKUP;
} else if (is_multivalue_index || is_vector_index) {
tree_type = ObDASIterTreeType::ITER_TREE_DOMAIN_LOOKUP;
} else if (OB_UNLIKELY(is_index_merge(attach_ctdef_))) {
tree_type = ObDASIterTreeType::ITER_TREE_INDEX_MERGE;
} else {
tree_type = OB_ISNULL(get_lookup_ctdef()) ? ObDASIterTreeType::ITER_TREE_PARTITION_SCAN
: ObDASIterTreeType::ITER_TREE_LOCAL_LOOKUP;
}
LOG_TRACE("get iter tree type", K(tree_type), K(scan_param_), KPC(attach_ctdef_));
return tree_type;
}
bool ObDASScanOp::is_index_merge(const ObDASBaseCtDef *attach_ctdef) const
{
bool bret = false;
if (attach_ctdef != nullptr) {
if (attach_ctdef->op_type_ == DAS_OP_INDEX_MERGE) {
bret = true;
} else if (attach_ctdef->op_type_ == DAS_OP_TABLE_LOOKUP) {
const ObDASBaseCtDef *rowkey_scan_ctdef =
static_cast<const ObDASTableLookupCtDef*>(attach_ctdef)->get_rowkey_scan_ctdef();
if (rowkey_scan_ctdef != nullptr) {
bret = rowkey_scan_ctdef->op_type_ == DAS_OP_INDEX_MERGE;
}
}
}
return bret;
}
int ObDASScanOp::init_related_tablet_ids(ObDASRelatedTabletID &related_tablet_ids)
{
int ret = OB_SUCCESS;
@ -373,6 +403,8 @@ int ObDASScanOp::init_related_tablet_ids(ObDASRelatedTabletID &related_tablet_id
related_tablet_ids.fwd_idx_tablet_id_,
related_tablet_ids.doc_id_idx_tablet_id_))) {
LOG_WARN("failed to get text ir tablet ids", K(ret));
} else if (OB_FAIL(get_index_merge_tablet_ids(related_tablet_ids.index_merge_tablet_ids_))) {
LOG_WARN("failed to get index merge tablet ids", K(ret));
}
return ret;
}
@ -468,7 +500,7 @@ int ObDASScanOp::open_op()
LOG_WARN("init scan param failed", K(ret));
} else if (FALSE_IT(tree_type = get_iter_tree_type())) {
} else if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type
|| ITER_TREE_TEXT_RETRIEVAL == tree_type) {
|| ITER_TREE_TEXT_RETRIEVAL == tree_type || ITER_TREE_INDEX_MERGE == tree_type) {
ObDASIter *result = nullptr;
if (OB_FAIL(init_related_tablet_ids(tablet_ids_))) {
LOG_WARN("failed to init related tablet ids", K(ret));
@ -516,7 +548,7 @@ int ObDASScanOp::release_op()
int ret = OB_SUCCESS;
ObDASIterTreeType tree_type = get_iter_tree_type();
if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type
|| ITER_TREE_TEXT_RETRIEVAL == tree_type) {
|| ITER_TREE_TEXT_RETRIEVAL == tree_type || ITER_TREE_INDEX_MERGE == tree_type) {
if (OB_NOT_NULL(result_)) {
ObDASIter *result = static_cast<ObDASIter*>(result_);
if (OB_FAIL(result->release())) {
@ -744,42 +776,47 @@ void ObDASScanOp::reset_access_datums_ptr(int64_t capacity)
if (scan_rtdef_->p_pd_expr_op_->is_vectorized()) {
int64_t reset_batch_size = capacity > 0 ? capacity : scan_rtdef_->eval_ctx_->max_batch_size_;
reset_batch_size = min(reset_batch_size, scan_rtdef_->eval_ctx_->max_batch_size_);
FOREACH_CNT(e, scan_ctdef_->pd_expr_spec_.access_exprs_) {
(*e)->locate_datums_for_update(*scan_rtdef_->eval_ctx_, reset_batch_size);
ObEvalInfo &info = (*e)->get_eval_info(*scan_rtdef_->eval_ctx_);
info.point_to_frame_ = true;
}
FOREACH_CNT(e, scan_ctdef_->pd_expr_spec_.pd_storage_aggregate_output_) {
(*e)->locate_datums_for_update(*scan_rtdef_->eval_ctx_, scan_rtdef_->eval_ctx_->max_batch_size_);
ObEvalInfo &info = (*e)->get_eval_info(*scan_rtdef_->eval_ctx_);
info.point_to_frame_ = true;
}
FOREACH_CNT(e, scan_ctdef_->pd_expr_spec_.ext_file_column_exprs_) {
(*e)->locate_datums_for_update(*scan_rtdef_->eval_ctx_, scan_rtdef_->eval_ctx_->max_batch_size_);
ObEvalInfo &info = (*e)->get_eval_info(*scan_rtdef_->eval_ctx_);
info.point_to_frame_ = true;
}
if (OB_NOT_NULL(scan_ctdef_->trans_info_expr_)) {
ObExpr *trans_expr = scan_ctdef_->trans_info_expr_;
trans_expr->locate_datums_for_update(*scan_rtdef_->eval_ctx_, reset_batch_size);
ObEvalInfo &info = trans_expr->get_eval_info(*scan_rtdef_->eval_ctx_);
info.point_to_frame_ = true;
if (attach_rtdef_ != nullptr) {
reset_access_datums_ptr(attach_rtdef_, reset_batch_size);
} else {
reset_access_datums_ptr(scan_rtdef_, reset_batch_size);
if (get_lookup_rtdef() != nullptr) {
reset_access_datums_ptr(get_lookup_rtdef(), reset_batch_size);
}
}
}
if (get_lookup_rtdef() != nullptr && get_lookup_rtdef()->p_pd_expr_op_->is_vectorized()) {
int64_t reset_batch_size = capacity > 0 ? capacity : scan_rtdef_->eval_ctx_->max_batch_size_;
reset_batch_size = min(reset_batch_size, scan_rtdef_->eval_ctx_->max_batch_size_);
FOREACH_CNT(e, get_lookup_ctdef()->pd_expr_spec_.access_exprs_) {
(*e)->locate_datums_for_update(*get_lookup_rtdef()->eval_ctx_, reset_batch_size);
ObEvalInfo &info = (*e)->get_eval_info(*get_lookup_rtdef()->eval_ctx_);
info.point_to_frame_ = true;
}
if (OB_NOT_NULL(get_lookup_ctdef()->trans_info_expr_)) {
ObExpr *trans_expr = get_lookup_ctdef()->trans_info_expr_;
trans_expr->locate_datums_for_update(*get_lookup_rtdef()->eval_ctx_, reset_batch_size);
ObEvalInfo &info = trans_expr->get_eval_info(*scan_rtdef_->eval_ctx_);
info.point_to_frame_ = true;
}
void ObDASScanOp::reset_access_datums_ptr(ObDASBaseRtDef *rtdef, int64_t capacity)
{
if (rtdef != nullptr) {
if (rtdef->op_type_== DAS_OP_TABLE_SCAN) {
const ObDASScanCtDef *scan_ctdef = static_cast<const ObDASScanCtDef*>(rtdef->ctdef_);
FOREACH_CNT(e, scan_ctdef->pd_expr_spec_.access_exprs_) {
(*e)->locate_datums_for_update(*rtdef->eval_ctx_, capacity);
ObEvalInfo &info = (*e)->get_eval_info(*rtdef->eval_ctx_);
info.point_to_frame_ = true;
}
FOREACH_CNT(e, scan_ctdef->pd_expr_spec_.pd_storage_aggregate_output_) {
(*e)->locate_datums_for_update(*rtdef->eval_ctx_, capacity);
ObEvalInfo &info = (*e)->get_eval_info(*rtdef->eval_ctx_);
info.point_to_frame_ = true;
}
FOREACH_CNT(e, scan_ctdef->pd_expr_spec_.ext_file_column_exprs_) {
(*e)->locate_datums_for_update(*rtdef->eval_ctx_, capacity);
ObEvalInfo &info = (*e)->get_eval_info(*rtdef->eval_ctx_);
info.point_to_frame_ = true;
}
if (scan_ctdef->trans_info_expr_ != nullptr) {
ObExpr *trans_expr = scan_ctdef->trans_info_expr_;
trans_expr->locate_datums_for_update(*rtdef->eval_ctx_, capacity);
ObEvalInfo &info = trans_expr->get_eval_info(*rtdef->eval_ctx_);
info.point_to_frame_ = true;
}
} else {
for (int64_t i = 0; i < rtdef->children_cnt_; i++) {
reset_access_datums_ptr(rtdef->children_[i], capacity);
}
}
}
}
@ -952,7 +989,7 @@ int ObDASScanOp::rescan()
"range_pos", scan_param_.range_array_pos_);
ObDASIterTreeType tree_type = get_iter_tree_type();
if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type
|| ITER_TREE_TEXT_RETRIEVAL == tree_type) {
|| ITER_TREE_TEXT_RETRIEVAL == tree_type || ITER_TREE_INDEX_MERGE == tree_type) {
if (OB_ISNULL(result_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr das iter tree", K(ret));
@ -996,7 +1033,7 @@ int ObDASScanOp::reuse_iter()
ObITabletScan &tsc_service = get_tsc_service();
ObLocalIndexLookupOp *lookup_op = get_lookup_op();
if (ITER_TREE_PARTITION_SCAN == tree_type || ITER_TREE_LOCAL_LOOKUP == tree_type
|| ITER_TREE_TEXT_RETRIEVAL == tree_type) {
|| ITER_TREE_TEXT_RETRIEVAL == tree_type || ITER_TREE_INDEX_MERGE == tree_type) {
if (OB_ISNULL(result_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr das iter tree", K(ret));
@ -1027,6 +1064,14 @@ int ObDASScanOp::reuse_iter()
}
break;
}
case ITER_TREE_INDEX_MERGE: {
ObDASIter *result_iter = static_cast<ObDASIter *>(result_);
if (OB_FAIL(ObDASIterUtils::set_index_merge_related_ids(
attach_ctdef_, tablet_ids_, ls_id_, result_iter))) {
LOG_WARN("failed to set index merge related ids", K(ret));
}
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
}
@ -1315,6 +1360,30 @@ int ObDASScanOp::get_text_ir_tablet_ids(
return ret;
}
int ObDASScanOp::get_index_merge_tablet_ids(common::ObIArray<common::ObTabletID> &index_merge_tablet_ids)
{
int ret = OB_SUCCESS;
int64_t N = related_tablet_ids_.count();
if (OB_UNLIKELY(related_ctdefs_.count() != related_tablet_ids_.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected related scan array not match", K(ret), K_(related_ctdefs), K_(related_tablet_ids));
} else if (OB_FAIL(index_merge_tablet_ids.prepare_allocate(N - 1))) { // need to remove main table
LOG_WARN("failed to prepare allocate index merge tablet ids", K(N), K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) {
const ObDASScanCtDef *ctdef = static_cast<const ObDASScanCtDef*>(related_ctdefs_.at(i));
if (OB_ISNULL(ctdef)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KPC(ctdef), K(related_tablet_ids_.count()), K(ret));
} else if (ctdef->is_index_merge_) {
OB_ASSERT(ctdef->index_merge_idx_ >= 0 && ctdef->index_merge_idx_ < N - 1);
index_merge_tablet_ids.at(ctdef->index_merge_idx_) = related_tablet_ids_.at(i);
}
}
}
return ret;
}
OB_SERIALIZE_MEMBER((ObDASScanOp, ObIDASTaskOp),
scan_param_.key_ranges_,
scan_ctdef_,

36
src/sql/das/ob_das_scan_op.h
View File

@ -18,6 +18,7 @@
#include "sql/engine/table/ob_index_lookup_op_impl.h"
#include "sql/das/ob_group_scan_iter.h"
#include "sql/das/iter/ob_das_iter.h"
#include "sql/rewrite/ob_query_range.h"
namespace oceanbase
{
@ -54,7 +55,10 @@ public:
doc_id_idx_(-1),
vec_vid_idx_(-1),
multivalue_idx_(-1),
multivalue_type_(0)
multivalue_type_(0),
index_merge_idx_(OB_INVALID_ID),
pre_query_range_(),
flags_(0)
{ }
//in das scan op, column described with column expr
virtual bool has_expr() const override { return true; }
@ -91,7 +95,10 @@ public:
K_(ir_scan_type),
K_(rowkey_exprs),
K_(table_scan_opt),
K_(vec_vid_idx));
K_(vec_vid_idx),
K_(index_merge_idx),
K_(pre_query_range),
K_(is_index_merge));
common::ObTableID ref_table_id_;
UIntFixedArray access_column_ids_;
int64_t schema_version_;
@ -118,6 +125,15 @@ public:
int64_t vec_vid_idx_;
int64_t multivalue_idx_;
int32_t multivalue_type_;
int64_t index_merge_idx_; // idx of the index scan node in index merge tree
ObQueryRange pre_query_range_;
union {
uint64_t flags_;
struct {
uint64_t is_index_merge_ : 1; // whether used for index merge
uint64_t reserved_ : 63;
};
};
};
struct ObDASScanRtDef : ObDASBaseRtDef
@ -144,7 +160,10 @@ public:
stmt_allocator_("StmtScanAlloc"),
scan_allocator_("TableScanAlloc"),
sample_info_(nullptr),
is_for_foreign_check_(false)
is_for_foreign_check_(false),
key_ranges_(),
ss_key_ranges_(),
mbr_filters_()
{ }
virtual ~ObDASScanRtDef();
bool enable_rich_format() const { return scan_flag_.enable_rich_format_; }
@ -159,7 +178,10 @@ public:
K_(timeout_ts),
K_(tx_lock_timeout),
K_(sql_mode),
K_(scan_flag));
K_(scan_flag),
K_(key_ranges),
K_(ss_key_ranges),
K_(mbr_filters));
int init_pd_op(ObExecContext &exec_ctx, const ObDASScanCtDef &scan_ctdef);
storage::ObRow2ExprsProjector *p_row2exprs_projector_;
ObPushdownOperator *p_pd_expr_op_;
@ -180,6 +202,9 @@ public:
common::ObWrapperAllocatorWithAttr scan_allocator_;
const common::SampleInfo *sample_info_; //Block(Row)SampleScan, only support local das scan
bool is_for_foreign_check_;
common::ObSEArray<common::ObNewRange, 1> key_ranges_;
common::ObSEArray<common::ObNewRange, 1> ss_key_ranges_;
common::ObSEArray<common::ObSpatialMBR, 1> mbr_filters_;
private:
union {
storage::ObRow2ExprsProjector row2exprs_projector_;
@ -240,6 +265,7 @@ public:
int rescan();
int reuse_iter();
void reset_access_datums_ptr(int64_t capacity = 0);
void reset_access_datums_ptr(ObDASBaseRtDef *rtdef, int64_t capacity);
ObLocalIndexLookupOp *get_lookup_op();
bool is_contain_trans_info() {return NULL != scan_ctdef_->trans_info_expr_; }
int do_table_scan();
@ -254,6 +280,7 @@ public:
common::ObTabletID &index_id_tid,
common::ObTabletID &snapshot_tid,
common::ObTabletID &com_aux_vec_tid);
int get_index_merge_tablet_ids(common::ObIArray<common::ObTabletID> &index_merge_tablet_ids);
bool enable_rich_format() const { return scan_rtdef_->enable_rich_format(); }
INHERIT_TO_STRING_KV("parent", ObIDASTaskOp,
KPC_(scan_ctdef),
@ -267,6 +294,7 @@ protected:
common::ObNewRowIterator *get_storage_scan_iter();
common::ObNewRowIterator *get_output_result_iter() { return result_; }
ObDASIterTreeType get_iter_tree_type() const;
bool is_index_merge(const ObDASBaseCtDef *attach_ctdef) const;
public:
ObSEArray<ObDatum *, 4> trans_info_array_;
protected:

127
src/sql/engine/table/ob_table_scan_op.cpp
View File

@ -942,7 +942,9 @@ int ObTableScanOp::prepare_all_das_tasks()
if (need_perform_real_batch_rescan()) {
grp_guard.switch_group_rescan_param(i);
}
if (OB_FAIL(prepare_single_scan_range(i))) {
if (MY_CTDEF.use_index_merge_ && OB_FAIL(prepare_index_merge_scan_range(i))) {
LOG_WARN("failed to prepare index merge scan range", K(ret));
} else if (!MY_CTDEF.use_index_merge_ && OB_FAIL(prepare_single_scan_range(i))) {
LOG_WARN("prepare single scan range failed", K(ret));
} else if (OB_FAIL(prepare_das_task())) {
LOG_WARN("prepare das task failed", K(ret));
@ -1172,7 +1174,11 @@ int ObTableScanOp::prepare_scan_range()
{
int ret = OB_SUCCESS;
if (!need_perform_real_batch_rescan()) {
ret = prepare_single_scan_range();
if (MY_CTDEF.use_index_merge_ && OB_FAIL(prepare_index_merge_scan_range())) {
LOG_WARN("failed to prepare index merge range", K(ret));
} else if (!MY_CTDEF.use_index_merge_ && OB_FAIL(prepare_single_scan_range())) {
LOG_WARN("failed to prepare single scan range", K(ret));
}
} else {
ret = prepare_batch_scan_range();
}
@ -1371,6 +1377,123 @@ int ObTableScanOp::prepare_single_scan_range(int64_t group_idx)
return ret;
}
// for index merge, disable equal range optimization
int ObTableScanOp::prepare_index_merge_scan_range(int64_t group_idx)
{
int ret = OB_SUCCESS;
ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_);
ObIAllocator &range_allocator = (table_rescan_allocator_ != nullptr ?
*table_rescan_allocator_ : ctx_.get_allocator());
ObDASBaseRtDef *attach_rtdef = tsc_rtdef_.attach_rtinfo_->attach_rtdef_;
if (OB_ISNULL(attach_rtdef)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KPC(attach_rtdef), K(ret));
} else {
ObDASBaseRtDef *index_merge_rtdef = attach_rtdef->op_type_ == DAS_OP_TABLE_LOOKUP ?
attach_rtdef->children_[0] : attach_rtdef;
if (OB_FAIL(prepare_range_for_each_index(group_idx, range_allocator, index_merge_rtdef))) {
LOG_WARN("failed to prepare range for each index", KPC(index_merge_rtdef), K(ret));
}
}
return ret;
}
int ObTableScanOp::prepare_range_for_each_index(int64_t group_idx,
ObIAllocator &allocator,
ObDASBaseRtDef *rtdef)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(rtdef)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KPC(rtdef), K(ret));
} else if (rtdef->op_type_ == DAS_OP_TABLE_SCAN) {
ObQueryRangeArray key_ranges;
ObQueryRangeArray ss_key_ranges;
ObDASScanRtDef *scan_rtdef = static_cast<ObDASScanRtDef*>(rtdef);
const ObDASScanCtDef *scan_ctdef = static_cast<const ObDASScanCtDef*>(rtdef->ctdef_);
const ObQueryRange &pre_query_range = scan_ctdef->pre_query_range_;
scan_rtdef->key_ranges_.reuse();
scan_rtdef->ss_key_ranges_.reuse();
scan_rtdef->mbr_filters_.reuse();
if (OB_UNLIKELY(!pre_query_range.has_range())) {
// virtual table, do nothing
} else if (pre_query_range.is_contain_geo_filters() &&
OB_FAIL(ObSQLUtils::extract_geo_query_range(
pre_query_range,
allocator,
ctx_,
key_ranges,
scan_rtdef->mbr_filters_,
ObBasicSessionInfo::create_dtc_params(ctx_.get_my_session())))) {
LOG_WARN("failed to extract pre query ranges", K(ret));
} else if (!pre_query_range.is_contain_geo_filters() &&
OB_FAIL(ObSQLUtils::extract_pre_query_range(
pre_query_range,
allocator,
ctx_,
key_ranges,
ObBasicSessionInfo::create_dtc_params(ctx_.get_my_session())))) {
LOG_WARN("failed to extract pre query ranges", K(ret));
} else if (OB_FAIL(pre_query_range.get_ss_tablet_ranges(allocator,
ctx_,
ss_key_ranges,
ObBasicSessionInfo::create_dtc_params(ctx_.get_my_session())))) {
LOG_WARN("failed to final extract index skip query range", K(ret));
} else if (!ss_key_ranges.empty()) {
// index skip scan, ranges from extract_pre_query_range/get_ss_tablet_ranges,
// prefix range and postfix range is single range
if (1 != ss_key_ranges.count() || 1 != key_ranges.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index skip scan range", K(ret), K(key_ranges), K(ss_key_ranges));
} else {
key_ranges.at(0)->table_id_ = scan_ctdef->ref_table_id_;
key_ranges.at(0)->group_idx_ = group_idx;
ss_key_ranges.at(0)->table_id_ = scan_ctdef->ref_table_id_;
ss_key_ranges.at(0)->group_idx_ = group_idx;
if (OB_FAIL(scan_rtdef->key_ranges_.push_back(*key_ranges.at(0))) ||
OB_FAIL(scan_rtdef->ss_key_ranges_.push_back(*ss_key_ranges.at(0)))) {
LOG_WARN("failed to push back ss key range", KPC(scan_rtdef), K(ret));
}
}
} else {
ObNewRange whole_range;
ObNewRange *key_range = nullptr;
whole_range.set_whole_range();
whole_range.table_id_ = scan_ctdef->ref_table_id_;
whole_range.group_idx_ = group_idx;
for (int64_t i = 0; OB_SUCC(ret) && i < key_ranges.count(); ++i) {
key_range = key_ranges.at(i);
key_range->table_id_ = scan_ctdef->ref_table_id_;
key_range->group_idx_ = group_idx;
if (OB_FAIL(scan_rtdef->key_ranges_.push_back(*key_range)) ||
OB_FAIL(scan_rtdef->ss_key_ranges_.push_back(whole_range))) {
LOG_WARN("failed to push back key range", KPC(scan_rtdef), K(ret));
}
}
}
if (OB_SUCC(ret) && MY_SPEC.is_vt_mapping_) {
OZ(vt_result_converter_->convert_key_ranges(scan_rtdef->key_ranges_));
}
} else if (rtdef->op_type_ == DAS_OP_SORT) {
OB_ASSERT(rtdef->children_ != nullptr && rtdef->children_cnt_ == 1);
if (OB_FAIL(prepare_range_for_each_index(group_idx, allocator, rtdef->children_[0]))) {
LOG_WARN("failed to prepare scan range for child", K(ret));
}
} else if (rtdef->op_type_ == DAS_OP_INDEX_MERGE) {
OB_ASSERT(rtdef->children_ != nullptr && rtdef->children_cnt_ == 2);
if (OB_FAIL(prepare_range_for_each_index(group_idx, allocator, rtdef->children_[0]))) {
LOG_WARN("failed to prepare scan range for left tree", K(ret));
} else if (OB_FAIL(prepare_range_for_each_index(group_idx, allocator, rtdef->children_[1]))) {
LOG_WARN("failed to prepare scan range for right tree", K(ret));
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected rtdef type", K(rtdef), K(ret));
}
return ret;
}
int ObTableScanOp::single_equal_scan_check_type(const ParamStore &param_store, bool& is_same_type)
{
int ret = OB_SUCCESS;

15
src/sql/engine/table/ob_table_scan_op.h
View File

@ -137,6 +137,7 @@ struct GroupRescanParamInfo
common::ObObjParam cur_param_; //current param in param store, used to restore paramstore state after the completion of group rescan.
};
typedef common::ObFixedArray<GroupRescanParamInfo, common::ObIAllocator> GroupRescanParamArray;
struct ObTableScanCtDef
{
OB_UNIS_VERSION(1);
@ -158,7 +159,9 @@ public:
{ }
const ExprFixedArray &get_das_output_exprs() const
{
return lookup_ctdef_ != nullptr ? lookup_ctdef_->result_output_ : scan_ctdef_.result_output_;
return attach_spec_.attach_ctdef_ != nullptr ? attach_spec_.get_result_output()
: lookup_ctdef_ != nullptr ? lookup_ctdef_->result_output_
: scan_ctdef_.result_output_;
}
const UIntFixedArray &get_full_acccess_cids() const
{
@ -179,7 +182,9 @@ public:
KPC_(das_dppr_tbl),
KPC_(calc_part_id_expr),
K_(global_index_rowkey_exprs),
K_(attach_spec));
K_(attach_spec),
K_(is_das_keep_order),
K_(use_index_merge));
//the query range of index scan/table scan
ObQueryRange pre_query_range_;
FlashBackItem flashback_item_;
@ -211,7 +216,8 @@ public:
uint64_t flags_;
struct {
uint64_t is_das_keep_order_ : 1; // whether das need keep ordering
uint64_t reserved_ : 63;
uint64_t use_index_merge_ : 1; // whether use index merge
uint64_t reserved_ : 62;
};
};
};
@ -485,7 +491,8 @@ protected:
int single_equal_scan_check_type(const ParamStore &param_store, bool& is_same_type);
bool need_extract_range() const { return MY_SPEC.tsc_ctdef_.pre_query_range_.has_range(); }
int prepare_single_scan_range(int64_t group_idx = 0);
int prepare_index_merge_scan_range(int64_t group_idx = 0);
int prepare_range_for_each_index(int64_t group_idx, ObIAllocator &allocator, ObDASBaseRtDef *rtdef);
int reuse_table_rescan_allocator();
int local_iter_rescan();

14
src/sql/optimizer/ob_index_info_cache.cpp
View File

@ -30,7 +30,8 @@ ObIndexInfoCache::~ObIndexInfoCache()
int ObIndexInfoCache::get_index_info_entry(const uint64_t table_id,
const uint64_t index_id,
IndexInfoEntry *&entry) const
IndexInfoEntry *&entry,
int64_t *idx) const
{
int ret = OB_SUCCESS;
entry = NULL;
@ -45,6 +46,9 @@ int ObIndexInfoCache::get_index_info_entry(const uint64_t table_id,
LOG_WARN("entry should not be null", K(ret));
} else if (index_entrys_[i]->get_index_id() == index_id) {
entry = index_entrys_[i];
if (idx != nullptr) {
*idx = i;
}
break;
}
}
@ -106,9 +110,17 @@ int ObIndexInfoCache::get_access_path_ordering(const uint64_t table_id,
int ObIndexInfoCache::add_index_info_entry(IndexInfoEntry *entry)
{
int ret = OB_SUCCESS;
IndexInfoEntry *old_entry;
int64_t idx = OB_INVALID_INDEX;
if (OB_ISNULL(entry)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("entry should not be null", K(ret));
} else if (OB_FAIL(get_index_info_entry(table_id_, entry->get_index_id(), old_entry, &idx))) {
LOG_WARN("failed to get index info entry", KPC(entry), K(ret));
} else if (old_entry != nullptr) {
// update index info entry
old_entry->~IndexInfoEntry();
index_entrys_[idx] = entry;
} else if (entry_count_ >= common::OB_MAX_INDEX_PER_TABLE + 1) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid entry count", K(ret), K_(entry_count),

4
src/sql/optimizer/ob_index_info_cache.h
View File

@ -223,12 +223,14 @@ public:
int get_query_range(const uint64_t table_id,
const uint64_t index_id,
const QueryRangeInfo *&query_range_info) const;
int get_access_path_ordering(const uint64_t table_id,
const uint64_t index_id,
const OrderingInfo *&ordering_info) const;
int get_index_info_entry(const uint64_t table_id,
const uint64_t index_id,
IndexInfoEntry *&entry) const;
IndexInfoEntry *&entry,
int64_t *idx = nullptr) const;
void set_table_id(const uint64_t table_id) { table_id_ = table_id; }
void set_base_table_id(const uint64_t base_table_id) { base_table_id_ = base_table_id; }
uint64_t get_table_id() const { return table_id_; }

516
src/sql/optimizer/ob_join_order.cpp
View File

@ -1782,6 +1782,8 @@ int ObJoinOrder::create_one_access_path(const uint64_t table_id,
ap->est_cost_info_.use_column_store_ = use_column_store;
ap->contain_das_op_ = ap->use_das_;
ap->is_ror_ = (ref_id == index_id) ? true
: range_info.get_equal_prefix_count() >= range_info.get_index_column_count();
if (OB_FAIL(init_sample_info_for_access_path(ap, table_id, table_item))) {
LOG_WARN("failed to init sample info", K(ret));
} else if (OB_FAIL(add_access_filters(ap,
@ -2742,6 +2744,9 @@ int ObJoinOrder::create_access_paths(const uint64_t table_id,
const ParamStore *params = NULL;
bool is_valid = true;
ObSQLSessionInfo *session_info = NULL;
bool use_index_merge = false;
ObArray<uint64_t> index_merge_list;
ObRawExpr* index_merge_root = NULL;
if (OB_ISNULL(get_plan()) ||
OB_ISNULL(stmt = get_plan()->get_stmt()) ||
OB_ISNULL(opt_ctx = &get_plan()->get_optimizer_context()) ||
@ -2756,6 +2761,23 @@ int ObJoinOrder::create_access_paths(const uint64_t table_id,
} else if (OB_FAIL(get_generated_col_index_qual(table_id,
helper.filters_, helper))) {
LOG_WARN("get prefix index qual failed");
} else if (OB_FAIL(check_can_use_index_merge(table_id,
ref_table_id,
helper,
use_index_merge,
index_merge_list,
index_merge_root))) {
LOG_WARN("failed to check can use index merge", K(ret));
} else if (OB_UNLIKELY(use_index_merge)) {
if (OB_FAIL(create_index_merge_path(table_id,
ref_table_id,
helper,
access_paths,
index_info_cache,
index_merge_list,
index_merge_root))) {
LOG_WARN("failed to create index merge path", K(index_merge_list), K(ret));
}
} else if (OB_FAIL(get_valid_index_ids(table_id,
ref_table_id,
candi_index_ids))) {
@ -2771,21 +2793,16 @@ int ObJoinOrder::create_access_paths(const uint64_t table_id,
valid_index_ids,
helper))) {
LOG_WARN("failed to add table by heuristics", K(ret));
} else if (!valid_index_ids.empty()) {
LOG_TRACE("table added using heuristics", K(table_id));
} else if (OB_FAIL(skyline_prunning_index(table_id,
ref_table_id,
stmt,
true,
index_info_cache,
candi_index_ids,
valid_index_ids,
helper.filters_))) {
} else if (valid_index_ids.empty() && OB_FAIL(skyline_prunning_index(table_id,
ref_table_id,
stmt,
true,
index_info_cache,
candi_index_ids,
valid_index_ids,
helper.filters_))) {
LOG_WARN("failed to pruning_index", K(table_id), K(ref_table_id), K(ret));
} else {
LOG_TRACE("table added using skyline", K(table_id), K(valid_index_ids));
}
if (OB_SUCC(ret)) {
helper.table_opt_info_->optimization_method_ = OptimizationMethod::COST_BASED;
for (int64_t i = 0; OB_SUCC(ret) && i < valid_index_ids.count(); ++i) {
bool is_create_basic_path = false;
@ -2850,38 +2867,267 @@ int ObJoinOrder::create_access_paths(const uint64_t table_id,
OB_FAIL(access_paths.push_back(das_column_store_access_path))) {
LOG_WARN("failed to push back access path", K(ret));
} else if (is_create_basic_path &&
use_row_store &&
OB_FAIL(create_one_access_path(table_id,
ref_table_id,
valid_index_ids.at(i),
index_info_cache,
helper,
basic_row_store_access_path,
false,
false,
use_skip_scan))) {
use_row_store &&
OB_FAIL(create_one_access_path(table_id,
ref_table_id,
valid_index_ids.at(i),
index_info_cache,
helper,
basic_row_store_access_path,
false,
false,
use_skip_scan))) {
LOG_WARN("failed to make index path", "index_table_id", valid_index_ids.at(i), K(ret));
} else if(OB_NOT_NULL(basic_row_store_access_path) &&
OB_FAIL(access_paths.push_back(basic_row_store_access_path))) {
LOG_WARN("failed to push back access path", K(ret));
} else if (is_create_basic_path &&
use_column_store &&
OB_FAIL(create_one_access_path(table_id,
ref_table_id,
valid_index_ids.at(i),
index_info_cache,
helper,
basic_column_store_access_path,
false,
true,
use_skip_scan))) {
use_column_store &&
OB_FAIL(create_one_access_path(table_id,
ref_table_id,
valid_index_ids.at(i),
index_info_cache,
helper,
basic_column_store_access_path,
false,
true,
use_skip_scan))) {
LOG_WARN("failed to make index path", "index_table_id", valid_index_ids.at(i), K(ret));
} else if( OB_NOT_NULL(basic_column_store_access_path) &&
OB_FAIL(access_paths.push_back(basic_column_store_access_path))) {
} else if(OB_NOT_NULL(basic_column_store_access_path) &&
OB_FAIL(access_paths.push_back(basic_column_store_access_path))) {
LOG_WARN("failed to push back access path", K(ret));
}
}
}
return ret;
}
int ObJoinOrder::check_can_use_index_merge(const uint64_t table_id,
const uint64_t ref_table_id,
PathHelper &helper,
bool &use_index_merge,
ObIArray<uint64_t> &index_merge_list,
ObRawExpr *&index_merge_root)
{
int ret = OB_SUCCESS;
use_index_merge = false;
index_merge_list.reuse();
index_merge_root = nullptr;
bool contains_invalid_index = false;
bool is_all_local_index = false;
bool is_all_global_index = false;
ObArray<ObArray<uint64_t>> merge_index_column_ids;
const LogTableHint *log_table_hint = nullptr;
const ObUnionMergeHint *union_merge_hint = nullptr;
ObSqlSchemaGuard *schema_guard = nullptr;
const ObDMLStmt *stmt = nullptr;
ObQueryCtx *query_ctx = nullptr;
ObRawExpr *root = nullptr;
OPT_TRACE_TITLE("BEGIN CHECK USE INDEX MERGE");
OPT_TRACE("table_id: ", table_id, "ref_table_id: ", ref_table_id);
OPT_TRACE("full query filters: ", helper.filters_);
if (OB_ISNULL(get_plan()) ||
OB_ISNULL(stmt = get_plan()->get_stmt()) ||
OB_ISNULL(schema_guard = get_plan()->get_optimizer_context().get_sql_schema_guard()) ||
OB_ISNULL(query_ctx = stmt->get_query_ctx())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get unexpected null", K(get_plan()), K(stmt), K(schema_guard), K(query_ctx), K(ret));
} else if (query_ctx->optimizer_features_enable_version_ < COMPAT_VERSION_4_3_4) {
LOG_TRACE("failed to index merge due to compat version < 4.3.4");
} else if (is_virtual_table(ref_table_id)) {
LOG_TRACE("failed to index merge due to virtual table");
} else if (FALSE_IT(log_table_hint = get_plan()->get_log_plan_hint().get_log_table_hint(table_id))) {
} else if (log_table_hint == nullptr || log_table_hint->merge_index_list_.count() < 2) {
OPT_TRACE("failed to index merge due to no valid index merge hint", log_table_hint);
LOG_TRACE("failed to index merge due to no valid index merge hint", K(log_table_hint));
} else if (OB_FAIL(check_index_merge_list(table_id,
ref_table_id,
log_table_hint->merge_index_list_,
contains_invalid_index,
is_all_local_index,
is_all_global_index,
merge_index_column_ids))) {
LOG_WARN("failed to check index merge list", K(ret));
} else if (contains_invalid_index) {
OPT_TRACE("failed to index merge due to contains invalid index");
LOG_TRACE("failed to index merge due to contains invalid index",
K(log_table_hint->merge_index_list_));
} else if (!is_all_local_index) {
OPT_TRACE("failed to index merge due to not all local index");
LOG_TRACE("failed to index merge due to not all local index",
K(log_table_hint->merge_index_list_));
} else {
// only Disjunctive Normal Form (DNF) and Conjunctive Normal Form (CNF) support index merge.
// for DNF, the number of predicate segments connected in the DNF must match the number of indexes
// specified in the hint. each part of predicate segment will be processed by the corresponding index.
// for CNF, we try to find the first disjunctive subformula which satisfies the index merge
// requirement and perform index merge with the subformula.
OPT_TRACE("got index merge list", log_table_hint->merge_index_list_);
int64_t index_cnt = log_table_hint->merge_index_list_.count();
bool found_valid_disjunctive_form = false;
for (int64_t i = 0; OB_SUCC(ret) && !found_valid_disjunctive_form && i < helper.filters_.count(); ++i) {
if (OB_ISNULL(root = helper.filters_.at(i))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("unexpected null filter", K(ret));
} else if (!IndexMergePath::is_disjunctive_clause(root)) {
OPT_TRACE("filter failed to index merge due to not match disjunctive clause", root);
LOG_TRACE("filter failed to index merge due to not match disjunctive clause", K(root));
} else if (root->get_param_count() != index_cnt) {
OPT_TRACE("filter failed to index merge due to filter and index count not matched",
root, "filter count", root->get_param_count(), "index count", index_cnt);
LOG_TRACE("filter failed to index merge due to not match disjunctive clause", K(root));
} else {
// check whether each filter overlap by the corresponding index columns
bool index_overlap_filter = true;
for (int64_t i = 0; OB_SUCC(ret) && i < index_cnt && index_overlap_filter; ++i) {
ObRawExpr *param_expr = root->get_param_expr(i);
ObArray<uint64_t> column_ids;
if (OB_ISNULL(param_expr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null filter", K(ret));
} else if (OB_FAIL(ObRawExprUtils::extract_column_ids(param_expr, column_ids))) {
LOG_WARN("failed to extract column ids", KPC(param_expr), K(ret));
} else if (!ObOptimizerUtil::overlap(column_ids, merge_index_column_ids.at(i))) {
index_overlap_filter = false;
OPT_TRACE("filter failed to index merge due to not overlap with index", root,
"filter", param_expr, "index", log_table_hint->merge_index_list_.at(i), "filter column ids",
column_ids, "index column ids", merge_index_column_ids.at(i));
LOG_TRACE("filter failed to index merge due to not overlap with index", KPC(param_expr), K(column_ids),
K(merge_index_column_ids.at(i)));
}
}
if (OB_SUCC(ret) && index_overlap_filter) {
if (OB_FAIL(index_merge_list.assign(log_table_hint->merge_index_list_))) {
LOG_WARN("failed to assign index merge list", K(ret));
} else {
found_valid_disjunctive_form = true;
use_index_merge = true;
index_merge_root = root;
}
}
}
}
}
OPT_TRACE("index merge list", index_merge_list, "use_index_merge", use_index_merge,
"index_merge_root", index_merge_root);
OPT_TRACE_TITLE("END CHECK USE INDEX MERGE");
LOG_TRACE("check can use index merge finished", K(table_id), K(ref_table_id),
K(use_index_merge), K(index_merge_list), K(index_merge_root));
return ret;
}
int ObJoinOrder::create_index_merge_path(const uint64_t table_id,
const uint64_t ref_table_id,
PathHelper &helper,
ObIArray<AccessPath *> &access_paths,
ObIndexInfoCache &index_info_cache,
const ObIArray<uint64_t> &index_merge_list,
ObRawExpr *index_merge_root)
{
int ret = OB_SUCCESS;
const ObDMLStmt *stmt = nullptr;
ObSqlSchemaGuard *schema_guard = nullptr;
IndexMergePath *index_merge_path = nullptr;
ObArray<ObRawExpr *> backup_filters; // backup helper filters
if (OB_ISNULL(get_plan()) ||
OB_ISNULL(stmt = get_plan()->get_stmt()) ||
OB_ISNULL(schema_guard = get_plan()->get_optimizer_context().get_sql_schema_guard()) ||
index_merge_list.count() < 2 ||
OB_ISNULL(index_merge_root) ||
OB_ISNULL(allocator_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("unexpected null", K(get_plan()), K(stmt), K(schema_guard), K(index_merge_list),
K(index_merge_root), K(allocator_), K(ret));
} else if (OB_FAIL(backup_filters.assign(helper.filters_))) {
LOG_WARN("failed to assign filters array", K(ret));
} else if (OB_ISNULL(index_merge_path = OB_NEWx(IndexMergePath, allocator_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to new a index merge path", K(ret));
} else {
// simply use das for all global index, otherwise not use das.
// since px is not supported, use das for all situations now.
bool use_das = true;
bool use_column_store = false;
OptSkipScanState use_skip_scan = OptSkipScanState::SS_UNSET;
index_info_cache.set_table_id(table_id);
index_info_cache.set_base_table_id(ref_table_id);
ObArray<ObString> index_name_list;
OB_ASSERT(index_merge_list.count() == index_merge_root->get_param_count());
/* create access path for each index scan and combine them to genrate a index merge path */
for (int64_t i = 0; OB_SUCC(ret) && i < index_merge_list.count(); i++) {
AccessPath *ap = nullptr;
IndexInfoEntry *index_info_entry = nullptr;
const ObTableSchema *index_schema = nullptr;
ObString index_name;
uint64_t index_id = index_merge_list.at(i);
// replace helper.filters_ with the corresponding filters of the current index table
// to extract the query range and create index table access path
helper.filters_.at(0) = index_merge_root->get_param_expr(i);
if (OB_FAIL(fill_index_info_entry(table_id, ref_table_id, index_id, index_info_entry, helper))) {
LOG_WARN("failed to fill index info entry", K(ret));
} else if (OB_ISNULL(index_info_entry)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to fill index info entry", K(ret));
} else if (FALSE_IT(index_info_entry->set_is_index_back(true))) {
// set force index back when use index merge
} else if (OB_FAIL(index_info_cache.add_index_info_entry(index_info_entry))) {
LOG_WARN("failed to add index info entry", K(ret));
} else if (OB_FAIL(create_one_access_path(table_id,
ref_table_id,
index_id,
index_info_cache,
helper,
ap,
use_das,
use_column_store,
use_skip_scan))) {
LOG_WARN("failed to create one access path", K(table_id), K(ref_table_id), K(index_id));
} else if (OB_FAIL(index_merge_path->add_index_scan_node(INDEX_MERGE_UNION, ap, allocator_))) {
LOG_WARN("failed to add index scan node", K(index_merge_path), K(ap));
} else if (ref_table_id == index_id) {
index_name = ObIndexHint::PRIMARY_KEY;
} else if (OB_FAIL(schema_guard->get_table_schema(table_id, index_id, stmt, index_schema))) {
ret = OB_SCHEMA_ERROR;
LOG_WARN("failed to get index table schema", K(index_id), K(ret));
} else if (OB_FAIL(index_schema->get_index_name(index_name))) {
LOG_WARN("failed to get index name", K(index_id), K(ret));
}
if (OB_SUCC(ret) && OB_FAIL(index_name_list.push_back(index_name))) {
LOG_WARN("failed to push back index name", K(ret));
}
} // for end
/* adjust index merge path ordering and scan direction */
if (OB_SUCC(ret)) {
ObArray<ObRawExpr*> rowkey_exprs;
ObOrderDirection direction = default_asc_direction();
if (!index_merge_path->is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid index merge path", KPC(index_merge_path), K(ret));
} else if (OB_FAIL(index_merge_path->filters_.assign(backup_filters))) {
LOG_WARN("failed to assign filters", K(ret));
} else if (OB_FAIL(get_plan()->get_rowkey_exprs(table_id, ref_table_id, rowkey_exprs))) {
LOG_WARN("failed to get rowkey exprs", K(ret));
} else if (FALSE_IT(index_merge_path->ordering_.reuse())) {
} else if (OB_FAIL(get_index_scan_direction(rowkey_exprs, stmt, get_plan()->get_equal_sets(), direction))) {
LOG_WARN("failed to get index scan direction", K(ret));
} else if (OB_FAIL(index_merge_path->set_scan_direction(direction))) {
LOG_WARN("failed to set scan direction for index merge path", K(ret), KPC(index_merge_path));
} else if (OB_FAIL(ObOptimizerUtil::make_sort_keys(rowkey_exprs,
index_merge_path->order_direction_,
index_merge_path->ordering_))) {
LOG_WARN("failed to make rowkey ordering for index merge path", K(ret));
} else if (OB_FAIL(index_merge_path->index_name_list_.assign(index_name_list))) {
LOG_WARN("failed to assign index name list", K(ret));
} else if (FALSE_IT(index_merge_path->force_used_by_hint_ = true)) {
} else if (OB_FAIL(access_paths.push_back(index_merge_path))) {
LOG_WARN("failed to push back index merge path", K(ret));
}
}
}
return ret;
}
@ -6161,6 +6407,206 @@ int AccessPath::compute_valid_inner_path()
return ret;
}
bool ObIndexMergeNode::is_valid() const
{
return is_leaf_node_ ?
(ap_ != NULL && left_node_ == NULL && right_node_ == NULL) :
(merge_type_ != INDEX_MERGE_INVALID &&
left_node_ != NULL &&
right_node_ != NULL &&
left_node_->is_valid() &&
right_node_->is_valid());
}
int ObIndexMergeNode::set_scan_direction(const ObOrderDirection &direction)
{
int ret = OB_SUCCESS;
if (is_leaf_node_) {
if (OB_ISNULL(ap_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(ap_), K(ret));
} else {
ap_->order_direction_ = direction;
}
} else if (OB_ISNULL(left_node_) || OB_ISNULL(right_node_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(left_node_), K(right_node_), K(ret));
} else if (OB_FAIL(SMART_CALL(left_node_->set_scan_direction(direction))) ||
OB_FAIL(SMART_CALL(right_node_->set_scan_direction(direction)))) {
LOG_WARN("failed to set scan direction", K(ret));
}
return ret;
}
int IndexMergePath::init(ObIndexMergeNode *node, common::ObIAllocator *allocator)
{
int ret = OB_SUCCESS;
const AccessPath *ap = nullptr;
if (OB_ISNULL(node) || OB_ISNULL(allocator) || OB_ISNULL(ap = node->ap_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KPC(node), K(allocator), K(ret));
} else if (OB_FAIL(assign(*ap, allocator))) {
LOG_WARN("failed to assign access path", K(ret));
} else {
root_ = node;
is_index_merge_ = true;
}
return ret;
}
int IndexMergePath::add_index_scan_node(const ObIndexMergeType merge_type,
AccessPath *ap,
common::ObIAllocator *allocator)
{
int ret = OB_SUCCESS;
ObIndexMergeNode *node = nullptr;
if (OB_ISNULL(ap) || OB_ISNULL(allocator) || merge_type == INDEX_MERGE_INVALID) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(merge_type), K(ap), K(allocator), K(ret));
} else if (OB_ISNULL(node = OB_NEWx(ObIndexMergeNode, allocator))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create a index merge node", K(ret));
} else if (OB_FAIL(add_var_to_array_no_dup(index_table_ids_, ap->index_id_))) {
LOG_WARN("failed to push back index table id", K(ret));
} else {
node->ap_ = ap;
node->is_leaf_node_ = true;
node->index_tid_ = ap->index_id_;
node->idx_ = index_cnt_;
node->is_ror_ = ap->is_ror_;
if (OB_ISNULL(root_)) {
// first add node, init index merge path with first index access path
if (OB_FAIL(init(node, allocator))) {
LOG_WARN("failed to init index merge path", K(ret));
}
} else {
ObIndexMergeNode *new_root = nullptr;
if (OB_ISNULL(new_root = OB_NEWx(ObIndexMergeNode, allocator))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create a index merge node", K(ret));
} else {
new_root->merge_type_ = merge_type;
new_root->left_node_ = root_;
new_root->right_node_ = node;
new_root->is_leaf_node_ = false;
root_ = new_root;
}
}
}
if (OB_SUCC(ret)) {
index_cnt_++;
}
return ret;
}
int IndexMergePath::set_scan_direction(const ObOrderDirection &direction)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(root_) || !root_->is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KPC(root_), K(ret));
} else if (OB_FAIL(root_->set_scan_direction(direction))) {
LOG_WARN("failed to set scan direction for index merge path", K(ret));
} else {
order_direction_ = direction;
}
return ret;
}
bool IndexMergePath::is_valid() const
{
return root_ == nullptr ? false : root_->is_valid();
}
bool IndexMergePath::is_disjunctive_clause(const ObRawExpr *root)
{
bool bret = false;
if (OB_NOT_NULL(root)) {
if (T_OP_OR == root->get_expr_type()) {
bret = true;
for (int64_t i = 0; bret && i < root->get_param_count(); i++) {
const ObRawExpr *expr = root->get_param_expr(i);
if (expr->has_flag(IS_SIMPLE_COND) || expr->has_flag(IS_RANGE_COND)) {
} else if (T_OP_AND == expr->get_expr_type()) {
bret = bret && is_simple_conjunctive_clause(expr);
} else {
bret = false;
}
}
}
}
return bret;
}
bool IndexMergePath::is_simple_conjunctive_clause(const ObRawExpr *root)
{
bool bret = false;
if (OB_NOT_NULL(root)) {
if (T_OP_AND == root->get_expr_type()) {
bret = true;
for (int64_t i = 0; bret && i < root->get_param_count(); i++) {
const ObRawExpr *expr = root->get_param_expr(i);
if (expr->has_flag(IS_SIMPLE_COND) || expr->has_flag(IS_RANGE_COND)) {
} else {
bret = false;
}
}
}
}
return bret;
}
int ObJoinOrder::check_index_merge_list(const uint64_t table_id,
const uint64_t ref_table_id,
const common::ObIArray<uint64_t> &index_list,
bool &contains_invalid_index,
bool &is_all_local_index,
bool &is_all_global_index,
common::ObIArray<common::ObArray<uint64_t>> &merge_index_column_ids)
{
int ret = OB_SUCCESS;
contains_invalid_index = false;
is_all_local_index = true;
is_all_global_index = true;
merge_index_column_ids.reuse();
const ObDMLStmt *stmt = nullptr;
ObSqlSchemaGuard *schema_guard = nullptr;
const ObTableSchema *index_schema = nullptr;
if (OB_ISNULL(get_plan()) ||
OB_ISNULL(stmt = get_plan()->get_stmt()) ||
OB_ISNULL(schema_guard = get_plan()->get_optimizer_context().get_sql_schema_guard())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr", K(get_plan()), K(stmt), K(schema_guard), K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && !contains_invalid_index && i < index_list.count(); i++) {
const ObTableID &index_id = index_list.at(i);
ObArray<uint64_t> index_column_ids;
bool is_global_index = false;
if (OB_UNLIKELY(index_id == 0)) {
// invalid index id, mark as invalid index merge path
contains_invalid_index = true;
} else if (OB_FAIL(schema_guard->get_table_schema(index_id, index_schema))) {
LOG_WARN("falied to get table schema", K(index_schema));
} else if (OB_ISNULL(index_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null index schema", K(index_id), K(ret));
} else if (FALSE_IT(is_global_index = index_schema->is_global_index_table())) {
} else if (OB_FAIL(index_schema->get_column_ids(index_column_ids))) {
LOG_WARN("failed to get column ids", K(index_id), K(ret));
} else if (OB_FAIL(merge_index_column_ids.push_back(index_column_ids))) {
LOG_WARN("failed to push back index column ids", K(ret));
} else if (OB_UNLIKELY(ref_table_id == index_id)) {
// skip index type check when use primary key
} else {
is_all_local_index &= !is_global_index;
is_all_global_index &= is_global_index;
}
}
return ret;
}
int FunctionTablePath::assign(const FunctionTablePath &other, common::ObIAllocator *allocator)
{
int ret = OB_SUCCESS;

108
src/sql/optimizer/ob_join_order.h
View File

@ -539,7 +539,9 @@ struct EstimateCostInfo {
for_update_(false),
use_skip_scan_(OptSkipScanState::SS_UNSET),
use_column_store_(false),
is_valid_inner_path_(false)
is_valid_inner_path_(false),
is_index_merge_(false),
is_ror_(false)
{
}
virtual ~AccessPath() {
@ -636,7 +638,9 @@ struct EstimateCostInfo {
K_(use_das),
K_(use_skip_scan),
K_(use_column_store),
K_(is_valid_inner_path));
K_(is_valid_inner_path),
K_(is_index_merge),
K_(is_ror));
public:
//member variables
uint64_t table_id_;
@ -661,10 +665,87 @@ struct EstimateCostInfo {
bool use_column_store_;
// mark this access path is inner path and contribute query range
bool is_valid_inner_path_;
bool is_index_merge_; // whether used for index merge
bool is_ror_; // indicate whether result from index table scan is ordered by primary key
private:
DISALLOW_COPY_AND_ASSIGN(AccessPath);
};
enum ObIndexMergeType : uint32_t
{
INDEX_MERGE_INVALID = 0,
INDEX_MERGE_UNION,
INDEX_MERGE_INTERSECT,
INDEX_HASH_UNON,
INDEX_HASH_INTERSECT
};
struct ObIndexMergeNode
{
public:
ObIndexMergeNode()
: merge_type_(INDEX_MERGE_INVALID),
left_node_(nullptr),
right_node_(nullptr),
ap_(nullptr),
is_leaf_node_(false),
index_tid_(OB_INVALID_ID),
idx_(OB_INVALID_ID),
is_ror_(false)
{}
bool is_valid() const;
int set_scan_direction(const ObOrderDirection &direction);
TO_STRING_KV(K_(merge_type),
K_(left_node),
K_(right_node),
K_(is_leaf_node),
K_(index_tid),
K_(is_ror));
public:
ObIndexMergeType merge_type_;
ObIndexMergeNode *left_node_;
ObIndexMergeNode *right_node_;
/*** for leaf node, which is, the index scan node ***/
AccessPath *ap_;
bool is_leaf_node_;
uint64_t index_tid_; // the table id of index table
int64_t idx_; // a unique identifier for each index node
bool is_ror_;
};
class IndexMergePath : public AccessPath
{
public:
IndexMergePath()
: AccessPath(OB_INVALID_ID, OB_INVALID_ID, OB_INVALID_ID, nullptr, NULLS_FIRST_ASC),
root_(nullptr),
index_cnt_(0),
filters_(),
force_used_by_hint_(false)
{}
int init(ObIndexMergeNode *node, common::ObIAllocator *allocator);
int add_index_scan_node(const ObIndexMergeType merge_type, AccessPath *ap, common::ObIAllocator *allocator);
bool is_valid() const;
int set_scan_direction(const ObOrderDirection &direction);
static bool is_disjunctive_clause(const ObRawExpr *root);
static bool is_simple_conjunctive_clause(const ObRawExpr *root);
public:
ObIndexMergeNode *root_;
int64_t index_cnt_;
common::ObSEArray<ObTableID, 2, common::ModulePageAllocator, true> index_table_ids_;
common::ObSEArray<ObString, 2, common::ModulePageAllocator, true> index_name_list_;
common::ObSEArray<ObRawExpr*, 4, common::ModulePageAllocator, true> filters_;
bool force_used_by_hint_;
private:
DISALLOW_COPY_AND_ASSIGN(IndexMergePath);
};
class JoinPath : public Path
{
public:
@ -1523,6 +1604,29 @@ struct NullAwareAntiJoinInfo {
bool use_column_store,
OptSkipScanState use_skip_scan);
int check_can_use_index_merge(const uint64_t table_id,
const uint64_t ref_table_id,
PathHelper &helper,
bool &use_index_merge,
ObIArray<uint64_t> &index_merge_list,
ObRawExpr *&index_merge_root);
int check_index_merge_list(const uint64_t table_id,
const uint64_t ref_table_id,
const ObIArray<uint64_t> &index_merge_list,
bool &contains_invalid_index,
bool &is_all_local_index,
bool &is_all_global_index,
common::ObIArray<common::ObArray<uint64_t>> &merge_index_column_ids);
int create_index_merge_path(const uint64_t table_id,
const uint64_t ref_table_id,
PathHelper &helper,
ObIArray<AccessPath *> &access_paths,
ObIndexInfoCache &index_info_cache,
const ObIArray<uint64_t> &index_merge_list,
ObRawExpr *index_merge_root);
int init_sample_info_for_access_path(AccessPath *ap,
const uint64_t table_id,
const TableItem *table_item);

2
src/sql/optimizer/ob_log_join.cpp
View File

@ -1417,6 +1417,8 @@ int ObLogJoin::check_if_disable_batch(ObLogicalOperator* root, bool &can_use_bat
can_use_batch_nlj = false;
} else if (ts->get_scan_direction() != default_asc_direction() && ts->get_scan_direction() != ObOrderDirection::UNORDERED) {
can_use_batch_nlj = false;
} else if (ts->use_index_merge()) {
can_use_batch_nlj = false;
} else {
SMART_VAR(ObTablePartitionInfo, tmp_info) {
ObTablePartitionInfo *tmp_info_ptr = &tmp_info;

18
src/sql/optimizer/ob_log_plan.cpp
View File

@ -2889,8 +2889,10 @@ int ObLogPlan::allocate_access_path(AccessPath *ap,
} else {
LOG_DEBUG("handle text ir expr in plan", K(ret), K(non_match_filters), K(match_filters));
}
} else if (OB_FAIL(scan->set_table_scan_filters(ap->filter_))) {
LOG_WARN("failed to set filters", K(ret));
} else if (scan->use_index_merge() && OB_FAIL(scan->set_index_merge_scan_filters(ap))) {
LOG_WARN("failed to set index merge filters", K(ret));
} else if (!scan->use_index_merge() && OB_FAIL(scan->set_table_scan_filters(ap->filter_))) {
LOG_WARN("failed to set table scan filters", K(ret));
} else if (OB_FAIL(append(scan->get_pushdown_filter_exprs(), ap->pushdown_filters_))) {
LOG_WARN("failed to append pushdown filters", K(ret));
} else if (ap->est_cost_info_.index_meta_info_.is_multivalue_index_ &&
@ -11276,6 +11278,18 @@ int ObLogPlan::collect_location_related_info(ObLogicalOperator &op)
}
}
if (OB_SUCC(ret) && tsc_op.use_index_merge()) {
ObArray<ObTableID> index_tids;
if (OB_FAIL(tsc_op.get_index_tids(index_tids))) {
LOG_WARN("failed to get index tids", K(ret));
} else if (OB_FAIL(append_array_no_dup(rel_info.related_ids_, index_tids))) {
LOG_WARN("failed to append index merge table ids", K(index_tids), K(ret));
} else if (OB_FAIL(add_var_to_array_no_dup(rel_info.related_ids_, tsc_op.get_real_ref_table_id()))) {
LOG_WARN("failed to append main table id", K(ret));
}
}
LOG_TRACE("collect location related info", K(rel_info));
if (OB_SUCC(ret) && OB_FAIL(optimizer_context_.get_loc_rel_infos().push_back(rel_info))) {
LOG_WARN("store location related info failed", K(ret));
}

2
src/sql/optimizer/ob_log_subplan_filter.cpp
View File

@ -456,6 +456,8 @@ int ObLogSubPlanFilter::check_if_match_das_group_rescan(ObLogicalOperator *root,
LOG_WARN("unexpected null", K(ret));
} else if (!tsc->use_das()) {
group_rescan = false;
} else if (tsc->use_index_merge()) {
group_rescan = false;
}
if (OB_SUCC(ret) && group_rescan) {
group_rescan = !(is_virtual_table(ap->ref_table_id_)

463
src/sql/optimizer/ob_log_table_scan.cpp
View File

@ -58,6 +58,8 @@ const char *ObLogTableScan::get_name() const
name = use_das() ? "DISTRIBUTED TABLE SKIP SCAN" : "TABLE SKIP SCAN";
} else if (EXTERNAL_TABLE == get_table_type()) {
name = "EXTERNAL TABLE SCAN";
} else if (use_index_merge()) {
name = use_das() ? "DISTRIBUTED INDEX MERGE SCAN" : "INDEX MERGE SCAN";
} else if (use_das()) {
if (is_get) {
name = "DISTRIBUTED TABLE GET";
@ -212,6 +214,8 @@ int ObLogTableScan::get_op_exprs(ObIArray<ObRawExpr*> &all_exprs)
LOG_WARN("failed to analyze filter monotonicity", K(ret));
} else if (OB_FAIL(get_filter_assist_exprs(all_exprs))) {
LOG_WARN("failed to get filter assist expr", K(ret));
} else if (use_index_merge() && OB_FAIL(append_array_no_dup(all_exprs, full_filters_))) {
LOG_WARN("failed to append index merge full filters", K(ret));
} else if (OB_FAIL(ObLogicalOperator::get_op_exprs(all_exprs))) {
LOG_WARN("failed to get exprs", K(ret));
} else { /*do nothing*/ }
@ -517,6 +521,15 @@ int ObLogTableScan::generate_access_exprs()
}
}
if (OB_SUCC(ret) && use_index_merge() && !full_filters_.empty()) {
ObArray<ObRawExpr*> column_exprs;
if (OB_FAIL(ObRawExprUtils::extract_column_exprs(full_filters_, column_exprs))) {
LOG_WARN("failed to extract column exprs", K(full_filters_), K(ret));
} else if (OB_FAIL(append_array_no_dup(access_exprs_, column_exprs))) {
LOG_WARN("failed to append array no dup", K(column_exprs), K(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_pseudo_column_like_exprs().count(); i++) {
ObRawExpr *expr = stmt->get_pseudo_column_like_exprs().at(i);
if (OB_ISNULL(expr)) {
@ -624,10 +637,43 @@ int ObLogTableScan::replace_index_back_pushdown_filters(ObRawExprReplacer &repla
LOG_WARN("failed to replace non pushdown expr", K(ret));
} else if (OB_FAIL(replace_exprs_action(replacer, lookup_pushdown_filters))) {
LOG_WARN("failed to replace lookup pushdown expr", K(ret));
} else if (OB_UNLIKELY(use_index_merge())) {
// when use index merge, we need to replace the filter exprs related to virtual generated columns
// for those main table participates as a branch of merge.
IndexMergePath *path = static_cast<IndexMergePath*>(access_path_);
if (OB_ISNULL(path)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null index merge path", K(ret));
} else if (OB_FAIL(replace_index_merge_pushdown_filters(path->root_, replacer))) {
LOG_WARN("failed to replace index merge pushdown filters", K(ret));
}
} else { /* do nothing */ }
return ret;
}
int ObLogTableScan::replace_index_merge_pushdown_filters(ObIndexMergeNode *node, ObRawExprReplacer &replacer)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(node) || !node->is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid index merge node", KPC(node), K(ret));
} else if (node->is_leaf_node_) {
if (OB_UNLIKELY(node->index_tid_ == ref_table_id_)) {
// main table participates as a branch of merge
ObArray<ObRawExpr*> scan_pushdown_filters;
if (OB_FAIL(get_index_filters(node->idx_, scan_pushdown_filters))) {
LOG_WARN("failed to get index scan pushdown filters", K(node->index_tid_), K(ret));
} else if (OB_FAIL(replace_exprs_action(replacer, scan_pushdown_filters))) {
LOG_WARN("failed to replace index scan pushdown filters", K(ret));
}
}
} else if (OB_FAIL(SMART_CALL(replace_index_merge_pushdown_filters(node->left_node_, replacer))) ||
OB_FAIL(SMART_CALL(replace_index_merge_pushdown_filters(node->right_node_, replacer)))) {
LOG_WARN("failed to set index merge pushdown filters", K(ref_table_id_), K(ret));
}
return ret;
}
int ObLogTableScan::has_nonpushdown_filter(bool &has_npd_filter)
{
int ret = OB_SUCCESS;
@ -711,6 +757,44 @@ int ObLogTableScan::extract_pushdown_filters(ObIArray<ObRawExpr*> &nonpushdown_f
return ret;
}
int ObLogTableScan::extract_nonpushdown_filters(const ObIArray<ObRawExpr*> &filters,
ObIArray<ObRawExpr*> &nonpushdown_filters,
ObIArray<ObRawExpr*> &pushdown_filters) const
{
int ret = OB_SUCCESS;
if (get_contains_fake_cte() ||
is_virtual_table(get_ref_table_id()) ||
EXTERNAL_TABLE == get_table_type()) {
// all filters can not push down to storage
if (OB_FAIL(nonpushdown_filters.assign(filters))) {
LOG_WARN("failed to assign full filter to non-pushdown filters", K(ret));
}
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < filters.count(); ++i) {
if (use_batch() && filters.at(i)->has_flag(CNT_DYNAMIC_PARAM)) {
//In Batch table scan the dynamic param filter do not push down to storage
if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) {
LOG_WARN("failed to push dynamic filter to non-pushdown filters", K(ret), K(i));
}
} else if (filters.at(i)->has_flag(CNT_PL_UDF) ||
filters.at(i)->has_flag(CNT_OBJ_ACCESS_EXPR)) {
//User Define Function/obj access expr filter do not push down to storage
if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) {
LOG_WARN("failed to push UDF/obj access filter to non-pushdown filters", K(ret), K(i));
}
} else if (filters.at(i)->has_flag(CNT_DYNAMIC_USER_VARIABLE) ||
filters.at(i)->has_flag(CNT_ASSIGN_EXPR)) {
if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) {
LOG_WARN("failed to push variable assign filter to non-pushdown filters", K(ret), K(i));
}
} else if (OB_FAIL(pushdown_filters.push_back(filters.at(i)))) {
LOG_WARN("failed to push back pushdown filter", K(ret));
}
}
}
return ret;
}
int ObLogTableScan::extract_virtual_gen_access_exprs(
ObIArray<ObRawExpr*> &access_exprs,
uint64_t scan_table_id)
@ -1060,7 +1144,10 @@ int ObLogTableScan::index_back_check()
{
int ret = OB_SUCCESS;
bool column_found = true;
if (!is_index_scan()) {
if (use_index_merge()) {
// force set index back when index merge
column_found = false;
} else if (!is_index_scan()) {
column_found = true;
} else {
for (int64_t i = 0; OB_SUCC(ret) && column_found && i < access_exprs_.count(); ++i) {
@ -1142,6 +1229,121 @@ int ObLogTableScan::set_table_scan_filters(const common::ObIArray<ObRawExpr *> &
return ret;
}
int ObLogTableScan::set_index_merge_scan_filters(const AccessPath *path)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(path) || !path->is_index_merge_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KPC(path));
} else {
const IndexMergePath *root_path = static_cast<const IndexMergePath*>(path);
if (OB_FAIL(full_filters_.assign(root_path->filters_))) {
LOG_WARN("failed to assign filters array", K(ret));
} else if (OB_FAIL(index_range_conds_.prepare_allocate(root_path->index_cnt_)) ||
OB_FAIL(index_filters_.prepare_allocate(root_path->index_cnt_))) {
LOG_WARN("failed to prepare allocate index range filters", K(ret));
} else if (OB_FAIL(set_index_table_scan_filters(root_path->root_))) {
LOG_WARN("failed to set index table scan filters", K(ret));
}
}
LOG_TRACE("index merge set range conds and filters", K(index_range_conds_), K(index_filters_));
return ret;
}
int ObLogTableScan::set_index_table_scan_filters(ObIndexMergeNode *node)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(node) || !node->is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KPC(node));
} else if (node->is_leaf_node_) {
bool is_get = false;
ObOptimizerContext *opt_ctx = nullptr;
ObSqlSchemaGuard *schema_guard = nullptr;
const share::schema::ObTableSchema *index_schema = nullptr;
AccessPath *ap = nullptr;
/*
* virtual table may have hash index,
* for hash index, if it is a get, we should still extract the range condition
*/
if (OB_ISNULL(get_plan())
|| OB_ISNULL(opt_ctx = &get_plan()->get_optimizer_context())
|| OB_ISNULL(schema_guard = opt_ctx->get_sql_schema_guard())
|| OB_ISNULL(ap = node->ap_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("unexpected nullptr", K(get_plan()), K(opt_ctx), K(schema_guard), K(ap), K(ret));
} else if (get_contains_fake_cte()) {
// do nothing
} else if (OB_FAIL(schema_guard->get_table_schema(table_id_, ap->index_id_, get_stmt(), index_schema))) {
LOG_WARN("failed to get table schema", K(ret));
} else if (OB_ISNULL(index_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret));
} else if (OB_FAIL(is_table_get(is_get))) {
LOG_WARN("failed to check is table get", K(ret));
} else if ((index_schema->is_ordered() || is_get) && NULL != ap->pre_query_range_) {
const ObIArray<ObRawExpr *> &range_exprs = ap->pre_query_range_->get_range_exprs();
ObArray<ObRawExpr *> scan_pushdown_filters;
ObArray<bool> filter_before_index_back;
ObArray<uint64_t> index_column_ids;
// extract the filters can be pushed down to index table scan
for (ObTableSchema::const_column_iterator iter = index_schema->column_begin();
OB_SUCC(ret) && iter != index_schema->column_end(); ++iter) {
if (OB_ISNULL(iter)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr column iter", K(iter), K(ret));
} else {
const ObColumnSchemaV2 *column_schema = *iter;
if (OB_ISNULL(column_schema)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr column schema", K(ret));
} else if (OB_FAIL(index_column_ids.push_back(column_schema->get_column_id()))) {
LOG_WARN("failed to push back column id", K(ret));
}
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObOptimizerUtil::check_filter_before_indexback(ap->filter_,
index_column_ids,
filter_before_index_back))) {
LOG_WARN("failed to check filter before index back", K(ap->filter_), K(ret));
} else {
OB_ASSERT(ap->filter_.count() == filter_before_index_back.count());
for (int64_t i = 0; OB_SUCC(ret) && i < ap->filter_.count(); i++) {
if (filter_before_index_back.at(i) && OB_FAIL(scan_pushdown_filters.push_back(ap->filter_.at(i)))) {
LOG_WARN("failed to push back filter", K(ret));
}
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < scan_pushdown_filters.count(); i++) {
bool found_expr = false;
for (int64_t j = 0; OB_SUCC(ret) && !found_expr && j < range_exprs.count(); j++) {
if (scan_pushdown_filters.at(i) == range_exprs.at(j)) {
//有重复表达式，忽略掉
found_expr = true;
} else { /* Do nothing */ }
}
// for virtual table, even if we extract query range, we need to maintain the condition into the filter
if (OB_SUCC(ret) && (!found_expr || (is_virtual_table(ref_table_id_)))) {
if (OB_FAIL(index_filters_.at(node->idx_).push_back(scan_pushdown_filters.at(i)))) {
LOG_WARN("add filter expr failed", KPC(node), K(ret));
} else { /* Do nothing */ }
}
if (OB_SUCC(ret) && found_expr) {
if (OB_FAIL(index_range_conds_.at(node->idx_).push_back(scan_pushdown_filters.at(i)))) {
LOG_WARN("failed to push back expr", K(ret));
} else { /*do nothing*/}
}
} //end for
}
} else if (OB_FAIL(SMART_CALL(set_index_table_scan_filters(node->left_node_))) ||
OB_FAIL(SMART_CALL(set_index_table_scan_filters(node->right_node_)))) {
LOG_WARN("failed to set index table filters", K(ret));
}
return ret;
}
int ObLogTableScan::pick_out_query_range_exprs()
{
int ret = OB_SUCCESS;
@ -1421,6 +1623,14 @@ int ObLogTableScan::get_plan_item_info(PlanText &plan_text,
LOG_WARN("BUF_PRINTF fails", K(ret));
} else { /* Do nothing */ }
if (OB_SUCC(ret) && use_index_merge()) {
if (OB_FAIL(BUF_PRINTF(", "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("use_index_merge=true"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else { /* Do nothing */ }
}
if (OB_SUCC(ret) && is_tsc_with_vid_) {
if (is_tsc_with_vid_ && OB_FAIL(BUF_PRINTF(", "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
@ -1524,6 +1734,34 @@ int ObLogTableScan::get_plan_object_info(PlanText &plan_text,
BEGIN_BUF_PRINT;
if (OB_FAIL(BUF_PRINTF("%.*s", name.length(), name.ptr()))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (use_index_merge()) {
if (OB_FAIL(BUF_PRINTF("%s", LEFT_BRACKET))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else {
ObArray<ObString> index_name_list;
if (OB_FAIL(get_index_name_list(index_name_list))) {
LOG_WARN("failed to get index name list", K(ret));
} else {
int64_t N = index_name_list.count();
for (int64_t i = 0; OB_SUCC(ret) && i < N - 1; i++) {
if (OB_FAIL(BUF_PRINTF("%.*s", index_name_list.at(i).length(), index_name_list.at(i).ptr()))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF(","))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(BUF_PRINTF("%.*s", index_name_list.at(N-1).length(), index_name_list.at(N-1).ptr()))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (is_descending_direction(get_scan_direction()) &&
OB_FAIL(BUF_PRINTF("%s", COMMA_REVERSE))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("%s", RIGHT_BRACKET))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
}
}
}
}
} else if (is_index_scan()) {
if (OB_FAIL(BUF_PRINTF("%s", LEFT_BRACKET))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
@ -1837,65 +2075,95 @@ int ObLogTableScan::print_range_annotation(char *buf,
ExplainType type)
{
int ret = OB_SUCCESS;
ObArray<ObRawExpr *> range_key;
for (int64_t i = 0; OB_SUCC(ret) && i < range_columns_.count(); ++i) {
if (OB_ISNULL(range_columns_.at(i).expr_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Expr in column item should not be NULL", K(i), K(ret));
} else if (OB_FAIL(range_key.push_back(range_columns_.at(i).expr_))) {
LOG_WARN("Fail to add range expr", K(i), K(ret));
} else { /* Do nothing */ }
}
if (OB_SUCC(ret)) {
EXPLAIN_PRINT_EXPRS(range_key, type);
if (use_index_merge()) {
ObArray<ObString> index_name_list;
if (OB_FAIL(get_index_name_list(index_name_list))) {
LOG_WARN("failed to get index name list", K(ret));
} else {
OB_ASSERT(index_name_list.count() == index_range_conds_.count());
for (int64_t i = 0; OB_SUCC(ret) && i < index_range_conds_.count(); ++i) {
const ObString &index_name = index_name_list.at(i);
const ObIArray<ObRawExpr*> &range_cond = index_range_conds_.at(i);
const ObIArray<ObRawExpr*> &filter = index_filters_.at(i);
if (OB_FAIL(BUF_PRINTF("index_name: %.*s, ", index_name.length(), index_name.ptr()))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
}
EXPLAIN_PRINT_EXPRS(range_cond, type);
if (OB_SUCC(ret)) {
if (OB_FAIL(BUF_PRINTF(", "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else { /* Do nothing */ }
}
EXPLAIN_PRINT_EXPRS(filter, type);
if (OB_SUCC(ret)) {
if (OB_FAIL(BUF_PRINTF("\n "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else { /* Do nothing */ }
}
}
}
} else {
ObArray<ObRawExpr *> range_key;
for (int64_t i = 0; OB_SUCC(ret) && i < range_columns_.count(); ++i) {
if (OB_ISNULL(range_columns_.at(i).expr_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Expr in column item should not be NULL", K(i), K(ret));
} else if (OB_FAIL(range_key.push_back(range_columns_.at(i).expr_))) {
LOG_WARN("Fail to add range expr", K(i), K(ret));
} else { /* Do nothing */ }
}
if (OB_SUCC(ret)) {
if (OB_FAIL(BUF_PRINTF(", "))) {
EXPLAIN_PRINT_EXPRS(range_key, type);
if (OB_SUCC(ret)) {
if (OB_FAIL(BUF_PRINTF(", "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("range"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else { /* Do nothing */ }
} else { /* Do nothing */ }
} else { /* Do nothing */ }
//When range is empty. Range is always true.
if (OB_SUCC(ret) && 0 >= ranges_.count()) {
if (OB_FAIL(BUF_PRINTF("("))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("range"))) {
} else if (OB_FAIL(BUF_PRINTF("MIN"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF(" ; "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("MAX"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF(")"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else { /* Do nothing */ }
} else { /* Do nothing */ }
} else { /* Do nothing */ }
//When range is empty. Range is always true.
if (OB_SUCC(ret) && 0 >= ranges_.count()) {
if (OB_FAIL(BUF_PRINTF("("))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("MIN"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF(" ; "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("MAX"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF(")"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else { /* Do nothing */ }
} else { /* Do nothing */ }
if (OB_SUCC(ret)) {
ret = print_ranges(buf, buf_len, pos, ranges_);
}
if (OB_SUCC(ret)) {
ret = print_ranges(buf, buf_len, pos, ranges_);
}
if (OB_SUCC(ret) && is_skip_scan()) {
int64_t skip_scan_offset = get_pre_query_range()->get_skip_scan_offset();
if (OB_FAIL(BUF_PRINTF("\n prefix_columns_cnt = %ld , skip_scan_range", skip_scan_offset))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (ss_ranges_.empty() && OB_FAIL(BUF_PRINTF("(MIN ; MAX)"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(print_ranges(buf, buf_len, pos, ss_ranges_))) {
LOG_WARN("failed to print index skip ranges", K(ret));
} else { /* Do nothing */ }
}
if (OB_SUCC(ret)) {
if (!range_conds_.empty()) {
//print range condition
const ObIArray<ObRawExpr*> &range_cond = range_conds_;
if (OB_FAIL(BUF_PRINTF(", "))) {
if (OB_SUCC(ret) && is_skip_scan()) {
int64_t skip_scan_offset = get_pre_query_range()->get_skip_scan_offset();
if (OB_FAIL(BUF_PRINTF("\n prefix_columns_cnt = %ld , skip_scan_range", skip_scan_offset))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("\n "))) {
} else if (ss_ranges_.empty() && OB_FAIL(BUF_PRINTF("(MIN ; MAX)"))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(print_ranges(buf, buf_len, pos, ss_ranges_))) {
LOG_WARN("failed to print index skip ranges", K(ret));
} else { /* Do nothing */ }
}
if (OB_SUCC(ret)) {
if (!range_conds_.empty()) {
//print range condition
const ObIArray<ObRawExpr*> &range_cond = range_conds_;
if (OB_FAIL(BUF_PRINTF(", "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
} else if (OB_FAIL(BUF_PRINTF("\n "))) {
LOG_WARN("BUF_PRINTF fails", K(ret));
}
EXPLAIN_PRINT_EXPRS(range_cond, type);
}
EXPLAIN_PRINT_EXPRS(range_cond, type);
}
}
@ -2019,7 +2287,8 @@ int ObLogTableScan::print_outline_data(PlanText &plan_text)
}
if (OB_FAIL(index_hint.print_hint(plan_text))) {
LOG_WARN("failed to print index hint", K(ret));
} else if (use_das()) {
}
if (OB_SUCC(ret) && use_das()) {
ObIndexHint use_das_hint(T_USE_DAS_HINT);
use_das_hint.set_qb_name(qb_name);
use_das_hint.get_table().set_table(*table_item);
@ -2035,8 +2304,18 @@ int ObLogTableScan::print_outline_data(PlanText &plan_text)
LOG_WARN("failed to print use column store hint", K(ret));
}
}
if (OB_SUCC(ret) && use_index_merge()) {
ObUnionMergeHint union_merge_hint;
union_merge_hint.set_qb_name(qb_name);
union_merge_hint.get_table().set_table(*table_item);
ObIArray<ObString> &index_name_list = union_merge_hint.get_index_name_list();
if (OB_FAIL(get_index_name_list(index_name_list))) {
LOG_WARN("failed to get index name list", K(ret));
} else if (OB_FAIL(union_merge_hint.print_hint(plan_text))) {
LOG_WARN("failed to print index merge hint", K(ret));
}
}
}
return ret;
}
@ -2074,6 +2353,10 @@ int ObLogTableScan::print_used_hint(PlanText &plan_text)
&& use_column_store() == table_hint->use_column_store_hint_->is_enable_hint()
&& OB_FAIL(table_hint->use_column_store_hint_->print_hint(plan_text))) {
LOG_WARN("failed to print use column_store hint", K(ret));
} else if (NULL != table_hint->union_merge_hint_
&& use_index_merge_by_hint() == table_hint->union_merge_hint_->is_enable_hint()
&& OB_FAIL(table_hint->union_merge_hint_->print_hint(plan_text))) {
LOG_WARN("failed to print use union merge hint", K(ret));
} else if (table_hint->index_list_.empty()) {
/*do nothing*/
} else if (OB_UNLIKELY(table_hint->index_list_.count() != table_hint->index_hints_.count())) {
@ -3170,6 +3453,82 @@ int ObLogTableScan::get_filter_assist_exprs(ObIArray<ObRawExpr *> &assist_exprs)
return ret;
}
bool ObLogTableScan::use_index_merge() const
{
bool bret = false;
if (OB_NOT_NULL(access_path_)) {
bret = access_path_->is_index_merge_;
}
return bret;
}
bool ObLogTableScan::use_index_merge_by_hint() const
{
bool bret = false;
if (use_index_merge()) {
bret = static_cast<IndexMergePath*>(access_path_)->force_used_by_hint_;
}
return bret;
}
int ObLogTableScan::get_index_range_conds(int64_t idx, ObIArray<ObRawExpr *> &index_range_conds) const
{
int ret = OB_SUCCESS;
index_range_conds.reuse();
if (OB_UNLIKELY(idx < 0 || idx >= index_range_conds_.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid idx of index range conds", K(idx), K(index_range_conds_.count()));
} else if (OB_FAIL(index_range_conds.assign(index_range_conds_.at(idx)))) {
LOG_WARN("failed to assign index range conds", K(ret));
}
return ret;
}
int ObLogTableScan::get_index_filters(int64_t idx, ObIArray<ObRawExpr *> &index_filters) const
{
int ret = OB_SUCCESS;
index_filters.reuse();
if (OB_UNLIKELY(idx < 0 || idx >= index_filters_.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid idx of index filters", K(idx), K(index_filters_.count()));
} else if (OB_FAIL(index_filters.assign(index_filters_.at(idx)))) {
LOG_WARN("failed to assign index filters", K(ret));
}
return ret;
}
int ObLogTableScan::get_index_tids(ObIArray<ObTableID> &index_tids) const
{
int ret = OB_SUCCESS;
index_tids.reuse();
if (OB_ISNULL(access_path_) || !access_path_->is_index_merge_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("unexpected null index merge path", KPC(access_path_));
} else {
IndexMergePath *index_merge_path = static_cast<IndexMergePath*>(access_path_);
if (OB_FAIL(index_tids.assign(index_merge_path->index_table_ids_))) {
LOG_WARN("failed to assign index merge table ids", K(ret));
}
}
return ret;
}
int ObLogTableScan::get_index_name_list(ObIArray<ObString> &index_name_list) const
{
int ret = OB_SUCCESS;
index_name_list.reuse();
if (OB_ISNULL(access_path_) || !access_path_->is_index_merge_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("unexpected null index merge path", KPC(access_path_));
} else {
IndexMergePath *index_merge_path = static_cast<IndexMergePath*>(access_path_);
if (OB_FAIL(index_name_list.assign(index_merge_path->index_name_list_))) {
LOG_WARN("failed to assign index merge table names", K(ret));
}
}
return ret;
}
int ObLogTableScan::check_das_need_scan_with_vid()
{
int ret = OB_SUCCESS;

43
src/sql/optimizer/ob_log_table_scan.h
View File

@ -11,7 +11,7 @@
*/
#ifndef OCEANBASE_SQL_OB_LOG_TABLE_SCAN_H
#define OCEANBASE_SQL_OB_LOG_TABLE_SCAN_H 1
#define OCEANBASE_SQL_OB_LOG_TABLE_SCAN_H
#include "sql/optimizer/ob_logical_operator.h"
#include "sql/optimizer/ob_log_operator_factory.h"
#include "sql/optimizer/ob_join_order.h"
@ -565,6 +565,9 @@ public:
bool is_need_feedback() const;
int set_table_scan_filters(const common::ObIArray<ObRawExpr *> &filters);
// for index merge, we need to set range conds and filters for each index scan
int set_index_merge_scan_filters(const AccessPath *path);
int set_index_table_scan_filters(ObIndexMergeNode *node);
inline common::ObIArray<ObRawExpr*> &get_range_conditions() { return range_conds_; }
const common::ObIArray<ObRawExpr*> &get_range_conditions() const { return range_conds_; }
inline void set_diverse_path_count(int64_t count) { diverse_path_count_ = count; }
@ -622,8 +625,12 @@ public:
ObIArray<ObRawExpr*> &scan_pushdown_filters,
ObIArray<ObRawExpr*> &lookup_pushdown_filters,
bool ignore_pd_filter = false) const;
int extract_nonpushdown_filters(const ObIArray<ObRawExpr*> &filters,
ObIArray<ObRawExpr*> &nonpushdown_filters,
ObIArray<ObRawExpr*> &pushdown_filters) const;
int has_nonpushdown_filter(bool &has_npd_filter);
int replace_index_back_pushdown_filters(ObRawExprReplacer &replacer);
int replace_index_merge_pushdown_filters(ObIndexMergeNode *node, ObRawExprReplacer &replacer);
int extract_virtual_gen_access_exprs(ObIArray<ObRawExpr*> &access_exprs,
uint64_t scan_table_id);
int adjust_print_access_info(ObIArray<ObRawExpr*> &access_exprs);
@ -664,6 +671,17 @@ public:
const ObColumnRefRawExpr *col_expr,
PushdownFilterMonotonicity &mono,
ObIArray<ObRawExpr *> &assist_exprs) const;
bool use_index_merge() const;
const ObIArray<ObRawExpr*> &get_full_filters() const { return full_filters_; }
const ObIArray<ObRawExpr*> &get_index_range_conds(int64_t idx) const { return index_range_conds_.at(idx); }
const ObIArray<ObRawExpr*> &get_index_filters(int64_t idx) const { return index_filters_.at(idx); }
int get_index_range_conds(int64_t idx, ObIArray<ObRawExpr *> &index_range_conds) const;
int get_index_filters(int64_t idx, ObIArray<ObRawExpr *> &index_filters) const;
int get_index_tids(ObIArray<ObTableID> &index_tids) const;
int get_index_name_list(ObIArray<ObString> &index_name_list) const;
bool use_index_merge_by_hint() const;
private: // member functions
//called when index_back_ set
int pick_out_query_range_exprs();
@ -717,6 +735,29 @@ protected: // memeber variables
const common::ObIArray<int64_t> *part_ids_;
common::ObSEArray<ObRawExpr *, 8, common::ModulePageAllocator, true> range_conds_;
// for index merge, we need to prepare range conds and filters for each index scan, and we need
// to store full query filters for final check.
// for example, consider following query:
// create table t1(c1 int primary key, c2 int, c3 int, c4 int);
// create index c2 on t1(c2) local;
// create index c3 on t1(c3) local;
// create index c4 on t1(c4) local;
// select /*+union_merge(t1 c2 c3 c4)*/ * from t1 where c1=1 or c2=1 or c4<1;
// when we choose index merge plan, range conds and filters need to be prepared, thus:
// ---------------------------------------------------------
// | index table | range conds | filters |
// ---------------------------------------------------------
// | c2 | NULL | c1 = 1 |
// | c3 | NULL | NULL |
// | c4 | c4 < 1 | NULL |
// ---------------------------------------------------------
// NOTE: only filters before index back can be pushed down to index scan.
// and full filters 'c1=1 or c2=1 or c4<1' will be used after lookup for final check.
typedef common::ObSEArray<ObRawExpr *, 2, common::ModulePageAllocator, true> ExprSEArray;
common::ObSEArray<ExprSEArray, 2, common::ModulePageAllocator, true> index_range_conds_;
common::ObSEArray<ExprSEArray, 2, common::ModulePageAllocator, true> index_filters_;
ExprSEArray full_filters_;
// index primary key columns.
// indicates use which columns to extract query range
common::ObSEArray<ColumnItem, 4, common::ModulePageAllocator, true> range_columns_;

1
src/sql/parser/sql_parser_mysql_mode.l
View File

@ -993,6 +993,7 @@ Timestamp{whitespace}?\"[^\"]*\" {
}
<hint>INDEX { return INDEX_HINT; }
<hint>NO_INDEX { return NO_INDEX_HINT; }
<hint>UNION_MERGE { return UNION_MERGE_HINT; }
<hint>USE_DAS { return USE_DAS_HINT; }
<hint>NO_USE_DAS { return NO_USE_DAS_HINT; }
<hint>INDEX_SS { return INDEX_SS_HINT; }

22
src/sql/parser/sql_parser_mysql_mode.y
View File

@ -181,7 +181,7 @@ COALESCE_AGGR NO_COALESCE_AGGR WITH_PULLUP WO_PULLUP
MV_REWRITE NO_MV_REWRITE
DECORRELATE NO_DECORRELATE
// optimize hint
INDEX_HINT FULL_HINT NO_INDEX_HINT USE_DAS_HINT NO_USE_DAS_HINT
INDEX_HINT FULL_HINT NO_INDEX_HINT USE_DAS_HINT NO_USE_DAS_HINT UNION_MERGE_HINT
INDEX_SS_HINT INDEX_SS_ASC_HINT INDEX_SS_DESC_HINT
USE_COLUMN_STORE_HINT NO_USE_COLUMN_STORE_HINT
LEADING_HINT ORDERED
@ -427,7 +427,7 @@ END_P SET_VAR DELIMITER
%type <node> relation_factor_in_hint relation_factor_in_hint_list relation_factor_in_pq_hint opt_relation_factor_in_hint_list relation_factor_in_use_join_hint_list relation_factor_in_mv_hint_list opt_relation_factor_in_mv_hint_list
%type <node> relation_factor_in_leading_hint_list joined_table tbl_name table_subquery table_subquery_alias
%type <node> relation_factor_with_star relation_with_star_list opt_with_star
%type <node> index_hint_type key_or_index index_hint_scope index_element index_list opt_index_list opt_index_prefix
%type <node> index_hint_type key_or_index index_hint_scope index_element index_list opt_index_list opt_index_prefix union_merge_list
%type <node> add_key_or_index_opt add_key_or_index add_unique_key_opt add_unique_key add_constraint_uniq_key_opt add_constraint_uniq_key add_constraint_pri_key_opt add_constraint_pri_key add_primary_key_opt add_primary_key add_spatial_index_opt add_spatial_index
%type <node> index_hint_definition index_hint_list
%type <node> intnum_list
@ -11506,6 +11506,18 @@ qb_name_list:
}
;
union_merge_list:
NAME_OB
{
$$ = $1;
}
| union_merge_list opt_comma NAME_OB
{
(void) $2;
malloc_non_terminal_node($$, result->malloc_pool_, T_LINK_NODE, 2, $1, $3);
}
;
optimize_hint:
INDEX_HINT '(' qb_name_option relation_factor_in_hint NAME_OB opt_index_prefix ')'
{
@ -11515,6 +11527,12 @@ INDEX_HINT '(' qb_name_option relation_factor_in_hint NAME_OB opt_index_prefix '
{
malloc_non_terminal_node($$, result->malloc_pool_, T_NO_INDEX_HINT, 3, $3, $4, $5);
}
| UNION_MERGE_HINT '(' qb_name_option relation_factor_in_hint union_merge_list ')'
{
ParseNode *index_list = NULL;
merge_nodes(index_list, result, T_UNION_MERGE_LIST, $5);
malloc_non_terminal_node($$, result->malloc_pool_, T_UNION_MERGE_HINT, 3, $3, $4, index_list);
}
| FULL_HINT '(' qb_name_option relation_factor_in_hint ')'
{
malloc_non_terminal_node($$, result->malloc_pool_, T_FULL_HINT, 2, $3, $4);

50
src/sql/resolver/dml/ob_dml_resolver.cpp
View File

@ -14621,6 +14621,12 @@ int ObDMLResolver::resolve_optimize_hint(const ParseNode &hint_node,
}
break;
}
case T_UNION_MERGE_HINT: {
if (OB_FAIL(resolve_union_merge_hint(hint_node, opt_hint))) {
LOG_WARN("failed to resolve union merge hint", K(ret));
}
break;
}
case T_ORDERED:
case T_LEADING: {
if (OB_FAIL(resolve_join_order_hint(hint_node, opt_hint))) {
@ -14871,6 +14877,50 @@ int ObDMLResolver::resolve_index_hint(const TableItem &table, const ParseNode &i
return ret;
}
int ObDMLResolver::resolve_union_merge_hint(const ParseNode &hint_node,
ObOptHint *&opt_hint)
{
int ret = OB_SUCCESS;
opt_hint = NULL;
ObUnionMergeHint *union_merge_hint = NULL;
ParseNode *table_node = NULL;
ParseNode *index_list_node = NULL;
ObString qb_name;
if (OB_UNLIKELY(3 != hint_node.num_child_)
|| OB_ISNULL(table_node = hint_node.children_[1])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected index merge hint", K(ret), K(hint_node.type_), K(hint_node.num_child_),
K(table_node));
} else if (OB_FAIL(ObQueryHint::create_hint(allocator_, hint_node.type_, union_merge_hint))) {
LOG_WARN("failed to create hint", K(ret));
} else if (OB_FAIL(resolve_qb_name_node(hint_node.children_[0], qb_name))) {
LOG_WARN("Failed to resolve qb name node", K(ret));
} else if (OB_FAIL(resolve_table_relation_in_hint(*table_node, union_merge_hint->get_table()))) {
LOG_WARN("Resolve table relation fail", K(ret));
} else if (OB_ISNULL(index_list_node = hint_node.children_[2])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected union merge hint", K(ret));
} else {
const ParseNode *index_node = NULL;
ObString index_name;
for (int32_t i = 0; OB_SUCC(ret) && i < index_list_node->num_child_; i++) {
if (OB_ISNULL(index_node = index_list_node->children_[i])) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected nullptr index node", K(ret));
} else {
index_name.assign_ptr(index_node->str_value_, static_cast<int32_t>(index_node->str_len_));
if (OB_FAIL(union_merge_hint->get_index_name_list().push_back(index_name))) {
LOG_WARN("failed to push back index name", K(index_name), K(union_merge_hint->get_index_name_list()));
}
}
}
union_merge_hint->set_qb_name(qb_name);
opt_hint = union_merge_hint;
}
LOG_TRACE("resolve union merge hint finished", KPC(union_merge_hint));
return ret;
}
int ObDMLResolver::resolve_join_order_hint(const ParseNode &hint_node,
ObOptHint *&opt_hint)
{

2
src/sql/resolver/dml/ob_dml_resolver.h
View File

@ -942,6 +942,8 @@ private:
ObOptHint *&opt_hint);
int resolve_index_hint(const TableItem &table, // resolved mysql mode index hint after table
const ParseNode &index_hint_node);
int resolve_union_merge_hint(const ParseNode &hint_node,
ObOptHint *&opt_hint);
int resolve_table_parallel_hint(const ParseNode &hint_node, ObOptHint *&opt_hint);
int resolve_join_order_hint(const ParseNode &hint_node, ObOptHint *&opt_hint);
int resolve_join_hint(const ParseNode &join_node, ObIArray<ObHint*> &join_hints);

33
src/sql/resolver/dml/ob_hint.cpp
View File

@ -1261,6 +1261,7 @@ const char* ObHint::get_hint_name(ObItemType type, bool is_enable_hint /* defaul
case T_FULL_HINT: return "FULL";
case T_NO_INDEX_HINT: return "NO_INDEX";
case T_USE_DAS_HINT: return is_enable_hint ? "USE_DAS" : "NO_USE_DAS";
case T_UNION_MERGE_HINT: return "UNION_MERGE";
case T_USE_COLUMN_STORE_HINT: return is_enable_hint ? "USE_COLUMN_TABLE" : "NO_USE_COLUMN_TABLE";
case T_INDEX_SS_HINT: return "INDEX_SS";
case T_INDEX_SS_ASC_HINT: return "INDEX_SS_ASC";
@ -1364,6 +1365,7 @@ int ObHint::deep_copy_hint_contain_table(ObIAllocator *allocator, ObHint *&hint)
case HINT_JOIN_FILTER: DEEP_COPY_NORMAL_HINT(ObJoinFilterHint); break;
case HINT_WIN_MAGIC: DEEP_COPY_NORMAL_HINT(ObWinMagicHint); break;
case HINT_COALESCE_AGGR: DEEP_COPY_NORMAL_HINT(ObCoalesceAggrHint); break;
case HINT_UNION_MERGE: DEEP_COPY_NORMAL_HINT(ObUnionMergeHint); break;
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected hint type to deep copy", K(ret), K(hint_class_));
@ -2276,6 +2278,37 @@ int ObIndexHint::print_hint_desc(PlanText &plan_text) const
return ret;
}
int ObUnionMergeHint::assign(const ObUnionMergeHint &other)
{
int ret = OB_SUCCESS;
if (OB_FAIL(table_.assign(other.table_))) {
LOG_WARN("failed to assign table", K(ret));
} else if (OB_FAIL(index_name_list_.assign(other.index_name_list_))) {
LOG_WARN("failed to assign index name list", K(ret));
} else if (OB_FAIL(ObOptHint::assign(other))) {
LOG_WARN("fail to assign hint", K(ret));
}
return ret;
}
int ObUnionMergeHint::print_hint_desc(PlanText &plan_text) const
{
int ret = OB_SUCCESS;
char *buf = plan_text.buf_;
int64_t &buf_len = plan_text.buf_len_;
int64_t &pos = plan_text.pos_;
if (OB_FAIL(table_.print_table_in_hint(plan_text))) {
LOG_WARN("fail to print table in hint", K(ret));
} else {
for (int64_t i = 0; i < index_name_list_.count(); i++) {
if (OB_FAIL(BUF_PRINTF(" \"%.*s\"", index_name_list_.at(i).length(), index_name_list_.at(i).ptr()))) {
LOG_WARN("fail to print index name", K(ret));
}
}
}
return ret;
}
int ObJoinHint::assign(const ObJoinHint &other)
{
int ret = OB_SUCCESS;

29
src/sql/resolver/dml/ob_hint.h
View File

@ -269,7 +269,8 @@ struct ObGlobalHint {
#define COMPAT_VERSION_4_3_1 (oceanbase::common::cal_version(4, 3, 1, 0))
#define COMPAT_VERSION_4_3_2 (oceanbase::common::cal_version(4, 3, 2, 0))
#define COMPAT_VERSION_4_3_3 (oceanbase::common::cal_version(4, 3, 3, 0))
#define LASTED_COMPAT_VERSION COMPAT_VERSION_4_3_3
#define COMPAT_VERSION_4_3_4 (oceanbase::common::cal_version(4, 3, 4, 0))
#define LASTED_COMPAT_VERSION COMPAT_VERSION_4_3_4
static bool is_valid_opt_features_version(uint64_t version)
{ return COMPAT_VERSION_4_0 <= version && LASTED_COMPAT_VERSION >= version; }
@ -572,7 +573,8 @@ public:
HINT_PQ_SET,
HINT_JOIN_FILTER,
HINT_TABLE_DYNAMIC_SAMPLING,
HINT_PQ
HINT_PQ,
HINT_UNION_MERGE
};
static const int64_t MAX_EXPR_STR_LENGTH_IN_HINT = 1024;
@ -649,6 +651,7 @@ public:
bool is_coalesce_aggr_hint() const {return HINT_COALESCE_AGGR == hint_class_; }
bool is_trans_added() const { return is_trans_added_; }
bool set_trans_added(bool is_trans_added) { return is_trans_added_ = is_trans_added; }
bool is_union_merge_hint() const { return T_UNION_MERGE_HINT == hint_type_; }
VIRTUAL_TO_STRING_KV("hint_type", get_type_name(hint_type_),
K_(hint_class), K_(qb_name),
@ -1031,6 +1034,28 @@ private:
int64_t index_prefix_;
};
class ObUnionMergeHint : public ObOptHint
{
public:
ObUnionMergeHint(ObItemType hint_type = T_UNION_MERGE_HINT)
: ObOptHint(hint_type)
{
set_hint_class(HINT_UNION_MERGE);
}
int assign(const ObUnionMergeHint &other);
virtual ~ObUnionMergeHint() {}
virtual int get_all_table_in_hint(ObIArray<ObTableInHint*> &all_tables) override { return all_tables.push_back(&table_); }
virtual int print_hint_desc(PlanText &plan_text) const override;
ObTableInHint &get_table() { return table_; }
const ObTableInHint &get_table() const { return table_; }
common::ObIArray<common::ObString> &get_index_name_list() { return index_name_list_; }
const common::ObIArray<common::ObString> &get_index_name_list() const { return index_name_list_; }
INHERIT_TO_STRING_KV("ObHint", ObHint, K_(table), K_(index_name_list));
private:
ObTableInHint table_;
common::ObSEArray<common::ObString, 2, common::ModulePageAllocator, true> index_name_list_;
};
class ObTableParallelHint : public ObOptHint
{
public:

52
src/sql/resolver/dml/ob_sql_hint.cpp
View File

@ -1345,7 +1345,8 @@ int ObStmtHint::merge_hint(ObHint &hint,
|| hint.is_join_filter_hint()
|| hint.is_table_parallel_hint()
|| hint.is_table_dynamic_sampling_hint()
|| hint.is_pq_subquery_hint()) {
|| hint.is_pq_subquery_hint()
|| hint.is_union_merge_hint()) {
if (OB_FAIL(add_var_to_array_no_dup(other_opt_hints_, &hint))) {
LOG_WARN("failed to add var to array", K(ret));
}
@ -1501,6 +1502,10 @@ int ObLogPlanHint::init_other_opt_hints(ObSqlSchemaGuard &schema_guard,
if (OB_FAIL(normal_hints_.push_back(hint))) {
LOG_WARN("failed to push back", K(ret));
}
} else if (hint->is_union_merge_hint()) {
if (OB_FAIL(add_union_merge_hint(stmt, query_hint, *static_cast<const ObUnionMergeHint*>(hint)))) {
LOG_WARN("failed to add union merge hint", K(ret));
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected hint type in other_opt_hints_", K(ret), K(*hint));
@ -1566,6 +1571,25 @@ int ObLogPlanHint::add_index_hint(const ObDMLStmt &stmt,
return ret;
}
int ObLogPlanHint::add_union_merge_hint(const ObDMLStmt &stmt,
const ObQueryHint &query_hint,
const ObUnionMergeHint &union_merge_hint)
{
int ret = OB_SUCCESS;
LogTableHint *log_table_hint = NULL;
if (OB_FAIL(get_log_table_hint_for_update(stmt, query_hint, union_merge_hint.get_table(),
true, log_table_hint))) {
LOG_WARN("failed to get log table hint by hint", K(ret));
} else if (NULL == log_table_hint) {
/* do nothing */
} else if (T_UNION_MERGE_HINT == union_merge_hint.get_hint_type()) {
if (NULL == log_table_hint->union_merge_hint_) {
log_table_hint->union_merge_hint_ = &union_merge_hint;
}
}
return ret;
}
int ObLogPlanHint::add_table_parallel_hint(const ObDMLStmt &stmt,
const ObQueryHint &query_hint,
const ObTableParallelHint &table_parallel_hint)
@ -1859,6 +1883,13 @@ int ObLogPlanHint::check_use_das(uint64_t table_id, bool &force_das, bool &force
return ret;
}
const ObUnionMergeHint *ObLogPlanHint::get_union_merge_hint(uint64_t table_id) const
{
int ret = OB_SUCCESS;
const LogTableHint *log_table_hint = get_log_table_hint(table_id);
return NULL == log_table_hint ? NULL : log_table_hint->union_merge_hint_;
}
int ObLogPlanHint::check_use_column_store(uint64_t table_id, bool &force_column_store, bool &force_no_column_store) const
{
int ret = OB_SUCCESS;
@ -2387,6 +2418,7 @@ int LogTableHint::assign(const LogTableHint &other)
parallel_hint_ = other.parallel_hint_;
use_das_hint_ = other.use_das_hint_;
use_column_store_hint_ = other.use_column_store_hint_;
union_merge_hint_ = other.union_merge_hint_;
if (OB_FAIL(index_list_.assign(other.index_list_))) {
LOG_WARN("failed to assign index list", K(ret));
} else if (OB_FAIL(index_hints_.assign(other.index_hints_))) {
@ -2407,7 +2439,7 @@ int LogTableHint::init_index_hints(ObSqlSchemaGuard &schema_guard)
if (OB_ISNULL(table_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected log index hint", K(ret), K(this));
} else if (index_hints_.empty()) {
} else if (index_hints_.empty() && union_merge_hint_ == nullptr) {
/* do nothing */
} else if (OB_FAIL(schema_guard.get_can_read_index_array(table_->ref_id_,
tids,
@ -2420,6 +2452,9 @@ int LogTableHint::init_index_hints(ObSqlSchemaGuard &schema_guard)
} else if (table_index_count > OB_MAX_INDEX_PER_TABLE) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Table index count is bigger than OB_MAX_INDEX_PER_TABLE", K(ret), K(table_index_count));
} else if (union_merge_hint_ != nullptr &&
OB_FAIL(merge_index_list_.prepare_allocate(union_merge_hint_->get_index_name_list().count()))) {
LOG_WARN("failed to prepare allocate merge index list", KPC(union_merge_hint_), K(ret));
} else {
LOG_TRACE("get readable index", K(table_index_count));
const share::schema::ObTableSchema *index_schema = NULL;
@ -2488,6 +2523,19 @@ int LogTableHint::init_index_hints(ObSqlSchemaGuard &schema_guard)
LOG_WARN("fail to push back", K(ret), K(hint_pos));
}
}
if (OB_SUCC(ret) && union_merge_hint_ != nullptr) {
for (int64_t i = 0; OB_SUCC(ret) && i < union_merge_hint_->get_index_name_list().count(); ++i) {
if (0 != union_merge_hint_->get_index_name_list().at(i).case_compare(index_name)) {
/* do nothing */
} else if (OB_UNLIKELY(i < 0 || i >= merge_index_list_.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(i), K(merge_index_list_.count()), K(ret));
} else {
merge_index_list_.at(i) = index_id;
}
}
}
}
}
if (OB_SUCC(ret)) {

13
src/sql/resolver/dml/ob_sql_hint.h
View File

@ -290,12 +290,14 @@ struct LogTableHint
parallel_hint_(NULL),
use_das_hint_(NULL),
use_column_store_hint_(NULL),
union_merge_hint_(NULL),
dynamic_sampling_hint_(NULL),
is_ds_hint_conflict_(false) {}
LogTableHint(const TableItem *table) : table_(table),
parallel_hint_(NULL),
use_das_hint_(NULL),
use_column_store_hint_(NULL),
union_merge_hint_(NULL),
dynamic_sampling_hint_(NULL),
is_ds_hint_conflict_(false) {}
int assign(const LogTableHint &other);
@ -305,7 +307,8 @@ struct LogTableHint
bool is_valid() const { return !index_list_.empty() || NULL != parallel_hint_
|| NULL != use_das_hint_ || !join_filter_hints_.empty()
|| dynamic_sampling_hint_ != NULL
|| NULL != use_column_store_hint_; }
|| NULL != use_column_store_hint_
|| NULL != union_merge_hint_; }
int get_join_filter_hint(const ObRelIds &left_tables,
bool part_join_filter,
const ObJoinFilterHint *&hint) const;
@ -319,7 +322,7 @@ struct LogTableHint
int get_index_prefix(const uint64_t index_id, int64_t &index_prefix) const;
TO_STRING_KV(K_(table), K_(index_list), K_(index_hints),
K_(parallel_hint), K_(use_das_hint),
K_(parallel_hint), K_(use_das_hint), K_(union_merge_hint),
K_(join_filter_hints), K_(left_tables),
KPC(dynamic_sampling_hint_), K(is_ds_hint_conflict_));
@ -329,6 +332,8 @@ struct LogTableHint
const ObTableParallelHint *parallel_hint_;
const ObIndexHint *use_das_hint_;
const ObIndexHint *use_column_store_hint_;
const ObUnionMergeHint *union_merge_hint_;
common::ObSEArray<uint64_t, 2, common::ModulePageAllocator, true> merge_index_list_;
ObSEArray<const ObJoinFilterHint*, 1, common::ModulePageAllocator, true> join_filter_hints_;
ObSEArray<ObRelIds, 1, common::ModulePageAllocator, true> left_tables_; // left table relids in join filter hint
const ObTableDynamicSamplingHint *dynamic_sampling_hint_;
@ -426,6 +431,9 @@ struct ObLogPlanHint
int add_index_hint(const ObDMLStmt &stmt,
const ObQueryHint &query_hint,
const ObIndexHint &index_hint);
int add_union_merge_hint(const ObDMLStmt &stmt,
const ObQueryHint &query_hint,
const ObUnionMergeHint &union_merge_hint);
int add_join_hint(const ObDMLStmt &stmt,
const ObQueryHint &query_hint,
const ObJoinHint &join_hint);
@ -451,6 +459,7 @@ struct ObLogPlanHint
bool config_disable,
JoinFilterPushdownHintInfo& info) const;
int check_use_das(uint64_t table_id, bool &force_das, bool &force_no_das) const;
const ObUnionMergeHint *get_union_merge_hint(uint64_t table_id) const;
int check_use_column_store(uint64_t table_id, bool &force_column_store, bool &force_no_column_store) const;
int check_use_skip_scan(uint64_t table_id, uint64_t index_id,
bool &force_skip_scan,