diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 4d4a17bd8f..b11374ad95 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -1487,3 +1487,6 @@ DEF_TIME(_wait_interval_after_truncate, OB_CLUSTER_PARAMETER, "30s", "[0s,)", DEF_CAP(_rebuild_replica_log_lag_threshold, OB_TENANT_PARAMETER, "0M", "[0M,)", "size of clog files that a replica lag behind leader to trigger rebuild, 0 means never trigger rebuild on purpose. Range: [0, +∞)", ObParameterAttr(Section::TRANS, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_BOOL(_enable_in_range_optimization, OB_TENANT_PARAMETER, "False", + "Enable extract query range optimization for in predicate", + ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 27c61172a3..3a63d9a949 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -3165,10 +3165,12 @@ int ObJoinOrder::extract_preliminary_query_range(const ObIArray &ran int ret = OB_SUCCESS; ObOptimizerContext *opt_ctx = NULL; const ParamStore *params = NULL; + ObSQLSessionInfo *session_info = NULL; if (OB_ISNULL(get_plan()) || OB_ISNULL(opt_ctx = &get_plan()->get_optimizer_context()) || OB_ISNULL(allocator_) || - OB_ISNULL(params = opt_ctx->get_params())) { + OB_ISNULL(params = opt_ctx->get_params()) || + OB_ISNULL(session_info = opt_ctx->get_session_info())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get unexpected null", K(get_plan()), K(opt_ctx), K(allocator_), K(params), K(ret)); @@ -3181,11 +3183,13 @@ int ObJoinOrder::extract_preliminary_query_range(const ObIArray &ran } else { tmp_qr = new(tmp_ptr)ObQueryRange(*allocator_); const ObDataTypeCastParams dtc_params = - ObBasicSessionInfo::create_dtc_params(opt_ctx->get_session_info()); + ObBasicSessionInfo::create_dtc_params(session_info); + bool is_in_range_optimization_enabled = session_info->is_in_range_optimization_enabled(); if (OB_FAIL(tmp_qr->preliminary_extract_query_range(range_columns, predicates, dtc_params, opt_ctx->get_exec_ctx(), &expr_constraints, - params))) { + params, false, true, + is_in_range_optimization_enabled))) { LOG_WARN("failed to preliminary extract query range", K(ret)); } } diff --git a/src/sql/optimizer/ob_opt_selectivity.cpp b/src/sql/optimizer/ob_opt_selectivity.cpp index 5b8982e22c..b0335d3013 100644 --- a/src/sql/optimizer/ob_opt_selectivity.cpp +++ b/src/sql/optimizer/ob_opt_selectivity.cpp @@ -3833,7 +3833,10 @@ int ObOptSelectivity::get_column_query_range(const OptSelectivityCtx &ctx, dtc_params, ctx.get_opt_ctx().get_exec_ctx(), NULL, - params))) { + params, + false, + true, + ctx.get_session_info()->is_in_range_optimization_enabled()))) { LOG_WARN("failed to preliminary extract query range", K(ret)); } else if (!query_range.need_deep_copy()) { if (OB_FAIL(query_range.direct_get_tablet_ranges(allocator, *exec_ctx, ranges, diff --git a/src/sql/optimizer/ob_select_log_plan.cpp b/src/sql/optimizer/ob_select_log_plan.cpp index 2e22b87e8b..4153f3eb1d 100644 --- a/src/sql/optimizer/ob_select_log_plan.cpp +++ b/src/sql/optimizer/ob_select_log_plan.cpp @@ -6335,7 +6335,8 @@ int ObSelectLogPlan::adjust_late_materialization_plan_structure(ObLogicalOperato ObSqlSchemaGuard *schema_guard = NULL; ObSEArray rowkey_ids; if (OB_ISNULL(stmt = get_stmt()) || OB_ISNULL(join) || - OB_ISNULL(index_scan) || OB_ISNULL(table_scan)) { + OB_ISNULL(index_scan) || OB_ISNULL(table_scan) || + OB_ISNULL(optimizer_context_.get_session_info())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(stmt), K(join), K(index_scan), K(table_scan), K(ret)); } else if (OB_UNLIKELY(log_op_def::LOG_JOIN != join->get_type())) { @@ -6401,6 +6402,7 @@ int ObSelectLogPlan::adjust_late_materialization_plan_structure(ObLogicalOperato if (OB_SUCC(ret)) { const ObDataTypeCastParams dtc_params = ObBasicSessionInfo::create_dtc_params(optimizer_context_.get_session_info()); + bool is_in_range_optimization_enabled = optimizer_context_.get_session_info()->is_in_range_optimization_enabled(); ObQueryRange *query_range = static_cast(get_allocator().alloc(sizeof(ObQueryRange))); const ParamStore *params = get_optimizer_context().get_params(); if (OB_ISNULL(query_range)) { @@ -6416,7 +6418,10 @@ int ObSelectLogPlan::adjust_late_materialization_plan_structure(ObLogicalOperato dtc_params, optimizer_context_.get_exec_ctx(), NULL, - params))) { + params, + false, + true, + is_in_range_optimization_enabled))) { LOG_WARN("failed to preliminary extract query range", K(ret)); } else if (OB_FAIL(table_scan->set_range_columns(range_columns))) { LOG_WARN("failed to set range columns", K(ret)); diff --git a/src/sql/optimizer/ob_table_location.cpp b/src/sql/optimizer/ob_table_location.cpp index c1ef72254d..691f1f810b 100644 --- a/src/sql/optimizer/ob_table_location.cpp +++ b/src/sql/optimizer/ob_table_location.cpp @@ -1291,7 +1291,8 @@ int ObTableLocation::init( if (OB_FAIL(record_in_dml_partition_info(stmt, exec_ctx, filter_exprs, is_in_hit_, table_schema))) { //这是一个特殊路径，针对in filter条件 LOG_WARN("fail to record_in_dml_partition_info", K(ret)); } else if (!is_in_hit_) { - if (OB_FAIL(record_not_insert_dml_partition_info(stmt, exec_ctx, table_schema, filter_exprs, dtc_params))) { + if (OB_FAIL(record_not_insert_dml_partition_info(stmt, exec_ctx, table_schema, filter_exprs, dtc_params, + session_info->is_in_range_optimization_enabled()))) { LOG_WARN("Fail to record select or update partition info", K(stmt_type_), K(ret)); } else if (OB_FAIL(get_not_insert_dml_part_sort_expr(stmt, sort_exprs))) { LOG_WARN("Failed to get not insert dml sort key with parts", K(ret)); @@ -1920,7 +1921,8 @@ int ObTableLocation::set_location_calc_node(const ObDMLStmt &stmt, ObPartLocCalcNode *&calc_node, ObPartLocCalcNode *&gen_col_node, bool &get_all, - bool &is_range_get) + bool &is_range_get, + const bool is_in_range_optimization_enabled) { int ret = OB_SUCCESS; ObSEArray part_columns; @@ -1949,7 +1951,8 @@ int ObTableLocation::set_location_calc_node(const ObDMLStmt &stmt, get_all, is_range_get, dtc_params, - exec_ctx))) { + exec_ctx, + is_in_range_optimization_enabled))) { LOG_WARN("Failed to get location calc node", K(ret)); } else if (gen_cols.count() > 0) { //analyze information with dependented column of generated column @@ -1960,7 +1963,8 @@ int ObTableLocation::set_location_calc_node(const ObDMLStmt &stmt, always_true, gen_col_node, dtc_params, - exec_ctx))) { + exec_ctx, + is_in_range_optimization_enabled))) { LOG_WARN("Get query range node error", K(ret)); } else if (always_true) { gen_col_node = NULL; @@ -2224,7 +2228,8 @@ int ObTableLocation::record_not_insert_dml_partition_info( ObExecContext *exec_ctx, const ObTableSchema *table_schema, const ObIArray &filter_exprs, - const ObDataTypeCastParams &dtc_params) + const ObDataTypeCastParams &dtc_params, + const bool is_in_range_optimization_enabled) { int ret = OB_SUCCESS; if (OB_FAIL(set_location_calc_node(stmt, @@ -2238,7 +2243,8 @@ int ObTableLocation::record_not_insert_dml_partition_info( calc_node_, gen_col_node_, part_get_all_, - is_part_range_get_))) { + is_part_range_get_, + is_in_range_optimization_enabled))) { LOG_WARN("failed to set location calc node for first-level partition", K(ret)); } else if (PARTITION_LEVEL_TWO == part_level_ && OB_FAIL(set_location_calc_node(stmt, @@ -2252,7 +2258,8 @@ int ObTableLocation::record_not_insert_dml_partition_info( subcalc_node_, sub_gen_col_node_, subpart_get_all_, - is_subpart_range_get_))) { + is_subpart_range_get_, + is_in_range_optimization_enabled))) { LOG_WARN("failed to set location calc node for second-level partition", K(ret)); } @@ -2335,7 +2342,8 @@ int ObTableLocation::get_location_calc_node(const ObPartitionLevel part_level, bool &get_all, bool &is_range_get, const ObDataTypeCastParams &dtc_params, - ObExecContext *exec_ctx) + ObExecContext *exec_ctx, + const bool is_in_range_optimization_enabled) { int ret = OB_SUCCESS; uint64_t column_id = OB_INVALID_ID; @@ -2363,7 +2371,8 @@ int ObTableLocation::get_location_calc_node(const ObPartitionLevel part_level, always_true, calc_node, dtc_params, - exec_ctx))) { + exec_ctx, + is_in_range_optimization_enabled))) { LOG_WARN("Get query range node error", K(ret)); } else if (always_true) { get_all = true; @@ -2405,7 +2414,7 @@ int ObTableLocation::get_location_calc_node(const ObPartitionLevel part_level, ObPartLocCalcNode *column_node = NULL; if (normal_filters.count() > 0) { if (OB_FAIL(get_query_range_node(part_level, partition_columns, filter_exprs, column_always_true, - column_node, dtc_params, exec_ctx))) { + column_node, dtc_params, exec_ctx, is_in_range_optimization_enabled))) { LOG_WARN("Failed to get query range node", K(ret)); } } @@ -2432,7 +2441,8 @@ int ObTableLocation::get_query_range_node(const ObPartitionLevel part_level, bool &always_true, ObPartLocCalcNode *&calc_node, const ObDataTypeCastParams &dtc_params, - ObExecContext *exec_ctx) + ObExecContext *exec_ctx, + const bool is_in_range_optimization_enabled) { int ret = OB_SUCCESS; bool phy_rowid_for_table_loc = (part_level == part_level_); @@ -2451,7 +2461,8 @@ int ObTableLocation::get_query_range_node(const ObPartitionLevel part_level, NULL, NULL, phy_rowid_for_table_loc, - false))) { + false, + is_in_range_optimization_enabled))) { LOG_WARN("Failed to pre extract query range", K(ret)); } else if (node->pre_query_range_.is_precise_whole_range()) { //pre query range is whole range, indicate that there are no partition condition in filters, diff --git a/src/sql/optimizer/ob_table_location.h b/src/sql/optimizer/ob_table_location.h index 3bb1aa20a4..557953fb72 100644 --- a/src/sql/optimizer/ob_table_location.h +++ b/src/sql/optimizer/ob_table_location.h @@ -922,7 +922,8 @@ private: ObExecContext *exec_ctx, const share::schema::ObTableSchema *table_schema, const common::ObIArray &filter_exprs, - const common::ObDataTypeCastParams &dtc_params); + const common::ObDataTypeCastParams &dtc_params, + const bool is_in_range_optimization_enabled); int add_se_value_expr(const ObRawExpr *value_expr, RowDesc &value_row_desc, @@ -950,7 +951,8 @@ private: bool &get_all, bool &is_range_get, const common::ObDataTypeCastParams &dtc_params, - ObExecContext *exec_ctx); + ObExecContext *exec_ctx, + const bool is_in_range_optimization_enabled); int analyze_filter(const common::ObIArray &partition_columns, const ObRawExpr *partition_expr, @@ -968,7 +970,8 @@ private: bool &always_true, ObPartLocCalcNode *&calc_node, const common::ObDataTypeCastParams &dtc_params, - ObExecContext *exec_ctx); + ObExecContext *exec_ctx, + const bool is_in_range_optimization_enabled); int extract_eq_op(ObExecContext *exec_ctx, const ObRawExpr *l_expr, @@ -1055,7 +1058,8 @@ private: ObPartLocCalcNode *&calc_node, ObPartLocCalcNode *&gen_col_node, bool &get_all, - bool &is_range_get); + bool &is_range_get, + const bool is_in_range_optimization_enabled); int calc_partition_ids_by_in_expr( ObExecContext &exec_ctx, diff --git a/src/sql/rewrite/ob_key_part.h b/src/sql/rewrite/ob_key_part.h index 0e61c2bd4a..39adfe497a 100644 --- a/src/sql/rewrite/ob_key_part.h +++ b/src/sql/rewrite/ob_key_part.h @@ -41,6 +41,7 @@ enum InType T_NOT_IN_KEY_PART }; class ObKeyPart; +const int64_t MAX_EXTRACT_IN_COLUMN_NUMBER = 6; class ObKeyPartId { @@ -204,12 +205,12 @@ struct InParamValsWrapper hash_code = hash(); return ret; } - ObSEArray param_vals_; + ObSEArray param_vals_; TO_STRING_KV(K_(param_vals)); }; -typedef ObSEArray OffsetsArr; -typedef ObSEArray InParamsArr; +typedef ObSEArray OffsetsArr; +typedef ObSEArray InParamsArr; struct ObInKeyPart { ObInKeyPart() diff --git a/src/sql/rewrite/ob_query_range.cpp b/src/sql/rewrite/ob_query_range.cpp index 58a2e22078..b8599e0b2f 100644 --- a/src/sql/rewrite/ob_query_range.cpp +++ b/src/sql/rewrite/ob_query_range.cpp @@ -251,7 +251,8 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu const ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, ExprConstrantArray *expr_constraints /* = NULL */, - const ParamsIArray *params /* = NULL */) + const ParamsIArray *params /* = NULL */, + const bool use_in_optimization /* = false */) { int ret = OB_SUCCESS; ObArenaAllocator ctx_allocator(ObModIds::OB_QUERY_RANGE_CTX); @@ -268,6 +269,7 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu GET_ALWAYS_TRUE_OR_FALSE(true, root); } else { if (OB_FAIL(preliminary_extract(expr_root, root, dtc_params, + use_in_optimization, T_OP_IN == expr_root->get_expr_type()))) { LOG_WARN("gen table range failed", K(ret)); } else if (query_range_ctx_->cur_expr_is_precise_ && root != NULL) { @@ -305,7 +307,7 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu } if (OB_SUCC(ret) && NULL != root) { ObSqlBitSet<> key_offsets; - if (OB_FAIL(refine_large_range_graph(root))) { + if (OB_FAIL(refine_large_range_graph(root, use_in_optimization))) { LOG_WARN("failed to refine large range graph", K(ret)); } else if (OB_FAIL(check_graph_type(*root))) { LOG_WARN("check graph type failed", K(ret)); @@ -686,7 +688,8 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu ExprConstrantArray *expr_constraints /* = NULL */, const ParamsIArray *params /* = NULL */, const bool phy_rowid_for_table_loc /* = false*/, - const bool ignore_calc_failure /* = true*/) + const bool ignore_calc_failure /* = true*/, + const bool use_in_optimization /* = false */) { int ret = OB_SUCCESS; ObKeyPartList and_ranges; @@ -695,7 +698,7 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu ObKeyPartList geo_ranges; bool has_geo_expr = false; - SQL_REWRITE_LOG(DEBUG, "preliminary extract", K(range_columns), K(root_exprs)); + SQL_REWRITE_LOG(DEBUG, "preliminary extract", K(range_columns), K(root_exprs), K(use_in_optimization)); ObSEArray candi_exprs; ObArenaAllocator ctx_allocator(ObModIds::OB_QUERY_RANGE_CTX); if (OB_FAIL(init_query_range_ctx(ctx_allocator, range_columns, exec_ctx, @@ -719,6 +722,7 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu if (OB_ISNULL(cur_expr)) { // continue } else if (OB_FAIL(preliminary_extract(cur_expr, temp_result, dtc_params, + use_in_optimization, T_OP_IN == cur_expr->get_expr_type()))) { LOG_WARN("Generate table range failed", K(ret)); } else if (NULL == temp_result) { @@ -783,7 +787,7 @@ int ObQueryRange::preliminary_extract_query_range(const ColumnIArray &range_colu } else if (contain_in_ && !query_range_ctx_->need_final_extract_ && OB_FAIL(rebuild_in_graph(temp_result))) { LOG_WARN("failed to rebuild and graph for in key", K(ret)); - } else if (OB_FAIL(refine_large_range_graph(temp_result))) { + } else if (OB_FAIL(refine_large_range_graph(temp_result, use_in_optimization))) { LOG_WARN("failed to refine large range graph", K(ret)); } else if (OB_FAIL(check_graph_type(*temp_result))) { LOG_WARN("check graph type failed", K(ret)); @@ -914,7 +918,7 @@ bool ObQueryRange::is_and_next_useless(ObKeyPart *cur_key, ObKeyPart *and_next, } // if the range size is large then RANGE_MAX_SIZE, remove some ranges according to pos_.offset_ -int ObQueryRange::refine_large_range_graph(ObKeyPart *&key_part) +int ObQueryRange::refine_large_range_graph(ObKeyPart *&key_part, bool use_in_optimization) { int ret = OB_SUCCESS; ObSEArray pre_key_parts; @@ -924,6 +928,7 @@ int ObQueryRange::refine_large_range_graph(ObKeyPart *&key_part) ObSEArray next_or_count; uint64_t cur_range_size = 1; bool need_refine = false; + int64_t max_range_size = use_in_optimization ? MAX_RANGE_SIZE_NEW : MAX_RANGE_SIZE_OLD; if (OB_ISNULL(key_part)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("keypart is null", K(ret), K(key_part)); @@ -937,7 +942,7 @@ int ObQueryRange::refine_large_range_graph(ObKeyPart *&key_part) if (OB_FAIL(compute_range_size(key_parts, or_count, next_key_parts, next_or_count, cur_range_size))) { LOG_WARN("failed to compute range size", K(ret)); - } else if (cur_range_size > MAX_RANGE_SIZE) { + } else if (cur_range_size > max_range_size) { need_refine = true; } else if (OB_FAIL(pre_key_parts.assign(key_parts))) { LOG_WARN("failed to assign array", K(ret), K(key_parts)); @@ -947,7 +952,7 @@ int ObQueryRange::refine_large_range_graph(ObKeyPart *&key_part) LOG_WARN("failed to assign array", K(ret), K(next_or_count)); } else { /* do nothing */ } } - range_size_ = need_refine ? MAX_RANGE_SIZE : + range_size_ = need_refine ? max_range_size : cur_range_size < RANGE_BUCKET_SIZE ? RANGE_BUCKET_SIZE : cur_range_size; if (OB_SUCC(ret) && need_refine) { ObKeyPart *first_keypart = NULL; @@ -2574,7 +2579,7 @@ int ObQueryRange::pre_extract_single_in_op(const ObOpRawExpr *b_expr, } else if (OB_ISNULL(r_expr = static_cast(b_expr->get_param_expr(1)))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("r_expr is null.", K(ret)); - } else if (r_expr->get_param_count() > 10000) { + } else if (r_expr->get_param_count() > MAX_RANGE_SIZE_OLD) { // do not extract range over MAX_RANGE_SIZE GET_ALWAYS_TRUE_OR_FALSE(true, out_key_part); query_range_ctx_->cur_expr_is_precise_ = false; @@ -2656,7 +2661,7 @@ int ObQueryRange::pre_extract_complex_in_op(const ObOpRawExpr *b_expr, } else if (OB_ISNULL(r_expr = static_cast(b_expr->get_param_expr(1)))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("r_expr is null.", K(ret)); - } else if (r_expr->get_param_count() > 10000) { + } else if (r_expr->get_param_count() > MAX_RANGE_SIZE_OLD) { GET_ALWAYS_TRUE_OR_FALSE(true, out_key_part); query_range_ctx_->cur_expr_is_precise_ = false; } else { @@ -2902,6 +2907,8 @@ int ObQueryRange::prepare_multi_in_info(const ObOpRawExpr *l_expr, if (OB_ISNULL(l_expr) || OB_ISNULL(r_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(l_expr), K(r_expr)); + } else if (OB_UNLIKELY(l_expr->get_param_count() > MAX_EXTRACT_IN_COLUMN_NUMBER)) { + // do nothiing } else if (OB_FAIL(idx_pos_map.create(l_expr->get_param_count(), "IdxKeyMap", "IdxKeyMap"))) { LOG_WARN("fail to init hashmap", K(ret)); } else if (OB_FAIL(idx_param_map.create(l_expr->get_param_count(), "IdxParamMap", "IdxParamMap"))) { @@ -3390,7 +3397,8 @@ int ObQueryRange::pre_extract_not_in_op(const ObOpRawExpr *b_expr, int ObQueryRange::pre_extract_and_or_op(const ObOpRawExpr *m_expr, ObKeyPart *&out_key_part, - const ObDataTypeCastParams &dtc_params) + const ObDataTypeCastParams &dtc_params, + const bool use_in_optimization) { int ret = OB_SUCCESS; if (OB_ISNULL(m_expr) || OB_ISNULL(query_range_ctx_)) { @@ -3402,7 +3410,7 @@ int ObQueryRange::pre_extract_and_or_op(const ObOpRawExpr *m_expr, for (int64_t i = 0; OB_SUCC(ret) && i < m_expr->get_param_count(); ++i) { ObKeyPart *tmp = NULL; query_range_ctx_->cur_expr_is_precise_ = false; - if (OB_FAIL(preliminary_extract(m_expr->get_param_expr(i), tmp, dtc_params))) { + if (OB_FAIL(preliminary_extract(m_expr->get_param_expr(i), tmp, dtc_params, use_in_optimization))) { LOG_WARN("preliminary_extract failed", K(ret)); } else if (T_OP_AND == m_expr->get_expr_type()) { if (OB_FAIL(add_and_item(key_part_list, tmp))) { @@ -3629,6 +3637,7 @@ int ObQueryRange::pre_extract_geo_op(const ObOpRawExpr *geo_expr, int ObQueryRange::preliminary_extract(const ObRawExpr *node, ObKeyPart *&out_key_part, const ObDataTypeCastParams &dtc_params, + const bool use_in_optimization, const bool is_single_in /* = false */) { int ret = OB_SUCCESS; @@ -3671,17 +3680,27 @@ int ObQueryRange::preliminary_extract(const ObRawExpr *node, LOG_WARN("extract not_btw failed", K(ret)); } } else if (T_OP_IN == node->get_expr_type()) { - if (OB_FAIL(pre_extract_in_op(b_expr, out_key_part, dtc_params))) { - LOG_WARN("extract single in_op failed", K(ret)); - } else if (!out_key_part->is_always_true() && !out_key_part->is_always_false()) { - contain_in_ = true; + if (use_in_optimization) { + if (OB_FAIL(pre_extract_in_op(b_expr, out_key_part, dtc_params))) { + LOG_WARN("extract single in_op failed", K(ret)); + } else if (!out_key_part->is_always_true() && !out_key_part->is_always_false()) { + contain_in_ = true; + } + } else { + if (is_single_in) { + if (OB_FAIL(pre_extract_single_in_op(b_expr, out_key_part, dtc_params))) { + LOG_WARN("extract single in_op failed", K(ret)); + } + } else if (OB_FAIL(pre_extract_complex_in_op(b_expr, out_key_part, dtc_params))) { + LOG_WARN("extract in_op failed", K(ret)); + } } } else if (T_OP_NOT_IN == node->get_expr_type()) { if (OB_FAIL(pre_extract_not_in_op(b_expr, out_key_part, dtc_params))) { LOG_WARN("extract in_op failed", K(ret)); } } else if (T_OP_AND == node->get_expr_type() || T_OP_OR == node->get_expr_type()) { - if (OB_FAIL(pre_extract_and_or_op(b_expr, out_key_part, dtc_params))) { + if (OB_FAIL(pre_extract_and_or_op(b_expr, out_key_part, dtc_params, use_in_optimization))) { LOG_WARN("extract and_or failed", K(ret)); } } else if (node->is_spatial_expr()) { @@ -5536,6 +5555,8 @@ int ObQueryRange::union_in_with_in(ObKeyPartList &or_list, } } else if (OB_FAIL(cur1->in_keypart_->union_in_key(cur2->in_keypart_))) { LOG_WARN("failed to union in key", K(ret)); + } else if (cur1->and_next_ != NULL && cur1->and_next_->equal_to(cur2->and_next_)) { + // keep and_next_ } else { cur1->and_next_ = NULL; } diff --git a/src/sql/rewrite/ob_query_range.h b/src/sql/rewrite/ob_query_range.h index 328601afc5..440cbefb41 100644 --- a/src/sql/rewrite/ob_query_range.h +++ b/src/sql/rewrite/ob_query_range.h @@ -381,7 +381,8 @@ public: const common::ObDataTypeCastParams &dtc_params, ObExecContext *exec_ctx, ExprConstrantArray *expr_constraints = NULL, - const ParamsIArray *params = NULL); + const ParamsIArray *params = NULL, + const bool use_in_optimization = false); /** * @brief * @param range_columns: columns used to extract range, index column or partition column @@ -403,7 +404,8 @@ public: ExprConstrantArray *expr_constraints = NULL, const ParamsIArray *params = NULL, const bool phy_rowid_for_table_loc = false, - const bool ignore_calc_failure = true); + const bool ignore_calc_failure = true, + const bool use_in_optimization = false); // final_extract_query_range extracts the final query range of its physical plan. // It will get the real-time value of some const which are unknow during physical plan generating. @@ -600,6 +602,7 @@ private: int preliminary_extract(const ObRawExpr *node, ObKeyPart *&out_key_part, const common::ObDataTypeCastParams &dtc_params, + const bool use_in_optimization, const bool is_single_in = false); int pre_extract_basic_cmp(const ObRawExpr *node, ObKeyPart *&out_key_part, @@ -630,7 +633,8 @@ private: const ObDataTypeCastParams &dtc_params); int pre_extract_and_or_op(const ObOpRawExpr *m_expr, ObKeyPart *&out_key_part, - const common::ObDataTypeCastParams &dtc_params); + const common::ObDataTypeCastParams &dtc_params, + const bool use_in_optimization); int pre_extract_const_op(const ObRawExpr *node, ObKeyPart *&out_key_part); int pre_extract_geo_op(const ObOpRawExpr *geo_expr, @@ -868,7 +872,7 @@ private: const ObObj *&obj_ptr, ObKeyPart &out_key_part, const ObDataTypeCastParams &dtc_params); - int refine_large_range_graph(ObKeyPart *&key_part); + int refine_large_range_graph(ObKeyPart *&key_part, bool use_in_optimization = false); int compute_range_size(const ObIArray &key_parts, const ObIArray &or_count, ObIArray &next_key_parts, @@ -891,7 +895,8 @@ private: ObKeyPart *&out_key_part); private: static const int64_t RANGE_BUCKET_SIZE = 1000; - static const int64_t MAX_RANGE_SIZE = 100000; + static const int64_t MAX_RANGE_SIZE_OLD = 10000; + static const int64_t MAX_RANGE_SIZE_NEW = 100000; static const int64_t MAX_NOT_IN_SIZE = 10; //do not extract range for not in row over this size typedef common::ObObjStore KeyPartStore; private: diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index fb6360b8b7..0f7ab42547 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -416,6 +416,17 @@ int ObSQLSessionInfo::is_groupby_placement_transformation_enabled(bool &transfor return ret; } +bool ObSQLSessionInfo::is_in_range_optimization_enabled() const +{ + bool bret = false; + int64_t tenant_id = get_effective_tenant_id(); + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); + if (tenant_config.is_valid()) { + bret = tenant_config->_enable_in_range_optimization; + } + return bret; +} + void ObSQLSessionInfo::destroy(bool skip_sys_var) { if (is_inited_) { diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index 3442fad7e0..cd1aab1d58 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -1044,6 +1044,7 @@ public: int is_force_temp_table_materialize(bool &force_materialize) const; int is_temp_table_transformation_enabled(bool &transformation_enabled) const; int is_groupby_placement_transformation_enabled(bool &transformation_enabled) const; + bool is_in_range_optimization_enabled() const; ObSessionDDLInfo &get_ddl_info() { return ddl_info_; } void set_ddl_info(const ObSessionDDLInfo &ddl_info) { ddl_info_ = ddl_info; } diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index 8d0c19732d..2e0d50bf2c 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -264,6 +264,7 @@ _enable_easy_keepalive _enable_fulltext_index _enable_hash_join_hasher _enable_hash_join_processor +_enable_in_range_optimization _enable_newsort _enable_new_sql_nio _enable_oracle_priv_check diff --git a/unittest/sql/rewrite/test_query_range_filter.result b/unittest/sql/rewrite/test_query_range_filter.result index 59200f0cb9..6e0022dceb 100644 --- a/unittest/sql/rewrite/test_query_range_filter.result +++ b/unittest/sql/rewrite/test_query_range_filter.result @@ -651,20 +651,20 @@ count of rang columns = 2 [{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] **rowkey num = 3** -**filter count = 0** +**filter count = 1** is not min_to_max_range ranges.count() = 4 -all_single_value_ranges = 0 -star_border_flag[0] = 0 -end_border_flag[0] = 0 -star_border_flag[1] = 0 -end_border_flag[1] = 0 -star_border_flag[2] = 0 -end_border_flag[2] = 0 -star_border_flag[3] = 0 -end_border_flag[3] = 0 +all_single_value_ranges = 1 +star_border_flag[0] = 1 +end_border_flag[0] = 1 +star_border_flag[1] = 1 +end_border_flag[1] = 1 +star_border_flag[2] = 1 +end_border_flag[2] = 1 +star_border_flag[3] = 1 +end_border_flag[3] = 1 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},{"BIGINT":1},MIN;{"BIGINT":1},{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":2},{"BIGINT":2},MIN;{"BIGINT":2},{"BIGINT":2},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] +[{"range":"table_id:3003,group_idx:0,[{"BIGINT":1},{"BIGINT":1},{"BIGINT":1};{"BIGINT":1},{"BIGINT":1},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3},{"BIGINT":1};{"BIGINT":3},{"BIGINT":3},{"BIGINT":1}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4},{"BIGINT":1};{"BIGINT":4},{"BIGINT":4},{"BIGINT":1}]"}] [16] (a, b, c) in ((?{1},?{1},?{1}),(?{2},?{2},?{2})) or (a,b) in ((?{3},?{3}),(?{4},?{4})) or a = ?{1} @@ -689,32 +689,32 @@ count of rang columns = 1 is not min_to_max_range ranges.count() = 4 all_single_value_ranges = 0 -star_border_flag[0] = 1 -end_border_flag[0] = 1 +star_border_flag[0] = 0 +end_border_flag[0] = 0 star_border_flag[1] = 1 end_border_flag[1] = 1 star_border_flag[2] = 1 end_border_flag[2] = 1 -star_border_flag[3] = 0 -end_border_flag[3] = 0 +star_border_flag[3] = 1 +end_border_flag[3] = 1 count of rang columns = 2 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}] +[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN;{"BIGINT":1},MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":3},{"BIGINT":3};{"BIGINT":3},{"BIGINT":3}]"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":4},{"BIGINT":4};{"BIGINT":4},{"BIGINT":4}]"}] **rowkey num = 3** **filter count = 0** is not min_to_max_range ranges.count() = 4 all_single_value_ranges = 0 -star_border_flag[0] = 1 -end_border_flag[0] = 1 -star_border_flag[1] = 0 -end_border_flag[1] = 0 +star_border_flag[0] = 0 +end_border_flag[0] = 0 +star_border_flag[1] = 1 +end_border_flag[1] = 1 star_border_flag[2] = 0 end_border_flag[2] = 0 star_border_flag[3] = 0 end_border_flag[3] = 0 count of rang columns = 3 -[{"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}] +[{"range":"table_id:3003,group_idx:0,({"BIGINT":1},MIN,MIN;{"BIGINT":1},MAX,MAX)"}, {"range":"table_id:3003,group_idx:0,[{"BIGINT":2},{"BIGINT":2},{"BIGINT":2};{"BIGINT":2},{"BIGINT":2},{"BIGINT":2}]"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":3},{"BIGINT":3},MIN;{"BIGINT":3},{"BIGINT":3},MAX)"}, {"range":"table_id:3003,group_idx:0,({"BIGINT":4},{"BIGINT":4},MIN;{"BIGINT":4},{"BIGINT":4},MAX)"}] [17] a = 1 and c = 1 or (a = 2 and b = 2)