From bcc15937eede00e4e6c7505c9177aaef82d07628 Mon Sep 17 00:00:00 2001 From: leslieyuchen Date: Wed, 15 Nov 2023 07:40:18 +0000 Subject: [PATCH] [CP] Fix issues related to user variables. --- .../code_generator/ob_static_engine_cg.cpp | 37 +++++++++++++++++-- src/sql/das/ob_das_ref.cpp | 10 ++--- src/sql/optimizer/ob_log_plan.cpp | 29 ++++++++------- src/sql/optimizer/ob_log_table_scan.cpp | 30 +++++++++++++-- src/sql/optimizer/ob_log_table_scan.h | 6 ++- src/sql/resolver/expr/ob_expr_info_flag.h | 2 + 6 files changed, 87 insertions(+), 27 deletions(-) diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index 2af0f9e234..3ee2dfec00 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -7000,9 +7000,40 @@ int ObStaticEngineCG::set_other_properties(const ObLogPlan &log_plan, ObPhysical } if (OB_SUCC(ret) && !log_plan.get_stmt()->is_explain_stmt()) { - if (OB_FAIL(generate_rt_exprs(log_plan.get_stmt()->get_query_ctx()->var_init_exprs_, - phy_plan.var_init_exprs_))) { - LOG_WARN("generate var init exprs failed", KR(ret)); + const ObIArray &var_init_exprs = log_plan.get_stmt()->get_query_ctx()->var_init_exprs_; + phy_plan.var_init_exprs_.set_capacity(var_init_exprs.count()); + for (int i = 0; OB_SUCC(ret) && i < var_init_exprs.count(); ++i) { + const ObRawExpr *var_init_expr = var_init_exprs.at(i); + LOG_DEBUG("start to generate var init expr", KPC(var_init_expr)); + /** + * What is a user variable initialization expression? + * In MySQL, the user variable assignment clause in SELECT FROM DUAL statement will always be + * executed before the statement is executed. + * However, in OceanBase, such clauses may not be executed due to short-circuit operation in the execution path, + * leading to uninitialized user variables. + * For example: SELECT * FROM t1, (SELECT @rownum:=0) AS init; + * In this statement, if t1 is an empty table, MySQL will still execute SELECT @rownum:=0, + * while OB will not. + * To accommodate this behavior, + * we collect the expressions that appear in SELECT FROM DUAL and can be executed independently, + * and execute them once as the initialization operation for user variables before the plan is executed. + **/ + if (!var_init_expr->has_flag(CNT_SUB_QUERY) && + !var_init_expr->has_flag(CNT_AGG) && + !var_init_expr->has_flag(CNT_WINDOW_FUNC) && + !var_init_expr->has_flag(CNT_ONETIME) && + !var_init_expr->has_flag(CNT_ALIAS) && + !var_init_expr->has_flag(CNT_DYNAMIC_PARAM) && + !ObOptimizerUtil::has_psedu_column(*var_init_expr) && + !ObOptimizerUtil::has_hierarchical_expr(*var_init_expr) && + var_init_expr->get_relation_ids().is_empty()) { + ObExpr *var_rt_expr = nullptr; + if (OB_FAIL(generate_rt_expr(*var_init_expr, var_rt_expr))) { + LOG_WARN("generate var init expr failed", KR(ret), KPC(var_init_expr)); + } else if (OB_FAIL(phy_plan.var_init_exprs_.push_back(var_rt_expr))) { + LOG_WARN("store var rt expr failed", KR(ret)); + } + } } } diff --git a/src/sql/das/ob_das_ref.cpp b/src/sql/das/ob_das_ref.cpp index 5237a0574a..99c9c92c18 100644 --- a/src/sql/das/ob_das_ref.cpp +++ b/src/sql/das/ob_das_ref.cpp @@ -479,18 +479,16 @@ int ObDASRef::close_all_task() } ret = COVER_SUCC(last_end_ret); - if (OB_SUCC(ret)) { - if (OB_ISNULL(session = exec_ctx_.get_my_session())) { - ret = OB_NOT_INIT; - LOG_WARN("session is nullptr", K(ret)); - } + if (OB_ISNULL(session = exec_ctx_.get_my_session())) { + ret = COVER_SUCC(OB_NOT_INIT); + LOG_WARN("session is nullptr", K(ret)); } bool merge_trans_result_fail = (ret != OB_SUCCESS); // any fail during merge trans_result, // need set trans_result incomplete, in order to // indicate some transaction participants info unknown if (merge_trans_result_fail && OB_NOT_NULL(session)) { - LOG_WARN("close all task fail, set trans_result to incomplete"); + LOG_WARN("close all task fail, set trans_result to incomplete", K(ret)); session->get_trans_result().set_incomplete(); } batched_tasks_.destroy(); diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index a5a449e1a8..dcb5b488cc 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -6687,13 +6687,21 @@ int ObLogPlan::try_push_aggr_into_table_scan(ObLogicalOperator *top, } else if (log_op_def::LOG_TABLE_SCAN == top->get_type()) { ObLogTableScan *scan_op = static_cast(top); bool is_get = false; + bool has_npd_filter = false; //has non-pushdown filter if (OB_FAIL(scan_op->is_table_get(is_get))) { LOG_WARN("failed to check is get", K(ret)); + } else if (OB_FAIL(scan_op->has_nonpushdown_filter(has_npd_filter))) { + LOG_WARN("check whether hash non-pushdown filter failed", K(ret)); } else if (is_get || + has_npd_filter || scan_op->get_index_back() || scan_op->is_sample_scan() || (is_descending_direction(scan_op->get_scan_direction()) && !groupby_columns.empty())) { - // can not push down + //aggr func cannot be pushed down to the storage layer in these scenarios: + //1. TSC has index lookup + //2. TSC is sample scan operator + //3. TSC contains filters that cannot be pushed down to the storage + //4. TSC is point get } else if (OB_FAIL(scan_op->get_pushdown_aggr_exprs().assign(aggr_items))) { LOG_WARN("failed to assign group exprs", K(ret)); } else if (OB_FAIL(scan_op->get_pushdown_groupby_columns().assign(groupby_columns))) { @@ -8702,17 +8710,12 @@ int ObLogPlan::try_push_limit_into_table_scan(ObLogicalOperator *top, ObRawExpr *new_limit_expr = NULL; ObRawExpr *new_offset_expr = NULL; - bool contain_udf = false; - common::ObIArray &filters = table_scan->get_filter_exprs(); - for(int i = 0; OB_SUCC(ret) && !contain_udf && i < filters.count(); i++) { - if(OB_ISNULL(filters.at(i))) { - //do nothing - } else if(filters.at(i)->has_flag(ObExprInfoFlag::CNT_PL_UDF)) { - contain_udf = true; - } - } - - if (OB_SUCC(ret) && !contain_udf && !is_virtual_table(table_scan->get_ref_table_id()) && + bool has_npd_filter = false; //has non-pushdown filter + //if TSC contains filters that cannot be pushdown to the storage + //the limit clause cannot be pushed down either. + if (OB_FAIL(table_scan->has_nonpushdown_filter(has_npd_filter))) { + LOG_WARN("check whether has non-pushdown filter failed", K(ret)); + } else if (!has_npd_filter && !is_virtual_table(table_scan->get_ref_table_id()) && table_scan->get_table_type() != schema::EXTERNAL_TABLE && !(OB_INVALID_ID != table_scan->get_dblink_id() && NULL != offset_expr) && !get_stmt()->is_calc_found_rows() && !table_scan->is_sample_scan() && @@ -14654,4 +14657,4 @@ int ObLogPlan::compute_duplicate_table_replicas(ObLogicalOperator *op) } } return ret; -} \ No newline at end of file +} diff --git a/src/sql/optimizer/ob_log_table_scan.cpp b/src/sql/optimizer/ob_log_table_scan.cpp index 5ff336c902..48ce7cd1d0 100644 --- a/src/sql/optimizer/ob_log_table_scan.cpp +++ b/src/sql/optimizer/ob_log_table_scan.cpp @@ -512,14 +512,35 @@ int ObLogTableScan::replace_index_back_pushdown_filters(ObRawExprReplacer &repla return ret; } +int ObLogTableScan::has_nonpushdown_filter(bool &has_npd_filter) +{ + int ret = OB_SUCCESS; + has_npd_filter = false; + ObArray nonpushdown_filters; + ObArray scan_pushdown_filters; + ObArray lookup_pushdown_filters; + if (OB_FAIL(extract_pushdown_filters(nonpushdown_filters, + scan_pushdown_filters, + lookup_pushdown_filters, + true /*ignore pushdown filters*/))) { + LOG_WARN("extract pushdnow filters failed", K(ret)); + } else if (!nonpushdown_filters.empty()) { + has_npd_filter = true; + } + return ret; +} + int ObLogTableScan::extract_pushdown_filters(ObIArray &nonpushdown_filters, ObIArray &scan_pushdown_filters, - ObIArray &lookup_pushdown_filters) + ObIArray &lookup_pushdown_filters, + bool ignore_pd_filter /*= false */) { int ret = OB_SUCCESS; const ObIArray &filters = get_filter_exprs(); const auto &flags = get_filter_before_index_flags(); - if (get_contains_fake_cte() || is_virtual_table(get_ref_table_id()) || EXTERNAL_TABLE == get_table_type()) { + if (get_contains_fake_cte() || + is_virtual_table(get_ref_table_id()) || + EXTERNAL_TABLE == get_table_type()) { //all filters can not push down to storage if (OB_FAIL(nonpushdown_filters.assign(filters))) { LOG_WARN("store non-pushdown filters failed", K(ret)); @@ -543,10 +564,13 @@ int ObLogTableScan::extract_pushdown_filters(ObIArray &nonpushdown_f if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) { LOG_WARN("push UDF filter store non-pushdown filter failed", K(ret), K(i)); } - } else if (filters.at(i)->has_flag(CNT_DYNAMIC_USER_VARIABLE)) { + } else if (filters.at(i)->has_flag(CNT_DYNAMIC_USER_VARIABLE) + || filters.at(i)->has_flag(CNT_ASSIGN_EXPR)) { if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) { LOG_WARN("push variable assign filter store non-pushdown filter failed", K(ret), K(i)); } + } else if (ignore_pd_filter) { + //ignore_pd_filter: only extract non-pushdown filters, ignore others } else if (!get_index_back()) { if (OB_FAIL(scan_pushdown_filters.push_back(filters.at(i)))) { LOG_WARN("store pushdown filter failed", K(ret)); diff --git a/src/sql/optimizer/ob_log_table_scan.h b/src/sql/optimizer/ob_log_table_scan.h index a2c7e2cb7b..5df6b5484f 100644 --- a/src/sql/optimizer/ob_log_table_scan.h +++ b/src/sql/optimizer/ob_log_table_scan.h @@ -467,8 +467,10 @@ public: int generate_ddl_output_column_ids(); int replace_gen_col_op_exprs(ObRawExprReplacer &replacer); int extract_pushdown_filters(ObIArray &nonpushdown_filters, - ObIArray &scan_pushdown_filters, - ObIArray &lookup_pushdown_filters); + ObIArray &scan_pushdown_filters, + ObIArray &lookup_pushdown_filters, + bool ignore_pd_filter = false); + int has_nonpushdown_filter(bool &has_npd_filter); int replace_index_back_pushdown_filters(ObRawExprReplacer &replacer); int extract_virtual_gen_access_exprs(ObIArray &access_exprs, uint64_t scan_table_id); diff --git a/src/sql/resolver/expr/ob_expr_info_flag.h b/src/sql/resolver/expr/ob_expr_info_flag.h index bb906d950b..5ed43ffe11 100644 --- a/src/sql/resolver/expr/ob_expr_info_flag.h +++ b/src/sql/resolver/expr/ob_expr_info_flag.h @@ -175,6 +175,7 @@ inline const char* get_expr_info_flag_str(const ObExprInfoFlag flag) case IS_PL_UDF: { ret = "IS_PL_UDF"; break; }; case IS_SEQ_EXPR: { ret = "IS_SEQ_EXPR"; break; } case IS_ENUM_OR_SET: { ret = "IS_ENUM_OR_SET"; break; } + case IS_ASSIGN_EXPR: { ret = "IS_ASSIGN_EXPR"; break; } case IS_CONST_EXPR: { ret = "IS_CONST_EXPR"; break; } case CNT_CONST_EXPR: { ret = "CNT_CONST_EXPR"; break; } case CNT_CONST: { ret = "CNT_CONST"; break; } @@ -212,6 +213,7 @@ inline const char* get_expr_info_flag_str(const ObExprInfoFlag flag) case CNT_SEQ_EXPR: { ret = "CNT_SEQ_EXPR"; break; } case CNT_DYNAMIC_PARAM: { ret = "CNT_DYNAMIC_PARAM"; break; } case CNT_ENUM_OR_SET: { ret = "CNT_ENUM_OR_SET"; break; } + case CNT_ASSIGN_EXPR: { ret = "CNT_ASSIGN_EXPR"; break; } case BE_USED: { ret = "BE_USED"; break; } case IS_SIMPLE_COND: { ret = "IS_SIMPLE_COND"; break; } case IS_RANGE_COND: { ret = "IS_RANGE_COND"; break; }