diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 977f4a2d01..b833746736 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -1665,7 +1665,7 @@ const int64_t OB_MAX_STATICS_PER_TABLE = 128; const uint64_t OB_DEFAULT_INDEX_ATTRIBUTES_SET = 0; const uint64_t OB_DEFAULT_INDEX_VISIBILITY = 0;//0 menas visible;1 means invisible -const uint64_t BNLJ_DEFAULT_GROUP_SIZE = 1000; +const uint64_t OB_MAX_BULK_JOIN_ROWS = 1000; const int64_t OB_INDEX_WRITE_START_DELAY = 20 * 1000 * 1000; //20s diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 838b7122db..64370d9efc 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -145,6 +145,7 @@ ob_set_subtarget(ob_sql engine_basic engine/basic/ob_count_op.cpp engine/basic/ob_expr_values_op.cpp engine/basic/ob_function_table_op.cpp + engine/basic/ob_group_join_buffer.cpp engine/basic/ob_hash_partitioning_infrastructure_op.cpp engine/basic/ob_limit_op.cpp engine/basic/ob_material_op.cpp diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index c6e0fb7e21..7e91409a29 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -856,7 +856,7 @@ int ObStaticEngineCG::generate_calc_exprs( bool contain_batch_stmt_parameter = false; if (OB_FAIL(ret)) { } else if (OB_FAIL(ObOptimizerUtil::check_contain_batch_stmt_parameter( - raw_expr, + raw_expr, contain_batch_stmt_parameter))) { LOG_WARN("failed to check contain batch stmt parameter", K(ret)); } else { @@ -868,6 +868,7 @@ int ObStaticEngineCG::generate_calc_exprs( && !raw_expr->is_op_pseudo_column_expr() && !has_exist_in_array(dep_exprs, flattened_cur_exprs_arr.at(i)) && (raw_expr->has_flag(CNT_VOLATILE_CONST) + || raw_expr->has_flag(CNT_DYNAMIC_PARAM) || contain_batch_stmt_parameter // 计算包含batch优化的折叠参数 || !raw_expr->is_const_expr())) { if (check_eval_once @@ -4069,7 +4070,7 @@ int ObStaticEngineCG::generate_join_spec(ObLogJoin &op, ObJoinSpec &spec) ObNestedLoopJoinSpec &nlj = static_cast(spec); if (op.enable_px_batch_rescan()) { nlj.enable_px_batch_rescan_ = true; - nlj.left_group_size_ = ObNestedLoopJoinOp::PX_RESCAN_BATCH_ROW_COUNT; + nlj.group_size_ = ObNestedLoopJoinOp::PX_RESCAN_BATCH_ROW_COUNT; } else { nlj.enable_px_batch_rescan_ = false; } @@ -4078,7 +4079,7 @@ int ObStaticEngineCG::generate_join_spec(ObLogJoin &op, ObJoinSpec &spec) ObNestedLoopJoinSpec &nlj = static_cast(spec); bool use_batch_nlj = op.can_use_batch_nlj(); if (use_batch_nlj) { - nlj.use_group_ = use_batch_nlj; + nlj.group_rescan_ = use_batch_nlj; } if (nlj.is_vectorized()) { @@ -4113,6 +4114,32 @@ int ObStaticEngineCG::generate_join_spec(ObLogJoin &op, ObJoinSpec &spec) } } } + + if (OB_SUCC(ret)) { + if (OB_FAIL(nlj.left_rescan_params_.init(op.get_above_pushdown_left_params().count()))) { + LOG_WARN("fail to init fixed array", K(ret)); + } else if (OB_FAIL(nlj.right_rescan_params_.init(op.get_above_pushdown_right_params().count()))) { + LOG_WARN("fail to init fixed array", K(ret)); + } else if (OB_FAIL(set_batch_exec_param(op.get_nl_params(), nlj_spec.rescan_params_))) { + LOG_WARN("fail to set batch exec param", K(ret)); + } + ARRAY_FOREACH(op.get_above_pushdown_left_params(), i) { + ObExecParamRawExpr* param_expr = op.get_above_pushdown_left_params().at(i); + if (OB_FAIL(batch_exec_param_caches_.push_back(BatchExecParamCache(param_expr, + &nlj, + true)))) { + LOG_WARN("fail to push back param expr", K(ret)); + } + } + ARRAY_FOREACH(op.get_above_pushdown_right_params(), i) { + ObExecParamRawExpr* param_expr = op.get_above_pushdown_right_params().at(i); + if (OB_FAIL(batch_exec_param_caches_.push_back(BatchExecParamCache(param_expr, + &nlj, + false)))) { + LOG_WARN("fail to push back param expr", K(ret)); + } + } + } } } } @@ -4513,6 +4540,30 @@ int ObStaticEngineCG::generate_spec( for (int64_t i = 0; OB_SUCC(ret) && i < ARRAYSIZEOF(exec_params); i++) { OZ(generate_param_spec(*exec_params[i], *setters[i])); } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(spec.left_rescan_params_.init(op.get_above_pushdown_left_params().count()))) { + LOG_WARN("fail to init fixed array", K(ret)); + } else if (OB_FAIL(spec.right_rescan_params_.init(op.get_above_pushdown_right_params().count()))) { + LOG_WARN("fail to init fixed array", K(ret)); + } else if (OB_FAIL(set_batch_exec_param(*exec_params[0], *setters[0]))) { + LOG_WARN("fail to set batch exec param", K(ret)); + } + ARRAY_FOREACH(op.get_above_pushdown_left_params(), i) { + ObExecParamRawExpr* param_expr = op.get_above_pushdown_left_params().at(i); + if (OB_FAIL(batch_exec_param_caches_.push_back(BatchExecParamCache(param_expr, + &spec, + true)))) { + LOG_WARN("fail to push back param expr", K(ret)); + } + } + ARRAY_FOREACH(op.get_above_pushdown_right_params(), i) { + ObExecParamRawExpr* param_expr = op.get_above_pushdown_right_params().at(i); + if (OB_FAIL(batch_exec_param_caches_.push_back(BatchExecParamCache(param_expr, + &spec, + false)))) { + LOG_WARN("fail to push back param expr", K(ret)); + } + } } if (OB_SUCC(ret)) { @@ -4586,7 +4637,7 @@ int ObStaticEngineCG::generate_spec( } } if (OB_SUCC(ret)) { - spec.enable_das_batch_rescans_ = op.get_enable_das_batch_rescans(); + spec.enable_das_batch_rescans_ = op.enable_das_batch_rescans(); } return ret; } @@ -6797,5 +6848,50 @@ int ObStaticEngineCG::check_fk_nested_dup_del(const uint64_t table_id, return ret; } +int ObStaticEngineCG::set_batch_exec_param(const ObIArray &exec_params, + const ObFixedArray& setters) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(exec_params.count() != setters.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("nl params should have the same length with rescan params", K(ret)); + } + ARRAY_FOREACH(exec_params, i) { + const ObExecParamRawExpr* param_expr = exec_params.at(i); + const ObDynamicParamSetter& setter = setters.at(i); + for (int64_t j = batch_exec_param_caches_.count() - 1; OB_SUCC(ret) && j >= 0; --j) { + const BatchExecParamCache &cache = batch_exec_param_caches_.at(j); + if (param_expr != cache.expr_) { + } else if (OB_ISNULL(cache.spec_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (cache.spec_->get_type() == PHY_SUBPLAN_FILTER) { + ObSubPlanFilterSpec *spf = static_cast(cache.spec_); + if (cache.is_left_param_ && + OB_FAIL(spf->left_rescan_params_.push_back(setter))) { + LOG_WARN("fail to push back left rescan params", K(ret)); + } else if (!cache.is_left_param_ && + OB_FAIL(spf->right_rescan_params_.push_back(setter))) { + LOG_WARN("fail to push back right rescan params", K(ret)); + } else if (OB_FAIL(batch_exec_param_caches_.remove(j))) { + LOG_WARN("fail to remove batch nl param caches", K(ret)); + } + } else if (cache.spec_->get_type() == PHY_NESTED_LOOP_JOIN) { + ObNestedLoopJoinSpec *nlj = static_cast(cache.spec_); + if (cache.is_left_param_ && + OB_FAIL(nlj->left_rescan_params_.push_back(setter))) { + LOG_WARN("fail to push back left rescan params", K(ret)); + } else if (!cache.is_left_param_ && + OB_FAIL(nlj->right_rescan_params_.push_back(setter))) { + LOG_WARN("fail to push back right rescan params", K(ret)); + } else if (OB_FAIL(batch_exec_param_caches_.remove(j))) { + LOG_WARN("fail to remove batch nl param caches", K(ret)); + } + } + } + } + return ret; +} + } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/code_generator/ob_static_engine_cg.h b/src/sql/code_generator/ob_static_engine_cg.h index a3b78dc1b6..e7ccc23d53 100644 --- a/src/sql/code_generator/ob_static_engine_cg.h +++ b/src/sql/code_generator/ob_static_engine_cg.h @@ -458,6 +458,7 @@ private: int add_output_datum_check_flag(ObOpSpec &spec); int generate_calc_part_id_expr(const ObRawExpr &src, const ObDASTableLocMeta *loc_meta, ObExpr *&dst); int check_only_one_unique_key(const ObLogPlan &log_plan, const ObTableSchema* table_schema, bool& only_one_unique_key); + bool is_simple_aggr_expr(const ObItemType &expr_type) { return T_FUN_COUNT == expr_type || T_FUN_SUM == expr_type || T_FUN_MAX == expr_type @@ -467,6 +468,31 @@ private: DASTableIdList &parent_tables, bool &is_dup); bool has_cycle_reference(DASTableIdList &parent_tables, const uint64_t table_id); + + int set_batch_exec_param(const ObIArray &exec_params, + const ObFixedArray& setters); +private: + struct BatchExecParamCache { + BatchExecParamCache(ObExecParamRawExpr* expr, ObOpSpec* spec, bool is_left) + : expr_(expr), spec_(spec), is_left_param_(is_left) {} + + BatchExecParamCache() + : expr_(NULL), spec_(NULL), is_left_param_(true) {} + + BatchExecParamCache(const BatchExecParamCache& other) + { + expr_ = other.expr_; + spec_ = other.spec_; + is_left_param_ = other.is_left_param_; + } + + TO_STRING_KV(K_(expr), + K_(is_left_param)); + + ObExecParamRawExpr* expr_; + ObOpSpec* spec_; + bool is_left_param_; + }; private: ObPhysicalPlan *phy_plan_; ObOptimizerContext *opt_ctx_; @@ -478,6 +504,7 @@ private: common::ObSEArray fake_cte_tables_; ObDmlCgService dml_cg_service_; ObTscCgService tsc_cg_service_; + common::ObSEArray batch_exec_param_caches_; }; } // end namespace sql diff --git a/src/sql/code_generator/ob_tsc_cg_service.cpp b/src/sql/code_generator/ob_tsc_cg_service.cpp index 1ef799490b..68cdde7d2a 100644 --- a/src/sql/code_generator/ob_tsc_cg_service.cpp +++ b/src/sql/code_generator/ob_tsc_cg_service.cpp @@ -144,7 +144,7 @@ int ObTscCgService::generate_tsc_ctdef(ObLogTableScan &op, ObTableScanCtDef &tsc } } } - if (OB_SUCC(ret) && op.use_batch()) { + if (OB_SUCC(ret)) { ObArray bnlj_params; OZ(op.extract_bnlj_param_idxs(bnlj_params)); OZ(tsc_ctdef.bnlj_param_idxs_.assign(bnlj_params)); @@ -362,9 +362,15 @@ int ObTscCgService::extract_pushdown_filters(const ObLogTableScan &op, //2. data table scan filter when TSC use the data table scan directly //lookup_pushdown_filters means that the data table filter when TSC use index scan and lookup the data table for (int64_t i = 0; OB_SUCC(ret) && i < filters.count(); ++i) { - if (filters.at(i)->has_flag(CNT_PL_UDF)) { + if (op.use_batch() && filters.at(i)->has_flag(CNT_DYNAMIC_PARAM)) { + //In Batch table scan the dynamic param filter do not push down to storage if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) { - LOG_WARN("store non-pushdown filter failed", K(ret), K(i)); + LOG_WARN("push dynamic filter to store non-pushdown filter failed", K(ret), K(i)); + } + } else if (filters.at(i)->has_flag(CNT_PL_UDF)) { + //User Define Function filter do not push down to storage + if (OB_FAIL(nonpushdown_filters.push_back(filters.at(i)))) { + LOG_WARN("push UDF filter store non-pushdown filter failed", K(ret), K(i)); } } else if (!op.get_index_back()) { if (OB_FAIL(scan_pushdown_filters.push_back(filters.at(i)))) { diff --git a/src/sql/das/ob_das_context.h b/src/sql/das/ob_das_context.h index f9c206e0a7..de5f7cc19a 100644 --- a/src/sql/das/ob_das_context.h +++ b/src/sql/das/ob_das_context.h @@ -51,6 +51,7 @@ public: snapshot_(), savepoint_(0), del_ctx_list_(allocator), + jump_read_group_id_(-1), flags_(0) { need_check_server_ = 1; @@ -120,6 +121,7 @@ private: //@todo: save snapshot version DASDelCtxList del_ctx_list_; public: + int64_t jump_read_group_id_; union { uint64_t flags_; struct { @@ -129,6 +131,7 @@ public: uint64_t reserved_ : 62; }; }; + }; } // namespace sql } // namespace oceanbase diff --git a/src/sql/das/ob_das_group_scan_op.cpp b/src/sql/das/ob_das_group_scan_op.cpp index a5686a4982..1c8bbe2308 100644 --- a/src/sql/das/ob_das_group_scan_op.cpp +++ b/src/sql/das/ob_das_group_scan_op.cpp @@ -152,6 +152,23 @@ int ObDASGroupScanOp::switch_scan_group() return ret; } +int ObDASGroupScanOp::set_scan_group(int64_t group_id) +{ + int ret = OB_SUCCESS; + ObLocalIndexLookupOp *lookup_op = get_lookup_op(); + // for lookup group scan, switch lookup group scan iter + if (NULL != lookup_op) { + ret = lookup_op->set_lookup_scan_group(group_id); + } else { + ret = iter_.set_scan_group(group_id); + } + + if (OB_FAIL(ret) && OB_ITER_END != ret) { + LOG_WARN("set scan group failed", K(ret), KP(lookup_op), K(iter_)); + } + return ret; +} + ObNewRowIterator *ObDASGroupScanOp::get_storage_scan_iter() { ObNewRowIterator *iter = NULL; @@ -313,6 +330,33 @@ int ObGroupLookupOp::switch_lookup_scan_group() return ret; } +int ObGroupLookupOp::set_lookup_scan_group(int64_t group_id) +{ + int ret = OB_SUCCESS; + state_ = OUTPUT_ROWS; + ObGroupScanIter *group_iter = NULL; + group_iter = static_cast(get_lookup_iter()); + if (NULL == group_iter) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguement", K(group_iter), K(ret)); + } else { + ret = group_iter->set_scan_group(group_id); + if(-1 == group_id) { + ++lookup_group_cnt_; + } else { + lookup_group_cnt_ = group_id + 1; + } + + if(lookup_group_cnt_ >= index_group_cnt_ && OB_ITER_END == ret && !index_end_) { + ret = OB_SUCCESS; + state_ = INDEX_SCAN; + } + } + + return ret; +} + + OB_SERIALIZE_MEMBER((ObDASGroupScanOp, ObDASScanOp), iter_, cur_group_idx_, group_size_); diff --git a/src/sql/das/ob_das_group_scan_op.h b/src/sql/das/ob_das_group_scan_op.h index 6684405f92..b42390091c 100644 --- a/src/sql/das/ob_das_group_scan_op.h +++ b/src/sql/das/ob_das_group_scan_op.h @@ -36,6 +36,7 @@ public: lookup_group_cnt_ = 1; } virtual int64_t get_index_group_cnt() override { return index_group_cnt_; } + virtual void set_index_group_cnt(int64_t group_cnt_) override {index_group_cnt_ = group_cnt_;} virtual void inc_index_group_cnt() override { ++index_group_cnt_; } virtual int64_t get_lookup_group_cnt() override { return lookup_group_cnt_; } virtual void inc_lookup_group_cnt() override { ++lookup_group_cnt_; } @@ -43,6 +44,10 @@ public: { return static_cast(rowkey_iter_)->switch_scan_group(); } + virtual int set_rowkey_scan_group(int64_t group_id) override + { + return static_cast(rowkey_iter_)->set_scan_group(group_id); + } virtual ObNewRowIterator *&get_lookup_storage_iter() override; int init_group_range(int64_t cur_group_idx, int64_t group_size) override; virtual bool need_next_index_batch() const override; @@ -50,6 +55,7 @@ public: int64_t group_size, ObExpr *group_id_expr); virtual int switch_lookup_scan_group() override; + virtual int set_lookup_scan_group(int64_t group_id) override; public: ObGroupScanIter group_iter_; int64_t index_group_cnt_; // number of groups fetched from index table @@ -65,7 +71,8 @@ public: int open_op() override; int release_op() override; virtual int rescan() override; - virtual int switch_scan_group(); + virtual int switch_scan_group() override; + virtual int set_scan_group(int64_t group_id) override; int64_t get_cur_group_idx() const { return iter_.get_cur_group_idx(); } void init_group_range(int64_t cur_group_idx, int64_t group_size); virtual ObLocalIndexLookupOp *get_lookup_op() override diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index 91a14a2833..1dfaf53079 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -796,6 +796,17 @@ int ObLocalIndexLookupOp::get_next_row() switch (state_) { case INDEX_SCAN: { lookup_rowkey_cnt_ = 0; + if (is_group_scan_) { + //See the comment in the ObLocalIndexLookupOp::get_next_rows + set_index_group_cnt(get_lookup_group_cnt()); + ret = set_rowkey_scan_group(get_lookup_group_cnt() - 1); + if (OB_SUCCESS != ret) { + LOG_WARN("set_rowkey_scan_group fail",K(get_lookup_group_cnt() - 1),K(ret)); + if (OB_ITER_END == ret) { + ret = OB_ERR_UNEXPECTED; + } + } + } int64_t start_group_idx = get_index_group_cnt() - 1; while (OB_SUCC(ret) && lookup_rowkey_cnt_ < default_row_batch_cnt) { index_rtdef_->p_pd_expr_op_->clear_evaluated_flag(); @@ -894,6 +905,22 @@ int ObLocalIndexLookupOp::get_next_rows(int64_t &count, int64_t capacity) case INDEX_SCAN: { int64_t rowkey_count = 0; lookup_rowkey_cnt_ = 0; + if (is_group_scan_) { + //Do the group scan jump read. + //Now we support jump read in GroupScan iter. + //Some of row read from index maybe jump. + //We need to sync index_group_cnt with lookup_group_cnt. + //Because in the rescan we manipulate the lookup_group_cnt. + set_index_group_cnt(get_lookup_group_cnt()); + ret = set_rowkey_scan_group(get_lookup_group_cnt() - 1); + if (OB_SUCCESS != ret) { + LOG_WARN("set_rowkey_scan_group fail",K(get_lookup_group_cnt() - 1),K(ret)); + if (OB_ITER_END == ret) { + ret = OB_ERR_UNEXPECTED; + } + } + } + int64_t start_group_idx = get_index_group_cnt() - 1; while (OB_SUCC(ret) && lookup_rowkey_cnt_ < default_row_batch_cnt) { int64_t batch_size = min(capacity, default_row_batch_cnt - lookup_rowkey_cnt_); @@ -997,7 +1024,9 @@ int ObLocalIndexLookupOp::get_next_rows(int64_t &count, int64_t capacity) int ObLocalIndexLookupOp::check_lookup_row_cnt() { int ret = OB_SUCCESS; + //In group scan the jump read may happend, so the lookup_group_cnt and lookup_rowkey_cnt_ mismatch. if (GCONF.enable_defensive_check() + && !is_group_scan_ && lookup_ctdef_->pd_expr_spec_.pushdown_filters_.empty()) { if (OB_UNLIKELY(lookup_rowkey_cnt_ != lookup_row_cnt_) && get_index_group_cnt() == get_lookup_group_cnt()) { diff --git a/src/sql/das/ob_das_scan_op.h b/src/sql/das/ob_das_scan_op.h index b716f9cb07..f905213ccc 100644 --- a/src/sql/das/ob_das_scan_op.h +++ b/src/sql/das/ob_das_scan_op.h @@ -185,6 +185,7 @@ public: bool is_group_scan() { return NULL != scan_ctdef_->group_id_expr_; } virtual bool need_all_output() { return false; } virtual int switch_scan_group() { return common::OB_SUCCESS; }; + virtual int set_scan_group(int64_t group_id) { UNUSED(group_id); return common::OB_NOT_IMPLEMENT; }; INHERIT_TO_STRING_KV("parent", ObIDASTaskOp, KPC_(scan_ctdef), KPC_(scan_rtdef), @@ -296,13 +297,16 @@ public: // for lookup group scan virtual int64_t get_index_group_cnt() { return 0; } virtual int64_t get_lookup_group_cnt() { return 0; } + virtual void set_index_group_cnt(int64_t group_cnt_) { UNUSED(group_cnt_); /*do nothing*/ } virtual void inc_index_group_cnt() { /*do nothing*/ } virtual void inc_lookup_group_cnt() { /*do nothing*/ } virtual int init_group_range(int64_t cur_group_idx, int64_t group_size) { return common::OB_NOT_IMPLEMENT; } virtual bool need_next_index_batch() const; virtual int switch_rowkey_scan_group() { return common::OB_NOT_IMPLEMENT; } + virtual int set_rowkey_scan_group(int64_t group_id) { UNUSED(group_id); return common::OB_NOT_IMPLEMENT; } virtual int switch_lookup_scan_group() { return common::OB_NOT_IMPLEMENT; } + virtual int set_lookup_scan_group(int64_t group_id) { UNUSED(group_id); return common::OB_NOT_IMPLEMENT; } virtual ObNewRowIterator *&get_lookup_storage_iter() { return lookup_iter_; } virtual ObNewRowIterator *get_lookup_iter() { return lookup_iter_; } int check_lookup_row_cnt(); diff --git a/src/sql/das/ob_group_scan_iter.cpp b/src/sql/das/ob_group_scan_iter.cpp index e52795458f..704d544c19 100644 --- a/src/sql/das/ob_group_scan_iter.cpp +++ b/src/sql/das/ob_group_scan_iter.cpp @@ -129,6 +129,8 @@ ObGroupScanIter::ObGroupScanIter() { } + +// 旧版本算法: // 1. 如果last_group_idx > cur_group_idx 则返回iter_end // 2. 如果last_group_idx = cur_group_idx // 则将缓存的行store到对应output行表达式的中, 设置last_group_idx = -1, 返回行 @@ -141,41 +143,70 @@ ObGroupScanIter::ObGroupScanIter() // 记录当前读到的行的group_idx到last_group_idx // 对当前行进行深拷贝暂存 // 返回iter end + + +// 2022-12-03 更新支持跳读算法: +// 1. 如果last_group_idx > cur_group_idx 则返回iter_end +// 2. 如果last_group_idx == cur_group_idx +// 则将缓存的行store到对应output行表达式的中, 设置last_group_idx = -1, 返回行 +// 3. 如果last_group_idx < cur_group_idx +// 3.1 循环条件 last_group_idx < cur_group_idx +// 从存储层获取一行数据 +// 如果ITER_END，存储层所有数据消费完了，返回ITER_END，设置last_group_idx为INT_MAX。 +// 设置last_group_idx为当前行的group_idx. +// 3.2 判断last_group_idx是否与当前cur_group_idx相同 +// a. 如果相同: +// 则返回数据，并且设置last_group_idx = -1; +// b. 如果不相同，说明last_group_idx > cur_group_idx出的循环: +// 对当前行进行深拷贝暂存 +// 返回iter end int ObGroupScanIter::get_next_row() { int ret = OB_SUCCESS; - int64_t group_idx = MIN_GROUP_INDEX; - if (last_group_idx_ < cur_group_idx_) { - ObDatum *datum_group_idx = NULL; - if (OB_FAIL(get_iter()->get_next_row())) { - if (OB_ITER_END != ret) { - LOG_WARN("fail to get next row", K(ret)); - } - } else if (OB_FAIL(group_id_expr_->eval(*row_store_.eval_ctx_, - datum_group_idx))) { - LOG_WARN("fail to eval group id", K(ret)); - } else if (FALSE_IT(group_idx = datum_group_idx->get_int())) { - } else if (group_idx == cur_group_idx_) { - // return result - } else { - if (OB_FAIL(row_store_.save(false, 0, 1))) { - LOG_WARN("fail to save last row", K(ret)); - } else { - last_group_idx_ = group_idx; - ret = OB_ITER_END; - } - } + if (last_group_idx_ > cur_group_idx_) { + ret = OB_ITER_END; } else if (last_group_idx_ == cur_group_idx_) { OZ(row_store_.to_expr(false, 0, 1)); last_group_idx_ = MIN_GROUP_INDEX; } else { - ret = OB_ITER_END; + //last_group_idx_ < cur_group_idx_ + //last_group_idx_ 是MIN_GROUP_INDEX或者需要跳读。 + ObDatum *datum_group_idx = NULL; + while(OB_SUCC(ret) && last_group_idx_ < cur_group_idx_) { + if (OB_FAIL(get_iter()->get_next_row())) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next row", K(ret)); + } else { + //store的OB_ITER_END说明后面没有数据了，以后都返回ITER_END. + last_group_idx_ = INT64_MAX; + } + } else if (OB_FAIL(group_id_expr_->eval(*row_store_.eval_ctx_, + datum_group_idx))) { + LOG_WARN("fail to eval group id", K(ret)); + } else { + last_group_idx_ = datum_group_idx->get_int(); + } + }// while end + if(OB_SUCC(ret)) { + if (last_group_idx_ == cur_group_idx_) { + // return result + last_group_idx_ = MIN_GROUP_INDEX; + } else { + //last_group_idx_ > cur_group_idx_ + if (OB_FAIL(row_store_.save(false, 0, 1))) { + LOG_WARN("fail to save last row", K(ret)); + } else { + ret = OB_ITER_END; + } + } + } } LOG_DEBUG("das group next row", K(ret), K(this), K(*this), K(*row_store_.eval_ctx_)); return ret; } +// 旧版本算法: // 1. 如果last_group_idx > cur_group_idx 则返回iter_end // 2. 如果last_group_idx = cur_group_idx // 2.1 遍历缓存的行, 计算对应group_idx, 当前行是否属于cur_group_idx: @@ -192,14 +223,48 @@ int ObGroupScanIter::get_next_row() // c. iter_end // 3.3 如果遍历结束, 则iter_end, last_group_idx = -1 + +// 2022-12-03 更新支持跳读算法: +// 1. 如果last_group_idx > cur_group_idx 则返回iter_end 算法结束。 +// 2. 如果row_store_里有数据 (即 MIN_GROUP_INDEX != last_group_idx_) +// 循环遍历row_store_。 +// 如果row_store_被消费完，设置last_group_idx = MIN_GROUP_INDEX，退出循环。 +// 如果row_store_中找到last_group_idx >= cur_group_idx，退出循环。 +// 3. 循环判断last_group_idx是否是MIN_GROUP_INDEX +// 3.1 如果last_group_idx != MIN_GROUP_INDEX说明row_store_里有数据，退出循环。 +// 3.2 从存储层读出一批数据 +// a. 完全ITER_END说明所有数据消费完了，算法结束，last_group_idx=INT64_MAX。 +// b. 如果能找到符合group_idx >= cur_group_idx +// 将数据深拷贝到row_store_中备用,更新last_group_idx = group_idx。 +// c. 保持last_group_idx为MIN_GROUP_INDEX，从存储层读取下一批数据 +// 此时，last_group_idx要么是INT64_MAX，要么last_group_idx>=cur_group_idx +// 4. 判断last_group_idx == cur_group_idx +// 4.1 如果相等，吐行，更新last_group_idx +// 4.2 如果不想等，此时必有last_group_idx>cur_group_idx返回ITER_END + + int ObGroupScanIter::get_next_rows(int64_t &count, int64_t capacity) { int ret = OB_SUCCESS; - LOG_DEBUG("das group before next row", K(last_group_idx_), K(cur_group_idx_)); int64_t storage_count = 0; int64_t ret_count = 0; int64_t group_idx = MIN_GROUP_INDEX; - if (last_group_idx_ < cur_group_idx_) { + LOG_DEBUG("das group before next row", K(last_group_idx_), K(cur_group_idx_)); + + if (last_group_idx_ > cur_group_idx_) { + ret = OB_ITER_END; + } else { + while(MIN_GROUP_INDEX != last_group_idx_ && last_group_idx_ < cur_group_idx_) { + row_store_.next_start_pos(); + last_group_idx_ = row_store_.cur_group_idx(); + //We should not assume OB_INVALID_INDEX == MIN_GROUP_INDEX + if (OB_INVALID_INDEX == last_group_idx_) { + last_group_idx_ = MIN_GROUP_INDEX; + } + }//while end + } + + while(OB_SUCC(ret) && MIN_GROUP_INDEX == last_group_idx_) { reset_expr_datum_ptr(); if (OB_FAIL(get_iter()->get_next_rows(storage_count, capacity))) { if (OB_ITER_END == ret && storage_count > 0) { @@ -216,51 +281,52 @@ int ObGroupScanIter::get_next_rows(int64_t &count, int64_t capacity) int64_t i = 0; for (i = 0; i < storage_count; i++) { group_idx = group_idx_batch[i].get_int(); - if (cur_group_idx_ == group_idx) { - ret_count++; - continue; - } else { + if (group_idx >= cur_group_idx_) { + int tmp_ret = row_store_.save(true, i, storage_count - i); + if (OB_SUCCESS != tmp_ret) { + LOG_WARN("fail to save batch result", K(tmp_ret)); + ret = tmp_ret; + } last_group_idx_ = group_idx; - ret = OB_ITER_END; break; } - } // for end; - if (i < storage_count) { - int tmp_ret = row_store_.save(true, i, storage_count - i); - if (OB_SUCCESS != tmp_ret) { - LOG_WARN("fail to save batch result", K(tmp_ret)); - ret = tmp_ret; + }//for end + } + }//while end + + if (OB_SUCC(ret)) { + if(last_group_idx_ == cur_group_idx_) { + int64_t start_pos = row_store_.get_start_pos(); + while(cur_group_idx_ == last_group_idx_) { + group_idx = row_store_.cur_group_idx(); + if (cur_group_idx_ == group_idx) { + row_store_.next_start_pos(); + ret_count++; + } else { + // if row store iter end, group_idx = MIN_GROUP_INDEX; + last_group_idx_ = group_idx; + //We should not assume OB_INVALID_INDEX == MIN_GROUP_INDEX + if (OB_INVALID_INDEX == last_group_idx_) { + last_group_idx_ = MIN_GROUP_INDEX; + } } + }//while end + if (ret_count > 0) { + OZ(row_store_.to_expr(true, start_pos, ret_count)); } else { - last_group_idx_ = MIN_GROUP_INDEX; + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ret count must be greater than 0", K(ret_count), K(row_store_), K(ret)); } - } - } else if (last_group_idx_ == cur_group_idx_) { - int64_t group_idx = MIN_GROUP_INDEX; - int64_t start_pos = row_store_.get_start_pos(); - while(cur_group_idx_ == last_group_idx_) { - group_idx = row_store_.cur_group_idx(); - if (cur_group_idx_ == group_idx) { - row_store_.next_start_pos(); - ret_count++; - } else { - // if row store iter end, group_idx = MIN_GROUP_INDEX; - last_group_idx_ = group_idx; + // when batch data in row store not end, and cur_group_idx != last_group_idx, + // means this group data is end + if (OB_SUCC(ret) && MIN_GROUP_INDEX != last_group_idx_ && cur_group_idx_ != last_group_idx_) { + ret = OB_ITER_END; } - } - if (ret_count > 0) { - OZ(row_store_.to_expr(true, start_pos, ret_count)); } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ret count must be greater than 0", K(ret_count), K(row_store_), K(ret)); - } - // when batch data in row store not end, and cur_group_idx != last_group_idx, - // means this group data is end - if (OB_SUCC(ret) && MIN_GROUP_INDEX != last_group_idx_ && cur_group_idx_ != last_group_idx_) { + OB_ASSERT(last_group_idx_ > cur_group_idx_); + OB_ASSERT(last_group_idx_ != INT64_MAX); ret = OB_ITER_END; } - } else { - ret = OB_ITER_END; } count = ret_count; LOG_DEBUG("das group after next row", K(last_group_idx_), K(group_idx), K(cur_group_idx_), @@ -311,6 +377,27 @@ int ObGroupScanIter::switch_scan_group() return ret; } +int ObGroupScanIter::set_scan_group(int64_t group_id) +{ + int ret = OB_SUCCESS; + //TODO shengle Unified interface with das rescan + if (row_store_.need_check_output_datum_) { + reset_expr_datum_ptr(); + } + + if (-1 == group_id) { + group_id = cur_group_idx_ + 1; + } + cur_group_idx_ = group_id; + if (cur_group_idx_ >= group_size_) { + ret = OB_ITER_END; + } + + LOG_DEBUG("set scan group", K(ret), K(*this), KP(this)); + + return ret; +} + OB_SERIALIZE_MEMBER(ObGroupScanIter, cur_group_idx_, group_size_); } // namespace sql diff --git a/src/sql/das/ob_group_scan_iter.h b/src/sql/das/ob_group_scan_iter.h index 4c7e960ff6..a268334374 100644 --- a/src/sql/das/ob_group_scan_iter.h +++ b/src/sql/das/ob_group_scan_iter.h @@ -75,6 +75,7 @@ public: ObGroupScanIter(); //virtual int rescan() override; int switch_scan_group(); + int set_scan_group(int64_t group_id); virtual int get_next_row(ObNewRow *&row) { return common::OB_NOT_IMPLEMENT; } ; virtual int get_next_row() override; virtual int get_next_rows(int64_t &count, int64_t capacity) override; diff --git a/src/sql/engine/basic/ob_group_join_buffer.cpp b/src/sql/engine/basic/ob_group_join_buffer.cpp new file mode 100644 index 0000000000..5f85a2d9bf --- /dev/null +++ b/src/sql/engine/basic/ob_group_join_buffer.cpp @@ -0,0 +1,1019 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SQL_ENG + +#include "ob_group_join_buffer.h" +#include "sql/engine/ob_exec_context.h" + +namespace oceanbase +{ +using namespace common; +namespace sql +{ +int ObBatchRowDatums::init(const ObExprPtrIArray *exprs, ObIAllocator *alloc, int32_t batch_size) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(alloc) || OB_ISNULL(exprs)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(alloc), KP(exprs)); + } else { + char *buf= (char *)alloc->alloc(ObBitVector::memory_size(batch_size) + + sizeof(ObDatum) * batch_size * exprs->count()); + if (NULL == buf) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", KR(ret)); + } else { + MEMSET(buf, 0, ObBitVector::memory_size(batch_size)); + skip_ = to_bit_vector(buf); + alloc_ = alloc; + exprs_ = exprs; + datums_ = reinterpret_cast(buf + ObBitVector::memory_size(batch_size)); + batch_size_ = batch_size; + size_ = 0; + saved_size_ = 0; + is_inited_ = true; + } + } + + return ret; +} + +void ObBatchRowDatums::from_exprs(ObEvalCtx &ctx, ObBitVector *skip, int64_t size) +{ + OB_ASSERT(size <= batch_size_); + OB_ASSERT(OB_NOT_NULL(skip) && OB_NOT_NULL(exprs_)); + for (int64_t i = 0; i < exprs_->count(); i++) { + ObExpr *expr = exprs_->at(i); + ObDatum *datums = expr->locate_batch_datums(ctx); + int64_t copy_size = (expr->is_batch_result() ? size: 1) * sizeof(ObDatum); + MEMCPY(datums_ + i * batch_size_, datums, copy_size); + } + size_ = size; + saved_size_ = size; + skip_->deep_copy(*skip, size); +} + +void ObBatchRowDatums::extend_save(ObEvalCtx &ctx, int64_t size) +{ + if (size > saved_size_) { + for (int64_t i = 0; i < exprs_->count(); i++) { + ObExpr *expr = exprs_->at(i); + if (expr->is_batch_result()) { + ObDatum *datums = expr->locate_batch_datums(ctx); + int64_t copy_size = (size - saved_size_) * sizeof(ObDatum); + MEMCPY(datums_ + i * batch_size_ + saved_size_, datums + saved_size_, copy_size); + } + } + saved_size_ = size; + } +} + +void ObBatchRowDatums::to_exprs(ObEvalCtx &ctx) +{ + if (saved_size_ > 0) { + for (int64_t i = 0; i < exprs_->count(); i++) { + ObExpr *expr = exprs_->at(i); + ObDatum *datums = expr->locate_batch_datums(ctx); + int64_t copy_size = (expr->is_batch_result() ? saved_size_: 1) * sizeof(ObDatum); + MEMCPY(datums, datums_ + i * batch_size_, copy_size); + } + } +} + +void ObBatchRowDatums::to_exprs(ObEvalCtx &ctx, int64_t from_idx, int64_t to_idx) +{ + OB_ASSERT(from_idx <= size_ && to_idx <= batch_size_); + OB_ASSERT(!skip_->exist(from_idx)); + for (int64_t i = 0; i < exprs_->count(); i++) { + ObExpr *expr = exprs_->at(i); + ObDatum *datums = expr->locate_batch_datums(ctx); + if (!expr->is_batch_result()) { + *datums = *(datums_ + i * batch_size_); + } else { + *(datums + to_idx) = *(datums_ + i * batch_size_ + from_idx) ; + } + } +} + +ObGroupJoinBufffer::ObGroupJoinBufffer() + : op_(NULL), spec_(NULL), ctx_(NULL), eval_ctx_(NULL), + left_(NULL), right_(NULL), rescan_params_(NULL), + left_rescan_params_(NULL), right_rescan_params_(NULL), mem_context_(NULL), + left_store_(), left_store_iter_(), left_store_group_idx_(), + above_left_group_params_(), above_right_group_params_(), + group_params_(), above_group_params_(), + last_row_(), last_batch_(), + right_cnt_(0), cur_group_idx_(0), left_store_read_(0), + above_group_idx_for_expand_(0), above_group_idx_for_read_(0), + above_group_size_(0), max_group_size_(0), + group_scan_size_(0), flags_(0) +{ + need_check_above_ = true; +} + +int ObGroupJoinBufffer::init(ObOperator *op, + const int64_t max_group_size, + const int64_t group_scan_size, + const common::ObIArray *rescan_params, + const common::ObIArray *left_rescan_params, + const common::ObIArray *right_rescan_params) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited())) { + ret = OB_INIT_TWICE; + LOG_WARN("batch info was already inited", KR(ret)); + } else if (OB_UNLIKELY(NULL != mem_context_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mem context should be null", KR(ret)); + } else if (OB_ISNULL(op)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("op is null", KR(ret), KP(op)); + } else if (OB_UNLIKELY(op->get_child_cnt() < 2)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("op should have at least 2 children", KR(ret), K(op->get_child_cnt())); + } else { + op_ = op; + spec_ = &op_->get_spec(); + ctx_ = &op_->get_exec_ctx(); + eval_ctx_ = &op_->get_eval_ctx(); + left_ = op_->get_child(); + right_ = op_->get_child(1); + right_cnt_ = op_->get_child_cnt() - 1; + max_group_size_ = max_group_size; + group_scan_size_ = group_scan_size; + rescan_params_ = rescan_params; + left_rescan_params_ = left_rescan_params; + right_rescan_params_ = right_rescan_params; + ObSQLSessionInfo *session = ctx_->get_my_session(); + uint64_t tenant_id =session->get_effective_tenant_id(); + lib::ContextParam param; + param.set_mem_attr(tenant_id, + ObModIds::OB_SQL_NLJ_CACHE, + ObCtxIds::WORK_AREA) + .set_properties(lib::USE_TL_PAGE_OPTIONAL); + if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { + LOG_WARN("create entity failed", KR(ret)); + } else if (OB_ISNULL(mem_context_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null memory entity", KR(ret)); + } else if (OB_FAIL(left_store_.init(UINT64_MAX, tenant_id, ObCtxIds::WORK_AREA))) { + LOG_WARN("init row store failed", KR(ret)); + } else { + left_store_.set_allocator(mem_context_->get_malloc_allocator()); + } + } + if (OB_SUCC(ret)) { + is_inited_ = true; + } + return ret; +} + +int ObGroupJoinBufffer::has_next_left_row(bool &has_next) +{ + int ret = OB_SUCCESS; + has_next = false; + if (left_store_iter_.is_valid() && left_store_iter_.has_next()) { + if (OB_UNLIKELY(left_store_read_ >= left_store_group_idx_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("left row read and group idx do not match", KR(ret), + K(left_store_read_), K(left_store_group_idx_.count())); + } else if (above_group_idx_for_read_ == left_store_group_idx_.at(left_store_read_)) { + // we are still reading results for the current rescan param, need to rescan right child + has_next = true; + } + } + return ret; +} + +int ObGroupJoinBufffer::init_above_group_params() +{ + int ret = OB_SUCCESS; + if (need_check_above_) { + need_check_above_ = false; + int64_t left_group_size = 0; + int64_t right_group_size = 0; + if (OB_FAIL(build_above_group_params(*left_rescan_params_, + above_left_group_params_, + left_group_size))) { + LOG_WARN("build above group params failed", KR(ret)); + } else if (OB_FAIL(build_above_group_params(*right_rescan_params_, + above_right_group_params_, + right_group_size))) { + LOG_WARN("build above group params failed", KR(ret)); + } else if (left_group_size > 0) { + // if above op only fills group params for our right child, + // then it is just single level group rescan. + if (OB_UNLIKELY(left_group_size != right_group_size && right_group_size > 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("left and right group sizes do not match", KR(ret), + K(left_group_size), K(right_group_size)); + } else { + is_multi_level_ = true; + } + } + } + if (is_multi_level_) { + LOG_TRACE("multi level group rescan", KR(ret), + K(spec_->get_id()), K(spec_->get_type()), K(is_multi_level_)); + } + return ret; +} + +int ObGroupJoinBufffer::fill_cur_row_group_param() +{ + int ret = OB_SUCCESS; + ObPhysicalPlanCtx *plan_ctx = ctx_->get_physical_plan_ctx(); + if (group_params_.empty() || cur_group_idx_ >= group_params_.at(0).count_) { + ret = OB_ERR_UNEXPECTED; + if (group_params_.empty()) { + LOG_WARN("empty group params", KR(ret), K(cur_group_idx_), K(group_params_.empty())); + } else { + LOG_WARN("row idx is unexpected", KR(ret), + K(cur_group_idx_), K(group_params_.count())); + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < group_params_.count(); i++) { + const ObDynamicParamSetter &rescan_param = rescan_params_->at(i); + int64_t param_idx = rescan_param.param_idx_; + ObExpr *dst = rescan_param.dst_; + ObDatum ¶m_datum = dst->locate_datum_for_write(*eval_ctx_); + ObSqlArrayObj &arr = group_params_.at(i); + if (OB_FAIL(param_datum.from_obj(arr.data_[cur_group_idx_], dst->obj_datum_map_))) { + LOG_WARN("fail to cast datum", K(ret)); + } else { + plan_ctx->get_param_store_for_update().at(param_idx) = arr.data_[cur_group_idx_]; + dst->set_evaluated_projected(*eval_ctx_); + } + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (is_multi_level_) { + for (int64_t i = 0; OB_SUCC(ret) && i < above_right_group_params_.count(); i++) { + if (NULL == above_right_group_params_.at(i)) { + // skip + } else { + // we need to fill group rescan params according to our group step + const ObDynamicParamSetter &rescan_param = right_rescan_params_->at(i); + int64_t param_idx = rescan_param.param_idx_; + ObExpr *dst = rescan_param.dst_; + ObDatum ¶m_datum = dst->locate_datum_for_write(*eval_ctx_); + ObSqlArrayObj *arr = above_right_group_params_.at(i); + if (OB_UNLIKELY(above_group_idx_for_read_ >= arr->count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected group idx", KR(ret), K(above_group_idx_for_read_), K(arr->count_)); + } else if (OB_FAIL(param_datum.from_obj(arr->data_[above_group_idx_for_read_], dst->obj_datum_map_))) { + LOG_WARN("cast datum failed", KR(ret)); + } else { + plan_ctx->get_param_store_for_update().at(param_idx) = arr->data_[above_group_idx_for_read_]; + dst->set_evaluated_projected(*eval_ctx_); + } + } + } + } + if (OB_SUCC(ret)) { + cur_group_idx_++; + } + return ret; +} + +int ObGroupJoinBufffer::rescan_left() +{ + int ret = OB_SUCCESS; + bool need_rescan = false; + if (is_multi_level_) { + // for multi level group rescan, we only rescan left if + // there is a new group of params for left child. + // note that if op_ is a child of SPF, param store can be + // filled with a new group before we finish processing the + // current group. in that case, we need to discard the old + // group and switch to the new batch. + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(*ctx_); + for (int64_t i = 0; !need_rescan && i < left_rescan_params_->count(); i++) { + int64_t param_idx = left_rescan_params_->at(i).param_idx_; + if (plan_ctx->get_param_store().at(param_idx).is_ext_sql_array()) { + need_rescan = true; + } + } + } else { + need_rescan = true; + } + if (OB_FAIL(ret)) { + // do nothing + } else if (need_rescan) { + if (OB_FAIL(set_above_group_size())) { + LOG_WARN("set above group size failed", KR(ret)); + } else if (OB_FAIL(left_->rescan())) { + LOG_WARN("rescan left failed", KR(ret), + K(left_->get_spec().get_id()), K(left_->op_name())); + } else { + is_left_end_ = false; + above_group_idx_for_expand_ = 0; + above_group_idx_for_read_ = 0; + } + } else { + above_group_idx_for_read_++; + } + return ret; +} + +int ObGroupJoinBufffer::rescan_right() +{ + int ret = OB_SUCCESS; + int save_ret = OB_SUCCESS; + if (skip_rescan_right_) { + skip_rescan_right_ = false; + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < right_cnt_; i++) { + int cur_ret = right_[i].rescan(); + if (OB_SUCC(cur_ret) || OB_ITER_END == cur_ret) { + if (0 == i) { + save_ret = cur_ret; + } + if (cur_ret != save_ret) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("rescan right children returned different codes", KR(ret), + KR(cur_ret), KR(save_ret), K(i), K(right_cnt_)); + } + } else { + ret = cur_ret; + LOG_WARN("rescan right failed", KR(ret), K(i), K(right_cnt_)); + } + } + } + return ret; +} + +int ObGroupJoinBufffer::fill_group_buffer() +{ + int ret = OB_SUCCESS; + if (!need_fill_group_buffer()) { + // do nothing + } else if (OB_FAIL(init_group_params())) { + LOG_WARN("init group params failed", KR(ret)); + } else if (is_left_end_) { + // we have fetched all rows from left + ret = OB_ITER_END; + } else { + common::ObSEArray left_params_backup; + common::ObSEArray right_params_backup; + if (OB_FAIL(backup_above_params(left_params_backup, right_params_backup))) { + LOG_WARN("backup above params failed", KR(ret)); + } else { + if (save_last_row_) { + if (OB_ISNULL(last_row_.get_store_row())) { + ret = OB_NOT_INIT; + LOG_WARN("store row is null", KR(ret), + K(save_last_row_), KP(last_row_.get_store_row())); + } else if (OB_FAIL(last_row_.restore(left_->get_spec().output_, *eval_ctx_))) { + LOG_WARN("restore last row failed", KR(ret)); + } + } + if (OB_SUCC(ret)) { + reset_buffer_state(); + if (OB_FAIL(last_row_.init( + mem_context_->get_malloc_allocator(), left_->get_spec().output_.count()))) { + LOG_WARN("failed to init right last row", KR(ret)); + } + } + bool ignore_end = false; + while (OB_SUCC(ret) && !is_full()) { + op_->clear_evaluated_flag(); + if (!rescan_params_->empty()) { + op_->set_pushdown_param_null(*rescan_params_); + } + if (OB_FAIL(get_next_left_row())) { + if (OB_ITER_END != ret) { + LOG_WARN("get next left row failed", KR(ret)); + } else { + is_left_end_ = true; + } + } else if (OB_FAIL(add_row_to_store())) { + LOG_WARN("add row to store failed", KR(ret)); + } else if (OB_FAIL(prepare_rescan_params())) { + LOG_WARN("prepare rescan params failed", KR(ret)); + } else if (OB_FAIL(deep_copy_dynamic_obj())) { + LOG_WARN("deep copy dynamic obj failed", KR(ret)); + } else { + ignore_end = true; + } + } + if (OB_SUCC(ret)) { + if (!rescan_params_->empty()) { + op_->set_pushdown_param_null(*rescan_params_); + } + if (OB_FAIL(last_row_.shadow_copy(left_->get_spec().output_, *eval_ctx_))) { + LOG_WARN("shadow copy last left row failed", KR(ret)); + } else { + save_last_row_ = true; + } + } + if (OB_SUCC(ret) || (ignore_end && OB_ITER_END == ret)) { + ret = OB_SUCCESS; + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(*ctx_); + if (OB_FAIL(left_store_.finish_add_row(false))) { + LOG_WARN("finish add row to row store failed", KR(ret)); + } else if (OB_FAIL(left_store_.begin(left_store_iter_))) { + LOG_WARN("begin iterator for chunk row store failed", KR(ret)); + } else if (OB_FAIL(bind_group_params_to_store())) { + LOG_WARN("bind group params to store failed", KR(ret)); + } else if (OB_FAIL(rescan_right())) { + if (OB_ITER_END == ret) { + ret = OB_ERR_UNEXPECTED; + } + LOG_WARN("rescan right failed", KR(ret)); + } else { + skip_rescan_right_ = true; + } + } + int save_ret = ret; + ret = OB_SUCCESS; + if (OB_FAIL(restore_above_params(left_params_backup, + right_params_backup))) { + LOG_WARN("restore above params failed", KR(ret), KR(save_ret)); + } else { + ret = save_ret; + } + } + } + return ret; +} + +int ObGroupJoinBufffer::batch_fill_group_buffer(const int64_t max_row_cnt, + const ObBatchRows *&batch_rows) +{ + int ret = OB_SUCCESS; + if (!need_fill_group_buffer()) { + // do nothing + } else if (OB_FAIL(init_group_params())) { + LOG_WARN("init group params failed", KR(ret)); + } else if (is_left_end_) { + ret = OB_ITER_END; + } else { + bool ignore_end = false; + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_); + if (save_last_batch_) { + if (!last_batch_.is_inited()) { + ret = OB_NOT_INIT; + LOG_WARN("last batch is not inited", KR(ret), + K(save_last_batch_), K(last_batch_.is_inited())); + } else { + last_batch_.to_exprs(*eval_ctx_); + save_last_batch_ = false; + } + } + if (OB_SUCC(ret)) { + reset_buffer_state(); + if (OB_FAIL(last_batch_.init(&left_->get_spec().output_, + &mem_context_->get_arena_allocator(), + spec_->max_batch_size_))) { + LOG_WARN("init batch failed", KR(ret)); + } + } + while (OB_SUCC(ret) && !is_full()) { + op_->clear_evaluated_flag(); + if (save_last_batch_) { + last_batch_.to_exprs(*eval_ctx_); + save_last_batch_ = false; + } + if (!rescan_params_->empty()) { + op_->set_pushdown_param_null(*rescan_params_); + } + if (OB_FAIL(get_next_left_batch(max_row_cnt, batch_rows))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next left batch failed", KR(ret)); + } + } else { + for (int64_t l_idx = 0; OB_SUCC(ret) && l_idx < batch_rows->size_; l_idx++) { + if (batch_rows->skip_->exist(l_idx)) { continue; } + batch_info_guard.set_batch_idx(l_idx); + batch_info_guard.set_batch_size(batch_rows->size_); + if (OB_FAIL(add_row_to_store())) { + LOG_WARN("store left row failed", KR(ret)); + } else if (OB_FAIL(prepare_rescan_params())) { + LOG_WARN("prepare rescan params failed", KR(ret)); + } else if (OB_FAIL(deep_copy_dynamic_obj())) { + LOG_WARN("deep copy dynamic obj failed", KR(ret)); + } + } + ignore_end = true; + } + if (OB_SUCC(ret)) { + if (!rescan_params_->empty()) { + op_->set_pushdown_param_null(*rescan_params_); + } + if (batch_rows->size_ == 0 && batch_rows->end_) { + // do nothing + } else { + last_batch_.from_exprs(*eval_ctx_, batch_rows->skip_, batch_rows->size_); + save_last_batch_ = true; + } + } + op_->clear_evaluated_flag(); + } + if (OB_SUCC(ret) || (ignore_end && OB_ITER_END == ret)) { + ret = OB_SUCCESS; + if (left_store_.get_row_cnt() <= 0) { + // this could happen if we have skipped all rows + ret = OB_ITER_END; + } else if (OB_FAIL(left_store_.finish_add_row(false))) { + LOG_WARN("finish add row to row store failed", KR(ret)); + } else if (OB_FAIL(left_store_.begin(left_store_iter_))) { + LOG_WARN("begin iterator for chunk row store failed", KR(ret)); + } else if (OB_FAIL(bind_group_params_to_store())) { + LOG_WARN("bind group params to store failed", KR(ret)); + } else if (OB_FAIL(rescan_right())) { + if (OB_ITER_END == ret) { + ret = OB_ERR_UNEXPECTED; + } + LOG_WARN("rescan right failed", KR(ret)); + } else { + skip_rescan_right_ = true; + } + } + } + return ret; +} + +int ObGroupJoinBufffer::get_next_row_from_store() +{ + int ret = OB_SUCCESS; + if (left_store_iter_.is_valid() && left_store_iter_.has_next()) { + if (OB_UNLIKELY(left_store_read_ >= left_store_group_idx_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("left row read and group idx do not match", KR(ret), + K(left_store_read_), K(left_store_group_idx_.count())); + } else if (above_group_idx_for_read_ == left_store_group_idx_.at(left_store_read_)) { + // we are still reading results for the current rescan param, + // need to rescan right child + if (OB_FAIL(left_store_iter_.get_next_row(left_->get_spec().output_, *eval_ctx_))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next row from iter failed", KR(ret)); + } + } else { + left_store_read_++; + } + } else { + // we have finished reading results for the current rescan param + ret = OB_ITER_END; + } + } else { + ret = OB_ITER_END; + } + return ret; +} + +int ObGroupJoinBufffer::get_next_batch_from_store(int64_t max_rows, int64_t &read_rows) +{ + int ret = OB_SUCCESS; + read_rows = 0; + if (left_store_iter_.is_valid() && left_store_iter_.has_next()) { + if (OB_UNLIKELY(left_store_read_ >= left_store_group_idx_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("left row read and group idx do not match", KR(ret), + K(left_store_read_), K(left_store_group_idx_.count())); + } else { + if (is_multi_level_) { + // left_store_iter_ = [1, 2, 3, 4, 5, 6, 7, 8] + // left_store_batch_idx_ = [0, 0, 0, 0, 1, 1, 1, 1] + // left_store_read_ = 0 + // max_rows = 5 + // + // As shown above, get next batch from left_store_iter_ with max_rows=5 + // can return rows 1 ~ 5 where row 5 does not belong to current rescan. + // Thus, we need to check and rewrite max_rows to 4 in this case. + int64_t tmp_max_rows = 0; + if (above_group_idx_for_read_ == left_store_group_idx_.at(left_store_read_)) { + for (int64_t i = left_store_read_; i < left_store_group_idx_.count(); i++) { + if (tmp_max_rows >= max_rows) { + break; + } else if (above_group_idx_for_read_ != left_store_group_idx_.at(i)) { + break; + } else { + tmp_max_rows++; + } + } + } + max_rows = tmp_max_rows; + } + if (max_rows > 0) { + // we are still reading results for the current rescan param, + // need to rescan right child + if (OB_FAIL(left_store_iter_.get_next_batch(left_->get_spec().output_, + *eval_ctx_, + max_rows, + read_rows))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next batch from iter failed", KR(ret)); + } + } else { + left_store_read_ += read_rows; + } + } else { + ret = OB_ITER_END; + } + } + } else { + ret = OB_ITER_END; + } + return ret; +} + +void ObGroupJoinBufffer::destroy() +{ + left_store_iter_.reset(); + left_store_.reset(); + left_store_group_idx_.destroy(); + above_left_group_params_.destroy(); + above_right_group_params_.destroy(); + last_row_.reset(); + if (NULL != mem_context_) { + DESTROY_CONTEXT(mem_context_); + mem_context_ = NULL; + } +} + +int ObGroupJoinBufffer::init_group_params() +{ + int ret = OB_SUCCESS; + if (!group_params_.empty()) { + for (int64_t i = 0; i < group_params_.count(); ++i) { + group_params_.at(i).count_ = 0; + } + } else if (OB_FAIL(group_params_.allocate_array(ctx_->get_allocator(), + rescan_params_->count()))) { + LOG_WARN("allocate group params array failed", KR(ret), K(rescan_params_->count())); + } else { + int64_t obj_buf_size = sizeof(ObObjParam) * max_group_size_; + for (int64_t i = 0; OB_SUCC(ret) && i < group_params_.count(); ++i) { + ObExpr *dst_expr = rescan_params_->at(i).dst_; + void *buf = ctx_->get_allocator().alloc(obj_buf_size); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", KR(ret), K(obj_buf_size)); + } else { + group_params_.at(i).data_ = reinterpret_cast(buf); + group_params_.at(i).count_ = 0; + group_params_.at(i).element_.set_meta_type(dst_expr->obj_meta_); + } + } + } + if (OB_FAIL(ret) || 0 == right_rescan_params_->count()) { + // do nothing + } else if (!above_group_params_.empty()) { + for (int64_t i = 0; i < above_group_params_.count(); i++) { + above_group_params_.at(i).count_ = 0; + } + } else if (OB_FAIL(above_group_params_.allocate_array( + ctx_->get_allocator(), right_rescan_params_->count()))) { + LOG_WARN("allocate above group params array failed", KR(ret), + K(right_rescan_params_->count())); + } else { + int64_t obj_buf_size = sizeof(ObObjParam) * max_group_size_; + for (int64_t i = 0; OB_SUCC(ret) && i < above_group_params_.count(); ++i) { + ObExpr *dst_expr = right_rescan_params_->at(i).dst_; + void *buf = ctx_->get_allocator().alloc(obj_buf_size); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed", KR(ret), K(obj_buf_size)); + } else { + above_group_params_.at(i).data_ = reinterpret_cast(buf); + above_group_params_.at(i).count_ = 0; + above_group_params_.at(i).element_.set_meta_type(dst_expr->obj_meta_); + } + } + } + return ret; +} + +int ObGroupJoinBufffer::deep_copy_dynamic_obj() +{ + int ret = OB_SUCCESS; + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(*ctx_); + ParamStore ¶m_store = plan_ctx->get_param_store_for_update(); + if (OB_ISNULL(mem_context_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("mem entity is not inited", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < rescan_params_->count(); ++i) { + const ObDynamicParamSetter &rescan_param = rescan_params_->at(i); + int64_t param_idx = rescan_param.param_idx_; + if (OB_FAIL(ob_write_obj(mem_context_->get_arena_allocator(), + param_store.at(param_idx), + group_params_.at(i).data_[group_params_.at(i).count_]))) { + LOG_WARN("deep copy dynamic param failed", KR(ret), K(i), K(param_idx)); + } else { + group_params_.at(i).count_++; + } + } + if (OB_FAIL(ret) || !is_multi_level_) { + // do nothing + } else { + if (OB_UNLIKELY(above_group_params_.count() != above_right_group_params_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected group params count", KR(ret), + K(above_group_params_.count()), + K(above_right_group_params_.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < above_group_params_.count(); i++) { + ObSqlArrayObj *arr = above_right_group_params_.at(i); + if (NULL == arr) { + // skip + } else if (OB_FAIL(ob_write_obj( + mem_context_->get_arena_allocator(), + arr->data_[above_group_idx_for_expand_], + above_group_params_.at(i).data_[above_group_params_.at(i).count_]))) { + LOG_WARN("deep copy dynamic param failed", KR(ret), K(i), K(above_group_idx_for_expand_)); + } else { + ++above_group_params_.at(i).count_; + } + } + } + return ret; +} + +int ObGroupJoinBufffer::bind_group_params_to_store() +{ + int ret = OB_SUCCESS; + int64_t param_cnt = rescan_params_->count(); + ParamStore ¶m_store = GET_PHY_PLAN_CTX(*ctx_)->get_param_store_for_update(); + if (OB_UNLIKELY(param_cnt != group_params_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param count is invalid", KR(ret), K(param_cnt), K(group_params_.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < param_cnt; i++) { + const ObDynamicParamSetter &rescan_param = rescan_params_->at(i); + int64_t param_idx = rescan_param.param_idx_; + int64_t array_obj_addr = reinterpret_cast(&group_params_.at(i)); + param_store.at(param_idx).set_extend(array_obj_addr, T_EXT_SQL_ARRAY); + } + if (OB_FAIL(ret)) { + // do nothing + } else if (is_multi_level_) { + if (OB_UNLIKELY(above_group_params_.count() != right_rescan_params_->count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param counts do not match", KR(ret), + K(above_group_params_.count()), + K(right_rescan_params_->count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < right_rescan_params_->count(); i++) { + int64_t param_idx = right_rescan_params_->at(i).param_idx_; + int64_t array_obj_addr = reinterpret_cast(&above_group_params_.at(i)); + param_store.at(param_idx).set_extend(array_obj_addr, T_EXT_SQL_ARRAY); + } + } + return ret; +} + +int ObGroupJoinBufffer::prepare_rescan_params() +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < rescan_params_->count(); i++) { + if (OB_FAIL(rescan_params_->at(i).set_dynamic_param(*eval_ctx_))) { + LOG_WARN("set dynamic param failed", KR(ret)); + } + } + return ret; +} + +int ObGroupJoinBufffer::get_next_left_row() +{ + int ret = OB_SUCCESS; + bool got_row = false; + while (OB_SUCC(ret) && !got_row) { + op_->clear_evaluated_flag(); + if (OB_FAIL(left_->get_next_row())) { + if (OB_ITER_END != ret) { + LOG_WARN("get next row from left failed", KR(ret)); + } else if (is_multi_level_) { + // for multi level group rescan, left_ may output more than 1 iterators, + // and we need to call left_->rescan() to switch to next iterator + ret = OB_SUCCESS; + if ((above_group_idx_for_expand_ + 1) >= above_group_size_) { + // wait for parent op to fill next group params + ret = OB_ITER_END; + } else { + above_group_idx_for_expand_++; + ObPhysicalPlanCtx *plan_ctx = ctx_->get_physical_plan_ctx(); + for (int64_t i = 0; OB_SUCC(ret) && i < above_left_group_params_.count(); i++) { + ObSqlArrayObj *array_obj = above_left_group_params_.at(i); + if (NULL == array_obj) { + // skip + } else { + const ObDynamicParamSetter &rescan_param = left_rescan_params_->at(i); + int64_t param_idx = rescan_param.param_idx_; + ObExpr *dst = rescan_param.dst_; + ObDatum ¶m_datum = dst->locate_datum_for_write(*eval_ctx_); + if (OB_FAIL(param_datum.from_obj(array_obj->data_[above_group_idx_for_expand_], + dst->obj_datum_map_))) { + LOG_WARN("cast datum failed", KR(ret)); + } else { + plan_ctx->get_param_store_for_update().at(param_idx) = + array_obj->data_[above_group_idx_for_expand_]; + dst->set_evaluated_projected(*eval_ctx_); + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(left_->rescan())) { + if (OB_ITER_END != ret) { + LOG_WARN("rescan left failed", KR(ret)); + } + } + } + } + } else { + got_row = true; + } + } + return ret; +} + +int ObGroupJoinBufffer::get_next_left_batch(const int64_t max_row_cnt, const ObBatchRows *&batch_rows) +{ + int ret = OB_SUCCESS; + bool got_row = false; + if (OB_FAIL(left_->get_next_batch(max_row_cnt, batch_rows))) { + LOG_WARN("get next batch from left failed", KR(ret)); + } else if (batch_rows->end_) { + if (is_multi_level_) { + // for multi level group rescan, left_ may output more than 1 iterators, + // and we need to call left_->rescan() to switch to next iterator + if ((above_group_idx_for_expand_ + 1) >= above_group_size_) { + // wait for parent op to fill next group params + ret = OB_ITER_END; + } else { + above_group_idx_for_expand_++; + ObPhysicalPlanCtx *plan_ctx = ctx_->get_physical_plan_ctx(); + for (int64_t i = 0; OB_SUCC(ret) && i < above_left_group_params_.count(); i++) { + ObSqlArrayObj *array_obj = above_left_group_params_.at(i); + if (NULL == array_obj) { + // skip + } else { + const ObDynamicParamSetter &rescan_param = left_rescan_params_->at(i); + int64_t param_idx = rescan_param.param_idx_; + ObExpr *dst = rescan_param.dst_; + ObDatum ¶m_datum = dst->locate_datum_for_write(*eval_ctx_); + if (OB_FAIL(param_datum.from_obj(array_obj->data_[above_group_idx_for_expand_], dst->obj_datum_map_))) { + LOG_WARN("cast datum failed", KR(ret)); + } else { + plan_ctx->get_param_store_for_update().at(param_idx) = + array_obj->data_[above_group_idx_for_expand_]; + dst->set_evaluated_projected(*eval_ctx_); + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(left_->rescan())) { + if (OB_ITER_END != ret) { + LOG_WARN("rescan left failed", KR(ret)); + } + } + } + } else { + ret = OB_ITER_END; + } + } + return ret; +} + +int ObGroupJoinBufffer::add_row_to_store() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(left_store_.add_row(left_->get_spec().output_, eval_ctx_))) { + LOG_WARN("add row failed", KR(ret)); + } else if (OB_FAIL(left_store_group_idx_.push_back(above_group_idx_for_expand_))) { + LOG_WARN("add index failed", KR(ret)); + } + return ret; +} + +int ObGroupJoinBufffer::build_above_group_params( + const common::ObIArray &above_rescan_params, + common::ObIArray &above_group_params, + int64_t &group_size) +{ + int ret = OB_SUCCESS; + group_size = 0; + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(*ctx_); + for (int64_t i = 0; OB_SUCC(ret) && i < above_rescan_params.count(); i++) { + int64_t param_idx = above_rescan_params.at(i).param_idx_; + const ObObjParam &obj_param = plan_ctx->get_param_store().at(param_idx); + ObSqlArrayObj *array_obj = NULL; + if (obj_param.is_ext_sql_array()) { + array_obj = reinterpret_cast(obj_param.get_ext()); + if (0 == group_size) { + group_size = array_obj->count_; + } else if (OB_UNLIKELY(group_size != array_obj->count_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group sizes do not match", KR(ret), + K(group_size), K(array_obj->count_)); + } + } + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(above_group_params.push_back(array_obj))) { + LOG_WARN("push array obj failed", KR(ret), K(i), KP(array_obj)); + } + } + return ret; +} + +int ObGroupJoinBufffer::set_above_group_size() { + int ret = OB_SUCCESS; + if (is_multi_level_) { + above_group_size_ = 0; + for (int i = 0; i < above_left_group_params_.count(); i++) { + if (NULL == above_left_group_params_.at(i)) { + // skip + } else { + above_group_size_ = above_left_group_params_.at(i)->count_; + break; + } + } + if (0 == above_group_size_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("above group size is invalid", KR(ret), K(above_group_size_)); + } + } + return ret; +} + +void ObGroupJoinBufffer::reset_buffer_state() +{ + cur_group_idx_ = 0; + left_store_read_ = 0; + left_store_iter_.reset(); + left_store_.reset(); + left_store_read_ = 0; + left_store_group_idx_.reuse(); + last_row_.reset(); + last_batch_.reset(); + save_last_row_ = false; + mem_context_->get_arena_allocator().reset(); +} + +int ObGroupJoinBufffer::backup_above_params(common::ObIArray &left_params_backup, + common::ObIArray &right_params_backup) +{ + int ret = OB_SUCCESS; + if (is_multi_level_) { + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(*ctx_); + left_params_backup.reuse(); + right_params_backup.reuse(); + for (int64_t i = 0; OB_SUCC(ret) && i < left_rescan_params_->count(); i++) { + int64_t param_idx = left_rescan_params_->at(i).param_idx_; + const ObObjParam &obj_param = plan_ctx->get_param_store().at(param_idx); + if (OB_FAIL(left_params_backup.push_back(obj_param))) { + LOG_WARN("push obj param failed", KR(ret), K(param_idx), K(obj_param)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < right_rescan_params_->count(); i++) { + int64_t param_idx = right_rescan_params_->at(i).param_idx_; + const ObObjParam &obj_param = plan_ctx->get_param_store().at(param_idx); + if (OB_FAIL(right_params_backup.push_back(obj_param))) { + LOG_WARN("push obj param failed", KR(ret), K(param_idx), K(obj_param)); + } + } + } + return ret; +} + +int ObGroupJoinBufffer::restore_above_params(common::ObIArray &left_params_backup, + common::ObIArray &right_params_backup) +{ + int ret = OB_SUCCESS; + if (is_multi_level_) { + if (OB_UNLIKELY(left_rescan_params_->count() != left_params_backup.count() + || right_rescan_params_->count() != right_params_backup.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("backup params count do not match", KR(ret), + K(left_rescan_params_->count()), K(left_params_backup.count()), + K(right_rescan_params_->count()), K(right_params_backup.count())); + } + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(*ctx_); + for (int64_t i = 0; OB_SUCC(ret) && i < left_rescan_params_->count(); i++) { + int64_t param_idx = left_rescan_params_->at(i).param_idx_; + plan_ctx->get_param_store_for_update().at(param_idx) = left_params_backup.at(i); + } + for (int64_t i = 0; OB_SUCC(ret) && i < right_rescan_params_->count(); i++) { + int64_t param_idx = right_rescan_params_->at(i).param_idx_; + plan_ctx->get_param_store_for_update().at(param_idx) = right_params_backup.at(i); + } + } + return ret; +} +} // end namespace sql +} // end namespace oceanbase diff --git a/src/sql/engine/basic/ob_group_join_buffer.h b/src/sql/engine/basic/ob_group_join_buffer.h new file mode 100644 index 0000000000..0ee6e95a49 --- /dev/null +++ b/src/sql/engine/basic/ob_group_join_buffer.h @@ -0,0 +1,169 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_BASIC_OB_GROUP_JOIN_BUFFER_H_ +#define OCEANBASE_BASIC_OB_GROUP_JOIN_BUFFER_H_ + +#include "sql/engine/basic/ob_chunk_datum_store.h" +#include "sql/engine/ob_operator.h" + +namespace oceanbase +{ +namespace sql +{ +struct ObBatchRowDatums +{ + ObBatchRowDatums() { reset(); } + int init(const ObExprPtrIArray *exprs, common::ObIAllocator *alloc, int32_t batch_size); + void reset() + { + alloc_ = NULL; + exprs_ = NULL; + batch_size_ = 0; + datums_ = NULL; + skip_ = NULL; + size_ = 0; + saved_size_ = 0; + is_inited_ = false; + } + void from_exprs(ObEvalCtx &ctx, ObBitVector *skip, int64_t size); + void extend_save(ObEvalCtx &ctx, int64_t size); + void to_exprs(ObEvalCtx &ctx); + void to_exprs(ObEvalCtx &ctx, int64_t from_idx, int64_t to_idx); + ObDatum &get_datum(int64_t row_id, int64_t col_id) + { + return datums_[col_id * batch_size_ + row_id]; + } + bool is_inited() const { return is_inited_; } + +public: + common::ObIAllocator *alloc_; + const ObExprPtrIArray *exprs_; + int32_t batch_size_; + ObDatum *datums_; + ObBitVector *skip_; + int32_t size_; + int32_t saved_size_; // record the saved size, include extend saved size +private: + bool is_inited_; +}; + +class ObGroupJoinBufffer +{ +public: + ObGroupJoinBufffer(); + ~ObGroupJoinBufffer() {} // does not free memory + int init(ObOperator *op, + const int64_t max_group_size, + const int64_t group_scan_size, + const common::ObIArray *rescan_params, + const common::ObIArray *left_rescan_params, + const common::ObIArray *right_rescan_params); + bool is_inited() const { return is_inited_; } + bool is_full() const { return left_store_.get_row_cnt() >= group_scan_size_; } + bool need_fill_group_buffer() { return !(left_store_iter_.is_valid() && left_store_iter_.has_next()); } + bool is_multi_level() const { return is_multi_level_; } + int has_next_left_row(bool &has_next); + int init_above_group_params(); + int fill_cur_row_group_param(); + int rescan_left(); + int rescan_right(); + int fill_group_buffer(); + int batch_fill_group_buffer(const int64_t max_row_cnt, const ObBatchRows *&batch_rows); + int get_next_row_from_store(); + int get_next_batch_from_store(int64_t max_rows, int64_t &read_rows); + void destroy(); +private: + int init_group_params(); + int deep_copy_dynamic_obj(); + int bind_group_params_to_store(); + int prepare_rescan_params(); + int get_next_left_row(); + int get_next_left_batch(const int64_t max_row_cnt, const ObBatchRows *&batch_rows); + int add_row_to_store(); + int build_above_group_params(const common::ObIArray &above_rescan_params, + common::ObIArray &above_group_params, + int64_t &group_size); + int set_above_group_size(); + void reset_buffer_state(); + int backup_above_params(common::ObIArray &left_params_backup, + common::ObIArray &right_params_backup); + int restore_above_params(common::ObIArray &left_params_backup, + common::ObIArray &right_params_backup); + +private: + ObOperator *op_; + const ObOpSpec *spec_; + ObExecContext *ctx_; + ObEvalCtx *eval_ctx_; + ObOperator *left_; + ObOperator * right_; + const common::ObIArray *rescan_params_; + const common::ObIArray *left_rescan_params_; + const common::ObIArray *right_rescan_params_; + lib::MemoryContext mem_context_; + // buffer for rows read from left child + ObChunkDatumStore left_store_; + ObChunkDatumStore::Iterator left_store_iter_; + // for multi level batch rescan + // NLJ 1 + // / \ + // TSC 1 NLJ 2 + // / \ + // TSC 2 TSC 3 + // During NLJ 2's rescan, NLJ 1 may have supplied a batch of params to TSC 2 and TSC 3. + // Thus, NLJ 2 need to keep track of NLJ 1's batch rescan pace and make sure only output + // rows corresponding to NLJ 2's current rescan. + // + // store batch index corresponding to NLJ above this op + common::ObSEArray left_store_group_idx_; + // for NLJ 2, above_left_batch_params_ stores batch rescan params used by TSC 2 and supplied by NLJ 1 + common::ObSEArray above_left_group_params_; + // for NLJ 2, above_right_batch_params_ stores batch rescan params used by TSC 3 and supplied by NLJ 1 + common::ObSEArray above_right_group_params_; + // batch rescan params supplied by this op + common::ObArrayWrap group_params_; + // for NLJ 2, we need to rewrite params in above_right_batch_params_ and align them with our rescan pace, + // above_bnlj_params_ stores batch params supplied by NLJ 1 and overwritten by NLJ 2 + common::ObArrayWrap above_group_params_; + ObChunkDatumStore::ShadowStoredRow last_row_; + ObBatchRowDatums last_batch_; + int64_t right_cnt_; + int64_t cur_group_idx_; + // rows read from left_store_iter_ + int64_t left_store_read_; + // index used when filling group buffer, + // see fill_group_buffer() and batch_fill_group_buffer() + int64_t above_group_idx_for_expand_; + // index used when reading from right child, + // see get_next_row_from_store() and get_next_batch_from_store() + int64_t above_group_idx_for_read_; + int64_t above_group_size_; + int64_t max_group_size_; + int64_t group_scan_size_; + union { + uint64_t flags_; + struct { + uint64_t is_inited_ : 1; + uint64_t need_check_above_ : 1; + uint64_t is_multi_level_ : 1; + uint64_t is_left_end_ : 1; + uint64_t save_last_row_ : 1; + uint64_t save_last_batch_ : 1; + uint64_t skip_rescan_right_ : 1; + uint64_t reserved_ : 57; + }; + }; +}; +} // end namespace sql +} // end namespace oceanbase +#endif // OCEANBASE_BASIC_OB_GROUP_JOIN_BUFFER_H_ diff --git a/src/sql/engine/join/ob_nested_loop_join_op.cpp b/src/sql/engine/join/ob_nested_loop_join_op.cpp index df4315e663..a1ed526fc3 100644 --- a/src/sql/engine/join/ob_nested_loop_join_op.cpp +++ b/src/sql/engine/join/ob_nested_loop_join_op.cpp @@ -23,7 +23,10 @@ namespace sql { OB_SERIALIZE_MEMBER((ObNestedLoopJoinSpec, ObBasicNestedLoopJoinSpec), - use_group_, left_group_size_, left_expr_ids_in_other_cond_); + group_rescan_, group_size_, + left_expr_ids_in_other_cond_, + left_rescan_params_, + right_rescan_params_); ObNestedLoopJoinOp::ObNestedLoopJoinOp(ObExecContext &exec_ctx, const ObOpSpec &spec, @@ -35,8 +38,8 @@ ObNestedLoopJoinOp::ObNestedLoopJoinOp(ObExecContext &exec_ctx, batch_state_(JS_FILL_LEFT), save_last_batch_(false), batch_mem_ctx_(NULL), stored_rows_(NULL), left_brs_(NULL), left_matched_(NULL), need_switch_iter_(false), iter_end_(false), op_max_batch_size_(0), - max_group_size_(BNLJ_DEFAULT_GROUP_SIZE), - bnlj_cur_idx_(0) + max_group_size_(OB_MAX_BULK_JOIN_ROWS), + group_join_buffer_() { state_operation_func_[JS_JOIN_END] = &ObNestedLoopJoinOp::join_end_operate; state_function_func_[JS_JOIN_END][FT_ITER_GOING] = NULL; @@ -61,8 +64,8 @@ int ObNestedLoopJoinOp::inner_open() LOG_WARN("failed to open in base class", K(ret)); } if (OB_SUCC(ret) && is_vectorized()) { - if (MY_SPEC.use_group_) { - max_group_size_ = BNLJ_DEFAULT_GROUP_SIZE + MY_SPEC.plan_->get_batch_size(); + if (MY_SPEC.group_rescan_) { + max_group_size_ = OB_MAX_BULK_JOIN_ROWS + MY_SPEC.plan_->get_batch_size(); } if (OB_ISNULL(batch_mem_ctx_)) { ObSQLSessionInfo *session = ctx_.get_my_session(); @@ -118,15 +121,25 @@ int ObNestedLoopJoinOp::inner_open() &(batch_mem_ctx_->get_arena_allocator()), MY_SPEC.max_batch_size_))) { LOG_WARN("fail to init batch", K(ret)); - } else if (MY_SPEC.use_group_ || MY_SPEC.enable_px_batch_rescan_) { + } else if (MY_SPEC.enable_px_batch_rescan_) { if (OB_FAIL(last_save_batch_.init(&left_->get_spec().output_, - &batch_mem_ctx_->get_arena_allocator(), - MY_SPEC.max_batch_size_))) { + &batch_mem_ctx_->get_arena_allocator(), + MY_SPEC.max_batch_size_))) { LOG_WARN("fail to init batch", K(ret)); } } } } + if (OB_SUCC(ret) && MY_SPEC.group_rescan_) { + if (OB_FAIL(group_join_buffer_.init(this, + max_group_size_, + MY_SPEC.group_size_, + &MY_SPEC.rescan_params_, + &MY_SPEC.left_rescan_params_, + &MY_SPEC.right_rescan_params_))) { + LOG_WARN("init batch info failed", KR(ret)); + } + } return ret; } //NLJ has its own switch_iterator @@ -156,12 +169,19 @@ int ObNestedLoopJoinOp::rescan() //NLJ's rescan should only drive left child's rescan, //the right child's rescan is defer to rescan_right_operator() driven by get_next_row(); defered_right_rescan_ = true; - if (OB_FAIL(left_->rescan())) { - LOG_WARN("rescan left child operator failed", K(ret), "child op_type", left_->op_name()); - } - if (OB_SUCC(ret)) { - if (OB_FAIL(inner_rescan())) { - LOG_WARN("failed to inner rescan", K(ret)); + if (!MY_SPEC.group_rescan_) { + if (OB_FAIL(left_->rescan())) { + LOG_WARN("rescan left child operator failed", KR(ret), "child op_type", left_->op_name()); + } else if (OB_FAIL(inner_rescan())) { + LOG_WARN("failed to inner rescan", KR(ret)); + } + } else { + if (OB_FAIL(group_join_buffer_.init_above_group_params())) { + LOG_WARN("init above bnlj params failed", KR(ret)); + } else if (OB_FAIL(group_join_buffer_.rescan_left())) { + LOG_WARN("rescan left failed", KR(ret)); + } else if (OB_FAIL(inner_rescan())) { + LOG_WARN("inner rescan failed", KR(ret)); } } @@ -268,33 +288,6 @@ int ObNestedLoopJoinOp::fill_cur_row_rescan_param() return ret; } -int ObNestedLoopJoinOp::fill_cur_row_bnlj_param() -{ - int ret = OB_SUCCESS; - ObPhysicalPlanCtx *plan_ctx = ctx_.get_physical_plan_ctx(); - if (bnlj_params_.empty() || bnlj_cur_idx_ >= bnlj_params_.at(0).count_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("row idx is unexpected", K(ret), - K(bnlj_cur_idx_), K(bnlj_params_.at(0).count_)); - } else { - int64_t param_cnt = bnlj_params_.count(); - for (int64_t i = 0; OB_SUCC(ret) && i < param_cnt; ++i) { - const ObDynamicParamSetter &rescan_param = get_spec().rescan_params_.at(i); - int64_t param_idx = rescan_param.param_idx_; - ObExpr *dst = rescan_param.dst_; - ObDatum ¶m_datum = dst->locate_datum_for_write(eval_ctx_); - ObSqlArrayObj &arr = bnlj_params_.at(i); - if (OB_FAIL(param_datum.from_obj(arr.data_[bnlj_cur_idx_], dst->obj_datum_map_))) { - LOG_WARN("fail to cast datum", K(ret)); - } else { - plan_ctx->get_param_store_for_update().at(param_idx) = arr.data_[bnlj_cur_idx_]; - dst->set_evaluated_projected(eval_ctx_); - } - } - } - return ret; -} - int ObNestedLoopJoinOp::join_row_with_semi_join() { int ret = OB_SUCCESS; @@ -356,7 +349,7 @@ int ObNestedLoopJoinOp::join_end_func_end() int ObNestedLoopJoinOp::read_left_operate() { int ret = OB_SUCCESS; - if (MY_SPEC.use_group_ || MY_SPEC.enable_px_batch_rescan_) { + if (MY_SPEC.group_rescan_ || MY_SPEC.enable_px_batch_rescan_) { if (OB_FAIL(group_read_left_operate()) && OB_ITER_END != ret) { LOG_WARN("failed to read left group", K(ret)); } @@ -408,230 +401,165 @@ int ObNestedLoopJoinOp::rescan_right_operator() } } else { brs_holder_.reset(); - if (MY_SPEC.use_group_ && !MY_SPEC.enable_px_batch_rescan_) { - if (OB_FAIL(fill_cur_row_bnlj_param())) { - LOG_WARN("fill bnlj param failed", K(ret)); - } else { - bnlj_cur_idx_++; - } - } } } return ret; } -int ObNestedLoopJoinOp::init_bnlj_params() -{ - int ret = OB_SUCCESS; - if (!bnlj_params_.empty()) { - //to reuse bnlj param buffer - for (int64_t i = 0; i < bnlj_params_.count(); ++i) { - bnlj_params_.at(i).count_ = 0; - } - } else if (OB_FAIL(bnlj_params_.allocate_array(ctx_.get_allocator(), - get_spec().rescan_params_.count()))) { - LOG_WARN("allocate bnlj params failed", K(ret), K(get_spec().rescan_params_.count())); - } else { - int64_t obj_buf_size = sizeof(ObObjParam) * max_group_size_; - for (int64_t i = 0; OB_SUCC(ret) && i < bnlj_params_.count(); ++i) { - ObExpr *dst_expr = get_spec().rescan_params_.at(i).dst_; - void *buf = ctx_.get_allocator().alloc(obj_buf_size); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory", K(ret), K(obj_buf_size)); - } else { - bnlj_params_.at(i).data_ = reinterpret_cast(buf); - bnlj_params_.at(i).count_ = 0; - bnlj_params_.at(i).element_.set_meta_type(dst_expr->obj_meta_); - } - } - } - return ret; -} - -int ObNestedLoopJoinOp::bind_bnlj_param_to_store() -{ - int ret = OB_SUCCESS; - int64_t param_cnt = get_spec().rescan_params_.count(); - ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - ParamStore ¶m_store = plan_ctx->get_param_store_for_update(); - if (OB_UNLIKELY(param_cnt != bnlj_params_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("bnlj param count is invalid", K(ret), K(param_cnt), K(bnlj_params_.count())); - } - for (int64_t i = 0; OB_SUCC(ret) && i < param_cnt; ++i) { - const ObDynamicParamSetter &rescan_param = get_spec().rescan_params_.at(i); - int64_t param_idx = rescan_param.param_idx_; - int64_t array_obj_addr = reinterpret_cast(&bnlj_params_.at(i)); - param_store.at(param_idx).set_extend(array_obj_addr, T_EXT_SQL_ARRAY); - } - return ret; -} - int ObNestedLoopJoinOp::group_read_left_operate() { int ret = OB_SUCCESS; - if (left_store_iter_.is_valid() && left_store_iter_.has_next()) { - // 重新设置右表 table scan result, result 为下一个 cache - if (MY_SPEC.enable_px_batch_rescan_) { - batch_rescan_ctl_.cur_idx_++; - } else if (OB_FAIL(rescan_right_operator())) { - // 这里是不期望有 OB_ITER_END - if (OB_ITER_END == ret) { - ret = OB_ERR_UNEXPECTED; + if (MY_SPEC.enable_px_batch_rescan_) { + if (left_store_iter_.is_valid() && left_store_iter_.has_next()) { + // 重新设置右表 table scan result, result 为下一个 cache + if (MY_SPEC.enable_px_batch_rescan_) { + batch_rescan_ctl_.cur_idx_++; } - LOG_WARN("failed to get next right row from group", K(ret)); - } - } else { - // 当前 row 对应的 cache 读完了, 左表拿新 cache, 设置右表 result_iter - if (!MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(init_bnlj_params())) { - LOG_WARN("Failed to init group rescan", K(ret)); - } else if (is_left_end_) { - ret = OB_ITER_END; } else { - if (OB_ISNULL(mem_context_)) { - ObSQLSessionInfo *session = ctx_.get_my_session(); - uint64_t tenant_id =session->get_effective_tenant_id(); - lib::ContextParam param; - param.set_mem_attr(tenant_id, - ObModIds::OB_SQL_NLJ_CACHE, - ObCtxIds::WORK_AREA) - .set_properties(lib::USE_TL_PAGE_OPTIONAL); - if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { - LOG_WARN("create entity failed", K(ret)); - } else if (OB_ISNULL(mem_context_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null memory entity returned", K(ret)); - } else if (OB_FAIL(left_store_.init(UINT64_MAX, tenant_id, ObCtxIds::WORK_AREA))) { - LOG_WARN("init row store failed", K(ret)); - } else { - left_store_.set_allocator(mem_context_->get_malloc_allocator()); - } - } - - bool ignore_end = false; - if (OB_SUCC(ret)) { - // 没有下一个了, 尝试填充 cache. - batch_rescan_ctl_.reuse(); - bnlj_cur_idx_ = 0; - left_store_iter_.reset(); - left_store_.reset(); - mem_context_->get_arena_allocator().reset(); - if (OB_ISNULL(last_store_row_.get_store_row())) { - if (save_last_row_) { + if (is_left_end_) { + ret = OB_ITER_END; + } else { + if (OB_ISNULL(mem_context_)) { + ObSQLSessionInfo *session = ctx_.get_my_session(); + uint64_t tenant_id =session->get_effective_tenant_id(); + lib::ContextParam param; + param.set_mem_attr(tenant_id, + ObModIds::OB_SQL_NLJ_CACHE, + ObCtxIds::WORK_AREA) + .set_properties(lib::USE_TL_PAGE_OPTIONAL); + if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { + LOG_WARN("create entity failed", K(ret)); + } else if (OB_ISNULL(mem_context_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected status: store row is null", K(ret)); - } else if (OB_FAIL(last_store_row_.init( - mem_context_->get_malloc_allocator(), left_->get_spec().output_.count()))) { - LOG_WARN("failed to init right last row", K(ret)); - } - } else if (save_last_row_) { - if (OB_FAIL(last_store_row_.restore(left_->get_spec().output_, eval_ctx_))) { - LOG_WARN("failed to restore left row", K(ret)); - } - } - save_last_row_ = false; - set_param_null(); - while (OB_SUCC(ret) && !is_full()) { - // need clear evaluated flag, since prepare_rescan_params() will evaluate expression. - clear_evaluated_flag(); - if (OB_FAIL(get_next_left_row())) { - if (OB_ITER_END != ret) { - LOG_WARN("failed to get next left row", K(ret)); - } else { - is_left_end_ = true; - } - } else if (OB_FAIL(left_store_.add_row(left_->get_spec().output_, &eval_ctx_))) { - LOG_WARN("failed to store left row", K(ret)); - // do nothing - } else if (OB_FAIL(prepare_rescan_params(true/*is_group*/))) { - LOG_WARN("failed to prepare rescan params", K(ret)); - // 下压参数数据是由被换的原始表达式计算生成, 比如c1 = c2 + 1--> c1 = ?; - // 下压参数?的值, 由c2+1计算而来, c2+1的内存是复用的, 如果此时不深拷贝 - // 计算query range的下压param, 则可能导致后面query range的结果和 - // 前面query range的obobj对应的ptr(string/number类型在obj中ptr)使用相同指针; - } else if (!MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(deep_copy_dynamic_obj())) { - LOG_WARN("fail to deep copy dynamic obj", K(ret)); + LOG_WARN("null memory entity returned", K(ret)); + } else if (OB_FAIL(left_store_.init(UINT64_MAX, tenant_id, ObCtxIds::WORK_AREA))) { + LOG_WARN("init row store failed", K(ret)); } else { - ignore_end = true; + left_store_.set_allocator(mem_context_->get_malloc_allocator()); } } + bool ignore_end = false; if (OB_SUCC(ret)) { - // here need to set param null, because dynamic datum ptr - // which from last batch row may invalid + // 没有下一个了, 尝试填充 cache. + batch_rescan_ctl_.reuse(); + left_store_iter_.reset(); + left_store_.reset(); + mem_context_->get_arena_allocator().reset(); + if (OB_ISNULL(last_store_row_.get_store_row())) { + if (save_last_row_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status: store row is null", K(ret)); + } else if (OB_FAIL(last_store_row_.init( + mem_context_->get_malloc_allocator(), left_->get_spec().output_.count()))) { + LOG_WARN("failed to init right last row", K(ret)); + } + } else if (save_last_row_) { + if (OB_FAIL(last_store_row_.restore(left_->get_spec().output_, eval_ctx_))) { + LOG_WARN("failed to restore left row", K(ret)); + } + } + save_last_row_ = false; set_param_null(); - if (OB_FAIL(last_store_row_.shadow_copy(left_->get_spec().output_, eval_ctx_))) { - LOG_WARN("failed to shadow copy last left row", K(ret)); - } else { - save_last_row_ = true; + while (OB_SUCC(ret) && !is_full()) { + // need clear evaluated flag, since prepare_rescan_params() will evaluate expression. + clear_evaluated_flag(); + if (OB_FAIL(get_next_left_row())) { + if (OB_ITER_END != ret) { + LOG_WARN("failed to get next left row", K(ret)); + } else { + is_left_end_ = true; + } + } else if (OB_FAIL(left_store_.add_row(left_->get_spec().output_, &eval_ctx_))) { + LOG_WARN("failed to store left row", K(ret)); + // do nothing + } else if (OB_FAIL(prepare_rescan_params(true/*is_group*/))) { + LOG_WARN("failed to prepare rescan params", K(ret)); + // 下压参数数据是由被换的原始表达式计算生成, 比如c1 = c2 + 1--> c1 = ?; + // 下压参数?的值, 由c2+1计算而来, c2+1的内存是复用的, 如果此时不深拷贝 + // 计算query range的下压param, 则可能导致后面query range的结果和 + // 前面query range的obobj对应的ptr(string/number类型在obj中ptr)使用相同指针; + } else { + ignore_end = true; + } + } + if (OB_SUCC(ret)) { + // here need to set param null, because dynamic datum ptr + // which from last batch row may invalid + set_param_null(); + if (OB_FAIL(last_store_row_.shadow_copy(left_->get_spec().output_, eval_ctx_))) { + LOG_WARN("failed to shadow copy last left row", K(ret)); + } else { + save_last_row_ = true; + } + } + } + if (OB_SUCC(ret) || (ignore_end && OB_ITER_END == ret)) { + ret = OB_SUCCESS; + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); + if (OB_FAIL(left_store_.finish_add_row(false))) { + LOG_WARN("failed to finish add row to row store", K(ret)); + } else if (OB_FAIL(left_store_.begin(left_store_iter_))) { + LOG_WARN("failed to begin iterator for chunk row store", K(ret)); } } } - - if (OB_SUCC(ret) || (ignore_end && OB_ITER_END == ret)) { - ret = OB_SUCCESS; - ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - if (OB_FAIL(left_store_.finish_add_row(false))) { - LOG_WARN("failed to finish add row to row store", K(ret)); - } else if (OB_FAIL(left_store_.begin(left_store_iter_))) { - LOG_WARN("failed to begin iterator for chunk row store", K(ret)); - } else if (!MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(bind_bnlj_param_to_store())) { - LOG_WARN("bind bnlj param to store failed", K(ret)); - } else if (!MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(rescan_right_operator())) { - LOG_WARN("failed to rescan right op", K(ret)); - } - } - } - } - - if (OB_SUCC(ret)) { - // 拿到下一行 ret = OB_SUCCESS; - clear_evaluated_flag(); - if (OB_FAIL(left_store_iter_.get_next_row(left_->get_spec().output_, - eval_ctx_))) { - LOG_WARN("Failed to get next row", K(ret)); - } else if (MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(fill_cur_row_rescan_param())) { - LOG_WARN("fail to fill cur row rescan param", K(ret)); - } else if (MY_SPEC.enable_px_batch_rescan_) { - OZ(right_->rescan()); - OX(brs_holder_.reset()); } if (OB_SUCC(ret)) { - left_row_joined_ = false; + // 拿到下一行 ret = OB_SUCCESS; + clear_evaluated_flag(); + if (OB_FAIL(left_store_iter_.get_next_row(left_->get_spec().output_, + eval_ctx_))) { + LOG_WARN("Failed to get next row", K(ret)); + } else if (MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(fill_cur_row_rescan_param())) { + LOG_WARN("fail to fill cur row rescan param", K(ret)); + } else if (MY_SPEC.enable_px_batch_rescan_) { + OZ(right_->rescan()); + OX(brs_holder_.reset()); + } + if (OB_SUCC(ret)) { + left_row_joined_ = false; + } } - } - return ret; -} - -int ObNestedLoopJoinOp::deep_copy_dynamic_obj() -{ - int ret = OB_SUCCESS; - int64_t param_cnt = get_spec().rescan_params_.count(); - ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - ParamStore ¶m_store = plan_ctx->get_param_store_for_update(); - if (OB_ISNULL(mem_context_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("mem entity not init", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < param_cnt; ++i) { - const ObDynamicParamSetter &rescan_param = get_spec().rescan_params_.at(i); - int64_t param_idx = rescan_param.param_idx_; - if (OB_FAIL(ob_write_obj(mem_context_->get_arena_allocator(), - param_store.at(param_idx), - bnlj_params_.at(i).data_[bnlj_params_.at(i).count_]))) { - LOG_WARN("deep copy dynamic param", K(ret)); + } else { + // das group rescan + bool has_next = false; + if (OB_FAIL(group_join_buffer_.fill_group_buffer())) { + if (OB_ITER_END != ret) { + LOG_WARN("fill group buffer failed", KR(ret)); + } + } else if (OB_FAIL(group_join_buffer_.has_next_left_row(has_next))) { + LOG_WARN("check has next failed", KR(ret)); + } else if (has_next) { + clear_evaluated_flag(); + if (OB_FAIL(group_join_buffer_.rescan_right())) { + if (OB_ITER_END == ret) { + ret = OB_ERR_UNEXPECTED; + } + LOG_WARN("rescan right failed", KR(ret)); + } else if (OB_FAIL(group_join_buffer_.fill_cur_row_group_param())) { + LOG_WARN("fill group param failed", KR(ret)); + } } else { - ++bnlj_params_.at(i).count_; + ret = OB_ITER_END; + } + if (OB_SUCC(ret)) { + clear_evaluated_flag(); + if (OB_FAIL(group_join_buffer_.get_next_row_from_store())) { + if (OB_ITER_END != ret) { + LOG_WARN("get next row failed", KR(ret)); + } + } else { + left_row_joined_ = false; + } } } - return ret; } int ObNestedLoopJoinOp::read_left_func_going() { int ret = OB_SUCCESS; - if (MY_SPEC.use_group_ || MY_SPEC.enable_px_batch_rescan_) { + if (MY_SPEC.group_rescan_ || MY_SPEC.enable_px_batch_rescan_) { // do nothing // group nested loop join 已经做过 rescan 了 } else if (OB_FAIL(prepare_rescan_params())) { @@ -694,13 +622,13 @@ int ObNestedLoopJoinOp::read_right_func_end() bool ObNestedLoopJoinOp::is_full() const { - return left_store_.get_row_cnt() >= MY_SPEC.left_group_size_; + return left_store_.get_row_cnt() >= MY_SPEC.group_size_; } int ObNestedLoopJoinOp::get_left_batch() { int ret = OB_SUCCESS; - if (MY_SPEC.use_group_ || MY_SPEC.enable_px_batch_rescan_) { + if (MY_SPEC.group_rescan_ || MY_SPEC.enable_px_batch_rescan_) { if (OB_FAIL(group_get_left_batch(left_brs_)) && OB_ITER_END != ret) { LOG_WARN("fail to get left batch", K(ret)); } @@ -735,127 +663,145 @@ int ObNestedLoopJoinOp::get_left_batch() int ObNestedLoopJoinOp::group_get_left_batch(const ObBatchRows *&left_brs) { int ret = OB_SUCCESS; - left_brs = &left_->get_brs(); - if (left_store_iter_.is_valid() && left_store_iter_.has_next()) { - // do nothing - } else { - if (!MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(init_bnlj_params())) { - LOG_WARN("Failed to init bnlj params", K(ret)); - } else if (is_left_end_) { - ret = OB_ITER_END; + if (MY_SPEC.enable_px_batch_rescan_) { + left_brs = &left_->get_brs(); + if (left_store_iter_.is_valid() && left_store_iter_.has_next()) { // do nothing } else { - if (OB_ISNULL(mem_context_)) { - ObSQLSessionInfo *session = ctx_.get_my_session(); - uint64_t tenant_id =session->get_effective_tenant_id(); - lib::ContextParam param; - param.set_mem_attr(tenant_id, - ObModIds::OB_SQL_NLJ_CACHE, - ObCtxIds::WORK_AREA) - .set_properties(lib::USE_TL_PAGE_OPTIONAL); - if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { - LOG_WARN("create entity failed", K(ret)); - } else if (OB_ISNULL(mem_context_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("null memory entity returned", K(ret)); - } else if (OB_FAIL(left_store_.init(UINT64_MAX, tenant_id, ObCtxIds::WORK_AREA))) { - LOG_WARN("init row store failed", K(ret)); - } else { - left_store_.set_allocator(mem_context_->get_malloc_allocator()); - } - } - if (OB_SUCC(ret)) { - // 没有下一个了, 尝试填充 cache. - batch_rescan_ctl_.reuse(); - bnlj_cur_idx_ = 0; - left_store_iter_.reset(); - left_store_.reset(); - mem_context_->get_arena_allocator().reset(); - save_last_row_ = false; - ObEvalCtx::BatchInfoScopeGuard batch_info_guard(eval_ctx_); - while (OB_SUCC(ret) && continue_fetching()) { - // need clear evaluated flag, since prepare_rescan_params() will evaluate expression. - clear_evaluated_flag(); - if (save_last_batch_) { - last_save_batch_.to_exprs(eval_ctx_); - save_last_batch_ = false; - } - set_param_null(); - if (OB_FAIL(left_->get_next_batch(op_max_batch_size_, left_brs_))) { - LOG_WARN("failed to get next left row", K(ret)); - } else if (left_brs_->end_) { - is_left_end_ = true; - } - for (int64_t l_idx = 0; OB_SUCC(ret) && l_idx < left_brs_->size_; l_idx++) { - if (left_brs_->skip_->exist(l_idx)) { continue; } - batch_info_guard.set_batch_idx(l_idx); - batch_info_guard.set_batch_size(left_brs_->size_); - if (OB_FAIL(left_store_.add_row(left_->get_spec().output_, &eval_ctx_))) { - LOG_WARN("failed to store left row", K(ret)); - // do nothing - } else if (OB_FAIL(prepare_rescan_params(true))) { - LOG_WARN("failed to prepare rescan params", K(ret)); - // 下压参数数据是由被换的原始表达式计算生成, 比如c1 = c2 + 1--> c1 = ?; - // 下压参数?的值, 由c2+1计算而来, c2+1的内存是复用的, 如果此时不深拷贝 - // 计算query range的下压param, 则可能导致后面query range的结果和 - // 前面query range的obobj对应的ptr(string/number类型在obj中ptr)使用相同指针; - } else if (!MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(deep_copy_dynamic_obj())) { - LOG_WARN("fail to deep copy dynamic obj", K(ret)); - } - } // for end - } - if (OB_SUCC(ret)) { - set_param_null(); - if (left_brs_->size_ == 0 && left_brs_->end_) { - // do nothing - } else { - last_save_batch_.from_exprs(eval_ctx_, left_brs_->skip_, left_brs_->size_); - save_last_batch_ = true; - } - } - clear_evaluated_flag(); - } - if (OB_SUCC(ret) ) { - if (left_store_.get_row_cnt() <= 0) { - ret = OB_ITER_END; - } else if (OB_FAIL(left_store_.finish_add_row(false))) { - LOG_WARN("failed to finish add row to row store", K(ret)); - } else if (OB_FAIL(left_store_.begin(left_store_iter_))) { - LOG_WARN("failed to begin iterator for chunk row store", K(ret)); - } else if (!MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(bind_bnlj_param_to_store())) { - LOG_WARN("bind bnlj param to store failed", K(ret)); - } else if (!MY_SPEC.enable_px_batch_rescan_ && OB_FAIL(rescan_right_operator())) { - LOG_WARN("failed to rescan right op", K(ret)); - } else { - need_switch_iter_ = false; - } - } - } - } - - if (OB_SUCC(ret)) { - int64_t read_size = 0; - int64_t max_size = MY_SPEC.max_batch_size_; - last_save_batch_.extend_save(eval_ctx_, max_size); - if (OB_FAIL(left_store_iter_.get_next_batch(left_->get_spec().output_, - eval_ctx_, - max_size, - read_size))) { - if (OB_ITER_END == ret) { + if (is_left_end_) { + ret = OB_ITER_END; // do nothing } else { - LOG_WARN("Failed to get next row", K(ret)); + if (OB_ISNULL(mem_context_)) { + ObSQLSessionInfo *session = ctx_.get_my_session(); + uint64_t tenant_id =session->get_effective_tenant_id(); + lib::ContextParam param; + param.set_mem_attr(tenant_id, + ObModIds::OB_SQL_NLJ_CACHE, + ObCtxIds::WORK_AREA) + .set_properties(lib::USE_TL_PAGE_OPTIONAL); + if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) { + LOG_WARN("create entity failed", K(ret)); + } else if (OB_ISNULL(mem_context_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null memory entity returned", K(ret)); + } else if (OB_FAIL(left_store_.init(UINT64_MAX, tenant_id, ObCtxIds::WORK_AREA))) { + LOG_WARN("init row store failed", K(ret)); + } else { + left_store_.set_allocator(mem_context_->get_malloc_allocator()); + } + } + if (OB_SUCC(ret)) { + // 没有下一个了, 尝试填充 cache. + batch_rescan_ctl_.reuse(); + left_store_iter_.reset(); + left_store_.reset(); + mem_context_->get_arena_allocator().reset(); + save_last_row_ = false; + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(eval_ctx_); + while (OB_SUCC(ret) && continue_fetching()) { + // need clear evaluated flag, since prepare_rescan_params() will evaluate expression. + clear_evaluated_flag(); + if (save_last_batch_) { + last_save_batch_.to_exprs(eval_ctx_); + save_last_batch_ = false; + } + set_param_null(); + if (OB_FAIL(left_->get_next_batch(op_max_batch_size_, left_brs_))) { + LOG_WARN("failed to get next left row", K(ret)); + } else if (left_brs_->end_) { + is_left_end_ = true; + } + for (int64_t l_idx = 0; OB_SUCC(ret) && l_idx < left_brs_->size_; l_idx++) { + if (left_brs_->skip_->exist(l_idx)) { continue; } + batch_info_guard.set_batch_idx(l_idx); + batch_info_guard.set_batch_size(left_brs_->size_); + if (OB_FAIL(left_store_.add_row(left_->get_spec().output_, &eval_ctx_))) { + LOG_WARN("failed to store left row", K(ret)); + // do nothing + } else if (OB_FAIL(prepare_rescan_params(true))) { + LOG_WARN("failed to prepare rescan params", K(ret)); + // 下压参数数据是由被换的原始表达式计算生成, 比如c1 = c2 + 1--> c1 = ?; + // 下压参数?的值, 由c2+1计算而来, c2+1的内存是复用的, 如果此时不深拷贝 + // 计算query range的下压param, 则可能导致后面query range的结果和 + // 前面query range的obobj对应的ptr(string/number类型在obj中ptr)使用相同指针; + } + } // for end + } + if (OB_SUCC(ret)) { + set_param_null(); + if (left_brs_->size_ == 0 && left_brs_->end_) { + // do nothing + } else { + last_save_batch_.from_exprs(eval_ctx_, left_brs_->skip_, left_brs_->size_); + save_last_batch_ = true; + } + } + clear_evaluated_flag(); + } + if (OB_SUCC(ret) ) { + if (left_store_.get_row_cnt() <= 0) { + ret = OB_ITER_END; + } else if (OB_FAIL(left_store_.finish_add_row(false))) { + LOG_WARN("failed to finish add row to row store", K(ret)); + } else if (OB_FAIL(left_store_.begin(left_store_iter_))) { + LOG_WARN("failed to begin iterator for chunk row store", K(ret)); + } else { + need_switch_iter_ = false; + } + } } } if (OB_SUCC(ret)) { - const_cast(left_brs)->skip_->reset(read_size); - const_cast(left_brs)->size_ = read_size; - const_cast(left_brs)->end_ = false; - left_row_joined_ = false; + int64_t read_size = 0; + int64_t max_size = MY_SPEC.max_batch_size_; + last_save_batch_.extend_save(eval_ctx_, max_size); + if (OB_FAIL(left_store_iter_.get_next_batch(left_->get_spec().output_, + eval_ctx_, + max_size, + read_size))) { + if (OB_ITER_END == ret) { + // do nothing + } else { + LOG_WARN("Failed to get next row", K(ret)); + } + } + + if (OB_SUCC(ret)) { + const_cast(left_brs)->skip_->reset(read_size); + const_cast(left_brs)->size_ = read_size; + const_cast(left_brs)->end_ = false; + left_row_joined_ = false; + } + } + } else { + // das group rescan + bool has_next = false; + if (OB_FAIL(group_join_buffer_.batch_fill_group_buffer(op_max_batch_size_, left_brs_))) { + if (OB_ITER_END != ret) { + LOG_WARN("batch fill group buffer failed", KR(ret)); + } + } else if (OB_FAIL(group_join_buffer_.has_next_left_row(has_next))) { + LOG_WARN("check has next failed", KR(ret)); + } else if (!has_next) { + ret = OB_ITER_END; + } + if (OB_SUCC(ret)) { + int64_t read_size = 0; + int64_t max_size = min(MY_SPEC.max_batch_size_, left_->get_spec().max_batch_size_); + if (OB_FAIL(group_join_buffer_.get_next_batch_from_store(max_size, read_size))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next batch from store failed", KR(ret)); + } + } else { + const_cast(left_brs)->skip_->reset(read_size); + const_cast(left_brs)->size_ = read_size; + const_cast(left_brs)->end_ = false; + left_row_joined_ = false; + } } } - return ret; } @@ -870,21 +816,20 @@ int ObNestedLoopJoinOp::process_left_batch() // Adding seperated guards for left/right children can also solve the problem, // we don't choose that way due to performance reason. batch_info_guard.set_batch_size(left_brs_->size_); - if (!MY_SPEC.use_group_ && !MY_SPEC.enable_px_batch_rescan_) { + if (!MY_SPEC.group_rescan_ && !MY_SPEC.enable_px_batch_rescan_) { batch_info_guard.set_batch_idx(l_idx); if (left_brs_->skip_->exist(l_idx)) { continue; } if (OB_FAIL(rescan_params_batch_one(l_idx))) { LOG_WARN("fail to rescan params", K(ret)); } - } else if (MY_SPEC.use_group_ && !MY_SPEC.enable_px_batch_rescan_) { - // after group rescan, first left row not need switch iter - if (!need_switch_iter_) { - need_switch_iter_ = true; - } else if (OB_FAIL(rescan_right_operator())) { + } else if (MY_SPEC.group_rescan_ && !MY_SPEC.enable_px_batch_rescan_) { + if (OB_FAIL(group_join_buffer_.rescan_right())) { if (OB_ITER_END == ret) { ret = OB_ERR_UNEXPECTED; } - LOG_WARN("fail to switch iterator", K(ret)); + LOG_WARN("rescan right failed", KR(ret)); + } else if (OB_FAIL(group_join_buffer_.fill_cur_row_group_param())) { + LOG_WARN("fill group param failed", KR(ret)); } } else if (MY_SPEC.enable_px_batch_rescan_) { // NOTE: left batch is ALWAYS continous, NO need to check skip for @@ -979,9 +924,9 @@ int ObNestedLoopJoinOp::calc_right_batch_matched_result( if (0 == conds.count()) { brs_.skip_->deep_copy(*right_brs->skip_, right_brs->size_); } else { - if (MY_SPEC.use_group_ || MY_SPEC.enable_px_batch_rescan_) { + if (MY_SPEC.enable_px_batch_rescan_) { last_save_batch_.extend_save(eval_ctx_, right_brs->size_); - } else { + } else if (!MY_SPEC.group_rescan_) { left_batch_.extend_save(eval_ctx_, right_brs->size_); } batch_info_guard.set_batch_size(right_brs->size_); @@ -1029,7 +974,7 @@ int ObNestedLoopJoinOp::output() brs_.skip_->bit_calculate(*left_batch_.skip_, *left_matched_, left_batch_.size_, [](const uint64_t l, const uint64_t r) { return (l | r); }); } - if (MY_SPEC.use_group_ || MY_SPEC.enable_px_batch_rescan_) { + if (MY_SPEC.enable_px_batch_rescan_) { last_save_batch_.extend_save(eval_ctx_, left_batch_.size_); } left_batch_.to_exprs(eval_ctx_); @@ -1049,9 +994,9 @@ int ObNestedLoopJoinOp::output() LOG_WARN("fail to get next batch", K(ret)); } } else { - if (MY_SPEC.use_group_ || MY_SPEC.enable_px_batch_rescan_) { + if (MY_SPEC.enable_px_batch_rescan_) { last_save_batch_.extend_save(eval_ctx_, read_rows); - } else { + } else if (!MY_SPEC.group_rescan_) { left_batch_.extend_save(eval_ctx_, read_rows); } for (int64_t i = 0; OB_SUCC(ret) && i < read_rows; i++) { @@ -1172,90 +1117,5 @@ int ObNestedLoopJoinOp::calc_other_conds_with_update_left_expr(bool &is_match, return ret; } - -int ObBatchRowDatums::init(const ObExprPtrIArray *exprs, ObIAllocator *alloc, int32_t batch_size) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(alloc) || OB_ISNULL(exprs)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KP(alloc), KP(exprs)); - } else { - char *buf= (char *)alloc->alloc(ObBitVector::memory_size(batch_size) - + sizeof(ObDatum) * batch_size * exprs->count()); - if (NULL == buf) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory", K(ret)); - } else { - MEMSET(buf, 0, ObBitVector::memory_size(batch_size)); - skip_ = to_bit_vector(buf); - alloc_ = alloc; - exprs_ = exprs; - datums_ = reinterpret_cast(buf + ObBitVector::memory_size(batch_size)); - batch_size_ = batch_size; - size_ = 0; - saved_size_ = 0; - } - } - - return ret; -} - -void ObBatchRowDatums::from_exprs(ObEvalCtx &ctx, ObBitVector *skip, int64_t size) -{ - OB_ASSERT(size <= batch_size_); - OB_ASSERT(OB_NOT_NULL(skip) && OB_NOT_NULL(exprs_)); - for (int64_t i = 0; i < exprs_->count(); i++) { - ObExpr *expr = exprs_->at(i); - ObDatum *datums = expr->locate_batch_datums(ctx); - int64_t copy_size = (expr->is_batch_result() ? size: 1) * sizeof(ObDatum); - MEMCPY(datums_ + i * batch_size_, datums, copy_size); - } - size_ = size; - saved_size_ = size; - skip_->deep_copy(*skip, size); -} - -void ObBatchRowDatums::extend_save(ObEvalCtx &ctx, int64_t size) -{ - if (size > saved_size_) { - for (int64_t i = 0; i < exprs_->count(); i++) { - ObExpr *expr = exprs_->at(i); - if (expr->is_batch_result()) { - ObDatum *datums = expr->locate_batch_datums(ctx); - int64_t copy_size = (size - saved_size_) * sizeof(ObDatum); - MEMCPY(datums_ + i * batch_size_ + saved_size_, datums + saved_size_, copy_size); - } - } - saved_size_ = size; - } -} - -void ObBatchRowDatums::to_exprs(ObEvalCtx &ctx) -{ - if (saved_size_ > 0) { - for (int64_t i = 0; i < exprs_->count(); i++) { - ObExpr *expr = exprs_->at(i); - ObDatum *datums = expr->locate_batch_datums(ctx); - int64_t copy_size = (expr->is_batch_result() ? saved_size_: 1) * sizeof(ObDatum); - MEMCPY(datums, datums_ + i * batch_size_, copy_size); - } - } -} - -void ObBatchRowDatums::to_exprs(ObEvalCtx &ctx, int64_t from_idx, int64_t to_idx) -{ - OB_ASSERT(from_idx <= size_ && to_idx <= batch_size_); - OB_ASSERT(!skip_->exist(from_idx)); - for (int64_t i = 0; i < exprs_->count(); i++) { - ObExpr *expr = exprs_->at(i); - ObDatum *datums = expr->locate_batch_datums(ctx); - if (!expr->is_batch_result()) { - *datums = *(datums_ + i * batch_size_); - } else { - *(datums + to_idx) = *(datums_ + i * batch_size_ + from_idx) ; - } - } -} - } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/engine/join/ob_nested_loop_join_op.h b/src/sql/engine/join/ob_nested_loop_join_op.h index 1ba0333a22..ce1c2a166b 100644 --- a/src/sql/engine/join/ob_nested_loop_join_op.h +++ b/src/sql/engine/join/ob_nested_loop_join_op.h @@ -15,6 +15,7 @@ #include "sql/engine/join/ob_basic_nested_loop_join_op.h" #include "sql/engine/basic/ob_chunk_datum_store.h" +#include "sql/engine/basic/ob_group_join_buffer.h" #include "sql/engine/basic/ob_material_op.h" namespace oceanbase @@ -27,48 +28,35 @@ class ObNestedLoopJoinSpec : public ObBasicNestedLoopJoinSpec public: ObNestedLoopJoinSpec(common::ObIAllocator &alloc, const ObPhyOperatorType type) : ObBasicNestedLoopJoinSpec(alloc, type), - use_group_(false), - left_group_size_(BNLJ_DEFAULT_GROUP_SIZE), - left_expr_ids_in_other_cond_(alloc) + group_rescan_(false), + group_size_(OB_MAX_BULK_JOIN_ROWS), + left_expr_ids_in_other_cond_(alloc), + left_rescan_params_(alloc), + right_rescan_params_(alloc) {} public: - // for group nested loop join. - bool use_group_; - int64_t left_group_size_; + // for group join buffer + bool group_rescan_; + int64_t group_size_; ObFixedArray, common::ObIAllocator> left_expr_ids_in_other_cond_; + // for multi level batch rescan + // NLJ 1 + // / \ + // TSC 1 NLJ 2 + // / \ + // TSC 2 TSC 3 + // As shown above, for NLJ 2, its left_rescan_params_ stores params used by TSC 2 and + // set by NLJ 1. + // Similarly, for NLJ 2, its right_rescan_params_ stores params used by TSC 3 and set + // by NLJ 1. + common::ObFixedArray left_rescan_params_; + common::ObFixedArray right_rescan_params_; private: DISALLOW_COPY_AND_ASSIGN(ObNestedLoopJoinSpec); }; - -struct ObBatchRowDatums -{ - ObBatchRowDatums() - : alloc_(NULL), exprs_(NULL), batch_size_(0), datums_(NULL), - skip_(NULL), size_(0), saved_size_(0) - {} - int init(const ObExprPtrIArray *exprs, common::ObIAllocator *alloc, int32_t batch_size); - void from_exprs(ObEvalCtx &ctx, ObBitVector *skip, int64_t size); - void extend_save(ObEvalCtx &ctx, int64_t size); - void to_exprs(ObEvalCtx &ctx); - void to_exprs(ObEvalCtx &ctx, int64_t from_idx, int64_t to_idx); - ObDatum &get_datum(int64_t row_id, int64_t col_id) - { - return datums_[col_id * batch_size_ + row_id]; - } - -public: - common::ObIAllocator *alloc_; - const ObExprPtrIArray *exprs_; - int32_t batch_size_; - ObDatum *datums_; - ObBitVector *skip_; - int32_t size_; - int32_t saved_size_; // record the saved size, include extend saved size -}; - // Nest loop join has no expression result overwrite problem: // // LEFT: @@ -124,11 +112,13 @@ public: batch_mem_ctx_ = nullptr; } } + if (MY_SPEC.group_rescan_) { + group_join_buffer_.destroy(); + } ObBasicNestedLoopJoinOp::destroy(); } ObBatchRescanCtl &get_batch_rescan_ctl() { return batch_rescan_ctl_; } int fill_cur_row_rescan_param(); - int fill_cur_row_bnlj_param(); // Skip restore if child is material operator to save duplicate work // Note: // this is a sister function of backup_right_child_exprs(), call it after @@ -174,7 +164,6 @@ private: int read_left_operate_batch(); int read_left_operate_group_batch(); int group_read_left_operate(); - int deep_copy_dynamic_obj(); int read_left_func_going(); int read_left_func_end(); // JS_READ_RIGHT state operation and transfer functions. @@ -199,8 +188,6 @@ private: ObEvalCtx::BatchInfoScopeGuard &batch_info_guard); int output(); int inner_get_next_batch(const int64_t max_row_cnt); - int init_bnlj_params(); - int bind_bnlj_param_to_store(); // for vectorized end bool continue_fetching() { return !(left_brs_->end_ || is_full());} @@ -228,13 +215,12 @@ public: ObChunkDatumStore::Iterator right_store_iter_; const ObBatchRows *left_brs_; ObBitVector *left_matched_; - common::ObArrayWrap bnlj_params_; bool need_switch_iter_; bool iter_end_; ObBatchResultHolder brs_holder_; int64_t op_max_batch_size_; int64_t max_group_size_; - int64_t bnlj_cur_idx_; + ObGroupJoinBufffer group_join_buffer_; // for vectorized end private: DISALLOW_COPY_AND_ASSIGN(ObNestedLoopJoinOp); diff --git a/src/sql/engine/subquery/ob_subplan_filter_op.cpp b/src/sql/engine/subquery/ob_subplan_filter_op.cpp index bf05b8f68d..6d0c89dee8 100644 --- a/src/sql/engine/subquery/ob_subplan_filter_op.cpp +++ b/src/sql/engine/subquery/ob_subplan_filter_op.cpp @@ -53,7 +53,10 @@ ObSubQueryIterator::ObSubQueryIterator(ObOperator &op) iter_brs_(NULL), batch_size_(0), batch_row_pos_(0), - iter_end_(false) + iter_end_(false), + is_new_batch_(false), + current_group_(0), + das_batch_params_recovery_() { } @@ -82,16 +85,79 @@ int ObSubQueryIterator::rewind(const bool reset_onetime_plan /* = false */) LOG_WARN("failed to rewind iterator", K(ret)); } } else { - if (OB_FAIL(op_.rescan())) { - LOG_WARN("failed to do rescan", K(ret)); + if (parent_->enable_left_das_batch()) { + if (OB_FAIL(alloc_das_batch_store())) { + LOG_WARN("Alloc DAS batch parameter store fail.", K(ret)); + } else { + //We use GroupParamBackupGuard to save and resume data in param store. + //1.For SPF operator, we have multiple right child, every time rescan we + //need switch the param in param store, But all of param store index in + //the same array, So we switch all of the param store, but resume them + //after we rescan. + //2.For SPF operator, we nedd to support jump read. Difference child params + //may in the difference group id, current child rescan should not influence + //other child's param store stage. + //3.For nesting SPF with other SPF, parent and child SPF may in difference + //stage rescan or get_next_row, the expr may reuse, So child SPF rescan + //should not change the param store stage. + //So we implement the GroupParamBackupGuard to make Paramstore like a stack, + //protect every time change the Paramstore will be resume. + bool new_group = is_new_batch_; + if (OB_SUCC(ret) && is_new_batch_) { + GroupParamBackupGuard guard(eval_ctx_, + das_batch_params_recovery_, + parent_->get_spec().rescan_params_, + parent_->get_spec().rescan_params_.count()); + + ret = parent_->bind_das_batch_params_to_store(); + if (OB_SUCC(ret) && OB_FAIL(op_.rescan())) { + if(OB_ITER_END == ret) { + ret = OB_ERR_UNEXPECTED; + } + LOG_WARN("failed to do rescan", K(ret)); + } + current_group_ = 0; + is_new_batch_ = 0; + } + + uint64_t parent_spf_group = 0; + if(OB_SUCC(ret)) { + parent_->get_current_group(parent_spf_group); + } + if (OB_SUCC(ret) && current_group_ < parent_spf_group) { + if (new_group) { + //If in lookup op, we need to call get next row to init all of iter. + op_.get_next_row(); + } + int64_t old_jump_read_group_id; + old_jump_read_group_id = op_.get_exec_ctx().get_das_ctx().jump_read_group_id_; + op_.get_exec_ctx().get_das_ctx().jump_read_group_id_ = parent_spf_group; + if (OB_FAIL(op_.rescan())) { + if(OB_ITER_END == ret) { + ret = OB_ERR_UNEXPECTED; + } + LOG_WARN("Das jump read rescan fail.", K(ret)); + } + op_.get_exec_ctx().get_das_ctx().jump_read_group_id_ = old_jump_read_group_id; + current_group_ = parent_spf_group; + } + } + } else { + //No batch branch + if (OB_SUCC(ret) && OB_FAIL(op_.rescan())) { + LOG_WARN("failed to do rescan", K(ret)); + } } } - iter_end_ = false; - //for vectorize mode, SPF iter may have a stored batch to process - //should reset them in rewind() - iter_brs_ = NULL; - batch_size_ = 0; - batch_row_pos_ = 0; + + if (OB_SUCC(ret)) { + iter_end_ = false; + //for vectorize mode, SPF iter may have a stored batch to process + //should reset them in rewind() + iter_brs_ = NULL; + batch_size_ = 0; + batch_row_pos_ = 0; + } return ret; } @@ -104,6 +170,9 @@ void ObSubQueryIterator::reuse() batch_size_ = 0; batch_row_pos_ = 0; iter_end_ = false; + is_new_batch_ = false; + current_group_ = 0; + das_batch_params_recovery_.reset(); } //TODO 移到对应的expr， 设置一个标记确保只计算一次 @@ -248,6 +317,55 @@ int ObSubQueryIterator::reset_hash_map() return ret; } +int ObSubQueryIterator::alloc_das_batch_store() +{ + int ret = OB_SUCCESS; + int64_t params_count = 0; + params_count = parent_->get_spec().rescan_params_.count(); + if (!das_batch_params_recovery_.empty()) { + //Do nothing + OB_ASSERT(params_count == das_batch_params_recovery_.count()); + } else { + ObIAllocator& alloc = op_.get_exec_ctx().get_allocator(); + if (OB_FAIL(das_batch_params_recovery_.allocate_array(alloc, params_count))) { + LOG_WARN("Alloc das batch params fail." , K(ret)); + } + } + return ret; +} + + +void GroupParamBackupGuard::save_das_batch_store() +{ + //int64_t params_count = 0; + //params_count = parent_->get_spec().rescan_params_.count(); + OB_ASSERT(!das_batch_params_recovery_.empty()); + OB_ASSERT(das_batch_params_recovery_.count() == params_count_); + ObPhysicalPlanCtx *phy_ctx = eval_ctx_.exec_ctx_.get_physical_plan_ctx(); + ParamStore ¶m_store = phy_ctx->get_param_store_for_update(); + for (int64_t i = 0; i < params_count_; ++i) { + const ObDynamicParamSetter &rescan_param = rescan_params_.at(i); + //Always shallow copy for state save. + das_batch_params_recovery_.at(i) = param_store.at(rescan_param.param_idx_); + } +} + +void GroupParamBackupGuard::resume_das_batch_store() +{ + OB_ASSERT(!das_batch_params_recovery_.empty()); + OB_ASSERT(das_batch_params_recovery_.count() == params_count_); + ObPhysicalPlanCtx *phy_ctx = eval_ctx_.exec_ctx_.get_physical_plan_ctx(); + ParamStore ¶m_store = phy_ctx->get_param_store_for_update(); + for (int64_t i = 0; i < params_count_; ++i) { + const ObDynamicParamSetter &rescan_param = rescan_params_.at(i); + //Always shallow copy for state resume. + param_store.at(rescan_param.param_idx_) = das_batch_params_recovery_.at(i); + ObExpr *dst = rescan_param.dst_; + ObDatum ¶m_datum = dst->locate_datum_for_write(eval_ctx_); + param_datum.from_obj(das_batch_params_recovery_.at(i), dst->obj_datum_map_); + } +} + ObSubPlanFilterSpec::ObSubPlanFilterSpec(ObIAllocator &alloc, const ObPhyOperatorType type) : ObOpSpec(alloc, type), rescan_params_(alloc), @@ -260,7 +378,9 @@ ObSubPlanFilterSpec::ObSubPlanFilterSpec(ObIAllocator &alloc, const ObPhyOperato enable_px_batch_rescans_(alloc), enable_das_batch_rescans_(false), filter_exprs_(alloc), - output_exprs_(alloc) + output_exprs_(alloc), + left_rescan_params_(alloc), + right_rescan_params_(alloc) { } @@ -275,7 +395,9 @@ OB_SERIALIZE_MEMBER((ObSubPlanFilterSpec, ObOpSpec), enable_px_batch_rescans_, enable_das_batch_rescans_, filter_exprs_, - output_exprs_); + output_exprs_, + left_rescan_params_, + right_rescan_params_); DEF_TO_STRING(ObSubPlanFilterSpec) { @@ -301,12 +423,15 @@ ObSubPlanFilterOp::ObSubPlanFilterOp( update_set_mem_(NULL), iter_end_(false), enable_left_px_batch_(false), - enable_left_das_batch_(false), + max_group_size_(0), + current_group_(0), + das_batch_params_(), left_rows_(), left_rows_iter_(), last_store_row_(), save_last_row_(false), is_left_end_(false), + left_row_idx_(0), batch_rescan_ctl_(), cur_params_(), cur_param_idxs_(), @@ -366,23 +491,35 @@ int ObSubPlanFilterOp::rescan() LOG_WARN("failed to inner rescan", K(ret)); } - if (OB_SUCC(ret) && enable_left_px_batch_) { + if (OB_SUCC(ret) && + (MY_SPEC.enable_das_batch_rescans_ || enable_left_px_batch_)) { left_rows_.reset(); left_rows_iter_.reset(); - batch_rescan_ctl_.reuse(); is_left_end_ = false; + save_last_row_ = false; + last_store_row_.reset(); + } + + if (OB_SUCC(ret) && MY_SPEC.enable_das_batch_rescans_) { + //We do not need alloc memory again in rescan. + //das_batch_params_.reset(); + current_group_ = 0; + } + + if (OB_SUCC(ret) && enable_left_px_batch_) { + batch_rescan_ctl_.reuse(); cur_params_.reset(); cur_param_idxs_.reset(); cur_param_expr_idxs_.reset(); - save_last_row_ = false; - last_store_row_.reset(); brs_holder_.reset(); } - for (int32_t i = 1; OB_SUCC(ret) && i < child_cnt_; ++i) { - if (OB_FAIL(children_[i]->rescan())) { - LOG_WARN("rescan child operator failed", K(ret), - "op", op_name(), "child", children_[i]->op_name()); + if (!MY_SPEC.enable_das_batch_rescans_) { + for (int32_t i = 1; OB_SUCC(ret) && i < child_cnt_; ++i) { + if (OB_FAIL(children_[i]->rescan())) { + LOG_WARN("rescan child operator failed", K(ret), + "op", op_name(), "child", children_[i]->op_name()); + } } } for (int32_t i = 1; OB_SUCC(ret) && i < child_cnt_; ++i) { @@ -495,7 +632,6 @@ int ObSubPlanFilterOp::inner_open() MY_SPEC.enable_px_batch_rescans_.at(i)) { enable_left_px_batch_ = true; } - enable_left_das_batch_ = MY_SPEC.enable_das_batch_rescans_; if (!MY_SPEC.exec_param_idxs_inited_) { //unittest or old version, do not init hashmap } else if (OB_FAIL(iter->init_mem_entity())) { @@ -514,7 +650,19 @@ int ObSubPlanFilterOp::inner_open() } } } - if (enable_left_px_batch_ && OB_ISNULL(last_store_row_mem_)) { + + //BATCH SUBPLAN FILTER { + if (OB_SUCC(ret) && MY_SPEC.enable_das_batch_rescans_) { + max_group_size_ = OB_MAX_BULK_JOIN_ROWS; + if(OB_FAIL(alloc_das_batch_params(max_group_size_+MY_SPEC.max_batch_size_))) { + LOG_WARN("Fail to alloc das batch params.", K(ret)); + } + } + //} BATCH SUBPLAN FILTER END + //left_rows used by px_batch and das batch. + if (OB_SUCC(ret) && + (enable_left_px_batch_ || MY_SPEC.enable_das_batch_rescans_) && + OB_ISNULL(last_store_row_mem_)) { ObSQLSessionInfo *session = ctx_.get_my_session(); uint64_t tenant_id =session->get_effective_tenant_id(); lib::ContextParam param; @@ -545,6 +693,9 @@ int ObSubPlanFilterOp::inner_close() { destroy_subplan_iters(); destroy_update_set_mem(); + if (MY_SPEC.enable_das_batch_rescans_) { + das_batch_params_.reset(); + } return OB_SUCCESS; } @@ -575,15 +726,36 @@ int ObSubPlanFilterOp::handle_next_row() OZ(prepare_onetime_exprs()); } if (OB_FAIL(ret)) { - } else if (enable_left_px_batch_) { + } else if (enable_left_px_batch_ || MY_SPEC.enable_das_batch_rescans_) { + //DAS batch spf is conflict with PX batch spf + OB_ASSERT(!(enable_left_px_batch_ && MY_SPEC.enable_das_batch_rescans_)); bool has_row = false; - int batch_count = PX_RESCAN_BATCH_ROW_COUNT; + int batch_count = 0; + batch_count = MY_SPEC.enable_das_batch_rescans_ ? max_group_size_ : PX_RESCAN_BATCH_ROW_COUNT; if (left_rows_iter_.is_valid() && left_rows_iter_.has_next()) { - batch_rescan_ctl_.cur_idx_++; + if(MY_SPEC.enable_das_batch_rescans_) { + //das batch branch + //Consume the remaining batch data in left store. + current_group_++; + } else { + OB_ASSERT(enable_left_px_batch_); + //px batch branch + batch_rescan_ctl_.cur_idx_++; + } } else if (is_left_end_) { ret = OB_ITER_END; } else { - batch_rescan_ctl_.reuse(); + //Accumulate a new batch into left store. + if(enable_left_px_batch_) { + batch_rescan_ctl_.reuse(); + } + if (MY_SPEC.enable_das_batch_rescans_) { + current_group_ = 0; + //Always OB_SUCCESS in current implement. + if(OB_FAIL(init_das_batch_params())) { + LOG_WARN("Failed to init das batch params", K(ret)); + } + } left_rows_iter_.reset(); left_rows_.reset(); last_store_row_mem_->get_arena_allocator().reset(); @@ -614,8 +786,10 @@ int ObSubPlanFilterOp::handle_next_row() } } else if (OB_FAIL(left_rows_.add_row(child_->get_spec().output_, &eval_ctx_))) { LOG_WARN("fail to add row", K(ret)); - } else if (OB_FAIL(prepare_rescan_params(true))) { + } else if (enable_left_px_batch_ && OB_FAIL(prepare_rescan_params(true))) { LOG_WARN("fail to prepare rescan params", K(ret)); + } else if (MY_SPEC.enable_das_batch_rescans_ && OB_FAIL(deep_copy_dynamic_obj())) { + LOG_WARN("fail to deep copy dynamic obj", K(ret)); } else { has_row = true; } @@ -633,15 +807,31 @@ int ObSubPlanFilterOp::handle_next_row() ret = OB_SUCCESS; OZ(left_rows_.finish_add_row(false)); OZ(left_rows_.begin(left_rows_iter_)); + if (MY_SPEC.enable_das_batch_rescans_) { + //Lazy batch rescan right iterator. + //Just set the flag, do the rescan when call the iter->rewind(). + for(int32_t i = 1; OB_SUCC(ret) && i < child_cnt_; ++i) { + Iterator *&iter = subplan_iters_.at(i - 1); + iter->set_new_batch(true); + } + } } } + //After accumulate a new batch or previous have remaining row. if (OB_SUCC(ret)) { clear_evaluated_flag(); // fetch datum from left_row_iter_ instead of child operator if (OB_FAIL(left_rows_iter_.get_next_row(child_->get_spec().output_, eval_ctx_))) { LOG_WARN("Failed to get next row", K(ret)); - } else { + } else if (enable_left_px_batch_) { + //px batch spf branch OZ(fill_cur_row_rescan_param()); + } else { + //das batch spf branch + OB_ASSERT(MY_SPEC.enable_das_batch_rescans_); + if (OB_FAIL(fill_cur_row_das_batch_param(eval_ctx_, current_group_))) { + LOG_WARN("Filed to prepare das batch rescan params", K(ret)); + } } } } else if (FALSE_IT(clear_evaluated_flag())) { @@ -790,6 +980,144 @@ int ObSubPlanFilterOp::handle_next_batch_with_px_rescan(const int64_t op_max_bat return ret; } +int ObSubPlanFilterOp::handle_next_batch_with_group_rescan(const int64_t op_max_batch_size) +{ + int ret = OB_SUCCESS; + const ObBatchRows *child_brs = NULL; + bool stop_fetch = false; + ObEvalCtx::BatchInfoScopeGuard guard(eval_ctx_); + uint64_t left_rows_total_cnt = 0; + if (left_rows_iter_.is_valid() && left_rows_iter_.has_next()) { + // fetch data from left store + } else { + // 1. material data from child into left_rows_ + // 2. prepare batch rescan params + left_rows_.reset(); + left_rows_iter_.reset(); + (void) brs_holder_.restore(); + current_group_ = 0; + if(OB_FAIL(init_das_batch_params())) { + LOG_WARN("Failed to init das batch params", K(ret)); + } + while (OB_SUCC(ret) && continue_fetching(left_rows_total_cnt, stop_fetch, true)) { + set_param_null(); + clear_evaluated_flag(); + int64_t store_row_cnt = -1; + if (OB_FAIL(child_->get_next_batch(op_max_batch_size, child_brs))) { + LOG_WARN("fail to get next batch", K(ret)); + } else if (OB_FAIL(left_rows_.add_batch(child_->get_spec().output_, eval_ctx_, + *child_brs->skip_, child_brs->size_, store_row_cnt))) { + LOG_WARN("fail to add expr datums to left_rows_", K(ret)); + } else { + stop_fetch = child_brs->end_; + left_rows_total_cnt += store_row_cnt; + guard.set_batch_size(child_brs->size_); + clear_evaluated_flag(); + // prepare group batch rescan parameter + for (int64_t l_idx = 0; OB_SUCC(ret) && l_idx < child_brs->size_; l_idx++) { + if (child_brs->skip_->exist(l_idx)) { continue; } + guard.set_batch_idx(l_idx); + if (OB_FAIL(deep_copy_dynamic_obj())) { + LOG_WARN("deep_copy_dynamic_obj", K(ret)); + } + } + } + } + if (OB_SUCC(ret)) { + if (!child_brs->end_) { + // backup child datums into brs_holder_ + OZ(brs_holder_.save(MY_SPEC.max_batch_size_)); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(left_rows_.finish_add_row(false))) { + LOG_WARN("prepare rescan params failed", K(ret)); + } else if (OB_FAIL(left_rows_.begin(left_rows_iter_))) { + LOG_WARN("prepare rescan params failed", K(ret)); + } else if (left_rows_total_cnt != left_rows_.get_row_cnt()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("left_rows row cnt is unexpectd", K(ret)); + } + } + + if (OB_SUCC(ret)) { + //Lazy batch rescan right iterator. + //Just set the flag, do the rescan when call the iter->rewind(). + for(int32_t i = 1; OB_SUCC(ret) && i < child_cnt_; ++i) { + Iterator *&iter = subplan_iters_.at(i - 1); + iter->set_new_batch(true); + } + } + } + } + + // fetch data from masterized left_rows(ChunkDatumStore) and do filtering + if (OB_SUCC(ret)) { + int64_t rows_fetched = 0; + clear_evaluated_flag(); + if (OB_FAIL(left_rows_iter_.get_next_batch(child_->get_spec().output_, + eval_ctx_, op_max_batch_size, rows_fetched))) { + if (OB_ITER_END == ret) { + brs_.size_ = rows_fetched; + brs_.end_ = true; + iter_end_ = true; + OB_ASSERT(0 == brs_.size_); + OB_ASSERT(0 == left_rows_total_cnt); + ret = OB_SUCCESS; + } else { + LOG_WARN("left_rows_iter_.get_next_batch failed", K(ret)); + } + } else { + // Note: rows are fetched from left_rows(ChunkDatumStore), so there is no + // skip row. + // Do not change brs_.skip_ + brs_.size_ = rows_fetched; + left_rows_total_cnt -= rows_fetched; // debug only + guard.set_batch_size(brs_.size_); + for (int64_t l_idx = 0; OB_SUCC(ret) && l_idx < brs_.size_; l_idx++) { + guard.set_batch_idx(l_idx); + if (OB_FAIL(fill_cur_row_das_batch_param(eval_ctx_, current_group_))) { + LOG_WARN("fill_cur_row_das_batch_param failed", K(ret)); + } else { + if (need_init_before_get_row_) { + for (int32_t i = 1; OB_SUCC(ret) && i < child_cnt_; ++i) { + Iterator *&iter = subplan_iters_.at(i - 1); + if (MY_SPEC.init_plan_idxs_.has_member(i)) { + OZ(iter->prepare_init_plan()); + } + } + need_init_before_get_row_ = false; + } + } + if (OB_SUCC(ret)) { + bool filtered = false; + if (OB_FAIL(filter_row(eval_ctx_, MY_SPEC.filter_exprs_, filtered))) { + LOG_WARN("fail to filter row", K(ret)); + } else if (filtered) { + brs_.skip_->set(l_idx); + } else { + ObDatum *datum = NULL; + FOREACH_CNT_X(e, spec_.output_, OB_SUCC(ret)) { + if (OB_FAIL((*e)->eval(eval_ctx_, datum))) { + LOG_WARN("expr evaluate failed", K(ret), K(*e)); + } + } + } + } + current_group_++; + } // for end + LOG_DEBUG("show batch_rescan_ctl_ info ", K(batch_rescan_ctl_), + K(rows_fetched), K(left_rows_total_cnt)); + } + } + FOREACH_CNT_X(e, spec_.output_, OB_SUCC(ret)) { + (*e)->get_eval_info(eval_ctx_).projected_ = true; + } + + return ret; +} + + int ObSubPlanFilterOp::inner_get_next_batch(const int64_t max_row_cnt) { int ret = OB_SUCCESS; @@ -799,7 +1127,13 @@ int ObSubPlanFilterOp::inner_get_next_batch(const int64_t max_row_cnt) } //从主表中获取一行数据 clear_evaluated_flag(); - if (enable_left_px_batch_) { + if(OB_FAIL(ret)) { + LOG_WARN("prepare_onetime_expr fail.", K(ret)); + } else if (MY_SPEC.enable_das_batch_rescans_) { + if (OB_FAIL(handle_next_batch_with_group_rescan(op_max_batch_size))) { + LOG_WARN("handle_next_batch_with_group_rescan failed", K(ret)); + } + } else if (enable_left_px_batch_) { if (OB_FAIL(handle_next_batch_with_px_rescan(op_max_batch_size))) { LOG_WARN("handle_next_batch_with_px_rescan failed", K(ret)); } @@ -1031,5 +1365,127 @@ int ObSubPlanFilterOp::handle_update_set() return ret; } +int ObSubPlanFilterOp::alloc_das_batch_params(uint64_t group_size) +{ + int ret = OB_SUCCESS; + if (das_batch_params_.empty()) { + ret = das_batch_params_.allocate_array(ctx_.get_allocator(), + MY_SPEC.rescan_params_.count()); + if(OB_SUCC(ret)) { + uint64_t obj_buf_size = sizeof(ObObjParam) * group_size; + for (int64_t i = 0; OB_SUCC(ret) && i < das_batch_params_.count(); ++i) { + void *buf = ctx_.get_allocator().alloc(obj_buf_size); + if (NULL == buf) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate das params array buf failed", K(ret), K(i), K(obj_buf_size)); + } else { + das_batch_params_.at(i).data_ = reinterpret_cast(buf); + das_batch_params_.at(i).count_ = 0; + ObExpr *dst_expr = MY_SPEC.rescan_params_.at(i).dst_; + das_batch_params_.at(i).element_.set_meta_type(dst_expr->obj_meta_); + } + } + } else { + LOG_WARN("allocate das params failed", KR(ret), K(MY_SPEC.rescan_params_.count())); + } + } + return ret; +} + +int ObSubPlanFilterOp::init_das_batch_params() +{ + OB_ASSERT(!das_batch_params_.empty()); + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < das_batch_params_.count(); ++i) { + das_batch_params_.at(i).count_ = 0; + } + return ret; +} + +int ObSubPlanFilterOp::deep_copy_dynamic_obj() +{ + int ret = OB_SUCCESS; + ObObjParam *param = NULL; + int64_t param_cnt = MY_SPEC.rescan_params_.count(); + if (OB_ISNULL(last_store_row_mem_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("mem entity not init", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < param_cnt; ++i) { + const ObDynamicParamSetter &rescan_param = MY_SPEC.rescan_params_.at(i); + if (OB_FAIL(rescan_param.set_dynamic_param(eval_ctx_, param))) { + LOG_WARN("fail to set dynamic param", K(ret)); + } else if (OB_ISNULL(param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param is null", K(ret)); + } else if (OB_FAIL(ob_write_obj(last_store_row_mem_->get_arena_allocator(), + *param, + das_batch_params_.at(i).data_[das_batch_params_.at(i).count_]))) { + LOG_WARN("deep copy dynamic param", KR(ret)); + } else { + ++das_batch_params_.at(i).count_; + } + } + return ret; +} + +int ObSubPlanFilterOp::fill_cur_row_das_batch_param(ObEvalCtx& eval_ctx, uint64_t current_group) const +{ + int ret = OB_SUCCESS; + ObPhysicalPlanCtx *plan_ctx = ctx_.get_physical_plan_ctx(); + if (das_batch_params_.empty() || current_group >= das_batch_params_.at(0).count_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("row idx is unexpected", K(ret), + K(current_group), K(das_batch_params_.at(0).count_)); + } else { + int64_t param_cnt = das_batch_params_.count(); + if (unlikely(MY_SPEC.rescan_params_.count() != param_cnt)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("das params count is invalid", KR(ret), K(param_cnt), K(das_batch_params_.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < param_cnt; ++i) { + const ObDynamicParamSetter &rescan_param = MY_SPEC.rescan_params_.at(i); + int64_t param_idx = rescan_param.param_idx_; + ObExpr *dst = rescan_param.dst_; + ObDatum ¶m_datum = dst->locate_datum_for_write(eval_ctx); + const ObSqlArrayObj &arr = das_batch_params_.at(i); + if (OB_FAIL(param_datum.from_obj(arr.data_[current_group], dst->obj_datum_map_))) { + LOG_WARN("fail to cast datum", K(ret)); + } else { + plan_ctx->get_param_store_for_update().at(param_idx) = arr.data_[current_group]; + dst->set_evaluated_projected(eval_ctx); + } + } + } + return ret; +} + +int ObSubPlanFilterOp::bind_das_batch_params_to_store() const +{ + int ret = OB_SUCCESS; + int64_t param_cnt = MY_SPEC.rescan_params_.count(); + ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); + ParamStore ¶m_store = plan_ctx->get_param_store_for_update(); + if (OB_UNLIKELY(param_cnt != das_batch_params_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("das params count is invalid", KR(ret), K(param_cnt), K(das_batch_params_.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < param_cnt; ++i) { + const ObDynamicParamSetter &rescan_param = MY_SPEC.rescan_params_.at(i); + int64_t param_idx = rescan_param.param_idx_; + int64_t array_obj_addr = reinterpret_cast(&das_batch_params_.at(i)); + param_store.at(param_idx).set_extend(array_obj_addr, T_EXT_SQL_ARRAY); + } + if (OB_SUCC(ret)) { + LOG_DEBUG("bind das param to store", K(das_batch_params_), K(plan_ctx->get_param_store())); + } + return ret; +} + +void ObSubPlanFilterOp::get_current_group(uint64_t& current_group) const +{ + current_group = current_group_; +} + } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/engine/subquery/ob_subplan_filter_op.h b/src/sql/engine/subquery/ob_subplan_filter_op.h index 2c5eb44dc8..9054153b39 100644 --- a/src/sql/engine/subquery/ob_subplan_filter_op.h +++ b/src/sql/engine/subquery/ob_subplan_filter_op.h @@ -93,6 +93,8 @@ public: int get_next_batch(const int64_t max_row_cnt, const ObBatchRows *&batch_rows); //for vectorized end bool is_onetime_plan() const { return onetime_plan_; } + + void set_new_batch(bool new_batch) { is_new_batch_ = new_batch;}; TO_STRING_KV(K(onetime_plan_), K(init_plan_), K(inited_)); //a row cache for hash optimizer to use @@ -102,6 +104,11 @@ public: private: + // for das batch spf + int alloc_das_batch_store(); + int save_das_batch_store(); + int resume_das_batch_store(); + // for das batch spf end ObOperator &op_; bool onetime_plan_; bool init_plan_; @@ -124,6 +131,12 @@ private: int64_t batch_row_pos_; bool iter_end_; // for vectorized end + + // for das batch spf + bool is_new_batch_; + uint64_t current_group_; + common::ObArrayWrap das_batch_params_recovery_; + // for das batch spf end }; class ObSubPlanFilterSpec : public ObOpSpec @@ -154,6 +167,8 @@ public: bool enable_das_batch_rescans_; ExprFixedArray filter_exprs_; ExprFixedArray output_exprs_; + common::ObFixedArray left_rescan_params_; + common::ObFixedArray right_rescan_params_; }; class ObSubPlanFilterOp : public ObOperator @@ -181,17 +196,28 @@ public: } int handle_next_row(); bool enable_px_batch_rescan() { return enable_left_px_batch_; } - bool enable_das_batch_rescan() { return enable_left_das_batch_; } //for vectorized int inner_get_next_batch(const int64_t max_row_cnt); // for vectorized end int init_left_cur_row(const int64_t column_cnt, ObExecContext &ctx); int fill_cur_row_rescan_param(); + + //for DAS batch SPF + int fill_cur_row_das_batch_param(ObEvalCtx& eval_ctx, uint64_t current_group) const; + int bind_das_batch_params_to_store() const; + void get_current_group(uint64_t& current_group) const; + bool enable_left_das_batch() const {return MY_SPEC.enable_das_batch_rescans_;} + //for DAS batch SPF end + + const ObSubPlanFilterSpec &get_spec() const + { return static_cast(spec_); } + public: ObBatchRescanCtl &get_batch_rescan_ctl() { return batch_rescan_ctl_; } static const int64_t PX_RESCAN_BATCH_ROW_COUNT = 8192; int handle_next_batch_with_px_rescan(const int64_t op_max_batch_size); + int handle_next_batch_with_group_rescan(const int64_t op_max_batch_size); private: void set_param_null() { set_pushdown_param_null(MY_SPEC.rescan_params_); }; void destroy_subplan_iters(); @@ -208,19 +234,31 @@ private: int prepare_onetime_exprs(); int prepare_onetime_exprs_inner(); int handle_update_set(); - bool continue_fetching(uint64_t left_rows_total_cnt, bool stop) + bool continue_fetching(uint64_t left_rows_total_cnt, bool stop, bool use_group = false) { - return (!stop && (left_rows_total_cnt < PX_RESCAN_BATCH_ROW_COUNT)); + return use_group? + (!stop && (left_rows_total_cnt < max_group_size_)) + :(!stop && (left_rows_total_cnt < PX_RESCAN_BATCH_ROW_COUNT)); } + // for das batch spf + int alloc_das_batch_params(uint64_t group_size); + int init_das_batch_params(); + int deep_copy_dynamic_obj(); + // for das batch spf end + private: common::ObSEArray subplan_iters_; lib::MemoryContext update_set_mem_; bool iter_end_; // for px batch rescan bool enable_left_px_batch_; + // for px batch rescan end // for das batch rescan - bool enable_left_das_batch_; + uint64_t max_group_size_; //Das batch rescan size; + uint64_t current_group_; //The group id in this time right iter rescan; + common::ObArrayWrap das_batch_params_; + // for das batch rescan end ObChunkDatumStore left_rows_; ObChunkDatumStore::Iterator left_rows_iter_; ObChunkDatumStore::ShadowStoredRow last_store_row_; @@ -237,6 +275,35 @@ private: ObBatchResultHolder brs_holder_; }; +class GroupParamBackupGuard +{ +public: + GroupParamBackupGuard(ObEvalCtx& eval_ctx, + common::ObArrayWrap& das_batch_params_recovery, + const common::ObFixedArray& rescan_params, + int64_t params_count) + : eval_ctx_(eval_ctx), + das_batch_params_recovery_(das_batch_params_recovery), + rescan_params_(rescan_params), + params_count_(params_count) + { + save_das_batch_store(); + } + ~GroupParamBackupGuard() + { + resume_das_batch_store(); + } +private: + void save_das_batch_store(); + void resume_das_batch_store(); +private: + ObEvalCtx& eval_ctx_; + common::ObArrayWrap& das_batch_params_recovery_; + const common::ObFixedArray& rescan_params_; + int64_t params_count_; +}; + + } // end namespace sql } // end namespace oceanbase diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index 496aae8c12..96ad4087b1 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -678,8 +678,8 @@ int ObTableScanOp::prepare_pushdown_limit_param() } else if (MY_SPEC.batch_scan_flag_) { //batch scan can not pushdown limit param to storage need_final_limit_ = true; - limit_param_.offset_ = 0; - limit_param_.limit_ = -1; + tsc_rtdef_.scan_rtdef_.limit_param_.offset_ = 0; + tsc_rtdef_.scan_rtdef_.limit_param_.limit_ = -1; } else if (tsc_rtdef_.has_lookup_limit() || das_ref_.get_das_task_cnt() > 1) { //for index back, need to final limit output rows in TableScan operator, //please see me for the reason: https://work.aone.alibaba-inc.com/issue/43232745 @@ -1226,8 +1226,8 @@ int ObTableScanOp::inner_open() } if (OB_SUCC(ret)) { // here need add plan batch_size, because in vectorized execution, - // left batch may greater than BNLJ_DEFAULT_GROUP_SIZE - max_group_size_ = BNLJ_DEFAULT_GROUP_SIZE + MY_SPEC.plan_->get_batch_size(); + // left batch may greater than OB_MAX_BULK_JOIN_ROWS + max_group_size_ = OB_MAX_BULK_JOIN_ROWS + MY_SPEC.plan_->get_batch_size(); if (MY_CTDEF.pre_query_range_.get_is_equal_and()) { int64_t column_count = MY_CTDEF.pre_query_range_.get_column_count(); size_t range_size = sizeof(ObNewRange) + sizeof(ObObj) * column_count * 2; @@ -1398,7 +1398,7 @@ int ObTableScanOp::inner_rescan() // replaced by NLJ. LOG_WARN("build batch nlj params failed", KR(ret)); } else if (!need_fetch_batch_result()) { - ret = switch_batch_iter(); + ret = set_batch_iter(ctx_.get_das_ctx().jump_read_group_id_); } else { if (is_virtual_table(MY_SPEC.ref_table_id_) || !das_ref_.is_all_local_task() @@ -1563,6 +1563,37 @@ int ObTableScanOp::switch_batch_iter() return ret; } +int ObTableScanOp::set_batch_iter(int64_t group_id) +{ + int ret = OB_SUCCESS; + for (DASTaskIter task_iter = das_ref_.begin_task_iter(); + OB_SUCC(ret) && !task_iter.is_end(); ++task_iter) { + ObDASGroupScanOp *group_scan_op = DAS_GROUP_SCAN_OP(*task_iter); + if (OB_FAIL(group_scan_op->set_scan_group(group_id))) { + if (OB_ITER_END != ret) { + LOG_WARN("switch batch iter failed", K(ret)); + } else { + iter_end_ = true; + } + } + } + if (OB_SUCC(ret) && !iter_end_) { + if (!das_ref_.has_task()) { + iter_end_ = true; + } else { + //prepare to output row + scan_result_ = das_ref_.begin_result_iter(); + if (OB_FAIL(update_output_tablet_id())) { + LOG_WARN("update output row pkey failed", K(ret), KPC(scan_result_.get_tablet_loc())); + } + } + } + return ret; +} + + + + int ObTableScanOp::get_next_row_with_das() { int ret = OB_SUCCESS; @@ -1589,12 +1620,31 @@ int ObTableScanOp::get_next_row_with_das() LOG_WARN("get next row from das result failed", K(ret)); } } else { - ++input_row_cnt_; + // We need do filter first before do the limit. + // See the issue 47201028. + bool filtered = false; + if (need_final_limit_ && !MY_SPEC.filters_.empty()) { + if (OB_FAIL(filter_row(filtered))) { + LOG_WARN("das get_next_row filter row failed", K(ret)); + } else { + if(filtered) { + //Do nothing + } else { + ++input_row_cnt_; + } + } + } else { + ++input_row_cnt_; + } if (need_final_limit_ && input_row_cnt_ <= limit_param_.offset_) { continue; } else { - ++output_row_cnt_; - got_row = true; + if (need_final_limit_ && !MY_SPEC.filters_.empty() && filtered) { + //Do nothing + } else { + ++output_row_cnt_; + got_row = true; + } } } } @@ -1635,7 +1685,26 @@ int ObTableScanOp::get_next_batch_with_das(int64_t &count, int64_t capacity) LOG_WARN("get next batch from das result failed", K(ret)); } } else { - input_row_cnt_ += count; + // We need do filter first before do the limit. + // See the issue 47201028. + if(need_final_limit_ && !MY_SPEC.filters_.empty() && count > 0) { + bool all_filtered = false; + if (OB_FAIL(filter_batch_rows(MY_SPEC.filters_, + *brs_.skip_, + count, + all_filtered))) { + LOG_WARN("filter batch failed in das get_next_batch", K(ret)); + } else if (all_filtered) { + //Do nothing. + brs_.skip_->reset(count); + } else { + int64_t skipped_rows_count = brs_.skip_->accumulate_bit_cnt(count); + input_row_cnt_ += count - skipped_rows_count; + brs_.skip_->reset(count); + } + } else { + input_row_cnt_ += count; + } } } if (OB_SUCC(ret) && need_final_limit_) { @@ -1671,9 +1740,29 @@ int ObTableScanOp::get_next_batch_with_das(int64_t &count, int64_t capacity) LOG_WARN("get next batch from das result failed", K(ret)); } } else { - got_batch = true; - output_row_cnt_ += count; - input_row_cnt_ += count; + // We need do filter first before do the limit. + // See the issue 47201028. + if (need_final_limit_ && !MY_SPEC.filters_.empty() && count > 0) { + bool all_filtered = false; + if (OB_FAIL(filter_batch_rows(MY_SPEC.filters_, + *brs_.skip_, + count, + all_filtered))) { + LOG_WARN("filter batch failed in das get_next_batch", K(ret)); + } else if (all_filtered) { + //Do nothing. + brs_.skip_->reset(count); + } else { + int64_t skipped_rows_count = brs_.skip_->accumulate_bit_cnt(count); + got_batch = true; + output_row_cnt_ += (count - skipped_rows_count); + input_row_cnt_ += (count - skipped_rows_count); + } + } else { + got_batch = true; + output_row_cnt_ += count; + input_row_cnt_ += count; + } } } return ret; diff --git a/src/sql/engine/table/ob_table_scan_op.h b/src/sql/engine/table/ob_table_scan_op.h index 5846edae35..47b624c734 100644 --- a/src/sql/engine/table/ob_table_scan_op.h +++ b/src/sql/engine/table/ob_table_scan_op.h @@ -383,6 +383,7 @@ protected: int local_iter_reuse(); int switch_batch_iter(); + int set_batch_iter(int64_t group_id); int calc_expr_int_value(const ObExpr &expr, int64_t &retval, bool &is_null_value); int init_table_scan_rtdef(); int init_das_scan_rtdef(const ObDASScanCtDef &das_ctdef, diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 68e1dcaa20..5c02599933 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -4906,18 +4906,26 @@ int JoinPath::can_use_batch_nlj(ObLogPlan *plan, || table_item->for_update_ || !access_path->subquery_exprs_.empty() ); - for (int64_t i = 0; OB_SUCC(ret) && use_batch_nlj && i < access_path->filter_.count(); ++i) { - const ObRawExpr *expr = access_path->filter_.at(i); - if (OB_ISNULL(expr)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("table filter is null", K(ret)); - } else if (OB_NOT_NULL(access_path->pre_query_range_) && - ObOptimizerUtil::find_item(access_path->pre_query_range_->get_range_exprs(), expr)) { - //range expr, do nothing - } else { - use_batch_nlj = !expr->has_flag(CNT_DYNAMIC_PARAM); + + if (use_batch_nlj) { + bool found_query_range = false; + for (int64_t i = 0; OB_SUCC(ret) && i < access_path->filter_.count() && !found_query_range; ++i) { + const ObRawExpr *expr = access_path->filter_.at(i); + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table filter is null", K(ret)); + } else if (OB_NOT_NULL(access_path->pre_query_range_) && + ObOptimizerUtil::find_item(access_path->pre_query_range_->get_range_exprs(), expr)) { + found_query_range = true; + } else { + //do nothing + //dynamic filter will keep in tsc operator + //static filter will pushdown to storage + } } + use_batch_nlj = use_batch_nlj && found_query_range; } + } return ret; } @@ -5006,7 +5014,8 @@ int JoinPath::can_use_das_batch_nlj(ObLogicalOperator* root, bool &use_batch_nlj if (OB_FAIL(SMART_CALL(can_use_das_batch_nlj(root->get_child(0), use_batch_nlj)))) { LOG_WARN("failed to check das batch nlj", K(ret)); } - } else if (log_op_def::LOG_SET == root->get_type()) { + } else if (log_op_def::LOG_SET == root->get_type() + || log_op_def::LOG_JOIN == root->get_type()) { use_batch_nlj = true; for (int64_t i = 0; OB_SUCC(ret) && use_batch_nlj && i < root->get_num_of_child(); ++i) { ObLogicalOperator *child = root->get_child(i); diff --git a/src/sql/optimizer/ob_log_join.cpp b/src/sql/optimizer/ob_log_join.cpp index 31e35beb79..0f473be2df 100644 --- a/src/sql/optimizer/ob_log_join.cpp +++ b/src/sql/optimizer/ob_log_join.cpp @@ -1158,15 +1158,43 @@ int ObLogJoin::set_use_batch(ObLogicalOperator* root) } else if (root->is_table_scan()) { ObLogTableScan *ts = static_cast(root); // dblink can not support batch nlj - if (!ts->get_range_conditions().empty()) { - ts->set_use_batch(can_use_batch_nlj_); + bool has_param = false; + ObSEArray idx_array; + if (OB_FAIL(ts->extract_bnlj_param_idxs(idx_array))) { + LOG_WARN("extract param indexes failed", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && !has_param && i < nl_params_.count(); i++) { + int64_t param_idx = nl_params_.at(i)->get_param_index(); + for (int64_t j = 0; OB_SUCC(ret) && j < idx_array.count(); j++) { + if (param_idx == idx_array.at(j)) { + has_param = true; + } + } + } + if (OB_SUCC(ret)) { + ts->set_use_batch(ts->use_batch() || (can_use_batch_nlj_ && has_param)); } } else if (root->get_num_of_child() == 1) { if (log_op_def::LOG_TABLE_LOOKUP == root->get_type()) { - ObLogTableLookup *tlu = static_cast(root); - tlu->set_use_batch(can_use_batch_nlj_); - } - if(OB_FAIL(SMART_CALL(set_use_batch(root->get_child(first_child))))) { + ObLogTableLookup *tlu = NULL; + ObLogTableScan *ts = NULL; + if (OB_ISNULL(tlu = static_cast(root)) + || OB_ISNULL(root->get_child(first_child)) + || !root->get_child(first_child)->is_table_scan() + || OB_ISNULL(ts = static_cast(root->get_child(first_child)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid input", K(ret)); + } else { + if (!ts->get_filter_exprs().empty()) { + tlu->set_use_batch(false); + ts->set_use_batch(false); + } else if (OB_FAIL(SMART_CALL(set_use_batch(root->get_child(first_child))))) { + LOG_WARN("failed to check use batch nlj", K(ret)); + } else { + tlu->set_use_batch(tlu->use_batch() || (can_use_batch_nlj_ && ts->use_batch())); + } + } + } else if (OB_FAIL(SMART_CALL(set_use_batch(root->get_child(first_child))))) { LOG_WARN("failed to check use batch nlj", K(ret)); } } else if (log_op_def::LOG_SET == root->get_type()) { @@ -1175,14 +1203,85 @@ int ObLogJoin::set_use_batch(ObLogicalOperator* root) if (OB_ISNULL(child)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid child", K(ret)); - } else if(OB_FAIL(SMART_CALL(set_use_batch(child)))) { + } else if (OB_FAIL(SMART_CALL(set_use_batch(child)))) { LOG_WARN("failed to check use batch nlj", K(ret)); } } + } else if (log_op_def::LOG_JOIN == root->get_type()) { + ObLogJoin *nlj = NULL; + ObLogicalOperator *child = NULL; + if (OB_ISNULL(nlj = static_cast(root)) + || OB_ISNULL(child = nlj->get_child(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid input", K(ret)); + } else if (OB_FAIL(SMART_CALL(set_use_batch(child)))) { + LOG_WARN("failed to check use batch nlj", K(ret)); + } else if (!nlj->can_use_batch_nlj()) { + // do nothing + } else if (OB_ISNULL(child = nlj->get_child(1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid child", K(ret)); + } else if (OB_FAIL(SMART_CALL(set_use_batch(child)))) { + LOG_WARN("failed to check use batch nlj", K(ret)); + } } else { /*do nothing*/ } return ret; } +int ObLogJoin::check_and_set_use_batch() +{ + int ret = OB_SUCCESS; + ObSQLSessionInfo *session_info = NULL; + ObLogPlan *plan = NULL; + if (OB_ISNULL(plan = get_plan()) + || OB_ISNULL(session_info = plan->get_optimizer_context().get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!can_use_batch_nlj_) { + // do nothing + } else if (OB_FAIL(session_info->get_nlj_batching_enabled(can_use_batch_nlj_))) { + LOG_WARN("failed to get enable batch variable", K(ret)); + } else if (NESTED_LOOP_JOIN != get_join_algo()) { + can_use_batch_nlj_ = false; + } + // check use batch + if (OB_SUCC(ret) && can_use_batch_nlj_) { + bool contains_invalid_startup = false; + bool contains_limit = false; + if (get_child(1)->get_type() == log_op_def::LOG_GRANULE_ITERATOR) { + can_use_batch_nlj_ = false; + } else if (OB_FAIL(plan->contains_startup_with_exec_param(get_child(1), + contains_invalid_startup))) { + LOG_WARN("failed to check contains invalid startup", K(ret)); + } else if (contains_invalid_startup) { + can_use_batch_nlj_ = false; + } else if (OB_FAIL(plan->contains_limit_or_pushdown_limit(get_child(1), contains_limit))) { + LOG_WARN("failed to check contains limit", K(ret)); + } else if (contains_limit) { + can_use_batch_nlj_ = false; + } else if (log_op_def::LOG_TABLE_LOOKUP == get_child(1)->get_type()) { + ObLogTableScan *ts = NULL; + if (OB_ISNULL(get_child(1)->get_child(0)) + || !get_child(1)->get_child(0)->is_table_scan()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid input", K(ret)); + } else if (OB_ISNULL(ts = static_cast(get_child(1)->get_child(0)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid input", K(ret)); + } else if (!ts->get_filter_exprs().empty()) { + can_use_batch_nlj_ = false; + } + } + } + // set use batch + if (OB_SUCC(ret) && can_use_batch_nlj_) { + if (OB_FAIL(set_use_batch(get_child(1)))) { + LOG_WARN("failed to set use batch nlj", K(ret)); + } + } + return ret; +} + bool ObLogJoin::is_my_exec_expr(const ObRawExpr *expr) { return ObOptimizerUtil::find_item(nl_params_, expr); diff --git a/src/sql/optimizer/ob_log_join.h b/src/sql/optimizer/ob_log_join.h index 853af2d465..5cdaebf5cf 100644 --- a/src/sql/optimizer/ob_log_join.h +++ b/src/sql/optimizer/ob_log_join.h @@ -150,11 +150,15 @@ namespace sql inline bool can_use_batch_nlj() const { return can_use_batch_nlj_; } void set_can_use_batch_nlj(bool can_use) { can_use_batch_nlj_ = can_use; } - int set_use_batch(ObLogicalOperator* root); + int check_and_set_use_batch(); void set_join_path(JoinPath *path) { join_path_ = path; } JoinPath *get_join_path() { return join_path_; } bool is_my_exec_expr(const ObRawExpr *expr); + common::ObIArray &get_above_pushdown_left_params() { return above_pushdown_left_params_; } + common::ObIArray &get_above_pushdown_right_params() { return above_pushdown_right_params_; } + private: + int set_use_batch(ObLogicalOperator* root); inline bool can_enable_gi_partition_pruning() { return (NESTED_LOOP_JOIN == join_algo_) @@ -210,6 +214,7 @@ namespace sql int print_join_tables_in_hint(const ObDMLStmt &stmt, planText &plan_text, const ObRelIds &table_set); + private: // all join predicates common::ObSEArray join_conditions_; //equal join condition, for merge-join @@ -233,6 +238,8 @@ namespace sql common::ObSEArray join_filter_infos_; bool can_use_batch_nlj_; JoinPath *join_path_; + common::ObSEArray above_pushdown_left_params_; + common::ObSEArray above_pushdown_right_params_; DISALLOW_COPY_AND_ASSIGN(ObLogJoin); }; diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index f6f8b7869b..d3d815297b 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -9303,178 +9303,138 @@ int ObLogPlan::plan_tree_traverse(const TraverseOp &operation, void *ctx) AllocBloomFilterContext bf_ctx; GenLinkStmtPostContext link_ctx(get_allocator(), get_optimizer_context().get_schema_guard()); CopyPartExprCtx copy_part_expr_ctx; - - // set up context - switch (operation) { - case PX_PIPE_BLOCKING: { - ctx = &pipe_block_ctx; - if (OB_FAIL(get_plan_root()->init_all_traverse_ctx(pipe_block_ctx))) { - LOG_WARN("init traverse ctx failed", K(ret)); - } - break; - } - case ALLOC_GI: { - ctx = &gi_ctx; - bool is_valid = true; - if (get_stmt()->is_insert_stmt() && - !static_cast(get_stmt())->value_from_select()) { - gi_ctx.is_valid_for_gi_ = false; - } else if (OB_FAIL(get_plan_root()->should_allocate_gi_for_dml(is_valid))) { - LOG_WARN("failed to check should allocate gi for dml", K(ret)); - } else { - gi_ctx.is_valid_for_gi_ = get_stmt()->is_dml_write_stmt() && is_valid; - } - break; - } - case ALLOC_MONITORING_DUMP: { - ctx = &md_ctx; - break; - } - case BLOOM_FILTER: { - ctx = &bf_ctx; - break; - } - case PROJECT_PRUNING: { - ctx = &output_deps; - break; - } - case COPY_PART_EXPR: { - ctx = ©_part_expr_ctx; - break; - } - case ALLOC_EXPR: { - if (OB_FAIL(alloc_expr_ctx.flattern_expr_map_.create(128, "ExprAlloc"))) { - LOG_WARN("failed to init hash map", K(ret)); - } else { - ctx = &alloc_expr_ctx; - } - break; - } - case OPERATOR_NUMBERING: { - ctx = &numbering_ctx; - break; - } - case EXCHANGE_NUMBERING: { - ctx = &numbering_exchange_ctx; - break; - } - case GEN_SIGNATURE: { - ctx = &hash_seed; - break; - } - case GEN_LOCATION_CONSTRAINT: { - ctx = &location_constraints; - break; - } - case PX_ESTIMATE_SIZE: - break; - case GEN_LINK_STMT: { - if (OB_FAIL(link_ctx.init())) { - LOG_WARN("failed to init link_ctx", K(operation), K(ret)); - } else { - ctx = &link_ctx; - } - break; - } - case EXPLAIN_COLLECT_WIDTH: - case EXPLAIN_WRITE_BUFFER: - case EXPLAIN_WRITE_BUFFER_OUTPUT: - case EXPLAIN_WRITE_BUFFER_OUTLINE: - case EXPLAIN_INDEX_SELECTION_INFO: - case ALLOC_STARTUP_EXPR: - default: - break; - } - if (OB_SUCC(ret)) { - if (((PX_ESTIMATE_SIZE == operation) || - (PX_PIPE_BLOCKING == operation) || - (PX_RESCAN == operation)) && - (get_optimizer_context().is_local_or_remote_plan())) { - /*do nothing*/ - } else if (ALLOC_GI == operation && - get_optimizer_context().is_local_or_remote_plan() && - !(gi_ctx.is_valid_for_gi_ && - get_optimizer_context().enable_batch_rpc())) { - /*do nothing*/ - } else if (OB_FAIL(get_plan_root()->do_plan_tree_traverse(operation, ctx))) { - LOG_WARN("failed to apply operation to operator", K(operation), K(ret)); - } else { - // remember signature in plan - if (GEN_SIGNATURE == operation) { - if (OB_ISNULL(ctx)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ctx is null", K(ret), K(ctx)); + SMART_VAR(ObBatchExecParamCtx, batch_exec_param_ctx) { + // set up context + switch (operation) { + case PX_PIPE_BLOCKING: { + ctx = &pipe_block_ctx; + if (OB_FAIL(get_plan_root()->init_all_traverse_ctx(pipe_block_ctx))) { + LOG_WARN("init traverse ctx failed", K(ret)); + } + break; + } + case ALLOC_GI: { + ctx = &gi_ctx; + bool is_valid = true; + if (get_stmt()->is_insert_stmt() && + !static_cast(get_stmt())->value_from_select()) { + gi_ctx.is_valid_for_gi_ = false; + } else if (OB_FAIL(get_plan_root()->should_allocate_gi_for_dml(is_valid))) { + LOG_WARN("failed to check should allocate gi for dml", K(ret)); } else { - hash_value_ = *static_cast(ctx); - LOG_TRACE("succ to generate plan hash value", "hash_value", hash_value_); + gi_ctx.is_valid_for_gi_ = get_stmt()->is_dml_write_stmt() && is_valid; } - } else if (GEN_LOCATION_CONSTRAINT == operation) { - ObSqlCtx *sql_ctx = NULL; - if (OB_ISNULL(optimizer_context_.get_exec_ctx()) - || OB_ISNULL(sql_ctx = optimizer_context_.get_exec_ctx()->get_sql_ctx())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(optimizer_context_.get_exec_ctx()), K(sql_ctx)); - } else if (OB_FAIL(remove_duplicate_constraint(location_constraints, - *sql_ctx))) { - LOG_WARN("fail to remove duplicate constraint", K(ret)); - } else if (OB_FAIL(calc_and_set_exec_pwj_map(location_constraints))) { - LOG_WARN("failed to calc and set exec pwj map", K(ret)); + break; + } + case ALLOC_MONITORING_DUMP: { + ctx = &md_ctx; + break; + } + case BLOOM_FILTER: { + ctx = &bf_ctx; + break; + } + case PROJECT_PRUNING: { + ctx = &output_deps; + break; + } + case COPY_PART_EXPR: { + ctx = ©_part_expr_ctx; + break; + } + case ALLOC_EXPR: { + if (OB_FAIL(alloc_expr_ctx.flattern_expr_map_.create(128, "ExprAlloc"))) { + LOG_WARN("failed to init hash map", K(ret)); + } else { + ctx = &alloc_expr_ctx; } - } else if (OPERATOR_NUMBERING == operation) { - NumberingCtx *num_ctx = static_cast(ctx); - max_op_id_ = num_ctx->op_id_; - LOG_TRACE("trace max operator id", K(max_op_id_), K(this)); - } else { /* Do nothing */ } - LOG_TRACE("succ to apply operaion to operator", K(operation), K(ret)); + break; + } + case OPERATOR_NUMBERING: { + ctx = &numbering_ctx; + break; + } + case EXCHANGE_NUMBERING: { + ctx = &numbering_exchange_ctx; + break; + } + case GEN_SIGNATURE: { + ctx = &hash_seed; + break; + } + case GEN_LOCATION_CONSTRAINT: { + ctx = &location_constraints; + break; + } + case PX_ESTIMATE_SIZE: + break; + case GEN_LINK_STMT: { + if (OB_FAIL(link_ctx.init())) { + LOG_WARN("failed to init link_ctx", K(operation), K(ret)); + } else { + ctx = &link_ctx; + } + break; + } + case COLLECT_BATCH_EXEC_PARAM: { + ctx = &batch_exec_param_ctx; + break; + } + case EXPLAIN_COLLECT_WIDTH: + case EXPLAIN_WRITE_BUFFER: + case EXPLAIN_WRITE_BUFFER_OUTPUT: + case EXPLAIN_WRITE_BUFFER_OUTLINE: + case EXPLAIN_INDEX_SELECTION_INFO: + case ALLOC_STARTUP_EXPR: + default: + break; + } + if (OB_SUCC(ret)) { + if (((PX_ESTIMATE_SIZE == operation) || + (PX_PIPE_BLOCKING == operation) || + (PX_RESCAN == operation)) && + (get_optimizer_context().is_local_or_remote_plan())) { + /*do nothing*/ + } else if (ALLOC_GI == operation && + get_optimizer_context().is_local_or_remote_plan() && + !(gi_ctx.is_valid_for_gi_ && + get_optimizer_context().enable_batch_rpc())) { + /*do nothing*/ + } else if (OB_FAIL(get_plan_root()->do_plan_tree_traverse(operation, ctx))) { + LOG_WARN("failed to apply operation to operator", K(operation), K(ret)); + } else { + // remember signature in plan + if (GEN_SIGNATURE == operation) { + if (OB_ISNULL(ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ctx is null", K(ret), K(ctx)); + } else { + hash_value_ = *static_cast(ctx); + LOG_TRACE("succ to generate plan hash value", "hash_value", hash_value_); + } + } else if (GEN_LOCATION_CONSTRAINT == operation) { + ObSqlCtx *sql_ctx = NULL; + if (OB_ISNULL(optimizer_context_.get_exec_ctx()) + || OB_ISNULL(sql_ctx = optimizer_context_.get_exec_ctx()->get_sql_ctx())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(optimizer_context_.get_exec_ctx()), K(sql_ctx)); + } else if (OB_FAIL(remove_duplicate_constraint(location_constraints, + *sql_ctx))) { + LOG_WARN("fail to remove duplicate constraint", K(ret)); + } else if (OB_FAIL(calc_and_set_exec_pwj_map(location_constraints))) { + LOG_WARN("failed to calc and set exec pwj map", K(ret)); + } + } else if (OPERATOR_NUMBERING == operation) { + NumberingCtx *num_ctx = static_cast(ctx); + max_op_id_ = num_ctx->op_id_; + LOG_TRACE("trace max operator id", K(max_op_id_), K(this)); + } else { /* Do nothing */ } + LOG_TRACE("succ to apply operaion to operator", K(operation), K(ret)); + } } } } - return ret; -} -int ObLogPlan::check_and_reset_batch_nlj(ObLogicalOperator *root) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(root)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid root", K(ret)); - } else if (root->get_type() == log_op_def::LOG_JOIN && - static_cast(root)->get_join_algo() == NESTED_LOOP_JOIN && - static_cast(root)->can_use_batch_nlj() && - OB_NOT_NULL(root->get_child(1))) { - bool need_reset = false; - bool contains_invalid_startup = false; - bool contains_limit = false; - if (root->get_child(1)->get_type() == log_op_def::LOG_GRANULE_ITERATOR) { - need_reset = true; - } else if (OB_FAIL(contains_startup_with_exec_param(root->get_child(1), - contains_invalid_startup))) { - LOG_WARN("failed to check contains invalid startup", K(ret)); - } else if (contains_invalid_startup) { - need_reset = true; - } else if (OB_FAIL(contains_limit_or_pushdown_limit(root->get_child(1), contains_limit))) { - LOG_WARN("failed to check contains limit", K(ret)); - } else if (contains_limit) { - need_reset = true; - } - if (OB_SUCC(ret) && need_reset) { - ObLogJoin *nl_join_op = static_cast(root); - nl_join_op->set_can_use_batch_nlj(false); - if (OB_FAIL(nl_join_op->set_use_batch(nl_join_op->get_child(1)))) { - LOG_WARN("failed to reset batch join flag", K(ret)); - } else {/*do nothing*/} - } - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < root->get_num_of_child(); ++i) { - ObLogicalOperator *child = root->get_child(i); - if (OB_ISNULL(child)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid child", K(ret)); - } else if (OB_FAIL(SMART_CALL(check_and_reset_batch_nlj(child)))) { - LOG_WARN("failed to refresh batch nlj", K(ret)); - } else {/*do nothing*/} - } - } return ret; } @@ -10741,7 +10701,8 @@ int ObLogPlan::generate_plan() GEN_LOCATION_CONSTRAINT, PX_ESTIMATE_SIZE, GEN_LINK_STMT, - ALLOC_STARTUP_EXPR))) { + ALLOC_STARTUP_EXPR, + COLLECT_BATCH_EXEC_PARAM))) { LOG_WARN("failed to do plan traverse", K(ret)); } else if (OB_FAIL(do_post_traverse_processing())) { LOG_WARN("failed to post traverse processing", K(ret)); @@ -10756,9 +10717,6 @@ int ObLogPlan::do_post_traverse_processing() if (OB_ISNULL(root = get_plan_root())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); - } else if (OB_FAIL(check_and_reset_batch_nlj(root))) { - // refresh nlj batch flag for right side gi or startup with exec param - LOG_WARN("failed to refresh batch nlj", K(ret)); } else if (OB_FAIL(calc_plan_resource())) { LOG_WARN("fail calc plan resource", K(ret)); } else { /*do nothing*/ } @@ -10929,9 +10887,9 @@ int ObLogPlan::adjust_final_plan_info(ObLogicalOperator *&op) } else if (OB_FAIL(op->reorder_filter_exprs())) { LOG_WARN("failed to reorder filter exprs", K(ret)); } else if (log_op_def::LOG_JOIN == op->get_type() && - OB_FAIL(static_cast(op)->set_use_batch(op->get_child(1)))) { + OB_FAIL(static_cast(op)->check_and_set_use_batch())) { LOG_WARN("failed to set use batch nlj", K(ret)); - } else if (log_op_def::LOG_SUBPLAN_FILTER == op->get_type() && false && // TODO: chenxuan open it + } else if (log_op_def::LOG_SUBPLAN_FILTER == op->get_type() && OB_FAIL(static_cast(op)->check_and_set_use_batch())) { LOG_WARN("failed to set use batch spf", K(ret)); } else { /*do nothing*/ } diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index 046bfcce8d..2650924ba1 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -1221,8 +1221,6 @@ public: int contains_limit_or_pushdown_limit(ObLogicalOperator *op, bool &contains); - int check_and_reset_batch_nlj(ObLogicalOperator *root); - /** * 递归处理expr里的SubQuery，遇到SubLink就生成一个SubPlan * @param expr diff --git a/src/sql/optimizer/ob_log_subplan_filter.cpp b/src/sql/optimizer/ob_log_subplan_filter.cpp index 9f064aa67f..07bf5ae307 100644 --- a/src/sql/optimizer/ob_log_subplan_filter.cpp +++ b/src/sql/optimizer/ob_log_subplan_filter.cpp @@ -210,7 +210,7 @@ int ObLogSubPlanFilter::est_cost() LOG_WARN("failed to get child", K(ret), K(first_child)); } else if (OB_FALSE_IT(first_child_card = first_child->get_card())) { } else if (OB_FAIL(inner_est_cost(first_child_card, op_cost_))) { - LOG_WARN("failed to est cost", K(ret)); + LOG_WARN("failed to est cost", K(ret)); } else { set_cost(op_cost_ + first_child->get_cost()); set_card(first_child_card); @@ -238,8 +238,8 @@ int ObLogSubPlanFilter::re_est_cost(EstimateCostInfo ¶m, double &card, doubl } else if (OB_ISNULL(child = get_child(ObLogicalOperator::first_child))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); - } else if (param.need_row_count_ >= 0 && - param.need_row_count_ < get_card() && + } else if (param.need_row_count_ >= 0 && + param.need_row_count_ < get_card() && sel > OB_DOUBLE_EPSINON && OB_FALSE_IT(param.need_row_count_ /= sel)) { } else if (OB_FAIL(SMART_CALL(child->re_est_cost(param, @@ -449,7 +449,6 @@ int ObLogSubPlanFilter::check_if_match_das_batch_rescan(ObLogicalOperator *root, bool &enable_das_batch_rescans) { int ret = OB_SUCCESS; - enable_das_batch_rescans = false; if (OB_ISNULL(root)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret)); @@ -457,39 +456,34 @@ int ObLogSubPlanFilter::check_if_match_das_batch_rescan(ObLogicalOperator *root, bool is_valid = false; ObLogTableScan *tsc = static_cast(root); if (!tsc->use_das()) { - // do nothing + enable_das_batch_rescans = false; } else if (OB_FAIL(ObOptimizerUtil::check_contribute_query_range(root, get_exec_params(), is_valid))) { LOG_WARN("failed to check query range contribution", K(ret)); - } else if (is_valid) { - enable_das_batch_rescans = true; - } - } else if (root->get_num_of_child() == 1 && - OB_FAIL(SMART_CALL(check_if_match_das_batch_rescan(root->get_child(0), - enable_das_batch_rescans)))) { - LOG_WARN("failed to check match das batch rescan", K(ret)); - } else {/*do nothing*/} - return ret; -} - -int ObLogSubPlanFilter::check_if_match_das_batch_rescan(bool &enable_das_batch_rescans) -{ - int ret = OB_SUCCESS; - enable_das_batch_rescans = true; - for (int64_t i = 1; OB_SUCC(ret) && enable_das_batch_rescans && i < get_num_of_child(); ++i) { - ObLogicalOperator *child = get_child(i); - if (OB_ISNULL(child)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret)); - } else if (get_initplan_idxs().has_member(i) || get_onetime_idxs().has_member(i)) { + } else if (!is_valid) { enable_das_batch_rescans = false; - } else if (OB_FAIL(check_if_match_das_batch_rescan(child, enable_das_batch_rescans))) { - LOG_WARN("failed to check match das batch rescan", K(ret)); - } else { - // do nothing + } else if (tsc->get_scan_direction() != default_asc_direction()) { + enable_das_batch_rescans = false; + } else {/*do nothing*/} + } else if (root->get_num_of_child() == 1) { + if (log_op_def::LOG_TABLE_LOOKUP == root->get_type()) { + ObLogTableLookup *tlu = static_cast(root); + ObLogTableScan *ts = static_cast(root->get_child(0)); + if (OB_ISNULL(tlu) || OB_ISNULL(ts)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (!ts->get_filter_exprs().empty()) { + enable_das_batch_rescans = false; + } } - } + if (OB_SUCC(ret)) { + if (OB_FAIL(SMART_CALL(check_if_match_das_batch_rescan(root->get_child(0), + enable_das_batch_rescans)))) { + LOG_WARN("failed to check match das batch rescan", K(ret)); + } + } + } else {/*do nothing*/} return ret; } @@ -502,12 +496,12 @@ int ObLogSubPlanFilter::set_use_das_batch(ObLogicalOperator* root) } else if (root->is_table_scan()) { ObLogTableScan *ts = static_cast(root); if (!ts->get_range_conditions().empty()) { - ts->set_use_batch(true); + ts->set_use_batch(enable_das_batch_rescans_); } } else if (root->get_num_of_child() == 1) { if (log_op_def::LOG_TABLE_LOOKUP == root->get_type()) { ObLogTableLookup *tlu = static_cast(root); - tlu->set_use_batch(true); + tlu->set_use_batch(enable_das_batch_rescans_); } if(OB_FAIL(SMART_CALL(set_use_das_batch(root->get_child(first_child))))) { LOG_WARN("failed to check use das batch", K(ret)); @@ -519,25 +513,42 @@ int ObLogSubPlanFilter::set_use_das_batch(ObLogicalOperator* root) int ObLogSubPlanFilter::check_and_set_use_batch() { int ret = OB_SUCCESS; - bool &enable_das_batch_rescans = get_enable_das_batch_rescans(); - if (DistAlgo::DIST_NONE_ALL != get_distributed_algo()) { - // do nothing - } else if (OB_FAIL(check_if_match_das_batch_rescan(enable_das_batch_rescans))) { - LOG_WARN("failed to check match das batch rescan", K(ret)); - } else if (!enable_das_batch_rescans) { - // do nothing - } else { - for (int64_t i = 1; OB_SUCC(ret) && i < get_num_of_child(); i ++) { - ObLogicalOperator *child = get_child(i); - if (OB_ISNULL(child)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret)); - } else if (OB_FAIL(set_use_das_batch(child))) { - LOG_WARN("failed to set use das batch rescan", K(ret)); - } - } - LOG_TRACE("spf das batch rescan", K(enable_das_batch_rescans)); + ObSQLSessionInfo *session_info = NULL; + if (OB_ISNULL(get_plan()) || OB_ISNULL(session_info = get_plan()->get_optimizer_context().get_session_info())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(session_info->get_nlj_batching_enabled(enable_das_batch_rescans_))) { + LOG_WARN("failed to get enable batch variable", K(ret)); } + // check use batch + for (int64_t i = 1; OB_SUCC(ret) && enable_das_batch_rescans_ && i < get_num_of_child(); i++) { + ObLogicalOperator *child = get_child(i); + if (OB_ISNULL(child)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (get_initplan_idxs().has_member(i) || get_onetime_idxs().has_member(i)) { + enable_das_batch_rescans_ = false; + } else if (!(child->get_type() == log_op_def::LOG_TABLE_SCAN + || child->get_type() == log_op_def::LOG_TABLE_LOOKUP + || child->get_type() == log_op_def::LOG_SUBPLAN_SCAN)) { + enable_das_batch_rescans_ = false; + } else if (OB_FAIL(check_if_match_das_batch_rescan(child, enable_das_batch_rescans_))) { + LOG_WARN("failed to check match das batch rescan", K(ret)); + } else { + // do nothing + } + } + // set use batch + for (int64_t i = 1; OB_SUCC(ret) && i < get_num_of_child(); i++) { + ObLogicalOperator *child = get_child(i); + if (OB_ISNULL(child)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(set_use_das_batch(child))) { + LOG_WARN("failed to set use das batch rescan", K(ret)); + } + } + LOG_TRACE("spf das batch rescan", K(ret), K(enable_das_batch_rescans_)); return ret; } diff --git a/src/sql/optimizer/ob_log_subplan_filter.h b/src/sql/optimizer/ob_log_subplan_filter.h index a0b9dbc875..07e4576b1e 100644 --- a/src/sql/optimizer/ob_log_subplan_filter.h +++ b/src/sql/optimizer/ob_log_subplan_filter.h @@ -97,13 +97,13 @@ public: virtual int compute_sharding_info() override; inline DistAlgo get_distributed_algo() { return dist_algo_; } inline void set_distributed_algo(const DistAlgo set_dist_algo) { dist_algo_ = set_dist_algo; } - + int add_px_batch_rescan_flag(bool flag) { return enable_px_batch_rescans_.push_back(flag); } common::ObIArray &get_px_batch_rescans() { return enable_px_batch_rescans_; } - - inline bool &get_enable_das_batch_rescans() { return enable_das_batch_rescans_; } + + inline bool enable_das_batch_rescans() { return enable_das_batch_rescans_; } + inline void set_enable_das_batch_rescans(bool flag) { enable_das_batch_rescans_ = flag; } int check_and_set_use_batch(); - int check_if_match_das_batch_rescan(bool &enable_das_batch_rescans); int check_if_match_das_batch_rescan(ObLogicalOperator *root, bool &enable_das_batch_rescans); int set_use_das_batch(ObLogicalOperator* root); @@ -111,6 +111,11 @@ public: int allocate_startup_expr_post() override; int allocate_subquery_id(); + + common::ObIArray &get_above_pushdown_left_params() { return above_pushdown_left_params_; } + + common::ObIArray &get_above_pushdown_right_params() { return above_pushdown_right_params_; } + private: int extract_exist_style_subquery_exprs(ObRawExpr *expr, ObIArray &exist_style_exprs); @@ -128,6 +133,9 @@ protected: common::ObBitSet<> one_time_idxs_; bool update_set_; common::ObSEArray enable_px_batch_rescans_; + + common::ObSEArray above_pushdown_left_params_; + common::ObSEArray above_pushdown_right_params_; bool enable_das_batch_rescans_; private: DISALLOW_COPY_AND_ASSIGN(ObLogSubPlanFilter); diff --git a/src/sql/optimizer/ob_log_table_scan.cpp b/src/sql/optimizer/ob_log_table_scan.cpp index fdcd449526..7f93a5d15d 100755 --- a/src/sql/optimizer/ob_log_table_scan.cpp +++ b/src/sql/optimizer/ob_log_table_scan.cpp @@ -1545,12 +1545,24 @@ int ObLogTableScan::get_phy_location_type(ObTableLocationType &location_type) int ObLogTableScan::extract_bnlj_param_idxs(ObIArray &bnlj_params) { int ret = OB_SUCCESS; - ObArray param_exprs; - if (OB_FAIL(ObRawExprUtils::extract_params(range_conds_, param_exprs))) { - LOG_WARN("extract params failed", K(ret)); + ObArray range_param_exprs; + ObArray filter_param_exprs; + if (OB_FAIL(ObRawExprUtils::extract_params(range_conds_, range_param_exprs))) { + LOG_WARN("extract range params failed", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_params(filter_exprs_, filter_param_exprs))) { + LOG_WARN("extract filter params failed", K(ret)); } - for (int64_t i = 0; OB_SUCC(ret) && i < param_exprs.count(); ++i) { - ObRawExpr *expr = param_exprs.at(i); + for (int64_t i = 0; OB_SUCC(ret) && i < range_param_exprs.count(); ++i) { + ObRawExpr *expr = range_param_exprs.at(i); + if (expr->has_flag(IS_DYNAMIC_PARAM)) { + ObConstRawExpr *exec_param = static_cast(expr); + if (OB_FAIL(add_var_to_array_no_dup(bnlj_params, exec_param->get_value().get_unknown()))) { + LOG_WARN("add var to array no dup failed", K(ret)); + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < filter_param_exprs.count(); ++i) { + ObRawExpr *expr = filter_param_exprs.at(i); if (expr->has_flag(IS_DYNAMIC_PARAM)) { ObConstRawExpr *exec_param = static_cast(expr); if (OB_FAIL(add_var_to_array_no_dup(bnlj_params, exec_param->get_value().get_unknown()))) { @@ -1559,7 +1571,7 @@ int ObLogTableScan::extract_bnlj_param_idxs(ObIArray &bnlj_params) } } if (OB_SUCC(ret)) { - LOG_DEBUG("extract bnlj params", K(param_exprs), K(bnlj_params)); + LOG_DEBUG("extract bnlj params", K(range_param_exprs), K(filter_param_exprs), K(bnlj_params)); } return ret; } diff --git a/src/sql/optimizer/ob_logical_operator.cpp b/src/sql/optimizer/ob_logical_operator.cpp index bccabc4e27..77dba13cda 100644 --- a/src/sql/optimizer/ob_logical_operator.cpp +++ b/src/sql/optimizer/ob_logical_operator.cpp @@ -1703,6 +1703,12 @@ int ObLogicalOperator::do_pre_traverse_operation(const TraverseOp &op, void *ctx } break; } + case COLLECT_BATCH_EXEC_PARAM: { + if (OB_FAIL(collect_batch_exec_param_pre(ctx))) { + LOG_WARN("failed to gen batch exec param pre", K(ret)); + } + break; + } default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected access of default branch", K(op), K(ret)); @@ -1873,6 +1879,12 @@ int ObLogicalOperator::do_post_traverse_operation(const TraverseOp &op, void *ct } break; } + case COLLECT_BATCH_EXEC_PARAM: { + if (OB_FAIL(collect_batch_exec_param_post(ctx))) { + LOG_WARN("failed to gen batch exec param post", K(ret)); + } + break; + } default: break; } @@ -5491,3 +5503,118 @@ int ObLogicalOperator::need_alloc_material_for_shared_hj(ObLogicalOperator &curr } return ret; } + +int ObLogicalOperator::collect_batch_exec_param_pre(void* ctx) +{ + int ret = OB_SUCCESS; + ObBatchExecParamCtx* param_ctx = static_cast(ctx); + if (OB_ISNULL(param_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (ObLogOpType::LOG_JOIN == get_type() && + NESTED_LOOP_JOIN == static_cast(this)->get_join_algo()) { + if (OB_FAIL(param_ctx->params_idx_.push_back(param_ctx->exec_params_.count()))) { + LOG_WARN("failed to push back params idx", K(ret)); + } + } else if (ObLogOpType::LOG_SUBPLAN_FILTER == get_type()) { + if (OB_FAIL(param_ctx->params_idx_.push_back(param_ctx->exec_params_.count()))) { + LOG_WARN("failed to push back params idx", K(ret)); + } + } else if (param_ctx->params_idx_.empty()) { + /* do nothing */ + } else { + ObSEArray param_exprs; + if (OB_FAIL(ObRawExprUtils::extract_params(get_filter_exprs(), param_exprs))) { + LOG_WARN("extract params failed", K(ret)); + } else if (OB_FAIL(ObRawExprUtils::extract_params(get_startup_exprs(), param_exprs))) { + LOG_WARN("extract params failed", K(ret)); + } else if (ObLogOpType::LOG_TABLE_SCAN == get_type() && + OB_FAIL(ObRawExprUtils::extract_params(static_cast(this)->get_range_conditions(), + param_exprs))) { + LOG_WARN("extract params failed", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < param_exprs.count(); ++i) { + ObRawExpr *expr = param_exprs.at(i); + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (expr->has_flag(IS_DYNAMIC_PARAM)) { + ObBatchExecParamCtx::ExecParam param(expr, branch_id_); + if (OB_FAIL(param_ctx->exec_params_.push_back(param))) { + LOG_WARN("failed to push back exec params", K(ret)); + } + } + } + } + return ret; +} + +int ObLogicalOperator::collect_batch_exec_param_post(void* ctx) +{ + int ret = OB_SUCCESS; + ObBatchExecParamCtx* param_ctx = static_cast(ctx); + if (OB_ISNULL(param_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (ObLogOpType::LOG_SUBPLAN_FILTER == get_type()) { + ObLogSubPlanFilter* filter_op = static_cast(this); + if (OB_FAIL(collect_batch_exec_param(ctx, + filter_op->get_exec_params(), + filter_op->get_above_pushdown_left_params(), + filter_op->get_above_pushdown_right_params()))) { + LOG_WARN("failed to collect batch exec param", K(ret)); + } + } else if (ObLogOpType::LOG_JOIN == get_type() && + NESTED_LOOP_JOIN == static_cast(this)->get_join_algo()) { + ObLogJoin* join_op = static_cast(this); + if (OB_FAIL(collect_batch_exec_param(ctx, + join_op->get_nl_params(), + join_op->get_above_pushdown_left_params(), + join_op->get_above_pushdown_right_params()))) { + LOG_WARN("failed to collect batch exec param", K(ret)); + } + } + return ret; +} + +int ObLogicalOperator::collect_batch_exec_param(void* ctx, + const ObIArray &exec_params, + ObIArray &left_above_params, + ObIArray &right_above_params) +{ + int ret = OB_SUCCESS; + int64_t start = -1; + ObBatchExecParamCtx* param_ctx = static_cast(ctx); + if (OB_ISNULL(param_ctx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_UNLIKELY(param_ctx->params_idx_.empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("params idx is null", K(ret)); + } else if (OB_FAIL(param_ctx->params_idx_.pop_back(start))) { + LOG_WARN("failed to pop back idx", K(ret)); + } else if (OB_UNLIKELY(start > param_ctx->exec_params_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("params idx is large than count", K(ret)); + } else { + for (int64_t i = param_ctx->exec_params_.count() - 1; OB_SUCC(ret) && i >= start; --i) { + const ObBatchExecParamCtx::ExecParam& param = param_ctx->exec_params_.at(i); + if (!ObOptimizerUtil::find_item(exec_params, param.expr_)) { + /* do nothing */ + } else if (OB_FAIL(param_ctx->exec_params_.remove(i))) { + LOG_WARN("failed to remove params", K(ret)); + } + } + for (int64_t i = start; OB_SUCC(ret) && i < param_ctx->exec_params_.count(); ++i) { + const ObBatchExecParamCtx::ExecParam& param = param_ctx->exec_params_.at(i); + if (param.branch_id_ > branch_id_ && + OB_FAIL(right_above_params.push_back(static_cast(param.expr_)))) { + LOG_WARN("failed to push back right params", K(ret)); + } else if (param.branch_id_ <= branch_id_ && + OB_FAIL(left_above_params.push_back(static_cast(param.expr_)))) { + LOG_WARN("failed to push back left params", K(ret)); + } + } + } + return ret; +} \ No newline at end of file diff --git a/src/sql/optimizer/ob_logical_operator.h b/src/sql/optimizer/ob_logical_operator.h index d2b6f5e3c8..851734c094 100644 --- a/src/sql/optimizer/ob_logical_operator.h +++ b/src/sql/optimizer/ob_logical_operator.h @@ -726,6 +726,35 @@ private: common::ObArray op_ctxs_; }; +struct ObBatchExecParamCtx +{ + struct ExecParam + { + ExecParam() : expr_(NULL), branch_id_(0) {} + + ExecParam(ObRawExpr *expr, uint64_t branch_id) + : expr_(expr), branch_id_(branch_id) {} + + ExecParam(const ExecParam &other) + : expr_(other.expr_), + branch_id_(other.branch_id_) {} + + ExecParam &operator=(const ExecParam &other) + { + expr_ = other.expr_; + branch_id_ = other.branch_id_; + return *this; + } + + ObRawExpr *expr_; + uint64_t branch_id_; + + TO_STRING_KV(K_(expr), K_(branch_id)); + }; + common::ObSEArray params_idx_; + common::ObSEArray exec_params_; +}; + struct ObErrLogDefine { ObErrLogDefine() : @@ -1612,6 +1641,12 @@ public: int has_window_function_below(bool &has_win_func) const; int get_pushdown_op(log_op_def::ObLogOpType op_type, const ObLogicalOperator *&op) const; + int collect_batch_exec_param_pre(void* ctx); + int collect_batch_exec_param_post(void* ctx); + int collect_batch_exec_param(void* ctx, + const ObIArray &exec_params, + ObIArray &left_above_params, + ObIArray &right_above_params); public: ObSEArray child_; ObSEArray equal_param_constraints_; diff --git a/src/sql/optimizer/ob_optimizer.h b/src/sql/optimizer/ob_optimizer.h index 044c5e0940..9fe6a4b5d1 100644 --- a/src/sql/optimizer/ob_optimizer.h +++ b/src/sql/optimizer/ob_optimizer.h @@ -94,6 +94,8 @@ namespace sql GEN_LINK_STMT, ALLOC_STARTUP_EXPR, COPY_PART_EXPR, + + COLLECT_BATCH_EXEC_PARAM, TRAVERSE_OP_END }; diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index facf8467f3..6ef1696520 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -7603,7 +7603,7 @@ int ObRawExprUtils::extract_params(common::ObIArray &exprs, return ret; } -int ObRawExprUtils::is_contain_params(common::ObIArray &exprs, +int ObRawExprUtils::is_contain_params(const common::ObIArray &exprs, bool &is_contain) { int ret = OB_SUCCESS; @@ -7786,7 +7786,7 @@ int ObRawExprUtils::get_col_ref_expr_recursively(ObRawExpr *expr, return ret; } -int ObRawExprUtils::is_contain_params(ObRawExpr *expr, bool &is_contain) +int ObRawExprUtils::is_contain_params(const ObRawExpr *expr, bool &is_contain) { int ret = OB_SUCCESS; if (OB_ISNULL(expr)) { @@ -7794,7 +7794,6 @@ int ObRawExprUtils::is_contain_params(ObRawExpr *expr, bool &is_contain) LOG_WARN("expr passed in is NULL", K(ret)); } else if (expr->is_param_expr()) { is_contain = true; - LOG_WARN("expr is param expr", KPC(expr)); } else { for (int64_t i = 0; OB_SUCC(ret) && !is_contain && i < expr->get_param_count(); ++i) { if (OB_FAIL(SMART_CALL(is_contain_params(expr->get_param_expr(i), is_contain)))) { diff --git a/src/sql/resolver/expr/ob_raw_expr_util.h b/src/sql/resolver/expr/ob_raw_expr_util.h index 6986faa4ed..b4f6257215 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.h +++ b/src/sql/resolver/expr/ob_raw_expr_util.h @@ -1001,8 +1001,8 @@ public: static int extract_params(common::ObIArray &exprs, common::ObIArray ¶ms); - static int is_contain_params(common::ObIArray &exprs, bool &is_contain); - static int is_contain_params(ObRawExpr *expr, bool &is_contain); + static int is_contain_params(const common::ObIArray &exprs, bool &is_contain); + static int is_contain_params(const ObRawExpr *expr, bool &is_contain); static int add_calc_tablet_id_on_calc_rowid_expr(const ObDMLStmt *dml_stmt, ObRawExprFactory &expr_factory, diff --git a/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result b/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result index fa77a0c655..08aebfc6f9 100644 --- a/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result +++ b/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result @@ -2662,9 +2662,9 @@ Outputs & filters: 2 - output([xy_t1.c2], [xy_t1.c1]), filter(nil), rowset=256, access([xy_t1.c2], [xy_t1.c1]), partitions(p0) 3 - output([1]), filter(nil), rowset=256, - access(nil), partitions(p0) + access([GROUP_ID]), partitions(p0) 4 - output([1]), filter(nil), rowset=256, - access(nil), partitions(p0) + access([GROUP_ID]), partitions(p0) select /*+ no_rewrite*/ sum(c1), sum(c2) from xy_t1 where not exists (select 1 from xy_t3 where c2 = xy_t1.c2) and exists (select 1 from xy_t2 where c2 = xy_t1.c2); +---------+---------+ @@ -4054,7 +4054,7 @@ Outputs & filters: 1 - output([xy_t2.c1], [xy_t2.c3], [xy_t2.c2]), filter(nil), rowset=256, access([xy_t2.c1], [xy_t2.c3], [xy_t2.c2]), partitions(p0) 2 - output([1]), filter([xy_t1.c3 < ?]), rowset=256, - access([xy_t1.c3]), partitions(p0) + access([GROUP_ID], [xy_t1.c3]), partitions(p0) select /*+ no_rewrite*/ c1, c2 from xy_t2 where not exists (select 1 from xy_t1 where xy_t1.c1 = xy_t2.c1 and xy_t1.c3 < xy_t2.c3) order by 1,2; +----+------+ @@ -4188,7 +4188,7 @@ Outputs & filters: 1 - output([xy_t2.c1], [xy_t2.c2]), filter(nil), rowset=256, access([xy_t2.c1], [xy_t2.c2]), partitions(p0) 2 - output([1]), filter([xy_t1.c1 > xy_t1.c2]), rowset=256, - access([xy_t1.c1], [xy_t1.c2]), partitions(p0) + access([GROUP_ID], [xy_t1.c1], [xy_t1.c2]), partitions(p0) select /*+ no_rewrite*/ c1, c2 from xy_t2 where not exists (select 1 from xy_t1 where xy_t1.c1 = xy_t2.c2 and xy_t1.c2 < xy_t2.c2) order by 1,2; +----+------+ diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result index 6dab74573e..3bf7fa84d7 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result @@ -1502,3 +1502,796 @@ drop table t2; drop table t3; drop table t4; set ob_enable_transformation = on; +explain_protocol: 0 +drop table if exists t1; +drop table if exists t2; +create table t1 (c1 int primary key, c2 decimal, c3 int, c4 varchar(20)) partition by hash(c1) partitions 2; +create table t2 (c1 int primary key, c2 decimal, c3 int, c4 varchar(20)); + +insert into t1 (c1, c2, c3, c4) values (1, 1, 1, 'a'); +insert into t1 (c1, c2, c3, c4) values (2, 2, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (12, 9, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (4, 3, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (3, 3, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (9, 10, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (7, 4, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (6, 7, null, 'a'); + +insert into t1 (c1, c2, c3, c4) values (18, 1, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (19, 2, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (20, 3, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (21, 5, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (22, 7, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (23, 7, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (24, 7, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (25, 7, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (26, 9, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (28, 3, null, 'a'); + +insert into t2 (c1, c2, c3, c4) values (1, 1, 1, 'a'); +insert into t2 (c1, c2, c3, c4) values (2, 2, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (3, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (11, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (5, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (15, 13, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (17, 12, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (6, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (7, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (10, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (13, 6, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (14, 17, null, 'a'); + +insert into t2 (c1, c2, c3, c4) values (18, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (19, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (20, 6, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (21, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (22, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (23, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (24, 6, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (26, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (27, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (29, 12, null, 'a'); + +explain_protocol: 3 +EXPLAIN EXTENDED_NOADDR select c1,c1 in (select c1 from t2 where t2.c1 >= t1.c1) as x from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |23 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |22 | +|2 | SUBPLAN FILTER | |1 |22 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [t1.c1 = ANY(subquery(1))]), filter(nil), rowset=256, + exec_params_([t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 >= ?]) + +select c1,c1 in (select c1 from t2 where t2.c1 >= t1.c1) as x from t1; ++----+---+ +| c1 | x | ++----+---+ +| 1 | 1 | +| 2 | 1 | +| 3 | 1 | +| 4 | 0 | +| 6 | 1 | +| 7 | 1 | +| 9 | 0 | +| 12 | 0 | +| 18 | 1 | +| 19 | 1 | +| 20 | 1 | +| 21 | 1 | +| 22 | 1 | +| 23 | 1 | +| 24 | 1 | +| 25 | 0 | +| 26 | 1 | +| 28 | 0 | ++----+---+ +EXPLAIN EXTENDED_NOADDR select c1, (select c1 from t2 where t2.c1 >= t1.c1 and t2.c2 > t1.c3 limit 1) as x from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |23 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |23 | +|2 | SUBPLAN FILTER | |1 |22 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE SCAN|t2 |1 |19 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, subquery(1))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, subquery(1))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [subquery(1)]), filter(nil), rowset=256, + exec_params_([t1.c1], [cast(t1.c3, DECIMAL(11, 0))]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1], [t1.c3]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1], [t1.c3]), filter(nil), rowset=256, + access([t1.c1], [t1.c3]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter([t2.c2 > ?]), rowset=256, + access([GROUP_ID], [t2.c1], [t2.c2]), partitions(p0), + limit(1), offset(nil), + is_index_back=false, filter_before_indexback[false], + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 >= ?]) + +select c1, (select c1 from t2 where t2.c1 >= t1.c1 and t2.c2 > t1.c3 limit 1) as x from t1; ++----+---+ +| c1 | x | ++----+---+ +| 1 | 2 | +| 2 | NULL | +| 3 | NULL | +| 4 | NULL | +| 6 | NULL | +| 7 | NULL | +| 9 | NULL | +| 12 | NULL | +| 18 | NULL | +| 19 | NULL | +| 20 | NULL | +| 21 | NULL | +| 22 | NULL | +| 23 | NULL | +| 24 | NULL | +| 25 | NULL | +| 26 | NULL | +| 28 | NULL | ++----+---+ +EXPLAIN EXTENDED_NOADDR select c1,c1 + (select c1 from t2 where t2.c1 = t1.c1) as x from t1; +Query Plan +===================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +----------------------------------------------------- +|0 |PX COORDINATOR | |1 |22 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |22 | +|2 | SUBPLAN FILTER | |1 |22 | +|3 | PX PARTITION ITERATOR| |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE GET|t2 |1 |18 | +===================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 + subquery(1))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 + subquery(1))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [subquery(1)]), filter(nil), rowset=256, + exec_params_([t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 = ?]) + +select c1,c1 + (select c1 from t2 where t2.c1 = t1.c1) as x from t1; ++----+----+ +| c1 | x | ++----+----+ +| 1 | 2 | +| 2 | 4 | +| 3 | 6 | +| 4 | NULL | +| 6 | 12 | +| 7 | 14 | +| 9 | NULL | +| 12 | NULL | +| 18 | 36 | +| 19 | 38 | +| 20 | 40 | +| 21 | 42 | +| 22 | 44 | +| 23 | 46 | +| 24 | 48 | +| 25 | NULL | +| 26 | 52 | +| 28 | NULL | ++----+----+ +EXPLAIN EXTENDED_NOADDR select c1,c1 in (select c1 from t2 where t2.c1 >= t1.c1) and c1 in (select c2 from t2 where t2.c1 >= t1.c1) as x from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |41 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |41 | +|2 | SUBPLAN FILTER | |1 |40 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|6 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [t1.c1 = ANY(subquery(1))], [cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2))]), filter(nil), rowset=256, + exec_params_([t1.c1], [t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 >= ?]) + 6 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 >= ?]) + +select c1,c1 in (select c1 from t2 where t2.c1 >= t1.c1) and c1 in (select c2 from t2 where t2.c1 >= t1.c1) as x from t1; ++----+------+ +| c1 | x | ++----+------+ +| 1 | 1 | +| 2 | 1 | +| 3 | 1 | +| 4 | 0 | +| 6 | 1 | +| 7 | 0 | +| 9 | 0 | +| 12 | 0 | +| 18 | 0 | +| 19 | 0 | +| 20 | 0 | +| 21 | 0 | +| 22 | 0 | +| 23 | 0 | +| 24 | 0 | +| 25 | 0 | +| 26 | 0 | +| 28 | 0 | ++----+------+ +EXPLAIN EXTENDED_NOADDR select c1,c1 in (select c1 from t2 where t2.c1 >= t1.c1) or c1 in (select c2 from t2 where t2.c1 >= t1.c1) as x from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |41 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |41 | +|2 | SUBPLAN FILTER | |1 |40 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|6 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [t1.c1 = ANY(subquery(1))], [cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2))]), filter(nil), rowset=256, + exec_params_([t1.c1], [t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 >= ?]) + 6 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 >= ?]) + +select c1,c1 in (select c1 from t2 where t2.c1 >= t1.c1) or c1 in (select c2 from t2 where t2.c1 >= t1.c1) as x from t1; ++----+------+ +| c1 | x | ++----+------+ +| 1 | 1 | +| 2 | 1 | +| 3 | 1 | +| 4 | 0 | +| 6 | 1 | +| 7 | 1 | +| 9 | 0 | +| 12 | 1 | +| 18 | 1 | +| 19 | 1 | +| 20 | 1 | +| 21 | 1 | +| 22 | 1 | +| 23 | 1 | +| 24 | 1 | +| 25 | 0 | +| 26 | 1 | +| 28 | 0 | ++----+------+ +EXPLAIN EXTENDED_NOADDR select c1,c1 in (select c1 from t2 where t2.c1 <= t1.c1) and c1 in (select c2 from t2 where t2.c1 <= t1.c1) as x from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |41 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |41 | +|2 | SUBPLAN FILTER | |1 |40 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|6 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [t1.c1 = ANY(subquery(1))], [cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2))]), filter(nil), rowset=256, + exec_params_([t1.c1], [t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 6 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + +select c1,c1 in (select c1 from t2 where t2.c1 <= t1.c1) and c1 in (select c2 from t2 where t2.c1 <= t1.c1) as x from t1; ++----+------+ +| c1 | x | ++----+------+ +| 1 | 1 | +| 2 | 1 | +| 3 | 1 | +| 4 | 0 | +| 6 | 0 | +| 7 | 0 | +| 9 | 0 | +| 12 | 0 | +| 18 | 0 | +| 19 | 0 | +| 20 | 0 | +| 21 | 0 | +| 22 | 0 | +| 23 | 0 | +| 24 | 0 | +| 25 | 0 | +| 26 | 0 | +| 28 | 0 | ++----+------+ +EXPLAIN EXTENDED_NOADDR select c1,c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as x from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |41 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |41 | +|2 | SUBPLAN FILTER | |1 |40 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|6 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [t1.c1 = ANY(subquery(1))], [cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2))]), filter(nil), rowset=256, + exec_params_([t1.c1], [t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 6 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + +select c1,c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as x from t1; ++----+------+ +| c1 | x | ++----+------+ +| 1 | 1 | +| 2 | 1 | +| 3 | 1 | +| 4 | 0 | +| 6 | 1 | +| 7 | 1 | +| 9 | 0 | +| 12 | 0 | +| 18 | 1 | +| 19 | 1 | +| 20 | 1 | +| 21 | 1 | +| 22 | 1 | +| 23 | 1 | +| 24 | 1 | +| 25 | 0 | +| 26 | 1 | +| 28 | 0 | ++----+------+ +EXPLAIN EXTENDED_NOADDR select c1,c1 in (select c1 from t2 where t2.c1 <= t1.c1) and c1 in (select c2 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |59 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |59 | +|2 | SUBPLAN FILTER | |1 |58 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|6 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|7 | DISTRIBUTED TABLE GET |t2 |1 |18 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)) OR t1.c1 + subquery(3))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2)) OR t1.c1 + subquery(3))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [t1.c1 = ANY(subquery(1))], [cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(2))], [subquery(3)]), filter(nil), rowset=256, + exec_params_([t1.c1], [t1.c1], [t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 6 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 7 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 = ?]) + +select c1,c1 in (select c1 from t2 where t2.c1 <= t1.c1) and c1 in (select c2 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x from t1; ++----+------+ +| c1 | x | ++----+------+ +| 1 | 1 | +| 2 | 1 | +| 3 | 1 | +| 4 | NULL | +| 6 | 1 | +| 7 | 1 | +| 9 | NULL | +| 12 | NULL | +| 18 | 1 | +| 19 | 1 | +| 20 | 1 | +| 21 | 1 | +| 22 | 1 | +| 23 | 1 | +| 24 | 1 | +| 25 | NULL | +| 26 | 1 | +| 28 | NULL | ++----+------+ + +EXPLAIN EXTENDED_NOADDR select c1,c1 in (select c1 from t2 where t2.c1=t1.c1) and c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x, c1+(select c1 from t2 where t2.c1 = t1.c1) or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as y,c1+(select c2 from t2 where t1.c1=t2.c1) > 10 and c1 in (select c1 from t2 where t1.c1>= t2.c1) as z from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |131 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |131 | +|2 | SUBPLAN FILTER | |1 |130 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE GET |t2 |1 |18 | +|6 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|7 | DISTRIBUTED TABLE GET |t2 |1 |18 | +|8 | DISTRIBUTED TABLE GET |t2 |1 |18 | +|9 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|10| DISTRIBUTED TABLE GET |t2 |1 |18 | +|11| DISTRIBUTED TABLE SCAN|t2 |8 |19 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND t1.c1 = ANY(subquery(2)) OR t1.c1 + subquery(3), t1.c1 + subquery(4) OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(5)), cast(t1.c1, DECIMAL(11, 0)) + subquery(6) > cast(10, DECIMAL(2, 0)) AND t1.c1 = ANY(subquery(7)))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND t1.c1 = ANY(subquery(2)) OR t1.c1 + subquery(3), t1.c1 + subquery(4) OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(5)), cast(t1.c1, DECIMAL(11, 0)) + subquery(6) > cast(10, DECIMAL(2, 0)) AND t1.c1 = ANY(subquery(7)))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [t1.c1 = ANY(subquery(1))], [t1.c1 = ANY(subquery(2))], [cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(5))], [t1.c1 = ANY(subquery(7))], [subquery(3)], [subquery(4)], [subquery(6)]), filter(nil), rowset=256, + exec_params_([t1.c1], [t1.c1], [t1.c1], [t1.c1], [t1.c1], [t1.c1], [t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 = ?]) + 6 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 7 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 = ?]) + 8 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 = ?]) + 9 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 10 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([? = t2.c1]) + 11 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([? >= t2.c1]) + +select c1,c1 in (select c1 from t2 where t2.c1=t1.c1) and c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x, c1+(select c1 from t2 where t2.c1 = t1.c1) or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as y,c1+(select c2 from t2 where t1.c1=t2.c1) > 10 and c1 in (select c1 from t2 where t1.c1>= t2.c1) as z from t1; ++----+------+------+------+ +| c1 | x | y | z | ++----+------+------+------+ +| 1 | 1 | 1 | 0 | +| 2 | 1 | 1 | 0 | +| 3 | 1 | 1 | 0 | +| 4 | NULL | NULL | 0 | +| 6 | 1 | 1 | 0 | +| 7 | 1 | 1 | 1 | +| 9 | NULL | NULL | 0 | +| 12 | NULL | NULL | 0 | +| 18 | 1 | 1 | 1 | +| 19 | 1 | 1 | 1 | +| 20 | 1 | 1 | 1 | +| 21 | 1 | 1 | 1 | +| 22 | 1 | 1 | 1 | +| 23 | 1 | 1 | 1 | +| 24 | 1 | 1 | 1 | +| 25 | NULL | NULL | 0 | +| 26 | 1 | 1 | 1 | +| 28 | NULL | NULL | 0 | ++----+------+------+------+ + +EXPLAIN EXTENDED_NOADDR select c1,c1 in (select c1 from t2 where t2.c1>=t1.c1) and c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x, c1+(select c1 from t2 where t2.c1 = t1.c1) > 30 or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as y,c1+(select c2 from t2 where t1.c1=t2.c1) > 10 and c1 in (select c1 from t2 where t1.c1>= t2.c1) as z from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |131 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |131 | +|2 | SUBPLAN FILTER | |1 |131 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|6 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|7 | DISTRIBUTED TABLE GET |t2 |1 |18 | +|8 | DISTRIBUTED TABLE GET |t2 |1 |18 | +|9 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|10| DISTRIBUTED TABLE GET |t2 |1 |18 | +|11| DISTRIBUTED TABLE SCAN|t2 |8 |19 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND t1.c1 = ANY(subquery(2)) OR t1.c1 + subquery(3), t1.c1 + subquery(4) > 30 OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(5)), cast(t1.c1, DECIMAL(11, 0)) + subquery(6) > cast(10, DECIMAL(2, 0)) AND t1.c1 = ANY(subquery(7)))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND t1.c1 = ANY(subquery(2)) OR t1.c1 + subquery(3), t1.c1 + subquery(4) > 30 OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(5)), cast(t1.c1, DECIMAL(11, 0)) + subquery(6) > cast(10, DECIMAL(2, 0)) AND t1.c1 = ANY(subquery(7)))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [t1.c1 = ANY(subquery(1))], [t1.c1 = ANY(subquery(2))], [cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(5))], [t1.c1 = ANY(subquery(7))], [subquery(3)], [subquery(4)], [subquery(6)]), filter(nil), rowset=256, + exec_params_([t1.c1], [t1.c1], [t1.c1], [t1.c1], [t1.c1], [t1.c1], [t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 >= ?]) + 6 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 7 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 = ?]) + 8 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 = ?]) + 9 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 10 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([? = t2.c1]) + 11 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([? >= t2.c1]) + +select c1,c1 in (select c1 from t2 where t2.c1>=t1.c1) and c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x, c1+(select c1 from t2 where t2.c1 = t1.c1) > 30 or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as y,c1+(select c2 from t2 where t1.c1=t2.c1) > 10 and c1 in (select c1 from t2 where t1.c1>= t2.c1) as z from t1; ++----+------+------+------+ +| c1 | x | y | z | ++----+------+------+------+ +| 1 | 1 | 1 | 0 | +| 2 | 1 | 1 | 0 | +| 3 | 1 | 1 | 0 | +| 4 | NULL | NULL | 0 | +| 6 | 1 | 0 | 0 | +| 7 | 1 | 0 | 1 | +| 9 | NULL | NULL | 0 | +| 12 | NULL | NULL | 0 | +| 18 | 1 | 1 | 1 | +| 19 | 1 | 1 | 1 | +| 20 | 1 | 1 | 1 | +| 21 | 1 | 1 | 1 | +| 22 | 1 | 1 | 1 | +| 23 | 1 | 1 | 1 | +| 24 | 1 | 1 | 1 | +| 25 | NULL | NULL | 0 | +| 26 | 1 | 1 | 1 | +| 28 | NULL | NULL | 0 | ++----+------+------+------+ + +EXPLAIN EXTENDED_NOADDR select c1,c1 in (select c1 from t2 where t2.c1>=t1.c1) and c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x, c1+(select c1 from t2 where t2.c1 = t1.c1) > 30 or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as y,c1+(select c2 from t2 where t1.c1=t2.c1) > 10 and c1 in (select c1 from t2 where t1.c1>= t2.c1) as z from t1; +Query Plan +====================================================== +|ID|OPERATOR |NAME |EST. ROWS|COST| +------------------------------------------------------ +|0 |PX COORDINATOR | |1 |131 | +|1 | EXCHANGE OUT DISTR |:EX10000|1 |131 | +|2 | SUBPLAN FILTER | |1 |131 | +|3 | PX PARTITION ITERATOR | |1 |4 | +|4 | TABLE SCAN |t1 |1 |4 | +|5 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|6 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|7 | DISTRIBUTED TABLE GET |t2 |1 |18 | +|8 | DISTRIBUTED TABLE GET |t2 |1 |18 | +|9 | DISTRIBUTED TABLE SCAN|t2 |8 |19 | +|10| DISTRIBUTED TABLE GET |t2 |1 |18 | +|11| DISTRIBUTED TABLE SCAN|t2 |8 |19 | +====================================================== + +Outputs & filters: +------------------------------------- + 0 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND t1.c1 = ANY(subquery(2)) OR t1.c1 + subquery(3), t1.c1 + subquery(4) > 30 OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(5)), cast(t1.c1, DECIMAL(11, 0)) + subquery(6) > cast(10, DECIMAL(2, 0)) AND t1.c1 = ANY(subquery(7)))]), filter(nil), rowset=256 + 1 - output([INTERNAL_FUNCTION(t1.c1, t1.c1 = ANY(subquery(1)) AND t1.c1 = ANY(subquery(2)) OR t1.c1 + subquery(3), t1.c1 + subquery(4) > 30 OR cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(5)), cast(t1.c1, DECIMAL(11, 0)) + subquery(6) > cast(10, DECIMAL(2, 0)) AND t1.c1 = ANY(subquery(7)))]), filter(nil), rowset=256, dop=1 + 2 - output([t1.c1], [t1.c1 = ANY(subquery(1))], [t1.c1 = ANY(subquery(2))], [cast(t1.c1, DECIMAL(11, 0)) = ANY(subquery(5))], [t1.c1 = ANY(subquery(7))], [subquery(3)], [subquery(4)], [subquery(6)]), filter(nil), rowset=256, + exec_params_([t1.c1], [t1.c1], [t1.c1], [t1.c1], [t1.c1], [t1.c1], [t1.c1]), onetime_exprs_(nil), init_plan_idxs_(nil), batch_das=true + 3 - output([t1.c1]), filter(nil), rowset=256, + force partition granule. + 4 - output([t1.c1]), filter(nil), rowset=256, + access([t1.c1]), partitions(p[0-1]), + is_index_back=false, + range_key([t1.c1]), range(MIN ; MAX)always true + 5 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 >= ?]) + 6 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 7 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 = ?]) + 8 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 = ?]) + 9 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([t2.c1 <= ?]) + 10 - output([t2.c2]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c2]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([? = t2.c1]) + 11 - output([t2.c1]), filter(nil), rowset=256, + access([GROUP_ID], [t2.c1]), partitions(p0), + is_index_back=false, + range_key([t2.c1]), range(MIN ; MAX)always true, + range_cond([? >= t2.c1]) + +select c1,c1 in (select c1 from t2 where t2.c1>=t1.c1) and c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x, c1+(select c1 from t2 where t2.c1 = t1.c1) > 30 or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as y,c1+(select c2 from t2 where t1.c1=t2.c1) > 10 and c1 in (select c1 from t2 where t1.c1>= t2.c1) as z from t1; ++----+------+------+------+ +| c1 | x | y | z | ++----+------+------+------+ +| 1 | 1 | 1 | 0 | +| 2 | 1 | 1 | 0 | +| 3 | 1 | 1 | 0 | +| 4 | NULL | NULL | 0 | +| 6 | 1 | 0 | 0 | +| 7 | 1 | 0 | 1 | +| 9 | NULL | NULL | 0 | +| 12 | NULL | NULL | 0 | +| 18 | 1 | 1 | 1 | +| 19 | 1 | 1 | 1 | +| 20 | 1 | 1 | 1 | +| 21 | 1 | 1 | 1 | +| 22 | 1 | 1 | 1 | +| 23 | 1 | 1 | 1 | +| 24 | 1 | 1 | 1 | +| 25 | NULL | NULL | 0 | +| 26 | 1 | 1 | 1 | +| 28 | NULL | NULL | 0 | ++----+------+------+------+ diff --git a/tools/deploy/mysql_test/test_suite/static_engine/t/subplan_filter.test b/tools/deploy/mysql_test/test_suite/static_engine/t/subplan_filter.test index 86908824d4..e7aadcbfdf 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/t/subplan_filter.test +++ b/tools/deploy/mysql_test/test_suite/static_engine/t/subplan_filter.test @@ -214,3 +214,79 @@ drop table t4; set ob_enable_transformation = on; connection syscon; --sleep 2 + +--enable_sorted_result +--explain_protocol 0 +--disable_warnings +drop table if exists t1; +drop table if exists t2; +--enable_warnings + +create table t1 (c1 int primary key, c2 decimal, c3 int, c4 varchar(20)) partition by hash(c1) partitions 2; +create table t2 (c1 int primary key, c2 decimal, c3 int, c4 varchar(20)); + +insert into t1 (c1, c2, c3, c4) values (1, 1, 1, 'a'); +insert into t1 (c1, c2, c3, c4) values (2, 2, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (12, 9, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (4, 3, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (3, 3, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (9, 10, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (7, 4, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (6, 7, null, 'a'); + +insert into t1 (c1, c2, c3, c4) values (18, 1, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (19, 2, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (20, 3, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (21, 5, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (22, 7, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (23, 7, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (24, 7, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (25, 7, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (26, 9, null, 'a'); +insert into t1 (c1, c2, c3, c4) values (28, 3, null, 'a'); + + +insert into t2 (c1, c2, c3, c4) values (1, 1, 1, 'a'); +insert into t2 (c1, c2, c3, c4) values (2, 2, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (3, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (11, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (5, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (15, 13, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (17, 12, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (6, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (7, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (10, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (13, 6, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (14, 17, null, 'a'); + + +insert into t2 (c1, c2, c3, c4) values (18, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (19, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (20, 6, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (21, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (22, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (23, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (24, 6, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (26, 3, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (27, 5, null, 'a'); +insert into t2 (c1, c2, c3, c4) values (29, 12, null, 'a'); + +--explain_protocol 3 +select c1,c1 in (select c1 from t2 where t2.c1 >= t1.c1) as x from t1; +#https://work.aone.alibaba-inc.com/issue/47201028 +select c1, (select c1 from t2 where t2.c1 >= t1.c1 and t2.c2 > t1.c3 limit 1) as x from t1; +select c1,c1 + (select c1 from t2 where t2.c1 = t1.c1) as x from t1; +select c1,c1 in (select c1 from t2 where t2.c1 >= t1.c1) and c1 in (select c2 from t2 where t2.c1 >= t1.c1) as x from t1; +select c1,c1 in (select c1 from t2 where t2.c1 >= t1.c1) or c1 in (select c2 from t2 where t2.c1 >= t1.c1) as x from t1; +select c1,c1 in (select c1 from t2 where t2.c1 <= t1.c1) and c1 in (select c2 from t2 where t2.c1 <= t1.c1) as x from t1; +select c1,c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as x from t1; +select c1,c1 in (select c1 from t2 where t2.c1 <= t1.c1) and c1 in (select c2 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x from t1; + + +select c1,c1 in (select c1 from t2 where t2.c1=t1.c1) and c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x, c1+(select c1 from t2 where t2.c1 = t1.c1) or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as y,c1+(select c2 from t2 where t1.c1=t2.c1) > 10 and c1 in (select c1 from t2 where t1.c1>= t2.c1) as z from t1; + + +select c1,c1 in (select c1 from t2 where t2.c1>=t1.c1) and c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x, c1+(select c1 from t2 where t2.c1 = t1.c1) > 30 or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as y,c1+(select c2 from t2 where t1.c1=t2.c1) > 10 and c1 in (select c1 from t2 where t1.c1>= t2.c1) as z from t1; + + +select c1,c1 in (select c1 from t2 where t2.c1>=t1.c1) and c1 in (select c1 from t2 where t2.c1 <= t1.c1) or c1 + (select c1 from t2 where t2.c1 = t1.c1) as x, c1+(select c1 from t2 where t2.c1 = t1.c1) > 30 or c1 in (select c2 from t2 where t2.c1 <= t1.c1) as y,c1+(select c2 from t2 where t1.c1=t2.c1) > 10 and c1 in (select c1 from t2 where t1.c1>= t2.c1) as z from t1;