hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix outline/hint generate plan bug

Browse Source
This commit is contained in:
obdev 2023-03-04 01:41:06 +00:00 committed by ob-robot
parent a038417955
commit 9eba955ac6
20 changed files with 694 additions and 431 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
src/sql/optimizer/ob_log_group_by.cpp
View File

@ -476,75 +476,53 @@ int ObLogGroupBy::inner_replace_op_exprs(
int ObLogGroupBy::print_outline_data(PlanText &plan_text)
{
int ret = OB_SUCCESS;
char *buf = plan_text.buf_;
int64_t &buf_len = plan_text.buf_len_;
int64_t &pos = plan_text.pos_;
const ObDMLStmt *stmt = NULL;
ObString qb_name;
const ObLogicalOperator *child = NULL;
const ObLogicalOperator *op = NULL;
const ObLogGroupBy *group_by_op = NULL;
if (is_push_down()) {
/* print outline in top group by */
} else if (OB_ISNULL(get_plan()) || OB_ISNULL(stmt = get_plan()->get_stmt())
|| OB_ISNULL(child = get_child(ObLogicalOperator::first_child))) {
} else if (OB_ISNULL(get_plan()) || OB_ISNULL(stmt = get_plan()->get_stmt())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected NULL", K(ret), K(get_plan()), K(stmt), K(child));
} else if (OB_FAIL(child->get_pushdown_op(log_op_def::LOG_GROUP_BY, op))) {
LOG_WARN("failed to get push down group by", K(ret));
LOG_WARN("unexpected NULL", K(ret), K(get_plan()), K(stmt));
} else if (OB_FAIL(stmt->get_qb_name(qb_name))) {
LOG_WARN("fail to get qb_name", K(ret), K(stmt->get_stmt_id()));
} else if (NULL != op &&
OB_FAIL(BUF_PRINTF("%s%s(@\"%.*s\")",
ObQueryHint::get_outline_indent(plan_text.is_oneline_),
ObHint::get_hint_name(T_GBY_PUSHDOWN),
qb_name.length(), qb_name.ptr()))) {
LOG_WARN("fail to print buffer", K(ret), K(buf), K(buf_len), K(pos));
} else if (OB_FALSE_IT(group_by_op = static_cast<const ObLogGroupBy*>(NULL == op ? this : op))) {
} else if (HASH_AGGREGATE == group_by_op->get_algo() &&
OB_FAIL(BUF_PRINTF("%s%s(@\"%.*s\")",
ObQueryHint::get_outline_indent(plan_text.is_oneline_),
ObHint::get_hint_name(T_USE_HASH_AGGREGATE),
qb_name.length(), qb_name.ptr()))) {
LOG_WARN("fail to print buffer", K(ret), K(buf), K(buf_len), K(pos));
} else {/*do nothing*/}
} else {
if (OB_SUCC(ret) && has_push_down_) {
ObOptHint hint(T_GBY_PUSHDOWN);
hint.set_qb_name(qb_name);
if (OB_FAIL(hint.print_hint(plan_text))) {
LOG_WARN("failed to print hint", K(ret), K(hint));
}
}
if (OB_SUCC(ret) && (use_hash_aggr_ || use_part_sort_)) {
ObAggHint hint(use_hash_aggr_ ? T_USE_HASH_AGGREGATE : T_NO_USE_HASH_AGGREGATE);
hint.set_qb_name(qb_name);
hint.set_use_partition_sort(use_part_sort_);
if (OB_FAIL(hint.print_hint(plan_text))) {
LOG_WARN("failed to print hint", K(ret), K(hint));
}
}
}
return ret;
}
int ObLogGroupBy::print_used_hint(PlanText &plan_text)
{
int ret = OB_SUCCESS;
const ObLogicalOperator *child = NULL;
const ObLogicalOperator *op = NULL;
const ObHint *hint = NULL;
if (is_push_down()) {
/* print outline in top group by */
} else if (OB_ISNULL(get_plan())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected NULL", K(ret), K(get_plan()));
} else if (OB_ISNULL(child = get_child(ObLogicalOperator::first_child))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected NULL", K(ret), K(child));
} else if (OB_FAIL(child->get_pushdown_op(log_op_def::LOG_GROUP_BY, op))) {
LOG_WARN("failed to get push down group by", K(ret));
} else {
const ObHint *use_hash = get_plan()->get_log_plan_hint().get_normal_hint(T_USE_HASH_AGGREGATE);
const ObHint *pushdown = get_plan()->get_log_plan_hint().get_normal_hint(T_GBY_PUSHDOWN);
if (NULL != use_hash) {
// for pushdown group by, need get pushdown algo
const ObLogGroupBy *group_by_op = static_cast<const ObLogGroupBy*>(NULL == op ? this : op);
bool match_hint = (HASH_AGGREGATE == group_by_op->get_algo() && use_hash->is_enable_hint())
|| (MERGE_AGGREGATE == group_by_op->get_algo() && use_hash->is_disable_hint());
if (match_hint && OB_FAIL(use_hash->print_hint(plan_text))) {
LOG_WARN("failed to print used hint for group by", K(ret), K(*use_hash));
}
}
if (OB_SUCC(ret) && NULL != pushdown) {
bool match_hint = NULL == op ? pushdown->is_disable_hint()
: pushdown->is_enable_hint();
if (match_hint && OB_FAIL(pushdown->print_hint(plan_text))) {
LOG_WARN("failed to print used hint for group by", K(ret), K(*pushdown));
}
}
} else if (NULL != (hint = get_plan()->get_log_plan_hint().get_normal_hint(T_GBY_PUSHDOWN))
&& hint->is_enable_hint() == has_push_down_
&& OB_FAIL(hint->print_hint(plan_text))) {
LOG_WARN("failed to print used hint for group by", K(ret), K(*hint));
} else if (NULL != (hint = get_plan()->get_log_plan_hint().get_normal_hint(T_USE_HASH_AGGREGATE))
&& hint->is_enable_hint() == use_hash_aggr_
&& static_cast<const ObAggHint*>(hint)->force_partition_sort() == use_part_sort_
&& OB_FAIL(hint->print_hint(plan_text))) {
LOG_WARN("failed to print used hint for group by", K(ret), K(*hint));
}
return ret;
}

11
src/sql/optimizer/ob_log_group_by.h
View File

@ -76,7 +76,10 @@ public:
aggr_stage_(ObThreeStageAggrStage::NONE_STAGE),
three_stage_info_(),
rollup_adaptive_info_(),
force_push_down_(false)
force_push_down_(false),
use_hash_aggr_(false),
has_push_down_(false),
use_part_sort_(false)
{}
virtual ~ObLogGroupBy()
{}
@ -197,6 +200,8 @@ public:
{ return ObRollupStatus::ROLLUP_COLLECTOR == rollup_adaptive_info_.rollup_status_; }
inline void set_force_push_down(bool force_push_down)
{ force_push_down_ = force_push_down; }
void set_group_by_outline_info(bool use_hash_aggr, bool has_push_down, bool use_part_sort = false)
{ use_hash_aggr_ = use_hash_aggr; has_push_down_ = has_push_down; use_part_sort_ = use_part_sort; }
virtual int get_plan_item_info(PlanText &plan_text,
ObSqlPlanItem &plan_item) override;
@ -230,6 +235,10 @@ private:
// for rollup distributor and collector
ObRollupAdaptiveInfo rollup_adaptive_info_;
bool force_push_down_; // control by _aggregation_optimization_settings
// use print outline
bool use_hash_aggr_;
bool has_push_down_;
bool use_part_sort_;
};
} // end of namespace sql
} // end of namespace oceanbase

17
src/sql/optimizer/ob_log_plan.cpp
View File

@ -6321,6 +6321,8 @@ int ObLogPlan::create_three_stage_group_plan(const ObIArray<ObRawExpr*> &group_b
} else if (OB_FAIL(third_group_by->set_rollup_info(third_rollup_status,
helper.rollup_id_expr_))) {
LOG_WARN("failed to set rollup parallel info", K(ret));
} else {
third_group_by->set_group_by_outline_info(HASH_AGGREGATE == second_aggr_algo, true);
}
}
return ret;
@ -6527,7 +6529,9 @@ int ObLogPlan::create_scala_group_plan(const ObIArray<ObAggFunRawExpr*> &aggr_it
is_from_povit,
origin_child_card))) {
LOG_WARN("failed to allocate scala group by as top", K(ret));
} else { /*do nothing*/ }
} else {
static_cast<ObLogGroupBy*>(top)->set_group_by_outline_info(false, false);
}
} else if (!groupby_helper.distinct_exprs_.empty() &&
OB_FAIL(top->check_sharding_compatible_with_reduce_expr(groupby_helper.distinct_exprs_,
is_partition_wise))) {
@ -6566,7 +6570,9 @@ int ObLogPlan::create_scala_group_plan(const ObIArray<ObAggFunRawExpr*> &aggr_it
is_from_povit,
origin_child_card))) {
LOG_WARN("failed to allocate scala group by as top", K(ret));
} else { /*do nothing*/ }
} else {
static_cast<ObLogGroupBy*>(top)->set_group_by_outline_info(false, groupby_helper.can_basic_pushdown_ || is_partition_wise); //zzydebug
}
}
return ret;
@ -6645,8 +6651,6 @@ int ObLogPlan::init_groupby_helper(const ObIArray<ObRawExpr*> &group_exprs,
ObSEArray<ObRawExpr*, 4> group_rollup_exprs;
bool push_group = false;
groupby_helper.is_scalar_group_by_ = true;
groupby_helper.force_use_hash_ = get_log_plan_hint().use_hash_aggregate();
groupby_helper.force_use_merge_ = get_log_plan_hint().use_merge_aggregate();
if (OB_FAIL(candidates_.get_best_plan(best_plan))) {
LOG_WARN("failed to get best plan", K(ret));
} else if (OB_ISNULL(best_plan) ||
@ -6659,6 +6663,11 @@ int ObLogPlan::init_groupby_helper(const ObIArray<ObRawExpr*> &group_exprs,
} else if (OB_FAIL(append(group_rollup_exprs, group_exprs)) ||
OB_FAIL(append(group_rollup_exprs, rollup_exprs))) {
LOG_WARN("failed to append group rollup exprs", K(ret));
} else if (OB_FAIL(get_log_plan_hint().get_aggregation_info(groupby_helper.force_use_hash_,
groupby_helper.force_use_merge_,
groupby_helper.force_part_sort_,
groupby_helper.force_normal_sort_))) {
LOG_WARN("failed to get aggregation info from hint", K(ret));
} else if (OB_FAIL(check_scalar_groupby_pushdown(aggr_items,
groupby_helper.can_scalar_pushdown_))) {
LOG_WARN("failed to check scalar group by pushdown", K(ret));

4
src/sql/optimizer/ob_log_plan.h
View File

@ -487,6 +487,8 @@ public:
can_rollup_pushdown_(false),
force_use_hash_(false),
force_use_merge_(false),
force_part_sort_(false),
force_normal_sort_(false),
is_scalar_group_by_(false),
distinct_exprs_(),
aggr_code_expr_(NULL),
@ -506,6 +508,8 @@ public:
bool can_rollup_pushdown_;
bool force_use_hash_; // has use_hash_aggregation/use_hash_distinct hint
bool force_use_merge_; // has no_use_hash_aggregation/no_use_hash_distinct hint
bool force_part_sort_; // force use partition sort for merge group by
bool force_normal_sort_; // disable use partition sort for merge group by
bool is_scalar_group_by_;
ObSEArray<ObRawExpr*, 8> distinct_exprs_;

2
src/sql/optimizer/ob_log_set.cpp
View File

@ -809,7 +809,7 @@ int ObLogSet::print_outline_data(PlanText &plan_text)
LOG_WARN("fail to print buffer", K(ret), K(buf), K(buf_len), K(pos));
} else if (OB_FAIL(construct_pq_set_hint(hint))) {
LOG_WARN("fail to construct pq set hint", K(ret));
} else if (hint.get_dist_methods().empty()) {
} else if (hint.get_dist_methods().empty() && hint.get_left_branch().empty()) {
/*do nothing*/
} else if (OB_FALSE_IT(hint.set_qb_name(qb_name))) {
} else if (hint.print_hint(plan_text)) {

4
src/sql/optimizer/ob_log_table_scan.cpp
View File

@ -1760,6 +1760,10 @@ int ObLogTableScan::print_used_hint(PlanText &plan_text)
} else if (NULL != table_hint->parallel_hint_ && table_hint->parallel_hint_->get_parallel() > 1
&& OB_FAIL(table_hint->parallel_hint_->print_hint(plan_text))) {
LOG_WARN("failed to print table parallel hint", K(ret));
} else if (NULL != table_hint->use_das_hint_
&& use_das() == table_hint->use_das_hint_->is_enable_hint()
&& OB_FAIL(table_hint->use_das_hint_->print_hint(plan_text))) {
LOG_WARN("failed to print use das hint", K(ret));
} else if (table_hint->index_list_.empty()) {
/*do nothing*/
} else if (OB_UNLIKELY(table_hint->index_list_.count() != table_hint->index_hints_.count())) {

223
src/sql/optimizer/ob_select_log_plan.cpp
View File

@ -332,19 +332,25 @@ int ObSelectLogPlan::candi_allocate_three_stage_group_by(const ObIArray<ObRawExp
is_partition_wise))) {
LOG_WARN("failed to check if sharding compatible with distinct expr", K(ret));
} else if (!candidate_plan.plan_tree_->is_distributed() || is_partition_wise) {
bool part_sort_valid = !groupby_helper.force_normal_sort_;
bool normal_sort_valid = !groupby_helper.force_part_sort_;
bool can_ignore_merge_plan = !(groupby_plans.empty() || groupby_helper.force_use_merge_);
if (OB_FAIL(create_merge_group_plan(reduce_exprs,
group_by_exprs,
group_directions,
rollup_exprs,
rollup_directions,
aggr_items,
having_exprs,
is_from_povit,
groupby_helper,
candidate_plan,
groupby_plans,
can_ignore_merge_plan))) {
if (OB_FAIL(update_part_sort_method(part_sort_valid, normal_sort_valid))) {
LOG_WARN("fail to update part sort method", K(ret));
} else if (OB_FAIL(create_merge_group_plan(reduce_exprs,
group_by_exprs,
group_directions,
rollup_exprs,
rollup_directions,
aggr_items,
having_exprs,
is_from_povit,
groupby_helper,
candidate_plan,
groupby_plans,
part_sort_valid,
normal_sort_valid,
can_ignore_merge_plan))) {
LOG_WARN("failed to create merge group by plan", K(ret));
}
} else {
@ -370,16 +376,25 @@ int ObSelectLogPlan::get_valid_aggr_algo(const ObIArray<ObRawExpr*> &group_by_ex
const GroupingOpHelper &groupby_helper,
const bool ignore_hint,
bool &use_hash_valid,
bool &use_merge_valid)
bool &use_merge_valid,
bool &part_sort_valid,
bool &normal_sort_valid)
{
int ret = OB_SUCCESS;
use_hash_valid = ignore_hint ? true : !groupby_helper.force_use_merge_;
use_merge_valid = ignore_hint ? true : !groupby_helper.force_use_hash_;
ObSQLSessionInfo *session = get_optimizer_context().get_session_info();
ObObj val;
if (OB_ISNULL(get_stmt()) || OB_ISNULL(session)) {
if (ignore_hint) {
use_hash_valid = true;
use_merge_valid = true;
part_sort_valid = true;
normal_sort_valid = true;
} else {
use_hash_valid = !groupby_helper.force_use_merge_;
use_merge_valid = !groupby_helper.force_use_hash_;
part_sort_valid = !groupby_helper.force_normal_sort_;
normal_sort_valid = !groupby_helper.force_part_sort_;
}
if (OB_ISNULL(get_stmt()) || OB_ISNULL(optimizer_context_.get_query_ctx())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(get_stmt()), K(session), K(ret));
LOG_WARN("get unexpected null", K(get_stmt()), K(optimizer_context_.get_query_ctx()), K(ret));
} else if (get_stmt()->has_rollup()
|| group_by_exprs.empty()
|| get_stmt()->has_distinct_or_concat_agg()) {
@ -387,6 +402,41 @@ int ObSelectLogPlan::get_valid_aggr_algo(const ObIArray<ObRawExpr*> &group_by_ex
//too much memory consumption for hash aggregate.
use_hash_valid = false;
}
if (OB_FAIL(ret)) {
} else if (!use_merge_valid) {
part_sort_valid = false;
normal_sort_valid = false;
} else if (group_by_exprs.empty()) {
part_sort_valid = false;
/* if normal_sort_valid set as false by hint, will retry by ignore hint */
} else if (OB_FAIL(update_part_sort_method(part_sort_valid, normal_sort_valid))) {
LOG_WARN("fail to update part sort method", K(ret));
}
return ret;
}
// update partition sort method with opt_param hint
int ObSelectLogPlan::update_part_sort_method(bool &part_sort_valid,
bool &normal_sort_valid)
{
int ret = OB_SUCCESS;
if (!part_sort_valid || !normal_sort_valid) {
/* has sort method in no_use_hash_aggregation hint, ignore opt_param hint */
} else {
bool use_part_sort = false;
bool is_exists_opt = false;
if (OB_FAIL(optimizer_context_.get_query_ctx()->get_global_hint()
.opt_params_.get_bool_opt_param(
ObOptParamHint::USE_PART_SORT_MGB, use_part_sort, is_exists_opt))) {
LOG_WARN("fail to check partition sort merge group by enabled", K(ret));
} else if (!is_exists_opt) {
/* has no opt_param hint */
} else {
part_sort_valid = use_part_sort;
normal_sort_valid = !use_part_sort;
}
}
return ret;
}
@ -406,8 +456,11 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray<ObRawExpr*> &
CandidatePlan candidate_plan;
bool use_hash_valid = false;
bool use_merge_valid = false;
bool part_sort_valid = false;
bool normal_sort_valid = false;
if (OB_FAIL(get_valid_aggr_algo(group_by_exprs, groupby_helper, ignore_hint,
use_hash_valid, use_merge_valid))) {
use_hash_valid, use_merge_valid,
part_sort_valid, normal_sort_valid))) {
LOG_WARN("failed to get valid aggr algo", K(ret));
}
// create hash group by plans
@ -436,7 +489,7 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray<ObRawExpr*> &
}
// create merge group by plans
if (OB_SUCC(ret) && use_merge_valid) {
bool can_ignore_merge_plan = !(groupby_plans.empty() || (!ignore_hint && groupby_helper.force_use_merge_));
bool can_ignore_merge_plan = !groupby_plans.empty();
for (int64_t i = 0; OB_SUCC(ret) && i < candidates_.candidate_plans_.count(); i++) {
candidate_plan = candidates_.candidate_plans_.at(i);
bool is_needed = false;
@ -469,6 +522,8 @@ int ObSelectLogPlan::candi_allocate_normal_group_by(const ObIArray<ObRawExpr*> &
groupby_helper,
candidate_plan,
groupby_plans,
part_sort_valid,
normal_sort_valid,
can_ignore_merge_plan))) {
LOG_WARN("failed to create merge group by plan", K(ret));
}
@ -628,6 +683,8 @@ int ObSelectLogPlan::create_rollup_pushdown_plan(const ObIArray<ObRawExpr*> &gro
} else if (rollup_collector->set_rollup_info(ObRollupStatus::ROLLUP_COLLECTOR,
groupby_helper.rollup_id_expr_)) {
LOG_WARN("failed to set rollup id expr", K(ret));
} else {
rollup_collector->set_group_by_outline_info(false, true);
}
return ret;
}
@ -665,6 +722,7 @@ int ObSelectLogPlan::create_hash_group_plan(const ObIArray<ObRawExpr*> &reduce_e
is_partition_wise))) {
LOG_WARN("failed to allocate group by as top", K(ret));
}
static_cast<ObLogGroupBy*>(top)->set_group_by_outline_info(true, false);
} else {
// allocate push down group by
if (groupby_helper.can_basic_pushdown_) {
@ -719,7 +777,9 @@ int ObSelectLogPlan::create_hash_group_plan(const ObIArray<ObRawExpr*> &reduce_e
groupby_helper.group_ndv_,
origin_child_card))) {
LOG_WARN("failed to allocate scala group by as top", K(ret));
} else { /*do nothing*/ }
} else {
static_cast<ObLogGroupBy*>(top)->set_group_by_outline_info(true, groupby_helper.can_basic_pushdown_);
}
}
}
return ret;
@ -865,65 +925,43 @@ int ObSelectLogPlan::create_merge_group_plan(const ObIArray<ObRawExpr*> &reduce_
GroupingOpHelper &groupby_helper,
CandidatePlan &candidate_plan,
ObIArray<CandidatePlan> &candidate_plans,
bool part_sort_valid,
bool normal_sort_valid,
bool can_ignore_merge_plan)
{
int ret = OB_SUCCESS;
bool use_part_sort = true;
bool is_exists_opt = false;
bool is_plan_valid = true;
bool gen_part_sort_mgb_plan = false;
const int64_t group_by_expr_cnt = group_by_exprs.count();
CandidatePlan part_sort_mgb_plan = candidate_plan;
if (OB_ISNULL(optimizer_context_.get_query_ctx())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret));
} else if (OB_FAIL(optimizer_context_.get_query_ctx()->get_global_hint()
.opt_params_.get_bool_opt_param(
ObOptParamHint::USE_PART_SORT_MGB, use_part_sort, is_exists_opt))) {
LOG_WARN("fail to check partition sort merge group by enabled", K(ret));
} else if (group_by_expr_cnt > 0 &&
(!is_exists_opt || use_part_sort) &&
OB_FAIL(inner_create_merge_group_plan(reduce_exprs,
group_by_exprs,
group_directions,
rollup_exprs,
rollup_directions,
aggr_items,
having_exprs,
is_from_povit,
groupby_helper,
part_sort_mgb_plan.plan_tree_,
is_plan_valid,
can_ignore_merge_plan,
use_part_sort))) {
LOG_WARN("failed to create partition sort merge group by plan", K(ret));
} else if (group_by_expr_cnt > 0 &&
NULL != part_sort_mgb_plan.plan_tree_ &&
is_plan_valid &&
(!is_exists_opt || use_part_sort) &&
FALSE_IT(gen_part_sort_mgb_plan = true)) {
} else if (gen_part_sort_mgb_plan &&
OB_FAIL(candidate_plans.push_back(part_sort_mgb_plan))) {
if (part_sort_valid && OB_FAIL(inner_create_merge_group_plan(reduce_exprs,
group_by_exprs,
group_directions,
rollup_exprs,
rollup_directions,
aggr_items,
having_exprs,
is_from_povit,
groupby_helper,
part_sort_mgb_plan.plan_tree_,
true,
can_ignore_merge_plan))) {
LOG_WARN("failed to create partition sort merge group by plan", K(ret));
} else if (part_sort_valid && NULL != part_sort_mgb_plan.plan_tree_
&& OB_FAIL(candidate_plans.push_back(part_sort_mgb_plan))) {
LOG_WARN("failed to push merge group by", K(ret));
} else if (FALSE_IT(can_ignore_merge_plan = can_ignore_merge_plan ? true : gen_part_sort_mgb_plan)) {
} else if (is_plan_valid &&
(!is_exists_opt || !use_part_sort) &&
OB_FAIL(inner_create_merge_group_plan(reduce_exprs,
group_by_exprs,
group_directions,
rollup_exprs,
rollup_directions,
aggr_items,
having_exprs,
is_from_povit,
groupby_helper,
candidate_plan.plan_tree_,
is_plan_valid,
can_ignore_merge_plan))) {
} else if (normal_sort_valid && OB_FAIL(inner_create_merge_group_plan(reduce_exprs,
group_by_exprs,
group_directions,
rollup_exprs,
rollup_directions,
aggr_items,
having_exprs,
is_from_povit,
groupby_helper,
candidate_plan.plan_tree_,
false,
can_ignore_merge_plan))) {
LOG_WARN("failed to create merge group by plan", K(ret));
} else if (is_plan_valid &&
(!is_exists_opt || !use_part_sort) &&
OB_FAIL(candidate_plans.push_back(candidate_plan))) {
} else if (normal_sort_valid && NULL != candidate_plan.plan_tree_ &&
OB_FAIL(candidate_plans.push_back(candidate_plan))) {
LOG_WARN("failed to push merge group by", K(ret));
}
return ret;
@ -939,9 +977,8 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray<ObRawExpr*> &r
const bool is_from_povit,
GroupingOpHelper &groupby_helper,
ObLogicalOperator *&top,
bool &is_plan_valid,
bool can_ignore_merge_plan,
bool use_part_sort)
bool use_part_sort,
bool can_ignore_merge_plan)
{
int ret = OB_SUCCESS;
int64_t prefix_pos = 0;
@ -950,8 +987,7 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray<ObRawExpr*> &r
bool push_need_sort = false;
bool is_partition_wise = false;
bool is_fetch_with_ties = false;
is_plan_valid = true;
int64_t part_cnt = 0;
int64_t part_cnt = use_part_sort ? group_by_exprs.count() : 0;
OrderItem hash_sortkey;
ObSEArray<ObRawExpr*, 1> dummy_exprs;
ObSEArray<ObAggFunRawExpr*, 1> dummy_aggrs;
@ -963,9 +999,9 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray<ObRawExpr*> &r
ObSEArray<ObOrderDirection, 4> adjusted_group_directions;
int64_t interesting_order_info = OrderingFlag::NOT_MATCH;
double origin_child_card = 0.0;
if (OB_ISNULL(top)) {
if (OB_ISNULL(top) || OB_UNLIKELY(0 == part_cnt && use_part_sort)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret));
LOG_WARN("get unexpected params", K(top), K(part_cnt), K(use_part_sort), K(ret));
} else if (OB_FALSE_IT(origin_child_card = top->get_card())) {
} else if (OB_FAIL(adjusted_group_by_exprs.assign(group_by_exprs)) ||
OB_FAIL(adjusted_group_directions.assign(group_directions))) {
@ -996,7 +1032,6 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray<ObRawExpr*> &r
OrderingCheckScope::CHECK_ALL & (~OrderingCheckScope::CHECK_GROUP),
interesting_order_info))) {
LOG_WARN("failed to compute stmt interesting order", K(ret));
} else if (FALSE_IT(part_cnt = use_part_sort ? group_by_exprs.count() : 0)) {
} else if (OB_FAIL(ObOptimizerUtil::check_need_sort(sort_exprs,
&sort_directions,
top->get_op_ordering(),
@ -1009,9 +1044,12 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray<ObRawExpr*> &r
prefix_pos,
part_cnt))) {
LOG_WARN("failed to check if need sort", K(ret));
} else if (need_sort && can_ignore_merge_plan && OrderingFlag::NOT_MATCH == interesting_order_info) {
// if need sort and no further op needs the output order, not generate merge groupby
is_plan_valid = false;
} else if ((!need_sort && use_part_sort) ||
(need_sort && can_ignore_merge_plan &&
OrderingFlag::NOT_MATCH == interesting_order_info)) {
// 1. need generate partition sort plan, but need not sort
// 2. if need sort and no further op needs the output order, not generate merge groupby
top = NULL;
} else if (top->is_distributed() &&
OB_FAIL(top->check_sharding_compatible_with_reduce_expr(reduce_exprs,
is_partition_wise))) {
@ -1030,8 +1068,10 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray<ObRawExpr*> &r
origin_child_card,
is_partition_wise))) {
LOG_WARN("failed to allocate group by as top", K(ret));
} else {
static_cast<ObLogGroupBy*>(top)->set_group_by_outline_info(false, false, use_part_sort);
}
} else if (need_sort && part_cnt > 0 &&
} else if (use_part_sort &&
OB_FAIL(create_hash_sortkey(part_cnt, sort_keys, hash_sortkey))) {
LOG_WARN("failed to create hash sort key", K(ret), K(part_cnt), K(sort_keys));
} else {
@ -1063,7 +1103,7 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray<ObRawExpr*> &r
!need_sort && top_is_local_order,
nullptr,
is_fetch_with_ties,
(need_sort && part_cnt > 0) ? &hash_sortkey : NULL))) {
use_part_sort ? &hash_sortkey : NULL))) {
LOG_WARN("failed to allcoate sort as top", K(ret));
} else if (OB_FAIL(allocate_group_by_as_top(top,
MERGE_AGGREGATE,
@ -1101,7 +1141,7 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray<ObRawExpr*> &r
top->get_is_local_order(),
nullptr,
is_fetch_with_ties,
(need_sort && part_cnt > 0) ? &hash_sortkey : NULL))) {
use_part_sort ? &hash_sortkey : NULL))) {
LOG_WARN("failed to allocate sort as top", K(ret));
} else if (OB_FAIL(allocate_group_by_as_top(top,
MERGE_AGGREGATE,
@ -1113,7 +1153,10 @@ int ObSelectLogPlan::inner_create_merge_group_plan(const ObIArray<ObRawExpr*> &r
groupby_helper.group_ndv_,
origin_child_card))) {
LOG_WARN("failed to allocate group by as top", K(ret));
} else { /*do nothing*/ }
} else {
static_cast<ObLogGroupBy*>(top)->set_group_by_outline_info(false,
groupby_helper.can_basic_pushdown_, use_part_sort);
}
}
}
return ret;

12
src/sql/optimizer/ob_select_log_plan.h
View File

@ -85,7 +85,10 @@ private:
const GroupingOpHelper &groupby_helper,
const bool ignore_hint,
bool &use_hash_valid,
bool &use_merge_valid);
bool &use_merge_valid,
bool &part_sort_valid,
bool &normal_sort_valid);
int update_part_sort_method(bool &part_sort_valid, bool &normal_sort_valid);
int candi_allocate_normal_group_by(const ObIArray<ObRawExpr*> &reduce_exprs,
const ObIArray<ObRawExpr*> &group_by_exprs,
const ObIArray<ObOrderDirection> &group_directions,
@ -160,6 +163,8 @@ private:
GroupingOpHelper &groupby_helper,
CandidatePlan &candidate_plan,
ObIArray<CandidatePlan> &candidate_plans,
bool part_sort_valid,
bool normal_sort_valid,
bool can_ignore_merge = false);
int generate_merge_group_sort_keys(ObLogicalOperator *top,
@ -671,9 +676,8 @@ private:
const bool is_from_povit,
GroupingOpHelper &groupby_helper,
ObLogicalOperator *&top,
bool &is_plan_valid,
bool can_ignore_merge = false,
bool use_part_sort = false);
bool use_part_sort,
bool can_ignore_merge = false);
DISALLOW_COPY_AND_ASSIGN(ObSelectLogPlan);
};

2
src/sql/parser/sql_parser_mysql_mode.l
View File

@ -950,6 +950,8 @@ Timestamp{whitespace}?\"[^\"]*\" {
<hint>USE_PLAN_CACHE { return USE_PLAN_CACHE; }
<hint>USE_HASH_AGGREGATION { return USE_HASH_AGGREGATION; }
<hint>NO_USE_HASH_AGGREGATION { return NO_USE_HASH_AGGREGATION; }
<hint>PARTITION_SORT { return PARTITION_SORT; }
<hint>NO_PARTITION_SORT { return NO_PARTITION_SORT; }
<hint>USE_LATE_MATERIALIZATION { return USE_LATE_MATERIALIZATION; }
<hint>NO_USE_LATE_MATERIALIZATION { return NO_USE_LATE_MATERIALIZATION; }
<hint>TRACE_LOG { return TRACE_LOG; }

14
src/sql/parser/sql_parser_mysql_mode.y
View File

@ -171,7 +171,8 @@ INDEX_SS_HINT INDEX_SS_ASC_HINT INDEX_SS_DESC_HINT
LEADING_HINT ORDERED
USE_NL USE_MERGE USE_HASH NO_USE_HASH NO_USE_MERGE NO_USE_NL
USE_NL_MATERIALIZATION NO_USE_NL_MATERIALIZATION
USE_HASH_AGGREGATION NO_USE_HASH_AGGREGATION
USE_HASH_AGGREGATION NO_USE_HASH_AGGREGATION
PARTITION_SORT NO_PARTITION_SORT
USE_LATE_MATERIALIZATION NO_USE_LATE_MATERIALIZATION
PX_JOIN_FILTER NO_PX_JOIN_FILTER PX_PART_JOIN_FILTER NO_PX_PART_JOIN_FILTER
PQ_MAP PQ_DISTRIBUTE PQ_DISTRIBUTE_WINDOW PQ_SET RANDOM_LOCAL BROADCAST BC2HOST LIST
@ -8850,6 +8851,17 @@ INDEX_HINT '(' qb_name_option relation_factor_in_hint NAME_OB ')'
| NO_USE_HASH_AGGREGATION opt_qb_name
{
malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_HASH_AGGREGATE, 1, $2);
$$->value_ = -1;
}
| NO_USE_HASH_AGGREGATION '(' qb_name_option NO_PARTITION_SORT ')'
{
malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_HASH_AGGREGATE, 1, $3);
$$->value_ = 0;
}
| NO_USE_HASH_AGGREGATION '(' qb_name_option PARTITION_SORT ')'
{
malloc_non_terminal_node($$, result->malloc_pool_, T_NO_USE_HASH_AGGREGATE, 1, $3);
$$->value_ = 1;
}
| USE_LATE_MATERIALIZATION opt_qb_name
{

37
src/sql/resolver/dml/ob_dml_resolver.cpp
View File

@ -11982,8 +11982,6 @@ int ObDMLResolver::resolve_optimize_hint(const ParseNode &hint_node,
}
case T_USE_LATE_MATERIALIZATION:
case T_NO_USE_LATE_MATERIALIZATION:
case T_USE_HASH_AGGREGATE:
case T_NO_USE_HASH_AGGREGATE:
case T_GBY_PUSHDOWN:
case T_NO_GBY_PUSHDOWN:
case T_USE_HASH_DISTINCT:
@ -11999,6 +11997,13 @@ int ObDMLResolver::resolve_optimize_hint(const ParseNode &hint_node,
}
break;
}
case T_USE_HASH_AGGREGATE:
case T_NO_USE_HASH_AGGREGATE: {
if (OB_FAIL(resolve_aggregation_hint(hint_node, opt_hint))) {
LOG_WARN("failed to resolve aggregation hint.", K(ret));
}
break;
}
case T_TABLE_PARALLEL: { // PARALLEL(qb_name tablespec 4)
if (OB_FAIL(resolve_table_parallel_hint(hint_node, opt_hint))) {
LOG_WARN("fail to resolve parallel in hint", K(ret));
@ -12751,6 +12756,34 @@ int ObDMLResolver::resolve_normal_optimize_hint(const ParseNode &hint_node,
return ret;
}
int ObDMLResolver::resolve_aggregation_hint(const ParseNode &hint_node,
ObOptHint *&hint)
{
int ret = OB_SUCCESS;
hint = NULL;
ObAggHint *agg_hint = NULL;
ObString qb_name;
if (OB_UNLIKELY(1 != hint_node.num_child_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("hint with qb name param has no one children.", K(ret));
} else if (OB_FAIL(ObQueryHint::create_hint(allocator_, hint_node.type_, agg_hint))) {
LOG_WARN("failed to create hint", K(ret));
} else if (OB_FAIL(resolve_qb_name_node(hint_node.children_[0], qb_name))) {
LOG_WARN("failed to resolve qb name node.", K(ret));
} else {
hint = agg_hint;
agg_hint->set_qb_name(qb_name);
if (T_NO_USE_HASH_AGGREGATE == hint_node.type_) {
if (1 == hint_node.value_) {
agg_hint->set_use_partition_sort(true);
} else if (0 == hint_node.value_) {
agg_hint->set_use_partition_sort(false);
}
}
}
return ret;
}
int ObDMLResolver::resolve_monitor_ids(const ParseNode &tracing_node,
ObIArray<ObMonitorHint> &monitoring_ids)
{

1
src/sql/resolver/dml/ob_dml_resolver.h
View File

@ -840,6 +840,7 @@ private:
int resolve_pq_distribute_hint(const ParseNode &hint_node, ObOptHint *&opt_hint);
int resolve_pq_set_hint(const ParseNode &hint_node, ObOptHint *&opt_hint);
int resolve_join_filter_hint(const ParseNode &join_node, ObOptHint *&opt_hint);
int resolve_aggregation_hint(const ParseNode &hint_node, ObOptHint *&hint);
int resolve_normal_transform_hint(const ParseNode &hint_node, ObTransHint *&hint);
int resolve_normal_optimize_hint(const ParseNode &hint_node, ObOptHint *&hint);
int resolve_view_merge_hint(const ParseNode &hint_node, ObTransHint *&hint);

24
src/sql/resolver/dml/ob_hint.cpp
View File

@ -2511,5 +2511,29 @@ int ObWindowDistHint::print_hint_desc(PlanText &plan_text) const
return ret;
}
int ObAggHint::assign(const ObAggHint &other)
{
int ret = OB_SUCCESS;
sort_method_valid_ = other.sort_method_valid_;
use_partition_sort_ = other.use_partition_sort_;
return ret;
}
int ObAggHint::print_hint_desc(PlanText &plan_text) const
{
int ret = OB_SUCCESS;
if (sort_method_valid_) {
char *buf = plan_text.buf_;
int64_t &buf_len = plan_text.buf_len_;
int64_t &pos = plan_text.pos_;
if (use_partition_sort_ && OB_FAIL(BUF_PRINTF("PARTITION_SORT"))) {
LOG_WARN("print failed", K(ret));
} else if (!use_partition_sort_ && OB_FAIL(BUF_PRINTF("NO_PARTITION_SORT"))) {
LOG_WARN("print failed", K(ret));
}
}
return ret;
}
}//end of namespace sql
}//end of namespace oceanbase

25
src/sql/resolver/dml/ob_hint.h
View File

@ -931,6 +931,31 @@ private:
Algos algos_;
};
class ObAggHint : public ObOptHint
{
public:
ObAggHint(ObItemType hint_type)
: ObOptHint(hint_type),
sort_method_valid_(false),
use_partition_sort_(false)
{
}
int assign(const ObAggHint &other);
virtual ~ObAggHint() {}
virtual int print_hint_desc(PlanText &plan_text) const override;
void set_use_partition_sort(bool use_part_sort) { sort_method_valid_ = is_disable_hint(); use_partition_sort_ = sort_method_valid_ && use_part_sort; }
void reset_use_partition_sort() { sort_method_valid_ = false; use_partition_sort_ = false; }
bool force_partition_sort() const { return is_disable_hint() && sort_method_valid_ && use_partition_sort_; }
bool force_normal_sort() const { return is_disable_hint() && sort_method_valid_ && !use_partition_sort_; }
INHERIT_TO_STRING_KV("ObHint", ObHint, K_(sort_method_valid), K_(use_partition_sort));
private:
bool sort_method_valid_;
bool use_partition_sort_;
};
struct ObDDLSchemaVersionHint
{
ObDDLSchemaVersionHint() : schema_version_(0) {}

26
src/sql/resolver/dml/ob_sql_hint.cpp
View File

@ -1756,6 +1756,32 @@ bool ObLogPlanHint::has_disable_hint(ObItemType hint_type) const
return NULL != cur_hint ? cur_hint->is_disable_hint() : is_outline_data_;
}
int ObLogPlanHint::get_aggregation_info(bool &force_use_hash,
bool &force_use_merge,
bool &force_part_sort,
bool &force_normal_sort) const
{
int ret = OB_SUCCESS;
force_use_hash = false;
force_use_merge = false;
force_part_sort = false;
force_normal_sort = false;
const ObAggHint *agg_hint = static_cast<const ObAggHint*>(get_normal_hint(T_USE_HASH_AGGREGATE));
if (NULL != agg_hint) {
force_use_hash = agg_hint->is_enable_hint();
force_use_merge = agg_hint->is_disable_hint();
force_part_sort = agg_hint->force_partition_sort();
force_normal_sort = agg_hint->force_normal_sort();
if (force_use_merge && !force_part_sort && !force_normal_sort && is_outline_data_) {
force_normal_sort = true;
}
} else if (is_outline_data_) {
force_use_merge = true;
force_normal_sort = true;
}
return ret;
}
const ObWindowDistHint *ObLogPlanHint::get_window_dist() const
{
return static_cast<const ObWindowDistHint*>(

4
src/sql/resolver/dml/ob_sql_hint.h
View File

@ -435,6 +435,10 @@ struct ObLogPlanHint
bool has_disable_hint(ObItemType hint_type) const;
bool use_join_filter(const ObRelIds &table_set) const;
bool no_use_join_filter(const ObRelIds &table_set) const;
int get_aggregation_info(bool &force_use_hash,
bool &force_use_merge,
bool &force_part_sort,
bool &force_normal_sort) const;
bool use_late_material() const { return has_enable_hint(T_USE_LATE_MATERIALIZATION); }
bool no_use_late_material() const { return has_disable_hint(T_USE_LATE_MATERIALIZATION); }

8
src/sql/rewrite/ob_transform_groupby_pullup.cpp
View File

@ -305,7 +305,7 @@ int ObTransformGroupByPullup::check_groupby_pullup_validity(ObDMLStmt *stmt,
} else if (OB_FAIL(check_hint_valid(*stmt, *table->ref_query_, hint_valid))) {
LOG_WARN("check hint failed", K(ret));
} else if (!hint_valid) {
is_valid = false;
// can not set is_valid as false, may pullup other table
OPT_TRACE("hint reject transform");
} else if (OB_FALSE_IT(myhint = static_cast<const ObViewMergeHint*>(sub_stmt->get_stmt_hint().get_normal_hint(T_MERGE_HINT)))) {
} else if (ignore_tables.has_member(stmt->get_table_bit_index(table->table_id_))) {
@ -1071,12 +1071,12 @@ int ObTransformGroupByPullup::need_transform(const common::ObIArray<ObParentDMLS
} else {
need_trans = query_hint->is_valid_outline_transform(ctx_->trans_list_loc_,
get_hint(table->ref_query_->get_stmt_hint()));
if (!need_trans) {
OPT_TRACE("outline reject transform");
}
LOG_DEBUG("need trans pullup0", K(need_trans));
}
}
if (OB_SUCC(ret) && !need_trans) {
OPT_TRACE("outline reject transform");
}
}
LOG_DEBUG("need trans pullup", K(need_trans));
return ret;

590
src/sql/rewrite/ob_transform_predicate_move_around.cpp
View File

@ -47,44 +47,52 @@ int ObTransformPredicateMoveAround::transform_one_stmt(
ObArray<ObRawExpr *> dummy_pullup;
ObArray<ObRawExpr *> dummy_pushdown;
ObArray<int64_t> dummy_list;
bool is_happened = false;
transed_stmts_.reset();
real_happened_ = false;
UNUSED(parent_stmts);
if (OB_ISNULL(stmt)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("stmt is null", K(ret), K(stmt));
} else if (!stmt_map_.created() && OB_FAIL(stmt_map_.create(20, ObModIds::OB_SQL_COMPILE))) {
LOG_WARN("failed to create stmt map", K(ret));
} else if (OB_FAIL(pullup_predicates(stmt, dummy_list, dummy_pullup))) {
LOG_WARN("failed to pull up predicates", K(ret));
} else if ((stmt->is_insert_stmt() || stmt->is_merge_stmt())
&& (OB_FAIL(create_equal_exprs_for_insert(static_cast<ObDelUpdStmt*>(stmt))))) {
LOG_WARN("failed to create equal exprs for insert", K(ret));
} else if (OB_FAIL(pushdown_predicates(stmt, dummy_pushdown))) {
LOG_WARN("failed to push down predicates", K(ret));
} else if (transed_stmts_.empty() || !real_happened_) {
// transform not happened actually
transed_stmts_.reuse();
} else if (OB_FAIL(sort_transed_stmts())) {
LOG_WARN("sort sort transed stmts failed", K(ret));
} else if (OB_FAIL(add_transform_hint(*stmt, &transed_stmts_))) {
LOG_WARN("add transform hint failed", K(ret));
} else {
trans_happened = true;
}
if (OB_SUCC(ret)) {
if (OB_FAIL(pullup_predicates(stmt, dummy_list, dummy_pullup))) {
LOG_WARN("failed to pull up predicates", K(ret));
} else if (!stmt->is_insert_stmt() && !stmt->is_merge_stmt()) {
// do nothing
} else if (OB_FAIL(create_equal_exprs_for_insert(static_cast<ObDelUpdStmt*>(stmt),
is_happened))) {
LOG_WARN("failed to create equal exprs for insert", K(ret));
} else if (is_happened) {
trans_happened = true;
}
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(pushdown_predicates(stmt, dummy_pushdown))) {
LOG_WARN("failed to push down predicates", K(ret));
} else if (!trans_happened && !trans_happened_) {
// do nothing
} else if (OB_FAIL(add_transform_hint(*stmt, &transed_stmts_))) {
LOG_WARN("add transform hint failed", K(ret));
return ret;
}
int ObTransformPredicateMoveAround::sort_transed_stmts()
{
int ret = OB_SUCCESS;
auto cmp_func = [](ObDMLStmt* l_stmt, ObDMLStmt* r_stmt) {
if (OB_ISNULL(l_stmt) || OB_ISNULL(r_stmt)) {
return false;
} else {
trans_happened = true;
return l_stmt->get_stmt_id() < r_stmt->get_stmt_id();
}
}
};
std::sort(transed_stmts_.begin(), transed_stmts_.end(), cmp_func);
return ret;
}
ObTransformPredicateMoveAround::ObTransformPredicateMoveAround(ObTransformerCtx *ctx)
: ObTransformRule(ctx, TransMethod::ROOT_ONLY, T_PRED_DEDUCE),
allocator_("PredDeduce")
allocator_("PredDeduce"),
real_happened_(false)
{}
ObTransformPredicateMoveAround::~ObTransformPredicateMoveAround()
@ -213,6 +221,17 @@ int ObTransformPredicateMoveAround::get_stmt_to_trans(ObDMLStmt *stmt, ObIArray<
} else if (OB_FAIL(stmt->get_from_subquery_stmts(child_stmts))) {
LOG_WARN("failed to get from subquery stmts", K(ret));
} else {
ObIArray<ObQueryRefRawExpr *> &subquery_exprs = stmt->get_subquery_exprs();
for (int64_t i = 0; OB_SUCC(ret) && i < subquery_exprs.count(); i++) {
if (OB_ISNULL(subquery_exprs.at(i))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (OB_FAIL(SMART_CALL(get_stmt_to_trans(subquery_exprs.at(i)->get_ref_stmt(),
stmt_to_trans)))) {
LOG_WARN("get stmt to trans from subquery failed", K(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < child_stmts.count(); i++) {
if (OB_FAIL(SMART_CALL(get_stmt_to_trans(child_stmts.at(i), stmt_to_trans)))) {
LOG_WARN("get stmt to trans failed", K(ret));
@ -252,6 +271,7 @@ int ObTransformPredicateMoveAround::pullup_predicates_from_set_stmt(ObDMLStmt *s
return ret;
}
//pullup predicates do not change stmt, need not hint check and allowed pullup pred from temp table
int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt,
ObIArray<int64_t> &sel_ids,
ObIArray<ObRawExpr *> &output_pullup_preds)
@ -259,7 +279,6 @@ int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt,
int ret = OB_SUCCESS;
bool is_overflow = false;
ObIArray<ObRawExpr *> *input_pullup_preds = NULL;
bool enable_no_pred_deduce = false;
OPT_TRACE_BEGIN_SECTION;
OPT_TRACE("try to pullup predicates");
if (OB_ISNULL(stmt)) {
@ -270,8 +289,6 @@ int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt,
} else if (is_overflow) {
ret = OB_SIZE_OVERFLOW;
LOG_WARN("too deep recursive", K(ret), K(is_overflow));
} else if (OB_FAIL(check_enable_no_pred_deduce(*stmt, enable_no_pred_deduce))) {
LOG_WARN("check_enable_no_pred_deduce failed", K(ret));
} else if (OB_FAIL(acquire_transform_params(stmt, input_pullup_preds))) {
LOG_WARN("failed to acquire pullup preds", K(ret));
} else if (OB_ISNULL(input_pullup_preds)) {
@ -281,9 +298,6 @@ int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt,
if (OB_FAIL(pullup_predicates_from_set_stmt(stmt, sel_ids, output_pullup_preds))) {
LOG_WARN("process set stmt failed", K(ret));
}
} else if (!enable_no_pred_deduce &&
OB_FAIL(preprocess(*stmt))) {
LOG_WARN("failed to preprocess stmt", K(ret));
} else if (OB_FAIL(generate_basic_table_pullup_preds(stmt, *input_pullup_preds))) {
LOG_WARN("add stmt check constraints", K(ret));
} else if (OB_FAIL(pullup_predicates_from_view(*stmt, sel_ids, *input_pullup_preds))) {
@ -303,107 +317,6 @@ int ObTransformPredicateMoveAround::pullup_predicates(ObDMLStmt *stmt,
return ret;
}
int ObTransformPredicateMoveAround::preprocess(ObDMLStmt &stmt)
{
int ret = OB_SUCCESS;
OPT_TRACE("try pullup semi condition");
for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_semi_infos().count(); ++i) {
if (OB_FAIL(preprocess_semi_info(stmt, stmt.get_semi_infos().at(i),
stmt.get_condition_exprs()))) {
LOG_WARN("failed to preprocess joined table", K(ret));
}
}
OPT_TRACE("try pullup on condition");
for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_joined_tables().count(); ++i) {
if (OB_FAIL(preprocess_joined_table(stmt, stmt.get_joined_tables().at(i),
stmt.get_condition_exprs()))) {
LOG_WARN("failed to preprocess joined table", K(ret));
}
}
return ret;
}
int ObTransformPredicateMoveAround::preprocess_joined_table(ObDMLStmt &stmt,
JoinedTable *join_table,
ObIArray<ObRawExpr*> &upper_conds)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(join_table) ||
OB_ISNULL(join_table->left_table_) ||
OB_ISNULL(join_table->right_table_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("joined table is null", K(ret));
} else if (join_table->is_inner_join() && !join_table->join_conditions_.empty()) {
if (OB_FAIL(append(upper_conds, join_table->join_conditions_))) {
LOG_WARN("failed to append conditions", K(ret));
} else {
join_table->join_conditions_.reset();
trans_happened_ = true;
if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, &stmt))) {
LOG_WARN("append failed", K(ret));
}
}
}
if (OB_SUCC(ret) && join_table->left_table_->is_joined_table() &&
(join_table->is_inner_join() ||join_table->is_left_join())) {
if (OB_FAIL(SMART_CALL(preprocess_joined_table(stmt,
static_cast<JoinedTable*>(join_table->left_table_), upper_conds)))) {
LOG_WARN("failed to process joined table", K(ret));
}
}
if (OB_SUCC(ret) && join_table->right_table_->is_joined_table() &&
(join_table->is_inner_join() || join_table->is_right_join())) {
if (OB_FAIL(SMART_CALL(preprocess_joined_table(stmt,
static_cast<JoinedTable*>(join_table->right_table_), upper_conds)))) {
LOG_WARN("failed to process joined table", K(ret));
}
}
return ret;
}
int ObTransformPredicateMoveAround::preprocess_semi_info(ObDMLStmt &stmt,
SemiInfo *semi_info,
ObIArray<ObRawExpr*> &upper_conds)
{
int ret = OB_SUCCESS;
ObSqlBitSet<> left_rel_ids;
if (OB_ISNULL(semi_info)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("semi info is null", K(ret));
} else if (semi_info->is_anti_join()) {
// do not pull up anti conditions to upper conds
} else if (OB_FAIL(stmt.get_table_rel_ids(semi_info->left_table_ids_, left_rel_ids))) {
LOG_WARN("failed to get table rel ids", K(ret));
} else {
ObRawExpr *expr = NULL;
ObIArray<ObRawExpr*> &semi_conds = semi_info->semi_conditions_;
ObSEArray<ObRawExpr*, 4> left_filters;
for (int64_t i = 0; OB_SUCC(ret) && i < semi_conds.count(); ++i) {
if (OB_ISNULL(expr = semi_conds.at(i))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (!expr->has_flag(CNT_COLUMN) ||
!expr->get_relation_ids().is_subset(left_rel_ids)) {
/*do nothing*/
} else if (OB_FAIL(left_filters.push_back(expr))) {
LOG_WARN("failed to push back expr", K(ret));
}
}
if (OB_FAIL(ret) || left_filters.empty()) {
/*do nothing*/
} else if (OB_FAIL(append(upper_conds, left_filters))) {
LOG_WARN("failed to append conditions", K(ret));
} else if (OB_FAIL(ObOptimizerUtil::remove_item(semi_conds, left_filters))) {
LOG_WARN("failed to remove item", K(ret));
} else {
trans_happened_ = true;
if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, &stmt))) {
LOG_WARN("append failed", K(ret));
}
}
}
return ret;
}
int ObTransformPredicateMoveAround::pullup_predicates_from_view(
ObDMLStmt &stmt, ObIArray<int64_t> &sel_ids, ObIArray<ObRawExpr *> &input_pullup_preds)
@ -907,7 +820,9 @@ int ObTransformPredicateMoveAround::generate_pullup_predicates(
int ret = OB_SUCCESS;
ObSEArray<ObRawExpr *, 4> local_preds;
ObSEArray<ObRawExpr *, 4> filter_preds;
if (OB_FAIL(append(filter_preds, select_stmt.get_condition_exprs()))) {
if (OB_FAIL(gather_pullup_preds_from_semi_outer_join(select_stmt, filter_preds))) {
LOG_WARN("failed to pullup preds from semi outer join", K(ret));
} else if (OB_FAIL(append(filter_preds, select_stmt.get_condition_exprs()))) {
LOG_WARN("failed to append conditions", K(ret));
} else if (OB_FAIL(append(filter_preds, select_stmt.get_having_exprs()))) {
LOG_WARN("failed to append having conditions", K(ret));
@ -924,6 +839,77 @@ int ObTransformPredicateMoveAround::generate_pullup_predicates(
return ret;
}
int ObTransformPredicateMoveAround::gather_pullup_preds_from_semi_outer_join(ObDMLStmt &stmt,
ObIArray<ObRawExpr*> &preds,
bool remove_preds /* default false*/)
{
int ret = OB_SUCCESS;
ObSqlBitSet<> left_rel_ids;
SemiInfo *semi_info = NULL;
ObRawExpr *expr = NULL;
ObSEArray<ObRawExpr*, 4> new_semi_conditions;
for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_semi_infos().count(); ++i) {
left_rel_ids.reuse();
new_semi_conditions.reuse();
if (OB_ISNULL(semi_info = stmt.get_semi_infos().at(i))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("semi info is null", K(ret));
} else if (semi_info->is_anti_join()) {
// do not pull up anti conditions to upper conds
} else if (OB_FAIL(stmt.get_table_rel_ids(semi_info->left_table_ids_, left_rel_ids))) {
LOG_WARN("failed to get table rel ids", K(ret));
} else {
for (int64_t j = 0; OB_SUCC(ret) && j < semi_info->semi_conditions_.count(); ++j) {
if (OB_ISNULL(expr = semi_info->semi_conditions_.at(j))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (!expr->has_flag(CNT_COLUMN) || !expr->get_relation_ids().is_subset(left_rel_ids)) {
if (remove_preds && OB_FAIL(new_semi_conditions.push_back(expr))) {
LOG_WARN("failed to push back expr", K(ret));
}
} else if (OB_FAIL(preds.push_back(expr))) {
LOG_WARN("failed to push back expr", K(ret));
}
}
if (OB_SUCC(ret) && remove_preds && semi_info->semi_conditions_.count() != new_semi_conditions.count()
&& OB_FAIL(semi_info->semi_conditions_.assign(new_semi_conditions))) {
LOG_WARN("failed to assign semi conditions", K(ret));
}
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_joined_tables().count(); ++i) {
if (OB_FAIL(gather_pullup_preds_from_join_table(stmt.get_joined_tables().at(i), preds, remove_preds))) {
LOG_WARN("failed to pullup preds from joined table", K(ret));
}
}
return ret;
}
int ObTransformPredicateMoveAround::gather_pullup_preds_from_join_table(TableItem *table,
ObIArray<ObRawExpr*> &preds,
bool remove_preds)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(table)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (table->is_joined_table()) {
JoinedTable *join_table = static_cast<JoinedTable*>(table);
if (join_table->is_inner_join() && OB_FAIL(append(preds, join_table->join_conditions_))) {
LOG_WARN("failed to append conditions", K(ret));
} else if ((join_table->is_inner_join() || join_table->is_left_join())
&& OB_FAIL(SMART_CALL(gather_pullup_preds_from_join_table(join_table->left_table_, preds, remove_preds)))) {
LOG_WARN("failed to pullup preds from joined table", K(ret));
} else if ((join_table->is_inner_join() || join_table->is_right_join())
&& OB_FAIL(SMART_CALL(gather_pullup_preds_from_join_table(join_table->right_table_, preds, remove_preds)))) {
LOG_WARN("failed to pullup preds from joined table", K(ret));
} else if (join_table->is_inner_join() && remove_preds) {
join_table->join_conditions_.reset();
}
}
return ret;
}
// be careful about anti join right table,
// do not pullup a filter which does not contain any column of the right table.
int ObTransformPredicateMoveAround::compute_pullup_predicates(
@ -1086,7 +1072,6 @@ int ObTransformPredicateMoveAround::pushdown_predicates(
bool has_group = false;
bool has_winfunc = false;
ObSEArray<ObRawExpr *, 4> candi_preds;
ObSEArray<ObRawExpr *, 4> old_where_preds;
ObIArray<ObRawExpr *> *pullup_preds = NULL;
ObSelectStmt *sel_stmt = static_cast<ObSelectStmt *>(stmt);
bool enable_no_pred_deduce = false;
@ -1099,8 +1084,6 @@ int ObTransformPredicateMoveAround::pushdown_predicates(
LOG_WARN("failed to acquire pull up preds", K(ret));
} else if (OB_FAIL(check_enable_no_pred_deduce(*stmt, enable_no_pred_deduce))) {
LOG_WARN("check_enable_no_pred_deduce failed", K(ret));
} else if (OB_FAIL(old_where_preds.assign(stmt->get_condition_exprs()))) {
LOG_WARN("failed to assign conditions", K(ret));
} else if (OB_ISNULL(pullup_preds)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("pullup predicate array is null", K(ret));
@ -1114,7 +1097,12 @@ int ObTransformPredicateMoveAround::pushdown_predicates(
pushdown_preds))) {
LOG_WARN("recursive pushdown preds into set stmt failed", K(ret));
} else {/*do nothing*/}
} else if (stmt->is_hierarchical_query()) {
} else if (stmt->is_hierarchical_query() || enable_no_pred_deduce) {
// do not transform for current level stmt, but need call function to transform child query
if (enable_no_pred_deduce) {
LOG_TRACE("NO PRED DEDUCE");
OPT_TRACE("hint disable transform");
}
ObArray<ObRawExpr *> dummy_preds;
for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_table_size(); ++i) {
TableItem *table = stmt->get_table_item(i);
@ -1127,10 +1115,9 @@ int ObTransformPredicateMoveAround::pushdown_predicates(
LOG_WARN("failed to push down predicates", K(ret));
} else {/*do nothing*/}
}
} else if (enable_no_pred_deduce) {
LOG_TRACE("NO PRED DEDUCE");
OPT_TRACE("hint disable transform");
} else {
const uint64_t pushdown_pred_count = pushdown_preds.count();
bool is_happened = false;
if (OB_FAIL(stmt->has_rownum(has_rownum))) {
LOG_WARN("failed to check stmt has rownum", K(ret));
} else if (stmt->is_select_stmt()) {
@ -1155,8 +1142,10 @@ int ObTransformPredicateMoveAround::pushdown_predicates(
} else {
pushdown_preds.reset();
}
if (OB_SUCC(ret) && pushdown_preds.count() != pushdown_pred_count) {
is_happened = true;
}
}
if (OB_SUCC(ret) && has_group) {
ObSEArray<ObRawExpr*, 4> old_having_exprs;
if (OB_FAIL(old_having_exprs.assign(sel_stmt->get_having_exprs()))) {
@ -1169,39 +1158,41 @@ int ObTransformPredicateMoveAround::pushdown_predicates(
} else if (OB_FAIL(pushdown_through_groupby(*sel_stmt, candi_preds))) {
LOG_WARN("failed to pushdown predicate", K(ret));
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(check_transform_happened(stmt, old_having_exprs, sel_stmt->get_having_exprs()))) {
if (OB_SUCC(ret) && OB_FAIL(check_conds_deduced(old_having_exprs, sel_stmt->get_having_exprs(), is_happened))) {
LOG_WARN("failed to check transform happened", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(pushdown_into_where(*stmt, *pullup_preds, candi_preds))) {
LOG_WARN("failed to push down predicates into where", K(ret));
} else if (OB_FAIL(check_transform_happened(stmt, old_where_preds, stmt->get_condition_exprs()))) {
LOG_WARN("failed to check transform happened", K(ret));
}
}
}
if (OB_SUCC(ret) && !stmt->is_set_stmt() && !stmt->is_hierarchical_query()) {
ObIArray<FromItem> &from_items = stmt->get_from_items();
ObIArray<SemiInfo*> &semi_infos = stmt->get_semi_infos();
ObSEArray<ObRawExpr *, 1> empty;
for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); ++i) {
if (OB_FAIL(pushdown_into_table(stmt, stmt->get_table_item(from_items.at(i)),
*pullup_preds, enable_no_pred_deduce ? empty : stmt->get_condition_exprs()))) {
LOG_WARN("failed to push down predicates", K(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < semi_infos.count(); ++i) {
if (OB_FAIL(pushdown_into_semi_info(stmt, semi_infos.at(i), *pullup_preds,
enable_no_pred_deduce ? empty : stmt->get_condition_exprs()))) {
LOG_WARN("failed to push down into semi info", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(check_transform_happened(stmt, old_where_preds, stmt->get_condition_exprs()))) {
LOG_WARN("failed to check transform happened", K(ret));
typedef ObSEArray<ObSEArray<ObRawExpr*, 16>, 4> PredsArray;
SMART_VAR(PredsArray, all_old_preds) {
ObIArray<FromItem> &from_items = stmt->get_from_items();
ObIArray<SemiInfo*> &semi_infos = stmt->get_semi_infos();
if ((!is_happened || !real_happened_) && OB_FAIL(store_all_preds(*stmt, all_old_preds))) {
LOG_WARN("failed to store all preds", K(ret));
} else if (OB_FAIL(gather_pullup_preds_from_semi_outer_join(*stmt, stmt->get_condition_exprs(), true))) {
LOG_WARN("failed to pullup preds from semi outer join", K(ret));
} else if (OB_FAIL(pushdown_into_where(*stmt, *pullup_preds, candi_preds))) {
LOG_WARN("failed to push down predicates into where", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); ++i) {
if (OB_FAIL(pushdown_into_table(stmt, stmt->get_table_item(from_items.at(i)),
*pullup_preds, stmt->get_condition_exprs()))) {
LOG_WARN("failed to push down predicates", K(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < semi_infos.count(); ++i) {
if (OB_FAIL(pushdown_into_semi_info(stmt, semi_infos.at(i), *pullup_preds,
stmt->get_condition_exprs()))) {
LOG_WARN("failed to push down into semi info", K(ret));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(check_transform_happened(*stmt, all_old_preds, is_happened))) {
LOG_WARN("failed to check transform happened", K(ret));
} else if (is_happened && OB_FAIL(add_var_to_array_no_dup(transed_stmts_, stmt))) {
LOG_WARN("append transed stmt failed", K(ret));
}
}
}
}
@ -1222,6 +1213,163 @@ int ObTransformPredicateMoveAround::pushdown_predicates(
return ret;
}
int ObTransformPredicateMoveAround::store_all_preds(const ObDMLStmt &stmt,
ObIArray<ObSEArray<ObRawExpr*, 16>> &all_preds)
{
int ret = OB_SUCCESS;
const ObIArray<JoinedTable*> &join_tables = stmt.get_joined_tables();
const ObIArray<SemiInfo*> &semi_infos = stmt.get_semi_infos();
ObIArray<ObRawExpr*> *preds = NULL;
all_preds.reuse();
if (OB_ISNULL(preds = all_preds.alloc_place_holder())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("Allocate ObRawExpr* ObSEArray from array error", K(ret));
} else if (OB_FAIL(preds->assign(stmt.get_condition_exprs()))) {
LOG_WARN("failed to assign conditions", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < join_tables.count(); ++i) {
if (OB_FAIL(store_join_conds(join_tables.at(i), all_preds))) {
LOG_WARN("failed to store join conds", K(ret));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < semi_infos.count(); ++i) {
if (OB_ISNULL(semi_infos.at(i))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (OB_ISNULL(preds = all_preds.alloc_place_holder())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("Allocate ObRawExpr* ObSEArray from array error", K(ret));
} else if (OB_FAIL(preds->assign(semi_infos.at(i)->semi_conditions_))) {
LOG_WARN("failed to assign conditions", K(ret));
}
}
return ret;
}
int ObTransformPredicateMoveAround::store_join_conds(const TableItem *table,
ObIArray<ObSEArray<ObRawExpr*, 16>> &all_preds)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(table)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (table->is_joined_table()) {
const JoinedTable *join_table = static_cast<const JoinedTable*>(table);
ObIArray<ObRawExpr*> *preds = NULL;
if (OB_ISNULL(preds = all_preds.alloc_place_holder())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("Allocate ObRawExpr* ObSEArray from array error", K(ret));
} else if (OB_FAIL(preds->assign(join_table->join_conditions_))) {
LOG_WARN("failed to assign conditions", K(ret));
} else if (OB_FAIL(SMART_CALL(store_join_conds(join_table->left_table_, all_preds)))) {
LOG_WARN("failed to pullup preds from joined table", K(ret));
} else if (OB_FAIL(SMART_CALL(store_join_conds(join_table->right_table_, all_preds)))) {
LOG_WARN("failed to pullup preds from joined table", K(ret));
}
}
return ret;
}
int ObTransformPredicateMoveAround::check_transform_happened(const ObDMLStmt &stmt,
const ObIArray<ObSEArray<ObRawExpr*, 16>> &all_preds,
bool &is_happened)
{
int ret = OB_SUCCESS;
const ObIArray<JoinedTable*> &join_tables = stmt.get_joined_tables();
const ObIArray<SemiInfo*> &semi_infos = stmt.get_semi_infos();
bool real_happened = is_happened && real_happened_;
if (real_happened) {
// do nothing
} else if (OB_UNLIKELY(all_preds.empty())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected empty", K(ret), K(all_preds.count()));
} else if (OB_FAIL(check_conds_deduced(all_preds.at(0), stmt.get_condition_exprs(), real_happened))) {
LOG_WARN("failed to push back conditions", K(ret));
} else {
uint64_t idx = 1;
for (int64_t i = 0; !real_happened && OB_SUCC(ret) && i < join_tables.count(); ++i) {
if (OB_FAIL(check_join_conds_deduced(join_tables.at(i), all_preds, idx, real_happened))) {
LOG_WARN("failed to check join conds deduced", K(ret));
}
}
if (OB_SUCC(ret) && OB_UNLIKELY(semi_infos.count() + idx > all_preds.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected count", K(ret), K(semi_infos.count()), K(idx), K(all_preds.count()));
}
for (int64_t i = 0; !real_happened && OB_SUCC(ret) && i < semi_infos.count(); ++i) {
if (OB_ISNULL(semi_infos.at(i))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (OB_FAIL(check_conds_deduced(all_preds.at(idx),
semi_infos.at(i)->semi_conditions_,
real_happened))) {
LOG_WARN("failed to check conds deduced", K(ret));
} else {
++idx;
}
}
if (OB_SUCC(ret)) {
is_happened |= real_happened;
}
}
return ret;
}
int ObTransformPredicateMoveAround::check_join_conds_deduced(const TableItem *table,
const ObIArray<ObSEArray<ObRawExpr*, 16>> &all_preds,
uint64_t &idx,
bool &is_happened)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(table)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret));
} else if (!is_happened && table->is_joined_table()) {
const JoinedTable *join_table = static_cast<const JoinedTable*>(table);
if (OB_UNLIKELY(all_preds.count() <= idx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected idx", K(ret), K(idx), K(all_preds.count()));
} else if (OB_FAIL(check_conds_deduced(all_preds.at(idx), join_table->join_conditions_, is_happened))) {
LOG_WARN("failed to check conds deduced", K(ret));
} else {
++idx;
}
if (OB_FAIL(ret) || is_happened) {
} else if (OB_FAIL(SMART_CALL(check_join_conds_deduced(join_table->left_table_, all_preds, idx, is_happened)))) {
LOG_WARN("failed to check join conds deduced", K(ret));
} else if (OB_FAIL(SMART_CALL(check_join_conds_deduced(join_table->right_table_, all_preds, idx, is_happened)))) {
LOG_WARN("failed to check join conds deduced", K(ret));
}
}
return ret;
}
int ObTransformPredicateMoveAround::check_conds_deduced(const ObIArray<ObRawExpr *> &old_conditions,
const ObIArray<ObRawExpr *> &new_conditions,
bool &is_happened)
{
int ret = OB_SUCCESS;
if (!is_happened || !real_happened_) {
bool happened = false;
for (int64_t i = 0; OB_SUCC(ret) && !happened && i < new_conditions.count(); ++i) {
if (!ObPredicateDeduce::find_equal_expr(old_conditions, new_conditions.at(i))) {
happened = true;
}
}
for (int64_t i = 0; OB_SUCC(ret) && !happened && i < old_conditions.count(); ++i) {
if (!ObPredicateDeduce::find_equal_expr(new_conditions, old_conditions.at(i))) {
happened = true;
}
}
if (OB_SUCC(ret) && happened) {
is_happened = true;
real_happened_ = true;
}
LOG_DEBUG("check transform happened", K(old_conditions), K(new_conditions), K(is_happened));
}
return ret;
}
int ObTransformPredicateMoveAround::pushdown_into_set_stmt(ObSelectStmt *stmt,
ObIArray<ObRawExpr *> &pullup_preds,
ObIArray<ObRawExpr *> &pushdown_preds)
@ -1281,7 +1429,6 @@ int ObTransformPredicateMoveAround::check_pushdown_predicates(ObSelectStmt *stmt
ObIArray<ObRawExpr *> &output_pushdown_preds)
{
int ret = OB_SUCCESS;
bool enable_no_pred_deduce = false;
if (OB_ISNULL(stmt)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("stmt is null", K(ret));
@ -2088,9 +2235,7 @@ int ObTransformPredicateMoveAround::pushdown_into_joined_table(
ObIArray<ObRawExpr *> &pushdown_preds)
{
int ret = OB_SUCCESS;
bool enable_no_pred_deduce = false;
ObSEArray<ObRawExpr *, 4> all_preds;
ObSEArray<ObRawExpr *, 4> old_join_condition;
/// STEP 1. deduce new join conditions
bool is_stack_overflow = false;
if (OB_FAIL(check_stack_overflow(is_stack_overflow))) {
@ -2102,8 +2247,6 @@ int ObTransformPredicateMoveAround::pushdown_into_joined_table(
OB_ISNULL(joined_table->right_table_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("params are invalid", K(ret), K(stmt), K(joined_table));
} else if (OB_FAIL(old_join_condition.assign(joined_table->join_conditions_))) {
LOG_WARN("failed to assign join condition", K(ret));
} else if (joined_table->is_left_join() ||
joined_table->is_right_join() ||
joined_table->is_inner_join()) {
@ -2146,10 +2289,6 @@ int ObTransformPredicateMoveAround::pushdown_into_joined_table(
LOG_WARN("failed to get related columns", K(ret));
} else if (OB_FAIL(transform_predicates(*stmt, all_preds, cols, new_preds))) {
LOG_WARN("failed to deduce predicates", K(ret));
} else if (OB_FAIL(check_enable_no_pred_deduce(*stmt, enable_no_pred_deduce))) {
LOG_WARN("check_enable_no_pred_deduce failed", K(ret));
} else if (enable_no_pred_deduce) {
OPT_TRACE("hint disable transform");
} else if (joined_table->is_inner_join()) {
if (OB_FAIL(accept_predicates(*stmt,
joined_table->join_conditions_,
@ -2270,58 +2409,10 @@ int ObTransformPredicateMoveAround::pushdown_into_joined_table(
//do nothing for full join
}
}
if (OB_FAIL(ret)) {
} else if (check_transform_happened(stmt, old_join_condition, joined_table->join_conditions_)) {
LOG_WARN("failed to check transform happened", K(ret));
}
}
return ret;
}
/**
* 只有当condition出现了新的谓词才说明发生了改写
* 如果condition的谓词减少了，定义不发生改写，整个查询是否
* 发生改写交由后续的推导判断，如果减少的谓词成功下推到下层视图了，才说明发生了改写
* 否则视为没改写
*/
int ObTransformPredicateMoveAround::check_transform_happened(ObDMLStmt *stmt,
ObIArray<ObRawExpr *> &old_conditions,
ObIArray<ObRawExpr *> &new_conditions)
{
int ret = OB_SUCCESS;
const ObQueryHint *query_hint = stmt->get_stmt_hint().query_hint_;
const ObTransHint *hint = static_cast<const ObTransHint *>(get_hint(stmt->get_stmt_hint()));
bool is_happened = false;
if (OB_ISNULL(query_hint)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null", K(ret), K(ctx_), K(query_hint));
} else if (query_hint->has_outline_data() && hint == NULL) {
if (OB_FAIL(new_conditions.assign(old_conditions))) {
LOG_WARN("assign failed", K(ret));
}
} else {
for (int64_t i = 0; OB_SUCC(ret) && !is_happened && i < new_conditions.count(); ++i) {
if (!ObPredicateDeduce::find_equal_expr(old_conditions, new_conditions.at(i))) {
is_happened = true;
}
}
for (int64_t i = 0; OB_SUCC(ret) && !is_happened && i < old_conditions.count(); ++i) {
if (!ObPredicateDeduce::find_equal_expr(new_conditions, old_conditions.at(i))) {
is_happened = true;
}
}
if (OB_SUCC(ret) && is_happened) {
trans_happened_ = true;
if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, stmt))) {
LOG_WARN("append failed", K(ret));
}
}
}
LOG_DEBUG("check_transform_happened", K(old_conditions), K(new_conditions), K(is_happened));
return ret;
}
int ObTransformPredicateMoveAround::get_pushdown_predicates(
ObDMLStmt &stmt, TableItem &table,
ObIArray<ObRawExpr *> &preds,
@ -2391,7 +2482,6 @@ int ObTransformPredicateMoveAround::pushdown_into_semi_info(ObDMLStmt *stmt,
{
int ret = OB_SUCCESS;
ObSEArray<ObRawExpr *, 4> all_preds;
ObSEArray<ObRawExpr*, 4> old_semi_conds;
ObSqlBitSet<> left_rel_ids;
ObSqlBitSet<> right_rel_ids;
TableItem *right_table = NULL;
@ -2400,14 +2490,11 @@ int ObTransformPredicateMoveAround::pushdown_into_semi_info(ObDMLStmt *stmt,
ObSEArray<ObRawExpr*, 4> all_cols;
ObSEArray<ObRawExpr*, 4> new_preds;
ObSEArray<ObRawExpr *, 16, common::ModulePageAllocator, true> empty;
bool enable_no_pred_deduce = false;
OPT_TRACE("try to transform semi conditions");
if (OB_ISNULL(stmt) || OB_ISNULL(semi_info) ||
OB_ISNULL(right_table = stmt->get_table_item_by_id(semi_info->right_table_id_))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("params have null", K(ret), K(stmt), K(semi_info), K(right_table));
} else if (OB_FAIL(old_semi_conds.assign(semi_info->semi_conditions_))) {
LOG_WARN("failed to assign exprs", K(ret));
} else if (OB_FAIL(stmt->get_table_rel_ids(semi_info->left_table_ids_, left_rel_ids))) {
LOG_WARN("failed to get left rel ids", K(ret));
} else if (OB_FAIL(stmt->get_table_rel_ids(semi_info->right_table_id_, right_rel_ids))) {
@ -2429,22 +2516,14 @@ int ObTransformPredicateMoveAround::pushdown_into_semi_info(ObDMLStmt *stmt,
LOG_WARN("failed to get related columns", K(ret));
} else if (OB_FAIL(transform_predicates(*stmt, all_preds, cols, new_preds))) {
LOG_WARN("failed to deduce predicates", K(ret));
} else if (OB_FAIL(check_enable_no_pred_deduce(*stmt, enable_no_pred_deduce))) {
LOG_WARN("check_enable_no_pred_deduce failed", K(ret));
} else if (!enable_no_pred_deduce && OB_FAIL(accept_predicates(*stmt, semi_info->semi_conditions_,
} else if (OB_FAIL(accept_predicates(*stmt, semi_info->semi_conditions_,
properites, new_preds))) {
LOG_WARN("failed to check different", K(ret));
} else if (OB_FAIL(pushdown_into_table(stmt, right_table, pullup_preds,
enable_no_pred_deduce ? empty : semi_info->semi_conditions_))) {
} else if (OB_FAIL(pushdown_into_table(stmt, right_table, pullup_preds, semi_info->semi_conditions_))) {
LOG_WARN("failed to push down predicates", K(ret));
} else if (OB_FAIL(pushdown_semi_info_right_filter(stmt, ctx_, semi_info))) {
LOG_WARN("failed to pushdown semi info right filter", K(ret));
}
if (OB_SUCC(ret) && OB_FAIL(check_transform_happened(stmt, old_semi_conds,
semi_info->semi_conditions_))) {
LOG_WARN("failed to check transform happened", K(ret));
}
return ret;
}
@ -2503,7 +2582,6 @@ int ObTransformPredicateMoveAround::check_has_shared_query_ref(ObRawExpr *expr,
// pushdown right table filter in semi condition:
// 1. if right table is a basic table, create a generate table.
// 2. pushdown the right table filters into the generate table.
// subquery in right filter can not be a shared subquery
int ObTransformPredicateMoveAround::pushdown_semi_info_right_filter(ObDMLStmt *stmt,
ObTransformerCtx *ctx,
SemiInfo *semi_info)
@ -2523,21 +2601,15 @@ int ObTransformPredicateMoveAround::pushdown_semi_info_right_filter(ObDMLStmt *s
LOG_WARN("failed to extract table exprs", K(ret));
} else if (right_filters.empty()) {
// do nothing
} else if (OB_FAIL(ObOptimizerUtil::remove_item(semi_info->semi_conditions_, right_filters))) {
LOG_WARN("failed to remove right filters", K(ret));
} else if (OB_FAIL(ObTransformUtils::can_push_down_filter_to_table(*right_table, can_push))) {
LOG_WARN("failed to check can push down", K(ret), K(*right_table));
} else if (can_push) {
ObArray<ObRawExpr *> new_right_filters;
if (OB_FAIL(ObTransformUtils::move_expr_into_view(*ctx_->expr_factory_,
*stmt,
*right_table,
right_filters,
new_right_filters))) {
LOG_WARN("failed to move expr into view", K(ret));
} else if (OB_FAIL(right_table->ref_query_->add_condition_exprs(new_right_filters))) {
LOG_WARN("failed to add condition exprs", K(ret));
}
// if a right filter can push down to right_table, has pushdown in pushdown_into_table,
// do not reture error here
} else if (OB_FAIL(ObOptimizerUtil::remove_item(semi_info->semi_conditions_, right_filters))) {
LOG_WARN("failed to remove right filters", K(ret));
} else if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, stmt))) {
LOG_WARN("append transed stmt failed", K(ret));
} else if (OB_FAIL(ObTransformUtils::replace_with_empty_view(ctx_,
stmt,
view_table,
@ -2603,12 +2675,17 @@ int ObTransformPredicateMoveAround::pushdown_into_table(ObDMLStmt *stmt,
if (OB_SUCC(ret) && table_item->is_generated_table()) {
// if predicates are pushed into the view, we can remove them from the upper stmt
ObSEArray<ObRawExpr *, 8> invalid_preds;
uint64_t old_candi_preds_count = 0;
if (OB_FAIL(rename_pushdown_predicates(*stmt, *table_item, rename_preds))) {
LOG_WARN("failed to rename predicates", K(ret));
} else if (OB_FAIL(choose_pushdown_preds(rename_preds, invalid_preds, candi_preds))) {
LOG_WARN("failed to choose predicates for pushdown", K(ret));
} else if (OB_FALSE_IT(old_candi_preds_count = candi_preds.count())) {
} else if (OB_FAIL(SMART_CALL(pushdown_predicates(table_item->ref_query_, candi_preds)))) {
LOG_WARN("failed to push down predicates", K(ret));
} else if (candi_preds.count() != old_candi_preds_count
&& OB_FAIL(add_var_to_array_no_dup(transed_stmts_, stmt))) {
LOG_WARN("append transed stmt failed", K(ret));
} else if (OB_FAIL(append(candi_preds, invalid_preds))) {
LOG_WARN("failed to append predicates", K(ret));
}
@ -2958,8 +3035,7 @@ int ObTransformPredicateMoveAround::get_columns_in_filters(
* @param preds
* @return
*/
int ObTransformPredicateMoveAround::create_equal_exprs_for_insert(ObDelUpdStmt *del_upd_stmt,
bool &is_happened)
int ObTransformPredicateMoveAround::create_equal_exprs_for_insert(ObDelUpdStmt *del_upd_stmt)
{
int ret = OB_SUCCESS;
ObSEArray<ObRawExpr * ,4> part_exprs;
@ -3114,11 +3190,10 @@ int ObTransformPredicateMoveAround::create_equal_exprs_for_insert(ObDelUpdStmt *
LOG_WARN("failed to add condition expr", K(ret));
} else if (is_not_null && OB_FAIL(ObTransformUtils::add_param_not_null_constraint(*ctx_, constraints))) {
LOG_WARN("failed to add param not null constraint", K(ret));
} else if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, static_cast<ObDMLStmt *>(del_upd_stmt)))) {
LOG_WARN("append failed", K(ret));
} else {
is_happened = true;
if (OB_FAIL(add_var_to_array_no_dup(transed_stmts_, static_cast<ObDMLStmt *>(del_upd_stmt)))) {
LOG_WARN("append failed", K(ret));
}
real_happened_ = true;
}
}
}
@ -3271,6 +3346,7 @@ int ObTransformPredicateMoveAround::construct_transform_hint(ObDMLStmt &stmt, vo
LOG_WARN("adjust stmt id failed", K(ret));
} else {
hint->set_qb_name(qb_name);
OPT_TRACE("transformed query blocks:", qb_name);
}
}
}

35
src/sql/rewrite/ob_transform_predicate_move_around.h
View File

@ -43,21 +43,14 @@ private:
const ObDMLStmt &stmt,
bool &need_trans) override;
int sort_transed_stmts();
int check_outline_valid_to_transform(const ObDMLStmt &stmt, bool &need_trans);
int pullup_predicates(ObDMLStmt *stmt,
ObIArray<int64_t> &select_list,
ObIArray<ObRawExpr *> &properties);
int preprocess(ObDMLStmt &stmt);
int preprocess_semi_info(ObDMLStmt &stmt,
SemiInfo *semi_info,
ObIArray<ObRawExpr*> &upper_conds);
int preprocess_joined_table(ObDMLStmt &stmt,
JoinedTable *join_table,
ObIArray<ObRawExpr*> &upper_conds);
int pullup_predicates_from_view(ObDMLStmt &stmt,
ObIArray<int64_t> &sel_ids,
ObIArray<ObRawExpr *> &input_pullup_preds);
@ -93,6 +86,12 @@ private:
ObIArray<int64_t> &sel_ids,
ObIArray<ObRawExpr *> &input_pullup_preds,
ObIArray<ObRawExpr *> &output_pullup_preds);
int gather_pullup_preds_from_semi_outer_join(ObDMLStmt &stmt,
ObIArray<ObRawExpr*> &preds,
bool remove_preds = false);
int gather_pullup_preds_from_join_table(TableItem *table,
ObIArray<ObRawExpr*> &preds,
bool remove_preds);
int remove_pullup_union_predicates(ObIArray<ObRawExpr *> &exprs);
@ -220,9 +219,18 @@ private:
ObIArray<ObRawExpr *> &pullup_preds,
ObIArray<ObRawExpr *> &pushdown_preds);
int check_transform_happened(ObDMLStmt *stmt,
ObIArray<ObRawExpr *> &old_conditions,
ObIArray<ObRawExpr *> &new_conditions);
int store_all_preds(const ObDMLStmt &stmt, ObIArray<ObSEArray<ObRawExpr*, 16>> &all_preds);
int store_join_conds(const TableItem *table, ObIArray<ObSEArray<ObRawExpr*, 16>> &all_preds);
int check_transform_happened(const ObDMLStmt &stmt,
const ObIArray<ObSEArray<ObRawExpr*, 16>> &all_preds,
bool &is_happened);
int check_join_conds_deduced(const TableItem *table,
const ObIArray<ObSEArray<ObRawExpr*, 16>> &all_preds,
uint64_t &idx,
bool &is_happened);
int check_conds_deduced(const ObIArray<ObRawExpr *> &old_conditions,
const ObIArray<ObRawExpr *> &new_conditions,
bool &is_happened);
int pushdown_through_winfunc(ObSelectStmt &stmt,
ObIArray<ObRawExpr *> &predicates,
@ -277,7 +285,7 @@ private:
ObIArray<int64_t> &sel_items,
ObIArray<ObRawExpr *> &columns);
int create_equal_exprs_for_insert(ObDelUpdStmt *del_upd_stmt, bool &is_happened);
int create_equal_exprs_for_insert(ObDelUpdStmt *del_upd_stmt);
int print_debug_info(const char *str, ObDMLStmt *stmt, ObIArray<ObRawExpr *> &preds);
@ -325,6 +333,7 @@ private:
Ob2DArray<PullupPreds *> stmt_pullup_preds_;
ObSEArray<ObDMLStmt *, 8> transed_stmts_;
ObSEArray<ObHint *, 4> applied_hints_;
bool real_happened_;
};
}

6
tools/deploy/mysql_test/test_suite/groupby/r/mysql/group_by_basic.result
View File

@ -360,15 +360,15 @@ insert into t1 (c1, c2, c3) values (2, 3, 2);
insert into t1 (c1, c2, c3) values (2, 3, 3);
select c2, avg(distinct c3) from t1 group by c2;
c2 avg(distinct c3)
2 2.5000
1 1.5000
2 2.5000
3 2.5000
select c1,c2, group_concat(distinct c3 order by c3 desc) from t1 group by c1,c2;
c1 c2 group_concat(distinct c3 order by c3 desc)
1 1 1
2 3 3,2
2 1 2
1 2 3,2
2 1 2
2 3 3,2
select c1,c2, group_concat(distinct c3 order by c3 desc) from t1 group by c1,c2 with rollup;
c1 c2 group_concat(distinct c3 order by c3 desc)
1 1 1