diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 75d886a1dc..6ad21d3489 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -65,6 +65,7 @@ import org.apache.doris.nereids.properties.DistributionSpecStorageGather; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup; +import org.apache.doris.nereids.rules.rewrite.MergeLimits; import org.apache.doris.nereids.stats.StatsErrorEstimator; import org.apache.doris.nereids.trees.UnaryNode; import org.apache.doris.nereids.trees.expressions.AggregateExpression; @@ -1536,8 +1537,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor physicalLimit, PlanTranslatorContext context) { PlanFragment inputFragment = physicalLimit.child(0).accept(this, context); PlanNode child = inputFragment.getPlanRoot(); - child.setOffset(physicalLimit.getOffset()); - child.setLimit(physicalLimit.getLimit()); + child.setLimit(MergeLimits.mergeLimit(physicalLimit.getLimit(), physicalLimit.getOffset(), child.getLimit())); + child.setOffset(MergeLimits.mergeOffset(physicalLimit.getOffset(), child.getOffset())); updateLegacyPlanIdToPhysicalPlan(child, physicalLimit); return inputFragment; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java index e1428983c1..e544eed07c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java @@ -47,11 +47,21 @@ public class MergeLimits extends OneRewriteRuleFactory { .then(upperLimit -> { LogicalLimit bottomLimit = upperLimit.child(); return new LogicalLimit<>( - Math.min(upperLimit.getLimit(), - Math.max(bottomLimit.getLimit() - upperLimit.getOffset(), 0)), - bottomLimit.getOffset() + upperLimit.getOffset(), + mergeLimit(upperLimit.getLimit(), upperLimit.getOffset(), bottomLimit.getLimit()), + mergeOffset(upperLimit.getOffset(), bottomLimit.getOffset()), upperLimit.child().getPhase(), bottomLimit.child() ); }).toRule(RuleType.MERGE_LIMITS); } + + public static long mergeOffset(long upperOffset, long bottomOffset) { + return upperOffset + bottomOffset; + } + + public static long mergeLimit(long upperLimit, long upperOffset, long bottomLimit) { + if (bottomLimit < 0) { + return upperLimit; + } + return Math.min(upperLimit, Math.max(bottomLimit - upperOffset, 0)); + } } diff --git a/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out b/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out index a02974e6cc..929e4ce0d6 100644 --- a/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out +++ b/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out @@ -1,70 +1,55 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !eliminate_not_null_basic_comparison -- PhysicalResultSink ---PhysicalDistribute -----filter(( not score IS NULL) and (t.score > 13)) -------PhysicalOlapScan[t] +--filter(( not score IS NULL) and (t.score > 13)) +----PhysicalOlapScan[t] -- !eliminate_not_null_in_clause -- PhysicalResultSink ---PhysicalDistribute -----filter(( not id IS NULL) and id IN (1, 2, 3)) -------PhysicalOlapScan[t] +--filter(( not id IS NULL) and id IN (1, 2, 3)) +----PhysicalOlapScan[t] -- !eliminate_not_null_not_equal -- PhysicalResultSink ---PhysicalDistribute -----filter(( not (score = 13)) and ( not score IS NULL)) -------PhysicalOlapScan[t] +--filter(( not (score = 13)) and ( not score IS NULL)) +----PhysicalOlapScan[t] -- !eliminate_not_null_string_function -- PhysicalResultSink ---PhysicalDistribute -----filter(( not name IS NULL) and (length(name) > 0)) -------PhysicalOlapScan[t] +--filter(( not name IS NULL) and (length(name) > 0)) +----PhysicalOlapScan[t] -- !eliminate_not_null_aggregate -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashAgg[LOCAL] ---------PhysicalProject -----------filter(( not score IS NULL) and (t.score > 0)) -------------PhysicalOlapScan[t] +--hashAgg[LOCAL] +----filter(( not score IS NULL) and (t.score > 0)) +------PhysicalOlapScan[t] -- !eliminate_not_null_between -- PhysicalResultSink ---PhysicalDistribute -----filter(( not score IS NULL) and (t.score <= 10) and (t.score >= 1)) -------PhysicalOlapScan[t] +--filter(( not score IS NULL) and (t.score <= 10) and (t.score >= 1)) +----PhysicalOlapScan[t] -- !eliminate_not_null_math_function -- PhysicalResultSink ---PhysicalDistribute -----filter(( not score IS NULL) and (abs(score) = 5)) -------PhysicalOlapScan[t] +--filter(( not score IS NULL) and (abs(score) = 5)) +----PhysicalOlapScan[t] -- !eliminate_not_null_complex_logic -- PhysicalResultSink ---PhysicalDistribute -----filter(( not score IS NULL) and ((t.score > 5) OR (t.id < 10))) -------PhysicalOlapScan[t] +--filter(( not score IS NULL) and ((t.score > 5) OR (t.id < 10))) +----PhysicalOlapScan[t] -- !eliminate_not_null_date_function -- PhysicalResultSink ---PhysicalDistribute -----filter(( not name IS NULL) and (year(cast(name as DATEV2)) = 2022)) -------PhysicalOlapScan[t] +--filter(( not name IS NULL) and (year(cast(name as DATEV2)) = 2022)) +----PhysicalOlapScan[t] -- !eliminate_not_null_with_subquery -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[LEFT_SEMI_JOIN] hashCondition=((t.score = t.score)) otherCondition=() build RFs:RF0 score->[score] ---------filter(( not score IS NULL) and (t.score > 0)) -----------PhysicalOlapScan[t] apply RFs: RF0 ---------PhysicalDistribute -----------PhysicalProject -------------filter((t.score > 0)) ---------------PhysicalOlapScan[t] +--hashJoin[LEFT_SEMI_JOIN] hashCondition=((t.score = t.score)) otherCondition=() build RFs:RF0 score->[score] +----filter(( not score IS NULL) and (t.score > 0)) +------PhysicalOlapScan[t] apply RFs: RF0 +----filter((t.score > 0)) +------PhysicalOlapScan[t] diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out index bbac000f08..355b4b231f 100644 --- a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out +++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out @@ -1,157 +1,103 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !pushdown_cross_join -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------NestedLoopJoin[INNER_JOIN](t1.msg > t2.msg) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--NestedLoopJoin[INNER_JOIN](t1.msg > t2.msg) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_cross_join -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=() ---------PhysicalDistribute -----------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=() +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_inner_join -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg > t2.msg)) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg > t2.msg)) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_left_join -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg > t2.msg)) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg > t2.msg)) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_right_join -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t2.msg < t1.msg)) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t2.msg < t1.msg)) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_full_join -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg < t2.msg)) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg < t2.msg)) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_cross_join -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------NestedLoopJoin[INNER_JOIN](t1.msg < t2.msg) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--NestedLoopJoin[INNER_JOIN](t1.msg < t2.msg) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_inner_join_hash -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=() ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=() +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_left_join_hash -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=() ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=() +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_right_join_hash -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t2.msg = t1.msg)) otherCondition=() ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t2.msg = t1.msg)) otherCondition=() +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_full_join_hash -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=() ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=() +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_cross_join_combine -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) ---------PhysicalDistribute -----------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_inner_join_combine -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_left_join_combine -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_right_join_combine -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t2.msg = t1.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t2.msg = t1.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_full_join_combine -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) ---------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] -- !pushdown_cross_join_combine -- PhysicalResultSink ---PhysicalDistribute -----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) ---------PhysicalDistribute -----------PhysicalOlapScan[t1] ---------PhysicalDistribute -----------PhysicalOlapScan[t2] +--hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE))) +----PhysicalOlapScan[t1] +----PhysicalOlapScan[t2] diff --git a/regression-test/suites/nereids_rules_p0/eliminate_not_null/eliminate_not_null.groovy b/regression-test/suites/nereids_rules_p0/eliminate_not_null/eliminate_not_null.groovy index 1b81108ada..cc318a3e85 100644 --- a/regression-test/suites/nereids_rules_p0/eliminate_not_null/eliminate_not_null.groovy +++ b/regression-test/suites/nereids_rules_p0/eliminate_not_null/eliminate_not_null.groovy @@ -18,6 +18,8 @@ suite("eliminate_not_null") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + sql "SET disable_join_reorder=true" sql """ DROP TABLE IF EXISTS t diff --git a/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy b/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy index dfb65dd246..735aaa939e 100644 --- a/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy +++ b/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy @@ -18,7 +18,9 @@ suite("filter_push_through_aggregate") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" - + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + sql "SET disable_join_reorder=true" + sql """ DROP TABLE IF EXISTS filter_push_through_aggregate """ diff --git a/regression-test/suites/nereids_rules_p0/filter_push_down/push_filter_inside_join.groovy b/regression-test/suites/nereids_rules_p0/filter_push_down/push_filter_inside_join.groovy index a19ddfd506..9dfadb9630 100644 --- a/regression-test/suites/nereids_rules_p0/filter_push_down/push_filter_inside_join.groovy +++ b/regression-test/suites/nereids_rules_p0/filter_push_down/push_filter_inside_join.groovy @@ -19,7 +19,8 @@ suite("push_filter_inside_join") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "use regression_test_nereids_rules_p0" - sql "set disable_join_reorder=true" + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + sql "SET disable_join_reorder=true" sql 'set be_number_for_test=3' // Push down > condition to cross join