diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 0bc6746674..7e87fd4c15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -41,6 +41,8 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; @@ -50,6 +52,7 @@ import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.TypeUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -149,7 +152,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac queryStructInfo.addPredicates(pulledUpExpressions); } SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, - queryToViewSlotMapping); + queryToViewSlotMapping, comparisonResult, cascadesContext); // Can not compensate, bail out if (compensatePredicates.isEmpty()) { materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), @@ -189,16 +192,15 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac if (rewrittenPlan == null) { continue; } - // checkout the output logical properties is the same with query - if (!checkOutput(queryPlan, rewrittenPlan, materializationContext)) { - continue; - } // run rbo job on mv rewritten plan CascadesContext rewrittenPlanContext = CascadesContext.initContext( cascadesContext.getStatementContext(), rewrittenPlan, cascadesContext.getCurrentJobContext().getRequiredProperties()); Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute(); rewrittenPlan = rewrittenPlanContext.getRewritePlan(); + if (!checkOutput(queryPlan, rewrittenPlan, materializationContext)) { + continue; + } // check the partitions used by rewritten plan is valid or not Set invalidPartitionsQueryUsed = calcInvalidPartitions(rewrittenPlan, materializationContext, cascadesContext); @@ -213,7 +215,6 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac .collect(Collectors.toSet())))); continue; } - materializationContext.setSuccess(true); recordIfRewritten(queryPlan, materializationContext); rewriteResults.add(rewrittenPlan); } @@ -315,20 +316,28 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac List rewrittenExpressions = new ArrayList<>(); for (int index = 0; index < sourceShuttledExpressions.size(); index++) { - Expression expressionToRewrite = sourceShuttledExpressions.get(index); - if (expressionToRewrite instanceof Literal) { - rewrittenExpressions.add(expressionToRewrite); + Expression expressionShuttledToRewrite = sourceShuttledExpressions.get(index); + if (expressionShuttledToRewrite instanceof Literal) { + rewrittenExpressions.add(expressionShuttledToRewrite); continue; } - final Set slotsToRewrite = expressionToRewrite.collectToSet( - expression -> expression instanceof Slot); - Expression replacedExpression = ExpressionUtils.replace(expressionToRewrite, + final Set slotsToRewrite = + expressionShuttledToRewrite.collectToSet(expression -> expression instanceof Slot); + Expression replacedExpression = ExpressionUtils.replace(expressionShuttledToRewrite, targetToTargetReplacementMapping); if (replacedExpression.anyMatch(slotsToRewrite::contains)) { // if contains any slot to rewrite, which means can not be rewritten by target, bail out return ImmutableList.of(); } Expression sourceExpression = sourceExpressionsToWrite.get(index); + if (sourceExpression instanceof NamedExpression + && replacedExpression.nullable() != sourceExpression.nullable()) { + // if enable join eliminate, query maybe inner join and mv maybe outer join. + // If the slot is at null generate side, the nullable maybe different between query and view + // So need to force to consistent. + replacedExpression = sourceExpression.nullable() + ? new Nullable(replacedExpression) : new NonNullable(replacedExpression); + } if (sourceExpression instanceof NamedExpression) { NamedExpression sourceNamedExpression = (NamedExpression) sourceExpression; replacedExpression = new Alias(sourceNamedExpression.getExprId(), replacedExpression, @@ -358,30 +367,87 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac * For another example as following: * predicate a = b in mv, and a = b and c = d in query, the compensatory predicate is c = d */ - protected SplitPredicate predicatesCompensate(StructInfo queryStructInfo, StructInfo viewStructInfo, - SlotMapping queryToViewSlotMapping) { + protected SplitPredicate predicatesCompensate( + StructInfo queryStructInfo, + StructInfo viewStructInfo, + SlotMapping queryToViewSlotMapping, + ComparisonResult comparisonResult, + CascadesContext cascadesContext + ) { + // viewEquivalenceClass to query based + SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); + final Set equalCompensateConjunctions = compensateEquivalence( + queryStructInfo, + viewStructInfo, + viewToQuerySlotMapping, + comparisonResult); + // range compensate + final Set rangeCompensatePredicates = compensateRangePredicate( + queryStructInfo, + viewStructInfo, + viewToQuerySlotMapping, + comparisonResult); + // residual compensate + final Set residualCompensatePredicates = compensateResidualPredicate( + queryStructInfo, + viewStructInfo, + viewToQuerySlotMapping, + comparisonResult); + // if the join type in query and mv plan is different, we should check and add filter on mv to make + // the mv join type is accord with query + Set> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot(); + // check query is use the null reject slot which view comparison need + if (!requireNoNullableViewSlot.isEmpty()) { + Set queryPulledUpPredicates = queryStructInfo.getPredicates().getPulledUpPredicates(); + Set nullRejectPredicates = ExpressionUtils.inferNotNull(queryPulledUpPredicates, + cascadesContext); + if (nullRejectPredicates.isEmpty() || queryPulledUpPredicates.containsAll(nullRejectPredicates)) { + // query has not null reject predicates, so return + return SplitPredicate.invalid(); + } + Set queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream() + .map(expression -> TypeUtils.isNotNull(expression).orElse(null)) + .filter(Objects::nonNull) + .map(expr -> ExpressionUtils.replace((Expression) expr, + queryToViewSlotMapping.toSlotReferenceMap())) + .collect(Collectors.toSet()); + if (requireNoNullableViewSlot.stream().anyMatch( + set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty())) { + return SplitPredicate.invalid(); + } + } + return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions), + rangeCompensatePredicates.isEmpty() ? BooleanLiteral.of(true) + : ExpressionUtils.and(rangeCompensatePredicates), + residualCompensatePredicates.isEmpty() ? BooleanLiteral.of(true) + : ExpressionUtils.and(residualCompensatePredicates)); + } + + protected Set compensateEquivalence(StructInfo queryStructInfo, + StructInfo viewStructInfo, + SlotMapping viewToQuerySlotMapping, + ComparisonResult comparisonResult) { EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass(); EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass(); - // viewEquivalenceClass to query based - Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse() - .toSlotReferenceMap(); - EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping); + Map viewToQuerySlotMap = viewToQuerySlotMapping.toSlotReferenceMap(); + EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMap); if (viewEquivalenceClassQueryBased == null) { - return SplitPredicate.empty(); + return ImmutableSet.of(); } - final List equalCompensateConjunctions = new ArrayList<>(); + final Set equalCompensateConjunctions = new HashSet<>(); if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) { equalCompensateConjunctions.add(BooleanLiteral.of(true)); } - if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) { - return SplitPredicate.empty(); + if (queryEquivalenceClass.isEmpty() + && !viewEquivalenceClass.isEmpty()) { + return ImmutableSet.of(); } - EquivalenceClassSetMapping queryToViewEquivalenceMapping = EquivalenceClassSetMapping.generate( - queryEquivalenceClass, viewEquivalenceClassQueryBased); + EquivalenceClassSetMapping queryToViewEquivalenceMapping = + EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased); // can not map all target equivalence class, can not compensate if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size() < viewEquivalenceClass.getEquivalenceSetList().size()) { - return SplitPredicate.empty(); + return ImmutableSet.of(); } // do equal compensate Set> mappedQueryEquivalenceSet = @@ -410,49 +476,57 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac } } ); - // TODO range predicates and residual predicates compensate, Simplify implementation. + return equalCompensateConjunctions; + } + + protected Set compensateResidualPredicate(StructInfo queryStructInfo, + StructInfo viewStructInfo, + SlotMapping viewToQuerySlotMapping, + ComparisonResult comparisonResult) { SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); - - // range compensate - List rangeCompensate = new ArrayList<>(); - Expression queryRangePredicate = querySplitPredicate.getRangePredicate(); - Expression viewRangePredicate = viewSplitPredicate.getRangePredicate(); - Expression viewRangePredicateQueryBased = ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping); - - Set queryRangeSet = Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); - Set viewRangeQueryBasedSet = Sets.newHashSet( - ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); - // query range predicate can not contain all view range predicate when view have range predicate, bail out - if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE) && !queryRangeSet.containsAll( - viewRangeQueryBasedSet)) { - return SplitPredicate.empty(); - } - queryRangeSet.removeAll(viewRangeQueryBasedSet); - rangeCompensate.addAll(queryRangeSet); - - // residual compensate - List residualCompensate = new ArrayList<>(); Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate(); Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate(); Expression viewResidualPredicateQueryBased = - ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping); + ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping.toSlotReferenceMap()); Set queryResidualSet = Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate)); Set viewResidualQueryBasedSet = Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased)); // query residual predicate can not contain all view residual predicate when view have residual predicate, // bail out - if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE) && !queryResidualSet.containsAll( - viewResidualQueryBasedSet)) { - return SplitPredicate.empty(); + if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE) + && !queryResidualSet.containsAll(viewResidualQueryBasedSet)) { + return ImmutableSet.of(); } queryResidualSet.removeAll(viewResidualQueryBasedSet); - residualCompensate.addAll(queryResidualSet); + return queryResidualSet; + } - return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions), - rangeCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(rangeCompensate), - residualCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(residualCompensate)); + protected Set compensateRangePredicate(StructInfo queryStructInfo, + StructInfo viewStructInfo, + SlotMapping viewToQuerySlotMapping, + ComparisonResult comparisonResult) { + // TODO range predicates and residual predicates compensate, Simplify implementation. + SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); + SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); + + Expression queryRangePredicate = querySplitPredicate.getRangePredicate(); + Expression viewRangePredicate = viewSplitPredicate.getRangePredicate(); + Expression viewRangePredicateQueryBased = + ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping.toSlotReferenceMap()); + + Set queryRangeSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); + Set viewRangeQueryBasedSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); + // query range predicate can not contain all view range predicate when view have range predicate, bail out + if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE) + && !queryRangeSet.containsAll(viewRangeQueryBasedSet)) { + return ImmutableSet.of(); + } + queryRangeSet.removeAll(viewRangeQueryBasedSet); + return queryRangeSet; } /** @@ -508,6 +582,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac } protected void recordIfRewritten(Plan plan, MaterializationContext context) { + context.setSuccess(true); if (plan.getGroupExpression().isPresent()) { context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 0a25a603ae..39a89c80c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -51,7 +51,7 @@ public class Predicates { return predicates; } - public Set getPulledUpPredicates() { + public Set getPulledUpPredicates() { return pulledUpPredicates; } @@ -103,7 +103,7 @@ public class Predicates { return residualPredicate.orElse(BooleanLiteral.TRUE); } - public static SplitPredicate empty() { + public static SplitPredicate invalid() { return new SplitPredicate(null, null, null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java index de9115468f..da8f582f64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java @@ -66,6 +66,9 @@ public class PredicatesSplitter { if (leftArgOnlyContainsColumnRef && rightArgOnlyContainsColumnRef) { equalPredicates.add(comparisonPredicate); return null; + } else if ((leftArgOnlyContainsColumnRef && rightArg instanceof Literal) + || (rightArgOnlyContainsColumnRef && leftArg instanceof Literal)) { + rangePredicates.add(comparisonPredicate); } else { residualPredicates.add(comparisonPredicate); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NonNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NonNullable.java index b9cec8e3ce..f008058c76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NonNullable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/NonNullable.java @@ -24,6 +24,10 @@ import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.types.DataType; +import com.google.common.base.Preconditions; + +import java.util.List; + /** * change nullable input col to non_nullable col */ @@ -39,4 +43,10 @@ public class NonNullable extends ScalarFunction implements UnaryExpression, Cust return FunctionSignature.ret(dataType).args(dataType); } + @Override + public Expression withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, + "the child expression of NonNullable should be only one"); + return new NonNullable(children.get(0)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nullable.java index 49db085a5f..1753acecbf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nullable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nullable.java @@ -24,6 +24,10 @@ import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.types.DataType; +import com.google.common.base.Preconditions; + +import java.util.List; + /** * change non_nullable input col to nullable col */ @@ -39,4 +43,10 @@ public class Nullable extends ScalarFunction implements UnaryExpression, CustomS return FunctionSignature.ret(dataType).args(dataType); } + @Override + public Expression withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, + "the child expression of NonNullable should be only one"); + return new Nullable(children.get(0)); + } } diff --git a/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out b/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out index abf0eb188e..62d3523c67 100644 --- a/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out +++ b/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out @@ -99,6 +99,14 @@ 6 6 +-- !query1_5_before -- +6 +6 + +-- !query1_5_after -- +6 +6 + -- !query2_0_before -- 4 4 @@ -239,6 +247,20 @@ 6 6 +-- !query3_4_before -- +1 1 +1 1 +1 1 +1 1 +1 1 + +-- !query3_4_after -- +1 1 +1 1 +1 1 +1 1 +1 1 + -- !query4_0_before -- 4 4 diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index 7a111bc169..02f628b4f6 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -22,8 +22,6 @@ suite("aggregate_with_roll_up") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" - // tmp disable to rewrite, will be removed in the future - sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 39958e309b..d1d01ebee8 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -22,9 +22,6 @@ suite("aggregate_without_roll_up") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" - // tmp disable to rewrite, will be removed in the future - sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'" - sql "SET global enable_auto_analyze = false" sql """ drop table if exists orders @@ -173,8 +170,8 @@ suite("aggregate_without_roll_up") { } } - // single table - // with filter +// // single table +// // with filter def mv1_0 = "select o_shippriority, o_comment, " + "sum(o_totalprice) as sum_total, " + "max(o_totalprice) as max_total, " + diff --git a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy index 6f4028d609..5fd4124f4d 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy @@ -22,8 +22,6 @@ suite("inner_join") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" - // tmp disable to rewrite, will be removed in the future - sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'" sql """ drop table if exists orders @@ -231,6 +229,21 @@ suite("inner_join") { order_qt_query1_4_after "${query1_4}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_4""" + def mv1_5 = """ + select lineitem.L_LINENUMBER, orders.O_CUSTKEY, l_partkey, o_shippriority + from lineitem + inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY; + """ + def query1_5 = """ + select lineitem.L_LINENUMBER + from lineitem + inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY + and o_shippriority = l_partkey; + """ + order_qt_query1_5_before "${query1_5}" + check_rewrite(mv1_5, query1_5, "mv1_5") + order_qt_query1_5_after "${query1_5}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5""" // filter outside + left def mv2_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + @@ -349,6 +362,26 @@ suite("inner_join") { order_qt_query3_3_after "${query3_3}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_3""" + // join derive, the mv is outer join with filter and query is inner join + // the predicate should be ComparisonPredicate + def mv3_4 = """ + select l_linenumber, o_custkey + from orders + left join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY + where o_custkey = 1; + """ + def query3_4 = """ + select IFNULL(orders.O_CUSTKEY, 0) as custkey_not_null, + case when l_linenumber in (1,2,3) then l_linenumber else o_custkey end as case_when + from orders + inner join lineitem on orders.O_ORDERKEY = lineitem.L_ORDERKEY + where o_custkey = 1 and l_linenumber > 0; + """ + order_qt_query3_4_before "${query3_4}" + check_rewrite(mv3_4, query3_4, "mv3_4") + order_qt_query3_4_after "${query3_4}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_4""" + // filter outside + left + right def mv4_0 = "select l_linenumber, o_custkey, o_orderkey, o_orderstatus " + diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy index 6420248418..a9e21f63cf 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy @@ -22,8 +22,6 @@ suite("outer_join") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" - // tmp disable to rewrite, will be removed in the future - sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'" sql """ drop table if exists orders @@ -259,13 +257,17 @@ suite("outer_join") { // filter outside + right - def mv3_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + - "from lineitem " + - "left join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " - def query3_0 = "select lineitem.L_LINENUMBER " + - "from lineitem " + - "left join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + - "where orders.O_ORDERSTATUS = 'o'" + def mv3_0 = """ + select lineitem.L_LINENUMBER, orders.O_CUSTKEY + from lineitem + left join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY; + """ + def query3_0 = """ + select lineitem.L_LINENUMBER + from lineitem + left join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY + where orders.O_ORDERSTATUS = 'o'; + """ order_qt_query3_0_before "${query3_0}" // use a filed not from mv, should not success check_not_match(mv3_0, query3_0, "mv3_0") @@ -303,13 +305,17 @@ suite("outer_join") { // filter outside + left + right - def mv4_0 = "select l_linenumber, o_custkey, o_orderkey, o_orderstatus " + - "from lineitem " + - "left join orders on lineitem.l_orderkey = orders.o_orderkey " - def query4_0 = "select lineitem.l_linenumber " + - "from lineitem " + - "left join orders on lineitem.l_orderkey = orders.o_orderkey " + - "where o_orderstatus = 'o' AND o_orderkey = 1" + def mv4_0 = """ + select l_linenumber, o_custkey, o_orderkey, o_orderstatus + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey; + """ + def query4_0 = """ + select lineitem.l_linenumber + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey + where o_orderstatus = 'o' AND o_orderkey = 1; + """ order_qt_query4_0_before "${query4_0}" check_rewrite(mv4_0, query4_0, "mv4_0") order_qt_query4_0_after "${query4_0}" @@ -364,7 +370,7 @@ suite("outer_join") { // self join test def mv8_0 = """ - select + select a.o_orderkey, count(distinct a.o_orderstatus) num1, SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate = '2023-12-08' AND b.o_orderdate = '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num2, @@ -381,7 +387,7 @@ suite("outer_join") { group by a.o_orderkey; """ def query8_0 = """ - select + select a.o_orderkey, SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate = '2023-12-08' AND b.o_orderdate = '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num2, SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate >= '2023-12-01' AND a.o_orderdate <= '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num3, diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy index 97208b4e0b..428e6a9e13 100644 --- a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy @@ -22,8 +22,6 @@ suite("partition_mv_rewrite") { sql "SET enable_fallback_to_original_planner=false" sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" - // tmp disable to rewrite, will be removed in the future - sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'" sql """ drop table if exists orders