diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java index 6c25337ac9..3dbd6cfcec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java @@ -22,9 +22,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleId; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.processor.post.RuntimeFilterContext; -import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -40,6 +38,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.thrift.TRuntimeFilterType; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -73,21 +72,15 @@ public class RuntimeFilterTranslator { } private class RuntimeFilterExpressionTranslator extends ExpressionTranslator { - Map nereidsExprIdToSlotRef; + SlotRef targetSlotRef; - RuntimeFilterExpressionTranslator(Map nereidsExprIdToSlotRef) { - this.nereidsExprIdToSlotRef = nereidsExprIdToSlotRef; + RuntimeFilterExpressionTranslator(SlotRef targetSlotRef) { + this.targetSlotRef = targetSlotRef; } @Override public Expr visitSlotReference(SlotReference slotReference, PlanTranslatorContext context) { - slotReference = context.getRuntimeTranslator().get() - .context.getCorrespondingOlapSlotReference(slotReference); - SlotRef slot = nereidsExprIdToSlotRef.get(slotReference.getExprId()); - if (slot == null) { - throw new AnalysisException("cannot find SlotRef for " + slotReference); - } - return slot; + return targetSlotRef; } } @@ -111,8 +104,8 @@ public class RuntimeFilterTranslator { for (int i = 0; i < filter.getTargetExpressions().size(); i++) { Slot curTargetSlot = filter.getTargetSlots().get(i); Expression curTargetExpression = filter.getTargetExpressions().get(i); - Expr target = context.getExprIdToOlapScanNodeSlotRef().get(curTargetSlot.getExprId()); - if (target == null) { + SlotRef targetSlotRef = context.getExprIdToOlapScanNodeSlotRef().get(curTargetSlot.getExprId()); + if (targetSlotRef == null) { context.setTargetNullCount(); hasInvalidTarget = true; break; @@ -120,10 +113,15 @@ public class RuntimeFilterTranslator { ScanNode scanNode = context.getScanNodeOfLegacyRuntimeFilterTarget().get(curTargetSlot); Expr targetExpr; if (curTargetSlot.equals(curTargetExpression)) { - targetExpr = target; + targetExpr = targetSlotRef; } else { - RuntimeFilterExpressionTranslator translator = new RuntimeFilterExpressionTranslator( - context.getExprIdToOlapScanNodeSlotRef()); + // map nereids target slot to original planner slot + Preconditions.checkArgument(curTargetExpression.getInputSlots().size() == 1, + "target expression is invalid, input slot num > 1; filter :" + filter); + Slot slotInTargetExpression = curTargetExpression.getInputSlots().iterator().next(); + Preconditions.checkArgument(slotInTargetExpression.equals(curTargetSlot) + || curTargetSlot.equals(context.getAliasTransferMap().get(slotInTargetExpression).second)); + RuntimeFilterExpressionTranslator translator = new RuntimeFilterExpressionTranslator(targetSlotRef); targetExpr = curTargetExpression.accept(translator, ctx); } @@ -131,7 +129,7 @@ public class RuntimeFilterTranslator { if (!src.getType().equals(targetExpr.getType()) && filter.getType() != TRuntimeFilterType.BITMAP) { targetExpr = new CastExpr(src.getType(), targetExpr); } - SlotRef targetSlot = target.getSrcSlotRef(); + SlotRef targetSlot = targetSlotRef.getSrcSlotRef(); TupleId targetTupleId = targetSlot.getDesc().getParent().getId(); SlotId targetSlotId = targetSlot.getSlotId(); scanNodeList.add(scanNode); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index 55d990365a..d7e3f95be6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -51,6 +51,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; @@ -283,8 +284,10 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { || (pair != null && pair.first instanceof PhysicalCTEConsumer)) { continue; } - join.pushDownRuntimeFilter(context, generator, join, equalTo.right(), - equalTo.left(), type, buildSideNdv, i); + if (equalTo.left().getInputSlots().size() == 1) { + join.pushDownRuntimeFilter(context, generator, join, equalTo.right(), + equalTo.left(), type, buildSideNdv, i); + } } } return join; @@ -336,7 +339,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { TRuntimeFilterType type = TRuntimeFilterType.BITMAP; Set targetSlots = bitmapContains.child(1).getInputSlots(); for (Slot targetSlot : targetSlots) { - if (!checkPushDownPreconditionsForJoin(join, ctx, targetSlot)) { + if (!checkProbeSlot(ctx, targetSlot)) { continue; } Slot scanSlot = ctx.getAliasTransferPair(targetSlot).second; @@ -495,6 +498,23 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { return relation; } + @Override + public PhysicalSetOperation visitPhysicalSetOperation(PhysicalSetOperation setOperation, CascadesContext context) { + setOperation.children().forEach(child -> child.accept(this, context)); + RuntimeFilterContext ctx = context.getRuntimeFilterContext(); + if (!setOperation.getRegularChildrenOutputs().isEmpty()) { + // example: RegularChildrenOutputs is empty + // "select 1 a, 2 b union all select 3, 4 union all select 10 e, 20 f;" + for (int i = 0; i < setOperation.getOutput().size(); i++) { + Pair childSlotPair = ctx.getAliasTransferPair(setOperation.getRegularChildOutput(0).get(0)); + if (childSlotPair != null) { + ctx.aliasTransferMapPut(setOperation.getOutput().get(i), childSlotPair); + } + } + } + return setOperation; + } + // runtime filter build side ndv private long getBuildSideNdv(AbstractPhysicalJoin join, ComparisonPredicate compare) { @@ -520,7 +540,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { Slot unwrappedSlot = checkTargetChild(targetExpression); // aliasTransMap doesn't contain the key, means that the path from the scan to the join // contains join with denied join type. for example: a left join b on a.id = b.id - if (!checkPushDownPreconditionsForJoin(rf.getBuilderNode(), ctx, unwrappedSlot)) { + if (!checkProbeSlot(ctx, unwrappedSlot)) { return false; } Slot cteSlot = ctx.getAliasTransferPair(unwrappedSlot).second; @@ -605,17 +625,13 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { } /** - * Check runtime filter push down pre-conditions, such as builder side join type, etc. + * check if slot is in ctx.aliasTransferMap */ - public static boolean checkPushDownPreconditionsForJoin(AbstractPhysicalJoin physicalJoin, - RuntimeFilterContext ctx, Slot slot) { + public static boolean checkProbeSlot(RuntimeFilterContext ctx, Slot slot) { if (slot == null || !ctx.aliasTransferMapContains(slot)) { return false; - } else if (DENIED_JOIN_TYPES.contains(physicalJoin.getJoinType()) || physicalJoin.isMarkJoin()) { - return false; - } else { - return true; } + return true; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java index 513da0e93d..8a4ae73a59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java @@ -21,6 +21,8 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import java.util.Map; + /** * This is the factory for all ExpressionVisitor instance. * All children instance of DefaultExpressionVisitor or ExpressionVisitor for common usage @@ -29,6 +31,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunctio public class ExpressionVisitors { public static final ContainsAggregateChecker CONTAINS_AGGREGATE_CHECKER = new ContainsAggregateChecker(); + public static final ExpressionMapReplacer EXPRESSION_MAP_REPLACER = new ExpressionMapReplacer(); private static class ContainsAggregateChecker extends DefaultExpressionVisitor { @Override @@ -54,4 +57,22 @@ public class ExpressionVisitors { return true; } } + + /** + * replace sub expr by Map + */ + public static class ExpressionMapReplacer + extends DefaultExpressionRewriter> { + + private ExpressionMapReplacer() { + } + + @Override + public Expression visit(Expression expr, Map replaceMap) { + if (replaceMap.containsKey(expr)) { + return replaceMap.get(expr); + } + return super.visit(expr, replaceMap); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java index 3ebf9390d8..26d3601bcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java @@ -88,7 +88,7 @@ public abstract class AbstractPhysicalPlan extends AbstractPlan implements Physi // aliasTransMap doesn't contain the key, means that the path from the scan to the join // contains join with denied join type. for example: a left join b on a.id = b.id - if (!RuntimeFilterGenerator.checkPushDownPreconditionsForJoin(builderNode, ctx, probeSlot)) { + if (!RuntimeFilterGenerator.checkProbeSlot(ctx, probeSlot)) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java index 9d81831d89..2ace7d8586 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java @@ -302,7 +302,7 @@ public class PhysicalHashAggregate extends PhysicalUnar // aliasTransMap doesn't contain the key, means that the path from the scan to the join // contains join with denied join type. for example: a left join b on a.id = b.id - if (!RuntimeFilterGenerator.checkPushDownPreconditionsForJoin(builderNode, ctx, probeSlot)) { + if (!RuntimeFilterGenerator.checkProbeSlot(ctx, probeSlot)) { return false; } PhysicalRelation scan = ctx.getAliasTransferPair(probeSlot).first; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java index 8b4b37d1d3..9d51d6f724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java @@ -208,7 +208,7 @@ public class PhysicalProject extends PhysicalUnary generator, AbstractPhysicalJoin builderNode, Expression src, Expression probeExpr, TRuntimeFilterType type, long buildSideNdv, int exprOrder) { - RuntimeFilterContext ctx = context.getRuntimeFilterContext(); boolean pushedDown = false; + int projIndex = -1; + Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr); + if (probeSlot == null) { + return false; + } + List output = getOutputs(); + for (int j = 0; j < output.size(); j++) { + NamedExpression expr = output.get(j); + if (expr.getName().equals(probeSlot.getName())) { + projIndex = j; + break; + } + } + if (projIndex == -1) { + return false; + } for (int i = 0; i < this.children().size(); i++) { + Map map = Maps.newHashMap(); + // probeExpr only has one input slot + map.put(probeExpr.getInputSlots().iterator().next(), regularChildrenOutputs.get(i).get(projIndex)); + Expression newProbeExpr = probeExpr.accept(ExpressionVisitors.EXPRESSION_MAP_REPLACER, map); AbstractPhysicalPlan child = (AbstractPhysicalPlan) this.child(i); - // TODO: replace this special logic with dynamic handling and the name matching - if (child instanceof PhysicalDistribute) { - child = (AbstractPhysicalPlan) child.child(0); - } - if (child instanceof PhysicalProject) { - PhysicalProject project = (PhysicalProject) child; - int projIndex = -1; - Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr); - if (probeSlot == null) { - continue; - } - for (int j = 0; j < project.getProjects().size(); j++) { - NamedExpression expr = project.getProjects().get(j); - if (expr.getName().equals(probeSlot.getName())) { - projIndex = j; - break; - } - } - if (projIndex < 0 || projIndex >= project.getProjects().size()) { - continue; - } - Expression newProbeExpr = project.getProjects().get(projIndex); - if (newProbeExpr instanceof Alias) { - newProbeExpr = newProbeExpr.child(0); - } - Slot newProbeSlot = RuntimeFilterGenerator.checkTargetChild(newProbeExpr); - if (!RuntimeFilterGenerator.checkPushDownPreconditionsForJoin(builderNode, ctx, newProbeSlot)) { - continue; - } - pushedDown |= child.pushDownRuntimeFilter(context, generator, builderNode, src, + pushedDown |= child.pushDownRuntimeFilter(context, generator, builderNode, src, newProbeExpr, type, buildSideNdv, exprOrder); - } } return pushedDown; } diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query14.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query14.out index 37d24f8831..301f0d5008 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query14.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query14.out @@ -9,9 +9,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = iss.i_item_sk)) otherCondition=() +------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = iss.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[ss_item_sk] --------------------PhysicalProject -----------------------PhysicalOlapScan[store_sales] apply RFs: RF1 +----------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 --------------------PhysicalDistribute[DistributionSpecReplicated] ----------------------PhysicalProject ------------------------PhysicalOlapScan[item] @@ -23,9 +23,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = d2.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = ics.i_item_sk)) otherCondition=() +------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = ics.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[cs_item_sk] --------------------PhysicalProject -----------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 +----------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3 --------------------PhysicalDistribute[DistributionSpecReplicated] ----------------------PhysicalProject ------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out index a357addc10..adb98ff685 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out @@ -55,10 +55,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------hashAgg[LOCAL] ----------------PhysicalUnion ------------------PhysicalProject ---------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk] +--------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk] +----------------------PhysicalProject +------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() +--------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------PhysicalProject @@ -66,18 +66,18 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------PhysicalProject ----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) -----------------------------PhysicalDistribute[DistributionSpecReplicated] -------------------------------PhysicalProject ---------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) -----------------------------------PhysicalOlapScan[date_dim] -----------------------PhysicalDistribute[DistributionSpecHash] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------PhysicalProject +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) +----------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject ---------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ws_sold_date_sk] +--------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ws_sold_date_sk] +----------------------PhysicalProject +------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() +--------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------PhysicalProject @@ -85,11 +85,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------PhysicalProject ----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) -----------------------------PhysicalDistribute[DistributionSpecReplicated] -------------------------------PhysicalProject ---------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) -----------------------------------PhysicalOlapScan[date_dim] -----------------------PhysicalDistribute[DistributionSpecHash] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------PhysicalProject +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) +----------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query25.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query25.out index 7aad2def19..28ac6fa609 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query25.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query25.out @@ -18,7 +18,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN] hashCondition=((d1.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ss_sold_date_sk] ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_customer_sk = catalog_sales.cs_bill_customer_sk) and (store_returns.sr_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF3 cs_bill_customer_sk->[ss_customer_sk,sr_customer_sk];RF4 cs_item_sk->[ss_item_sk,sr_item_sk] +------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_customer_sk = catalog_sales.cs_bill_customer_sk) and (store_returns.sr_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF3 cs_bill_customer_sk->[sr_customer_sk,ss_customer_sk];RF4 cs_item_sk->[sr_item_sk,ss_item_sk] --------------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = store_returns.sr_customer_sk) and (store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query29.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query29.out index 132428dfe8..86f4118708 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query29.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query29.out @@ -18,7 +18,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN] hashCondition=((d1.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ss_sold_date_sk] ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_customer_sk = catalog_sales.cs_bill_customer_sk) and (store_returns.sr_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF3 cs_bill_customer_sk->[ss_customer_sk,sr_customer_sk];RF4 cs_item_sk->[ss_item_sk,sr_item_sk] +------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_customer_sk = catalog_sales.cs_bill_customer_sk) and (store_returns.sr_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF3 cs_bill_customer_sk->[sr_customer_sk,ss_customer_sk];RF4 cs_item_sk->[sr_item_sk,ss_item_sk] --------------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = store_returns.sr_customer_sk) and (store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out index c3aef2e8e0..fb20900a0b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query47.out @@ -49,3 +49,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) + diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out index 8ff4bb6350..18d9a45e7d 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query57.out @@ -49,3 +49,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) + diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query14.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query14.out index 1814cca24d..a4d7cb036a 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query14.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query14.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------hashJoin[INNER_JOIN] hashCondition=((item.i_brand_id = t.brand_id) and (item.i_category_id = t.category_id) and (item.i_class_id = t.class_id)) otherCondition=() build RFs:RF6 i_brand_id->[i_brand_id];RF7 i_class_id->[i_class_id];RF8 i_category_id->[i_category_id] +------hashJoin[INNER_JOIN] hashCondition=((item.i_brand_id = t.brand_id) and (item.i_category_id = t.category_id) and (item.i_class_id = t.class_id)) otherCondition=() build RFs:RF6 i_brand_id->[i_brand_id,i_brand_id,i_brand_id];RF7 i_class_id->[i_class_id,i_class_id,i_class_id];RF8 i_category_id->[i_category_id,i_category_id,i_category_id] --------PhysicalIntersect ----------PhysicalDistribute[DistributionSpecHash] ------------PhysicalProject @@ -28,7 +28,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 RF3 --------------------PhysicalDistribute[DistributionSpecReplicated] ----------------------PhysicalProject -------------------------PhysicalOlapScan[item] +------------------------PhysicalOlapScan[item] apply RFs: RF6 RF7 RF8 ----------------PhysicalDistribute[DistributionSpecReplicated] ------------------PhysicalProject --------------------filter((d2.d_year <= 2002) and (d2.d_year >= 2000)) @@ -42,7 +42,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalOlapScan[web_sales] apply RFs: RF4 RF5 --------------------PhysicalDistribute[DistributionSpecReplicated] ----------------------PhysicalProject -------------------------PhysicalOlapScan[item] +------------------------PhysicalOlapScan[item] apply RFs: RF6 RF7 RF8 ----------------PhysicalDistribute[DistributionSpecReplicated] ------------------PhysicalProject --------------------filter((d3.d_year <= 2002) and (d3.d_year >= 2000)) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out index 4a4d01a429..ee86e70357 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out @@ -55,10 +55,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------hashAgg[LOCAL] ----------------PhysicalUnion ------------------PhysicalProject ---------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk] +--------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk] +----------------------PhysicalProject +------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=() +--------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------PhysicalProject @@ -66,18 +66,18 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------PhysicalProject ----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) -----------------------------PhysicalDistribute[DistributionSpecReplicated] -------------------------------PhysicalProject ---------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) -----------------------------------PhysicalOlapScan[date_dim] -----------------------PhysicalDistribute[DistributionSpecHash] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------PhysicalProject +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) +----------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject ---------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ws_sold_date_sk] +--------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[ws_sold_date_sk] +----------------------PhysicalProject +------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=() +--------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------PhysicalProject @@ -85,11 +85,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------PhysicalProject ----------------------------------PhysicalCteConsumer ( cteId=CTEId#2 ) -----------------------------PhysicalDistribute[DistributionSpecReplicated] -------------------------------PhysicalProject ---------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) -----------------------------------PhysicalOlapScan[date_dim] -----------------------PhysicalDistribute[DistributionSpecHash] +--------------------------PhysicalDistribute[DistributionSpecHash] +----------------------------PhysicalProject +------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------PhysicalDistribute[DistributionSpecReplicated] ------------------------PhysicalProject ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000)) +----------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query25.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query25.out index 7787fcee2c..e5078f3321 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query25.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query25.out @@ -18,7 +18,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN] hashCondition=((d1.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ss_sold_date_sk] ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_customer_sk = catalog_sales.cs_bill_customer_sk) and (store_returns.sr_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF3 cs_bill_customer_sk->[ss_customer_sk,sr_customer_sk];RF4 cs_item_sk->[ss_item_sk,sr_item_sk] +------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_customer_sk = catalog_sales.cs_bill_customer_sk) and (store_returns.sr_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF3 cs_bill_customer_sk->[sr_customer_sk,ss_customer_sk];RF4 cs_item_sk->[sr_item_sk,ss_item_sk] --------------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = store_returns.sr_customer_sk) and (store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF0 sr_customer_sk->[ss_customer_sk];RF1 sr_item_sk->[ss_item_sk];RF2 sr_ticket_number->[ss_ticket_number] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query29.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query29.out index cb96901a76..10b336e57b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query29.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query29.out @@ -18,7 +18,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN] hashCondition=((d1.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ss_sold_date_sk] ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_customer_sk = catalog_sales.cs_bill_customer_sk) and (store_returns.sr_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF3 cs_bill_customer_sk->[ss_customer_sk,sr_customer_sk];RF4 cs_item_sk->[ss_item_sk,sr_item_sk] +------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_customer_sk = catalog_sales.cs_bill_customer_sk) and (store_returns.sr_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF3 cs_bill_customer_sk->[sr_customer_sk,ss_customer_sk];RF4 cs_item_sk->[sr_item_sk,ss_item_sk] --------------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = store_returns.sr_customer_sk) and (store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF0 sr_customer_sk->[ss_customer_sk];RF1 sr_item_sk->[ss_item_sk];RF2 sr_ticket_number->[ss_ticket_number] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out index dbc3759558..59d526865c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query47.out @@ -49,3 +49,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 2001)) --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) + diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out index 40ecf4c154..52cd80d56c 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query57.out @@ -49,3 +49,4 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalProject ------------------------filter((if((avg_monthly_sales > 0.0000), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000) and (v2.avg_monthly_sales > 0.0000) and (v2.d_year = 1999)) --------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +