database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[pick](nereids)Runtime filter pushdown refactor for branch-2.1 (#34682)

Browse Source 
* [refactor](Nereids)refactor runtime filter generator (#34275)

1. unify the process of generating rf for hash join and for nested loop join
2. fix some bugs in generating rf
3. remove some duplicated check

(cherry picked from commit 07267faac0d9c6ef3bb1fd4ee101b4c761c8a2f2)

* [refactor](nereids) do not deny a runtime filter by removing an entry in aliasMap (#34559)

in current version, there are 2 approaches to verify whether a join condition can be used to generate a runtime filter, they are
1. remove the output slot from aliasMap
2. pushDownVisitor.visit(...) return false
the 1st approach has some drawbacks, we prefer to the 2ed approach.
In this pr, all the cases are handled by the 2ed approach, and remove the related code for the 1st approach.

(cherry picked from commit a29082bf31e66efa2df193b38347e610f2bf7464)

* rebase
This commit is contained in:
minghong
2024-05-11 09:44:24 +08:00
committed by GitHub
parent e38801968d
commit dd1b54cf62
21 changed files with 865 additions and 869 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/clucene

Submodule be/src/clucene updated: d3de160871...9f849a47f7

2
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -1094,7 +1094,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
}
@Override
public PlanFragment visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer,
public PlanFragment visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer,
PlanTranslatorContext context) {
CTEId cteId = cteConsumer.getCteId();

25
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java
View File

@ -45,7 +45,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@ -131,6 +130,7 @@ public class RuntimeFilterContext {
private int targetNullCount = 0;
private final Map<Plan, Set<PhysicalRelation>> relationsUsedByPlan = Maps.newHashMap();
private final List<ExpandRF> expandedRF = Lists.newArrayList();
/**
@ -160,6 +160,23 @@ public class RuntimeFilterContext {
this.limits = new FilterSizeLimits(sessionVariable);
}
public void setRelationsUsedByPlan(Plan plan, Set<PhysicalRelation> relations) {
relationsUsedByPlan.put(plan, relations);
}
/**
* return true, if the relation is in the subtree
*/
public boolean isRelationUseByPlan(Plan plan, PhysicalRelation relation) {
Set<PhysicalRelation> relations = relationsUsedByPlan.get(plan);
if (relations == null) {
relations = Sets.newHashSet();
RuntimeFilterGenerator.getAllScanInfo(plan, relations);
relationsUsedByPlan.put(plan, relations);
}
return relations.contains(relation);
}
public SessionVariable getSessionVariable() {
return sessionVariable;
}
@ -273,10 +290,6 @@ public class RuntimeFilterContext {
return aliasTransferMap;
}
public Pair<PhysicalRelation, Slot> aliasTransferMapRemove(NamedExpression slot) {
return aliasTransferMap.remove(slot);
}
public Pair<PhysicalRelation, Slot> getAliasTransferPair(NamedExpression slot) {
return aliasTransferMap.get(slot);
}
@ -354,7 +367,7 @@ public class RuntimeFilterContext {
}
public List<ExprId> getTargetExprIdByFilterJoin(AbstractPhysicalJoin join) {
return joinToTargetExprId.getOrDefault(join, new ArrayList<>());
return joinToTargetExprId.getOrDefault(join, Lists.newArrayList());
}
public SlotReference getCorrespondingOlapSlotReference(SlotReference slot) {

107
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
View File

@ -52,8 +52,6 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow;
import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.JoinUtils;
@ -63,7 +61,6 @@ import org.apache.doris.thrift.TMinMaxRuntimeFilterType;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -259,8 +256,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
join.right().accept(this, context);
join.left().accept(this, context);
if (RuntimeFilterGenerator.DENIED_JOIN_TYPES.contains(join.getJoinType()) || join.isMarkJoin()) {
join.right().getOutput().forEach(slot ->
context.getRuntimeFilterContext().aliasTransferMapRemove(slot));
// do not generate RF on this join
return join;
}
RuntimeFilterContext ctx = context.getRuntimeFilterContext();
List<TRuntimeFilterType> legalTypes = Arrays.stream(TRuntimeFilterType.values())
@ -285,8 +282,16 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
continue;
}
if (equalTo.left().getInputSlots().size() == 1) {
join.pushDownRuntimeFilter(context, generator, join, equalTo.right(),
equalTo.left(), type, buildSideNdv, i);
RuntimeFilterPushDownVisitor.PushDownContext pushDownContext =
RuntimeFilterPushDownVisitor.PushDownContext.createPushDownContextForHashJoin(
equalTo.right(), equalTo.left(), ctx, generator, type, join,
context.getStatementContext().isHasUnknownColStats(), buildSideNdv, i);
// pushDownContext is not valid, if the target is an agg result.
// Currently, we only apply RF on PhysicalScan. So skip this rf.
// example: (select sum(x) as s from A) T join B on T.s=B.s
if (pushDownContext.isValid()) {
join.accept(new RuntimeFilterPushDownVisitor(), pushDownContext);
}
}
}
}
@ -301,7 +306,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
}
@Override
public PhysicalCTEProducer visitPhysicalCTEProducer(PhysicalCTEProducer producer, CascadesContext context) {
public PhysicalCTEProducer<? extends Plan> visitPhysicalCTEProducer(PhysicalCTEProducer<? extends Plan> producer,
CascadesContext context) {
CTEId cteId = producer.getCteId();
context.getRuntimeFilterContext().getCteProduceMap().put(cteId, producer);
Set<CTEId> processedCTE = context.getRuntimeFilterContext().getProcessedCTE();
@ -336,24 +342,12 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
} else {
bitmapContains = (BitmapContains) bitmapRuntimeFilterCondition;
}
TRuntimeFilterType type = TRuntimeFilterType.BITMAP;
Set<Slot> targetSlots = bitmapContains.child(1).getInputSlots();
for (Slot targetSlot : targetSlots) {
if (!checkProbeSlot(ctx, targetSlot)) {
continue;
}
Slot scanSlot = ctx.getAliasTransferPair(targetSlot).second;
PhysicalRelation scan = ctx.getAliasTransferPair(targetSlot).first;
RuntimeFilter filter = new RuntimeFilter(generator.getNextId(),
bitmapContains.child(0), ImmutableList.of(scanSlot),
ImmutableList.of(bitmapContains.child(1)), type, i, join, isNot, -1L,
false, scan);
scan.addAppliedRuntimeFilter(filter);
ctx.addJoinToTargetMap(join, scanSlot.getExprId());
ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter);
ctx.setTargetsOnScanNode(ctx.getAliasTransferPair(targetSlot).first,
scanSlot);
join.addBitmapRuntimeFilterCondition(bitmapRuntimeFilterCondition);
RuntimeFilterPushDownVisitor.PushDownContext pushDownContext =
RuntimeFilterPushDownVisitor.PushDownContext.createPushDownContextForBitMapFilter(
bitmapContains.child(0), bitmapContains.child(1), ctx, generator, join,
-1, i, isNot);
if (pushDownContext.isValid()) {
join.accept(new RuntimeFilterPushDownVisitor(), pushDownContext);
}
}
}
@ -423,15 +417,13 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
Slot olapScanSlot = pair.second;
PhysicalRelation scan = pair.first;
Preconditions.checkState(olapScanSlot != null && scan != null);
long buildSideNdv = getBuildSideNdv(join, compare);
RuntimeFilter filter = new RuntimeFilter(generator.getNextId(),
compare.child(1), ImmutableList.of(olapScanSlot), ImmutableList.of(olapScanSlot),
TRuntimeFilterType.MIN_MAX, exprOrder, join, true, buildSideNdv, false,
getMinMaxType(compare), scan);
scan.addAppliedRuntimeFilter(filter);
ctx.addJoinToTargetMap(join, olapScanSlot.getExprId());
ctx.setTargetExprIdToFilter(olapScanSlot.getExprId(), filter);
ctx.setTargetsOnScanNode(scan, olapScanSlot);
RuntimeFilterPushDownVisitor.PushDownContext pushDownContext =
RuntimeFilterPushDownVisitor.PushDownContext.createPushDownContextForNljMinMaxFilter(
compare.child(1), compare.child(0), ctx, generator, join,
exprOrder, getMinMaxType(compare));
if (pushDownContext.isValid()) {
join.accept(new RuntimeFilterPushDownVisitor(), pushDownContext);
}
}
}
}
@ -442,10 +434,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
// TODO: we need to support all type join
join.right().accept(this, context);
join.left().accept(this, context);
if (RuntimeFilterGenerator.DENIED_JOIN_TYPES.contains(join.getJoinType()) || join.isMarkJoin()) {
join.right().getOutput().forEach(slot ->
context.getRuntimeFilterContext().aliasTransferMapRemove(slot));
// do not generate RF on this join
return join;
}
RuntimeFilterContext ctx = context.getRuntimeFilterContext();
@ -555,7 +545,9 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
break;
}
}
Preconditions.checkState(targetExpr != null);
Preconditions.checkState(targetExpr != null,
"cannot find runtime filter cte.target: "
+ cteSlot + "in project " + project.toString());
if (targetExpr instanceof SlotReference && checkCanPushDownIntoBasicTable(project)) {
Map<Slot, PhysicalRelation> pushDownBasicTableInfos = getPushDownBasicTablesInfos(project,
(SlotReference) targetExpr, ctx);
@ -594,37 +586,6 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
return false;
}
@Override
public PhysicalPlan visitPhysicalTopN(PhysicalTopN<? extends Plan> topN, CascadesContext context) {
topN.child().accept(this, context);
PhysicalPlan child = (PhysicalPlan) topN.child();
for (Slot slot : child.getOutput()) {
context.getRuntimeFilterContext().aliasTransferMapRemove(slot);
}
return topN;
}
@Override
public PhysicalPlan visitPhysicalWindow(PhysicalWindow<? extends Plan> window, CascadesContext context) {
window.child().accept(this, context);
Set<SlotReference> commonPartitionKeys = window.getCommonPartitionKeyFromWindowExpressions();
window.child().getOutput().stream().filter(slot -> !commonPartitionKeys.contains(slot)).forEach(
slot -> context.getRuntimeFilterContext().aliasTransferMapRemove(slot)
);
return window;
}
/**
* Check runtime filter push down project/distribute pre-conditions.
*/
public static boolean checkPushDownPreconditionsForProjectOrDistribute(RuntimeFilterContext ctx, Slot slot) {
if (slot == null || !ctx.aliasTransferMapContains(slot)) {
return false;
} else {
return true;
}
}
/**
* check if slot is in ctx.aliasTransferMap
*/
@ -703,12 +664,12 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
/**
* Get all relation node from current root plan.
*/
public static void getAllScanInfo(PhysicalPlan root, Set<PhysicalRelation> scans) {
public static void getAllScanInfo(Plan root, Set<PhysicalRelation> scans) {
if (root instanceof PhysicalRelation) {
scans.add((PhysicalRelation) root);
} else {
for (Object child : root.children()) {
getAllScanInfo((PhysicalPlan) child, scans);
for (Plan child : root.children()) {
getAllScanInfo(child, scans);
}
}
}

16
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruner.java
View File

@ -94,7 +94,7 @@ public class RuntimeFilterPruner extends PlanPostProcessor {
}
@Override
public PhysicalNestedLoopJoin visitPhysicalNestedLoopJoin(
public PhysicalNestedLoopJoin<? extends Plan, ? extends Plan> visitPhysicalNestedLoopJoin(
PhysicalNestedLoopJoin<? extends Plan, ? extends Plan> join,
CascadesContext context) {
join.right().accept(this, context);
@ -108,28 +108,32 @@ public class RuntimeFilterPruner extends PlanPostProcessor {
}
@Override
public PhysicalCTEAnchor visitPhysicalCTEAnchor(PhysicalCTEAnchor<? extends Plan, ? extends Plan> cteAnchor,
CascadesContext context) {
public PhysicalCTEAnchor<? extends Plan, ? extends Plan> visitPhysicalCTEAnchor(
PhysicalCTEAnchor<? extends Plan, ? extends Plan> cteAnchor,
CascadesContext context) {
cteAnchor.child(0).accept(this, context);
cteAnchor.child(1).accept(this, context);
return cteAnchor;
}
@Override
public PhysicalTopN visitPhysicalTopN(PhysicalTopN<? extends Plan> topN, CascadesContext context) {
public PhysicalTopN<? extends Plan> visitPhysicalTopN(PhysicalTopN<? extends Plan> topN, CascadesContext context) {
topN.child().accept(this, context);
context.getRuntimeFilterContext().addEffectiveSrcNode(topN, RuntimeFilterContext.EffectiveSrcType.NATIVE);
return topN;
}
public PhysicalLimit visitPhysicalLimit(PhysicalLimit<? extends Plan> limit, CascadesContext context) {
public PhysicalLimit<? extends Plan> visitPhysicalLimit(
PhysicalLimit<? extends Plan> limit,
CascadesContext context) {
limit.child().accept(this, context);
context.getRuntimeFilterContext().addEffectiveSrcNode(limit, RuntimeFilterContext.EffectiveSrcType.NATIVE);
return limit;
}
@Override
public PhysicalHashJoin visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ? extends Plan> join,
public PhysicalHashJoin<? extends Plan, ? extends Plan> visitPhysicalHashJoin(
PhysicalHashJoin<? extends Plan, ? extends Plan> join,
CascadesContext context) {
join.right().accept(this, context);
RuntimeFilterContext rfContext = context.getRuntimeFilterContext();

424
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java Normal file
View File

@ -0,0 +1,424 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.processor.post;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.processor.post.RuntimeFilterPushDownVisitor.PushDownContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow;
import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.types.coercion.NumericType;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TMinMaxRuntimeFilterType;
import org.apache.doris.thrift.TRuntimeFilterType;
import cfjd.com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* push down rf
*/
public class RuntimeFilterPushDownVisitor extends PlanVisitor<Boolean, PushDownContext> {
// the context to push down rf by join condition: probeExpr = srcExpr
/**
* PushDownContext
*/
public static class PushDownContext {
final Expression srcExpr;
final Expression probeExpr;
final Slot probeSlot;
final RuntimeFilterContext rfContext;
final IdGenerator<RuntimeFilterId> rfIdGen;
final TRuntimeFilterType type;
final AbstractPhysicalJoin<? extends Plan, ? extends Plan> builderNode;
final boolean hasUnknownColStats;
final long buildSideNdv;
final int exprOrder;
final Pair<PhysicalRelation, Slot> finalTarget;
//bitmap rf used only
final boolean isNot;
// only used for Min_Max runtime filter
final TMinMaxRuntimeFilterType singleSideMinMax;
/**
* push down context
*/
// for hash join runtime filter
private PushDownContext(Expression srcExpr, Expression probeExpr, RuntimeFilterContext rfContext,
IdGenerator<RuntimeFilterId> rfIdGen, TRuntimeFilterType type,
AbstractPhysicalJoin<? extends Plan, ? extends Plan> builderNode,
boolean hasUnknownColStats, long buildSideNdv, int exprOrder, boolean isNot,
TMinMaxRuntimeFilterType singleSideMinMax) {
this.probeExpr = probeExpr;
this.rfContext = rfContext;
this.srcExpr = srcExpr;
this.rfIdGen = rfIdGen;
this.type = type;
this.builderNode = builderNode;
this.hasUnknownColStats = hasUnknownColStats;
this.buildSideNdv = buildSideNdv;
this.exprOrder = exprOrder;
this.isNot = isNot;
Expression expr = getSingleNumericSlotOrExpressionCoveredByCast(probeExpr);
if (expr instanceof Slot) {
probeSlot = (Slot) expr;
finalTarget = rfContext.getAliasTransferPair((Slot) probeSlot);
} else {
finalTarget = null;
probeSlot = null;
}
this.singleSideMinMax = singleSideMinMax;
}
// for BitMap runtime filter
public static PushDownContext createPushDownContextForBitMapFilter(Expression srcExpr, Expression probeExpr,
RuntimeFilterContext rfContext,
IdGenerator<RuntimeFilterId> rfIdGen,
AbstractPhysicalJoin<? extends Plan, ? extends Plan> builderNode,
long buildSideNdv, int exprOrder, boolean isNot) {
return new PushDownContext(srcExpr, probeExpr, rfContext, rfIdGen, TRuntimeFilterType.BITMAP, builderNode,
false, buildSideNdv,
exprOrder, isNot, TMinMaxRuntimeFilterType.MIN_MAX);
}
// for NLJ min-max runtime filter
public static PushDownContext createPushDownContextForNljMinMaxFilter(Expression srcExpr, Expression probeExpr,
RuntimeFilterContext rfContext,
IdGenerator<RuntimeFilterId> rfIdGen,
AbstractPhysicalJoin<? extends Plan, ? extends Plan> builderNode,
int exprOrder,
TMinMaxRuntimeFilterType singleSideMinMax) {
return new PushDownContext(srcExpr, probeExpr, rfContext, rfIdGen, TRuntimeFilterType.MIN_MAX, builderNode,
false, -1,
exprOrder, false, singleSideMinMax);
}
public static PushDownContext createPushDownContextForHashJoin(Expression srcExpr, Expression probeExpr,
RuntimeFilterContext rfContext,
IdGenerator<RuntimeFilterId> rfIdGen, TRuntimeFilterType type,
AbstractPhysicalJoin<? extends Plan, ? extends Plan> builderNode,
boolean hasUnknownColStats, long buildSideNdv, int exprOrder) {
return new PushDownContext(srcExpr, probeExpr, rfContext, rfIdGen, type, builderNode,
hasUnknownColStats, buildSideNdv,
exprOrder, false, TMinMaxRuntimeFilterType.MIN_MAX);
}
public boolean isValid() {
return finalTarget != null && probeSlot != null;
}
public PushDownContext withNewProbeExpression(Expression newProbe) {
return new PushDownContext(srcExpr, newProbe, this.rfContext, rfIdGen, type, builderNode,
hasUnknownColStats, buildSideNdv, exprOrder, isNot, singleSideMinMax);
}
private Expression getSingleNumericSlotOrExpressionCoveredByCast(Expression expression) {
if (expression.getInputSlots().size() == 1) {
Slot slot = expression.getInputSlots().iterator().next();
if (slot.getDataType() instanceof NumericType) {
return expression.getInputSlots().iterator().next();
}
}
// for other datatype, only support cast.
// example: T1 join T2 on subStr(T1.a, 1,4) = subStr(T2.a, 1,4)
// the cost of subStr is too high, and hence we do not generate RF subStr(T2.a, 1,4)->subStr(T1.a, 1,4)
while (expression instanceof Cast) {
expression = ((Cast) expression).child();
}
return expression;
}
}
@Override
public Boolean visit(Plan plan, PushDownContext ctx) {
boolean pushed = false;
for (Plan child : plan.children()) {
pushed |= child.accept(this, ctx);
}
return pushed;
}
@Override
public Boolean visitPhysicalRelation(PhysicalRelation scan, PushDownContext ctx) {
Preconditions.checkArgument(ctx.isValid(),
"runtime filter pushDownContext is invalid");
PhysicalRelation relation = ctx.finalTarget.first;
Slot scanSlot = ctx.finalTarget.second;
Slot probeSlot = ctx.probeSlot;
if (!relation.equals(scan)) {
return Boolean.FALSE;
}
TRuntimeFilterType type = ctx.type;
if (type == TRuntimeFilterType.IN_OR_BLOOM
&& ctx.rfContext.getSessionVariable().getEnablePipelineEngine()
&& RuntimeFilterGenerator.hasRemoteTarget(ctx.builderNode, scan)
&& !ctx.builderNode.isBroadCastJoin()) {
type = TRuntimeFilterType.BLOOM;
}
RuntimeFilter filter = ctx.rfContext.getRuntimeFilterBySrcAndType(ctx.srcExpr, type, ctx.builderNode);
if (filter != null) {
if (!filter.hasTargetScan(scan)) {
// A join B on A.a1=B.b and A.a1 = A.a2
// RF B.b->(A.a1, A.a2)
// however, RF(B.b->A.a2) is implied by RF(B.a->A.a1) and A.a1=A.a2
// we skip RF(B.b->A.a2)
scan.addAppliedRuntimeFilter(filter);
filter.addTargetSlot(scanSlot, ctx.probeExpr, scan);
ctx.rfContext.addJoinToTargetMap(ctx.builderNode, scanSlot.getExprId());
ctx.rfContext.setTargetExprIdToFilter(scanSlot.getExprId(), filter);
ctx.rfContext.setTargetsOnScanNode(ctx.rfContext.getAliasTransferPair(probeSlot).first, scanSlot);
}
} else {
filter = new RuntimeFilter(ctx.rfIdGen.getNextId(),
ctx.srcExpr, ImmutableList.of(scanSlot), ImmutableList.of(ctx.probeExpr),
type, ctx.exprOrder, ctx.builderNode, ctx.isNot, ctx.buildSideNdv,
!ctx.hasUnknownColStats, ctx.singleSideMinMax, scan);
scan.addAppliedRuntimeFilter(filter);
ctx.rfContext.addJoinToTargetMap(ctx.builderNode, scanSlot.getExprId());
ctx.rfContext.setTargetExprIdToFilter(scanSlot.getExprId(), filter);
ctx.rfContext.setTargetsOnScanNode(ctx.rfContext.getAliasTransferPair(probeSlot).first, scanSlot);
ctx.rfContext.setRuntimeFilterIdentityToFilter(ctx.srcExpr, type, ctx.builderNode, filter);
}
return true;
}
@Override
public Boolean visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ? extends Plan> join,
PushDownContext ctx) {
boolean pushed = false;
if (ctx.builderNode instanceof PhysicalHashJoin) {
/*
hashJoin( t1.A <=> t2.A )
+---->left outer Join(t1.B=T3.B)
+--->t1
+--->t3
+---->t2
RF(t1.A <=> t2.A) cannot be pushed down through left outer join
*/
EqualPredicate equal = (EqualPredicate) ctx.builderNode.getHashJoinConjuncts().get(ctx.exprOrder);
if (equal instanceof NullSafeEqual) {
if (join.getJoinType().isOuterJoin()) {
return false;
}
}
}
AbstractPhysicalPlan leftNode = (AbstractPhysicalPlan) join.child(0);
AbstractPhysicalPlan rightNode = (AbstractPhysicalPlan) join.child(1);
Set<Expression> probExprList = Sets.newLinkedHashSet();
probExprList.add(ctx.probeExpr);
Pair<PhysicalRelation, Slot> srcPair = ctx.rfContext.getAliasTransferMap().get(ctx.srcExpr);
PhysicalRelation srcNode = (srcPair == null) ? null : srcPair.first;
Pair<PhysicalRelation, Slot> targetPair = ctx.rfContext.getAliasTransferMap().get(ctx.probeExpr);
if (targetPair == null) {
/* cases for "targetPair is null"
when probeExpr is output slot of setOperator, targetPair is null
*/
return false;
}
PhysicalRelation target1 = targetPair.first;
PhysicalRelation target2 = null;
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().expandRuntimeFilterByInnerJoin) {
if (!join.equals(ctx.builderNode)
&& (join.getJoinType() == JoinType.INNER_JOIN || join.getJoinType().isSemiJoin())) {
for (Expression expr : join.getHashJoinConjuncts()) {
EqualPredicate equalTo = (EqualPredicate) expr;
if (ctx.probeExpr.equals(equalTo.left())) {
probExprList.add(equalTo.right());
targetPair = ctx.rfContext.getAliasTransferMap().get(equalTo.right());
target2 = (targetPair == null) ? null : targetPair.first;
} else if (ctx.probeExpr.equals(equalTo.right())) {
probExprList.add(equalTo.left());
targetPair = ctx.rfContext.getAliasTransferMap().get(equalTo.left());
target2 = (targetPair == null) ? null : targetPair.first;
}
if (target2 != null) {
ctx.rfContext.getExpandedRF().add(
new RuntimeFilterContext.ExpandRF(join, srcNode, target1, target2, equalTo));
}
}
probExprList.remove(ctx.srcExpr);
}
}
for (Expression prob : probExprList) {
PushDownContext ctxForChild = prob.equals(ctx.probeExpr) ? ctx : ctx.withNewProbeExpression(prob);
if (ctx.rfContext.isRelationUseByPlan(leftNode, ctxForChild.finalTarget.first)) {
pushed |= leftNode.accept(this, ctxForChild);
}
if (ctx.rfContext.isRelationUseByPlan(rightNode, ctxForChild.finalTarget.first)) {
pushed |= rightNode.accept(this, ctxForChild);
}
}
return pushed;
}
@Override
public Boolean visitPhysicalNestedLoopJoin(PhysicalNestedLoopJoin<? extends Plan, ? extends Plan> join,
PushDownContext ctx) {
if (ctx.builderNode instanceof PhysicalHashJoin) {
/*
hashJoin( t1.A <=> t2.A )
+---->left outer Join(t1.B=T3.B)
+--->t1
+--->t3
+---->t2
RF(t1.A <=> t2.A) cannot be pushed down through left outer join
*/
EqualPredicate equal = (EqualPredicate) ctx.builderNode.getHashJoinConjuncts().get(ctx.exprOrder);
if (equal instanceof NullSafeEqual) {
if (join.getJoinType().isOuterJoin()) {
return false;
}
}
}
boolean pushed = false;
if (ctx.rfContext.isRelationUseByPlan(join.left(), ctx.finalTarget.first)) {
pushed |= join.left().accept(this, ctx);
}
if (ctx.rfContext.isRelationUseByPlan(join.right(), ctx.finalTarget.first)) {
pushed |= join.right().accept(this, ctx);
}
return pushed;
}
@Override
public Boolean visitPhysicalProject(PhysicalProject<? extends Plan> project, PushDownContext ctx) {
// project ( A+1 as x)
// probeExpr: abs(x) => abs(A+1)
PushDownContext ctxProjectProbeExpr = ctx;
if (ctx.probeExpr instanceof SlotReference) {
for (NamedExpression namedExpression : project.getProjects()) {
if (namedExpression instanceof Alias
&& namedExpression.getExprId() == ((SlotReference) ctx.probeExpr).getExprId()) {
if (((Alias) namedExpression).child().getInputSlots().size() > 1) {
// only support one-slot probeExpr
return false;
}
ctxProjectProbeExpr = ctx.withNewProbeExpression(((Alias) namedExpression).child());
break;
}
}
} else {
for (NamedExpression namedExpression : project.getProjects()) {
if (namedExpression instanceof Alias
&& ctx.probeExpr.getInputSlots().contains(namedExpression.toSlot())) {
if (((Alias) namedExpression).child().getInputSlots().size() > 1) {
// only support one-slot probeExpr
return false;
}
Map<Expression, Expression> map = Maps.newHashMap();
// probeExpr only has one input slot
map.put(ctx.probeExpr.getInputSlots().iterator().next(),
((Alias) namedExpression).child());
Expression newProbeExpr = ctx.probeExpr.accept(ExpressionVisitors.EXPRESSION_MAP_REPLACER, map);
ctxProjectProbeExpr = ctx.withNewProbeExpression(newProbeExpr);
break;
}
}
}
return project.child().accept(this, ctxProjectProbeExpr);
}
@Override
public Boolean visitPhysicalSetOperation(PhysicalSetOperation setOperation, PushDownContext ctx) {
boolean pushedDown = false;
int projIndex = -1;
Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(ctx.probeExpr);
if (probeSlot == null) {
return false;
}
List<NamedExpression> output = setOperation.getOutputs();
for (int j = 0; j < output.size(); j++) {
NamedExpression expr = output.get(j);
if (expr.getName().equals(probeSlot.getName())) {
projIndex = j;
break;
}
}
if (projIndex == -1) {
return false;
}
for (int i = 0; i < setOperation.children().size(); i++) {
Map<Expression, Expression> map = Maps.newHashMap();
// probeExpr only has one input slot
map.put(ctx.probeExpr.getInputSlots().iterator().next(),
setOperation.getRegularChildrenOutputs().get(i).get(projIndex));
Expression newProbeExpr = ctx.probeExpr.accept(ExpressionVisitors.EXPRESSION_MAP_REPLACER, map);
PushDownContext childPushDownContext = ctx.withNewProbeExpression(newProbeExpr);
if (childPushDownContext.isValid()) {
/*
* childPushDownContext is not valid, for example:
* setop
* +--->scan t1(A, B)
* +--->select 0, 0
* push down context for "select 0, 0" is invalid
*/
pushedDown |= setOperation.child(i).accept(this, ctx.withNewProbeExpression(newProbeExpr));
}
}
return pushedDown;
}
@Override
public Boolean visitPhysicalTopN(PhysicalTopN<? extends Plan> topN, PushDownContext ctx) {
return false;
}
@Override
public Boolean visitPhysicalWindow(PhysicalWindow<? extends Plan> window, PushDownContext ctx) {
Set<SlotReference> commonPartitionKeys = window.getCommonPartitionKeyFromWindowExpressions();
if (commonPartitionKeys.containsAll(ctx.probeExpr.getInputSlots())) {
return window.child().accept(this, ctx);
}
return false;
}
}

79
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java
View File

@ -17,26 +17,17 @@
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.processor.post.RuntimeFilterContext;
import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.AbstractPlan;
import org.apache.doris.nereids.trees.plans.Explainable;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.util.MutableState;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@ -74,76 +65,6 @@ public abstract class AbstractPhysicalPlan extends AbstractPlan implements Physi
return physicalProperties;
}
/**
* Pushing down runtime filter into different plan node, such as olap scan node, cte sender node, etc.
*/
public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<RuntimeFilterId> generator,
AbstractPhysicalJoin<?, ?> builderNode,
Expression src, Expression probeExpr,
TRuntimeFilterType type, long buildSideNdv, int exprOrder) {
RuntimeFilterContext ctx = context.getRuntimeFilterContext();
// currently, we can ensure children in the two side are corresponding to the equal_to's.
// so right maybe an expression and left is a slot
Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr);
// aliasTransMap doesn't contain the key, means that the path from the scan to the join
// contains join with denied join type. for example: a left join b on a.id = b.id
if (!RuntimeFilterGenerator.checkProbeSlot(ctx, probeSlot)) {
return false;
}
boolean pushedDown = false;
for (Object child : children) {
AbstractPhysicalPlan childPlan = (AbstractPhysicalPlan) child;
pushedDown |= childPlan.pushDownRuntimeFilter(context, generator, builderNode, src, probeExpr,
type, buildSideNdv, exprOrder);
}
if (pushedDown) {
return true;
}
Slot scanSlot = ctx.getAliasTransferPair(probeSlot).second;
PhysicalRelation scan = ctx.getAliasTransferPair(probeSlot).first;
if (!RuntimeFilterGenerator.checkPushDownPreconditionsForRelation(this, scan)) {
return false;
}
// in-filter is not friendly to pipeline
if (type == TRuntimeFilterType.IN_OR_BLOOM
&& ctx.getSessionVariable().getEnablePipelineEngine()
&& RuntimeFilterGenerator.hasRemoteTarget(builderNode, scan)
&& !builderNode.isBroadCastJoin()) {
type = TRuntimeFilterType.BLOOM;
}
org.apache.doris.nereids.trees.plans.physical.RuntimeFilter filter =
ctx.getRuntimeFilterBySrcAndType(src, type, builderNode);
Preconditions.checkState(scanSlot != null, "scan slot is null");
if (filter != null) {
if (!filter.hasTargetScan(scan)) {
// A join B on A.a1=B.b and A.a1 = A.a2
// RF B.b->(A.a1, A.a2)
// however, RF(B.b->A.a2) is implied by RF(B.a->A.a1) and A.a1=A.a2
// we skip RF(B.b->A.a2)
this.addAppliedRuntimeFilter(filter);
filter.addTargetSlot(scanSlot, probeExpr, scan);
ctx.addJoinToTargetMap(builderNode, scanSlot.getExprId());
ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter);
ctx.setTargetsOnScanNode(ctx.getAliasTransferPair(probeSlot).first, scanSlot);
}
} else {
filter = new RuntimeFilter(generator.getNextId(),
src, ImmutableList.of(scanSlot), ImmutableList.of(probeExpr),
type, exprOrder, builderNode, buildSideNdv,
!context.getStatementContext().isHasUnknownColStats(), scan);
this.addAppliedRuntimeFilter(filter);
ctx.addJoinToTargetMap(builderNode, scanSlot.getExprId());
ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter);
ctx.setTargetsOnScanNode(ctx.getAliasTransferPair(probeSlot).first, scanSlot);
ctx.setRuntimeFilterIdentityToFilter(src, type, builderNode, filter);
}
return true;
}
@Override
public Plan getExplainPlan(ConnectContext ctx) {
return this;

16
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalCTEConsumer.java
View File

@ -17,22 +17,17 @@
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.CTEId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
@ -152,17 +147,6 @@ public class PhysicalCTEConsumer extends PhysicalRelation {
return shapeBuilder.toString();
}
@Override
public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<RuntimeFilterId> generator,
AbstractPhysicalJoin<?, ?> builderNode,
Expression src, Expression probeExpr,
TRuntimeFilterType type, long buildSideNdv, int exprOrder) {
// push down rf on cte sender
// TODO: refactor pushing down into cte internal here
return super.pushDownRuntimeFilter(context, generator, builderNode,
src, probeExpr, type, buildSideNdv, exprOrder);
}
@Override
public boolean canPushDownRuntimeFilter() {
return true;

36
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java
View File

@ -17,11 +17,7 @@
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.processor.post.RuntimeFilterContext;
import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator;
import org.apache.doris.nereids.properties.DistributionSpec;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
@ -31,9 +27,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@ -126,36 +120,6 @@ public class PhysicalDistribute<CHILD_TYPE extends Plan> extends PhysicalUnary<C
getLogicalProperties(), physicalProperties, statistics, child());
}
@Override
public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<RuntimeFilterId> generator,
AbstractPhysicalJoin<?, ?> builderNode, Expression src, Expression probeExpr,
TRuntimeFilterType type, long buildSideNdv, int exprOrder) {
RuntimeFilterContext ctx = context.getRuntimeFilterContext();
// currently, we can ensure children in the two side are corresponding to the equal_to's.
// so right maybe an expression and left is a slot
Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr);
if (probeSlot == null) {
return false;
}
if (RuntimeFilterGenerator.checkPushDownPreconditionsForProjectOrDistribute(ctx, probeSlot)) {
PhysicalRelation scan = ctx.getAliasTransferPair(probeSlot).first;
if (!RuntimeFilterGenerator.checkPushDownPreconditionsForRelation(this, scan)) {
return false;
}
// TODO: global rf need merge stage which is heavy
// add some rule, such as bc only is allowed for
// pushing down through distribute, currently always pushing.
AbstractPhysicalPlan childPlan = (AbstractPhysicalPlan) child(0);
return childPlan.pushDownRuntimeFilter(context, generator, builderNode, src, probeExpr,
type, buildSideNdv, exprOrder);
} else {
// if probe slot doesn't exist in aliasTransferMap, then try to pass it to child
AbstractPhysicalPlan childPlan = (AbstractPhysicalPlan) child(0);
return childPlan.pushDownRuntimeFilter(context, generator, builderNode, src, probeExpr,
type, buildSideNdv, exprOrder);
}
}
@Override
public List<Slot> computeOutput() {
return child().getOutput();

30
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashAggregate.java
View File

@ -17,11 +17,7 @@
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.processor.post.RuntimeFilterContext;
import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.properties.RequireProperties;
@ -37,9 +33,7 @@ import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@ -291,30 +285,6 @@ public class PhysicalHashAggregate<CHILD_TYPE extends Plan> extends PhysicalUnar
return builder.toString();
}
@Override
public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<RuntimeFilterId> generator,
AbstractPhysicalJoin<?, ?> builderNode, Expression src, Expression probeExpr,
TRuntimeFilterType type, long buildSideNdv, int exprOrder) {
RuntimeFilterContext ctx = context.getRuntimeFilterContext();
// currently, we can ensure children in the two side are corresponding to the equal_to's.
// so right maybe an expression and left is a slot
Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr);
// aliasTransMap doesn't contain the key, means that the path from the scan to the join
// contains join with denied join type. for example: a left join b on a.id = b.id
if (!RuntimeFilterGenerator.checkProbeSlot(ctx, probeSlot)) {
return false;
}
PhysicalRelation scan = ctx.getAliasTransferPair(probeSlot).first;
if (!RuntimeFilterGenerator.checkPushDownPreconditionsForRelation(this, scan)) {
return false;
}
AbstractPhysicalPlan child = (AbstractPhysicalPlan) child(0);
return child.pushDownRuntimeFilter(context, generator, builderNode,
src, probeExpr, type, buildSideNdv, exprOrder);
}
@Override
public List<Slot> computeOutput() {
return outputExpressions.stream()

105
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
View File

@ -17,34 +17,23 @@
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.hint.DistributeHint;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.processor.post.RuntimeFilterContext;
import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.MutableState;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.List;
@ -201,100 +190,6 @@ public class PhysicalHashJoin<
getLogicalProperties(), physicalProperties, statistics, left(), right());
}
@Override
public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<RuntimeFilterId> generator,
AbstractPhysicalJoin<?, ?> builderNode, Expression srcExpr, Expression probeExpr,
TRuntimeFilterType type, long buildSideNdv, int exprOrder) {
if (RuntimeFilterGenerator.DENIED_JOIN_TYPES.contains(getJoinType()) || isMarkJoin()) {
if (builderNode instanceof PhysicalHashJoin) {
PhysicalHashJoin<?, ?> builderJoin = (PhysicalHashJoin<?, ?>) builderNode;
if (builderJoin == this) {
return false;
}
EqualPredicate equal = (EqualPredicate) builderNode.getHashJoinConjuncts().get(exprOrder);
if (equal instanceof NullSafeEqual) {
if (this.joinType.isOuterJoin()) {
return false;
}
}
}
}
EqualPredicate equal = (EqualPredicate) builderNode.getHashJoinConjuncts().get(exprOrder);
if (equal instanceof NullSafeEqual) {
if (this.joinType.isOuterJoin()) {
return false;
}
}
RuntimeFilterContext ctx = context.getRuntimeFilterContext();
// if rf built between plan nodes containing cte both, for example both src slot and target slot are from cte,
// or two sub-queries both containing cte, disable this rf since this kind of cross-cte rf will make one side
// of cte to wait for a long time until another side cte consumer finished, which will make the rf into
// not ready state.
AbstractPhysicalPlan builderLeftNode = (AbstractPhysicalPlan) builderNode.child(0);
AbstractPhysicalPlan builderRightNode = (AbstractPhysicalPlan) builderNode.child(1);
Preconditions.checkState(builderLeftNode != null && builderRightNode != null,
"builder join node child node is null");
// if (RuntimeFilterGenerator.hasCTEConsumerDescendant(builderLeftNode)
// && RuntimeFilterGenerator.hasCTEConsumerDescendant(builderRightNode)) {
// return false;
// }
boolean pushedDown = false;
AbstractPhysicalPlan leftNode = (AbstractPhysicalPlan) child(0);
AbstractPhysicalPlan rightNode = (AbstractPhysicalPlan) child(1);
Preconditions.checkState(leftNode != null && rightNode != null,
"join child node is null");
Set<Expression> probExprList = Sets.newLinkedHashSet();
probExprList.add(probeExpr);
Pair<PhysicalRelation, Slot> srcPair = ctx.getAliasTransferMap().get(srcExpr);
PhysicalRelation srcNode = (srcPair == null) ? null : srcPair.first;
Pair<PhysicalRelation, Slot> targetPair = ctx.getAliasTransferMap().get(probeExpr);
if (targetPair == null) {
/* cases for "targetPair is null"
1. when probeExpr is output slot of setOperator, targetPair is null
2. join (topn(table1), table2)
when aliasTransferMap goes through topn, all slots from table1
are cleared to avoid generate rf(table2->table1)
*/
return false;
}
PhysicalRelation target1 = targetPair.first;
PhysicalRelation target2 = null;
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().expandRuntimeFilterByInnerJoin) {
if (!this.equals(builderNode)
&& (this.getJoinType() == JoinType.INNER_JOIN || this.getJoinType().isSemiJoin())) {
for (Expression expr : this.getHashJoinConjuncts()) {
EqualPredicate equalTo = (EqualPredicate) expr;
if (probeExpr.equals(equalTo.left())) {
probExprList.add(equalTo.right());
targetPair = ctx.getAliasTransferMap().get(equalTo.right());
target2 = (targetPair == null) ? null : targetPair.first;
} else if (probeExpr.equals(equalTo.right())) {
probExprList.add(equalTo.left());
targetPair = ctx.getAliasTransferMap().get(equalTo.left());
target2 = (targetPair == null) ? null : targetPair.first;
}
if (target2 != null) {
ctx.getExpandedRF().add(
new RuntimeFilterContext.ExpandRF(this, srcNode, target1, target2, equalTo));
}
}
probExprList.remove(srcExpr);
}
}
for (Expression prob : probExprList) {
pushedDown |= leftNode.pushDownRuntimeFilter(context, generator, builderNode,
srcExpr, prob, type, buildSideNdv, exprOrder);
pushedDown |= rightNode.pushDownRuntimeFilter(context, generator, builderNode,
srcExpr, prob, type, buildSideNdv, exprOrder);
}
return pushedDown;
}
@Override
public String shapeInfo() {
StringBuilder builder = new StringBuilder();

20
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java
View File

@ -17,16 +17,12 @@
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.hint.DistributeHint;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.DistributeType;
import org.apache.doris.nereids.trees.plans.JoinType;
@ -35,9 +31,7 @@ import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.MutableState;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
@ -244,18 +238,4 @@ public class PhysicalNestedLoopJoin<
hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts, markJoinSlotReference, groupExpression,
null, physicalProperties, statistics, left(), right());
}
@Override
public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<RuntimeFilterId> generator,
AbstractPhysicalJoin<?, ?> builderNode, Expression srcExpr,
Expression probeExpr, TRuntimeFilterType type, long buildSideNdv,
int exprOrder) {
EqualPredicate equal = (EqualPredicate) builderNode.getHashJoinConjuncts().get(exprOrder);
if (equal instanceof NullSafeEqual && this.joinType.isOuterJoin()) {
return false;
}
return super.pushDownRuntimeFilter(context, generator, builderNode, srcExpr, probeExpr, type, buildSideNdv,
exprOrder);
}
}

75
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalProject.java
View File

@ -17,11 +17,7 @@
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.processor.post.RuntimeFilterContext;
import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Add;
@ -34,11 +30,8 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.algebra.Project;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@ -169,74 +162,6 @@ public class PhysicalProject<CHILD_TYPE extends Plan> extends PhysicalUnary<CHIL
);
}
@Override
public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<RuntimeFilterId> generator,
AbstractPhysicalJoin<?, ?> builderNode, Expression src, Expression probeExpr,
TRuntimeFilterType type, long buildSideNdv, int exprOrder) {
RuntimeFilterContext ctx = context.getRuntimeFilterContext();
// currently, we can ensure children in the two side are corresponding to the equal_to's.
// so right maybe an expression and left is a slot
Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr);
if (probeSlot == null) {
return false;
}
if (RuntimeFilterGenerator.checkPushDownPreconditionsForProjectOrDistribute(ctx, probeSlot)) {
PhysicalRelation scan = ctx.getAliasTransferPair(probeSlot).first;
Preconditions.checkState(scan != null, "scan is null");
if (scan instanceof PhysicalCTEConsumer) {
// update the probeExpr
int projIndex = -1;
for (int i = 0; i < getProjects().size(); i++) {
NamedExpression expr = getProjects().get(i);
if (expr.getName().equals(probeSlot.getName())) {
projIndex = i;
break;
}
}
if (projIndex < 0 || projIndex >= getProjects().size()) {
// the pushed down path can't contain the probe expr
return false;
}
NamedExpression newProbeExpr = this.getProjects().get(projIndex);
if (newProbeExpr instanceof Alias) {
Expression child = ExpressionUtils.getExpressionCoveredByCast(newProbeExpr.child(0));
if (child instanceof NamedExpression) {
newProbeExpr = (NamedExpression) child;
} else {
return false;
}
}
Slot newProbeSlot = RuntimeFilterGenerator.checkTargetChild(newProbeExpr);
if (!RuntimeFilterGenerator.checkProbeSlot(ctx, newProbeSlot)) {
return false;
}
scan = ctx.getAliasTransferPair(newProbeSlot).first;
probeExpr = newProbeExpr;
}
if (!RuntimeFilterGenerator.checkPushDownPreconditionsForRelation(this, scan)) {
return false;
}
if (probeExpr instanceof SlotReference) {
for (NamedExpression namedExpression : projects) {
if (namedExpression instanceof Alias
&& namedExpression.getExprId() == ((SlotReference) probeExpr).getExprId()) {
probeExpr = ((Alias) namedExpression).child();
break;
}
}
}
AbstractPhysicalPlan child = (AbstractPhysicalPlan) child(0);
return child.pushDownRuntimeFilter(context, generator, builderNode,
src, probeExpr, type, buildSideNdv, exprOrder);
} else {
// if probe slot doesn't exist in aliasTransferMap, then try to pass it to child
AbstractPhysicalPlan child = (AbstractPhysicalPlan) child(0);
return child.pushDownRuntimeFilter(context, generator, builderNode,
src, probeExpr, type, buildSideNdv, exprOrder);
}
}
@Override
public List<Slot> computeOutput() {
List<NamedExpression> output = projects;

41
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java
View File

@ -17,30 +17,22 @@
package org.apache.doris.nereids.trees.plans.physical;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.planner.RuntimeFilterId;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@ -145,39 +137,6 @@ public abstract class PhysicalSetOperation extends AbstractPhysicalPlan implemen
return children.size();
}
@Override
public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<RuntimeFilterId> generator,
AbstractPhysicalJoin<?, ?> builderNode, Expression src, Expression probeExpr,
TRuntimeFilterType type, long buildSideNdv, int exprOrder) {
boolean pushedDown = false;
int projIndex = -1;
Slot probeSlot = RuntimeFilterGenerator.checkTargetChild(probeExpr);
if (probeSlot == null) {
return false;
}
List<NamedExpression> output = getOutputs();
for (int j = 0; j < output.size(); j++) {
NamedExpression expr = output.get(j);
if (expr.getName().equals(probeSlot.getName())) {
projIndex = j;
break;
}
}
if (projIndex == -1) {
return false;
}
for (int i = 0; i < this.children().size(); i++) {
Map<Expression, Expression> map = Maps.newHashMap();
// probeExpr only has one input slot
map.put(probeExpr.getInputSlots().iterator().next(), regularChildrenOutputs.get(i).get(projIndex));
Expression newProbeExpr = probeExpr.accept(ExpressionVisitors.EXPRESSION_MAP_REPLACER, map);
AbstractPhysicalPlan child = (AbstractPhysicalPlan) this.child(i);
pushedDown |= child.pushDownRuntimeFilter(context, generator, builderNode, src,
newProbeExpr, type, buildSideNdv, exprOrder);
}
return pushedDown;
}
@Override
public List<Slot> computeOutput() {
return outputs.stream()

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java
View File

@ -55,7 +55,6 @@ import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin;
import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
@ -274,10 +273,6 @@ public abstract class PlanVisitor<R, C> implements CommandVisitor<R, C>, Relatio
return visit(cteAnchor, context);
}
public R visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer, C context) {
return visit(cteConsumer, context);
}
public R visitPhysicalCTEProducer(PhysicalCTEProducer<? extends Plan> cteProducer, C context) {
return visit(cteProducer, context);
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
View File

@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeOlapScan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
@ -176,4 +177,8 @@ public interface RelationVisitor<R, C> {
default R visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, C context) {
return visitPhysicalRelation(tvfRelation, context);
}
default R visitPhysicalCTEConsumer(PhysicalCTEConsumer consumer, C context) {
return visitPhysicalRelation(consumer, context);
}
}

10
regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query35.out
View File

@ -13,19 +13,19 @@ PhysicalResultSink
--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3
------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5
----------------------------PhysicalDistribute[DistributionSpecReplicated]
------------------------------PhysicalProject
--------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 1999))
----------------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
--------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk]
----------------------------PhysicalDistribute[DistributionSpecHash]
------------------------------PhysicalProject
--------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=()
--------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk]
----------------------------------PhysicalDistribute[DistributionSpecHash]
------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ss_customer_sk]
--------------------------------------PhysicalDistribute[DistributionSpecHash]
@ -49,7 +49,7 @@ PhysicalResultSink
----------------------------------------------------PhysicalOlapScan[date_dim]
----------------------------------------PhysicalDistribute[DistributionSpecHash]
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[customer]
--------------------------------------------PhysicalOlapScan[customer] apply RFs: RF3 RF4
----------------------------------PhysicalDistribute[DistributionSpecHash]
------------------------------------PhysicalProject
--------------------------------------PhysicalOlapScan[customer_address]

4
regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query35.out
View File

@ -13,9 +13,9 @@ PhysicalResultSink
--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3
------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5
----------------------------PhysicalDistribute[DistributionSpecReplicated]
------------------------------PhysicalProject
--------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))

10
regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query35.out
View File

@ -13,19 +13,19 @@ PhysicalResultSink
--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = catalog_sales.cs_ship_customer_sk)) otherCondition=()
----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
----------------------------PhysicalProject
------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3
------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5
----------------------------PhysicalDistribute[DistributionSpecReplicated]
------------------------------PhysicalProject
--------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 2001))
----------------------------------PhysicalOlapScan[date_dim]
----------------------PhysicalDistribute[DistributionSpecHash]
------------------------PhysicalProject
--------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=()
--------------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk]
----------------------------PhysicalDistribute[DistributionSpecHash]
------------------------------PhysicalProject
--------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=()
--------------------------------hashJoin[INNER_JOIN] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk]
----------------------------------PhysicalDistribute[DistributionSpecHash]
------------------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ss_customer_sk]
--------------------------------------PhysicalDistribute[DistributionSpecHash]
@ -49,7 +49,7 @@ PhysicalResultSink
----------------------------------------------------PhysicalOlapScan[date_dim]
----------------------------------------PhysicalDistribute[DistributionSpecHash]
------------------------------------------PhysicalProject
--------------------------------------------PhysicalOlapScan[customer]
--------------------------------------------PhysicalOlapScan[customer] apply RFs: RF3 RF4
----------------------------------PhysicalDistribute[DistributionSpecHash]
------------------------------------PhysicalProject
--------------------------------------PhysicalOlapScan[customer_address]

2
regression-test/data/nereids_tpch_shape_sf1000_p0/runtime_filter/test_pushdown_setop.out
View File

@ -23,7 +23,7 @@ PhysicalResultSink
----PhysicalDistribute[DistributionSpecGather]
------hashAgg[LOCAL]
--------PhysicalProject
----------hashJoin[INNER_JOIN] hashCondition=((expr_abs(l_linenumber) = expr_cast(r_regionkey as LARGEINT))) otherCondition=() build RFs:RF0 expr_cast(r_regionkey as LARGEINT)->[abs(l_linenumber),abs(o_orderkey)]
----------hashJoin[INNER_JOIN] hashCondition=((expr_abs(l_linenumber) = expr_cast(r_regionkey as LARGEINT))) otherCondition=() build RFs:RF0 expr_cast(r_regionkey as LARGEINT)->[abs(cast(l_linenumber as BIGINT)),abs(o_orderkey)]
------------PhysicalProject
--------------PhysicalExcept
----------------PhysicalDistribute[DistributionSpecHash]

720
regression-test/suites/query_p0/join/test_join.groovy
View File

File diff suppressed because it is too large Load Diff