database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](nereids)set flag to indicate if bloom filter size is calculated by ndv (#30278)

Browse Source 
set flag to indicate if bloom filter size is calculated by ndv
This commit is contained in:
minghong
2024-01-26 09:59:21 +08:00
committed by yiguolei
parent 5986d5415e
commit 2284575afa
6 changed files with 40 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java
View File

@ -165,6 +165,7 @@ public class RuntimeFilterTranslator {
scanNode, targetExpr, true, isLocalTarget));
}
origFilter.setBitmapFilterNotIn(filter.isBitmapFilterNotIn());
origFilter.setBloomFilterSizeCalculatedByNdv(filter.isBloomFilterSizeCalculatedByNdv());
org.apache.doris.planner.RuntimeFilter finalizedFilter = finalize(origFilter);
scanNodeList.stream().filter(e -> e.getStatisticalType() == StatisticalType.CTE_SCAN_NODE)
.forEach(f -> {

8
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
View File

@ -186,7 +186,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
PhysicalRelation scan = ctx.getAliasTransferPair(targetSlot).first;
RuntimeFilter filter = new RuntimeFilter(generator.getNextId(),
bitmapContains.child(0), ImmutableList.of(scanSlot),
ImmutableList.of(bitmapContains.child(1)), type, i, join, isNot, -1L, scan);
ImmutableList.of(bitmapContains.child(1)), type, i, join, isNot, -1L,
false, scan);
scan.addAppliedRuntimeFilter(filter);
ctx.addJoinToTargetMap(join, scanSlot.getExprId());
ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter);
@ -265,7 +266,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
long buildSideNdv = getBuildSideNdv(join, compare);
RuntimeFilter filter = new RuntimeFilter(generator.getNextId(),
compare.child(1), ImmutableList.of(olapScanSlot), ImmutableList.of(olapScanSlot),
TRuntimeFilterType.MIN_MAX, exprOrder, join, true, buildSideNdv,
TRuntimeFilterType.MIN_MAX, exprOrder, join, true, buildSideNdv, false,
getMinMaxType(compare), scan);
scan.addAppliedRuntimeFilter(filter);
ctx.addJoinToTargetMap(join, olapScanSlot.getExprId());
@ -628,7 +629,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
// build multi-target runtime filter
// since always on different join, set the expr_order as 0
RuntimeFilter filter = new RuntimeFilter(generator.getNextId(),
equalTo.right(), targetList, targetExpressions, type, 0, join, buildSideNdv, cteNode);
equalTo.right(), targetList, targetExpressions, type, 0, join, buildSideNdv, true,
cteNode);
targetNodes.forEach(node -> node.addAppliedRuntimeFilter(filter));
for (Slot slot : targetList) {
ctx.setTargetExprIdToFilter(slot.getExprId(), filter);

3
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java
View File

@ -134,7 +134,8 @@ public abstract class AbstractPhysicalPlan extends AbstractPlan implements Physi
} else {
filter = new RuntimeFilter(generator.getNextId(),
src, ImmutableList.of(scanSlot), ImmutableList.of(probeExpr),
type, exprOrder, builderNode, buildSideNdv, scan);
type, exprOrder, builderNode, buildSideNdv,
!context.getStatementContext().isHasUnknownColStats(), scan);
this.addAppliedRuntimeFilter(filter);
ctx.addJoinToTargetMap(builderNode, scanSlot.getExprId());
ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter);

19
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java
View File

@ -51,22 +51,27 @@ public class RuntimeFilter {
private final List<PhysicalRelation> targetScans = Lists.newArrayList();
private final boolean bloomFilterSizeCalculatedByNdv;
/**
* constructor
*/
public RuntimeFilter(RuntimeFilterId id, Expression src, List<Slot> targets, List<Expression> targetExpressions,
TRuntimeFilterType type, int exprOrder, AbstractPhysicalJoin builderNode, long buildSideNdv,
boolean bloomFilterSizeCalculatedByNdv,
PhysicalRelation scan) {
this(id, src, targets, targetExpressions, type, exprOrder,
builderNode, false, buildSideNdv, TMinMaxRuntimeFilterType.MIN_MAX, scan);
builderNode, false, buildSideNdv, bloomFilterSizeCalculatedByNdv,
TMinMaxRuntimeFilterType.MIN_MAX, scan);
}
public RuntimeFilter(RuntimeFilterId id, Expression src, List<Slot> targets, List<Expression> targetExpressions,
TRuntimeFilterType type, int exprOrder, AbstractPhysicalJoin builderNode,
boolean bitmapFilterNotIn, long buildSideNdv,
boolean bitmapFilterNotIn, long buildSideNdv, boolean bloomFilterSizeCalculatedByNdv,
PhysicalRelation scan) {
this(id, src, targets, targetExpressions, type, exprOrder,
builderNode, bitmapFilterNotIn, buildSideNdv, TMinMaxRuntimeFilterType.MIN_MAX, scan);
builderNode, bitmapFilterNotIn, buildSideNdv, bloomFilterSizeCalculatedByNdv,
TMinMaxRuntimeFilterType.MIN_MAX, scan);
}
/**
@ -74,7 +79,8 @@ public class RuntimeFilter {
*/
public RuntimeFilter(RuntimeFilterId id, Expression src, List<Slot> targets, List<Expression> targetExpressions,
TRuntimeFilterType type, int exprOrder, AbstractPhysicalJoin builderNode,
boolean bitmapFilterNotIn, long buildSideNdv, TMinMaxRuntimeFilterType tMinMaxType,
boolean bitmapFilterNotIn, long buildSideNdv, boolean bloomFilterSizeCalculatedByNdv,
TMinMaxRuntimeFilterType tMinMaxType,
PhysicalRelation scan) {
this.id = id;
this.srcSlot = src;
@ -84,6 +90,7 @@ public class RuntimeFilter {
this.exprOrder = exprOrder;
this.builderNode = builderNode;
this.bitmapFilterNotIn = bitmapFilterNotIn;
this.bloomFilterSizeCalculatedByNdv = bloomFilterSizeCalculatedByNdv;
this.buildSideNdv = buildSideNdv <= 0 ? -1L : buildSideNdv;
this.tMinMaxType = tMinMaxType;
builderNode.addRuntimeFilter(this);
@ -167,4 +174,8 @@ public class RuntimeFilter {
.append("]");
return sb.toString();
}
public boolean isBloomFilterSizeCalculatedByNdv() {
return bloomFilterSizeCalculatedByNdv;
}
}

12
fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
View File

@ -112,6 +112,8 @@ public final class RuntimeFilter {
private TMinMaxRuntimeFilterType tMinMaxRuntimeFilterType;
private boolean bloomFilterSizeCalculatedByNdv = false;
/**
* Internal representation of a runtime filter target.
*/
@ -244,6 +246,7 @@ public final class RuntimeFilter {
tFilter.setMinMaxType(tMinMaxRuntimeFilterType);
}
tFilter.setOptRemoteRf(optRemoteRf);
tFilter.setBloomFilterSizeCalculatedByNdv(bloomFilterSizeCalculatedByNdv);
return tFilter;
}
@ -741,4 +744,13 @@ public final class RuntimeFilter {
}
return filterStr.toString();
}
public boolean isBloomFilterSizeCalculatedByNdv() {
return bloomFilterSizeCalculatedByNdv;
}
public void setBloomFilterSizeCalculatedByNdv(boolean bloomFilterSizeCalculatedByNdv) {
this.bloomFilterSizeCalculatedByNdv = bloomFilterSizeCalculatedByNdv;
}
}

5
gensrc/thrift/PlanNodes.thrift
View File

@ -1181,6 +1181,11 @@ struct TRuntimeFilterDesc {
// for min/max rf
13: optional TMinMaxRuntimeFilterType min_max_type;
// true, if bloom filter size is calculated by ndv
// if bloom_filter_size_calculated_by_ndv=false, BE could calculate filter size according to the actural row count, and
// ignore bloom_filter_size_bytes
14: optional bool bloom_filter_size_calculated_by_ndv;
}