From 2284575afa72de61623d8db82a7d80b80cb7ede1 Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 26 Jan 2024 09:59:21 +0800 Subject: [PATCH] [opt](nereids)set flag to indicate if bloom filter size is calculated by ndv (#30278) set flag to indicate if bloom filter size is calculated by ndv --- .../translator/RuntimeFilterTranslator.java | 1 + .../post/RuntimeFilterGenerator.java | 8 +++++--- .../plans/physical/AbstractPhysicalPlan.java | 3 ++- .../trees/plans/physical/RuntimeFilter.java | 19 +++++++++++++++---- .../apache/doris/planner/RuntimeFilter.java | 12 ++++++++++++ gensrc/thrift/PlanNodes.thrift | 5 +++++ 6 files changed, 40 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java index c0787c6012..a2fbc28ecd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java @@ -165,6 +165,7 @@ public class RuntimeFilterTranslator { scanNode, targetExpr, true, isLocalTarget)); } origFilter.setBitmapFilterNotIn(filter.isBitmapFilterNotIn()); + origFilter.setBloomFilterSizeCalculatedByNdv(filter.isBloomFilterSizeCalculatedByNdv()); org.apache.doris.planner.RuntimeFilter finalizedFilter = finalize(origFilter); scanNodeList.stream().filter(e -> e.getStatisticalType() == StatisticalType.CTE_SCAN_NODE) .forEach(f -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index 85f246998d..2a5d37e090 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -186,7 +186,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { PhysicalRelation scan = ctx.getAliasTransferPair(targetSlot).first; RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), bitmapContains.child(0), ImmutableList.of(scanSlot), - ImmutableList.of(bitmapContains.child(1)), type, i, join, isNot, -1L, scan); + ImmutableList.of(bitmapContains.child(1)), type, i, join, isNot, -1L, + false, scan); scan.addAppliedRuntimeFilter(filter); ctx.addJoinToTargetMap(join, scanSlot.getExprId()); ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter); @@ -265,7 +266,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { long buildSideNdv = getBuildSideNdv(join, compare); RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), compare.child(1), ImmutableList.of(olapScanSlot), ImmutableList.of(olapScanSlot), - TRuntimeFilterType.MIN_MAX, exprOrder, join, true, buildSideNdv, + TRuntimeFilterType.MIN_MAX, exprOrder, join, true, buildSideNdv, false, getMinMaxType(compare), scan); scan.addAppliedRuntimeFilter(filter); ctx.addJoinToTargetMap(join, olapScanSlot.getExprId()); @@ -628,7 +629,8 @@ public class RuntimeFilterGenerator extends PlanPostProcessor { // build multi-target runtime filter // since always on different join, set the expr_order as 0 RuntimeFilter filter = new RuntimeFilter(generator.getNextId(), - equalTo.right(), targetList, targetExpressions, type, 0, join, buildSideNdv, cteNode); + equalTo.right(), targetList, targetExpressions, type, 0, join, buildSideNdv, true, + cteNode); targetNodes.forEach(node -> node.addAppliedRuntimeFilter(filter)); for (Slot slot : targetList) { ctx.setTargetExprIdToFilter(slot.getExprId(), filter); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java index 108fa78f80..5d8d4046f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalPlan.java @@ -134,7 +134,8 @@ public abstract class AbstractPhysicalPlan extends AbstractPlan implements Physi } else { filter = new RuntimeFilter(generator.getNextId(), src, ImmutableList.of(scanSlot), ImmutableList.of(probeExpr), - type, exprOrder, builderNode, buildSideNdv, scan); + type, exprOrder, builderNode, buildSideNdv, + !context.getStatementContext().isHasUnknownColStats(), scan); this.addAppliedRuntimeFilter(filter); ctx.addJoinToTargetMap(builderNode, scanSlot.getExprId()); ctx.setTargetExprIdToFilter(scanSlot.getExprId(), filter); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java index 596b1c287a..f9b269ec30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java @@ -51,22 +51,27 @@ public class RuntimeFilter { private final List targetScans = Lists.newArrayList(); + private final boolean bloomFilterSizeCalculatedByNdv; + /** * constructor */ public RuntimeFilter(RuntimeFilterId id, Expression src, List targets, List targetExpressions, TRuntimeFilterType type, int exprOrder, AbstractPhysicalJoin builderNode, long buildSideNdv, + boolean bloomFilterSizeCalculatedByNdv, PhysicalRelation scan) { this(id, src, targets, targetExpressions, type, exprOrder, - builderNode, false, buildSideNdv, TMinMaxRuntimeFilterType.MIN_MAX, scan); + builderNode, false, buildSideNdv, bloomFilterSizeCalculatedByNdv, + TMinMaxRuntimeFilterType.MIN_MAX, scan); } public RuntimeFilter(RuntimeFilterId id, Expression src, List targets, List targetExpressions, TRuntimeFilterType type, int exprOrder, AbstractPhysicalJoin builderNode, - boolean bitmapFilterNotIn, long buildSideNdv, + boolean bitmapFilterNotIn, long buildSideNdv, boolean bloomFilterSizeCalculatedByNdv, PhysicalRelation scan) { this(id, src, targets, targetExpressions, type, exprOrder, - builderNode, bitmapFilterNotIn, buildSideNdv, TMinMaxRuntimeFilterType.MIN_MAX, scan); + builderNode, bitmapFilterNotIn, buildSideNdv, bloomFilterSizeCalculatedByNdv, + TMinMaxRuntimeFilterType.MIN_MAX, scan); } /** @@ -74,7 +79,8 @@ public class RuntimeFilter { */ public RuntimeFilter(RuntimeFilterId id, Expression src, List targets, List targetExpressions, TRuntimeFilterType type, int exprOrder, AbstractPhysicalJoin builderNode, - boolean bitmapFilterNotIn, long buildSideNdv, TMinMaxRuntimeFilterType tMinMaxType, + boolean bitmapFilterNotIn, long buildSideNdv, boolean bloomFilterSizeCalculatedByNdv, + TMinMaxRuntimeFilterType tMinMaxType, PhysicalRelation scan) { this.id = id; this.srcSlot = src; @@ -84,6 +90,7 @@ public class RuntimeFilter { this.exprOrder = exprOrder; this.builderNode = builderNode; this.bitmapFilterNotIn = bitmapFilterNotIn; + this.bloomFilterSizeCalculatedByNdv = bloomFilterSizeCalculatedByNdv; this.buildSideNdv = buildSideNdv <= 0 ? -1L : buildSideNdv; this.tMinMaxType = tMinMaxType; builderNode.addRuntimeFilter(this); @@ -167,4 +174,8 @@ public class RuntimeFilter { .append("]"); return sb.toString(); } + + public boolean isBloomFilterSizeCalculatedByNdv() { + return bloomFilterSizeCalculatedByNdv; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java index 646a07221a..58d4d3eabe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java @@ -112,6 +112,8 @@ public final class RuntimeFilter { private TMinMaxRuntimeFilterType tMinMaxRuntimeFilterType; + private boolean bloomFilterSizeCalculatedByNdv = false; + /** * Internal representation of a runtime filter target. */ @@ -244,6 +246,7 @@ public final class RuntimeFilter { tFilter.setMinMaxType(tMinMaxRuntimeFilterType); } tFilter.setOptRemoteRf(optRemoteRf); + tFilter.setBloomFilterSizeCalculatedByNdv(bloomFilterSizeCalculatedByNdv); return tFilter; } @@ -741,4 +744,13 @@ public final class RuntimeFilter { } return filterStr.toString(); } + + + public boolean isBloomFilterSizeCalculatedByNdv() { + return bloomFilterSizeCalculatedByNdv; + } + + public void setBloomFilterSizeCalculatedByNdv(boolean bloomFilterSizeCalculatedByNdv) { + this.bloomFilterSizeCalculatedByNdv = bloomFilterSizeCalculatedByNdv; + } } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 192cd56c8a..4c23a1afd9 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -1181,6 +1181,11 @@ struct TRuntimeFilterDesc { // for min/max rf 13: optional TMinMaxRuntimeFilterType min_max_type; + + // true, if bloom filter size is calculated by ndv + // if bloom_filter_size_calculated_by_ndv=false, BE could calculate filter size according to the actural row count, and + // ignore bloom_filter_size_bytes + 14: optional bool bloom_filter_size_calculated_by_ndv; }