database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improvement](pipeline) Use bloom runtime filter by default for pipeline engine (#18177)

Browse Source
This commit is contained in:
Gabriel
2023-04-03 15:31:48 +08:00
committed by GitHub
parent 368a2f7ace
commit 96a64dc9e8
2 changed files with 13 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java
View File

@ -35,6 +35,7 @@ import org.apache.doris.planner.HashJoinNode.DistributionMode;
import org.apache.doris.planner.JoinNodeBase;
import org.apache.doris.planner.RuntimeFilter.RuntimeFilterTarget;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TRuntimeFilterType;
import com.google.common.collect.ImmutableList;
@ -149,6 +150,12 @@ public class RuntimeFilterTranslator {
origFilter.markFinalized();
origFilter.assignToPlanNodes();
origFilter.extractTargetsPosition();
// Number of parallel instances are large for pipeline engine, so we prefer bloom filter.
if (!origFilter.hasRemoteTargets() && origFilter.getType() == TRuntimeFilterType.IN_OR_BLOOM
&& ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().enablePipelineEngine()) {
origFilter.setType(TRuntimeFilterType.BLOOM);
}
return origFilter;
}
}

6
fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilterGenerator.java
View File

@ -190,6 +190,12 @@ public final class RuntimeFilterGenerator {
int numBloomFilters = 0;
for (RuntimeFilter filter : filters) {
filter.extractTargetsPosition();
// Number of parallel instances are large for pipeline engine, so we prefer bloom filter.
if (!filter.hasRemoteTargets() && filter.getType() == TRuntimeFilterType.IN_OR_BLOOM
&& ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().enablePipelineEngine()) {
filter.setType(TRuntimeFilterType.BLOOM);
}
if (filter.getType() == TRuntimeFilterType.BLOOM) {
if (numBloomFilters >= maxNumBloomFilters) {
continue;