database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](nereids) do not use in_filter in pipeline mode (#19028)

Browse Source 
1. in pipeline in_or_bloom filter replaced by bloom filter
2. do not set broadcast row limit
This commit is contained in:
minghong
2023-04-25 19:02:12 +08:00
committed by GitHub
parent fa0f3a2859
commit 8ea69ca11c
66 changed files with 193 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
View File

@ -102,6 +102,10 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
if (type == TRuntimeFilterType.BITMAP) {
continue;
}
// in-filter is not friendly to pipeline
if (type == TRuntimeFilterType.IN_OR_BLOOM && ctx.getSessionVariable().enablePipelineEngine()) {
type = TRuntimeFilterType.BLOOM;
}
// currently, we can ensure children in the two side are corresponding to the equal_to's.
// so right maybe an expression and left is a slot
Slot unwrappedSlot = checkTargetChild(equalTo.left());

6
fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilterGenerator.java
View File

@ -190,12 +190,6 @@ public final class RuntimeFilterGenerator {
int numBloomFilters = 0;
for (RuntimeFilter filter : filters) {
filter.extractTargetsPosition();
// Number of parallel instances are large for pipeline engine, so we prefer bloom filter.
if (filter.hasRemoteTargets() && filter.getType() == TRuntimeFilterType.IN_OR_BLOOM
&& ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().enablePipelineEngine()) {
filter.setType(TRuntimeFilterType.BLOOM);
}
if (filter.getType() == TRuntimeFilterType.BLOOM) {
if (numBloomFilters >= maxNumBloomFilters) {
continue;

2
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -649,7 +649,7 @@ public class SessionVariable implements Serializable, Writable {
private double broadcastRightTableScaleFactor = 10.0;
@VariableMgr.VarAttr(name = BROADCAST_ROW_COUNT_LIMIT, needForward = true)
private double broadcastRowCountLimit = 15000000;
private double broadcastRowCountLimit = 30000000;
@VariableMgr.VarAttr(name = BROADCAST_HASHTABLE_MEM_LIMIT_PERCENTAGE, needForward = true)
private double broadcastHashtableMemLimitPercentage = 0.2;