database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improvement](runtime filter) Improve merge phase (#18828)

Browse Source
This commit is contained in:
Gabriel
2023-04-26 21:01:20 +08:00
committed by GitHub
parent 0271886c36
commit aabcab9dbe
20 changed files with 685 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
View File

@ -190,8 +190,12 @@ public final class RuntimeFilter {
tFilter.setIsBroadcastJoin(isBroadcastJoin);
tFilter.setHasLocalTargets(hasLocalTargets);
tFilter.setHasRemoteTargets(hasRemoteTargets);
boolean optRemoteRf = true;
for (RuntimeFilterTarget target : targets) {
tFilter.putToPlanIdToTargetExpr(target.node.getId().asInt(), target.expr.treeToThrift());
// TODO: now only support SlotRef
optRemoteRf = optRemoteRf && hasRemoteTargets && runtimeFilterType == TRuntimeFilterType.BLOOM
&& target.expr instanceof SlotRef;
}
tFilter.setType(runtimeFilterType);
tFilter.setBloomFilterSizeBytes(filterSizeBytes);
@ -199,6 +203,7 @@ public final class RuntimeFilter {
tFilter.setBitmapTargetExpr(targets.get(0).expr.treeToThrift());
tFilter.setBitmapFilterNotIn(bitmapFilterNotIn);
}
tFilter.setOptRemoteRf(optRemoteRf);
return tFilter;
}

43
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -96,7 +96,7 @@ import org.apache.doris.thrift.TQueryType;
import org.apache.doris.thrift.TReportExecStatusParams;
import org.apache.doris.thrift.TResourceLimit;
import org.apache.doris.thrift.TRuntimeFilterParams;
import org.apache.doris.thrift.TRuntimeFilterTargetParams;
import org.apache.doris.thrift.TRuntimeFilterTargetParamsV2;
import org.apache.doris.thrift.TScanRangeLocation;
import org.apache.doris.thrift.TScanRangeLocations;
import org.apache.doris.thrift.TScanRangeParams;
@ -3122,12 +3122,24 @@ public class Coordinator {
if (instanceExecParam.instanceId.equals(runtimeFilterMergeInstanceId)) {
for (Map.Entry<RuntimeFilterId, List<FRuntimeFilterTargetParam>> entry
: ridToTargetParam.entrySet()) {
List<TRuntimeFilterTargetParams> targetParams = Lists.newArrayList();
Map<TNetworkAddress, TRuntimeFilterTargetParamsV2> targetParams = new HashMap<>();
for (FRuntimeFilterTargetParam targetParam : entry.getValue()) {
targetParams.add(new TRuntimeFilterTargetParams(targetParam.targetFragmentInstanceId,
targetParam.targetFragmentInstanceAddr));
if (targetParams.containsKey(targetParam.targetFragmentInstanceAddr)) {
targetParams.get(targetParam.targetFragmentInstanceAddr).target_fragment_instance_ids
.add(targetParam.targetFragmentInstanceId);
} else {
targetParams.put(targetParam.targetFragmentInstanceAddr,
new TRuntimeFilterTargetParamsV2());
targetParams.get(targetParam.targetFragmentInstanceAddr).target_fragment_instance_addr
= targetParam.targetFragmentInstanceAddr;
targetParams.get(targetParam.targetFragmentInstanceAddr).target_fragment_instance_ids
= new ArrayList<>();
targetParams.get(targetParam.targetFragmentInstanceAddr).target_fragment_instance_ids
.add(targetParam.targetFragmentInstanceId);
}
}
params.params.runtime_filter_params.putToRidToTargetParam(entry.getKey().asInt(), targetParams);
params.params.runtime_filter_params.putToRidToTargetParamv2(entry.getKey().asInt(),
new ArrayList<TRuntimeFilterTargetParamsV2>(targetParams.values()));
}
for (Map.Entry<RuntimeFilterId, Integer> entry : ridToBuilderNum.entrySet()) {
params.params.runtime_filter_params.putToRuntimeFilterBuilderNum(
@ -3197,12 +3209,25 @@ public class Coordinator {
if (instanceExecParam.instanceId.equals(runtimeFilterMergeInstanceId)) {
for (Map.Entry<RuntimeFilterId, List<FRuntimeFilterTargetParam>> entry
: ridToTargetParam.entrySet()) {
List<TRuntimeFilterTargetParams> targetParams = Lists.newArrayList();
Map<TNetworkAddress, TRuntimeFilterTargetParamsV2> targetParams = new HashMap<>();
for (FRuntimeFilterTargetParam targetParam : entry.getValue()) {
targetParams.add(new TRuntimeFilterTargetParams(targetParam.targetFragmentInstanceId,
targetParam.targetFragmentInstanceAddr));
if (targetParams.containsKey(targetParam.targetFragmentInstanceAddr)) {
targetParams.get(targetParam.targetFragmentInstanceAddr).target_fragment_instance_ids
.add(targetParam.targetFragmentInstanceId);
} else {
targetParams.put(targetParam.targetFragmentInstanceAddr,
new TRuntimeFilterTargetParamsV2());
targetParams.get(targetParam.targetFragmentInstanceAddr).target_fragment_instance_addr
= targetParam.targetFragmentInstanceAddr;
targetParams.get(targetParam.targetFragmentInstanceAddr).target_fragment_instance_ids
= new ArrayList<>();
targetParams.get(targetParam.targetFragmentInstanceAddr).target_fragment_instance_ids
.add(targetParam.targetFragmentInstanceId);
}
}
localParams.runtime_filter_params.putToRidToTargetParam(entry.getKey().asInt(), targetParams);
localParams.runtime_filter_params.putToRidToTargetParamv2(entry.getKey().asInt(),
new ArrayList<TRuntimeFilterTargetParamsV2>(targetParams.values()));
}
for (Map.Entry<RuntimeFilterId, Integer> entry : ridToBuilderNum.entrySet()) {
localParams.runtime_filter_params.putToRuntimeFilterBuilderNum(