database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Opt](rf) Opt broadcast join remote runtime filter merge and wait (#29439)

Browse Source
This commit is contained in:
HappenLee
2024-01-03 11:21:28 +08:00
committed by GitHub
parent 067a9a3a22
commit 14e7eb7624
8 changed files with 32 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
View File

@ -604,6 +604,10 @@ public final class RuntimeFilter {
isBroadcastJoin = isBroadcast;
}
public boolean isBroadcast() {
return isBroadcastJoin;
}
public void computeNdvEstimate() {
if (ndvEstimate < 0) {
ndvEstimate = builderNode.getChild(1).getCardinalityAfterFilter();

12
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -3627,6 +3627,9 @@ public class Coordinator implements CoordInterface {
params.params.setRuntimeFilterParams(new TRuntimeFilterParams());
params.params.runtime_filter_params.setRuntimeFilterMergeAddr(runtimeFilterMergeAddr);
if (instanceExecParam.instanceId.equals(runtimeFilterMergeInstanceId)) {
Set<Integer> broadCastRf = assignedRuntimeFilters.stream().filter(RuntimeFilter::isBroadcast)
.map(r -> r.getFilterId().asInt()).collect(Collectors.toSet());
for (RuntimeFilter rf : assignedRuntimeFilters) {
if (!ridToTargetParam.containsKey(rf.getFilterId())) {
continue;
@ -3668,7 +3671,8 @@ public class Coordinator implements CoordInterface {
}
for (Map.Entry<RuntimeFilterId, Integer> entry : ridToBuilderNum.entrySet()) {
params.params.runtime_filter_params.putToRuntimeFilterBuilderNum(
entry.getKey().asInt(), entry.getValue());
entry.getKey().asInt(), broadCastRf.contains(entry.getKey().asInt())
? 1 : entry.getValue());
}
for (RuntimeFilter rf : assignedRuntimeFilters) {
params.params.runtime_filter_params.putToRidToRuntimeFilter(
@ -3753,6 +3757,9 @@ public class Coordinator implements CoordInterface {
localParams.setRuntimeFilterParams(new TRuntimeFilterParams());
localParams.runtime_filter_params.setRuntimeFilterMergeAddr(runtimeFilterMergeAddr);
if (instanceExecParam.instanceId.equals(runtimeFilterMergeInstanceId)) {
Set<Integer> broadCastRf = assignedRuntimeFilters.stream().filter(RuntimeFilter::isBroadcast)
.map(r -> r.getFilterId().asInt()).collect(Collectors.toSet());
for (RuntimeFilter rf : assignedRuntimeFilters) {
if (!ridToTargetParam.containsKey(rf.getFilterId())) {
continue;
@ -3795,7 +3802,8 @@ public class Coordinator implements CoordInterface {
}
for (Map.Entry<RuntimeFilterId, Integer> entry : ridToBuilderNum.entrySet()) {
localParams.runtime_filter_params.putToRuntimeFilterBuilderNum(
entry.getKey().asInt(), entry.getValue());
entry.getKey().asInt(), broadCastRf.contains(entry.getKey().asInt()) ? 1 :
entry.getValue());
}
for (RuntimeFilter rf : assignedRuntimeFilters) {
localParams.runtime_filter_params.putToRidToRuntimeFilter(