database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[pipelineX](local shuffle) Fix bucket hash shuffle (#28202)

Browse Source
This commit is contained in:
Gabriel
2023-12-10 00:35:00 +08:00
committed by GitHub
parent 61379b141e
commit 5aa90a3bce
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -3596,6 +3596,7 @@ public class Coordinator implements CoordInterface {
}
Map<TNetworkAddress, TPipelineFragmentParams> res = new HashMap();
Map<TNetworkAddress, Integer> instanceIdx = new HashMap();
for (int i = 0; i < instanceExecParams.size(); ++i) {
final FInstanceExecParam instanceExecParam = instanceExecParams.get(i);
if (!res.containsKey(instanceExecParam.host)) {
@ -3625,10 +3626,16 @@ public class Coordinator implements CoordInterface {
params.setNumBuckets(fragment.getBucketNum());
res.put(instanceExecParam.host, params);
res.get(instanceExecParam.host).setBucketSeqToInstanceIdx(new HashMap<Integer, Integer>());
instanceIdx.put(instanceExecParam.host, 0);
}
// Set each bucket belongs to which instance on this BE.
// This is used for LocalExchange(BUCKET_HASH_SHUFFLE).
int instanceId = instanceIdx.get(instanceExecParam.host);
for (int bucket : instanceExecParam.bucketSeqSet) {
res.get(instanceExecParam.host).getBucketSeqToInstanceIdx().put(bucket, i);
res.get(instanceExecParam.host).getBucketSeqToInstanceIdx().put(bucket, instanceId);
}
instanceIdx.replace(instanceExecParam.host, ++instanceId);
TPipelineFragmentParams params = res.get(instanceExecParam.host);
TPipelineInstanceParams localParams = new TPipelineInstanceParams();