database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](move-memtable) pass num local sink to backends (#26897)

Browse Source
This commit is contained in:
Kaijie Chen
2023-11-14 08:28:49 +08:00
committed by GitHub
parent de62c00f4e
commit b19abac5e2
19 changed files with 177 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
View File

@ -300,6 +300,7 @@ public class StreamLoadPlanner {
params.setParams(execParams);
params.setLoadStreamPerNode(taskInfo.getStreamPerNode());
params.setTotalLoadStreams(taskInfo.getStreamPerNode());
params.setNumLocalSink(1);
TQueryOptions queryOptions = new TQueryOptions();
queryOptions.setQueryType(TQueryType.LOAD);
queryOptions.setQueryTimeout(timeout);
@ -503,6 +504,7 @@ public class StreamLoadPlanner {
pipParams.setNumSenders(1);
pipParams.setLoadStreamPerNode(taskInfo.getStreamPerNode());
pipParams.setTotalLoadStreams(taskInfo.getStreamPerNode());
pipParams.setNumLocalSink(1);
TPipelineInstanceParams localParams = new TPipelineInstanceParams();
localParams.setFragmentInstanceId(new TUniqueId(loadId.hi, loadId.lo + fragmentInstanceIdIndex));

18
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -683,7 +683,7 @@ public class Coordinator implements CoordInterface {
int backendIdx = 0;
int profileFragmentId = 0;
long memoryLimit = queryOptions.getMemLimit();
Set<Long> backendsWithOlapTableSink = Sets.newHashSet();
Map<Long, Integer> numSinkOnBackend = Maps.newHashMap();
beToExecStates.clear();
// If #fragments >=2, use twoPhaseExecution with exec_plan_fragments_prepare and exec_plan_fragments_start,
// else use exec_plan_fragments directly.
@ -753,7 +753,7 @@ public class Coordinator implements CoordInterface {
states.addState(execState);
if (tParam.getFragment().getOutputSink() != null
&& tParam.getFragment().getOutputSink().getType() == TDataSinkType.OLAP_TABLE_SINK) {
backendsWithOlapTableSink.add(execState.backend.getId());
numSinkOnBackend.merge(execState.backend.getId(), 1, Integer::sum);
}
++backendIdx;
}
@ -765,7 +765,10 @@ public class Coordinator implements CoordInterface {
if (tParam.getFragment().getOutputSink() != null
&& tParam.getFragment().getOutputSink().getType() == TDataSinkType.OLAP_TABLE_SINK) {
tParam.setLoadStreamPerNode(loadStreamPerNode);
tParam.setTotalLoadStreams(backendsWithOlapTableSink.size() * loadStreamPerNode);
tParam.setTotalLoadStreams(numSinkOnBackend.size() * loadStreamPerNode);
tParam.setNumLocalSink(numSinkOnBackend.get(tParam.getBackendId()));
LOG.info("num local sink for backend {} is {}", tParam.getBackendId(),
numSinkOnBackend.get(tParam.getBackendId()));
}
}
profileFragmentId += 1;
@ -844,7 +847,7 @@ public class Coordinator implements CoordInterface {
}
}
Set<Long> backendsWithOlapTableSink = Sets.newHashSet();
int numBackendsWithSink = 0;
// 3. group PipelineExecContext by BE.
// So that we can use one RPC to send all fragment instances of a BE.
for (Map.Entry<TNetworkAddress, TPipelineFragmentParams> entry : tParams.entrySet()) {
@ -881,7 +884,7 @@ public class Coordinator implements CoordInterface {
if (entry.getValue().getFragment().getOutputSink() != null
&& entry.getValue().getFragment().getOutputSink().getType()
== TDataSinkType.OLAP_TABLE_SINK) {
backendsWithOlapTableSink.add(backendId);
numBackendsWithSink++;
}
++backendIdx;
}
@ -894,7 +897,10 @@ public class Coordinator implements CoordInterface {
&& entry.getValue().getFragment().getOutputSink().getType()
== TDataSinkType.OLAP_TABLE_SINK) {
entry.getValue().setLoadStreamPerNode(loadStreamPerNode);
entry.getValue().setTotalLoadStreams(backendsWithOlapTableSink.size() * loadStreamPerNode);
entry.getValue().setTotalLoadStreams(numBackendsWithSink * loadStreamPerNode);
entry.getValue().setNumLocalSink(entry.getValue().getLocalParams().size());
LOG.info("num local sink for backend {} is {}", entry.getValue().getBackendId(),
entry.getValue().getNumLocalSink());
}
}