database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](move-memtable) pass load stream num to backends (#26198)

Browse Source
This commit is contained in:
Kaijie Chen
2023-11-08 16:16:33 +08:00
committed by GitHub
parent 6637f9c15f
commit 58bf79f79e
25 changed files with 313 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
View File

@ -298,6 +298,8 @@ public class StreamLoadPlanner {
perNodeScanRange.put(scanNode.getId().asInt(), scanRangeParams);
execParams.setPerNodeScanRanges(perNodeScanRange);
params.setParams(execParams);
params.setLoadStreamPerNode(taskInfo.getStreamPerNode());
params.setTotalLoadStreams(taskInfo.getStreamPerNode());
TQueryOptions queryOptions = new TQueryOptions();
queryOptions.setQueryType(TQueryType.LOAD);
queryOptions.setQueryTimeout(timeout);
@ -499,6 +501,8 @@ public class StreamLoadPlanner {
pipParams.per_exch_num_senders = Maps.newHashMap();
pipParams.destinations = Lists.newArrayList();
pipParams.setNumSenders(1);
pipParams.setLoadStreamPerNode(taskInfo.getStreamPerNode());
pipParams.setTotalLoadStreams(taskInfo.getStreamPerNode());
TPipelineInstanceParams localParams = new TPipelineInstanceParams();
localParams.setFragmentInstanceId(new TUniqueId(loadId.hi, loadId.lo + fragmentInstanceIdIndex));

35
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -78,6 +78,7 @@ import org.apache.doris.system.SystemInfoService;
import org.apache.doris.task.LoadEtlTask;
import org.apache.doris.thrift.PaloInternalServiceVersion;
import org.apache.doris.thrift.TBrokerScanRange;
import org.apache.doris.thrift.TDataSinkType;
import org.apache.doris.thrift.TDescriptorTable;
import org.apache.doris.thrift.TDetailedReportParams;
import org.apache.doris.thrift.TErrorTabletInfo;
@ -688,6 +689,7 @@ public class Coordinator implements CoordInterface {
int backendIdx = 0;
int profileFragmentId = 0;
long memoryLimit = queryOptions.getMemLimit();
Set<Long> backendsWithOlapTableSink = Sets.newHashSet();
beToExecStates.clear();
// If #fragments >=2, use twoPhaseExecution with exec_plan_fragments_prepare and exec_plan_fragments_start,
// else use exec_plan_fragments directly.
@ -755,8 +757,23 @@ public class Coordinator implements CoordInterface {
beToExecStates.putIfAbsent(execState.backend.getId(), states);
}
states.addState(execState);
if (tParam.getFragment().getOutputSink() != null
&& tParam.getFragment().getOutputSink().getType() == TDataSinkType.OLAP_TABLE_SINK) {
backendsWithOlapTableSink.add(execState.backend.getId());
}
++backendIdx;
}
int loadStreamPerNode = 1;
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) {
loadStreamPerNode = ConnectContext.get().getSessionVariable().getLoadStreamPerNode();
}
for (TExecPlanFragmentParams tParam : tParams) {
if (tParam.getFragment().getOutputSink() != null
&& tParam.getFragment().getOutputSink().getType() == TDataSinkType.OLAP_TABLE_SINK) {
tParam.setLoadStreamPerNode(loadStreamPerNode);
tParam.setTotalLoadStreams(backendsWithOlapTableSink.size() * loadStreamPerNode);
}
}
profileFragmentId += 1;
} // end for fragments
@ -845,6 +862,7 @@ public class Coordinator implements CoordInterface {
}
}
Set<Long> backendsWithOlapTableSink = Sets.newHashSet();
// 3. group PipelineExecContext by BE.
// So that we can use one RPC to send all fragment instances of a BE.
for (Map.Entry<TNetworkAddress, TPipelineFragmentParams> entry : tParams.entrySet()) {
@ -878,8 +896,25 @@ public class Coordinator implements CoordInterface {
}
ctxs.addContext(pipelineExecContext);
if (entry.getValue().getFragment().getOutputSink() != null
&& entry.getValue().getFragment().getOutputSink().getType()
== TDataSinkType.OLAP_TABLE_SINK) {
backendsWithOlapTableSink.add(backendId);
}
++backendIdx;
}
int loadStreamPerNode = 1;
if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable() != null) {
loadStreamPerNode = ConnectContext.get().getSessionVariable().getLoadStreamPerNode();
}
for (Map.Entry<TNetworkAddress, TPipelineFragmentParams> entry : tParams.entrySet()) {
if (entry.getValue().getFragment().getOutputSink() != null
&& entry.getValue().getFragment().getOutputSink().getType()
== TDataSinkType.OLAP_TABLE_SINK) {
entry.getValue().setLoadStreamPerNode(loadStreamPerNode);
entry.getValue().setTotalLoadStreams(backendsWithOlapTableSink.size() * loadStreamPerNode);
}
}
profileFragmentId += 1;
} // end for fragments

13
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -411,6 +411,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_MEMTABLE_ON_SINK_NODE =
"enable_memtable_on_sink_node";
public static final String LOAD_STREAM_PER_NODE = "load_stream_per_node";
public static final String ENABLE_UNIQUE_KEY_PARTIAL_UPDATE = "enable_unique_key_partial_update";
public static final String INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD = "inverted_index_conjunction_opt_threshold";
@ -1236,6 +1238,9 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_MEMTABLE_ON_SINK_NODE, needForward = true)
public boolean enableMemtableOnSinkNode = false;
@VariableMgr.VarAttr(name = LOAD_STREAM_PER_NODE)
public int loadStreamPerNode = 20;
@VariableMgr.VarAttr(name = ENABLE_INSERT_GROUP_COMMIT)
public boolean enableInsertGroupCommit = false;
@ -2405,6 +2410,14 @@ public class SessionVariable implements Serializable, Writable {
this.enableUniqueKeyPartialUpdate = enableUniqueKeyPartialUpdate;
}
public int getLoadStreamPerNode() {
return loadStreamPerNode;
}
public void setLoadStreamPerNode(int loadStreamPerNode) {
this.loadStreamPerNode = loadStreamPerNode;
}
/**
* Serialize to thrift object.
* Used for rest api.

4
fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java
View File

@ -125,6 +125,10 @@ public interface LoadTaskInfo {
return false;
}
default int getStreamPerNode() {
return 20;
}
class ImportColumnDescs {
public List<ImportColumnDesc> descs = Lists.newArrayList();
public boolean isColumnDescsRewrited = false;

13
fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java
View File

@ -89,6 +89,7 @@ public class StreamLoadTask implements LoadTaskInfo {
private boolean enableProfile = false;
private boolean memtableOnSinkNode = false;
private int streamPerNode = 20;
private byte enclose = 0;
@ -309,6 +310,15 @@ public class StreamLoadTask implements LoadTaskInfo {
this.memtableOnSinkNode = memtableOnSinkNode;
}
@Override
public int getStreamPerNode() {
return streamPerNode;
}
public void setStreamPerNode(int streamPerNode) {
this.streamPerNode = streamPerNode;
}
public static StreamLoadTask fromTStreamLoadPutRequest(TStreamLoadPutRequest request) throws UserException {
StreamLoadTask streamLoadTask = new StreamLoadTask(request.getLoadId(), request.getTxnId(),
request.getFileType(), request.getFormatType(),
@ -447,6 +457,9 @@ public class StreamLoadTask implements LoadTaskInfo {
if (request.isSetMemtableOnSinkNode()) {
this.memtableOnSinkNode = request.isMemtableOnSinkNode();
}
if (request.isSetStreamPerNode()) {
this.streamPerNode = request.getStreamPerNode();
}
}
// used for stream load