database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](load) make default load mem limit configurable (#12348)

Browse Source 
* make LoadMemLimit valid for broker load, stream load and routine load

Co-authored-by: wuhangze <wuhangze@jd.com>
This commit is contained in:
Henry2SS
2022-09-12 10:25:01 +08:00
committed by GitHub
parent fc605779ed
commit ecfefae715
7 changed files with 14 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/load/Load.java
View File

@ -296,7 +296,7 @@ public class Load {
// resource info
if (ConnectContext.get() != null) {
job.setResourceInfo(ConnectContext.get().toResourceCtx());
job.setExecMemLimit(ConnectContext.get().getSessionVariable().getMaxExecMemByte());
job.setExecMemLimit(ConnectContext.get().getSessionVariable().getLoadMemLimit());
}
// job properties

2
fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
View File

@ -289,6 +289,8 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
}
if (stmt.getExecMemLimit() != -1) {
this.execMemLimit = stmt.getExecMemLimit();
} else if (ConnectContext.get() != null) {
this.execMemLimit = ConnectContext.get().getSessionVariable().getLoadMemLimit();
}
if (stmt.getSendBatchParallelism() > 0) {
this.sendBatchParallelism = stmt.getSendBatchParallelism();

2
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -382,7 +382,7 @@ public class SessionVariable implements Serializable, Writable {
public boolean forwardToMaster = true;
@VariableMgr.VarAttr(name = LOAD_MEM_LIMIT)
public long loadMemLimit = 0L;
public long loadMemLimit = 2 * 1024 * 1024 * 1024L; // 2GB as default
@VariableMgr.VarAttr(name = USE_V2_ROLLUP)
public boolean useV2Rollup = false;

3
fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java
View File

@ -30,6 +30,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.load.loadv2.LoadTask;
import org.apache.doris.qe.VariableMgr;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TFileType;
import org.apache.doris.thrift.TStreamLoadPutRequest;
@ -290,6 +291,8 @@ public class StreamLoadTask implements LoadTaskInfo {
}
if (request.isSetExecMemLimit()) {
execMemLimit = request.getExecMemLimit();
} else {
execMemLimit = VariableMgr.getDefaultSessionVariable().getLoadMemLimit();
}
if (request.getFormatType() == TFileFormatType.FORMAT_JSON) {
if (request.getJsonpaths() != null) {