database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improve](move-memtbale) enable move memtable in routine load (#28974)

Browse Source
This commit is contained in:
HHoflittlefish777
2024-01-06 18:22:01 +08:00
committed by GitHub
parent 85dd606fd1
commit db17f5fe79
6 changed files with 25 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java
View File

@ -108,6 +108,7 @@ public class KafkaTaskInfo extends RoutineLoadTaskInfo {
} else {
tRoutineLoadTask.setFormat(TFileFormatType.FORMAT_CSV_PLAIN);
}
tRoutineLoadTask.setMemtableOnSinkNode(routineLoadJob.isMemtableOnSinkNode());
return tRoutineLoadTask;
}

18
fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
View File

@ -202,6 +202,8 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
protected String sequenceCol;
protected boolean memtableOnSinkNode = false;
/**
* RoutineLoad support json data.
* Require Params:
@ -268,6 +270,9 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
public RoutineLoadJob(long id, LoadDataSourceType type) {
this.id = id;
this.dataSourceType = type;
if (ConnectContext.get() != null) {
this.memtableOnSinkNode = ConnectContext.get().getSessionVariable().enableMemtableOnSinkNode;
}
}
public RoutineLoadJob(Long id, String name,
@ -283,6 +288,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
if (ConnectContext.get() != null) {
SessionVariable var = ConnectContext.get().getSessionVariable();
sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode()));
this.memtableOnSinkNode = ConnectContext.get().getSessionVariable().enableMemtableOnSinkNode;
} else {
sessionVariables.put(SessionVariable.SQL_MODE, String.valueOf(SqlModeHelper.MODE_DEFAULT));
}
@ -304,6 +310,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
if (ConnectContext.get() != null) {
SessionVariable var = ConnectContext.get().getSessionVariable();
sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode()));
this.memtableOnSinkNode = ConnectContext.get().getSessionVariable().enableMemtableOnSinkNode;
} else {
sessionVariables.put(SessionVariable.SQL_MODE, String.valueOf(SqlModeHelper.MODE_DEFAULT));
}
@ -686,6 +693,15 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
return !Strings.isNullOrEmpty(sequenceCol);
}
@Override
public boolean isMemtableOnSinkNode() {
return memtableOnSinkNode;
}
public void setMemtableOnSinkNode(boolean memtableOnSinkNode) {
this.memtableOnSinkNode = memtableOnSinkNode;
}
public void setComment(String comment) {
this.comment = comment;
}
@ -874,11 +890,11 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback impl
}
private void initPlanner() throws UserException {
Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(dbId);
// for multi table load job, the table name is dynamic,we will set table when task scheduling.
if (isMultiTable) {
return;
}
Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(dbId);
planner = new StreamLoadPlanner(db,
(OlapTable) db.getTableOrMetaException(this.tableId, Table.TableType.OLAP), this);
}