database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](insert) refactor group commit insert into (#25795)

Browse Source
This commit is contained in:
meiyi
2023-11-03 12:02:40 +08:00
committed by GitHub
parent 61edb9e4d8
commit b19f275714
11 changed files with 182 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
View File

@ -53,18 +53,18 @@ import org.apache.doris.planner.GroupCommitOlapTableSink;
import org.apache.doris.planner.OlapTableSink;
import org.apache.doris.planner.StreamLoadPlanner;
import org.apache.doris.planner.external.jdbc.JdbcTableSink;
import org.apache.doris.proto.InternalService;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.tablefunction.GroupCommitTableValuedFunction;
import org.apache.doris.task.StreamLoadTask;
import org.apache.doris.thrift.TExecPlanFragmentParams;
import org.apache.doris.thrift.TExecPlanFragmentParamsList;
import org.apache.doris.thrift.TFileCompressType;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.thrift.TFileType;
import org.apache.doris.thrift.TMergeType;
import org.apache.doris.thrift.TPlan;
import org.apache.doris.thrift.TPlanFragment;
import org.apache.doris.thrift.TQueryOptions;
import org.apache.doris.thrift.TScanRangeParams;
import org.apache.doris.thrift.TStreamLoadPutRequest;
@ -162,9 +162,7 @@ public class NativeInsertStmt extends InsertStmt {
private boolean isGroupCommit = false;
private int baseSchemaVersion = -1;
private TUniqueId loadId = null;
private ByteString planBytes = null;
private ByteString tableBytes = null;
private ByteString rangeBytes = null;
private ByteString execPlanFragmentParamsBytes = null;
private long tableId = -1;
// true if be generates an insert from group commit tvf stmt and executes to load data
public boolean isGroupCommitTvf = false;
@ -1074,10 +1072,35 @@ public class NativeInsertStmt extends InsertStmt {
}
private void analyzeGroupCommit() {
if (ConnectContext.get().getSessionVariable().enableInsertGroupCommit && targetTable instanceof OlapTable
if (ConnectContext.get().getSessionVariable().enableInsertGroupCommit
&& targetTable instanceof OlapTable
&& !ConnectContext.get().isTxnModel()
&& getQueryStmt() instanceof SelectStmt
&& ((SelectStmt) getQueryStmt()).getTableRefs().isEmpty() && targetPartitionNames == null) {
&& ((SelectStmt) getQueryStmt()).getTableRefs().isEmpty() && targetPartitionNames == null
&& (label == null || Strings.isNullOrEmpty(label.getLabelName()))
&& (analyzer == null || analyzer != null && !analyzer.isReAnalyze())) {
SelectStmt selectStmt = (SelectStmt) queryStmt;
if (selectStmt.getValueList() != null) {
for (List<Expr> row : selectStmt.getValueList().getRows()) {
for (Expr expr : row) {
if (!expr.isLiteralOrCastExpr()) {
return;
}
}
}
} else {
SelectList selectList = selectStmt.getSelectList();
if (selectList != null) {
List<SelectListItem> items = selectList.getItems();
if (items != null) {
for (SelectListItem item : items) {
if (item.getExpr() != null && !item.getExpr().isLiteralOrCastExpr()) {
return;
}
}
}
}
}
isGroupCommit = true;
}
}
@ -1088,7 +1111,7 @@ public class NativeInsertStmt extends InsertStmt {
public void planForGroupCommit(TUniqueId queryId) throws UserException, TException {
OlapTable olapTable = (OlapTable) getTargetTable();
if (planBytes != null && olapTable.getBaseSchemaVersion() == baseSchemaVersion) {
if (execPlanFragmentParamsBytes != null && olapTable.getBaseSchemaVersion() == baseSchemaVersion) {
return;
}
if (!targetColumns.isEmpty()) {
@ -1110,9 +1133,6 @@ public class NativeInsertStmt extends InsertStmt {
StreamLoadPlanner planner = new StreamLoadPlanner((Database) getDbObj(), olapTable, streamLoadTask);
// Will using load id as query id in fragment
TExecPlanFragmentParams tRequest = planner.plan(streamLoadTask.getId());
DescriptorTable descTable = planner.getDescTable();
TPlanFragment fragment = tRequest.getFragment();
TPlan plan = fragment.getPlan();
for (Map.Entry<Integer, List<TScanRangeParams>> entry : tRequest.params.per_node_scan_ranges.entrySet()) {
for (TScanRangeParams scanRangeParams : entry.getValue()) {
scanRangeParams.scan_range.ext_scan_range.file_scan_range.params.setFormatType(
@ -1126,32 +1146,26 @@ public class NativeInsertStmt extends InsertStmt {
Preconditions.checkState(scanRangeParams.size() == 1);
// save plan message to be reused for prepare stmt
loadId = queryId;
planBytes = ByteString.copyFrom(new TSerializer().serialize(plan));
tableBytes = ByteString.copyFrom(new TSerializer().serialize(descTable.toThrift()));
rangeBytes = ByteString.copyFrom(new TSerializer().serialize(scanRangeParams.get(0)));
baseSchemaVersion = olapTable.getBaseSchemaVersion();
// see BackendServiceProxy#execPlanFragmentsAsync
TExecPlanFragmentParamsList paramsList = new TExecPlanFragmentParamsList();
paramsList.addToParamsList(tRequest);
execPlanFragmentParamsBytes = ByteString.copyFrom(new TSerializer().serialize(paramsList));
}
public InternalService.PExecPlanFragmentRequest getExecPlanFragmentRequest() {
return InternalService.PExecPlanFragmentRequest.newBuilder().setRequest(execPlanFragmentParamsBytes)
.setCompact(false).setVersion(InternalService.PFragmentRequestVersion.VERSION_2).build();
}
public TUniqueId getLoadId() {
return loadId;
}
public ByteString getPlanBytes() {
return planBytes;
}
public ByteString getTableBytes() {
return tableBytes;
}
public int getBaseSchemaVersion() {
return baseSchemaVersion;
}
public ByteString getRangeBytes() {
return rangeBytes;
}
public void setIsFromDeleteOrUpdateStmt(boolean isFromDeleteOrUpdateStmt) {
this.isFromDeleteOrUpdateStmt = isFromDeleteOrUpdateStmt;
}

4
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -1845,10 +1845,8 @@ public class StmtExecutor {
PGroupCommitInsertRequest request = PGroupCommitInsertRequest.newBuilder()
.setDbId(insertStmt.getTargetTable().getDatabase().getId())
.setTableId(insertStmt.getTargetTable().getId())
.setDescTbl(nativeInsertStmt.getTableBytes())
.setBaseSchemaVersion(nativeInsertStmt.getBaseSchemaVersion())
.setPlanNode(nativeInsertStmt.getPlanBytes())
.setScanRangeParams(nativeInsertStmt.getRangeBytes())
.setExecPlanFragmentRequest(nativeInsertStmt.getExecPlanFragmentRequest())
.setLoadId(Types.PUniqueId.newBuilder().setHi(loadId.hi).setLo(loadId.lo)
.build()).addAllData(rows)
.build();