database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](load)Pass hidden column to load columns (#17004)

Browse Source 
The LoadScanProvider doesn't get Hidden Columns from stream load parameter.
This may cause stream load delete operation fail. This pr is to pass the hidden columns to LoadScanProvider.
This commit is contained in:
Jibing-Li
2023-02-23 13:54:36 +08:00
committed by GitHub
parent 37960e83d3
commit bc619ce5be
4 changed files with 14 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
View File

@ -207,7 +207,7 @@ public class StreamLoadPlanner {
fileStatus.setSize(-1); // must set to -1, means stream.
}
fileScanNode.setLoadInfo(loadId, taskInfo.getTxnId(), destTable, BrokerDesc.createForStreamLoad(),
fileGroup, fileStatus, taskInfo.isStrictMode(), taskInfo.getFileType());
fileGroup, fileStatus, taskInfo.isStrictMode(), taskInfo.getFileType(), taskInfo.getHiddenColumns());
scanNode = fileScanNode;
scanNode.init(analyzer);

5
fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java vendored
View File

@ -158,9 +158,10 @@ public class ExternalFileScanNode extends ExternalScanNode {
// Only for stream load/routine load job.
public void setLoadInfo(TUniqueId loadId, long txnId, Table targetTable, BrokerDesc brokerDesc,
BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, TFileType fileType) {
BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, TFileType fileType,
List<String> hiddenColumns) {
FileGroupInfo fileGroupInfo = new FileGroupInfo(loadId, txnId, targetTable, brokerDesc,
fileGroup, fileStatus, strictMode, fileType);
fileGroup, fileStatus, strictMode, fileType, hiddenColumns);
fileGroupInfos.add(fileGroupInfo);
this.type = Type.LOAD;
}

9
fe/fe-core/src/main/java/org/apache/doris/planner/external/FileGroupInfo.java vendored
View File

@ -84,6 +84,7 @@ public class FileGroupInfo {
private long bytesPerInstance = 0;
// used for stream load, FILE_LOCAL or FILE_STREAM
private TFileType fileType;
private List<String> hiddenColumns = null;
// for broker load
public FileGroupInfo(long loadJobId, long txnId, Table targetTable, BrokerDesc brokerDesc,
@ -104,7 +105,8 @@ public class FileGroupInfo {
// for stream load
public FileGroupInfo(TUniqueId loadId, long txnId, Table targetTable, BrokerDesc brokerDesc,
BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, TFileType fileType) {
BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode,
TFileType fileType, List<String> hiddenColumns) {
this.jobType = JobType.STREAM_LOAD;
this.loadId = loadId;
this.txnId = txnId;
@ -116,6 +118,7 @@ public class FileGroupInfo {
this.filesAdded = 1;
this.strictMode = strictMode;
this.fileType = fileType;
this.hiddenColumns = hiddenColumns;
}
public Table getTargetTable() {
@ -152,6 +155,10 @@ public class FileGroupInfo {
return sb.toString();
}
public List<String> getHiddenColumns() {
return hiddenColumns;
}
public void getFileStatusAndCalcInstance(BackendPolicy backendPolicy) throws UserException {
if (filesAdded == 0) {
throw new UserException("No source file in this table(" + targetTable.getName() + ").");

3
fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java vendored
View File

@ -206,7 +206,8 @@ public class LoadScanProvider implements FileScanProviderIf {
List<Integer> srcSlotIds = Lists.newArrayList();
Load.initColumns(fileGroupInfo.getTargetTable(), columnDescs, context.fileGroup.getColumnToHadoopFunction(),
context.exprMap, analyzer, context.srcTupleDescriptor, context.srcSlotDescByName, srcSlotIds,
formatType(context.fileGroup.getFileFormat(), ""), null, VectorizedUtil.isVectorized());
formatType(context.fileGroup.getFileFormat(), ""), fileGroupInfo.getHiddenColumns(),
VectorizedUtil.isVectorized());
int columnCountFromPath = 0;
if (context.fileGroup.getColumnNamesFromPath() != null) {