From bc619ce5bea20541dfdd26a62f57089ad5cd16fb Mon Sep 17 00:00:00 2001
From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com>
Date: Thu, 23 Feb 2023 13:54:36 +0800
Subject: [PATCH] [Fix](load)Pass hidden column to load columns (#17004)

The LoadScanProvider doesn't get Hidden Columns from stream load parameter.
This may cause stream load delete operation fail. This pr is to pass the hidden columns to LoadScanProvider.
---
 .../java/org/apache/doris/planner/StreamLoadPlanner.java | 2 +-
 .../doris/planner/external/ExternalFileScanNode.java     | 5 +++--
 .../org/apache/doris/planner/external/FileGroupInfo.java | 9 ++++++++-
 .../apache/doris/planner/external/LoadScanProvider.java  | 3 ++-
 4 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
index 692e5c755b..9b5e15da75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java
@@ -207,7 +207,7 @@ public class StreamLoadPlanner {
             fileStatus.setSize(-1); // must set to -1, means stream.
         }
         fileScanNode.setLoadInfo(loadId, taskInfo.getTxnId(), destTable, BrokerDesc.createForStreamLoad(),
-                fileGroup, fileStatus, taskInfo.isStrictMode(), taskInfo.getFileType());
+                fileGroup, fileStatus, taskInfo.isStrictMode(), taskInfo.getFileType(), taskInfo.getHiddenColumns());
         scanNode = fileScanNode;
 
         scanNode.init(analyzer);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
index 68e75640f9..1d7b81c6f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
@@ -158,9 +158,10 @@ public class ExternalFileScanNode extends ExternalScanNode {
 
     // Only for stream load/routine load job.
     public void setLoadInfo(TUniqueId loadId, long txnId, Table targetTable, BrokerDesc brokerDesc,
-            BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, TFileType fileType) {
+            BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, TFileType fileType,
+            List<String> hiddenColumns) {
         FileGroupInfo fileGroupInfo = new FileGroupInfo(loadId, txnId, targetTable, brokerDesc,
-                fileGroup, fileStatus, strictMode, fileType);
+                fileGroup, fileStatus, strictMode, fileType, hiddenColumns);
         fileGroupInfos.add(fileGroupInfo);
         this.type = Type.LOAD;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileGroupInfo.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileGroupInfo.java
index 339fc663d9..d7d90dbec6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileGroupInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileGroupInfo.java
@@ -84,6 +84,7 @@ public class FileGroupInfo {
     private long bytesPerInstance = 0;
     // used for stream load, FILE_LOCAL or FILE_STREAM
     private TFileType fileType;
+    private List<String> hiddenColumns = null;
 
     // for broker load
     public FileGroupInfo(long loadJobId, long txnId, Table targetTable, BrokerDesc brokerDesc,
@@ -104,7 +105,8 @@ public class FileGroupInfo {
 
     // for stream load
     public FileGroupInfo(TUniqueId loadId, long txnId, Table targetTable, BrokerDesc brokerDesc,
-            BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode, TFileType fileType) {
+            BrokerFileGroup fileGroup, TBrokerFileStatus fileStatus, boolean strictMode,
+            TFileType fileType, List<String> hiddenColumns) {
         this.jobType = JobType.STREAM_LOAD;
         this.loadId = loadId;
         this.txnId = txnId;
@@ -116,6 +118,7 @@ public class FileGroupInfo {
         this.filesAdded = 1;
         this.strictMode = strictMode;
         this.fileType = fileType;
+        this.hiddenColumns = hiddenColumns;
     }
 
     public Table getTargetTable() {
@@ -152,6 +155,10 @@ public class FileGroupInfo {
         return sb.toString();
     }
 
+    public List<String> getHiddenColumns() {
+        return hiddenColumns;
+    }
+
     public void getFileStatusAndCalcInstance(BackendPolicy backendPolicy) throws UserException {
         if (filesAdded == 0) {
             throw new UserException("No source file in this table(" + targetTable.getName() + ").");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java
index 5a5dd7b061..d8e644e277 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/LoadScanProvider.java
@@ -206,7 +206,8 @@ public class LoadScanProvider implements FileScanProviderIf {
         List<Integer> srcSlotIds = Lists.newArrayList();
         Load.initColumns(fileGroupInfo.getTargetTable(), columnDescs, context.fileGroup.getColumnToHadoopFunction(),
                 context.exprMap, analyzer, context.srcTupleDescriptor, context.srcSlotDescByName, srcSlotIds,
-                formatType(context.fileGroup.getFileFormat(), ""), null, VectorizedUtil.isVectorized());
+                formatType(context.fileGroup.getFileFormat(), ""), fileGroupInfo.getHiddenColumns(),
+                VectorizedUtil.isVectorized());
 
         int columnCountFromPath = 0;
         if (context.fileGroup.getColumnNamesFromPath() != null) {