From cfbf13710bbd895a9e66a813c88d7f019f425fa8 Mon Sep 17 00:00:00 2001 From: deardeng <565620795@qq.com> Date: Wed, 11 May 2022 22:27:03 -0700 Subject: [PATCH] [fix](broker-load) can't load parquet file with column name case sensitive with Doris column (#9358) --- fe/fe-core/src/main/java/org/apache/doris/load/Load.java | 8 ++++++-- .../java/org/apache/doris/planner/BrokerScanNode.java | 4 ++-- .../java/org/apache/doris/planner/StreamLoadScanNode.java | 7 +++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index 905f950737..ea381977e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -1049,8 +1049,12 @@ public class Load { for (ImportColumnDesc importColumnDesc : copiedColumnExprs) { // make column name case match with real column name String columnName = importColumnDesc.getColumnName(); - String realColName = tbl.getColumn(columnName) == null ? columnName - : tbl.getColumn(columnName).getName(); + String realColName; + if (tbl.getColumn(columnName) == null || importColumnDesc.getExpr() == null) { + realColName = columnName; + } else { + realColName = tbl.getColumn(columnName).getName(); + } if (importColumnDesc.getExpr() != null) { Expr expr = transformHadoopFunctionExpr(tbl, realColName, importColumnDesc.getExpr()); exprsByName.put(realColName, expr); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java index a7b9493b88..6f004f2c9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -247,8 +247,8 @@ public class BrokerScanNode extends LoadScanNode { */ private void initColumns(ParamCreateContext context) throws UserException { context.srcTupleDescriptor = analyzer.getDescTbl().createTupleDescriptor(); - context.slotDescByName = Maps.newHashMap(); - context.exprMap = Maps.newHashMap(); + context.slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + context.exprMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); // for load job, column exprs is got from file group // for query, there is no column exprs, they will be got from table's schema in "Load.initColumns" diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 0930bad5f0..65a265d9c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -66,8 +66,11 @@ public class StreamLoadScanNode extends LoadScanNode { private TupleDescriptor srcTupleDesc; private TBrokerScanRange brokerScanRange; - private Map slotDescByName = Maps.newHashMap(); - private Map exprsByName = Maps.newHashMap(); + // If use case sensitive map, for example, + // the column name 「A」 in the table and the mapping '(a) set (A = a)' in load sql, + // Slotdescbyname stores「a」, later will use 「a」to get table's 「A」 column info, will throw exception. + private final Map slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + private final Map exprsByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); // used to construct for streaming loading public StreamLoadScanNode(