database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](broker-load) can't load parquet file with column name case sensitive with Doris column (#9358)

Browse Source
This commit is contained in:
deardeng
2022-05-11 22:27:03 -07:00
committed by GitHub
parent 122cc3b772
commit cfbf13710b
3 changed files with 13 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/load/Load.java
View File

@ -1049,8 +1049,12 @@ public class Load {
for (ImportColumnDesc importColumnDesc : copiedColumnExprs) {
// make column name case match with real column name
String columnName = importColumnDesc.getColumnName();
String realColName = tbl.getColumn(columnName) == null ? columnName
: tbl.getColumn(columnName).getName();
String realColName;
if (tbl.getColumn(columnName) == null || importColumnDesc.getExpr() == null) {
realColName = columnName;
} else {
realColName = tbl.getColumn(columnName).getName();
}
if (importColumnDesc.getExpr() != null) {
Expr expr = transformHadoopFunctionExpr(tbl, realColName, importColumnDesc.getExpr());
exprsByName.put(realColName, expr);

4
fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java
View File

@ -247,8 +247,8 @@ public class BrokerScanNode extends LoadScanNode {
*/
private void initColumns(ParamCreateContext context) throws UserException {
context.srcTupleDescriptor = analyzer.getDescTbl().createTupleDescriptor();
context.slotDescByName = Maps.newHashMap();
context.exprMap = Maps.newHashMap();
context.slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
context.exprMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
// for load job, column exprs is got from file group
// for query, there is no column exprs, they will be got from table's schema in "Load.initColumns"

7
fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java
View File

@ -66,8 +66,11 @@ public class StreamLoadScanNode extends LoadScanNode {
private TupleDescriptor srcTupleDesc;
private TBrokerScanRange brokerScanRange;
private Map<String, SlotDescriptor> slotDescByName = Maps.newHashMap();
private Map<String, Expr> exprsByName = Maps.newHashMap();
// If use case sensitive map, for example,
// the column name 「A」 in the table and the mapping '(a) set (A = a)' in load sql，
// Slotdescbyname stores「a」, later will use 「a」to get table's 「A」 column info, will throw exception.
private final Map<String, SlotDescriptor> slotDescByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
private final Map<String, Expr> exprsByName = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
// used to construct for streaming loading
public StreamLoadScanNode(