database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

branch-2.1: [fix](hudi) check requiredFields exists in splitHudiColumnNames in HudiJniScanner #51430 (#51504)

Browse Source 
Cherry-picked from #51430

Co-authored-by: Socrates <suyiteng@selectdb.com>
This commit is contained in:
github-actions[bot]
2025-06-16 14:54:40 +08:00
committed by GitHub
parent 56d87c7f4d
commit c6ff4994f3
1 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java
View File

@ -170,9 +170,6 @@ public class HadoopHudiJniScanner extends JniScanner {
for (int i = 0; i < fields.length; i++) {
Object fieldData = rowInspector.getStructFieldData(rowData, structFields[i]);
columnValue.setRow(fieldData);
// LOG.info("rows: {}, column: {}, col name: {}, col type: {}, inspector: {}",
// numRows, i, types[i].getName(), types[i].getType().name(),
// fieldInspectors[i].getTypeName());
columnValue.setField(types[i], fieldInspectors[i]);
appendData(i, columnValue);
}
@ -211,9 +208,15 @@ public class HadoopHudiJniScanner extends JniScanner {
.boxed()
.collect(Collectors.toMap(i -> splitHudiColumnNames[i], i -> hudiColumnTypes[i]));
requiredTypes = Arrays.stream(requiredFields)
.map(field -> ColumnType.parseType(field, hudiColNameToType.get(field)))
.toArray(ColumnType[]::new);
requiredTypes = new ColumnType[requiredFields.length];
for (int i = 0; i < requiredFields.length; i++) {
String requiredField = requiredFields[i];
if (!hudiColNameToType.containsKey(requiredField)) {
throw new IllegalArgumentException(
"Required field " + requiredField + " not found in Hudi column names: " + splitHudiColumnNames);
}
requiredTypes[i] = ColumnType.parseType(requiredField, hudiColNameToType.get(requiredField));
}
requiredColumnIds = Arrays.stream(requiredFields)
.mapToInt(hudiColNameToIdx::get)