database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](hive) report error with escape char and null format (#39700) (#39869)

Browse Source 
bp #39700

Co-authored-by: Socrates <suxiaogang223@icloud.com>
This commit is contained in:
Mingyu Chen
2024-08-24 09:23:03 +08:00
committed by GitHub
parent a6f267c479
commit b9da934b16
3 changed files with 65 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
View File

@ -95,6 +95,11 @@ public class HiveScanNode extends FileQueryScanNode {
public static final String PROP_MAP_KV_DELIMITER = "mapkey.delim";
public static final String DEFAULT_MAP_KV_DELIMITER = "\003";
public static final String PROP_ESCAPE_DELIMITER = "escape.delim";
public static final String DEFAULT_ESCAPE_DELIMIER = "\\";
public static final String PROP_NULL_FORMAT = "serialization.null.format";
public static final String DEFAULT_NULL_FORMAT = "\\N";
protected final HMSExternalTable hmsTable;
private HiveTransaction hiveTransaction = null;
@ -476,6 +481,21 @@ public class HiveScanNode extends FileQueryScanNode {
textParams.setEnclose(serdeParams.get(PROP_QUOTE_CHAR).getBytes()[0]);
}
// TODO: support escape char and null format in csv_reader
Optional<String> escapeChar = HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(),
PROP_ESCAPE_DELIMITER);
if (escapeChar.isPresent() && !escapeChar.get().equals(DEFAULT_ESCAPE_DELIMIER)) {
throw new UserException(
"not support serde prop " + PROP_ESCAPE_DELIMITER + " in hive text reading");
}
Optional<String> nullFormat = HiveMetaStoreClientHelper.getSerdeProperty(hmsTable.getRemoteTable(),
PROP_NULL_FORMAT);
if (nullFormat.isPresent() && !nullFormat.get().equals(DEFAULT_NULL_FORMAT)) {
throw new UserException(
"not support serde prop " + PROP_NULL_FORMAT + " in hive text reading");
}
TFileAttributes fileAttributes = new TFileAttributes();
fileAttributes.setTextParams(textParams);
fileAttributes.setHeaderType("");