database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](multi-catalog)add the FAQ for Aliyun DLF and add the fs.xx.impl check (#25594)

Browse Source 
1. add the FAQ for Aliyun DLF，include the DLF configuration and Jindo SDK locations
2. add the fs.xx.impl check when load cache files
This commit is contained in:
slothever
2023-10-31 17:09:52 +08:00
committed by GitHub
parent a698e14fb2
commit 994db8b400
6 changed files with 57 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
View File

@ -406,8 +406,10 @@ public class HiveMetaStoreCache {
if (uri.getScheme() != null) {
String scheme = uri.getScheme();
updateJobConf("fs." + scheme + ".impl.disable.cache", "true");
if (!scheme.equals("hdfs") && !scheme.equals("viewfs")) {
updateJobConf("fs." + scheme + ".impl", PropertyConverter.getHadoopFSImplByScheme(scheme));
if (jobConf.get("fs." + scheme + ".impl") == null) {
if (!scheme.equals("hdfs") && !scheme.equals("viewfs")) {
updateJobConf("fs." + scheme + ".impl", PropertyConverter.getHadoopFSImplByScheme(scheme));
}
}
}
} catch (Exception e) {

7
fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
View File

@ -332,8 +332,8 @@ public class PropertyConverter {
region + ".oss-dls.aliyuncs.com");
}
}
ossProperties.put("fs.oss.impl", "com.aliyun.emr.fs.oss.JindoOssFileSystem");
ossProperties.put("fs.AbstractFileSystem.oss.impl", "com.aliyun.emr.fs.oss.OSS");
ossProperties.put("fs.oss.impl", "com.aliyun.jindodata.oss.JindoOssFileSystem");
ossProperties.put("fs.AbstractFileSystem.oss.impl", "com.aliyun.jindodata.oss.OSS");
}
private static Map<String, String> convertToCOSProperties(Map<String, String> props, CloudCredential credential) {
@ -454,7 +454,8 @@ public class PropertyConverter {
if (!Strings.isNullOrEmpty(region)) {
boolean hdfsEnabled = Boolean.parseBoolean(props.getOrDefault(OssProperties.OSS_HDFS_ENABLED, "false"));
if (hdfsEnabled) {
props.putIfAbsent("fs.oss.impl", "com.aliyun.emr.fs.oss.JindoOssFileSystem");
props.putIfAbsent("fs.oss.impl", "com.aliyun.jindodata.oss.JindoOssFileSystem");
props.put("fs.AbstractFileSystem.oss.impl", "com.aliyun.jindodata.oss.OSS");
props.putIfAbsent(OssProperties.REGION, region);
// example: cn-shanghai.oss-dls.aliyuncs.com
// from https://www.alibabacloud.com/help/en/e-mapreduce/latest/oss-kusisurumen