database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improvement](multicatalog) support read tencent dlc table on lakefs (#36891)

Browse Source 
bp #36823
This commit is contained in:
Yulei-Yang
2024-06-27 14:03:48 +08:00
committed by GitHub
parent 22cb7b8fcb
commit 8a1ebba1cc
4 changed files with 19 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
View File

@ -71,6 +71,7 @@ public class FeConstants {
public static final String FS_PREFIX_BOS = "bos";
public static final String FS_PREFIX_COS = "cos";
public static final String FS_PREFIX_COSN = "cosn";
public static final String FS_PREFIX_LAKEFS = "lakefs";
public static final String FS_PREFIX_OBS = "obs";
public static final String FS_PREFIX_OFS = "ofs";
public static final String FS_PREFIX_GFS = "gfs";

15
fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
View File

@ -62,6 +62,7 @@ public class LocationPath {
COSN, // Tencent
OFS, // Tencent CHDFS
GFS, // Tencent GooseFs,
LAKEFS, // used by Tencent DLC
OSS, // Alibaba,
OSS_HDFS, // JindoFS on OSS
JFS, // JuiceFS,
@ -163,6 +164,10 @@ public class LocationPath {
locationType = LocationType.COSN;
this.location = location;
break;
case FeConstants.FS_PREFIX_LAKEFS:
locationType = LocationType.COSN;
this.location = normalizedLakefsPath(location);
break;
case FeConstants.FS_PREFIX_VIEWFS:
locationType = LocationType.VIEWFS;
this.location = location;
@ -277,6 +282,15 @@ public class LocationPath {
}
}
private static String normalizedLakefsPath(String location) {
int atIndex = location.indexOf("@dlc");
if (atIndex != -1) {
return "lakefs://" + location.substring(atIndex + 1);
} else {
return location;
}
}
public static Pair<FileSystemType, String> getFSIdentity(String location, String bindBrokerName) {
LocationPath locationPath = new LocationPath(location);
FileSystemType fsType = (bindBrokerName != null) ? FileSystemType.BROKER : locationPath.getFileSystemType();
@ -351,6 +365,7 @@ public class LocationPath {
case GCS:
// ATTN, for COSN, on FE side, use HadoopFS to access, but on BE, use S3 client to access.
case COSN:
case LAKEFS:
// now we only support S3 client for object storage on BE
return TFileType.FILE_S3;
case HDFS:

3
fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
View File

@ -188,7 +188,7 @@ public class PropertyConverter {
return OBSFileSystem.class.getName();
} else if (fsScheme.equalsIgnoreCase("oss")) {
return AliyunOSSFileSystem.class.getName();
} else if (fsScheme.equalsIgnoreCase("cosn")) {
} else if (fsScheme.equalsIgnoreCase("cosn") || fsScheme.equalsIgnoreCase("lakefs")) {
return CosFileSystem.class.getName();
} else {
return S3AFileSystem.class.getName();
@ -354,6 +354,7 @@ public class PropertyConverter {
cosProperties.put(CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY, props.get(CosProperties.ENDPOINT));
cosProperties.put("fs.cosn.impl.disable.cache", "true");
cosProperties.put("fs.cosn.impl", getHadoopFSImplByScheme("cosn"));
cosProperties.put("fs.lakefs.impl", getHadoopFSImplByScheme("lakefs"));
if (credential.isWhole()) {
cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_ID_KEY, credential.getAccessKey());
cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_KEY_KEY, credential.getSecretKey());

2
fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
View File

@ -470,7 +470,7 @@ public class PropertyConverterTest extends TestWithFeService {
+ " 'cos.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName0, CosProperties.COS_PREFIX,
"cos.ap-beijing.myqcloud.com", query0, 12, 17);
"cos.ap-beijing.myqcloud.com", query0, 12, 18);
String catalogName1 = "hms_oss";
String query1 = "create catalog " + catalogName1 + " properties (\n"