database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](multicatlog) fix read hive/iceberg catalog on cosn & fix read data via broker (#22087)

Browse Source 
* [fix](multicatlog) fix read hive/iceberg catalog on cosn & fix read data via broker

* Update FileSystemFactory.java
This commit is contained in:
Yulei-Yang
2023-08-10 14:44:53 +08:00
committed by GitHub
parent f2658dc7bd
commit f7d00d467a
4 changed files with 18 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
View File

@ -80,6 +80,7 @@ public class FeConstants {
public static String FS_PREFIX_GCS = "gs";
public static String FS_PREFIX_BOS = "bos";
public static String FS_PREFIX_COS = "cos";
public static String FS_PREFIX_COSN = "cosn";
public static String FS_PREFIX_OBS = "obs";
public static String FS_PREFIX_OFS = "ofs";
public static String FS_PREFIX_GFS = "gfs";

7
fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
View File

@ -53,9 +53,10 @@ public class S3Util {
public static boolean isObjStorage(String location) {
return isObjStorageUseS3Client(location)
|| location.startsWith(FeConstants.FS_PREFIX_COS)
|| location.startsWith(FeConstants.FS_PREFIX_OSS)
|| location.startsWith(FeConstants.FS_PREFIX_OBS);
// if treat cosn(tencent hadoop-cos) as a s3 file system, may bring incompatible issues
|| (location.startsWith(FeConstants.FS_PREFIX_COS) && !location.startsWith(FeConstants.FS_PREFIX_COSN))
|| location.startsWith(FeConstants.FS_PREFIX_OSS)
|| location.startsWith(FeConstants.FS_PREFIX_OBS);
}
private static boolean isObjStorageUseS3Client(String location) {

3
fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
View File

@ -66,7 +66,8 @@ public class FileSystemFactory {
fsType = FileSystemType.S3;
} else if (location.startsWith(FeConstants.FS_PREFIX_HDFS) || location.startsWith(FeConstants.FS_PREFIX_GFS)) {
fsType = FileSystemType.DFS;
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS) || location.startsWith(FeConstants.FS_PREFIX_COSN)) {
// ofs:// and cosn:// use the same underlying file system: Tencent Cloud HDFS, aka CHDFS)) {
fsType = FileSystemType.OFS;
} else if (location.startsWith(FeConstants.FS_PREFIX_JFS)) {
fsType = FileSystemType.JFS;

16
fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java vendored
View File

@ -353,12 +353,16 @@ public abstract class FileQueryScanNode extends FileScanNode {
params.setHdfsParams(tHdfsParams);
}
if (locationType == TFileType.FILE_BROKER && !params.isSetBrokerAddresses()) {
FsBroker broker = Env.getCurrentEnv().getBrokerMgr().getAnyAliveBroker();
if (broker == null) {
throw new UserException("No alive broker.");
if (locationType == TFileType.FILE_BROKER) {
params.setProperties(locationProperties);
if (!params.isSetBrokerAddresses()) {
FsBroker broker = Env.getCurrentEnv().getBrokerMgr().getAnyAliveBroker();
if (broker == null) {
throw new UserException("No alive broker.");
}
params.addToBrokerAddresses(new TNetworkAddress(broker.host, broker.port));
}
params.addToBrokerAddresses(new TNetworkAddress(broker.host, broker.port));
}
} else if (locationType == TFileType.FILE_S3 && !params.isSetProperties()) {
params.setProperties(locationProperties);
@ -450,6 +454,8 @@ public abstract class FileQueryScanNode extends FileScanNode {
return Optional.of(TFileType.FILE_LOCAL);
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
return Optional.of(TFileType.FILE_BROKER);
} else if (location.startsWith(FeConstants.FS_PREFIX_COSN)) {
return Optional.of(TFileType.FILE_BROKER);
} else if (location.startsWith(FeConstants.FS_PREFIX_GFS)) {
return Optional.of(TFileType.FILE_BROKER);
} else if (location.startsWith(FeConstants.FS_PREFIX_JFS)) {