database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](multi-catalog) when endpoint has no region, need a suggestion (#19203)

Browse Source 
solve the problem

```
 mysql> CREATE CATALOG iceberg PROPERTIES (  
    'type'='iceberg', 
    'iceberg.catalog.type'='rest',                                                                                                                                         
    'uri' = 'http://0.0.0.0:8888,  
    "AWS_ACCESS_KEY" = "admin",                                                                                                                                           
    "AWS_SECRET_KEY" = "password", 
    "AWS_REGION" = "us-east-1",                  
    "AWS_ENDPOINT" = "http://minio:9000"
);  
show databases; 

ERROR 1105 (HY000): IllegalArgumentException, msg: java.lang.IllegalArgumentException: The value of property fs.s3a.endpoint.region must not be null   
```
This commit is contained in:
slothever
2023-04-29 00:05:41 +08:00
committed by GitHub
parent 4a10d146bf
commit d006143330
2 changed files with 17 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.datasource.property;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
@ -126,7 +127,8 @@ public class PropertyConverter {
CloudCredentialWithEndpoint credential) {
// Old properties to new properties
properties.put(S3Properties.ENDPOINT, credential.getEndpoint());
properties.put(S3Properties.REGION, credential.getRegion());
properties.put(S3Properties.REGION,
checkRegion(credential.getEndpoint(), credential.getRegion(), S3Properties.Env.REGION));
properties.put(S3Properties.ACCESS_KEY, credential.getAccessKey());
properties.put(S3Properties.SECRET_KEY, credential.getSecretKey());
if (properties.containsKey(S3Properties.Env.TOKEN)) {
@ -149,7 +151,8 @@ public class PropertyConverter {
Map<String, String> s3Properties = Maps.newHashMap();
String endpoint = properties.get(S3Properties.ENDPOINT);
s3Properties.put(Constants.ENDPOINT, endpoint);
s3Properties.put(Constants.AWS_REGION, S3Properties.getRegionOfEndpoint(endpoint));
s3Properties.put(Constants.AWS_REGION,
checkRegion(endpoint, properties.get(S3Properties.REGION), S3Properties.REGION));
if (properties.containsKey(S3Properties.MAX_CONNECTIONS)) {
s3Properties.put(Constants.MAXIMUM_CONNECTIONS, properties.get(S3Properties.MAX_CONNECTIONS));
}
@ -164,6 +167,17 @@ public class PropertyConverter {
return s3Properties;
}
private static String checkRegion(String endpoint, String region, String regionKey) {
if (Strings.isNullOrEmpty(region)) {
region = S3Properties.getRegionOfEndpoint(endpoint);
}
if (Strings.isNullOrEmpty(region)) {
String errorMsg = String.format("Required property '%s' when region is not in endpoint.", regionKey);
Util.logAndThrowRuntimeException(LOG, errorMsg, new IllegalArgumentException(errorMsg));
}
return region;
}
private static void setS3FsAccess(Map<String, String> s3Properties, Map<String, String> properties,
CloudCredential credential) {
s3Properties.put(Constants.MAX_ERROR_RETRIES, "2");

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
View File

@ -113,7 +113,7 @@ public class S3Properties extends BaseProperties {
throw new IllegalArgumentException("Missing 'AWS_ENDPOINT' property. ");
}
String endpoint = props.get(Env.ENDPOINT);
String region = props.getOrDefault(S3Properties.REGION, S3Properties.getRegionOfEndpoint(endpoint));
String region = props.getOrDefault(Env.REGION, S3Properties.getRegionOfEndpoint(endpoint));
return new CloudCredentialWithEndpoint(endpoint, region, credential);
}