From b6dd56fee0bd5e5d0d4f721943ea1f571d58e693 Mon Sep 17 00:00:00 2001
From: slothever <18522955+wsjz@users.noreply.github.com>
Date: Fri, 18 Aug 2023 18:37:21 +0800
Subject: [PATCH] [fix](multi-catalog)fix compability issue for s3 endpoint
 (#23175)

---
 .../property/PropertyConverter.java           | 40 ++++++++++++-------
 .../property/constants/OssProperties.java     |  1 +
 2 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index 963efb62d6..754a885f4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -123,28 +123,40 @@ public class PropertyConverter {
         } else if (props.containsKey(MinioProperties.ENDPOINT)) {
             return convertToMinioProperties(props, MinioProperties.getCredential(props));
         } else if (props.containsKey(S3Properties.ENDPOINT)) {
-            CloudCredential credential = S3Properties.getCredential(props);
+            CloudCredential s3Credential = S3Properties.getCredential(props);
+            Map<String, String> s3Properties = convertToS3Properties(props, s3Credential);
             String s3CliEndpoint = props.get(S3Properties.ENDPOINT);
-            if (s3CliEndpoint.contains(CosProperties.COS_PREFIX)) {
-                props.putIfAbsent(CosProperties.ENDPOINT, s3CliEndpoint);
-                // CosN is not compatible with S3, when use s3 properties, will convert to cosn properties.
-                return convertToCOSProperties(props, credential);
-            }
-            return convertToS3Properties(props, S3Properties.getCredential(props));
+            return convertToCompatibleS3Properties(props, s3CliEndpoint, s3Credential, s3Properties);
         } else if (props.containsKey(S3Properties.Env.ENDPOINT)) {
             // checkout env in the end
             // compatible with the s3,obs,oss,cos when they use aws client.
             CloudCredentialWithEndpoint envCredentials = S3Properties.getEnvironmentCredentialWithEndpoint(props);
-            if (envCredentials.getEndpoint().contains(CosProperties.COS_PREFIX)) {
-                props.putIfAbsent(CosProperties.ENDPOINT, envCredentials.getEndpoint());
-                // CosN is not compatible with S3, when use s3 properties, will convert to cosn properties.
-                return convertToCOSProperties(props, envCredentials);
-            }
-            return convertToS3EnvProperties(props, envCredentials, false);
+            Map<String, String> s3Properties = convertToS3EnvProperties(props, envCredentials, false);
+            String s3CliEndpoint = envCredentials.getEndpoint();
+            return convertToCompatibleS3Properties(props, s3CliEndpoint, envCredentials, s3Properties);
         }
         return props;
     }
 
+    private static Map<String, String> convertToCompatibleS3Properties(Map<String, String> props,
+                                                                       String s3CliEndpoint,
+                                                                       CloudCredential credential,
+                                                                       Map<String, String> s3Properties) {
+        Map<String, String> heteroProps = new HashMap<>(s3Properties);
+        if (s3CliEndpoint.contains(CosProperties.COS_PREFIX)) {
+            props.putIfAbsent(CosProperties.ENDPOINT, s3CliEndpoint);
+            // CosN is not compatible with S3, when use s3 properties, will convert to cosn properties.
+            heteroProps.putAll(convertToCOSProperties(props, credential));
+        } else if (s3CliEndpoint.contains(ObsProperties.OBS_PREFIX)) {
+            props.putIfAbsent(ObsProperties.ENDPOINT, s3CliEndpoint);
+            heteroProps.putAll(convertToOBSProperties(props, credential));
+        } else if (s3CliEndpoint.contains(OssProperties.OSS_REGION_PREFIX)) {
+            props.putIfAbsent(OssProperties.ENDPOINT, s3CliEndpoint);
+            heteroProps.putAll(convertToOSSProperties(props, credential));
+        }
+        return heteroProps;
+    }
+
 
     private static Map<String, String> convertToOBSProperties(Map<String, String> props,
                                                               CloudCredential credential) {
@@ -237,7 +249,7 @@ public class PropertyConverter {
                                       CloudCredential credential) {
         s3Properties.put(Constants.MAX_ERROR_RETRIES, "2");
         s3Properties.put("fs.s3.impl.disable.cache", "true");
-        s3Properties.put("fs.s3.impl", S3AFileSystem.class.getName());
+        s3Properties.putIfAbsent("fs.s3.impl", S3AFileSystem.class.getName());
         String defaultProviderList = String.join(",", S3Properties.AWS_CREDENTIALS_PROVIDERS);
         String credentialsProviders = s3Properties
                 .getOrDefault(S3Properties.CREDENTIALS_PROVIDER, defaultProviderList);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java
index 210bc5814a..d4fa0e1c65 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java
@@ -26,6 +26,7 @@ import java.util.Map;
 public class OssProperties extends BaseProperties {
 
     public static final String OSS_PREFIX = "oss.";
+    public static final String OSS_REGION_PREFIX = "oss-";
     public static final String OSS_FS_PREFIX = "fs.oss";
 
     public static final String ENDPOINT = "oss.endpoint";