From 689b2c7bc555bb4b3a5951828d028785d2c5be14 Mon Sep 17 00:00:00 2001
From: Mingyu Chen <morningman@163.com>
Date: Mon, 19 Feb 2024 23:46:13 +0800
Subject: [PATCH] [fix](s3client) fix unable to access COS  (#31111)

---
 .../org/apache/doris/catalog/ResourceMgr.java |  2 +-
 .../org/apache/doris/catalog/S3Resource.java  | 36 +++++++++----------
 .../org/apache/doris/common/util/S3Util.java  |  2 +-
 .../cold_heat_separation/policy/create.groovy |  2 +-
 4 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
index 34dfb3fa91..93cb194c90 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
@@ -78,7 +78,7 @@ public class ResourceMgr implements Writable {
         Resource resource = Resource.fromStmt(stmt);
         if (createResource(resource, stmt.isIfNotExists())) {
             Env.getCurrentEnv().getEditLog().logCreateResource(resource);
-            LOG.info("Create resource success. Resource: {}", resource);
+            LOG.info("Create resource success. Resource: {}", resource.getName());
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
index 8c7373dde6..2013b4ff49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
@@ -111,18 +111,15 @@ public class S3Resource extends Resource {
         if (needCheck) {
             String bucketName = properties.get(S3Properties.BUCKET);
             String rootPath = properties.get(S3Properties.ROOT_PATH);
-            boolean available = pingS3(credential, bucketName, rootPath, properties);
-            if (!available) {
-                throw new DdlException("S3 can't use, please check your properties");
-            }
+            pingS3(credential, bucketName, rootPath, properties);
         }
         // optional
         S3Properties.optionalS3Property(properties);
         this.properties = properties;
     }
 
-    private static boolean pingS3(CloudCredentialWithEndpoint credential, String bucketName, String rootPath,
-                                  Map<String, String> properties) {
+    private static void pingS3(CloudCredentialWithEndpoint credential, String bucketName, String rootPath,
+            Map<String, String> properties) throws DdlException {
         String bucket = "s3://" + bucketName + "/";
         Map<String, String> propertiesPing = new HashMap<>();
         propertiesPing.put(S3Properties.Env.ACCESS_KEY, credential.getAccessKey());
@@ -136,24 +133,27 @@ public class S3Resource extends Resource {
         String testFile = bucket + rootPath + "/test-object-valid.txt";
         String content = "doris will be better";
         if (FeConstants.runningUnitTest) {
-            return true;
+            return;
         }
+        Status status = Status.OK;
         try {
-            Status status = fileSystem.directUpload(content, testFile);
+            status = fileSystem.directUpload(content, testFile);
             if (status != Status.OK) {
-                LOG.warn("ping update file status: {}, properties: {}", status, propertiesPing);
-                return false;
+                throw new DdlException(
+                        "ping s3 failed(upload), status: " + status + ", properties: " + new PrintableMap<>(
+                                propertiesPing, "=", true, false, true, false));
             }
         } finally {
-            Status delete = fileSystem.delete(testFile);
-            if (delete != Status.OK) {
-                LOG.warn("ping delete file status: {}, properties: {}", delete, propertiesPing);
-                return false;
+            if (status.ok()) {
+                Status delete = fileSystem.delete(testFile);
+                if (delete != Status.OK) {
+                    LOG.warn("delete test file failed, status: {}, properties: {}", delete, new PrintableMap<>(
+                            propertiesPing, "=", true, false, true, false));
+                }
             }
         }
 
         LOG.info("success to ping s3");
-        return true;
     }
 
     @Override
@@ -182,11 +182,7 @@ public class S3Resource extends Resource {
             String rootPath = properties.getOrDefault(S3Properties.ROOT_PATH,
                     this.properties.get(S3Properties.ROOT_PATH));
 
-            boolean available = pingS3(getS3PingCredentials(changedProperties),
-                        bucketName, rootPath, changedProperties);
-            if (!available) {
-                throw new DdlException("S3 can't use, please check your properties");
-            }
+            pingS3(getS3PingCredentials(changedProperties), bucketName, rootPath, changedProperties);
         }
 
         // modify properties
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
index 05bb2f6a10..0195783191 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
@@ -92,7 +92,7 @@ public class S3Util {
                 // use virtual hosted-style access
                 .serviceConfiguration(S3Configuration.builder()
                         .chunkedEncodingEnabled(false)
-                        .pathStyleAccessEnabled(false)
+                        .pathStyleAccessEnabled(true)
                         .build())
                 .build();
     }
diff --git a/regression-test/suites/cold_heat_separation/policy/create.groovy b/regression-test/suites/cold_heat_separation/policy/create.groovy
index 81f8fa5001..b382f05a4e 100644
--- a/regression-test/suites/cold_heat_separation/policy/create.groovy
+++ b/regression-test/suites/cold_heat_separation/policy/create.groovy
@@ -197,7 +197,7 @@ suite("create_policy") {
             "AWS_MAX_CONNECTIONS" = "50",
             "AWS_REQUEST_TIMEOUT_MS" = "3000",
             "AWS_CONNECTION_TIMEOUT_MS" = "1000",
-            "AWS_BUCKET" = "test-bucket",
+            "AWS_BUCKET" = "test-bucket"
         );
         """
         // errCode = 2, detailMessage = Missing [s3_validity_check] in properties.