database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](s3client) fix unable to access COS (#31111)

Browse Source
This commit is contained in:
Mingyu Chen
2024-02-19 23:46:13 +08:00
committed by yiguolei
parent 02bded2688
commit 689b2c7bc5
4 changed files with 19 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java
View File

@ -78,7 +78,7 @@ public class ResourceMgr implements Writable {
Resource resource = Resource.fromStmt(stmt);
if (createResource(resource, stmt.isIfNotExists())) {
Env.getCurrentEnv().getEditLog().logCreateResource(resource);
LOG.info("Create resource success. Resource: {}", resource);
LOG.info("Create resource success. Resource: {}", resource.getName());
}
}

36
fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
View File

@ -111,18 +111,15 @@ public class S3Resource extends Resource {
if (needCheck) {
String bucketName = properties.get(S3Properties.BUCKET);
String rootPath = properties.get(S3Properties.ROOT_PATH);
boolean available = pingS3(credential, bucketName, rootPath, properties);
if (!available) {
throw new DdlException("S3 can't use, please check your properties");
}
pingS3(credential, bucketName, rootPath, properties);
}
// optional
S3Properties.optionalS3Property(properties);
this.properties = properties;
}
private static boolean pingS3(CloudCredentialWithEndpoint credential, String bucketName, String rootPath,
Map<String, String> properties) {
private static void pingS3(CloudCredentialWithEndpoint credential, String bucketName, String rootPath,
Map<String, String> properties) throws DdlException {
String bucket = "s3://" + bucketName + "/";
Map<String, String> propertiesPing = new HashMap<>();
propertiesPing.put(S3Properties.Env.ACCESS_KEY, credential.getAccessKey());
@ -136,24 +133,27 @@ public class S3Resource extends Resource {
String testFile = bucket + rootPath + "/test-object-valid.txt";
String content = "doris will be better";
if (FeConstants.runningUnitTest) {
return true;
return;
}
Status status = Status.OK;
try {
Status status = fileSystem.directUpload(content, testFile);
status = fileSystem.directUpload(content, testFile);
if (status != Status.OK) {
LOG.warn("ping update file status: {}, properties: {}", status, propertiesPing);
return false;
throw new DdlException(
"ping s3 failed(upload), status: " + status + ", properties: " + new PrintableMap<>(
propertiesPing, "=", true, false, true, false));
}
} finally {
Status delete = fileSystem.delete(testFile);
if (delete != Status.OK) {
LOG.warn("ping delete file status: {}, properties: {}", delete, propertiesPing);
return false;
if (status.ok()) {
Status delete = fileSystem.delete(testFile);
if (delete != Status.OK) {
LOG.warn("delete test file failed, status: {}, properties: {}", delete, new PrintableMap<>(
propertiesPing, "=", true, false, true, false));
}
}
}
LOG.info("success to ping s3");
return true;
}
@Override
@ -182,11 +182,7 @@ public class S3Resource extends Resource {
String rootPath = properties.getOrDefault(S3Properties.ROOT_PATH,
this.properties.get(S3Properties.ROOT_PATH));
boolean available = pingS3(getS3PingCredentials(changedProperties),
bucketName, rootPath, changedProperties);
if (!available) {
throw new DdlException("S3 can't use, please check your properties");
}
pingS3(getS3PingCredentials(changedProperties), bucketName, rootPath, changedProperties);
}
// modify properties

2
fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
View File

@ -92,7 +92,7 @@ public class S3Util {
// use virtual hosted-style access
.serviceConfiguration(S3Configuration.builder()
.chunkedEncodingEnabled(false)
.pathStyleAccessEnabled(false)
.pathStyleAccessEnabled(true)
.build())
.build();
}