From 0dcbc5ee18b1a2947f314b94ffbbf710103d5c5c Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Thu, 26 Oct 2023 01:19:28 +0800 Subject: [PATCH] [fix](backup) missing use_path_style properties for minio (#25803) Follow #25496. In #25496, I fixed the issue that the aws s3 properties are invalid when passing from FE to BE. But it missed the `use_path_style` property, which is useful for minio access. This PR fix it. --- be/src/util/s3_util.h | 2 +- .../property/S3ClientBEProperties.java | 13 ++ .../property/constants/S3Properties.java | 2 + .../property/PropertyConverterTest.java | 125 +++++++++++++++++- 4 files changed, 140 insertions(+), 2 deletions(-) diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 1f00a82aa9..247c6e5ef2 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -64,7 +64,7 @@ const static std::string S3_REGION = "AWS_REGION"; const static std::string S3_TOKEN = "AWS_TOKEN"; const static std::string S3_MAX_CONN_SIZE = "AWS_MAX_CONN_SIZE"; const static std::string S3_REQUEST_TIMEOUT_MS = "AWS_REQUEST_TIMEOUT_MS"; -const static std::string S3_CONN_TIMEOUT_MS = "AWS_CONN_TIMEOUT_MS"; +const static std::string S3_CONN_TIMEOUT_MS = "AWS_CONNECTION_TIMEOUT_MS"; struct S3Conf { std::string ak; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java index fe912c1038..20da63656e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java @@ -23,6 +23,7 @@ import org.apache.doris.datasource.property.constants.MinioProperties; import org.apache.doris.datasource.property.constants.ObsProperties; import org.apache.doris.datasource.property.constants.OssProperties; import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.constants.S3Properties.Env; import java.util.HashMap; import java.util.Map; @@ -71,6 +72,18 @@ public class S3ClientBEProperties { if (properties.containsKey(S3Properties.BUCKET)) { beProperties.put(S3Properties.Env.BUCKET, properties.get(S3Properties.BUCKET)); } + if (properties.containsKey(S3Properties.MAX_CONNECTIONS)) { + beProperties.put(Env.MAX_CONNECTIONS, properties.get(S3Properties.MAX_CONNECTIONS)); + } + if (properties.containsKey(S3Properties.REQUEST_TIMEOUT_MS)) { + beProperties.put(Env.REQUEST_TIMEOUT_MS, properties.get(S3Properties.REQUEST_TIMEOUT_MS)); + } + if (properties.containsKey(S3Properties.CONNECTION_TIMEOUT_MS)) { + beProperties.put(Env.CONNECTION_TIMEOUT_MS, properties.get(S3Properties.CONNECTION_TIMEOUT_MS)); + } + if (properties.containsKey(PropertyConverter.USE_PATH_STYLE)) { + beProperties.put(PropertyConverter.USE_PATH_STYLE, properties.get(PropertyConverter.USE_PATH_STYLE)); + } return beProperties; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java index a1da49257b..ea3a8333b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java @@ -150,6 +150,8 @@ public class S3Properties extends BaseProperties { } else if (entry.getKey().startsWith(MinioProperties.MINIO_PREFIX)) { String s3Key = entry.getKey().replace(MinioProperties.MINIO_PREFIX, S3Properties.S3_PREFIX); s3Properties.put(s3Key, entry.getValue()); + } else { + s3Properties.put(entry.getKey(), entry.getValue()); } } return s3Properties; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java index bf6f7bf6b9..153f989b50 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java @@ -36,6 +36,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.HMSExternalCatalog; import org.apache.doris.datasource.property.constants.CosProperties; +import org.apache.doris.datasource.property.constants.GCSProperties; import org.apache.doris.datasource.property.constants.MinioProperties; import org.apache.doris.datasource.property.constants.ObsProperties; import org.apache.doris.datasource.property.constants.OssProperties; @@ -46,6 +47,7 @@ import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -405,11 +407,132 @@ public class PropertyConverterTest extends TestWithFeService { return (HMSExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(name); } - @Test public void testSerialization() throws Exception { MetaContext metaContext = new MetaContext(); metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT); metaContext.setThreadLocalInfo(); } + + @Test + public void testS3PropertiesConvertor() { + // 1. AWS + Map origProp = Maps.newHashMap(); + origProp.put(S3Properties.Env.ACCESS_KEY, "ak"); + origProp.put(S3Properties.Env.SECRET_KEY, "sk"); + origProp.put(S3Properties.Env.ENDPOINT, "endpoint"); + origProp.put(S3Properties.Env.REGION, "region"); + origProp.put(PropertyConverter.USE_PATH_STYLE, "true"); + Map beProperties = S3ClientBEProperties.getBeFSProperties(origProp); + Assertions.assertEquals(5, beProperties.size()); + Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY)); + Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY)); + Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT)); + Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION)); + Assertions.assertEquals("true", beProperties.get(PropertyConverter.USE_PATH_STYLE)); + + // 2. s3. + origProp = Maps.newHashMap(); + origProp.put(S3Properties.ACCESS_KEY, "ak"); + origProp.put(S3Properties.SECRET_KEY, "sk"); + origProp.put(S3Properties.ENDPOINT, "endpoint"); + origProp.put(S3Properties.REGION, "region"); + origProp.put(PropertyConverter.USE_PATH_STYLE, "false"); + beProperties = S3ClientBEProperties.getBeFSProperties(origProp); + Assertions.assertEquals(5, beProperties.size()); + Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY)); + Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY)); + Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT)); + Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION)); + Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE)); + + // 3. minio. + origProp = Maps.newHashMap(); + origProp.put(MinioProperties.ACCESS_KEY, "ak"); + origProp.put(MinioProperties.SECRET_KEY, "sk"); + origProp.put(MinioProperties.ENDPOINT, "endpoint"); + origProp.put(MinioProperties.REGION, "region"); + origProp.put(PropertyConverter.USE_PATH_STYLE, "false"); + beProperties = S3ClientBEProperties.getBeFSProperties(origProp); + Assertions.assertEquals(5, beProperties.size()); + Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY)); + Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY)); + Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT)); + Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION)); + Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE)); + + // 3.1 minio without region + origProp = Maps.newHashMap(); + origProp.put(MinioProperties.ACCESS_KEY, "ak"); + origProp.put(MinioProperties.SECRET_KEY, "sk"); + origProp.put(MinioProperties.ENDPOINT, "endpoint"); + origProp.put(PropertyConverter.USE_PATH_STYLE, "false"); + beProperties = S3ClientBEProperties.getBeFSProperties(origProp); + Assertions.assertEquals(5, beProperties.size()); + Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY)); + Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY)); + Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT)); + Assertions.assertEquals(MinioProperties.DEFAULT_REGION, beProperties.get(S3Properties.Env.REGION)); + Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE)); + + // 4. obs + origProp = Maps.newHashMap(); + origProp.put(ObsProperties.ACCESS_KEY, "ak"); + origProp.put(ObsProperties.SECRET_KEY, "sk"); + origProp.put(ObsProperties.ENDPOINT, "endpoint"); + origProp.put(ObsProperties.REGION, "region"); + origProp.put(PropertyConverter.USE_PATH_STYLE, "false"); + beProperties = S3ClientBEProperties.getBeFSProperties(origProp); + Assertions.assertEquals(5, beProperties.size()); + Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY)); + Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY)); + Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT)); + Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION)); + Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE)); + + // 4. oss + origProp = Maps.newHashMap(); + origProp.put(OssProperties.ACCESS_KEY, "ak"); + origProp.put(OssProperties.SECRET_KEY, "sk"); + origProp.put(OssProperties.ENDPOINT, "endpoint"); + origProp.put(OssProperties.REGION, "region"); + origProp.put(PropertyConverter.USE_PATH_STYLE, "false"); + beProperties = S3ClientBEProperties.getBeFSProperties(origProp); + Assertions.assertEquals(5, beProperties.size()); + Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY)); + Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY)); + Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT)); + Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION)); + Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE)); + + // 4. cos + origProp = Maps.newHashMap(); + origProp.put(CosProperties.ACCESS_KEY, "ak"); + origProp.put(CosProperties.SECRET_KEY, "sk"); + origProp.put(CosProperties.ENDPOINT, "endpoint"); + origProp.put(CosProperties.REGION, "region"); + origProp.put(PropertyConverter.USE_PATH_STYLE, "false"); + beProperties = S3ClientBEProperties.getBeFSProperties(origProp); + Assertions.assertEquals(5, beProperties.size()); + Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY)); + Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY)); + Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT)); + Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION)); + Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE)); + + // 5. gs + origProp = Maps.newHashMap(); + origProp.put(GCSProperties.ACCESS_KEY, "ak"); + origProp.put(GCSProperties.SECRET_KEY, "sk"); + origProp.put(GCSProperties.ENDPOINT, "endpoint"); + origProp.put(GCSProperties.REGION, "region"); + origProp.put(PropertyConverter.USE_PATH_STYLE, "false"); + beProperties = S3ClientBEProperties.getBeFSProperties(origProp); + Assertions.assertEquals(5, beProperties.size()); + Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY)); + Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY)); + Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT)); + Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION)); + Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE)); + } }