database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](backup) missing use_path_style properties for minio (#25803)

Browse Source 
Follow #25496.
In #25496, I fixed the issue that the aws s3 properties are invalid when passing from FE to BE.
But it missed the `use_path_style` property, which is useful for minio access.
This PR fix it.
This commit is contained in:
Mingyu Chen
2023-10-26 01:19:28 +08:00
committed by GitHub
parent 0fb57a6db8
commit 0dcbc5ee18
4 changed files with 140 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/util/s3_util.h
View File

@ -64,7 +64,7 @@ const static std::string S3_REGION = "AWS_REGION";
const static std::string S3_TOKEN = "AWS_TOKEN";
const static std::string S3_MAX_CONN_SIZE = "AWS_MAX_CONN_SIZE";
const static std::string S3_REQUEST_TIMEOUT_MS = "AWS_REQUEST_TIMEOUT_MS";
const static std::string S3_CONN_TIMEOUT_MS = "AWS_CONN_TIMEOUT_MS";
const static std::string S3_CONN_TIMEOUT_MS = "AWS_CONNECTION_TIMEOUT_MS";
struct S3Conf {
std::string ak;

13
fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.datasource.property.constants.MinioProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
import org.apache.doris.datasource.property.constants.OssProperties;
import org.apache.doris.datasource.property.constants.S3Properties;
import org.apache.doris.datasource.property.constants.S3Properties.Env;
import java.util.HashMap;
import java.util.Map;
@ -71,6 +72,18 @@ public class S3ClientBEProperties {
if (properties.containsKey(S3Properties.BUCKET)) {
beProperties.put(S3Properties.Env.BUCKET, properties.get(S3Properties.BUCKET));
}
if (properties.containsKey(S3Properties.MAX_CONNECTIONS)) {
beProperties.put(Env.MAX_CONNECTIONS, properties.get(S3Properties.MAX_CONNECTIONS));
}
if (properties.containsKey(S3Properties.REQUEST_TIMEOUT_MS)) {
beProperties.put(Env.REQUEST_TIMEOUT_MS, properties.get(S3Properties.REQUEST_TIMEOUT_MS));
}
if (properties.containsKey(S3Properties.CONNECTION_TIMEOUT_MS)) {
beProperties.put(Env.CONNECTION_TIMEOUT_MS, properties.get(S3Properties.CONNECTION_TIMEOUT_MS));
}
if (properties.containsKey(PropertyConverter.USE_PATH_STYLE)) {
beProperties.put(PropertyConverter.USE_PATH_STYLE, properties.get(PropertyConverter.USE_PATH_STYLE));
}
return beProperties;
}
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
View File

@ -150,6 +150,8 @@ public class S3Properties extends BaseProperties {
} else if (entry.getKey().startsWith(MinioProperties.MINIO_PREFIX)) {
String s3Key = entry.getKey().replace(MinioProperties.MINIO_PREFIX, S3Properties.S3_PREFIX);
s3Properties.put(s3Key, entry.getValue());
} else {
s3Properties.put(entry.getKey(), entry.getValue());
}
}
return s3Properties;

125
fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
View File

@ -36,6 +36,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.datasource.property.constants.CosProperties;
import org.apache.doris.datasource.property.constants.GCSProperties;
import org.apache.doris.datasource.property.constants.MinioProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
import org.apache.doris.datasource.property.constants.OssProperties;
@ -46,6 +47,7 @@ import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@ -405,11 +407,132 @@ public class PropertyConverterTest extends TestWithFeService {
return (HMSExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(name);
}
@Test
public void testSerialization() throws Exception {
MetaContext metaContext = new MetaContext();
metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT);
metaContext.setThreadLocalInfo();
}
@Test
public void testS3PropertiesConvertor() {
// 1. AWS
Map<String, String> origProp = Maps.newHashMap();
origProp.put(S3Properties.Env.ACCESS_KEY, "ak");
origProp.put(S3Properties.Env.SECRET_KEY, "sk");
origProp.put(S3Properties.Env.ENDPOINT, "endpoint");
origProp.put(S3Properties.Env.REGION, "region");
origProp.put(PropertyConverter.USE_PATH_STYLE, "true");
Map<String, String> beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
Assertions.assertEquals(5, beProperties.size());
Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY));
Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY));
Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT));
Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION));
Assertions.assertEquals("true", beProperties.get(PropertyConverter.USE_PATH_STYLE));
// 2. s3.
origProp = Maps.newHashMap();
origProp.put(S3Properties.ACCESS_KEY, "ak");
origProp.put(S3Properties.SECRET_KEY, "sk");
origProp.put(S3Properties.ENDPOINT, "endpoint");
origProp.put(S3Properties.REGION, "region");
origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
Assertions.assertEquals(5, beProperties.size());
Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY));
Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY));
Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT));
Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION));
Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE));
// 3. minio.
origProp = Maps.newHashMap();
origProp.put(MinioProperties.ACCESS_KEY, "ak");
origProp.put(MinioProperties.SECRET_KEY, "sk");
origProp.put(MinioProperties.ENDPOINT, "endpoint");
origProp.put(MinioProperties.REGION, "region");
origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
Assertions.assertEquals(5, beProperties.size());
Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY));
Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY));
Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT));
Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION));
Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE));
// 3.1 minio without region
origProp = Maps.newHashMap();
origProp.put(MinioProperties.ACCESS_KEY, "ak");
origProp.put(MinioProperties.SECRET_KEY, "sk");
origProp.put(MinioProperties.ENDPOINT, "endpoint");
origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
Assertions.assertEquals(5, beProperties.size());
Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY));
Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY));
Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT));
Assertions.assertEquals(MinioProperties.DEFAULT_REGION, beProperties.get(S3Properties.Env.REGION));
Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE));
// 4. obs
origProp = Maps.newHashMap();
origProp.put(ObsProperties.ACCESS_KEY, "ak");
origProp.put(ObsProperties.SECRET_KEY, "sk");
origProp.put(ObsProperties.ENDPOINT, "endpoint");
origProp.put(ObsProperties.REGION, "region");
origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
Assertions.assertEquals(5, beProperties.size());
Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY));
Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY));
Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT));
Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION));
Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE));
// 4. oss
origProp = Maps.newHashMap();
origProp.put(OssProperties.ACCESS_KEY, "ak");
origProp.put(OssProperties.SECRET_KEY, "sk");
origProp.put(OssProperties.ENDPOINT, "endpoint");
origProp.put(OssProperties.REGION, "region");
origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
Assertions.assertEquals(5, beProperties.size());
Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY));
Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY));
Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT));
Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION));
Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE));
// 4. cos
origProp = Maps.newHashMap();
origProp.put(CosProperties.ACCESS_KEY, "ak");
origProp.put(CosProperties.SECRET_KEY, "sk");
origProp.put(CosProperties.ENDPOINT, "endpoint");
origProp.put(CosProperties.REGION, "region");
origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
Assertions.assertEquals(5, beProperties.size());
Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY));
Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY));
Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT));
Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION));
Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE));
// 5. gs
origProp = Maps.newHashMap();
origProp.put(GCSProperties.ACCESS_KEY, "ak");
origProp.put(GCSProperties.SECRET_KEY, "sk");
origProp.put(GCSProperties.ENDPOINT, "endpoint");
origProp.put(GCSProperties.REGION, "region");
origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
Assertions.assertEquals(5, beProperties.size());
Assertions.assertEquals("ak", beProperties.get(S3Properties.Env.ACCESS_KEY));
Assertions.assertEquals("sk", beProperties.get(S3Properties.Env.SECRET_KEY));
Assertions.assertEquals("endpoint", beProperties.get(S3Properties.Env.ENDPOINT));
Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION));
Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE));
}
}