database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](glue)support access glue iceberg with credential list (#30473)

Browse Source 
merge from #30292
This commit is contained in:
slothever
2024-01-28 18:23:07 +08:00
committed by GitHub
parent f988686708
commit b1a9370004
17 changed files with 239 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
be/src/util/s3_util.cpp
View File

@ -19,6 +19,7 @@
#include <aws/core/auth/AWSAuthSigner.h>
#include <aws/core/auth/AWSCredentials.h>
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/utils/logging/LogLevel.h>
#include <aws/core/utils/logging/LogSystemInterface.h>
#include <aws/core/utils/memory/stl/AWSStringStream.h>
@ -124,8 +125,7 @@ S3ClientFactory& S3ClientFactory::instance() {
bool S3ClientFactory::is_s3_conf_valid(const std::map<std::string, std::string>& prop) {
StringCaseMap<std::string> properties(prop.begin(), prop.end());
if (properties.find(S3_AK) == properties.end() || properties.find(S3_SK) == properties.end() ||
properties.find(S3_ENDPOINT) == properties.end() ||
if (properties.find(S3_ENDPOINT) == properties.end() ||
properties.find(S3_REGION) == properties.end()) {
DCHECK(false) << "aws properties is incorrect.";
LOG(ERROR) << "aws properties is incorrect.";
@ -135,7 +135,7 @@ bool S3ClientFactory::is_s3_conf_valid(const std::map<std::string, std::string>&
}
bool S3ClientFactory::is_s3_conf_valid(const S3Conf& s3_conf) {
return !s3_conf.ak.empty() && !s3_conf.sk.empty() && !s3_conf.endpoint.empty();
return !s3_conf.endpoint.empty();
}
std::shared_ptr<Aws::S3::S3Client> S3ClientFactory::create(const S3Conf& s3_conf) {
@ -152,12 +152,6 @@ std::shared_ptr<Aws::S3::S3Client> S3ClientFactory::create(const S3Conf& s3_conf
}
}
Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
DCHECK(!aws_cred.IsExpiredOrEmpty());
if (!s3_conf.token.empty()) {
aws_cred.SetSessionToken(s3_conf.token);
}
Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration();
aws_config.endpointOverride = s3_conf.endpoint;
aws_config.region = s3_conf.region;
@ -180,11 +174,25 @@ std::shared_ptr<Aws::S3::S3Client> S3ClientFactory::create(const S3Conf& s3_conf
if (s3_conf.connect_timeout_ms > 0) {
aws_config.connectTimeoutMs = s3_conf.connect_timeout_ms;
}
std::shared_ptr<Aws::S3::S3Client> new_client = std::make_shared<Aws::S3::S3Client>(
std::move(aws_cred), std::move(aws_config),
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
s3_conf.use_virtual_addressing);
std::shared_ptr<Aws::S3::S3Client> new_client;
if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
DCHECK(!aws_cred.IsExpiredOrEmpty());
if (!s3_conf.token.empty()) {
aws_cred.SetSessionToken(s3_conf.token);
}
new_client = std::make_shared<Aws::S3::S3Client>(
std::move(aws_cred), std::move(aws_config),
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
s3_conf.use_virtual_addressing);
} else {
std::shared_ptr<Aws::Auth::AWSCredentialsProvider> aws_provider_chain =
std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>();
new_client = std::make_shared<Aws::S3::S3Client>(
std::move(aws_provider_chain), std::move(aws_config),
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
s3_conf.use_virtual_addressing);
}
{
std::lock_guard l(_lock);
@ -199,8 +207,10 @@ Status S3ClientFactory::convert_properties_to_s3_conf(
return Status::InvalidArgument("S3 properties are incorrect, please check properties.");
}
StringCaseMap<std::string> properties(prop.begin(), prop.end());
s3_conf->ak = properties.find(S3_AK)->second;
s3_conf->sk = properties.find(S3_SK)->second;
if (properties.find(S3_AK) != properties.end() && properties.find(S3_SK) != properties.end()) {
s3_conf->ak = properties.find(S3_AK)->second;
s3_conf->sk = properties.find(S3_SK)->second;
}
if (properties.find(S3_TOKEN) != properties.end()) {
s3_conf->token = properties.find(S3_TOKEN)->second;
}

13
bin/start_be.sh
View File

@ -36,7 +36,6 @@ OPTS="$(getopt \
eval set -- "${OPTS}"
RUN_DAEMON=0
RUN_IN_AWS=0
RUN_CONSOLE=0
while true; do
case "$1" in
@ -44,10 +43,6 @@ while true; do
RUN_DAEMON=1
shift
;;
--aws)
RUN_IN_AWS=1
shift
;;
--console)
RUN_CONSOLE=1
shift
@ -242,10 +237,7 @@ else
LIMIT="/bin/limit3 -c 0 -n 65536"
fi
## If you are not running in aws cloud, disable this env since https://github.com/aws/aws-sdk-cpp/issues/1410.
if [[ "${RUN_IN_AWS}" -eq 0 ]]; then
export AWS_EC2_METADATA_DISABLED=true
fi
export AWS_MAX_ATTEMPTS=2
## set asan and ubsan env to generate core file
export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:detect_container_overflow=0
@ -348,9 +340,6 @@ else
export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}"
fi
export AWS_EC2_METADATA_DISABLED=true
export AWS_MAX_ATTEMPTS=2
if [[ "${RUN_DAEMON}" -eq 1 ]]; then
nohup ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1 </dev/null &
elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then

13
conf/be.conf
View File

@ -80,3 +80,16 @@ ssl_private_key_path = "$DORIS_HOME/conf/key.pem"
# sys_log_verbose_modules = *
# log_buffer_level = -1
# palo_cgroups
# aws sdk log level
# Off = 0,
# Fatal = 1,
# Error = 2,
# Warn = 3,
# Info = 4,
# Debug = 5,
# Trace = 6
# Default to turn off aws sdk log, because aws sdk errors that need to be cared will be output through Doris logs
aws_log_level=0
## If you are not running in aws cloud, you can disable EC2 metadata
AWS_EC2_METADATA_DISABLED=true

4
docs/dev.json
View File

@ -315,6 +315,10 @@
"message": "多源数据目录",
"description": "The label for category Lakehouse.Multi Catalog in sidebar docs"
},
"sidebar.docs.category.Cloud Service Authentication": {
"message": "云服务认证接入",
"description": "The label for category Lakehouse.Cloud Service Authentication in sidebar docs"
},
"sidebar.docs.category.External Table": {
"message": "外部表",
"description": "The label for category Lakehouse.External Table in sidebar docs"

61
docs/en/docs/lakehouse/cloud-auth/cloud-auth.md Normal file
View File

@ -0,0 +1,61 @@
---
{
"title": "Cloud Service Authentication",
"language": "en"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Overview
When accessing a service on the cloud, we need to provide the credentials needed to access the service so that the service can be authenticated by IAM of cloud vendors.
## AWS
Now Doris support two types of authentication to access AWS service.
### Catalog Credentials
The Catalog supports filling in basic Credentials properties, such as:
1. For S3: `s3.endpoint`，`s3.access_key`，`s3.secret_key`。
2. For Glue: `glue.endpoint`，`glue.access_key`，`glue.secret_key`。
When access Glue though Iceberg Catalog, we can access tables on Glue by filling in the following properties:
```sql
CREATE CATALOG glue PROPERTIES (
"type"="iceberg",
"iceberg.catalog.type" = "glue",
"glue.endpoint" = "https://glue.us-east-1.amazonaws.com",
"glue.access_key" = "ak",
"glue.secret_key" = "sk"
);
```
### System Credentials
For applications running on AWS resources, such as EC2 instances, this approach enhances security by avoiding hardcoded credentials.
If we create the Catalog but not fill any Credentials in properties, the `DefaultAWSCredentialsProviderChain` will be used to read in the system environment variables or instance profile.
For details about how to configure environment variables and system properties, see: [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) .
- The configurable environment variables are: `AWS_ACCESS_KEY_ID`、`AWS_SECRET_ACCESS_KEY`、`AWS_SESSION_TOKEN`、`AWS_ROLE_ARN`、`AWS_WEB_IDENTITY_TOKEN_FILE` and so on.
- In addition, you can also use [aws configure](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) to configure Credentials, the Credentials file will be written to the `~/.aws` directory.

15
docs/en/docs/lakehouse/multi-catalog/iceberg.md
View File

@ -78,6 +78,17 @@ CREATE CATALOG iceberg_hadoop_ha PROPERTIES (
);
```
```sql
CREATE CATALOG iceberg_s3 PROPERTIES (
'type'='iceberg',
'iceberg.catalog.type' = 'hadoop',
'warehouse' = 's3://bucket/dir/key',
's3.endpoint' = 's3.us-east-1.amazonaws.com',
's3.access_key' = 'ak',
's3.secret_key' = 'sk'
);
```
#### Hive Metastore
```sql
@ -108,7 +119,9 @@ CREATE CATALOG glue PROPERTIES (
);
```
For Iceberg properties, see [Iceberg Glue Catalog](https://iceberg.apache.org/docs/latest/aws/#glue-catalog)
1. For Iceberg properties, see [Iceberg Glue Catalog](https://iceberg.apache.org/docs/latest/aws/#glue-catalog).
2. If you do not fill the credentials(`glue.access_key` and `glue.secret_key`) in glue catalog, the default DefaultAWSCredentialsProviderChain will be used, and it will read credentials and the system environment variables or instance profile properties on AWS EC2.
#### Alibaba Cloud DLF

7
docs/sidebars.json
View File

@ -238,6 +238,13 @@
"lakehouse/multi-catalog/jdbc"
]
},
{
"type": "category",
"label": "Cloud Service Authentication",
"items": [
"lakehouse/cloud-auth/cloud-auth"
]
},
"lakehouse/file",
"lakehouse/filecache",
"lakehouse/external-statistics",

61
docs/zh-CN/docs/lakehouse/cloud-auth/cloud-auth.md Normal file
View File

@ -0,0 +1,61 @@
---
{
"title": "云服务认证接入",
"language": "zh-CN"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# 概述
当访问云上的服务时，我们需要提供访问服务所需要的凭证，以便服务能够通过各云厂商IAM的认证。
## AWS
现在Doris访问AWS服务时，能够支持两种类型的身份认证。
### 使用Catalog属性认证
Catalog支持填写基本的Credentials属性，比如：
1. 访问S3时，可以使用s3.endpoint，s3.access_key，s3.secret_key。
2. 访问Glue时，可以使用glue.endpoint，glue.access_key，glue.secret_key。
以Iceberg Catalog访问Glue为例，我们可以填写以下属性访问在Glue上托管的表：
```sql
CREATE CATALOG glue PROPERTIES (
"type"="iceberg",
"iceberg.catalog.type" = "glue",
"glue.endpoint" = "https://glue.us-east-1.amazonaws.com",
"glue.access_key" = "ak",
"glue.secret_key" = "sk"
);
```
### 使用系统属性认证
用于运行在AWS资源(如EC2实例)上的应用程序。可以避免硬编码写入Credentials，能够增强数据安全性。
当我们在创建Catalog时，未填写Credentials属性，那么此时会使用DefaultAWSCredentialsProviderChain，它能够读取系统环境变量或者instance profile中配置的属性。
配置环境变量和系统属性的方式可以参考：[AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) 。
- 可以选择的配置的环境变量有：`AWS_ACCESS_KEY_ID`、`AWS_SECRET_ACCESS_KEY`、`AWS_SESSION_TOKEN`、`AWS_ROLE_ARN`、`AWS_WEB_IDENTITY_TOKEN_FILE`等
- 另外，还可以使用[aws configure](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html)直接配置Credentials信息，同时在`~/.aws`目录下生成credentials文件。

15
docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
View File

@ -78,6 +78,17 @@ CREATE CATALOG iceberg_hadoop_ha PROPERTIES (
);
```
```sql
CREATE CATALOG iceberg_s3 PROPERTIES (
'type'='iceberg',
'iceberg.catalog.type' = 'hadoop',
'warehouse' = 's3://bucket/dir/key',
's3.endpoint' = 's3.us-east-1.amazonaws.com',
's3.access_key' = 'ak',
's3.secret_key' = 'sk'
);
```
#### Hive Metastore
```sql
@ -108,7 +119,9 @@ CREATE CATALOG glue PROPERTIES (
);
```
Iceberg 属性详情参见 [Iceberg Glue Catalog](https://iceberg.apache.org/docs/latest/aws/#glue-catalog)
1. Iceberg 属性详情参见 [Iceberg Glue Catalog](https://iceberg.apache.org/docs/latest/aws/#glue-catalog)
2. 如果在AWS服务（如EC2）中，不填写Credentials相关信息（`glue.access_key`和`glue.secret_key`），Doris就会使用默认的DefaultAWSCredentialsProviderChain，它会读取系统环境变量或者InstanceProfile中配置的属性。
#### 阿里云 DLF

7
fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.avro;
import org.apache.avro.Schema;
import org.apache.avro.mapred.AvroWrapper;
import org.apache.avro.mapred.Pair;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -58,8 +59,10 @@ public class S3FileReader extends AvroReader {
@Override
public void open(AvroFileContext avroFileContext, boolean tableSchema) throws IOException {
Configuration conf = new Configuration();
conf.set(AvroProperties.FS_S3A_ACCESS_KEY, accessKey);
conf.set(AvroProperties.FS_S3A_SECRET_KEY, secretKey);
if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)) {
conf.set(AvroProperties.FS_S3A_ACCESS_KEY, accessKey);
conf.set(AvroProperties.FS_S3A_SECRET_KEY, secretKey);
}
conf.set(AvroProperties.FS_S3A_ENDPOINT, endpoint);
conf.set(AvroProperties.FS_S3A_REGION, region);
path = new Path(s3aUri);

20
fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
View File

@ -21,8 +21,15 @@ import org.apache.doris.datasource.credentials.CloudCredential;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider;
import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
@ -39,7 +46,7 @@ import java.time.Duration;
public class S3Util {
public static S3Client buildS3Client(URI endpoint, String region, CloudCredential credential) {
StaticCredentialsProvider scp;
AwsCredentialsProvider scp;
AwsCredentials awsCredential;
if (!credential.isTemporary()) {
awsCredential = AwsBasicCredentials.create(credential.getAccessKey(), credential.getSecretKey());
@ -47,7 +54,16 @@ public class S3Util {
awsCredential = AwsSessionCredentials.create(credential.getAccessKey(), credential.getSecretKey(),
credential.getSessionToken());
}
scp = StaticCredentialsProvider.create(awsCredential);
if (!credential.isWhole()) {
scp = AwsCredentialsProviderChain.of(
SystemPropertyCredentialsProvider.create(),
EnvironmentVariableCredentialsProvider.create(),
WebIdentityTokenFileCredentialsProvider.create(),
ProfileCredentialsProvider.create(),
InstanceProfileCredentialsProvider.create());
} else {
scp = StaticCredentialsProvider.create(awsCredential);
}
EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy
.builder()
.baseDelay(Duration.ofSeconds(1))

6
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.datasource.property.PropertyConverter;
import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.hadoop.HadoopCatalog;
@ -54,10 +55,11 @@ public class IcebergHadoopExternalCatalog extends IcebergExternalCatalog {
protected void initLocalObjectsImpl() {
icebergCatalogType = ICEBERG_HADOOP;
HadoopCatalog hadoopCatalog = new HadoopCatalog();
hadoopCatalog.setConf(getConfiguration());
Configuration conf = getConfiguration();
// initialize hive catalog
Map<String, String> catalogProperties = new HashMap<>();
String warehouse = catalogProperty.getProperties().get(CatalogProperties.WAREHOUSE_LOCATION);
String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION);
hadoopCatalog.setConf(conf);
catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
hadoopCatalog.initialize(icebergCatalogType, catalogProperties);
catalog = hadoopCatalog;

7
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java
View File

@ -18,9 +18,9 @@
package org.apache.doris.datasource.iceberg;
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.credentials.DataLakeAWSCredentialsProvider;
import org.apache.doris.datasource.iceberg.rest.DorisIcebergRestResolvedIO;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.constants.S3Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.Constants;
@ -58,9 +58,8 @@ public class IcebergRestExternalCatalog extends IcebergExternalCatalog {
private Configuration replaceS3Properties(Configuration conf) {
Map<String, String> catalogProperties = catalogProperty.getHadoopProperties();
String credentials = catalogProperties
.getOrDefault(Constants.AWS_CREDENTIALS_PROVIDER, DataLakeAWSCredentialsProvider.class.getName());
conf.set(Constants.AWS_CREDENTIALS_PROVIDER, credentials);
String defaultProviderList = String.join(",", S3Properties.AWS_CREDENTIALS_PROVIDERS);
conf.set(Constants.AWS_CREDENTIALS_PROVIDER, defaultProviderList);
String usePahStyle = catalogProperties.getOrDefault(PropertyConverter.USE_PATH_STYLE, "true");
// Set path style
conf.set(PropertyConverter.USE_PATH_STYLE, usePahStyle);

4
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
View File

@ -56,7 +56,7 @@ public class S3Properties extends BaseProperties {
public static final String ROOT_PATH = "s3.root.path";
public static final String BUCKET = "s3.bucket";
public static final String VALIDITY_CHECK = "s3_validity_check";
public static final List<String> REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY);
public static final List<String> REQUIRED_FIELDS = Arrays.asList(ENDPOINT);
public static final List<String> TVF_REQUIRED_FIELDS = Arrays.asList(ACCESS_KEY, SECRET_KEY);
public static final List<String> FS_KEYS = Arrays.asList(ENDPOINT, REGION, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN,
ROOT_PATH, BUCKET, MAX_CONNECTIONS, REQUEST_TIMEOUT_MS, CONNECTION_TIMEOUT_MS);
@ -100,7 +100,7 @@ public class S3Properties extends BaseProperties {
public static final String DEFAULT_MAX_CONNECTIONS = "50";
public static final String DEFAULT_REQUEST_TIMEOUT_MS = "3000";
public static final String DEFAULT_CONNECTION_TIMEOUT_MS = "1000";
public static final List<String> REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY);
public static final List<String> REQUIRED_FIELDS = Arrays.asList(ENDPOINT);
public static final List<String> FS_KEYS = Arrays.asList(ENDPOINT, REGION, ACCESS_KEY, SECRET_KEY, TOKEN,
ROOT_PATH, BUCKET, MAX_CONNECTIONS, REQUEST_TIMEOUT_MS, CONNECTION_TIMEOUT_MS);
}

7
fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
View File

@ -117,12 +117,7 @@ public class S3TableValuedFunction extends ExternalFileTableValuedFunction {
if (Strings.isNullOrEmpty(props.get(S3Properties.REGION))) {
throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.REGION));
}
if (Strings.isNullOrEmpty(props.get(S3Properties.ACCESS_KEY))) {
throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.ACCESS_KEY));
}
if (Strings.isNullOrEmpty(props.get(S3Properties.SECRET_KEY))) {
throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.SECRET_KEY));
}
// do not check ak and sk, because we can read them from system environment.
}
private String getEndpointAndSetVirtualBucket(S3URI s3uri, Map<String, String> props)

3
fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
View File

@ -55,6 +55,7 @@ import com.google.common.collect.Maps;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -76,7 +77,7 @@ public class PropertyConverterTest extends TestWithFeService {
List<String> withoutPrefix = ImmutableList.of("endpoint", "access_key", "secret_key");
checkSet.addAll(withoutPrefix);
checkSet.addAll(S3Properties.Env.REQUIRED_FIELDS);
checkSet.addAll(Arrays.asList(S3Properties.ENDPOINT, S3Properties.ACCESS_KEY, S3Properties.SECRET_KEY));
expectedCredential.put("access_key", "akk");
expectedCredential.put("secret_key", "skk");
}

6
regression-test/suites/cold_heat_separation/policy/create.groovy
View File

@ -220,8 +220,8 @@ suite("create_policy") {
"s3_validity_check" = "false"
);
"""
// errCode = 2, detailMessage = Missing [AWS_ACCESS_KEY] in properties.
assertEquals(failed_create_2, null)
// can read AWS_ACCESS_KEY from environment variable
assertEquals(failed_create_2, [[0]])
}
if (has_created_2.size() == 0) {
@ -240,7 +240,7 @@ suite("create_policy") {
"s3_validity_check" = "false"
);
"""
// errCode = 2, detailMessage = Missing [AWS_SECRET_KEY] in properties.
// can read AWS_SECRET_KEY from environment variables
assertEquals(failed_create_2, null)
}