database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](s3) move s3 providers to fe-common to be accessible for jni reader (#35779)

Browse Source 
backport: #35690

`PropertyConverter.setS3FsAccess` has add customized s3 providers:
```
public static final List<String> AWS_CREDENTIALS_PROVIDERS = Arrays.asList(
            DataLakeAWSCredentialsProvider.class.getName(),
            TemporaryAWSCredentialsProvider.class.getName(),
            SimpleAWSCredentialsProvider.class.getName(),
            EnvironmentVariableCredentialsProvider.class.getName(),
            IAMInstanceCredentialsProvider.class.getName());
```
And these providers are set as configuration value of
`fs.s3a.aws.credentials.provider`, which will be used as configuration
to build s3 reader in JNI readers. However,
`DataLakeAWSCredentialsProvider` is in `fe-core`, that is not dependent
by JNI readers, so we have to move s3 providers to `fe-common'.
This commit is contained in:
Ashin Gau
2024-06-03 14:04:39 +08:00
committed by GitHub
parent d4956bfaf5
commit 4f0365e0bf
24 changed files with 32 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
View File

@ -20,9 +20,9 @@ package org.apache.doris.catalog;
import org.apache.doris.backup.Status;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.common.proc.BaseProcResult;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.datasource.property.constants.S3Properties;
import org.apache.doris.fs.remote.S3FileSystem;

2
fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.common.util;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;

60
fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredential.java
View File

@ -1,60 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.datasource.credentials;
import org.apache.commons.lang3.StringUtils;
public class CloudCredential {
private String accessKey;
private String secretKey;
private String sessionToken;
public CloudCredential() {}
public String getAccessKey() {
return accessKey;
}
public void setAccessKey(String accessKey) {
this.accessKey = accessKey;
}
public String getSecretKey() {
return secretKey;
}
public void setSecretKey(String secretKey) {
this.secretKey = secretKey;
}
public String getSessionToken() {
return sessionToken;
}
public void setSessionToken(String sessionToken) {
this.sessionToken = sessionToken;
}
public boolean isWhole() {
return !StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey);
}
public boolean isTemporary() {
return !StringUtils.isEmpty(sessionToken);
}
}

61
fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredentialWithEndpoint.java
View File

@ -1,61 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.datasource.credentials;
public class CloudCredentialWithEndpoint extends CloudCredential {
private String endpoint;
private String region;
public CloudCredentialWithEndpoint(String endpoint, String region, CloudCredential credential) {
this.endpoint = endpoint;
this.region = region;
setAccessKey(credential.getAccessKey());
setSecretKey(credential.getSecretKey());
setSessionToken(credential.getSessionToken());
}
public CloudCredentialWithEndpoint(String endpoint, String region, String accessKey, String secretKey) {
this(endpoint, region, accessKey, secretKey, null);
}
public CloudCredentialWithEndpoint(String endpoint, String region, String accessKey,
String secretKey, String token) {
this.endpoint = endpoint;
this.region = region;
setAccessKey(accessKey);
setSecretKey(secretKey);
setSessionToken(token);
}
public String getEndpoint() {
return endpoint;
}
public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
}

59
fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/DataLakeAWSCredentialsProvider.java
View File

@ -1,59 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.datasource.credentials;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.BasicSessionCredentials;
import com.amazonaws.util.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.s3a.Constants;
public class DataLakeAWSCredentialsProvider implements AWSCredentialsProvider {
private final Configuration conf;
public DataLakeAWSCredentialsProvider(Configuration conf) {
this.conf = conf;
}
@Override
public AWSCredentials getCredentials() {
String accessKey = StringUtils.trim(conf.get(Constants.ACCESS_KEY));
String secretKey = StringUtils.trim(conf.get(Constants.SECRET_KEY));
String sessionToken = StringUtils.trim(conf.get(Constants.SESSION_TOKEN));
if (!StringUtils.isNullOrEmpty(accessKey) && !StringUtils.isNullOrEmpty(secretKey)) {
return (StringUtils.isNullOrEmpty(sessionToken) ? new BasicAWSCredentials(accessKey,
secretKey) : new BasicSessionCredentials(accessKey, secretKey, sessionToken));
} else {
throw new SdkClientException(
"Unable to load AWS credentials from hive conf (fs.s3a.access.key and fs.s3a.secret.key)");
}
}
@Override
public void refresh() {
}
@Override
public String toString() {
return getClass().getSimpleName();
}
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
View File

@ -17,8 +17,8 @@
package org.apache.doris.datasource.iceberg.dlf;
import org.apache.doris.common.credentials.CloudCredential;
import org.apache.doris.common.util.S3Util;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.datasource.iceberg.HiveCompatibleCatalog;
import org.apache.doris.datasource.iceberg.dlf.client.DLFCachedClientPool;
import org.apache.doris.datasource.property.PropertyConverter;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
View File

@ -19,11 +19,11 @@ package org.apache.doris.datasource.maxcompute;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.credentials.CloudCredential;
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.SessionContext;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.datasource.property.constants.MCProperties;
import com.aliyun.odps.Odps;

4
fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
View File

@ -17,12 +17,12 @@
package org.apache.doris.datasource.property;
import org.apache.doris.common.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.InitCatalogLog.Type;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.property.constants.CosProperties;
import org.apache.doris.datasource.property.constants.DLFProperties;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import java.util.Map;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import java.util.Arrays;
import java.util.List;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import com.aliyun.datalake.metastore.common.DataLakeConfig;
import com.google.common.collect.ImmutableList;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import java.util.Arrays;
import java.util.List;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import com.amazonaws.glue.catalog.util.AWSGlueConfig;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import java.util.Map;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import java.util.Arrays;
import java.util.List;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java
View File

@ -18,7 +18,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import java.util.Arrays;
import java.util.List;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import java.util.Arrays;
import java.util.List;

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java
View File

@ -17,7 +17,7 @@
package org.apache.doris.datasource.property.constants;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredential;
import com.google.common.collect.Maps;

6
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
View File

@ -19,9 +19,9 @@ package org.apache.doris.datasource.property.constants;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.datasource.credentials.DataLakeAWSCredentialsProvider;
import org.apache.doris.common.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.common.credentials.DataLakeAWSCredentialsProvider;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.thrift.TS3StorageParam;

2
fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
View File

@ -20,9 +20,9 @@ package org.apache.doris.fs.obj;
import org.apache.doris.backup.Status;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.credentials.CloudCredential;
import org.apache.doris.common.util.S3URI;
import org.apache.doris.common.util.S3Util;
import org.apache.doris.datasource.credentials.CloudCredential;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.constants.S3Properties;

2
fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java
View File

@ -22,8 +22,8 @@ import org.apache.doris.analysis.StorageBackend.StorageType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.UserException;
import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.common.util.S3URI;
import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.S3ClientBEProperties;
import org.apache.doris.datasource.property.constants.S3Properties;