From 4f0365e0bfa502b3237c1ddd89830480b13a5c4c Mon Sep 17 00:00:00 2001 From: Ashin Gau Date: Mon, 3 Jun 2024 14:04:39 +0800 Subject: [PATCH] [fix](s3) move s3 providers to fe-common to be accessible for jni reader (#35779) backport: #35690 `PropertyConverter.setS3FsAccess` has add customized s3 providers: ``` public static final List AWS_CREDENTIALS_PROVIDERS = Arrays.asList( DataLakeAWSCredentialsProvider.class.getName(), TemporaryAWSCredentialsProvider.class.getName(), SimpleAWSCredentialsProvider.class.getName(), EnvironmentVariableCredentialsProvider.class.getName(), IAMInstanceCredentialsProvider.class.getName()); ``` And these providers are set as configuration value of `fs.s3a.aws.credentials.provider`, which will be used as configuration to build s3 reader in JNI readers. However, `DataLakeAWSCredentialsProvider` is in `fe-core`, that is not dependent by JNI readers, so we have to move s3 providers to `fe-common'. --- fe/be-java-extensions/avro-scanner/pom.xml | 31 ------------------- fe/fe-common/pom.xml | 8 +++++ .../common}/credentials/CloudCredential.java | 2 +- .../CloudCredentialWithEndpoint.java | 2 +- .../DataLakeAWSCredentialsProvider.java | 2 +- fe/fe-core/pom.xml | 4 --- .../org/apache/doris/catalog/S3Resource.java | 2 +- .../org/apache/doris/common/util/S3Util.java | 2 +- .../datasource/iceberg/dlf/DLFCatalog.java | 2 +- .../maxcompute/MaxComputeExternalCatalog.java | 2 +- .../property/PropertyConverter.java | 4 +-- .../property/constants/BaseProperties.java | 2 +- .../property/constants/CosProperties.java | 2 +- .../property/constants/DLFProperties.java | 2 +- .../property/constants/GCSProperties.java | 2 +- .../property/constants/GlueProperties.java | 2 +- .../property/constants/MCProperties.java | 2 +- .../property/constants/MinioProperties.java | 2 +- .../property/constants/ObsProperties.java | 2 +- .../property/constants/OssProperties.java | 2 +- .../property/constants/PaimonProperties.java | 2 +- .../property/constants/S3Properties.java | 6 ++-- .../org/apache/doris/fs/obj/S3ObjStorage.java | 2 +- .../tablefunction/S3TableValuedFunction.java | 2 +- 24 files changed, 32 insertions(+), 59 deletions(-) rename fe/{fe-core/src/main/java/org/apache/doris/datasource => fe-common/src/main/java/org/apache/doris/common}/credentials/CloudCredential.java (97%) rename fe/{fe-core/src/main/java/org/apache/doris/datasource => fe-common/src/main/java/org/apache/doris/common}/credentials/CloudCredentialWithEndpoint.java (97%) rename fe/{fe-core/src/main/java/org/apache/doris/datasource => fe-common/src/main/java/org/apache/doris/common}/credentials/DataLakeAWSCredentialsProvider.java (97%) diff --git a/fe/be-java-extensions/avro-scanner/pom.xml b/fe/be-java-extensions/avro-scanner/pom.xml index 878e4a33e0..8d56a53a7d 100644 --- a/fe/be-java-extensions/avro-scanner/pom.xml +++ b/fe/be-java-extensions/avro-scanner/pom.xml @@ -56,37 +56,6 @@ under the License. hadoop-common provided - - org.apache.hadoop - hadoop-aws - - - slf4j-log4j12 - org.slf4j - - - log4j - log4j - - - servlet-api - javax.servlet - - - aws-java-sdk-s3 - com.amazonaws - - - aws-java-sdk-bundle - com.amazonaws - - - - - com.amazonaws - aws-java-sdk-bundle - ${aws-java-sdk.version} - org.apache.hadoop hadoop-hdfs diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index c4b1b29b35..04700c0290 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -108,6 +108,14 @@ under the License. provided + + org.apache.hadoop + hadoop-aws + + + com.amazonaws + aws-java-sdk-s3 + doris-fe-common diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredential.java b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredential.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredential.java rename to fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredential.java index e80af33410..59e8e44ed2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredential.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredential.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.datasource.credentials; +package org.apache.doris.common.credentials; import org.apache.commons.lang3.StringUtils; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredentialWithEndpoint.java b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredentialWithEndpoint.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredentialWithEndpoint.java rename to fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredentialWithEndpoint.java index 7f80612ea6..085ad03005 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredentialWithEndpoint.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredentialWithEndpoint.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.datasource.credentials; +package org.apache.doris.common.credentials; public class CloudCredentialWithEndpoint extends CloudCredential { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/DataLakeAWSCredentialsProvider.java b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/DataLakeAWSCredentialsProvider.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/DataLakeAWSCredentialsProvider.java rename to fe/fe-common/src/main/java/org/apache/doris/common/credentials/DataLakeAWSCredentialsProvider.java index 9901b9c668..600d444022 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/DataLakeAWSCredentialsProvider.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/DataLakeAWSCredentialsProvider.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.datasource.credentials; +package org.apache.doris.common.credentials; import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentials; diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 21a6fac91f..ef5cd1f5d0 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -386,10 +386,6 @@ under the License. - - org.apache.hadoop - hadoop-aws - com.amazonaws aws-java-sdk-s3 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 8b9b5f6af3..e1cde40c4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -20,9 +20,9 @@ package org.apache.doris.catalog; import org.apache.doris.backup.Status; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint; import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.fs.remote.S3FileSystem; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java index 57b53627c6..9e5e258a3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java @@ -17,7 +17,7 @@ package org.apache.doris.common.util; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentials; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java index ca5ccd5f35..e9c406715c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java @@ -17,8 +17,8 @@ package org.apache.doris.datasource.iceberg.dlf; +import org.apache.doris.common.credentials.CloudCredential; import org.apache.doris.common.util.S3Util; -import org.apache.doris.datasource.credentials.CloudCredential; import org.apache.doris.datasource.iceberg.HiveCompatibleCatalog; import org.apache.doris.datasource.iceberg.dlf.client.DLFCachedClientPool; import org.apache.doris.datasource.property.PropertyConverter; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java index 6c875d0e29..ce200ef4fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java @@ -19,11 +19,11 @@ package org.apache.doris.datasource.maxcompute; import org.apache.doris.common.DdlException; +import org.apache.doris.common.credentials.CloudCredential; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; -import org.apache.doris.datasource.credentials.CloudCredential; import org.apache.doris.datasource.property.constants.MCProperties; import com.aliyun.odps.Odps; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index e3ba2bdce2..425ea6cdcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -17,12 +17,12 @@ package org.apache.doris.datasource.property; +import org.apache.doris.common.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; import org.apache.doris.common.util.LocationPath; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InitCatalogLog.Type; -import org.apache.doris.datasource.credentials.CloudCredential; -import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.property.constants.CosProperties; import org.apache.doris.datasource.property.constants.DLFProperties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java index ef79886a86..6ea98607df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java index c806a49c61..bdf99a76fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java index dd0682114d..d8a3b7d477 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import com.aliyun.datalake.metastore.common.DataLakeConfig; import com.google.common.collect.ImmutableList; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java index 8a42f417a3..2ab005d020 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java index bf30a05099..4e1598fe32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import com.amazonaws.glue.catalog.util.AWSGlueConfig; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java index e3059cee4d..de1983f575 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java index a286718ea8..0499f5fd6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java index 7d95f9679f..19ff097fc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java @@ -18,7 +18,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java index d4fa0e1c65..56497748b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java index 72560614ea..9873984775 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import com.google.common.collect.Maps; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java index 747da72a95..1dfea17be6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java @@ -19,9 +19,9 @@ package org.apache.doris.datasource.property.constants; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import org.apache.doris.datasource.credentials.CloudCredential; -import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint; -import org.apache.doris.datasource.credentials.DataLakeAWSCredentialsProvider; +import org.apache.doris.common.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; +import org.apache.doris.common.credentials.DataLakeAWSCredentialsProvider; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.thrift.TS3StorageParam; diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java index 11ec72923d..2e5c9711a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java @@ -20,9 +20,9 @@ package org.apache.doris.fs.obj; import org.apache.doris.backup.Status; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; +import org.apache.doris.common.credentials.CloudCredential; import org.apache.doris.common.util.S3URI; import org.apache.doris.common.util.S3Util; -import org.apache.doris.datasource.credentials.CloudCredential; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.S3Properties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 8476f1c978..196496caf7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -22,8 +22,8 @@ import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; +import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; import org.apache.doris.common.util.S3URI; -import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.S3ClientBEProperties; import org.apache.doris.datasource.property.constants.S3Properties;