database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](multi-catalog)fix paimon meta properties convert (#37249) (#37958)

Browse Source 
bp #37249

Co-authored-by: slothever <18522955+wsjz@users.noreply.github.com>
This commit is contained in:
Mingyu Chen
2024-07-17 01:04:33 +08:00
committed by GitHub
parent b6e5281a1c
commit ba66ff5768
9 changed files with 204 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
View File

@ -20,9 +20,11 @@ package org.apache.doris.datasource.paimon;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.security.authentication.AuthenticationConfig;
import org.apache.doris.common.security.authentication.HadoopUGI;
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.SessionContext;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.constants.HMSProperties;
import org.apache.doris.datasource.property.constants.PaimonProperties;
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
@ -55,8 +57,11 @@ public abstract class PaimonExternalCatalog extends ExternalCatalog {
PaimonProperties.WAREHOUSE
);
public PaimonExternalCatalog(long catalogId, String name, String comment) {
public PaimonExternalCatalog(long catalogId, String name, String resource,
Map<String, String> props, String comment) {
super(catalogId, name, InitCatalogLog.Type.PAIMON, comment);
props = PropertyConverter.convertToMetaProperties(props);
catalogProperty = new CatalogProperty(resource, props);
}
@Override

4
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.datasource.paimon;
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.constants.CosProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
@ -34,8 +33,7 @@ public class PaimonFileExternalCatalog extends PaimonExternalCatalog {
public PaimonFileExternalCatalog(long catalogId, String name, String resource,
Map<String, String> props, String comment) {
super(catalogId, name, comment);
catalogProperty = new CatalogProperty(resource, props);
super(catalogId, name, resource, props, comment);
}
@Override

4
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java
View File

@ -18,7 +18,6 @@
package org.apache.doris.datasource.paimon;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.property.constants.HMSProperties;
import org.apache.doris.datasource.property.constants.PaimonProperties;
@ -37,8 +36,7 @@ public class PaimonHMSExternalCatalog extends PaimonExternalCatalog {
public PaimonHMSExternalCatalog(long catalogId, String name, String resource,
Map<String, String> props, String comment) {
super(catalogId, name, comment);
catalogProperty = new CatalogProperty(resource, props);
super(catalogId, name, resource, props, comment);
}
@Override

8
fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
View File

@ -20,7 +20,6 @@ package org.apache.doris.datasource.property;
import org.apache.doris.common.credentials.CloudCredential;
import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.InitCatalogLog.Type;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
@ -250,13 +249,14 @@ public class PropertyConverter {
return s3Properties;
}
private static String checkRegion(String endpoint, String region, String regionKey) {
public static String checkRegion(String endpoint, String region, String regionKey) {
if (Strings.isNullOrEmpty(region)) {
region = S3Properties.getRegionOfEndpoint(endpoint);
}
if (Strings.isNullOrEmpty(region)) {
String errorMsg = String.format("Required property '%s' when region is not in endpoint.", regionKey);
Util.logAndThrowRuntimeException(LOG, errorMsg, new IllegalArgumentException(errorMsg));
String errorMsg = String.format("No '%s' info found, using SDK default region: us-east-1", regionKey);
LOG.warn(errorMsg);
return "us-east-1";
}
return region;
}

3
fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
View File

@ -55,7 +55,8 @@ public class S3ClientBEProperties {
Map<String, String> beProperties = new HashMap<>();
String endpoint = properties.get(S3Properties.ENDPOINT);
beProperties.put(S3Properties.Env.ENDPOINT, endpoint);
String region = S3Properties.getRegionOfEndpoint(endpoint);
String region = PropertyConverter.checkRegion(endpoint, properties.get(S3Properties.Env.REGION),
S3Properties.Env.REGION);
beProperties.put(S3Properties.Env.REGION, properties.getOrDefault(S3Properties.REGION, region));
if (properties.containsKey(S3Properties.ACCESS_KEY)) {
beProperties.put(S3Properties.Env.ACCESS_KEY, properties.get(S3Properties.ACCESS_KEY));

12
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
View File

@ -40,6 +40,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
public class S3Properties extends BaseProperties {
@ -76,6 +77,8 @@ public class S3Properties extends BaseProperties {
WebIdentityTokenCredentialsProvider.class.getName(),
IAMInstanceCredentialsProvider.class.getName());
private static final Pattern IPV4_PORT_PATTERN = Pattern.compile("((?:\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5})");
public static Map<String, String> credentialToMap(CloudCredentialWithEndpoint credential) {
Map<String, String> resMap = new HashMap<>();
resMap.put(S3Properties.ENDPOINT, credential.getEndpoint());
@ -125,11 +128,18 @@ public class S3Properties extends BaseProperties {
}
String endpoint = props.get(Env.ENDPOINT);
String region = props.getOrDefault(Env.REGION, S3Properties.getRegionOfEndpoint(endpoint));
props.putIfAbsent(Env.REGION, PropertyConverter.checkRegion(endpoint, region, Env.REGION));
return new CloudCredentialWithEndpoint(endpoint, region, credential);
}
public static String getRegionOfEndpoint(String endpoint) {
String[] endpointSplit = endpoint.split("\\.");
if (IPV4_PORT_PATTERN.matcher(endpoint).find()) {
// if endpoint contains '192.168.0.1:8999', return null region
return null;
}
String[] endpointSplit = endpoint.replace("http://", "")
.replace("https://", "")
.split("\\.");
if (endpointSplit.length < 2) {
return null;
}