database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](multi-catalog) add max compute custom odps and tunnel url (#31390)

Browse Source 
add max compute custom odps and tunnel url
This commit is contained in:
slothever
2024-02-29 15:30:29 +08:00
committed by yiguolei
parent b9a87c63f7
commit 0b5b7175d6
11 changed files with 131 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java
View File

@ -35,6 +35,7 @@ import com.aliyun.odps.tunnel.TableTunnel;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Iterator;
@ -55,6 +56,8 @@ public class MaxComputeExternalCatalog extends ExternalCatalog {
private boolean enablePublicAccess;
private static final String odpsUrlTemplate = "http://service.{}.maxcompute.aliyun-inc.com/api";
private static final String tunnelUrlTemplate = "http://dt.{}.maxcompute.aliyun-inc.com";
private static String odpsUrl;
private static String tunnelUrl;
private static final List<String> REQUIRED_PROPERTIES = ImmutableList.of(
MCProperties.REGION,
MCProperties.PROJECT
@ -64,6 +67,8 @@ public class MaxComputeExternalCatalog extends ExternalCatalog {
String comment) {
super(catalogId, name, InitCatalogLog.Type.MAX_COMPUTE, comment);
catalogProperty = new CatalogProperty(resource, props);
odpsUrl = props.getOrDefault(MCProperties.ODPS_ENDPOINT, "");
tunnelUrl = props.getOrDefault(MCProperties.TUNNEL_SDK_ENDPOINT, "");
}
@Override
@ -92,16 +97,28 @@ public class MaxComputeExternalCatalog extends ExternalCatalog {
Account account = new AliyunAccount(accessKey, secretKey);
this.odps = new Odps(account);
enablePublicAccess = Boolean.parseBoolean(props.getOrDefault(MCProperties.PUBLIC_ACCESS, "false"));
String odpsUrl = odpsUrlTemplate.replace("{}", region);
if (enablePublicAccess) {
odpsUrl = odpsUrl.replace("-inc", "");
}
odps.setEndpoint(odpsUrl);
setOdpsUrl(region);
odps.setDefaultProject(defaultProject);
tunnel = new TableTunnel(odps);
String tunnelUrl = tunnelUrlTemplate.replace("{}", region);
if (enablePublicAccess) {
tunnelUrl = tunnelUrl.replace("-inc", "");
setTunnelUrl(region);
}
private void setOdpsUrl(String region) {
if (StringUtils.isEmpty(odpsUrl)) {
odpsUrl = odpsUrlTemplate.replace("{}", region);
if (enablePublicAccess) {
odpsUrl = odpsUrl.replace("-inc", "");
}
}
odps.setEndpoint(odpsUrl);
}
private void setTunnelUrl(String region) {
if (StringUtils.isEmpty(tunnelUrl)) {
tunnelUrl = tunnelUrlTemplate.replace("{}", region);
if (enablePublicAccess) {
tunnelUrl = tunnelUrl.replace("-inc", "");
}
}
tunnel.setEndpoint(tunnelUrl);
}
@ -215,4 +232,12 @@ public class MaxComputeExternalCatalog extends ExternalCatalog {
}
}
}
public String getOdpsUrl() {
return odpsUrl;
}
public String getTunnelUrl() {
return tunnelUrl;
}
}

2
fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalTable.java
View File

@ -269,6 +269,8 @@ public class MaxComputeExternalTable extends ExternalTable {
tMcTable.setRegion(mcCatalog.getRegion());
tMcTable.setAccessKey(mcCatalog.getAccessKey());
tMcTable.setSecretKey(mcCatalog.getSecretKey());
tMcTable.setOdpsUrl(mcCatalog.getOdpsUrl());
tMcTable.setTunnelUrl(mcCatalog.getTunnelUrl());
tMcTable.setPublicAccess(String.valueOf(mcCatalog.enablePublicAccess()));
// use mc project as dbName
tMcTable.setProject(dbName);

2
fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java
View File

@ -31,6 +31,8 @@ public class MCProperties extends BaseProperties {
public static final String SECRET_KEY = "mc.secret_key";
public static final String SESSION_TOKEN = "mc.session_token";
public static final String PUBLIC_ACCESS = "mc.public_access";
public static final String ODPS_ENDPOINT = "mc.odps_endpoint";
public static final String TUNNEL_SDK_ENDPOINT = "mc.tunnel_endpoint";
public static CloudCredential getCredential(Map<String, String> props) {
return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN);