database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[branch-2.1][improvement](jdbc catalog) improvement some jdbc catalog properties check order (#38770)

Browse Source 
pick (#38439)

1. Move the execution of testJdbcConnection() to checkWhenCreating
instead of the constructor
2. Move the logic of renaming lower_case_table_names to
lower_case_meta_names to setDefaultPropsIfMissing
This commit is contained in:
zy-kkk
2024-08-05 09:14:04 +08:00
committed by GitHub
parent 607c0b82a9
commit 9430b27e68
4 changed files with 52 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
View File

@ -122,7 +122,7 @@ public class CatalogFactory {
catalog = new EsExternalCatalog(catalogId, name, resource, props, comment);
break;
case "jdbc":
catalog = new JdbcExternalCatalog(catalogId, name, resource, props, comment, isReplay);
catalog = new JdbcExternalCatalog(catalogId, name, resource, props, comment);
break;
case "iceberg":
catalog = IcebergExternalCatalogFactory.createCatalog(catalogId, name, resource, props, comment);

4
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java
View File

@ -115,6 +115,10 @@ public class CatalogProperty implements Writable {
this.properties.put(key, val);
}
public void deleteProperty(String key) {
this.properties.remove(key);
}
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, GsonUtils.GSON.toJson(this));

57
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
View File

@ -72,11 +72,10 @@ public class JdbcExternalCatalog extends ExternalCatalog {
private transient JdbcClient jdbcClient;
public JdbcExternalCatalog(long catalogId, String name, String resource, Map<String, String> props,
String comment, boolean isReplay)
String comment)
throws DdlException {
super(catalogId, name, InitCatalogLog.Type.JDBC, comment);
this.catalogProperty = new CatalogProperty(resource, processCompatibleProperties(props, isReplay));
testJdbcConnection(isReplay);
this.catalogProperty = new CatalogProperty(resource, processCompatibleProperties(props));
}
@Override
@ -99,6 +98,21 @@ public class JdbcExternalCatalog extends ExternalCatalog {
getConnectionPoolMaxWaitTime(), getConnectionPoolMaxLifeTime());
}
@Override
public void setDefaultPropsIfMissing(boolean isReplay) {
super.setDefaultPropsIfMissing(isReplay);
// Modify lower_case_table_names to lower_case_meta_names if it exists
if (catalogProperty.getProperties().containsKey("lower_case_table_names") && isReplay) {
String lowerCaseTableNamesValue = catalogProperty.getProperties().get("lower_case_table_names");
catalogProperty.addProperty("lower_case_meta_names", lowerCaseTableNamesValue);
catalogProperty.deleteProperty("lower_case_table_names");
LOG.info("Modify lower_case_table_names to lower_case_meta_names, value: {}", lowerCaseTableNamesValue);
} else if (catalogProperty.getProperties().containsKey("lower_case_table_names") && !isReplay) {
throw new IllegalArgumentException("Jdbc catalog property lower_case_table_names is not supported,"
+ " please use lower_case_meta_names instead.");
}
}
@Override
public void onRefresh(boolean invalidCache) {
super.onRefresh(invalidCache);
@ -115,24 +129,12 @@ public class JdbcExternalCatalog extends ExternalCatalog {
}
}
protected Map<String, String> processCompatibleProperties(Map<String, String> props, boolean isReplay)
protected Map<String, String> processCompatibleProperties(Map<String, String> props)
throws DdlException {
Map<String, String> properties = Maps.newHashMap();
for (Map.Entry<String, String> kv : props.entrySet()) {
properties.put(StringUtils.removeStart(kv.getKey(), JdbcResource.JDBC_PROPERTIES_PREFIX), kv.getValue());
}
// Modify lower_case_table_names to lower_case_meta_names if it exists
if (properties.containsKey("lower_case_table_names") && isReplay) {
String lowerCaseTableNamesValue = properties.get("lower_case_table_names");
properties.put("lower_case_meta_names", lowerCaseTableNamesValue);
properties.remove("lower_case_table_names");
LOG.info("Modify lower_case_table_names to lower_case_meta_names, value: {}", lowerCaseTableNamesValue);
} else if (properties.containsKey("lower_case_table_names") && !isReplay) {
throw new DdlException("Jdbc catalog property lower_case_table_names is not supported,"
+ " please use lower_case_meta_names instead");
}
String jdbcUrl = properties.getOrDefault(JdbcResource.JDBC_URL, "");
if (!Strings.isNullOrEmpty(jdbcUrl)) {
jdbcUrl = JdbcResource.handleJdbcUrl(jdbcUrl);
@ -272,6 +274,7 @@ public class JdbcExternalCatalog extends ExternalCatalog {
catalogProperty.addProperty(JdbcResource.CHECK_SUM, computedChecksum);
}
}
testJdbcConnection();
}
/**
@ -313,22 +316,20 @@ public class JdbcExternalCatalog extends ExternalCatalog {
jdbcTable.setConnectionPoolKeepAlive(this.isConnectionPoolKeepAlive());
}
private void testJdbcConnection(boolean isReplay) throws DdlException {
private void testJdbcConnection() throws DdlException {
if (FeConstants.runningUnitTest) {
// skip test connection in unit test
return;
}
if (!isReplay) {
if (isTestConnection()) {
try {
initLocalObjectsImpl();
testFeToJdbcConnection();
testBeToJdbcConnection();
} finally {
if (jdbcClient != null) {
jdbcClient.closeClient();
jdbcClient = null;
}
if (isTestConnection()) {
try {
initLocalObjectsImpl();
testFeToJdbcConnection();
testBeToJdbcConnection();
} finally {
if (jdbcClient != null) {
jdbcClient.closeClient();
jdbcClient = null;
}
}
}