From ba82cd10c6397bc9c01bdb2b37c4f536daf5a5d3 Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Fri, 3 Mar 2023 00:47:46 +0800 Subject: [PATCH] [Enhencement](Jdbc catalog) Add two optional properties for jdbc catalog (#17245) 1. The first property is `only_specified_database`: In the past, `Jdbc Catalog` will synchronize all database from source database. Now we add a parameter called `only_specified_database` to jdbc catalog to allow only the specified database to be synchronized, eg: ```sql create resource if not exists ${resource_name} properties( "type"="jdbc", "user"="root", "password"="123456", "jdbc_url" = "jdbc:mysql://172.18.0.1:${mysql_port}/doris_test?useSSL=false", "driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar", "driver_class" = "com.mysql.cj.jdbc.Driver", "only_specified_database" = "true" ); ``` if `only_specified_database` is `true`, jdbc catalog will only synchronize the database which is specified in `jdbc_url`. 2. The second property is `lower_case_table_names`: This property will synchronize jdbc external data source table names in lower case. ```sql create resource if not exists ${resource_name} properties( "type"="jdbc", "user"="doris_test", "password"="123456", "jdbc_url" = "jdbc:oracle:thin:@172.18.0.1:${oracle_port}:${SID}", "driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/ojdbc8.jar", "driver_class" = "oracle.jdbc.driver.OracleDriver", "lower_case_table_names" = "true" ); ``` --- docs/en/docs/lakehouse/multi-catalog/jdbc.md | 36 ++++++------- .../docs/lakehouse/multi-catalog/jdbc.md | 44 +++++++-------- .../apache/doris/catalog/JdbcResource.java | 50 +++++++++++++++-- .../external/JdbcExternalDatabase.java | 1 - .../doris/datasource/JdbcExternalCatalog.java | 11 +++- .../doris/external/jdbc/JdbcClient.java | 53 ++++++++++++++++++- .../test_mysql_jdbc_catalog.out | 3 ++ .../test_oracle_jdbc_catalog.out | 16 ++++++ .../jdbc_catalog_p0/test_pg_jdbc_catalog.out | 3 ++ .../test_mysql_jdbc_catalog.groovy | 19 +++++++ .../test_oracle_jdbc_catalog.groovy | 41 +++++++++++++- .../test_pg_jdbc_catalog.groovy | 17 ++++++ 12 files changed, 242 insertions(+), 52 deletions(-) diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md b/docs/en/docs/lakehouse/multi-catalog/jdbc.md index 55d90b1a76..bd006608fe 100644 --- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md @@ -37,11 +37,9 @@ Once connected, Doris will ingest metadata of databases and tables from the exte ## Create Catalog - - 1. MySQL - + ```sql CREATE CATALOG jdbc_mysql PROPERTIES ( @@ -54,11 +52,9 @@ Once connected, Doris will ingest metadata of databases and tables from the exte ) ``` - - 2. PostgreSQL - + ```sql CREATE CATALOG jdbc_postgresql PROPERTIES ( @@ -79,11 +75,9 @@ Once connected, Doris will ingest metadata of databases and tables from the exte | Database | Schema | | Table | Table | - - 3. Oracle - + ```sql CREATE CATALOG jdbc_oracle PROPERTIES ( @@ -96,19 +90,17 @@ Once connected, Doris will ingest metadata of databases and tables from the exte ); ``` - As for data mapping from Oracle to Doris, one Database in Doris corresponds to one User (for example, "helowin" in `jdbc_url` above), and one Table in that Database corresponds to one table that the User has access to. In conclusion, the mapping relations are as follows: + As for data mapping from Oracle to Doris, one Database in Doris corresponds to one User, and one Table in that Database corresponds to one table that the User has access to. In conclusion, the mapping relations are as follows: - | Doris | PostgreSQL | + | Doris | Oracle | | -------- | ---------- | | Catalog | Database | | Database | User | | Table | Table | - - 4. Clickhouse - + ```sql CREATE CATALOG jdbc_clickhouse PROPERTIES ( @@ -121,11 +113,9 @@ Once connected, Doris will ingest metadata of databases and tables from the exte ); ``` - - 5. SQLServer - + ```sql CREATE CATALOG sqlserver_catalog PROPERTIES ( @@ -145,12 +135,10 @@ Once connected, Doris will ingest metadata of databases and tables from the exte | Catalog | Database | | Database | Schema | | Table | Table | - - 6. Doris - + Jdbc Catalog also support to connect another Doris database: @@ -176,6 +164,8 @@ Currently, Jdbc Catalog only support to use 5.x version of JDBC jar package to c | `jdbc_url ` | Yes | | JDBC connection string | | `driver_url ` | Yes | | JDBC Driver Jar | | `driver_class ` | Yes | | JDBC Driver Class | +| `only_specified_database` | No | "false" | Whether only the database specified to be synchronized. | +| `lower_case_table_names` | No | "false" | Whether to synchronize jdbc external data source table names in lower case. | > `driver_url` can be specified in three ways: > @@ -185,6 +175,12 @@ Currently, Jdbc Catalog only support to use 5.x version of JDBC jar package to c > > 3. HTTP address. For example, `https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-5.1.47.jar`. The system will download the Driver file from the HTTP address. This only supports HTTP services with no authentication requirements. +> `only_specified_database`: +> +> When the JDBC is connected, you can specify which database/schema to connect. For example, you can specify the DataBase in mysql `jdbc_url`; you can specify the CurrentSchema in PG `jdbc_url`. `only_specified_database` specifies whether only the database specified to be synchronized. +> +> If you connect the Oracle database when using this property, please use the version of the jar package above 8 or more (such as ojdbc8.jar). + ## Query diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md index 2a894727ca..0556eb72dc 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md @@ -37,11 +37,9 @@ JDBC Catalog 通过标准 JDBC 协议，连接其他数据源。 ## 创建 Catalog - - 1. MySQL - + ```sql CREATE CATALOG jdbc_mysql PROPERTIES ( @@ -54,11 +52,9 @@ CREATE CATALOG jdbc_mysql PROPERTIES ( ) ``` - - 2. PostgreSQL - + ```sql CREATE CATALOG jdbc_postgresql PROPERTIES ( @@ -71,7 +67,7 @@ CREATE CATALOG jdbc_postgresql PROPERTIES ( ); ``` -映射 PostgreSQL 时，Doris 的一个 Database 对应于 PostgreSQL 中指定 Catalog（如示例中 `jdbc_url` 参数中 "demo"）下的一个 Schema。而 Doris 的 Database 下的 Table 则对应于 PostgreSQL 中，Schema 下的 Tables。即映射关系如下： +映射 PostgreSQL 时，Doris 的一个 Database 对应于 PostgreSQL 中指定Catalog下的一个 Schema（如示例中 `jdbc_url` 参数中 "demo"下的schemas）。而 Doris 的 Database 下的 Table 则对应于 PostgreSQL 中，Schema 下的 Tables。即映射关系如下： |Doris | PostgreSQL | |---|---| @@ -79,11 +75,9 @@ CREATE CATALOG jdbc_postgresql PROPERTIES ( | Database | Schema | | Table | Table | - - 3. Oracle - + ```sql CREATE CATALOG jdbc_oracle PROPERTIES ( @@ -96,19 +90,17 @@ CREATE CATALOG jdbc_oracle PROPERTIES ( ); ``` -映射 Oracle 时，Doris 的一个 Database 对应于 Oracle 中的一个 User（如示例中 `jdbc_url` 参数中 "helowin"）。而 Doris 的 Database 下的 Table 则对应于 Oracle 中，该 User 下的有权限访问的 Table。即映射关系如下： +映射 Oracle 时，Doris 的一个 Database 对应于 Oracle 中的一个 User。而 Doris 的 Database 下的 Table 则对应于 Oracle 中，该 User 下的有权限访问的 Table。即映射关系如下： -|Doris | PostgreSQL | +|Doris | Oracle | |---|---| | Catalog | Database | | Database | User | | Table | Table | - - 4. Clickhouse - + ```sql CREATE CATALOG jdbc_clickhouse PROPERTIES ( @@ -121,11 +113,9 @@ CREATE CATALOG jdbc_clickhouse PROPERTIES ( ); ``` - - 5. SQLServer - + ```sql CREATE CATALOG sqlserver_catalog PROPERTIES ( @@ -146,11 +136,9 @@ CREATE CATALOG sqlserver_catalog PROPERTIES ( | Database | Schema | | Table | Table | - - 6. Doris - + Jdbc Catalog也支持连接另一个Doris数据库： @@ -173,9 +161,11 @@ CREATE CATALOG doris_catalog PROPERTIES ( --- | --- | --- | --- `user` | 是 | | 对应数据库的用户名 | `password` | 是 | | 对应数据库的密码 | -`jdbc_url ` | 是 | | JDBC 连接串 | -`driver_url ` | 是 | | JDBC Driver Jar 包名称* | -`driver_class ` | 是 | | JDBC Driver Class 名称 | +`jdbc_url` | 是 | | JDBC 连接串 | +`driver_url` | 是 | | JDBC Driver Jar 包名称* | +`driver_class` | 是 | | JDBC Driver Class 名称 | +`only_specified_database` | 否 | "false" | 指定是否只同步指定的 database | +`lower_case_table_names` | 否 | "false" | 是否以小写的形式同步jdbc外部数据源的表名 | > `driver_url` 可以通过以下三种方式指定： > @@ -185,6 +175,12 @@ CREATE CATALOG doris_catalog PROPERTIES ( > > 3. Http 地址。如：`https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-5.1.47.jar`。系统会从这个 http 地址下载 Driver 文件。仅支持无认证的 http 服务。 +> `only_specified_database`: +> +> 在jdbc连接时可以指定链接到哪个database/schema, 如：mysql中jdbc_url中可以指定database, pg的jdbc_url中可以指定currentSchema。`only_specified_database=true` 可以只同步指定的 database。 +> +> 如果使用该参数时连接oracle数据库，要求使用ojdbc8.jar以上版本jar包。 + ## 数据查询 ```sql diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index d43098b7db..ce9baf316c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -39,6 +39,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.List; import java.util.Map; @@ -80,8 +81,23 @@ public class JdbcResource extends Resource { public static final String DRIVER_CLASS = "driver_class"; public static final String DRIVER_URL = "driver_url"; public static final String TYPE = "type"; + public static final String ONLY_SPECIFIED_DATABASE = "only_specified_database"; + public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; public static final String CHECK_SUM = "checksum"; + private static final List OPTIONAL_PROPERTIES = Lists.newArrayList( + ONLY_SPECIFIED_DATABASE, + LOWER_CASE_TABLE_NAMES + ); + + // The default value of optional properties + private static final Map OPTIONAL_PROPERTIES_DEFAULT_VALUE = Maps.newHashMap(); + + static { + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ONLY_SPECIFIED_DATABASE, "false"); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(LOWER_CASE_TABLE_NAMES, "false"); + } + // timeout for both connection and read. 10 seconds is long enough. private static final int HTTP_TIMEOUT_MS = 10000; @SerializedName(value = "configs") @@ -121,6 +137,8 @@ public class JdbcResource extends Resource { replaceIfEffectiveValue(this.configs, USER, properties.get(USER)); replaceIfEffectiveValue(this.configs, PASSWORD, properties.get(PASSWORD)); replaceIfEffectiveValue(this.configs, TYPE, properties.get(TYPE)); + replaceIfEffectiveValue(this.configs, ONLY_SPECIFIED_DATABASE, properties.get(ONLY_SPECIFIED_DATABASE)); + replaceIfEffectiveValue(this.configs, LOWER_CASE_TABLE_NAMES, properties.get(LOWER_CASE_TABLE_NAMES)); this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL))); super.modifyProperties(properties); } @@ -135,6 +153,8 @@ public class JdbcResource extends Resource { copiedProperties.remove(USER); copiedProperties.remove(PASSWORD); copiedProperties.remove(TYPE); + copiedProperties.remove(ONLY_SPECIFIED_DATABASE); + copiedProperties.remove(LOWER_CASE_TABLE_NAMES); if (!copiedProperties.isEmpty()) { throw new AnalysisException("Unknown JDBC catalog resource properties: " + copiedProperties); } @@ -144,22 +164,46 @@ public class JdbcResource extends Resource { protected void setProperties(Map properties) throws DdlException { Preconditions.checkState(properties != null); for (String key : properties.keySet()) { - if (!DRIVER_URL.equals(key) && !JDBC_URL.equals(key) && !USER.equals(key) && !PASSWORD.equals(key) - && !TYPE.equals(key) && !DRIVER_CLASS.equals(key)) { - throw new DdlException("JDBC resource Property of " + key + " is unknown"); + switch (key) { + case DRIVER_URL: + case JDBC_URL: + case USER: + case PASSWORD: + case TYPE: + case DRIVER_CLASS: + case ONLY_SPECIFIED_DATABASE: // optional argument + case LOWER_CASE_TABLE_NAMES: // optional argument + break; + default: + throw new DdlException("JDBC resource Property of " + key + " is unknown"); } } configs = properties; + handleOptionalArguments(); checkProperties(DRIVER_URL); checkProperties(DRIVER_CLASS); checkProperties(JDBC_URL); checkProperties(USER); checkProperties(PASSWORD); checkProperties(TYPE); + checkProperties(ONLY_SPECIFIED_DATABASE); + checkProperties(LOWER_CASE_TABLE_NAMES); this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL))); configs.put(CHECK_SUM, computeObjectChecksum(getProperty(DRIVER_URL))); } + /** + * This function used to handle optional arguments + * eg: only_specified_database、lower_case_table_names + */ + private void handleOptionalArguments() { + for (String s : OPTIONAL_PROPERTIES) { + if (!configs.containsKey(s)) { + configs.put(s, OPTIONAL_PROPERTIES_DEFAULT_VALUE.get(s)); + } + } + } + @Override public Map getCopiedProperties() { return Maps.newHashMap(configs); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java index f0772d8f26..6eb6340f94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java @@ -55,7 +55,6 @@ public class JdbcExternalDatabase extends ExternalDatabase im super(extCatalog, id, name); } - // TODO(ftw): drew out the public multiple parts @Override protected void init() { InitDatabaseLog initDatabaseLog = new InitDatabaseLog(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java index 1bcf418570..4fd92a2c07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java @@ -120,9 +120,18 @@ public class JdbcExternalCatalog extends ExternalCatalog { return catalogProperty.getOrDefault(JdbcResource.CHECK_SUM, ""); } + public String getOnlySpecifiedDatabase() { + return catalogProperty.getOrDefault(JdbcResource.ONLY_SPECIFIED_DATABASE, "false"); + } + + public String getLowerCaseTableNames() { + return catalogProperty.getOrDefault(JdbcResource.LOWER_CASE_TABLE_NAMES, "false"); + } + @Override protected void initLocalObjectsImpl() { - jdbcClient = new JdbcClient(getJdbcUser(), getJdbcPasswd(), getJdbcUrl(), getDriverUrl(), getDriverClass()); + jdbcClient = new JdbcClient(getJdbcUser(), getJdbcPasswd(), getJdbcUrl(), getDriverUrl(), getDriverClass(), + getOnlySpecifiedDatabase(), getLowerCaseTableNames()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java index c46235d3c3..7a08bddda5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java @@ -26,6 +26,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import lombok.Data; @@ -42,6 +43,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.List; +import java.util.Map; @Getter public class JdbcClient { @@ -56,8 +58,18 @@ public class JdbcClient { private HikariDataSource dataSource = null; - public JdbcClient(String user, String password, String jdbcUrl, String driverUrl, String driverClass) { + private boolean isOnlySpecifiedDatabase = false; + + private boolean isLowerCaseTableNames = false; + + // only used when isLowerCaseTableNames = true. + private Map lowerTableToRealTable = Maps.newHashMap(); + + public JdbcClient(String user, String password, String jdbcUrl, String driverUrl, String driverClass, + String onlySpecifiedDatabase, String isLowerCaseTableNames) { this.jdbcUser = user; + this.isOnlySpecifiedDatabase = Boolean.valueOf(onlySpecifiedDatabase).booleanValue(); + this.isLowerCaseTableNames = Boolean.valueOf(isLowerCaseTableNames).booleanValue(); try { this.dbType = JdbcResource.parseDbType(jdbcUrl); } catch (DdlException e) { @@ -153,6 +165,9 @@ public class JdbcClient { Connection conn = getConnection(); Statement stmt = null; ResultSet rs = null; + if (isOnlySpecifiedDatabase) { + return getSpecifiedDatabase(conn); + } List databaseNames = Lists.newArrayList(); try { stmt = conn.createStatement(); @@ -186,6 +201,30 @@ public class JdbcClient { return databaseNames; } + public List getSpecifiedDatabase(Connection conn) { + List databaseNames = Lists.newArrayList(); + try { + switch (dbType) { + case JdbcResource.MYSQL: + case JdbcResource.CLICKHOUSE: + databaseNames.add(conn.getCatalog()); + break; + case JdbcResource.POSTGRESQL: + case JdbcResource.ORACLE: + case JdbcResource.SQLSERVER: + databaseNames.add(conn.getSchema()); + break; + default: + throw new JdbcClientException("Not supported jdbc type"); + } + } catch (SQLException e) { + throw new JdbcClientException("failed to get specified database name from jdbc", e); + } finally { + close(conn); + } + return databaseNames; + } + /** * get all tables of one database */ @@ -210,7 +249,12 @@ public class JdbcClient { throw new JdbcClientException("Unknown database type"); } while (rs.next()) { - tablesName.add(rs.getString("TABLE_NAME")); + String tableName = rs.getString("TABLE_NAME"); + if (isLowerCaseTableNames) { + lowerTableToRealTable.put(tableName.toLowerCase(), tableName); + tableName = tableName.toLowerCase(); + } + tablesName.add(tableName); } } catch (SQLException e) { throw new JdbcClientException("failed to get all tables for db %s", e, dbName); @@ -280,6 +324,11 @@ public class JdbcClient { Connection conn = getConnection(); ResultSet rs = null; List tableSchema = Lists.newArrayList(); + // if isLowerCaseTableNames == true, tableName is lower case + // but databaseMetaData.getColumns() is case sensitive + if (isLowerCaseTableNames) { + tableName = lowerTableToRealTable.get(tableName); + } try { DatabaseMetaData databaseMetaData = conn.getMetaData(); // getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) diff --git a/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out b/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out index 7b1a18c3a9..b7d6e8346e 100644 --- a/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out +++ b/regression-test/data/jdbc_catalog_p0/test_mysql_jdbc_catalog.out @@ -172,6 +172,9 @@ doris3 20 -- !test_insert4 -- 1 abcHa1.12345 1.123450xkalowadawd 2022-10-01 3.14159 1 2 0 100000 1.2345678 24.000 07:09:51 2022 2022-11-27T07:09:51 2022-11-27T07:09:51 +-- !specified_database -- +doris_test + -- !ex_tb1 -- {"k1":"v1", "k2":"v2"} diff --git a/regression-test/data/jdbc_catalog_p0/test_oracle_jdbc_catalog.out b/regression-test/data/jdbc_catalog_p0/test_oracle_jdbc_catalog.out index 67595ff0a9..fdfda85e67 100644 --- a/regression-test/data/jdbc_catalog_p0/test_oracle_jdbc_catalog.out +++ b/regression-test/data/jdbc_catalog_p0/test_oracle_jdbc_catalog.out @@ -79,3 +79,19 @@ doris2 19 doris3 20 doris3 20 +-- !specified_database -- +DORIS_TEST + +-- !lower_case_table_names1 -- +1 111 123 7456123.89 573 34 673.43 34.1264 60.0 23.231 + +-- !lower_case_table_names2 -- +1 1 china beijing alice abcdefghrjkmnopq +2 2 china shanghai bob abcdefghrjkmnopq +3 3 Americ new york Jerry abcdefghrjkmnopq + +-- !lower_case_table_names3 -- +1 99 9999 999999999 999999999999999999 999 99999 9999999999 9999999999999999999 +2 -99 -9999 -999999999 -999999999999999999 -999 -99999 -9999999999 -9999999999999999999 +3 10 100 999999999 999999999999999999 999 99999 9999999999 9999999999999999999 + diff --git a/regression-test/data/jdbc_catalog_p0/test_pg_jdbc_catalog.out b/regression-test/data/jdbc_catalog_p0/test_pg_jdbc_catalog.out index ac0ff72029..67a1b4914c 100644 --- a/regression-test/data/jdbc_catalog_p0/test_pg_jdbc_catalog.out +++ b/regression-test/data/jdbc_catalog_p0/test_pg_jdbc_catalog.out @@ -2149,6 +2149,9 @@ doris2 19 doris3 20 doris3 20 +-- !specified_database -- +doris_test + -- !test_old -- 123 abc 123 abc diff --git a/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog.groovy b/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog.groovy index a57aa6a4e8..35246942e5 100644 --- a/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog.groovy @@ -121,6 +121,25 @@ suite("test_mysql_jdbc_catalog", "p0") { sql """ drop catalog if exists ${catalog_name} """ sql """ drop resource if exists ${resource_name} """ + // test only_specified_database argument + sql """create resource if not exists ${resource_name} properties( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:mysql://127.0.0.1:${mysql_port}/doris_test?useSSL=false", + "driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true" + );""" + + sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}""" + sql """switch ${catalog_name}""" + + qt_specified_database """ show databases; """ + + sql """ drop catalog if exists ${catalog_name} """ + sql """ drop resource if exists ${resource_name} """ + // test old create-catalog syntax for compatibility sql """ CREATE CATALOG ${catalog_name} PROPERTIES ( "type"="jdbc", diff --git a/regression-test/suites/jdbc_catalog_p0/test_oracle_jdbc_catalog.groovy b/regression-test/suites/jdbc_catalog_p0/test_oracle_jdbc_catalog.groovy index 078e97df4d..ee20b76706 100644 --- a/regression-test/suites/jdbc_catalog_p0/test_oracle_jdbc_catalog.groovy +++ b/regression-test/suites/jdbc_catalog_p0/test_oracle_jdbc_catalog.groovy @@ -86,6 +86,45 @@ suite("test_oracle_jdbc_catalog", "p0") { order_qt_test_insert3 """ select name, age from ${test_insert} where id = '${uuid2}' order by age """ sql """drop catalog if exists ${catalog_name} """ - sql """drop resource if exists jdbc_resource_catalog_pg""" + sql """drop resource if exists ${resource_name}""" + + // test only_specified_database argument + sql """create resource if not exists ${resource_name} properties( + "type"="jdbc", + "user"="doris_test", + "password"="123456", + "jdbc_url" = "jdbc:oracle:thin:@127.0.0.1:${oracle_port}:${SID}", + "driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/ojdbc8.jar", + "driver_class" = "oracle.jdbc.driver.OracleDriver", + "only_specified_database" = "true" + );""" + sql """ CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ + sql """ switch ${catalog_name} """ + + qt_specified_database """ show databases; """ + sql """drop catalog if exists ${catalog_name} """ + sql """drop resource if exists ${resource_name}""" + + // test lower_case_table_names argument + sql """create resource if not exists ${resource_name} properties( + "type"="jdbc", + "user"="doris_test", + "password"="123456", + "jdbc_url" = "jdbc:oracle:thin:@127.0.0.1:${oracle_port}:${SID}", + "driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/ojdbc8.jar", + "driver_class" = "oracle.jdbc.driver.OracleDriver", + "lower_case_table_names" = "true" + );""" + sql """ CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ + sql """ switch ${catalog_name} """ + sql """ use ${ex_db_name}""" + + qt_lower_case_table_names1 """ select * from test_num order by ID; """ + qt_lower_case_table_names2 """ select * from test_char order by ID; """ + qt_lower_case_table_names3 """ select * from test_int order by ID; """ + + sql """drop catalog if exists ${catalog_name} """ + sql """drop resource if exists ${resource_name}""" + } } diff --git a/regression-test/suites/jdbc_catalog_p0/test_pg_jdbc_catalog.groovy b/regression-test/suites/jdbc_catalog_p0/test_pg_jdbc_catalog.groovy index 883d15be9e..d4e5036a22 100644 --- a/regression-test/suites/jdbc_catalog_p0/test_pg_jdbc_catalog.groovy +++ b/regression-test/suites/jdbc_catalog_p0/test_pg_jdbc_catalog.groovy @@ -89,6 +89,23 @@ suite("test_pg_jdbc_catalog", "p0") { sql """drop catalog if exists ${catalog_name} """ sql """drop resource if exists ${resource_name}""" + // test only_specified_database argument + sql """create resource if not exists ${resource_name} properties( + "type"="jdbc", + "user"="postgres", + "password"="123456", + "jdbc_url" = "jdbc:postgresql://127.0.0.1:${pg_port}/postgres?currentSchema=doris_test&useSSL=false", + "driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/postgresql-42.5.0.jar", + "driver_class" = "org.postgresql.Driver", + "only_specified_database" = "true" + );""" + sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """ + sql """switch ${catalog_name} """ + qt_specified_database """ show databases; """ + + sql """drop catalog if exists ${catalog_name} """ + sql """drop resource if exists ${resource_name}""" + // test old create-catalog syntax for compatibility sql """ CREATE CATALOG ${catalog_name} PROPERTIES ( "type"="jdbc",