From b477839bce30da8e3d9110f270362c8b50aafd08 Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Fri, 17 Nov 2023 23:51:47 +0800 Subject: [PATCH] [enhancement](jdbc catalog) Add lowercase column name mapping to Jdbc data source & optimize database and table mapping (#27124) This PR adds the processing of lowercase Column names in Oracle Jdbc Catalog. In the previous behavior, we changed all Oracle columns to uppercase queries by default, but could not handle the lowercase case. This PR can solve this situation and improve All Jdbc Catalog works --- .../oracle/init/03-create-table.sql | 7 ++ .../docker-compose/oracle/init/04-insert.sql | 2 + docs/en/docs/lakehouse/multi-catalog/jdbc.md | 19 ++- .../docs/lakehouse/multi-catalog/jdbc.md | 35 +++--- .../org/apache/doris/catalog/JdbcTable.java | 41 +++++-- .../catalog/external/JdbcExternalTable.java | 2 + .../datasource/jdbc/client/JdbcClient.java | 108 +++++++++++------- .../jdbc/client/JdbcMySQLClient.java | 23 ++-- .../jdbc/client/JdbcOracleClient.java | 25 ++-- .../planner/external/jdbc/JdbcScanNode.java | 16 ++- .../planner/external/odbc/OdbcScanNode.java | 2 +- .../jdbc/test_oracle_jdbc_catalog.out | 9 ++ .../jdbc/test_oracle_jdbc_catalog.groovy | 3 + 13 files changed, 196 insertions(+), 96 deletions(-) diff --git a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql index 046f18c0dc..24bf6a7a85 100644 --- a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql @@ -132,6 +132,13 @@ age number(2), score number(3,1) ); +CREATE TABLE "DORIS_TEST"."student3" +( +"id" NUMBER(5,0), +"NAME" VARCHAR2(20), +"AGE" NUMBER(2,0), +"SCORE" NUMBER(3,1) +); create table doris_test.test_all_types ( id int, diff --git a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql index 888cdf5bf7..663851cfa4 100644 --- a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql @@ -86,6 +86,8 @@ insert into doris_test."student2" values (2, 'bob', 21, 90.5); insert into doris_test."student2" values (3, 'jerry', 23, 88.0); insert into doris_test."student2" values (4, 'andy', 21, 93); +insert into doris_test."student3" values(1, 'doris', 3, 1.0); + insert into doris_test.test_all_types values (1, 111, 123, 7456123.89, 573, 34, 673.43, 34.1264, 56.2, 23.231, 99, 9999, 999999999, 999999999999999999, 999, 99999, 9999999999, 9999999999999999999, diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md b/docs/en/docs/lakehouse/multi-catalog/jdbc.md index 06385bf529..b31bd7f99f 100644 --- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md @@ -52,7 +52,7 @@ PROPERTIES ("key"="value", ...) | `driver_url ` | Yes | | JDBC Driver Jar | | `driver_class ` | Yes | | JDBC Driver Class | | `only_specified_database` | No | "false" | Whether only the database specified to be synchronized. | -| `lower_case_table_names` | No | "false" | Whether to synchronize jdbc external data source table names in lower case. | +| `lower_case_table_names` | No | "false" | Whether to synchronize the database name, table name and column name of jdbc external data source in lowercase. | | `include_database_list` | No | "" | When only_specified_database=true，only synchronize the specified databases. split with ','. db name is case sensitive. | | `exclude_database_list` | No | "" | When only_specified_database=true，do not synchronize the specified databases. split with ','. db name is case sensitive. | @@ -68,7 +68,7 @@ PROPERTIES ("key"="value", ...) ### Lowercase table name synchronization -When `lower_case_table_names` is set to `true`, Doris is able to query non-lowercase databases and tables by maintaining a mapping of lowercase names to actual names on the remote system +When `lower_case_table_names` is set to `true`, Doris is able to query non-lowercase databases and tables and columns by maintaining a mapping of lowercase names to actual names on the remote system **Notice:** @@ -78,9 +78,9 @@ When `lower_case_table_names` is set to `true`, Doris is able to query non-lower For other databases, you still need to specify the real library name and table name when querying. -2. In Doris 2.0.3 and later versions, it is valid for all databases. When querying, all library names and table names will be converted into real names and then queried. If you upgrade from an old version to 2.0. 3, `Refresh ` is required to take effect. +2. In Doris 2.0.3 and later versions, it is valid for all databases. When querying, all database names and table names and columns will be converted into real names and then queried. If you upgrade from an old version to 2.0. 3, `Refresh ` is required to take effect. - However, if the database or table names differ only in case, such as `Doris` and `doris`, Doris cannot query them due to ambiguity. + However, if the database or table or column names differ only in case, such as `Doris` and `doris`, Doris cannot query them due to ambiguity. 3. When the FE parameter's `lower_case_table_names` is set to `1` or `2`, the JDBC Catalog's `lower_case_table_names` parameter must be set to `true`. If the FE parameter's `lower_case_table_names` is set to `0`, the JDBC Catalog parameter can be `true` or `false` and defaults to `false`. This ensures consistency and predictability in how Doris handles internal and external table configurations. @@ -113,8 +113,8 @@ In some cases, the keywords in the database might be used as the field names. Fo ### Predicate Pushdown 1. When executing a query like `where dt = '2022-01-01'`, Doris can push down these filtering conditions to the external data source, thereby directly excluding data that does not meet the conditions at the data source level, reducing the number of unqualified Necessary data acquisition and transfer. This greatly improves query performance while also reducing the load on external data sources. - -2. When `enable_func_pushdown` is set to true, the function condition after where will also be pushed down to the external data source. Currently, only MySQL is supported. If you encounter a function that MySQL does not support, you can set this parameter to false, at present, Doris will automatically identify some functions not supported by MySQL to filter the push-down conditions, which can be checked by explain sql. + +2. When `enable_func_pushdown` is set to true, the function conditions after where will also be pushed down to the external data source. Currently, only MySQL and ClickHouse are supported. If you encounter a function that is not supported by MySQL or ClickHouse, you can set this parameter to false. , currently Doris will automatically identify some functions not supported by MySQL and functions supported by CLickHouse for push-down condition filtering, which can be viewed through explain sql. Functions that are currently not pushed down include: @@ -123,6 +123,13 @@ Functions that are currently not pushed down include: | DATE_TRUNC | | MONEY_FORMAT | +Functions that are currently pushed down include: + +| ClickHouse | +|:--------------:| +| FROM_UNIXTIME | +| UNIX_TIMESTAMP | + ### Line Limit If there is a limit keyword in the query, Doris will translate it into semantics suitable for different data sources. diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md index 83fe819417..48fb1f253f 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md @@ -44,16 +44,16 @@ PROPERTIES ("key"="value", ...) ## 参数说明 -| 参数 | 必须 | 默认值 | 说明 | -|---------------------------|-----|---------|---------------------------------------------------------------------------------------------| -| `user` | 是 | | 对应数据库的用户名 | -| `password` | 是 | | 对应数据库的密码 | -| `jdbc_url` | 是 | | JDBC 连接串 | -| `driver_url` | 是 | | JDBC Driver Jar 包名称 | -| `driver_class` | 是 | | JDBC Driver Class 名称 | -| `lower_case_table_names` | 否 | "false" | 是否以小写的形式同步jdbc外部数据源的库名和表名 | -| `only_specified_database` | 否 | "false" | 指定是否只同步指定的 database | -| `include_database_list` | 否 | "" | 当only_specified_database=true时，指定同步多个database，以','分隔。db名称是大小写敏感的。 | +| 参数 | 必须 | 默认值 | 说明 | +|---------------------------|-----|---------|-----------------------------------------------------------------------| +| `user` | 是 | | 对应数据库的用户名 | +| `password` | 是 | | 对应数据库的密码 | +| `jdbc_url` | 是 | | JDBC 连接串 | +| `driver_url` | 是 | | JDBC Driver Jar 包名称 | +| `driver_class` | 是 | | JDBC Driver Class 名称 | +| `lower_case_table_names` | 否 | "false" | 是否以小写的形式同步jdbc外部数据源的库名和表名以及列名 | +| `only_specified_database` | 否 | "false" | 指定是否只同步指定的 database | +| `include_database_list` | 否 | "" | 当only_specified_database=true时，指定同步多个database，以','分隔。db名称是大小写敏感的。 | | `exclude_database_list` | 否 | "" | 当only_specified_database=true时，指定不需要同步的多个database，以','分割。db名称是大小写敏感的。 | ### 驱动包路径 @@ -68,7 +68,7 @@ PROPERTIES ("key"="value", ...) ### 小写表名同步 -当 `lower_case_table_names` 设置为 `true` 时，Doris 通过维护小写名称到远程系统中实际名称的映射，能够查询非小写的数据库和表 +当 `lower_case_table_names` 设置为 `true` 时，Doris 通过维护小写名称到远程系统中实际名称的映射，能够查询非小写的数据库和表以及列 **注意：** @@ -78,9 +78,9 @@ PROPERTIES ("key"="value", ...) 对于其他数据库，仍需要在查询时指定真实的库名和表名。 -2. 在 Doris 2.0.3 及之后的版本，对所有的数据库都有效，在查询时，会将所有的库名和表名转换为真实的名称，再去查询，如果是从老版本升级到 2.0.3 ，需要 `Refresh ` 才能生效。 +2. 在 Doris 2.0.3 及之后的版本，对所有的数据库都有效，在查询时，会将所有的库名和表名以及列名转换为真实的名称，再去查询，如果是从老版本升级到 2.0.3 ，需要 `Refresh ` 才能生效。 - 但是，如果数据库或者表名只有大小写不同，例如 `Doris` 和 `doris`，则 Doris 由于歧义而无法查询它们。 + 但是，如果库名、表名或列名只有大小写不同，例如 `Doris` 和 `doris`，则 Doris 由于歧义而无法查询它们。 3. 当 FE 参数的 `lower_case_table_names` 设置为 `1` 或 `2` 时，JDBC Catalog 的 `lower_case_table_names` 参数必须设置为 `true`。如果 FE 参数的 `lower_case_table_names` 设置为 `0`，则 JDBC Catalog 的参数可以为 `true` 或 `false`，默认为 `false`。这确保了 Doris 在处理内部和外部表配置时的一致性和可预测性。 @@ -114,7 +114,7 @@ select * from mysql_catalog.mysql_database.mysql_table where k1 > 1000 and k3 =' 1. 当执行类似于 `where dt = '2022-01-01'` 这样的查询时，Doris 能够将这些过滤条件下推到外部数据源，从而直接在数据源层面排除不符合条件的数据，减少了不必要的数据获取和传输。这大大提高了查询性能，同时也降低了对外部数据源的负载。 -2. 当 `enable_func_pushdown` 设置为true，会将 where 之后的函数条件也下推到外部数据源，目前仅支持 MySQL，如遇到 MySQL 不支持的函数，可以将此参数设置为 false，目前 Doris 会自动识别部分 MySQL 不支持的函数进行下推条件过滤，可通过 explain sql 查看。 +2. 当 `enable_func_pushdown` 设置为true，会将 where 之后的函数条件也下推到外部数据源，目前仅支持 MySQL 以及 ClickHouse，如遇到 MySQL 或 ClickHouse 不支持的函数，可以将此参数设置为 false，目前 Doris 会自动识别部分 MySQL 不支持的函数以及 CLickHouse 支持的函数进行下推条件过滤，可通过 explain sql 查看。 目前不会下推的函数有： @@ -123,6 +123,13 @@ select * from mysql_catalog.mysql_database.mysql_table where k1 > 1000 and k3 =' | DATE_TRUNC | | MONEY_FORMAT | +目前会下推的函数有： + +| ClickHouse | +|:--------------:| +| FROM_UNIXTIME | +| UNIX_TIMESTAMP | + ### 行数限制 如果在查询中带有 limit 关键字，Doris 会将其转译成适合不同数据源的语义。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index 795f00e7bf..fe397929ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -27,6 +27,8 @@ import org.apache.doris.thrift.TOdbcTableType; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.collect.Maps; import lombok.Setter; @@ -47,9 +49,12 @@ import java.util.stream.Collectors; public class JdbcTable extends Table { private static final Logger LOG = LogManager.getLogger(JdbcTable.class); + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final String TABLE = "table"; private static final String REAL_DATABASE = "real_database"; private static final String REAL_TABLE = "real_table"; + private static final String REAL_COLUMNS = "real_columns"; private static final String RESOURCE = "resource"; private static final String TABLE_TYPE = "table_type"; private static final String URL = "jdbc_url"; @@ -65,6 +70,7 @@ public class JdbcTable extends Table { // real name only for jdbc catalog private String realDatabaseName; private String realTableName; + private Map realColumnNames; private String jdbcTypeName; @@ -110,7 +116,7 @@ public class JdbcTable extends Table { sb.append(getProperRealFullTableName(TABLE_TYPE_MAP.get(getTableTypeName()))); sb.append("("); List transformedInsertCols = insertCols.stream() - .map(col -> databaseProperName(TABLE_TYPE_MAP.get(getTableTypeName()), col)) + .map(col -> getProperRealColumnName(TABLE_TYPE_MAP.get(getTableTypeName()), col)) .collect(Collectors.toList()); sb.append(String.join(",", transformedInsertCols)); sb.append(")"); @@ -200,6 +206,7 @@ public class JdbcTable extends Table { serializeMap.put(CHECK_SUM, checkSum); serializeMap.put(REAL_DATABASE, realDatabaseName); serializeMap.put(REAL_TABLE, realTableName); + serializeMap.put(REAL_COLUMNS, objectMapper.writeValueAsString(realColumnNames)); int size = (int) serializeMap.values().stream().filter(v -> { return v != null; @@ -236,6 +243,9 @@ public class JdbcTable extends Table { checkSum = serializeMap.get(CHECK_SUM); realDatabaseName = serializeMap.get(REAL_DATABASE); realTableName = serializeMap.get(REAL_TABLE); + String realColumnNamesJson = serializeMap.get(REAL_COLUMNS); + realColumnNames = objectMapper.readValue(realColumnNamesJson, new TypeReference>() { + }); } public String getResourceName() { @@ -263,6 +273,14 @@ public class JdbcTable extends Table { } } + public String getProperRealColumnName(TOdbcTableType tableType, String columnName) { + if (realColumnNames == null || realColumnNames.isEmpty() || !realColumnNames.containsKey(columnName)) { + return databaseProperName(tableType, columnName); + } else { + return properNameWithRealName(tableType, realColumnNames.get(columnName)); + } + } + public String getTableTypeName() { return jdbcTypeName; } @@ -358,14 +376,13 @@ public class JdbcTable extends Table { * @param wrapEnd The character(s) to be added at the end of each name component. * @param toUpperCase If true, convert the name to upper case. * @param toLowerCase If true, convert the name to lower case. - *

- * Note: If both toUpperCase and toLowerCase are true, the name will ultimately be converted to lower case. - *

- * The name is expected to be in the format of 'schemaName.tableName'. If there is no '.', - * the function will treat the entire string as one name component. - * If there is a '.', the function will treat the string before the first '.' as the schema name - * and the string after the '.' as the table name. - * + *

+ * Note: If both toUpperCase and toLowerCase are true, the name will ultimately be converted to lower case. + *

+ * The name is expected to be in the format of 'schemaName.tableName'. If there is no '.', + * the function will treat the entire string as one name component. + * If there is a '.', the function will treat the string before the first '.' as the schema name + * and the string after the '.' as the table name. * @return The formatted name. */ public static String formatName(String name, String wrapStart, String wrapEnd, boolean toUpperCase, @@ -386,18 +403,18 @@ public class JdbcTable extends Table { /** * Formats a database name according to the database type. - * + *

* Rules: * - MYSQL, OCEANBASE: Wrap with backticks (`), case unchanged. Example: mySchema.myTable -> `mySchema.myTable` * - SQLSERVER: Wrap with square brackets ([]), case unchanged. Example: mySchema.myTable -> [mySchema].[myTable] * - POSTGRESQL, CLICKHOUSE, TRINO, OCEANBASE_ORACLE, SAP_HANA: Wrap with double quotes ("), case unchanged. - * Example: mySchema.myTable -> "mySchema"."myTable" + * Example: mySchema.myTable -> "mySchema"."myTable" * - ORACLE: Wrap with double quotes ("), convert to upper case. Example: mySchema.myTable -> "MYSCHEMA"."MYTABLE" * For other types, the name is returned as is. * * @param tableType The database type. * @param name The name to be formatted, expected in 'schemaName.tableName' format. If no '.', treats entire string - * as one name component. If '.', treats string before first '.' as schema name and after as table name. + * as one name component. If '.', treats string before first '.' as schema name and after as table name. * @return The formatted name. */ public static String databaseProperName(TOdbcTableType tableType, String name) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java index a02c59080f..9c5159a2bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java @@ -91,6 +91,8 @@ public class JdbcExternalTable extends ExternalTable { jdbcTable.setRealDatabaseName(((JdbcExternalCatalog) catalog).getJdbcClient().getRealDatabaseName(this.dbName)); jdbcTable.setRealTableName( ((JdbcExternalCatalog) catalog).getJdbcClient().getRealTableName(this.dbName, this.name)); + jdbcTable.setRealColumnNames(((JdbcExternalCatalog) catalog).getJdbcClient().getRealColumnNames(this.dbName, + this.name)); jdbcTable.setJdbcTypeName(jdbcCatalog.getDatabaseTypeName()); jdbcTable.setJdbcUrl(jdbcCatalog.getJdbcUrl()); jdbcTable.setJdbcUser(jdbcCatalog.getJdbcUser()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index 97052caccd..b4e2604bfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -67,10 +67,15 @@ public abstract class JdbcClient { // only used when isLowerCaseTableNames = true. protected final ConcurrentHashMap lowerDBToRealDB = new ConcurrentHashMap<>(); // only used when isLowerCaseTableNames = true. - protected final ConcurrentHashMap lowerTableToRealTable = new ConcurrentHashMap<>(); + protected final ConcurrentHashMap> lowerTableToRealTable + = new ConcurrentHashMap<>(); + // only used when isLowerCaseTableNames = true. + protected final ConcurrentHashMap>> + lowerColumnToRealColumn = new ConcurrentHashMap<>(); private final AtomicBoolean dbNamesLoaded = new AtomicBoolean(false); private final AtomicBoolean tableNamesLoaded = new AtomicBoolean(false); + private final AtomicBoolean columnNamesLoaded = new AtomicBoolean(false); public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) { String dbType = parseDbType(jdbcClientConfig.getJdbcUrl()); @@ -178,7 +183,7 @@ public abstract class JdbcClient { if (closeable != null) { try { closeable.close(); - } catch (Exception e) { + } catch (Exception e) { throw new JdbcClientException("Can not close : ", e); } } @@ -186,8 +191,10 @@ public abstract class JdbcClient { } // This part used to process meta-information of database, table and column. + /** * get all database name through JDBC + * * @return list of database names */ public List getDatabaseNameList() { @@ -208,6 +215,8 @@ public abstract class JdbcClient { if (isLowerCaseTableNames) { lowerDBToRealDB.put(databaseName.toLowerCase(), databaseName); databaseName = databaseName.toLowerCase(); + } else { + lowerDBToRealDB.put(databaseName, databaseName); } tempDatabaseNames.add(databaseName); } @@ -237,20 +246,20 @@ public abstract class JdbcClient { * get all tables of one database */ public List getTablesNameList(String dbName) { - String currentDbName = dbName; List tablesName = Lists.newArrayList(); String[] tableTypes = getTableTypes(); - if (isLowerCaseTableNames) { - currentDbName = getRealDatabaseName(dbName); - } - String finalDbName = currentDbName; + String finalDbName = getRealDatabaseName(dbName); processTable(finalDbName, null, tableTypes, (rs) -> { try { while (rs.next()) { String tableName = rs.getString("TABLE_NAME"); if (isLowerCaseTableNames) { - lowerTableToRealTable.put(tableName.toLowerCase(), tableName); + lowerTableToRealTable.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); + lowerTableToRealTable.get(finalDbName).put(tableName.toLowerCase(), tableName); tableName = tableName.toLowerCase(); + } else { + lowerTableToRealTable.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); + lowerTableToRealTable.get(finalDbName).put(tableName, tableName); } tablesName.add(tableName); } @@ -262,16 +271,10 @@ public abstract class JdbcClient { } public boolean isTableExist(String dbName, String tableName) { - String currentDbName = dbName; - String currentTableName = tableName; final boolean[] isExist = {false}; - if (isLowerCaseTableNames) { - currentDbName = getRealDatabaseName(dbName); - currentTableName = getRealTableName(dbName, tableName); - } String[] tableTypes = getTableTypes(); - String finalTableName = currentTableName; - String finalDbName = currentDbName; + String finalDbName = getRealDatabaseName(dbName); + String finalTableName = getRealTableName(dbName, tableName); processTable(finalDbName, finalTableName, tableTypes, (rs) -> { try { if (rs.next()) { @@ -292,23 +295,25 @@ public abstract class JdbcClient { Connection conn = getConnection(); ResultSet rs = null; List tableSchema = Lists.newArrayList(); - // if isLowerCaseTableNames == true, tableName is lower case - // but databaseMetaData.getColumns() is case sensitive - String currentDbName = dbName; - String currentTableName = tableName; - if (isLowerCaseTableNames) { - currentDbName = getRealDatabaseName(dbName); - currentTableName = getRealTableName(dbName, tableName); - } - String finalDbName = currentDbName; - String finalTableName = currentTableName; + String finalDbName = getRealDatabaseName(dbName); + String finalTableName = getRealTableName(dbName, tableName); try { DatabaseMetaData databaseMetaData = conn.getMetaData(); String catalogName = getCatalogName(conn); rs = getColumns(databaseMetaData, catalogName, finalDbName, finalTableName); while (rs.next()) { + lowerColumnToRealColumn.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); + lowerColumnToRealColumn.get(finalDbName).putIfAbsent(finalTableName, new ConcurrentHashMap<>()); JdbcFieldSchema field = new JdbcFieldSchema(); - field.setColumnName(rs.getString("COLUMN_NAME")); + String columnName = rs.getString("COLUMN_NAME"); + if (isLowerCaseTableNames) { + lowerColumnToRealColumn.get(finalDbName).get(finalTableName) + .put(columnName.toLowerCase(), columnName); + columnName = columnName.toLowerCase(); + } else { + lowerColumnToRealColumn.get(finalDbName).get(finalTableName).put(columnName, columnName); + } + field.setColumnName(columnName); field.setDataType(rs.getInt("DATA_TYPE")); field.setDataTypeName(rs.getString("TYPE_NAME")); /* @@ -352,11 +357,9 @@ public abstract class JdbcClient { } public String getRealDatabaseName(String dbname) { - if (!isLowerCaseTableNames) { - return dbname; - } - - if (lowerDBToRealDB.isEmpty() || !lowerDBToRealDB.containsKey(dbname)) { + if (lowerDBToRealDB == null + || lowerDBToRealDB.isEmpty() + || !lowerDBToRealDB.containsKey(dbname)) { loadDatabaseNamesIfNeeded(); } @@ -364,15 +367,34 @@ public abstract class JdbcClient { } public String getRealTableName(String dbName, String tableName) { - if (!isLowerCaseTableNames) { - return tableName; - } - - if (lowerTableToRealTable.isEmpty() || !lowerTableToRealTable.containsKey(tableName)) { + String realDbName = getRealDatabaseName(dbName); + if (lowerTableToRealTable == null + || lowerTableToRealTable.isEmpty() + || !lowerTableToRealTable.containsKey(realDbName) + || lowerTableToRealTable.get(realDbName) == null + || lowerTableToRealTable.get(realDbName).isEmpty() + || !lowerTableToRealTable.get(realDbName).containsKey(tableName) + || lowerTableToRealTable.get(realDbName).get(tableName) == null) { loadTableNamesIfNeeded(dbName); } - return lowerTableToRealTable.get(tableName); + return lowerTableToRealTable.get(realDbName).get(tableName); + } + + public Map getRealColumnNames(String dbName, String tableName) { + String realDbName = getRealDatabaseName(dbName); + String realTableName = getRealTableName(dbName, tableName); + if (lowerColumnToRealColumn == null + || lowerColumnToRealColumn.isEmpty() + || !lowerColumnToRealColumn.containsKey(realDbName) + || lowerColumnToRealColumn.get(realDbName) == null + || lowerColumnToRealColumn.get(realDbName).isEmpty() + || !lowerColumnToRealColumn.get(realDbName).containsKey(realTableName) + || lowerColumnToRealColumn.get(realDbName).get(realTableName) == null + || lowerColumnToRealColumn.get(realDbName).get(realTableName).isEmpty()) { + loadColumnNamesIfNeeded(dbName, tableName); + } + return lowerColumnToRealColumn.get(realDbName).get(realTableName); } private void loadDatabaseNamesIfNeeded() { @@ -387,6 +409,12 @@ public abstract class JdbcClient { } } + private void loadColumnNamesIfNeeded(String dbName, String tableName) { + if (columnNamesLoaded.compareAndSet(false, true)) { + getJdbcColumnsInfo(dbName, tableName); + } + } + // protected methods,for subclass to override protected String getCatalogName(Connection conn) throws SQLException { return null; @@ -411,7 +439,7 @@ public abstract class JdbcClient { } protected void processTable(String dbName, String tableName, String[] tableTypes, - Consumer resultSetConsumer) { + Consumer resultSetConsumer) { Connection conn = getConnection(); ResultSet rs = null; try { @@ -435,7 +463,7 @@ public abstract class JdbcClient { } protected ResultSet getColumns(DatabaseMetaData databaseMetaData, String catalogName, String schemaName, - String tableName) throws SQLException { + String tableName) throws SQLException { return databaseMetaData.getColumns(catalogName, schemaName, tableName, null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index 61ba2a0db4..3273f4b759 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -35,6 +35,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; public class JdbcMySQLClient extends JdbcClient { @@ -117,21 +118,27 @@ public class JdbcMySQLClient extends JdbcClient { Connection conn = getConnection(); ResultSet rs = null; List tableSchema = com.google.common.collect.Lists.newArrayList(); - // if isLowerCaseTableNames == true, tableName is lower case - // but databaseMetaData.getColumns() is case sensitive - if (isLowerCaseTableNames) { - dbName = lowerDBToRealDB.get(dbName); - tableName = lowerTableToRealTable.get(tableName); - } + String finalDbName = getRealDatabaseName(dbName); + String finalTableName = getRealTableName(dbName, tableName); try { DatabaseMetaData databaseMetaData = conn.getMetaData(); String catalogName = getCatalogName(conn); - rs = getColumns(databaseMetaData, catalogName, dbName, tableName); + rs = getColumns(databaseMetaData, catalogName, finalDbName, finalTableName); List primaryKeys = getPrimaryKeys(databaseMetaData, catalogName, dbName, tableName); Map mapFieldtoType = null; while (rs.next()) { + lowerColumnToRealColumn.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); + lowerColumnToRealColumn.get(finalDbName).putIfAbsent(finalTableName, new ConcurrentHashMap<>()); JdbcFieldSchema field = new JdbcFieldSchema(); - field.setColumnName(rs.getString("COLUMN_NAME")); + String columnName = rs.getString("COLUMN_NAME"); + if (isLowerCaseTableNames) { + lowerColumnToRealColumn.get(finalDbName).get(finalTableName) + .put(columnName.toLowerCase(), columnName); + columnName = columnName.toLowerCase(); + } else { + lowerColumnToRealColumn.get(finalDbName).get(finalTableName).put(columnName, columnName); + } + field.setColumnName(columnName); field.setDataType(rs.getInt("DATA_TYPE")); // in mysql-jdbc-connector-8.0.*, TYPE_NAME of the HLL column in doris will be "UNKNOWN" diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index d0a9f2c3de..270b5b4bdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -28,6 +28,7 @@ import java.sql.DatabaseMetaData; import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; public class JdbcOracleClient extends JdbcClient { @@ -61,6 +62,8 @@ public class JdbcOracleClient extends JdbcClient { if (isLowerCaseTableNames) { lowerDBToRealDB.put(databaseName.toLowerCase(), databaseName); databaseName = databaseName.toLowerCase(); + } else { + lowerDBToRealDB.put(databaseName, databaseName); } tempDatabaseNames.add(databaseName); } @@ -91,14 +94,8 @@ public class JdbcOracleClient extends JdbcClient { Connection conn = getConnection(); ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String currentDbName = dbName; - String currentTableName = tableName; - if (isLowerCaseTableNames) { - currentDbName = getRealDatabaseName(dbName); - currentTableName = getRealTableName(dbName, tableName); - } - String finalDbName = currentDbName; - String finalTableName = currentTableName; + String finalDbName = getRealDatabaseName(dbName); + String finalTableName = getRealTableName(dbName, tableName); try { DatabaseMetaData databaseMetaData = conn.getMetaData(); String catalogName = getCatalogName(conn); @@ -119,8 +116,18 @@ public class JdbcOracleClient extends JdbcClient { if (isModify && isTableModified(rs.getString("TABLE_NAME"), finalTableName)) { continue; } + lowerColumnToRealColumn.putIfAbsent(finalDbName, new ConcurrentHashMap<>()); + lowerColumnToRealColumn.get(finalDbName).putIfAbsent(finalTableName, new ConcurrentHashMap<>()); JdbcFieldSchema field = new JdbcFieldSchema(); - field.setColumnName(rs.getString("COLUMN_NAME")); + String columnName = rs.getString("COLUMN_NAME"); + if (isLowerCaseTableNames) { + lowerColumnToRealColumn.get(finalDbName).get(finalTableName) + .put(columnName.toLowerCase(), columnName); + columnName = columnName.toLowerCase(); + } else { + lowerColumnToRealColumn.get(finalDbName).get(finalTableName).put(columnName, columnName); + } + field.setColumnName(columnName); field.setDataType(rs.getInt("DATA_TYPE")); field.setDataTypeName(rs.getString("TYPE_NAME")); /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java index 23a44ed643..4eccaff477 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java @@ -138,7 +138,7 @@ public class JdbcScanNode extends ExternalScanNode { List pushDownConjuncts = collectConjunctsToPushDown(conjunctsList, errors); for (Expr individualConjunct : pushDownConjuncts) { - String filter = conjunctExprToString(jdbcType, individualConjunct); + String filter = conjunctExprToString(jdbcType, individualConjunct, tbl); filters.add(filter); conjuncts.remove(individualConjunct); } @@ -169,9 +169,9 @@ public class JdbcScanNode extends ExternalScanNode { continue; } Column col = slot.getColumn(); - columns.add(JdbcTable.databaseProperName(jdbcType, col.getName())); + columns.add(tbl.getProperRealColumnName(jdbcType, col.getName())); } - if (0 == columns.size()) { + if (columns.isEmpty()) { columns.add("*"); } } @@ -324,12 +324,12 @@ public class JdbcScanNode extends ExternalScanNode { return !fnExprList.isEmpty(); } - public static String conjunctExprToString(TOdbcTableType tableType, Expr expr) { + public static String conjunctExprToString(TOdbcTableType tableType, Expr expr, JdbcTable tbl) { if (expr instanceof CompoundPredicate) { StringBuilder result = new StringBuilder(); CompoundPredicate compoundPredicate = (CompoundPredicate) expr; for (Expr child : compoundPredicate.getChildren()) { - result.append(conjunctExprToString(tableType, child)); + result.append(conjunctExprToString(tableType, child, tbl)); result.append(" ").append(compoundPredicate.getOp().toString()).append(" "); } // Remove the last operator @@ -357,7 +357,11 @@ public class JdbcScanNode extends ExternalScanNode { ArrayList children = expr.getChildren(); String filter; if (children.get(0) instanceof SlotRef) { - filter = JdbcTable.databaseProperName(tableType, children.get(0).toMySql()); + if (tbl != null) { + filter = tbl.getProperRealColumnName(tableType, children.get(0).toMySql()); + } else { + filter = JdbcTable.databaseProperName(tableType, children.get(0).toMySql()); + } } else { filter = children.get(0).toMySql(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java index bf4e835e4f..68950ebab8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java @@ -184,7 +184,7 @@ public class OdbcScanNode extends ExternalScanNode { ArrayList odbcConjuncts = Expr.cloneList(conjuncts, sMap); for (Expr p : odbcConjuncts) { if (shouldPushDownConjunct(odbcType, p)) { - String filter = JdbcScanNode.conjunctExprToString(odbcType, p); + String filter = JdbcScanNode.conjunctExprToString(odbcType, p, null); filters.add(filter); conjuncts.remove(p); } diff --git a/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out index 519c7ab131..0545ddafac 100644 --- a/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out @@ -187,6 +187,15 @@ DORIS_TEST 3 jerry 23 88.0 4 andy 21 93.0 +-- !lower_case_column_names1 -- +1 doris 3 1.0 + +-- !lower_case_column_names2 -- +1 doris 3 1.0 + +-- !lower_case_column_names3 -- +1 doris 3 1.0 + -- !query_clob -- 10010 liantong 10086 yidong diff --git a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy index e762f2f4aa..f3463763b8 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy @@ -214,6 +214,9 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d // test lower case name order_qt_lower_case_table_names4 """ select * from student2 order by id; """ + order_qt_lower_case_column_names1 """ select * from student3 order by id; """ + order_qt_lower_case_column_names2 """ select * from student3 where id = 1 order by id; """ + order_qt_lower_case_column_names3 """ select * from student3 where id = 1 and name = 'doris' order by id; """ sql """drop catalog if exists ${catalog_name} """