From ce489cf7233a8dee38d39851d56ef5f45e8a83fa Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Mon, 21 Nov 2022 10:33:53 +0800 Subject: [PATCH] [Feature](JDBC)support clickhouse jdbc external table (#14244) --- .../ecosystem/external-table/jdbc-of-doris.md | 30 +++++++++++++++-- .../ecosystem/external-table/jdbc-of-doris.md | 32 +++++++++++++++++-- .../org/apache/doris/catalog/JdbcTable.java | 1 + .../apache/doris/planner/JdbcScanNode.java | 3 +- .../org/apache/doris/udf/JdbcExecutor.java | 7 +++- gensrc/thrift/Types.thrift | 3 +- 6 files changed, 69 insertions(+), 7 deletions(-) diff --git a/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md b/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md index 29a7f6cb77..552cd7e848 100644 --- a/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md +++ b/docs/en/docs/ecosystem/external-table/jdbc-of-doris.md @@ -148,6 +148,10 @@ PROPERTIES ( At present, only this version has been tested, and other versions will be added after testing +#### 4.ClickHouse +| ClickHouse Version | ClickHouse JDBC Driver Version | +|-------------|---------------------------------------| +| 22 | clickhouse-jdbc-0.3.2-patch11-all.jar | ## Type matching @@ -215,12 +219,34 @@ There are different data types among different databases. Here is a list of the | DATETIME | DATETIME | | DECIMAL | DECIMAL | +### ClickHouse + +| ClickHouse | Doris | +|:----------:|:--------:| +| BOOLEAN | BOOLEAN | +| CHAR | CHAR | +| VARCHAR | VARCHAR | +| STRING | STRING | +| DATE | DATE | +| Float32 | FLOAT | +| Float64 | DOUBLE | +| Int8 | TINYINT | +| Int16 | SMALLINT | +| Int32 | INT | +| Int64 | BIGINT | +| Int128 | LARGEINT | +| DATETIME | DATETIME | +| DECIMAL | DECIMAL | + +**Note:** +- For some specific types in ClickHouse, For example, UUID,IPv4,IPv6, and Enum8 can be matched with Doris's Varchar/String type. However, in the display of IPv4 and IPv6, an extra `/` is displayed before the data, which needs to be processed by the `split_part` function +- For the Geo type Point of ClickHouse, the match cannot be made ## Q&A -1. Besides mysql, Oracle, PostgreSQL, and SQL Server support more databases +1. Besides mysql, Oracle, PostgreSQL, SQL Server and ClickHouse support more databases -At present, Doris only adapts to MySQL, Oracle, SQL Server, and PostgreSQL. And planning to adapt other databases. In principle, any database that supports JDBC access can be accessed through the JDBC facade. If you need to access other appearances, you are welcome to modify the code and contribute to Doris. +At present, Doris only adapts to MySQL, Oracle, PostgreSQL, SQL Server and ClickHouse. And planning to adapt other databases. In principle, any database that supports JDBC access can be accessed through the JDBC facade. If you need to access other appearances, you are welcome to modify the code and contribute to Doris. 2. Read the Emoji expression on the surface of MySQL, and there is garbled code diff --git a/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md b/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md index 647b7aaf16..dbae5a2bba 100644 --- a/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md +++ b/docs/zh-CN/docs/ecosystem/external-table/jdbc-of-doris.md @@ -146,6 +146,11 @@ PROPERTIES ( 目前只测试了这一个版本其他版本测试后补充 +#### 4.ClickHouse测试 +| ClickHouse版本 | ClickHouse JDBC驱动版本 | +|----------| ------------------- | +| 22 | clickhouse-jdbc-0.3.2-patch11-all.jar | + ## 类型匹配 @@ -213,11 +218,34 @@ PROPERTIES ( | DATETIME | DATETIME | | DECIMAL | DECIMAL | +### ClickHouse + +| ClickHouse | Doris | +|:----------:|:--------:| +| BOOLEAN | BOOLEAN | +| CHAR | CHAR | +| VARCHAR | VARCHAR | +| STRING | STRING | +| DATE | DATE | +| Float32 | FLOAT | +| Float64 | DOUBLE | +| Int8 | TINYINT | +| Int16 | SMALLINT | +| Int32 | INT | +| Int64 | BIGINT | +| Int128 | LARGEINT | +| DATETIME | DATETIME | +| DECIMAL | DECIMAL | + +**注意：** +- 对于ClickHouse里的一些特殊类型，如UUID,IPv4,IPv6,Enum8可以用Doris的Varchar/String类型来匹配,但是在显示上IPv4,IPv6会额外在数据最前面显示一个`/`,需要自己用`split_part`函数处理 +- 对于ClickHouse的Geo类型Point,无法进行匹配 + ## Q&A -1. 除了MySQL,Oracle,PostgreSQL,SQLServer是否能够支持更多的数据库 +1. 除了MySQL,Oracle,PostgreSQL,SQLServer,ClickHouse是否能够支持更多的数据库 - 目前Doris只适配了MySQL，PostgreSQL,SQLServer,Oracle.关于其他的数据库的适配工作正在规划之中，原则上来说任何支持JDBC访问的数据库都能通过JDBC外表来访问。如果您有访问其他外表的需求，欢迎修改代码并贡献给Doris。 + 目前Doris只适配了MySQL,Oracle,PostgreSQL,SQLServer,ClickHouse.关于其他的数据库的适配工作正在规划之中，原则上来说任何支持JDBC访问的数据库都能通过JDBC外表来访问。如果您有访问其他外表的需求，欢迎修改代码并贡献给Doris。 2. 读写mysql外表的emoji表情出现乱码 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index 22151660cb..bba585904b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -71,6 +71,7 @@ public class JdbcTable extends Table { tempMap.put("postgresql", TOdbcTableType.POSTGRESQL); tempMap.put("sqlserver", TOdbcTableType.SQLSERVER); tempMap.put("oracle", TOdbcTableType.ORACLE); + tempMap.put("clickhouse", TOdbcTableType.CLICKHOUSE); TABLE_TYPE_MAP = Collections.unmodifiableMap(tempMap); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java index edbe898758..551135e730 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JdbcScanNode.java @@ -138,7 +138,8 @@ public class JdbcScanNode extends ScanNode { if (shouldPushDownLimit() && (jdbcType == TOdbcTableType.MYSQL || jdbcType == TOdbcTableType.POSTGRESQL - || jdbcType == TOdbcTableType.MONGODB)) { + || jdbcType == TOdbcTableType.MONGODB + || jdbcType == TOdbcTableType.CLICKHOUSE)) { sql.append(" LIMIT ").append(limit); } diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java index ffb905b7a1..21ef37775d 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java +++ b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java @@ -174,7 +174,12 @@ public class JdbcExecutor { } public long convertDateToLong(Object obj) { - LocalDate date = ((Date) obj).toLocalDate(); + LocalDate date; + if (obj instanceof LocalDate) { + date = (LocalDate) obj; + } else { + date = ((Date) obj).toLocalDate(); + } long time = UdfUtils.convertToDateTime(date.getYear(), date.getMonthValue(), date.getDayOfMonth(), 0, 0, 0, true); return time; diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 994880e69c..45b0daa885 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -553,7 +553,8 @@ enum TOdbcTableType { POSTGRESQL, SQLSERVER, REDIS, - MONGODB + MONGODB, + CLICKHOUSE } enum TKeysType {