database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](multi-catalog) support postgresql jdbc catalog (#15570)

Browse Source 
support postgresql jdbc catalog
This commit is contained in:
Tiewei Fang
2023-01-06 11:00:59 +08:00
committed by GitHub
parent b57500d0c3
commit df2da89b89
17 changed files with 2611 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
be/src/vec/exec/vjdbc_connector.cpp
View File

@ -42,8 +42,8 @@ const char* JDBC_EXECUTOR_GET_TYPES_SIGNATURE = "()Ljava/util/List;";
const char* JDBC_EXECUTOR_GET_ARR_LIST_SIGNATURE = "(Ljava/lang/Object;)Ljava/util/List;";
const char* JDBC_EXECUTOR_GET_ARR_TYPE_SIGNATURE = "()I";
const char* JDBC_EXECUTOR_CLOSE_SIGNATURE = "()V";
const char* JDBC_EXECUTOR_CONVERT_DATE_SIGNATURE = "(Ljava/lang/Object;)J";
const char* JDBC_EXECUTOR_CONVERT_DATETIME_SIGNATURE = "(Ljava/lang/Object;)J";
const char* JDBC_EXECUTOR_CONVERT_DATE_SIGNATURE = "(Ljava/lang/Object;Z)J";
const char* JDBC_EXECUTOR_CONVERT_DATETIME_SIGNATURE = "(Ljava/lang/Object;Z)J";
const char* JDBC_EXECUTOR_TRANSACTION_SIGNATURE = "()V";
JdbcConnector::JdbcConnector(const JdbcConnectorParam& param)
@ -491,17 +491,30 @@ Status JdbcConnector::_insert_column_data(JNIEnv* env, jobject jobj, const TypeD
data.length());
break;
}
case TYPE_DATE: {
int64_t num = _jobject_to_date(env, jobj);
int64_t num = _jobject_to_date(env, jobj, false);
reinterpret_cast<vectorized::ColumnVector<vectorized::Int64>*>(col_ptr)->insert_value(num);
break;
}
case TYPE_DATEV2: {
int64_t num = _jobject_to_date(env, jobj, true);
uint32_t num2 = static_cast<uint32_t>(num);
reinterpret_cast<vectorized::ColumnVector<vectorized::UInt32>*>(col_ptr)->insert_value(
num2);
break;
}
case TYPE_DATETIME: {
int64_t num = _jobject_to_datetime(env, jobj);
int64_t num = _jobject_to_datetime(env, jobj, false);
reinterpret_cast<vectorized::ColumnVector<vectorized::Int64>*>(col_ptr)->insert_value(num);
break;
}
case TYPE_DATETIMEV2: {
int64_t num = _jobject_to_datetime(env, jobj, true);
uint64_t num2 = static_cast<uint64_t>(num);
reinterpret_cast<vectorized::ColumnVector<vectorized::UInt64>*>(col_ptr)->insert_value(
num2);
break;
}
case TYPE_LARGEINT: {
StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS;
std::string data = _jobject_to_string(env, jobj);
@ -674,14 +687,14 @@ std::string JdbcConnector::_jobject_to_string(JNIEnv* env, jobject jobj) {
return str;
}
int64_t JdbcConnector::_jobject_to_date(JNIEnv* env, jobject jobj) {
int64_t JdbcConnector::_jobject_to_date(JNIEnv* env, jobject jobj, bool is_date_v2) {
return env->CallNonvirtualLongMethod(_executor_obj, _executor_clazz, _executor_convert_date_id,
jobj);
jobj, is_date_v2);
}
int64_t JdbcConnector::_jobject_to_datetime(JNIEnv* env, jobject jobj) {
int64_t JdbcConnector::_jobject_to_datetime(JNIEnv* env, jobject jobj, bool is_datetime_v2) {
return env->CallNonvirtualLongMethod(_executor_obj, _executor_clazz,
_executor_convert_datetime_id, jobj);
_executor_convert_datetime_id, jobj, is_datetime_v2);
}
Status JdbcConnector::begin_trans() {

4
be/src/vec/exec/vjdbc_connector.h
View File

@ -79,8 +79,8 @@ private:
vectorized::IColumn* column_ptr, int column_index,
std::string_view column_name);
std::string _jobject_to_string(JNIEnv* env, jobject jobj);
int64_t _jobject_to_date(JNIEnv* env, jobject jobj);
int64_t _jobject_to_datetime(JNIEnv* env, jobject jobj);
int64_t _jobject_to_date(JNIEnv* env, jobject jobj, bool is_date_v2);
int64_t _jobject_to_datetime(JNIEnv* env, jobject jobj, bool is_datetime_v2);
Status _cast_string_to_array(const SlotDescriptor* slot_desc, Block* block, int column_index,
int rows);

1
docker/thirdparties/docker-compose/postgresql/init/01-create-schema.sql
View File

@ -16,3 +16,4 @@
-- under the License.
create schema doris_test;
create schema catalog_pg_test;

28
docker/thirdparties/docker-compose/postgresql/init/02-create-table.sql
View File

@ -122,3 +122,31 @@ CREATE TABLE doris_test.test9 (
id5 bigint
);
CREATE TABLE catalog_pg_test.test10 (
ID INT NOT NULL,
bit_value bit,
real_value real,
cidr_value cidr,
inet_value inet,
macaddr_value macaddr,
bitn_value bit(10),
bitnv_value bit varying(10),
serial4_value serial4,
jsonb_value jsonb
);
CREATE TABLE catalog_pg_test.test11 (
ID INT PRIMARY KEY NOT NULL,
point_value point,
line_value line,
lseg_value lseg,
box_value box,
path_value path,
polygon_value polygon,
circle_value circle
);
CREATE TABLE catalog_pg_test.test12 (
ID INT NOT NULL,
uuid_value uuid
);

33
docker/thirdparties/docker-compose/postgresql/init/03-insert.sql
View File

@ -2622,3 +2622,36 @@ insert into doris_test.test9 values
(32767, 2147483647, false, 'testtsetab', 9223372036854775807),
(null, null, null, null, null);
insert into catalog_pg_test.test10 values
(1, cast(0 as bit), 12.123456, '10.16.10.14','10.16.10.14','ff:ff:ff:ff:ff:AA',
'1010101010', cast(10 as bit(5)), 1, '{"id":1}');
insert into catalog_pg_test.test10 values
(2, cast(0 as bit), 12.123456, '10.16.10.14','10.16.10.14','ff:ff:ff:ff:ff:ff',
cast(10 as bit(10)), cast(10 as bit(10)), 2, '{"id":1}');
insert into catalog_pg_test.test10 values
(3, cast(0 as bit), 12.123456, '10.16.10.14','10.16.10.14','ff:ff:ff:ff:ff:ff',
cast(10 as bit(10)), cast(10 as bit(7)), 3, '{"id":1}');
insert into catalog_pg_test.test11 values
(1, '(1,1)', '{1,1,1}', '(1,1),(2,2)', '(1,1),(2,2)', '(1,1),(2,2),(2,1)',
'((1,1),(2,2),(2,1))', '<(0,0),1>');
insert into catalog_pg_test.test11 values
(2, '(1,1)', '{1,1,1}', '(1,1),(2,2)', '(1,1),(2,2)', '[(1,1),(2,2),(2,1)]',
'((1,1),(2,2),(2,1))', '<(0,0),1>');
insert into catalog_pg_test.test11 values
(3, '(1.1,1.1)', '{1,1,1}', '(1,1),(2,2)', '(1,1),(2,2)', '[(1,1),(2,2),(2,1)]',
'((1,1),(2,2),(2,1))', '<(0,0),1>');
insert into catalog_pg_test.test11 values
(4, '(1.12,1.12)', '{1,1,1}', '(1,1),(2,2)', '(1,1),(2,2)', '[(1,1),(2,2),(2,1)]',
'((1,1),(2,2),(2,1))', '<(0,0),1>');
insert into catalog_pg_test.test11 values
(5, '(1.12345,1.12345)', '{1,1,1}', '(1,1),(2,2)', '(1,1),(2,2)', '[(1,1),(2,2),(2,1)]',
'((1,1),(2,2),(2,1))', '<(0,0),1>');
insert into catalog_pg_test.test11 values
(6, '(1.12345,1.12345)', '{1.1,1.1,1.1}', '(1.2,1.2),(2.3,2.3)', '(1.4,1.4),(2.5,2.5)', '[(1.6,1.6),(2.7,2.7),(2.8,1.8)]',
'((1.9,1.9),(2.3,2.3),(2.4,1.4))', '<(0.2,0.2),1.1>');
insert into catalog_pg_test.test12 values
(1, '980dd890-f7fe-4fff-999d-873516108b2e');
insert into catalog_pg_test.test12 values
(2, '980dd890-f7fe-4fff-999d-873516108b2e');

50
docs/en/docs/ecosystem/external-table/multi-catalog.md
View File

@ -372,6 +372,8 @@ Parameter | Description
The following example creates a Catalog connection named jdbc. This jdbc Catalog will connect to the specified database according to the 'jdbc.jdbc_url' parameter(`jdbc::mysql` in the example, so connect to the mysql database). Currently, only the MYSQL database type is supported.
**mysql catalog example**
```sql
-- 1.2.0+ Version
CREATE RESOURCE mysql_resource PROPERTIES (
@ -392,6 +394,28 @@ CREATE CATALOG jdbc PROPERTIES (
)
```
**postgresql catalog example**
```sql
-- 1.2.0+ Version
CREATE RESOURCE pg_resource PROPERTIES (
"type"="jdbc",
"user"="postgres",
"password"="123456",
"jdbc_url" = "jdbc:postgresql://127.0.0.1:5449/demo",
"driver_url" = "file:/path/to/postgresql-42.5.1.jar",
"driver_class" = "org.postgresql.Driver"
);
CREATE CATALOG jdbc WITH RESOURCE pg_resource;
-- 1.2.0 Version
CREATE CATALOG jdbc PROPERTIES (
"type"="jdbc",
"jdbc.jdbc_url" = "jdbc:postgresql://127.0.0.1:5449/demo",
...
)
```
Where `jdbc.driver_url` can be a remote jar package
```sql
@ -424,6 +448,8 @@ MySQL [(none)]> show catalogs;
2 rows in set (0.02 sec)
```
> Note: In the `postgresql catalog`, a database for doris corresponds to a schema in the postgresql specified catalog (specified in the `jdbc.jdbc_url` parameter), tables under this database corresponds to tables under this postgresql's schema.
Switch to the jdbc catalog with the `SWITCH` command and view the databases in it:
```sql
@ -640,6 +666,30 @@ For Hive/Iceberge/Hudi
| VARCHAR | STRING | |
| TINYTEXT、TEXT、MEDIUMTEXT、LONGTEXT、TINYBLOB、BLOB、MEDIUMBLOB、LONGBLOB、TINYSTRING、STRING、MEDIUMSTRING、LONGSTRING、BINARY、VARBINARY、JSON、SET、BIT | STRING | |
#### POSTGRESQL
POSTGRESQL Type | Doris Type | Comment |
|---|---|---|
| boolean | BOOLEAN | |
| smallint/int2 | SMALLINT | |
| integer/int4 | INT | |
| bigint/int8 | BIGINT | |
| decimal/numeric | DECIMAL | |
| real/float4 | FLOAT | |
| double precision | DOUBLE | |
| smallserial | SMALLINT | |
| serial | INT | |
| bigserial | BIGINT | |
| char | CHAR | |
| varchar/text | STRING | |
| timestamp | DATETIME | |
| date | DATE | |
| time | STRING | |
| interval | STRING | |
| point/line/lseg/box/path/polygon/circle | STRING | |
| cidr/inet/macaddr | STRING | |
| bit/bit(n)/bit varying(n) | STRING | `bit` type corresponds to the `STRING` type of DORIS. The data read is `true/false`, not `1/0` |
| uuid/josnb | STRING | |
## Privilege Management
Using Doris to access the databases and tables in the External Catalog is not controlled by the permissions of the external data source itself, but relies on Doris's own permission access management.

36
docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md
View File

@ -118,6 +118,7 @@ CREATE CATALOG catalog_name PROPERTIES (
```
3. Create catalog jdbc
**mysql**
```sql
-- 1.2.0+ Version
@ -134,14 +135,39 @@ CREATE CATALOG catalog_name PROPERTIES (
-- 1.2.0 Version
CREATE CATALOG jdbc PROPERTIES (
"type"="jdbc",
"user"="root",
"password"="123456",
"jdbc_url" = "jdbc:mysql://127.0.0.1:3316/doris_test?useSSL=false",
"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",
"driver_class" = "com.mysql.cj.jdbc.Driver"
"jdbc.user"="root",
"jdbc.password"="123456",
"jdbc.jdbc_url" = "jdbc:mysql://127.0.0.1:3316/doris_test?useSSL=false",
"jdbc.driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",
"jdbc.driver_class" = "com.mysql.cj.jdbc.Driver"
);
```
**postgresql**
```sql
-- 1.2.0+ Version
CREATE RESOURCE pg_resource PROPERTIES (
"type"="jdbc",
"user"="postgres",
"password"="123456",
"jdbc_url" = "jdbc:postgresql://127.0.0.1:5432/demo",
"driver_url" = "file:/path/to/postgresql-42.5.1.jar",
"driver_class" = "org.postgresql.Driver"
);
CREATE CATALOG jdbc WITH RESOURCE pg_resource;
-- 1.2.0 Version
CREATE CATALOG jdbc PROPERTIES (
"type"="jdbc",
"jdbc.user"="postgres",
"jdbc.password"="123456",
"jdbc.jdbc_url" = "jdbc:postgresql://127.0.0.1:5432/demo",
"jdbc.driver_url" = "file:/path/to/postgresql-42.5.1.jar",
"jdbc.driver_class" = "org.postgresql.Driver"
);
```
### Keywords
CREATE, CATALOG

54
docs/zh-CN/docs/ecosystem/external-table/multi-catalog.md
View File

@ -443,7 +443,10 @@ mysql> select * from test;
### 连接JDBC
以下示例，用于创建一个名为 jdbc 的 Catalog, 通过jdbc 连接指定的Mysql。
jdbc Catalog会根据`jdbc.jdbc_url` 来连接指定的数据库（示例中是`jdbc::mysql`, 所以连接MYSQL数据库），当前只支持MYSQL数据库类型。
jdbc Catalog会根据`jdbc.jdbc_url` 来连接指定的数据库（示例中是`jdbc::mysql`, 所以连接MYSQL数据库），当前支持MYSQL、POSTGRESQL数据库类型。
**MYSQL catalog示例**
```sql
-- 1.2.0+ 版本
CREATE RESOURCE mysql_resource PROPERTIES (
@ -464,6 +467,28 @@ CREATE CATALOG jdbc PROPERTIES (
)
```
**POSTGRESQL catalog示例**
```sql
-- 1.2.0+ 版本
CREATE RESOURCE pg_resource PROPERTIES (
"type"="jdbc",
"user"="postgres",
"password"="123456",
"jdbc_url" = "jdbc:postgresql://127.0.0.1:5449/demo",
"driver_url" = "file:/path/to/postgresql-42.5.1.jar",
"driver_class" = "org.postgresql.Driver"
);
CREATE CATALOG jdbc WITH RESOURCE pg_resource;
-- 1.2.0 版本
CREATE CATALOG jdbc PROPERTIES (
"type"="jdbc",
"jdbc.jdbc_url" = "jdbc:postgresql://127.0.0.1:5449/demo",
...
)
```
其中`jdbc.driver_url`可以是远程jar包：
```sql
@ -518,6 +543,8 @@ MySQL [(none)]> show databases;
9 rows in set (0.67 sec)
```
> 注意：在postgresql catalog中，doris的一个database对应于postgresql中指定catalog（`jdbc.jdbc_url`参数中指定的catalog）下的一个schema，database下的tables则对应于postgresql该schema下的tables。
查看`db1`数据库下的表，并查询：
```sql
MySQL [demo]> use db1;
@ -637,6 +664,31 @@ MySQL [db1]> select * from tbl1;
| CHAR | CHAR | |
| VARCHAR | STRING | |
| TINYTEXT、TEXT、MEDIUMTEXT、LONGTEXT、TINYBLOB、BLOB、MEDIUMBLOB、LONGBLOB、TINYSTRING、STRING、MEDIUMSTRING、LONGSTRING、BINARY、VARBINARY、JSON、SET、BIT | STRING | |
#### POSTGRESQL
POSTGRESQL Type | Doris Type | Comment |
|---|---|---|
| boolean | BOOLEAN | |
| smallint/int2 | SMALLINT | |
| integer/int4 | INT | |
| bigint/int8 | BIGINT | |
| decimal/numeric | DECIMAL | |
| real/float4 | FLOAT | |
| double precision | DOUBLE | |
| smallserial | SMALLINT | |
| serial | INT | |
| bigserial | BIGINT | |
| char | CHAR | |
| varchar/text | STRING | |
| timestamp | DATETIME | |
| date | DATE | |
| time | STRING | |
| interval | STRING | |
| point/line/lseg/box/path/polygon/circle | STRING | |
| cidr/inet/macaddr | STRING | |
| bit/bit(n)/bit varying(n) | STRING | `bit`类型映射为doris的`STRING`类型，读出的数据是`true/false`, 而不是`1/0` |
| uuid/josnb | STRING | |
## 权限管理
使用 Doris 对 External Catalog 中库表进行访问，并不受外部数据目录自身的权限控制，而是依赖 Doris 自身的权限访问管理功能。

36
docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md
View File

@ -122,6 +122,7 @@ CREATE CATALOG catalog_name PROPERTIES (
```
3. 新建数据目录 jdbc
**mysql**
```sql
-- 1.2.0+ 版本
@ -138,14 +139,39 @@ CREATE CATALOG catalog_name PROPERTIES (
-- 1.2.0 版本
CREATE CATALOG jdbc PROPERTIES (
"type"="jdbc",
"user"="root",
"password"="123456",
"jdbc_url" = "jdbc:mysql://127.0.0.1:3316/doris_test?useSSL=false",
"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",
"driver_class" = "com.mysql.cj.jdbc.Driver"
"jdbc.user"="root",
"jdbc.password"="123456",
"jdbc.jdbc_url" = "jdbc:mysql://127.0.0.1:3316/doris_test?useSSL=false",
"jdbc.driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",
"jdbc.driver_class" = "com.mysql.cj.jdbc.Driver"
);
```
**postgresql**
```sql
-- 1.2.0+ 版本
CREATE RESOURCE pg_resource PROPERTIES (
"type"="jdbc",
"user"="postgres",
"password"="123456",
"jdbc_url" = "jdbc:postgresql://127.0.0.1:5432/demo",
"driver_url" = "file:/path/to/postgresql-42.5.1.jar",
"driver_class" = "org.postgresql.Driver"
);
CREATE CATALOG jdbc WITH RESOURCE pg_resource;
-- 1.2.0 版本
CREATE CATALOG jdbc PROPERTIES (
"type"="jdbc",
"jdbc.user"="postgres",
"jdbc.password"="123456",
"jdbc.jdbc_url" = "jdbc:postgresql://127.0.0.1:5432/demo",
"jdbc.driver_url" = "file:/path/to/postgresql-42.5.1.jar",
"jdbc.driver_class" = "org.postgresql.Driver"
);
```
### Keywords
CREATE, CATALOG

2
fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
View File

@ -67,7 +67,7 @@ public class JdbcResource extends Resource {
public static final String DRIVER_CLASS = "driver_class";
public static final String DRIVER_URL = "driver_url";
public static final String TYPE = "type";
private static final String CHECK_SUM = "checksum";
public static final String CHECK_SUM = "checksum";
// timeout for both connection and read. 10 seconds is long enough.
private static final int HTTP_TIMEOUT_MS = 10000;

7
fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java
View File

@ -50,7 +50,7 @@ public class JdbcExternalCatalog extends ExternalCatalog {
private String jdbcUrl;
private String driverUrl;
private String driverClass;
private String checkSum;
private String checkSum = "";
public JdbcExternalCatalog(
long catalogId, String name, String resource, Map<String, String> props) throws DdlException {
@ -84,6 +84,7 @@ public class JdbcExternalCatalog extends ExternalCatalog {
properties.put(JdbcResource.JDBC_URL, jdbcUrl);
driverUrl = properties.getOrDefault(JdbcResource.DRIVER_URL, "");
driverClass = properties.getOrDefault(JdbcResource.DRIVER_CLASS, "");
checkSum = properties.getOrDefault(JdbcResource.CHECK_SUM, "");
return properties;
}
@ -119,7 +120,9 @@ public class JdbcExternalCatalog extends ExternalCatalog {
protected void initLocalObjectsImpl() {
jdbcClient = new JdbcClient(jdbcUser, jdbcPasswd, jdbcUrl, driverUrl, driverClass);
databaseTypeName = jdbcClient.getDbType();
checkSum = jdbcClient.getCheckSum();
if (checkSum.isEmpty()) {
checkSum = jdbcClient.getCheckSum();
}
}
@Override

104
fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java vendored
View File

@ -52,9 +52,9 @@ import java.util.List;
public class JdbcClient {
private static final Logger LOG = LogManager.getLogger(JdbcClient.class);
private static final String MYSQL = "MYSQL";
private static final String POSTGRESQL = "POSTGRESQL";
// private static final String ORACLE = "ORACLE";
// private static final String SQLSERVER = "SQLSERVER";
// private static final String POSTGRESQL = "POSTGRESQL";
private static final int HTTP_TIMEOUT_MS = 10000;
private String dbType;
@ -108,15 +108,14 @@ public class JdbcClient {
public String parseDbType(String url) {
if (url.startsWith("jdbc:mysql") || url.startsWith("jdbc:mariadb")) {
return MYSQL;
} else if (url.startsWith("jdbc:postgresql")) {
return POSTGRESQL;
}
// else if (url.startsWith("jdbc:oracle")) {
// return ORACLE;
// }
// else if (url.startsWith("jdbc:sqlserver")) {
// return SQLSERVER;
// } else if (url.startsWith("jdbc:postgresql")) {
// return POSTGRESQL;
// }
throw new JdbcClientException("Unsupported jdbc database type, please check jdbcUrl: " + jdbcUrl);
}
@ -181,7 +180,18 @@ public class JdbcClient {
List<String> databaseNames = Lists.newArrayList();
try {
stmt = conn.createStatement();
rs = stmt.executeQuery("SHOW DATABASES");
switch (dbType) {
case MYSQL:
rs = stmt.executeQuery("SHOW DATABASES");
break;
case POSTGRESQL:
rs = stmt.executeQuery("SELECT schema_name FROM information_schema.schemata "
+ "where schema_owner='" + jdbcUser + "';");
break;
default:
throw new JdbcClientException("Not supported jdbc type");
}
while (rs.next()) {
databaseNames.add(rs.getString(1));
}
@ -207,6 +217,9 @@ public class JdbcClient {
case MYSQL:
rs = databaseMetaData.getTables(dbName, null, null, types);
break;
case POSTGRESQL:
rs = databaseMetaData.getTables(null, dbName, null, types);
break;
default:
throw new JdbcClientException("Unknown database type");
}
@ -292,7 +305,16 @@ public class JdbcClient {
// Can contain single-character wildcards ("_"), or multi-character wildcards ("%")
// columnNamePattern - column name, `null` means get all columns
// Can contain single-character wildcards ("_"), or multi-character wildcards ("%")
rs = databaseMetaData.getColumns(dbName, null, tableName, null);
switch (dbType) {
case MYSQL:
rs = databaseMetaData.getColumns(dbName, null, tableName, null);
break;
case POSTGRESQL:
rs = databaseMetaData.getColumns(null, dbName, tableName, null);
break;
default:
throw new JdbcClientException("Unknown database type");
}
while (rs.next()) {
JdbcFieldSchema field = new JdbcFieldSchema();
field.setColumnName(rs.getString("COLUMN_NAME"));
@ -318,6 +340,8 @@ public class JdbcClient {
switch (dbType) {
case MYSQL:
return mysqlTypeToDoris(fieldSchema);
case POSTGRESQL:
return postgresqlTypeToDoris(fieldSchema);
default:
throw new JdbcClientException("Unknown database type");
}
@ -373,7 +397,6 @@ public class JdbcClient {
case "DATE":
return ScalarType.getDefaultDateType(Type.DATE);
case "TIMESTAMP":
return ScalarType.getDefaultDateType(Type.DATETIME);
case "DATETIME":
return ScalarType.getDefaultDateType(Type.DATETIME);
case "FLOAT":
@ -418,7 +441,72 @@ public class JdbcClient {
return ScalarType.createStringType();
default:
throw new JdbcClientException("Can not convert mysql data type to doris data type for type ["
+ mysqlType + "]");
+ mysqlType + "]");
}
}
public Type postgresqlTypeToDoris(JdbcFieldSchema fieldSchema) {
String pgType = fieldSchema.getDataTypeName();
switch (pgType) {
case "int2":
case "smallserial":
return Type.SMALLINT;
case "int4":
case "serial":
return Type.INT;
case "int8":
case "bigserial":
return Type.BIGINT;
case "numeric": {
int precision = fieldSchema.getColumnSize();
int scale = fieldSchema.getDecimalDigits();
if (precision <= ScalarType.MAX_DECIMAL128_PRECISION) {
if (!Config.enable_decimal_conversion && precision > ScalarType.MAX_DECIMALV2_PRECISION) {
return ScalarType.createStringType();
}
return ScalarType.createDecimalType(precision, scale);
} else {
return ScalarType.createStringType();
}
}
case "float4":
return Type.FLOAT;
case "float8":
return Type.DOUBLE;
case "bpchar":
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.columnSize);
return charType;
case "timestamp":
case "timestamptz":
return ScalarType.getDefaultDateType(Type.DATETIME);
case "date":
return ScalarType.getDefaultDateType(Type.DATE);
case "bool":
return Type.BOOLEAN;
case "bit":
case "point":
case "line":
case "lseg":
case "box":
case "path":
case "polygon":
case "circle":
case "varchar":
case "text":
case "time":
case "timetz":
case "interval":
case "cidr":
case "inet":
case "macaddr":
case "varbit":
case "jsonb":
case "uuid":
return ScalarType.createStringType();
default:
throw new JdbcClientException("Can not convert postgresql data type to doris data type for type ["
+ pgType + "]");
}
}

3
fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View File

@ -2313,6 +2313,9 @@ public class ShowExecutor {
public void handleShowCatalogs() throws AnalysisException {
ShowCatalogStmt showStmt = (ShowCatalogStmt) stmt;
if (ctx.getCurrentCatalog() == null) {
throw new AnalysisException("Current catalog is not exist, please switch catalog.");
}
resultSet = Env.getCurrentEnv().getCatalogMgr().showCatalogs(showStmt, ctx.getCurrentCatalog().getName());
}

17
fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
View File

@ -204,19 +204,21 @@ public class JdbcExecutor {
}
}
public long convertDateToLong(Object obj) {
public long convertDateToLong(Object obj, boolean isDateV2) {
LocalDate date;
if (obj instanceof LocalDate) {
date = (LocalDate) obj;
} else {
date = ((Date) obj).toLocalDate();
}
long time = UdfUtils.convertToDateTime(date.getYear(), date.getMonthValue(), date.getDayOfMonth(),
if (isDateV2) {
return UdfUtils.convertToDateV2(date.getYear(), date.getMonthValue(), date.getDayOfMonth());
}
return UdfUtils.convertToDateTime(date.getYear(), date.getMonthValue(), date.getDayOfMonth(),
0, 0, 0, true);
return time;
}
public long convertDateTimeToLong(Object obj) {
public long convertDateTimeToLong(Object obj, boolean isDateTimeV2) {
LocalDateTime date;
// TODO: not for sure: https://bugs.mysql.com/bug.php?id=101413
if (obj instanceof LocalDateTime) {
@ -224,9 +226,12 @@ public class JdbcExecutor {
} else {
date = ((Timestamp) obj).toLocalDateTime();
}
long time = UdfUtils.convertToDateTime(date.getYear(), date.getMonthValue(), date.getDayOfMonth(),
if (isDateTimeV2) {
return UdfUtils.convertToDateTimeV2(date.getYear(), date.getMonthValue(), date.getDayOfMonth(),
date.getHour(), date.getMinute(), date.getSecond());
}
return UdfUtils.convertToDateTime(date.getYear(), date.getMonthValue(), date.getDayOfMonth(),
date.getHour(), date.getMinute(), date.getSecond(), false);
return time;
}
private void init(String driverUrl, String sql, int batchSize, String driverClass, String jdbcUrl, String jdbcUser,

2143
regression-test/data/jdbc_catalog_p0/test_pg_jdbc_catalog.out Normal file
View File

File diff suppressed because one or more lines are too long

12
regression-test/suites/jdbc_catalog_p0/test_mysql_jdbc_catalog.groovy
View File

@ -18,6 +18,7 @@
suite("test_mysql_jdbc_catalog", "p0") {
String enabled = context.config.otherConfigs.get("enableJdbcTest")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
String resource_name = "jdbc_resource_catalog_mysql"
String catalog_name = "mysql_jdbc_catalog";
String internal_db_name = "regression_test_jdbc_catalog_p0";
String ex_db_name = "doris_test";
@ -47,8 +48,9 @@ suite("test_mysql_jdbc_catalog", "p0") {
sql """ADMIN SET FRONTEND CONFIG ("enable_decimal_conversion" = "true");"""
sql """drop catalog if exists ${catalog_name} """
sql """ drop resource if exists ${resource_name} """
sql """create resource if not exists jdbc_resource_catalog_mysql properties(
sql """create resource if not exists ${resource_name} properties(
"type"="jdbc",
"user"="root",
"password"="123456",
@ -56,8 +58,8 @@ suite("test_mysql_jdbc_catalog", "p0") {
"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",
"driver_class" = "com.mysql.cj.jdbc.Driver"
);"""
// if use 'com.mysql.cj.jdbc.Driver' here, it will report: ClassNotFound
sql """CREATE CATALOG ${catalog_name} WITH RESOURCE jdbc_resource_catalog_mysql"""
sql """CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name}"""
sql """ drop table if exists ${inDorisTable} """
@ -100,7 +102,7 @@ suite("test_mysql_jdbc_catalog", "p0") {
order_qt_ex_tb20 """ select * from ${ex_tb20} order by decimal_normal; """
sql """drop catalog if exists ${catalog_name} """
sql """drop resource if exists jdbc_resource_catalog_mysql"""
sql """drop resource if exists ${resource_name}"""
// test old create-catalog syntax for compatibility
sql """ CREATE CATALOG ${catalog_name} PROPERTIES (
@ -114,6 +116,6 @@ suite("test_mysql_jdbc_catalog", "p0") {
sql """switch ${catalog_name}"""
sql """use ${ex_db_name}"""
order_qt_ex_tb1 """ select * from ${ex_tb1} order by id; """
sql """drop resource if exists jdbc_resource_catalog_mysql"""
sql """drop resource if exists ${resource_name}"""
}
}

94
regression-test/suites/jdbc_catalog_p0/test_pg_jdbc_catalog.groovy Normal file
View File

@ -0,0 +1,94 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_pg_jdbc_catalog", "p0") {
String enabled = context.config.otherConfigs.get("enableJdbcTest")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
String resource_name = "jdbc_resource_catalog_pg";
String catalog_name = "pg_jdbc_catalog";
String internal_db_name = "regression_test_jdbc_catalog_p0";
String ex_schema_name = "doris_test";
String ex_schema_name2 = "catalog_pg_test";
String pg_port = context.config.otherConfigs.get("pg_14_port");
String inDorisTable = "doris_in_tb";
sql """drop catalog if exists ${catalog_name} """
sql """drop resource if exists ${resource_name}"""
sql """create resource if not exists ${resource_name} properties(
"type"="jdbc",
"user"="postgres",
"password"="123456",
"jdbc_url" = "jdbc:postgresql://127.0.0.1:${pg_port}/postgres?currentSchema=doris_test&useSSL=false",
"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/postgresql-42.5.0.jar",
"driver_class" = "org.postgresql.Driver"
);"""
sql """CREATE CATALOG ${catalog_name} WITH RESOURCE jdbc_resource_catalog_pg"""
sql """ drop table if exists ${inDorisTable} """
sql """
CREATE TABLE ${inDorisTable} (
`id` INT NULL COMMENT "主键id",
`name` string NULL COMMENT "名字"
) DISTRIBUTED BY HASH(id) BUCKETS 10
PROPERTIES("replication_num" = "1");
"""
sql """switch ${catalog_name}"""
sql """ use ${ex_schema_name}"""
order_qt_test0 """ select * from test3 order by id; """
sql """ insert into internal.${internal_db_name}.${inDorisTable} select id, name from test3; """
order_qt_in_tb """ select id, name from internal.${internal_db_name}.${inDorisTable} order by id; """
order_qt_test1 """ select * from test1 order by k8; """
order_qt_test2 """ select * from test2 order by id; """
order_qt_test3 """ select * from test2_item order by id; """
order_qt_test4 """ select * from test2_view order by id; """
order_qt_test5 """ select * from test3 order by id; """
order_qt_test6 """ select * from test4 order by id; """
order_qt_test7 """ select * from test5 order by id; """
order_qt_test8 """ select * from test6 order by id; """
order_qt_test9 """ select * from test7 order by id; """
order_qt_test10 """ select * from test8 order by id; """
order_qt_test11 """ select * from test9 order by id1; """
sql """ use ${ex_schema_name2}"""
order_qt_test12 """ select * from test10 order by id; """
order_qt_test13 """ select * from test11 order by id; """
order_qt_test14 """ select * from test12 order by id; """
sql """drop catalog if exists ${catalog_name} """
sql """drop resource if exists jdbc_resource_catalog_pg"""
// test old create-catalog syntax for compatibility
sql """ CREATE CATALOG ${catalog_name} PROPERTIES (
"type"="jdbc",
"jdbc.user"="postgres",
"jdbc.password"="123456",
"jdbc.jdbc_url" = "jdbc:postgresql://127.0.0.1:${pg_port}/postgres?useSSL=false&currentSchema=doris_test",
"jdbc.driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/postgresql-42.5.0.jar",
"jdbc.driver_class" = "org.postgresql.Driver");
"""
sql """switch ${catalog_name}"""
sql """use ${ex_schema_name}"""
order_qt_test_old """ select * from test3 order by id; """
sql """drop resource if exists jdbc_resource_catalog_pg"""
}
}