diff --git a/docs/en/docs/lakehouse/external-table/jdbc.md b/docs/en/docs/lakehouse/external-table/jdbc.md index 104f3a0f69..6f6695b3dc 100644 --- a/docs/en/docs/lakehouse/external-table/jdbc.md +++ b/docs/en/docs/lakehouse/external-table/jdbc.md @@ -238,7 +238,7 @@ PROPERTIES ( | OceanBase Version | OceanBase JDBC Driver Version | |-------------------|-------------------------------| -| 4.1.0 | oceanbase-client-2.4.2.jar | +| 3.2.3 | oceanbase-client-2.4.2.jar | ```sql CREATE EXTERNAL RESOURCE jdbc_oceanbase @@ -248,7 +248,8 @@ properties ( "password"="", "jdbc_url" = "jdbc:oceanbase://localhost:2881/test", "driver_url" = "file:///path/to/oceanbase-client-2.4.2.jar", - "driver_class" = "com.oceanbase.jdbc.Driverr" + "driver_class" = "com.oceanbase.jdbc.Driver", + "oceanbase_mode" = "mysql" or "oracle" ); CREATE EXTERNAL TABLE `ext_oceanbase` ( @@ -261,6 +262,10 @@ PROPERTIES ( ); ``` +> **Note:** +> +> When creating an OceanBase external table, you only need to specify the `oceanbase mode` parameter when creating a resource, and the table type of the table to be created is oceanbase + ## Type Mapping The followings list how data types in different databases are mapped in Doris. @@ -400,20 +405,8 @@ The followings list how data types in different databases are mapped in Doris. ### OceanBase -| OceanBase | Doris | -|:---------------:|:--------:| -| BOOLEAN | STRING | -| BIT | STRING | -| TINYINT | TINYINT | -| SMALLINT | SMALLINT | -| INT | INT | -| BIGINT | BIGINT | -| VARCHAR | VARCHAR | -| DATE | DATE | -| FLOAT | FLOAT | -| DATETIME | DATETIME | -| DOUBLE | DOUBLE | -| DECIMAL | DECIMAL | +For MySQL mode, please refer to [MySQL type mapping](#MySQL) +For Oracle mode, please refer to [Oracle type mapping](#Oracle) ## Q&A diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md b/docs/en/docs/lakehouse/multi-catalog/jdbc.md index f8448119a6..83bc625fb2 100644 --- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md @@ -205,29 +205,40 @@ When Trino is mapped, Doris's Database corresponds to a Schema in Trino that spe ```sql -CREATE CATALOG jdbc_oceanbase PROPERTIES ( +CREATE CATALOG jdbc_oceanbase_mysql PROPERTIES ( "type"="jdbc", "user"="root", "password"="123456", "jdbc_url" = "jdbc:oceanbase://127.0.0.1:2881/demo", "driver_url" = "oceanbase-client-2.4.2.jar", - "driver_class" = "com.oceanbase.jdbc.Drive" + "driver_class" = "com.oceanbase.jdbc.Drive", + "oceanbase_mode" = "mysql" +) + +CREATE CATALOG jdbc_oceanbase_oracle PROPERTIES ( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:oceanbase://127.0.0.1:2881/demo", + "driver_url" = "oceanbase-client-2.4.2.jar", + "driver_class" = "com.oceanbase.jdbc.Drive", + "oceanbase_mode" = "oracle" ) ``` ### Parameter Description -| Parameter | Required or Not | Default Value | Description | -| --------------- | --------------- | ------------- | -------------------------------------------------- | -| `user` | Yes | | Username in relation to the corresponding database | -| `password` | Yes | | Password for the corresponding database | -| `jdbc_url ` | Yes | | JDBC connection string | -| `driver_url ` | Yes | | JDBC Driver Jar | -| `driver_class ` | Yes | | JDBC Driver Class | -| `only_specified_database` | No | "false" | Whether only the database specified to be synchronized. | -| `lower_case_table_names` | No | "false" | Whether to synchronize jdbc external data source table names in lower case. | -| `specified_database_list` | No | "" | When only_specified_database=true，only synchronize the specified databases. split with ','. db name is case sensitive.| - +| Parameter | Required or Not | Default Value | Description | +|---------------------------|-----------------|---------------| -------------------------------------------------- | +| `user` | Yes | | Username in relation to the corresponding database | +| `password` | Yes | | Password for the corresponding database | +| `jdbc_url ` | Yes | | JDBC connection string | +| `driver_url ` | Yes | | JDBC Driver Jar | +| `driver_class ` | Yes | | JDBC Driver Class | +| `only_specified_database` | No | "false" | Whether only the database specified to be synchronized. | +| `lower_case_table_names` | No | "false" | Whether to synchronize jdbc external data source table names in lower case. | +| `specified_database_list` | No | "" | When only_specified_database=true，only synchronize the specified databases. split with ','. db name is case sensitive.| +| `oceanbase_mode` | No | "" | When the connected external data source is OceanBase, the mode must be specified as mysql or oracle | > `driver_url` can be specified in three ways: > > 1. File name. For example, `mysql-connector-java-5.1.47.jar`. Please place the Jar file package in `jdbc_drivers/` under the FE/BE deployment directory in advance so the system can locate the file. You can change the location of the file by modifying `jdbc_drivers_dir` in fe.conf and be.conf. @@ -459,32 +470,8 @@ Currently, only Hive connected to Trino has been tested. Other data sources conn ### OceanBase -| OceanBase Type | Doris Type | Comment | -|------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------------| -| TINYINT | TINYINT | | -| SMALLINT | SMALLINT | | -| MEDIUMINT | INT | | -| INT | INT | | -| BIGINT | BIGINT | | -| UNSIGNED TINYINT | SMALLINT | Doris does not support UNSIGNED data types so UNSIGNED TINYINT will be mapped to SMALLINT. | -| UNSIGNED MEDIUMINT | INT | Doris does not support UNSIGNED data types so UNSIGNED MEDIUMINT will be mapped to INT. | -| UNSIGNED INT | BIGINT | Doris does not support UNSIGNED data types so UNSIGNED INT will be mapped to BIGINT. | -| UNSIGNED BIGINT | LARGEINT | Doris does not support UNSIGNED data types so UNSIGNED BIGINT will be mapped to LARGEINT. | -| FLOAT | FLOAT | | -| DOUBLE | DOUBLE | | -| DECIMAL | DECIMALV3 | | -| DATE | DATEV2 | | -| TIMESTAMP | DATETIMEV2 | | -| DATETIME | DATETIMEV2 | | -| YEAR | DATETIMEV2 | | -| TIME | STRING | | -| CHAR | CHAR | | -| VARCHAR | VARCHAR | | -| BOOLEAN、TINYTEXT、TEXT、MEDIUMTEXT、LONGTEXT、TINYBLOB、BLOB、MEDIUMBLOB、LONGBLOB、TINYSTRING、STRING、MEDIUMSTRING、LONGSTRING、BINARY、VARBINARY、JSON、SET、BIT | STRING | | -| Other | UNSUPPORTED | | - -**Note:** -At present, only the MySQL mode of OceanBase has been adapted, while the Oracle mode has not been fully tested +For MySQL mode, please refer to [MySQL type mapping](#MySQL) +For Oracle mode, please refer to [Oracle type mapping](#Oracle) ## FAQ diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md index a04ffd43b7..92eb2345ee 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md @@ -241,10 +241,10 @@ CREATE CATALOG catalog_name PROPERTIES ( "driver_url" = "file:///path/to/mssql-jdbc-11.2.3.jre8.jar", "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver" ); - CREATE CATALOG sqlserver_catlog WITH RESOURCE sqlserver_resource; + CREATE CATALOG sqlserver_catalog WITH RESOURCE sqlserver_resource; -- The second way - CREATE CATALOG sqlserver_catlog PROPERTIES ( + CREATE CATALOG sqlserver_catalog PROPERTIES ( "type"="jdbc", "user"="SA", "password"="Doris123456", @@ -265,7 +265,7 @@ CREATE CATALOG catalog_name PROPERTIES ( "driver_url" = "file:///path/to/ngdbc.jar", "driver_class" = "com.sap.db.jdbc.Driver" ); - CREATE CATALOG saphana_catlog WITH RESOURCE saphana_resource; + CREATE CATALOG saphana_catalog WITH RESOURCE saphana_resource; -- The second way CREATE CATALOG saphana_catalog PROPERTIES ( @@ -289,7 +289,7 @@ CREATE CATALOG catalog_name PROPERTIES ( "driver_url" = "file:///path/to/trino-jdbc-389.jar", "driver_class" = "io.trino.jdbc.TrinoDriver" ); - CREATE CATALOG trino_catlog WITH RESOURCE trino_resource; + CREATE CATALOG trino_catalog WITH RESOURCE trino_resource; -- The second way CREATE CATALOG trino_catalog PROPERTIES ( @@ -311,9 +311,10 @@ CREATE CATALOG catalog_name PROPERTIES ( "password"="", "jdbc_url" = "jdbc:oceanbase://localhost:2881/demo", "driver_url" = "file:///path/to/oceanbase-client-2.4.2.jar", - "driver_class" = "com.oceanbase.jdbc.Driver" + "driver_class" = "com.oceanbase.jdbc.Driver", + "oceanbase_mode" = "mysql" or "oracle" ); - CREATE CATALOG oceanbase_catlog WITH RESOURCE oceanbase_resource; + CREATE CATALOG oceanbase_catalog WITH RESOURCE oceanbase_resource; -- The second way CREATE CATALOG oceanbase_catalog PROPERTIES ( @@ -322,7 +323,8 @@ CREATE CATALOG catalog_name PROPERTIES ( "password"="", "jdbc_url" = "jdbc:oceanbase://localhost:2881/demo", "driver_url" = "file:///path/to/oceanbase-client-2.4.2.jar", - "driver_class" = "com.oceanbase.jdbc.Driver" + "driver_class" = "com.oceanbase.jdbc.Driver", + "oceanbase_mode" = "mysql" or "oracle" ); ``` diff --git a/docs/zh-CN/docs/lakehouse/external-table/jdbc.md b/docs/zh-CN/docs/lakehouse/external-table/jdbc.md index 88b084316f..0aec57b9f6 100644 --- a/docs/zh-CN/docs/lakehouse/external-table/jdbc.md +++ b/docs/zh-CN/docs/lakehouse/external-table/jdbc.md @@ -232,7 +232,7 @@ PROPERTIES ( | OceanBase 版本 | OceanBase JDBC驱动版本 | |--------------|--------------------| -| 4.1.0 | oceanbase-client-2.4.2.jar | +| 3.2.3 | oceanbase-client-2.4.2.jar | ```sql CREATE EXTERNAL RESOURCE jdbc_oceanbase @@ -242,7 +242,8 @@ properties ( "password"="", "jdbc_url" = "jdbc:oceanbase://localhost:2881/test", "driver_url" = "file:///path/to/oceanbase-client-2.4.2.jar", - "driver_class" = "com.oceanbase.jdbc.Driverr" + "driver_class" = "com.oceanbase.jdbc.Driver", + "oceanbase_mode" = "mysql" or "oracle" ); CREATE EXTERNAL TABLE `ext_oceanbase` ( @@ -254,6 +255,9 @@ PROPERTIES ( "table_type"="oceanbase" ); ``` +> **注意：** +> +> 在创建OceanBase外表时，只需在创建Resource时指定`oceanbase_mode`参数，创建外表的table_type为oceanbase。 ## 类型匹配 @@ -393,21 +397,8 @@ PROPERTIES ( ### OceanBase -| OceanBase | Doris | -|:---------------:|:--------:| -| BOOLEAN | STRING | -| BIT | STRING | -| TINYINT | TINYINT | -| SMALLINT | SMALLINT | -| INT | INT | -| BIGINT | BIGINT | -| VARCHAR | VARCHAR | -| DATE | DATE | -| FLOAT | FLOAT | -| DATETIME | DATETIME | -| DOUBLE | DOUBLE | -| DECIMAL | DECIMAL | - +MySQL 模式请参考 [MySQL类型映射](#MySQL) +Oracle 模式请参考 [Oracle类型映射](#Oracle) ## Q&A diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md index f31114ea37..b1671e6b62 100644 --- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md +++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md @@ -206,28 +206,41 @@ CREATE CATALOG trino_catalog PROPERTIES ( ```sql -CREATE CATALOG jdbc_oceanbase PROPERTIES ( +CREATE CATALOG jdbc_oceanbase_mysql PROPERTIES ( "type"="jdbc", "user"="root", "password"="123456", "jdbc_url" = "jdbc:oceanbase://127.0.0.1:2881/demo", "driver_url" = "oceanbase-client-2.4.2.jar", - "driver_class" = "com.oceanbase.jdbc.Drive" + "driver_class" = "com.oceanbase.jdbc.Drive", + "oceanbase_mode" = "mysql" +) + +CREATE CATALOG jdbc_oceanbase_oracle PROPERTIES ( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:oceanbase://127.0.0.1:2881/demo", + "driver_url" = "oceanbase-client-2.4.2.jar", + "driver_class" = "com.oceanbase.jdbc.Drive", + "oceanbase_mode" = "oracle" ) ``` ### 参数说明 -参数 | 是否必须 | 默认值 | 说明 ---- | --- | --- | --- -`user` | 是 | | 对应数据库的用户名 | -`password` | 是 | | 对应数据库的密码 | -`jdbc_url` | 是 | | JDBC 连接串 | -`driver_url` | 是 | | JDBC Driver Jar 包名称* | -`driver_class` | 是 | | JDBC Driver Class 名称 | -`only_specified_database` | 否 | "false" | 指定是否只同步指定的 database | -`lower_case_table_names` | 否 | "false" | 是否以小写的形式同步jdbc外部数据源的表名 | -`specified_database_list` | 否 | "" | 当only_specified_database=true时，指定同步多个database，以','分隔。db名称是大小写敏感的。 | +| 参数 | 是否必须 | 默认值 | 说明 | +|---------------------------|----------|-----------|------------------------------------------------------------------- | +| `user` | 是 | | 对应数据库的用户名 | +| `password` | 是 | | 对应数据库的密码 | +| `jdbc_url` | 是 | | JDBC 连接串 | +| `driver_url` | 是 | | JDBC Driver Jar 包名称* | +| `driver_class` | 是 | | JDBC Driver Class 名称 | +| `only_specified_database` | 否 | "false" | 指定是否只同步指定的 database | +| `lower_case_table_names` | 否 | "false" | 是否以小写的形式同步jdbc外部数据源的表名 | +| `specified_database_list` | 否 | "" | 当only_specified_database=true时，指定同步多个database，以','分隔。db名称是大小写敏感的。 | +| `oceanbase_mode` | 否 | "" | 当连接的外部数据源为OceanBase时，必须为其指定模式为mysql或oracle | + > `driver_url` 可以通过以下三种方式指定： > @@ -457,32 +470,8 @@ set enable_odbc_transcation = true; ### OceanBase -| OceanBase Type | Doris Type | Comment | -|-----------------------------------------------------------------------------------------------------------------------------------------------------|-------------|-------------------------------| -| TINYINT | TINYINT | | -| SMALLINT | SMALLINT | | -| MEDIUMINT | INT | | -| INT | INT | | -| BIGINT | BIGINT | | -| UNSIGNED TINYINT | SMALLINT | Doris没有UNSIGNED数据类型，所以扩大一个数量级 | -| UNSIGNED MEDIUMINT | INT | Doris没有UNSIGNED数据类型，所以扩大一个数量级 | -| UNSIGNED INT | BIGINT | Doris没有UNSIGNED数据类型，所以扩大一个数量级 | -| UNSIGNED BIGINT | LARGEINT | Doris没有UNSIGNED数据类型，所以扩大一个数量级 | -| FLOAT | FLOAT | | -| DOUBLE | DOUBLE | | -| DECIMAL | DECIMALV3 | | -| DATE | DATEV2 | | -| TIMESTAMP | DATETIMEV2 | | -| DATETIME | DATETIMEV2 | | -| YEAR | DATETIMEV2 | | -| TIME | STRING | | -| CHAR | CHAR | | -| VARCHAR | VARCHAR | | -| BOOLEAN、TINYTEXT、TEXT、MEDIUMTEXT、LONGTEXT、TINYBLOB、BLOB、MEDIUMBLOB、LONGBLOB、TINYSTRING、STRING、MEDIUMSTRING、LONGSTRING、BINARY、VARBINARY、JSON、SET、BIT | STRING | | -| Other | UNSUPPORTED | | - -**Note:** -目前仅针对OceanBase的MySQL模式做了适配，Oracle模式并未完整测试 +MySQL 模式请参考 [MySQL类型映射](#MySQL) +Oracle 模式请参考 [Oracle类型映射](#Oracle) ## 常见问题 diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md index e8d27676e7..30ad218161 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-CATALOG.md @@ -247,10 +247,10 @@ CREATE CATALOG catalog_name PROPERTIES ( "driver_url" = "file:///path/to/mssql-jdbc-11.2.3.jre8.jar", "driver_class" = "com.microsoft.sqlserver.jdbc.SQLServerDriver" ); - CREATE CATALOG sqlserver_catlog WITH RESOURCE sqlserver_resource; + CREATE CATALOG sqlserver_catalog WITH RESOURCE sqlserver_resource; -- 方式二 - CREATE CATALOG sqlserver_catlog PROPERTIES ( + CREATE CATALOG sqlserver_catalog PROPERTIES ( "type"="jdbc", "user"="SA", "password"="Doris123456", @@ -271,7 +271,7 @@ CREATE CATALOG catalog_name PROPERTIES ( "driver_url" = "file:///path/to/ngdbc.jar", "driver_class" = "com.sap.db.jdbc.Driver" ); - CREATE CATALOG saphana_catlog WITH RESOURCE saphana_resource; + CREATE CATALOG saphana_catalog WITH RESOURCE saphana_resource; -- 方式二 CREATE CATALOG saphana_catalog PROPERTIES ( @@ -295,7 +295,7 @@ CREATE CATALOG catalog_name PROPERTIES ( "driver_url" = "file:///path/to/trino-jdbc-389.jar", "driver_class" = "io.trino.jdbc.TrinoDriver" ); - CREATE CATALOG trino_catlog WITH RESOURCE trino_resource; + CREATE CATALOG trino_catalog WITH RESOURCE trino_resource; -- 方式二 CREATE CATALOG trino_catalog PROPERTIES ( @@ -317,9 +317,10 @@ CREATE CATALOG catalog_name PROPERTIES ( "password"="", "jdbc_url" = "jdbc:oceanbase://localhost:2881/demo", "driver_url" = "file:///path/to/oceanbase-client-2.4.2.jar", - "driver_class" = "com.oceanbase.jdbc.Driver" + "driver_class" = "com.oceanbase.jdbc.Driver", + "oceanbase_mode" = "mysql" or "oracle" ); - CREATE CATALOG oceanbase_catlog WITH RESOURCE oceanbase_resource; + CREATE CATALOG oceanbase_catalog WITH RESOURCE oceanbase_resource; -- 方式二 CREATE CATALOG oceanbase_catalog PROPERTIES ( @@ -328,7 +329,8 @@ CREATE CATALOG catalog_name PROPERTIES ( "password"="", "jdbc_url" = "jdbc:oceanbase://localhost:2881/demo", "driver_url" = "file:///path/to/oceanbase-client-2.4.2.jar", - "driver_class" = "com.oceanbase.jdbc.Driver" + "driver_class" = "com.oceanbase.jdbc.Driver", + "oceanbase_mode" = "mysql" or "oracle" ); ``` diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index c406fcafad..536744b724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -69,6 +69,7 @@ public class JdbcResource extends Resource { public static final String JDBC_CLICKHOUSE = "jdbc:clickhouse"; public static final String JDBC_SAP_HANA = "jdbc:sap"; public static final String JDBC_TRINO = "jdbc:trino"; + public static final String JDBC_OCEANBASE = "jdbc:oceanbase"; public static final String MYSQL = "MYSQL"; public static final String POSTGRESQL = "POSTGRESQL"; @@ -77,6 +78,8 @@ public class JdbcResource extends Resource { public static final String CLICKHOUSE = "CLICKHOUSE"; public static final String SAP_HANA = "SAP_HANA"; public static final String TRINO = "TRINO"; + public static final String OCEANBASE = "OCEANBASE"; + public static final String OCEANBASE_ORACLE = "OCEANBASE_ORACLE"; public static final String JDBC_PROPERTIES_PREFIX = "jdbc."; public static final String JDBC_URL = "jdbc_url"; @@ -87,6 +90,7 @@ public class JdbcResource extends Resource { public static final String TYPE = "type"; public static final String ONLY_SPECIFIED_DATABASE = "only_specified_database"; public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; + public static final String OCEANBASE_MODE = "oceanbase_mode"; public static final String CHECK_SUM = "checksum"; private static final ImmutableList ALL_PROPERTIES = new ImmutableList.Builder().add( JDBC_URL, @@ -97,12 +101,14 @@ public class JdbcResource extends Resource { TYPE, ONLY_SPECIFIED_DATABASE, LOWER_CASE_TABLE_NAMES, - SPECIFIED_DATABASE_LIST + SPECIFIED_DATABASE_LIST, + OCEANBASE_MODE ).build(); private static final ImmutableList OPTIONAL_PROPERTIES = new ImmutableList.Builder().add( ONLY_SPECIFIED_DATABASE, LOWER_CASE_TABLE_NAMES, - SPECIFIED_DATABASE_LIST + SPECIFIED_DATABASE_LIST, + OCEANBASE_MODE ).build(); // The default value of optional properties @@ -113,6 +119,7 @@ public class JdbcResource extends Resource { OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ONLY_SPECIFIED_DATABASE, "false"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(LOWER_CASE_TABLE_NAMES, "false"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(SPECIFIED_DATABASE_LIST, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(OCEANBASE_MODE, ""); } // timeout for both connection and read. 10 seconds is long enough. @@ -139,7 +146,7 @@ public class JdbcResource extends Resource { for (String propertyKey : ALL_PROPERTIES) { replaceIfEffectiveValue(this.configs, propertyKey, properties.get(propertyKey)); } - this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL))); + this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL), getProperty(OCEANBASE_MODE))); super.modifyProperties(properties); } @@ -172,7 +179,7 @@ public class JdbcResource extends Resource { throw new DdlException("JdbcResource Missing " + property + " in properties"); } } - this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL))); + this.configs.put(JDBC_URL, handleJdbcUrl(getProperty(JDBC_URL), getProperty(OCEANBASE_MODE))); configs.put(CHECK_SUM, computeObjectChecksum(getProperty(DRIVER_URL))); } @@ -255,7 +262,7 @@ public class JdbcResource extends Resource { } } - public static String parseDbType(String url) throws DdlException { + public static String parseDbType(String url, String oceanbaseMode) throws DdlException { if (url.startsWith(JDBC_MYSQL) || url.startsWith(JDBC_MARIADB)) { return MYSQL; } else if (url.startsWith(JDBC_POSTGRESQL)) { @@ -270,15 +277,27 @@ public class JdbcResource extends Resource { return SAP_HANA; } else if (url.startsWith(JDBC_TRINO)) { return TRINO; + } else if (url.startsWith(JDBC_OCEANBASE)) { + if (oceanbaseMode == null || oceanbaseMode.isEmpty()) { + throw new DdlException("OceanBase mode must be specified for OceanBase databases" + + "(either 'mysql' or 'oracle')"); + } + if (oceanbaseMode.equalsIgnoreCase("mysql")) { + return OCEANBASE; + } else if (oceanbaseMode.equalsIgnoreCase("oracle")) { + return OCEANBASE_ORACLE; + } else { + throw new DdlException("Invalid OceanBase mode: " + oceanbaseMode + ". Must be 'mysql' or 'oracle'"); + } } throw new DdlException("Unsupported jdbc database type, please check jdbcUrl: " + url); } - public static String handleJdbcUrl(String jdbcUrl) throws DdlException { + public static String handleJdbcUrl(String jdbcUrl, String oceanbaseMode) throws DdlException { // delete all space in jdbcUrl String newJdbcUrl = jdbcUrl.replaceAll(" ", ""); - String dbType = parseDbType(newJdbcUrl); - if (dbType.equals(MYSQL)) { + String dbType = parseDbType(newJdbcUrl, oceanbaseMode); + if (dbType.equals(MYSQL) || dbType.equals(OCEANBASE)) { // `yearIsDateType` is a parameter of JDBC, and the default is true. // We force the use of `yearIsDateType=false` newJdbcUrl = checkAndSetJdbcBoolParam(newJdbcUrl, "yearIsDateType", "true", "false"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index aeaa1fd60b..3e28529f46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -39,6 +39,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -76,6 +77,8 @@ public class JdbcTable extends Table { tempMap.put("clickhouse", TOdbcTableType.CLICKHOUSE); tempMap.put("sap_hana", TOdbcTableType.SAP_HANA); tempMap.put("trino", TOdbcTableType.TRINO); + tempMap.put("oceanbase", TOdbcTableType.OCEANBASE); + tempMap.put("oceanbase_oracle", TOdbcTableType.OCEANBASE_ORACLE); TABLE_TYPE_MAP = Collections.unmodifiableMap(tempMap); } @@ -267,7 +270,11 @@ public class JdbcTable extends Table { if (Strings.isNullOrEmpty(jdbcTypeName)) { throw new DdlException("property " + TABLE_TYPE + " must be set"); } - if (!TABLE_TYPE_MAP.containsKey(jdbcTypeName.toLowerCase())) { + + Map tableTypeMapWithoutOceanbaseOracle = new HashMap<>(TABLE_TYPE_MAP); + tableTypeMapWithoutOceanbaseOracle.remove("oceanbase_oracle"); + + if (!tableTypeMapWithoutOceanbaseOracle.containsKey(jdbcTypeName.toLowerCase())) { throw new DdlException("Unknown jdbc table type: " + jdbcTypeName); } @@ -286,5 +293,19 @@ public class JdbcTable extends Table { driverClass = jdbcResource.getProperty(DRIVER_CLASS); driverUrl = jdbcResource.getProperty(DRIVER_URL); checkSum = jdbcResource.getProperty(CHECK_SUM); + + // get oceanbase_mode + String oceanbaseMode = jdbcResource.getProperty("oceanbase_mode"); + + // by oceanbase_mode set jdbcTypeName + if ("oceanbase".equalsIgnoreCase(jdbcTypeName)) { + if ("mysql".equalsIgnoreCase(oceanbaseMode)) { + jdbcTypeName = "oceanbase"; + } else if ("oracle".equalsIgnoreCase(oceanbaseMode)) { + jdbcTypeName = "oceanbase_oracle"; + } else { + throw new DdlException("Unknown oceanbase_mode: " + oceanbaseMode); + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java index 9211a80300..5693255be3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java @@ -112,9 +112,15 @@ public class OdbcTable extends Table { return list.stream().map(s -> "\"" + s + "\"").collect(Collectors.joining(".")); } + private static String oceanbaseOracleProperName(String name) { + List list = Arrays.asList(name.split("\\.")); + return list.stream().map(s -> "\"" + s + "\"").collect(Collectors.joining(".")); + } + public static String databaseProperName(TOdbcTableType tableType, String name) { switch (tableType) { case MYSQL: + case OCEANBASE: return mysqlProperName(name); case SQLSERVER: return mssqlProperName(name); @@ -128,6 +134,8 @@ public class OdbcTable extends Table { return saphanaProperName(name); case TRINO: return trinoProperName(name); + case OCEANBASE_ORACLE: + return oceanbaseOracleProperName(name); default: return name; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java index 5ebbb512b2..256737197d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java @@ -80,8 +80,9 @@ public class JdbcExternalCatalog extends ExternalCatalog { properties.put(StringUtils.removeStart(kv.getKey(), JdbcResource.JDBC_PROPERTIES_PREFIX), kv.getValue()); } String jdbcUrl = properties.getOrDefault(JdbcResource.JDBC_URL, ""); + String oceanbaseMode = properties.getOrDefault(JdbcResource.OCEANBASE_MODE, ""); if (!Strings.isNullOrEmpty(jdbcUrl)) { - jdbcUrl = JdbcResource.handleJdbcUrl(jdbcUrl); + jdbcUrl = JdbcResource.handleJdbcUrl(jdbcUrl, oceanbaseMode); properties.put(JdbcResource.JDBC_URL, jdbcUrl); } @@ -132,10 +133,14 @@ public class JdbcExternalCatalog extends ExternalCatalog { return catalogProperty.getOrDefault(JdbcResource.SPECIFIED_DATABASE_LIST, ""); } + public String getOceanBaseMode() { + return catalogProperty.getOrDefault(JdbcResource.OCEANBASE_MODE, ""); + } + @Override protected void initLocalObjectsImpl() { jdbcClient = new JdbcClient(getJdbcUser(), getJdbcPasswd(), getJdbcUrl(), getDriverUrl(), getDriverClass(), - getOnlySpecifiedDatabase(), getLowerCaseTableNames(), getSpecifiedDatabaseMap()); + getOnlySpecifiedDatabase(), getLowerCaseTableNames(), getSpecifiedDatabaseMap(), getOceanBaseMode()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java index 08e04321f8..bcab8f09ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java @@ -66,16 +66,20 @@ public class JdbcClient { // only used when isLowerCaseTableNames = true. private Map lowerTableToRealTable = Maps.newHashMap(); + private String oceanbaseMode = ""; + public JdbcClient(String user, String password, String jdbcUrl, String driverUrl, String driverClass, - String onlySpecifiedDatabase, String isLowerCaseTableNames, Map specifiedDatabaseMap) { + String onlySpecifiedDatabase, String isLowerCaseTableNames, Map specifiedDatabaseMap, + String oceanbaseMode) { this.jdbcUser = user; this.isOnlySpecifiedDatabase = Boolean.valueOf(onlySpecifiedDatabase).booleanValue(); this.isLowerCaseTableNames = Boolean.valueOf(isLowerCaseTableNames).booleanValue(); if (specifiedDatabaseMap != null) { this.specifiedDatabaseMap = specifiedDatabaseMap; } + this.oceanbaseMode = oceanbaseMode; try { - this.dbType = JdbcResource.parseDbType(jdbcUrl); + this.dbType = JdbcResource.parseDbType(jdbcUrl, oceanbaseMode); } catch (DdlException e) { throw new JdbcClientException("Failed to parse db type from jdbcUrl: " + jdbcUrl, e); } @@ -185,6 +189,7 @@ public class JdbcClient { switch (dbType) { case JdbcResource.MYSQL: case JdbcResource.CLICKHOUSE: + case JdbcResource.OCEANBASE: rs = stmt.executeQuery("SHOW DATABASES"); break; case JdbcResource.POSTGRESQL: @@ -192,6 +197,7 @@ public class JdbcClient { + "'" + jdbcUser + "', nspname, 'USAGE');"); break; case JdbcResource.ORACLE: + case JdbcResource.OCEANBASE_ORACLE: rs = stmt.executeQuery("SELECT DISTINCT OWNER FROM all_tables"); break; case JdbcResource.SQLSERVER: @@ -233,6 +239,7 @@ public class JdbcClient { switch (dbType) { case JdbcResource.MYSQL: case JdbcResource.CLICKHOUSE: + case JdbcResource.OCEANBASE: databaseNames.add(conn.getCatalog()); break; case JdbcResource.POSTGRESQL: @@ -240,6 +247,7 @@ public class JdbcClient { case JdbcResource.SQLSERVER: case JdbcResource.SAP_HANA: case JdbcResource.TRINO: + case JdbcResource.OCEANBASE_ORACLE: databaseNames.add(conn.getSchema()); break; default: @@ -266,6 +274,7 @@ public class JdbcClient { String catalogName = conn.getCatalog(); switch (dbType) { case JdbcResource.MYSQL: + case JdbcResource.OCEANBASE: rs = databaseMetaData.getTables(dbName, null, null, types); break; case JdbcResource.POSTGRESQL: @@ -273,6 +282,7 @@ public class JdbcClient { case JdbcResource.CLICKHOUSE: case JdbcResource.SQLSERVER: case JdbcResource.SAP_HANA: + case JdbcResource.OCEANBASE_ORACLE: rs = databaseMetaData.getTables(null, dbName, null, types); break; case JdbcResource.TRINO: @@ -306,6 +316,7 @@ public class JdbcClient { String catalogName = conn.getCatalog(); switch (dbType) { case JdbcResource.MYSQL: + case JdbcResource.OCEANBASE: rs = databaseMetaData.getTables(dbName, null, tableName, types); break; case JdbcResource.POSTGRESQL: @@ -313,6 +324,7 @@ public class JdbcClient { case JdbcResource.CLICKHOUSE: case JdbcResource.SQLSERVER: case JdbcResource.SAP_HANA: + case JdbcResource.OCEANBASE_ORACLE: rs = databaseMetaData.getTables(null, dbName, null, types); break; case JdbcResource.TRINO: @@ -381,6 +393,7 @@ public class JdbcClient { // Can contain single-character wildcards ("_"), or multi-character wildcards ("%") switch (dbType) { case JdbcResource.MYSQL: + case JdbcResource.OCEANBASE: rs = databaseMetaData.getColumns(dbName, null, tableName, null); break; case JdbcResource.POSTGRESQL: @@ -388,6 +401,7 @@ public class JdbcClient { case JdbcResource.CLICKHOUSE: case JdbcResource.SQLSERVER: case JdbcResource.SAP_HANA: + case JdbcResource.OCEANBASE_ORACLE: rs = databaseMetaData.getColumns(null, dbName, tableName, null); break; case JdbcResource.TRINO: @@ -427,12 +441,14 @@ public class JdbcClient { public Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { switch (dbType) { case JdbcResource.MYSQL: + case JdbcResource.OCEANBASE: return mysqlTypeToDoris(fieldSchema); case JdbcResource.POSTGRESQL: return postgresqlTypeToDoris(fieldSchema); case JdbcResource.CLICKHOUSE: return clickhouseTypeToDoris(fieldSchema); case JdbcResource.ORACLE: + case JdbcResource.OCEANBASE_ORACLE: return oracleTypeToDoris(fieldSchema); case JdbcResource.SQLSERVER: return sqlserverTypeToDoris(fieldSchema); @@ -876,7 +892,6 @@ public class JdbcClient { } } - private Type createDecimalOrStringType(int precision, int scale) { if (precision <= ScalarType.MAX_DECIMAL128_PRECISION) { return ScalarType.createDecimalV3Type(precision, scale); diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 9aa6188e22..f654bff157 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -384,7 +384,9 @@ enum TOdbcTableType { MONGODB, CLICKHOUSE, SAP_HANA, - TRINO + TRINO, + OCEANBASE, + OCEANBASE_ORACLE } struct TJdbcExecutorCtorParams {