From 104284bd6f4d0aa815b09abefce463182665a5cc Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Mon, 13 May 2024 22:37:35 +0800 Subject: [PATCH] [Refactor](jdbc catalog) Enhance Field Handling in JdbcFieldSchema Using Optional for Better Null Safety (#34730) In some cases, using Resultset.getxxx cannot correctly handle null data, so I use Optional to enhance the fields of JdbcFieldSchema for better type mapping when some results are null. --- .../hive/PostgreSQLJdbcHMSCachedClient.java | 1 + .../jdbc/client/JdbcClickHouseClient.java | 7 +- .../datasource/jdbc/client/JdbcClient.java | 55 +---------- .../datasource/jdbc/client/JdbcDB2Client.java | 13 +-- .../jdbc/client/JdbcMySQLClient.java | 60 +++++------- .../jdbc/client/JdbcOceanBaseClient.java | 1 + .../jdbc/client/JdbcOracleClient.java | 31 ++----- .../jdbc/client/JdbcPostgreSQLClient.java | 13 +-- .../jdbc/client/JdbcSQLServerClient.java | 9 +- .../jdbc/client/JdbcSapHanaClient.java | 17 ++-- .../jdbc/client/JdbcTrinoClient.java | 11 ++- .../datasource/jdbc/util/JdbcFieldSchema.java | 91 +++++++++++++++++++ 12 files changed, 165 insertions(+), 144 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java index a0c2aa29ab..932118001e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java @@ -24,6 +24,7 @@ import org.apache.doris.datasource.DatabaseMetadata; import org.apache.doris.datasource.TableMetadata; import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException; import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import org.apache.doris.thrift.TOdbcTableType; import com.google.common.base.Joiner; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java index 35e6c83af1..bdf0cbbc93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java @@ -20,6 +20,9 @@ package org.apache.doris.datasource.jdbc.client; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; + +import java.util.Optional; public class JdbcClickHouseClient extends JdbcClient { @@ -35,7 +38,7 @@ public class JdbcClickHouseClient extends JdbcClient { @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { - String ckType = fieldSchema.getDataTypeName(); + String ckType = fieldSchema.getDataTypeName().orElse("unknown"); if (ckType.startsWith("LowCardinality")) { fieldSchema.setAllowNull(true); @@ -81,7 +84,7 @@ public class JdbcClickHouseClient extends JdbcClient { if (ckType.startsWith("Array")) { String cktype = ckType.substring(6, ckType.length() - 1); - fieldSchema.setDataTypeName(cktype); + fieldSchema.setDataTypeName(Optional.of(cktype)); Type type = jdbcTypeToDoris(fieldSchema); return ArrayType.create(type, true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index 604e54277f..0ae2966079 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -24,11 +24,11 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.jdbc.JdbcIdentifierMapping; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.zaxxer.hikari.HikariDataSource; -import lombok.Data; import lombok.Getter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -255,14 +255,7 @@ public abstract class JdbcClient { public List getSchemaFromResultSetMetaData(ResultSetMetaData metaData) throws SQLException { List schemas = Lists.newArrayList(); for (int i = 1; i <= metaData.getColumnCount(); i++) { - JdbcFieldSchema field = new JdbcFieldSchema(); - field.setColumnName(metaData.getColumnName(i)); - field.setDataType(metaData.getColumnType(i)); - field.setDataTypeName(metaData.getColumnTypeName(i)); - field.setColumnSize(metaData.getColumnDisplaySize(i)); - field.setDecimalDigits(metaData.getScale(i)); - field.setNumPrecRadix(metaData.getPrecision(i)); - schemas.add(field); + schemas.add(new JdbcFieldSchema(metaData, i)); } return schemas; } @@ -347,27 +340,7 @@ public abstract class JdbcClient { String catalogName = getCatalogName(conn); rs = getRemoteColumns(databaseMetaData, catalogName, remoteDbName, remoteTableName); while (rs.next()) { - JdbcFieldSchema field = new JdbcFieldSchema(); - field.setColumnName(rs.getString("COLUMN_NAME")); - field.setDataType(rs.getInt("DATA_TYPE")); - field.setDataTypeName(rs.getString("TYPE_NAME")); - /* - We used this method to retrieve the key column of the JDBC table, but since we only tested mysql, - we kept the default key behavior in the parent class and only overwrite it in the mysql subclass - */ - field.setColumnSize(rs.getInt("COLUMN_SIZE")); - field.setDecimalDigits(rs.getInt("DECIMAL_DIGITS")); - field.setNumPrecRadix(rs.getInt("NUM_PREC_RADIX")); - /* - Whether it is allowed to be NULL - 0 (columnNoNulls) - 1 (columnNullable) - 2 (columnNullableUnknown) - */ - field.setAllowNull(rs.getInt("NULLABLE") != 0); - field.setRemarks(rs.getString("REMARKS")); - field.setCharOctetLength(rs.getInt("CHAR_OCTET_LENGTH")); - tableSchema.add(field); + tableSchema.add(new JdbcFieldSchema(rs)); } } catch (SQLException e) { throw new JdbcClientException("failed to get jdbc columns info for remote table `%s.%s`: %s", @@ -478,28 +451,6 @@ public abstract class JdbcClient { return jdbcLowerCaseMetaMatching.setColumnNameMapping(remoteDbName, remoteTableName, remoteColumns); } - @Data - protected static class JdbcFieldSchema { - protected String columnName; - // The SQL type of the corresponding java.sql.types (Type ID) - protected int dataType; - // The SQL type of the corresponding java.sql.types (Type Name) - protected String dataTypeName; - // For CHAR/DATA, columnSize means the maximum number of chars. - // For NUMERIC/DECIMAL, columnSize means precision. - protected int columnSize; - protected int decimalDigits; - // Base number (usually 10 or 2) - protected int numPrecRadix; - // column description - protected String remarks; - // This length is the maximum number of bytes for CHAR type - // for utf8 encoding, if columnSize=10, then charOctetLength=30 - // because for utf8 encoding, a Chinese character takes up 3 bytes - protected int charOctetLength; - protected boolean isAllowNull; - } - protected abstract Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema); protected Type createDecimalOrStringType(int precision, int scale) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java index af89f33eb9..dafb00ca9e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java @@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import com.google.common.collect.Lists; @@ -63,7 +64,7 @@ public class JdbcDB2Client extends JdbcClient { @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { - String db2Type = fieldSchema.getDataTypeName(); + String db2Type = fieldSchema.getDataTypeName().orElse("unknown"); switch (db2Type) { case "SMALLINT": return Type.SMALLINT; @@ -73,8 +74,8 @@ public class JdbcDB2Client extends JdbcClient { return Type.BIGINT; case "DECFLOAT": case "DECIMAL": { - int precision = fieldSchema.getColumnSize(); - int scale = fieldSchema.getDecimalDigits(); + int precision = fieldSchema.getColumnSize().orElse(0); + int scale = fieldSchema.getDecimalDigits().orElse(0); return createDecimalOrStringType(precision, scale); } case "DOUBLE": @@ -83,18 +84,18 @@ public class JdbcDB2Client extends JdbcClient { return Type.FLOAT; case "CHAR": ScalarType charType = ScalarType.createType(PrimitiveType.CHAR); - charType.setLength(fieldSchema.columnSize); + charType.setLength(fieldSchema.getColumnSize().orElse(0)); return charType; case "VARCHAR": case "LONG VARCHAR": ScalarType varcharType = ScalarType.createType(PrimitiveType.VARCHAR); - varcharType.setLength(fieldSchema.columnSize); + varcharType.setLength(fieldSchema.getColumnSize().orElse(0)); return varcharType; case "DATE": return ScalarType.createDateV2Type(); case "TIMESTAMP": { // postgres can support microsecond - int scale = fieldSchema.getDecimalDigits(); + int scale = fieldSchema.getDecimalDigits().orElse(0); if (scale > 6) { scale = 6; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index 6ea5ea127d..eee254482d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -122,40 +123,21 @@ public class JdbcMySQLClient extends JdbcClient { public List getJdbcColumnsInfo(String localDbName, String localTableName) { Connection conn = getConnection(); ResultSet rs = null; - List tableSchema = com.google.common.collect.Lists.newArrayList(); + List tableSchema = Lists.newArrayList(); String remoteDbName = getRemoteDatabaseName(localDbName); String remoteTableName = getRemoteTableName(localDbName, localTableName); try { DatabaseMetaData databaseMetaData = conn.getMetaData(); String catalogName = getCatalogName(conn); rs = getRemoteColumns(databaseMetaData, catalogName, remoteDbName, remoteTableName); - Map mapFieldtoType = null; - while (rs.next()) { - JdbcFieldSchema field = new JdbcFieldSchema(); - field.setColumnName(rs.getString("COLUMN_NAME")); - field.setDataType(rs.getInt("DATA_TYPE")); - // in mysql-jdbc-connector-8.0.*, TYPE_NAME of the HLL column in doris will be "UNKNOWN" - // in mysql-jdbc-connector-5.1.*, TYPE_NAME of the HLL column in doris will be "HLL" - // in mysql-jdbc-connector-8.0.*, TYPE_NAME of BITMAP column in doris will be "BIT" - // in mysql-jdbc-connector-5.1.*, TYPE_NAME of BITMAP column in doris will be "BITMAP" - field.setDataTypeName(rs.getString("TYPE_NAME")); - if (isDoris) { - mapFieldtoType = getColumnsDataTypeUseQuery(remoteDbName, remoteTableName); - field.setDataTypeName(mapFieldtoType.get(rs.getString("COLUMN_NAME"))); - } - field.setColumnSize(rs.getInt("COLUMN_SIZE")); - field.setDecimalDigits(rs.getInt("DECIMAL_DIGITS")); - field.setNumPrecRadix(rs.getInt("NUM_PREC_RADIX")); - /* - Whether it is allowed to be NULL - 0 (columnNoNulls) - 1 (columnNullable) - 2 (columnNullableUnknown) - */ - field.setAllowNull(rs.getInt("NULLABLE") != 0); - field.setRemarks(rs.getString("REMARKS")); - field.setCharOctetLength(rs.getInt("CHAR_OCTET_LENGTH")); + Map mapFieldtoType = Maps.newHashMap(); + if (isDoris) { + mapFieldtoType = getColumnsDataTypeUseQuery(remoteDbName, remoteTableName); + } + + while (rs.next()) { + JdbcFieldSchema field = new JdbcFieldSchema(rs, mapFieldtoType); tableSchema.add(field); } } catch (SQLException e) { @@ -184,12 +166,12 @@ public class JdbcMySQLClient extends JdbcClient { protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { // For Doris type if (isDoris) { - return dorisTypeToDoris(fieldSchema.getDataTypeName().toUpperCase()); + return dorisTypeToDoris(fieldSchema.getDataTypeName().orElse("unknown").toUpperCase()); } // For mysql type: "INT UNSIGNED": - // fieldSchema.getDataTypeName().split(" ")[0] == "INT" - // fieldSchema.getDataTypeName().split(" ")[1] == "UNSIGNED" - String[] typeFields = fieldSchema.getDataTypeName().split(" "); + // fieldSchema.getDataTypeName().orElse("unknown").split(" ")[0] == "INT" + // fieldSchema.getDataTypeName().orElse("unknown").split(" ")[1] == "UNSIGNED" + String[] typeFields = fieldSchema.getDataTypeName().orElse("unknown").split(" "); String mysqlType = typeFields[0]; // For unsigned int, should extend the type. if (typeFields.length > 1 && "UNSIGNED".equals(typeFields[1])) { @@ -204,8 +186,8 @@ public class JdbcMySQLClient extends JdbcClient { case "BIGINT": return Type.LARGEINT; case "DECIMAL": { - int precision = fieldSchema.getColumnSize() + 1; - int scale = fieldSchema.getDecimalDigits(); + int precision = fieldSchema.getColumnSize().orElse(0) + 1; + int scale = fieldSchema.getDecimalDigits().orElse(0); return createDecimalOrStringType(precision, scale); } case "DOUBLE": @@ -242,7 +224,7 @@ public class JdbcMySQLClient extends JdbcClient { case "DATETIME": { // mysql can support microsecond // use columnSize to calculate the precision of timestamp/datetime - int columnSize = fieldSchema.getColumnSize(); + int columnSize = fieldSchema.getColumnSize().orElse(0); int scale = columnSize > 19 ? columnSize - 20 : 0; if (scale > 6) { scale = 6; @@ -257,18 +239,18 @@ public class JdbcMySQLClient extends JdbcClient { case "DOUBLE": return Type.DOUBLE; case "DECIMAL": { - int precision = fieldSchema.getColumnSize(); - int scale = fieldSchema.getDecimalDigits(); + int precision = fieldSchema.getColumnSize().orElse(0); + int scale = fieldSchema.getDecimalDigits().orElse(0); return createDecimalOrStringType(precision, scale); } case "CHAR": ScalarType charType = ScalarType.createType(PrimitiveType.CHAR); - charType.setLength(fieldSchema.columnSize); + charType.setLength(fieldSchema.getColumnSize().orElse(0)); return charType; case "VARCHAR": - return ScalarType.createVarcharType(fieldSchema.columnSize); + return ScalarType.createVarcharType(fieldSchema.getColumnSize().orElse(0)); case "BIT": - if (fieldSchema.getColumnSize() == 1) { + if (fieldSchema.getColumnSize().orElse(0) == 1) { return Type.BOOLEAN; } else { return ScalarType.createStringType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java index 8628d62d70..b8c5b61aea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java @@ -19,6 +19,7 @@ package org.apache.doris.datasource.jdbc.client; import org.apache.doris.catalog.JdbcResource; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import java.sql.Connection; import java.sql.ResultSet; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 484028ed92..0efa94d9bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -69,27 +70,7 @@ public class JdbcOracleClient extends JdbcClient { if (isModify && isTableModified(rs.getString("TABLE_NAME"), remoteTableName)) { continue; } - JdbcFieldSchema field = new JdbcFieldSchema(); - field.setColumnName(rs.getString("COLUMN_NAME")); - field.setDataType(rs.getInt("DATA_TYPE")); - field.setDataTypeName(rs.getString("TYPE_NAME")); - /* - We used this method to retrieve the key column of the JDBC table, but since we only tested mysql, - we kept the default key behavior in the parent class and only overwrite it in the mysql subclass - */ - field.setColumnSize(rs.getInt("COLUMN_SIZE")); - field.setDecimalDigits(rs.getInt("DECIMAL_DIGITS")); - field.setNumPrecRadix(rs.getInt("NUM_PREC_RADIX")); - /* - Whether it is allowed to be NULL - 0 (columnNoNulls) - 1 (columnNullable) - 2 (columnNullableUnknown) - */ - field.setAllowNull(rs.getInt("NULLABLE") != 0); - field.setRemarks(rs.getString("REMARKS")); - field.setCharOctetLength(rs.getInt("CHAR_OCTET_LENGTH")); - tableSchema.add(field); + tableSchema.add(new JdbcFieldSchema(rs)); } } catch (SQLException e) { throw new JdbcClientException("failed to get table name list from jdbc for table %s:%s", e, remoteTableName, @@ -126,7 +107,7 @@ public class JdbcOracleClient extends JdbcClient { @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { - String oracleType = fieldSchema.getDataTypeName(); + String oracleType = fieldSchema.getDataTypeName().orElse("unknown"); if (oracleType.startsWith("INTERVAL")) { oracleType = oracleType.substring(0, 8); } else if (oracleType.startsWith("TIMESTAMP")) { @@ -134,7 +115,7 @@ public class JdbcOracleClient extends JdbcClient { return Type.UNSUPPORTED; } // oracle can support nanosecond, will lose precision - int scale = fieldSchema.getDecimalDigits(); + int scale = fieldSchema.getDecimalDigits().orElse(0); if (scale > 6) { scale = 6; } @@ -160,8 +141,8 @@ public class JdbcOracleClient extends JdbcClient { * In this case, doris can not determine p and s, so doris can not determine data type. */ case "NUMBER": - int precision = fieldSchema.getColumnSize(); - int scale = fieldSchema.getDecimalDigits(); + int precision = fieldSchema.getColumnSize().orElse(0); + int scale = fieldSchema.getDecimalDigits().orElse(0); if (scale <= 0) { precision -= scale; if (precision < 3) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java index 3d0bbc4e56..e6694ffdc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java @@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; public class JdbcPostgreSQLClient extends JdbcClient { @@ -34,7 +35,7 @@ public class JdbcPostgreSQLClient extends JdbcClient { @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { - String pgType = fieldSchema.getDataTypeName(); + String pgType = fieldSchema.getDataTypeName().orElse("unknown"); switch (pgType) { case "int2": case "smallserial": @@ -46,8 +47,8 @@ public class JdbcPostgreSQLClient extends JdbcClient { case "bigserial": return Type.BIGINT; case "numeric": { - int precision = fieldSchema.getColumnSize(); - int scale = fieldSchema.getDecimalDigits(); + int precision = fieldSchema.getColumnSize().orElse(0); + int scale = fieldSchema.getDecimalDigits().orElse(0); return createDecimalOrStringType(precision, scale); } case "float4": @@ -56,12 +57,12 @@ public class JdbcPostgreSQLClient extends JdbcClient { return Type.DOUBLE; case "bpchar": ScalarType charType = ScalarType.createType(PrimitiveType.CHAR); - charType.setLength(fieldSchema.columnSize); + charType.setLength(fieldSchema.getColumnSize().orElse(0)); return charType; case "timestamp": case "timestamptz": { // postgres can support microsecond - int scale = fieldSchema.getDecimalDigits(); + int scale = fieldSchema.getDecimalDigits().orElse(0); if (scale > 6) { scale = 6; } @@ -72,7 +73,7 @@ public class JdbcPostgreSQLClient extends JdbcClient { case "bool": return Type.BOOLEAN; case "bit": - if (fieldSchema.getColumnSize() == 1) { + if (fieldSchema.getColumnSize().orElse(0) == 1) { return Type.BOOLEAN; } else { return ScalarType.createStringType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java index e3d96421a1..1bb3ece7f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java @@ -19,6 +19,7 @@ package org.apache.doris.datasource.jdbc.client; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; public class JdbcSQLServerClient extends JdbcClient { @@ -28,7 +29,7 @@ public class JdbcSQLServerClient extends JdbcClient { @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { - String originSqlserverType = fieldSchema.getDataTypeName(); + String originSqlserverType = fieldSchema.getDataTypeName().orElse("unknown"); // For sqlserver IDENTITY type, such as 'INT IDENTITY' // originSqlserverType is "int identity", so we only get "int". String sqlserverType = originSqlserverType.split(" ")[0]; @@ -52,8 +53,8 @@ public class JdbcSQLServerClient extends JdbcClient { return ScalarType.createDecimalV3Type(10, 4); case "decimal": case "numeric": { - int precision = fieldSchema.getColumnSize(); - int scale = fieldSchema.getDecimalDigits(); + int precision = fieldSchema.getColumnSize().orElse(0); + int scale = fieldSchema.getDecimalDigits().orElse(0); return ScalarType.createDecimalV3Type(precision, scale); } case "date": @@ -62,7 +63,7 @@ public class JdbcSQLServerClient extends JdbcClient { case "datetime2": case "smalldatetime": { // postgres can support microsecond - int scale = fieldSchema.getDecimalDigits(); + int scale = fieldSchema.getDecimalDigits().orElse(0); if (scale > 6) { scale = 6; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java index 34dde63c80..f10d082bad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java @@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; public class JdbcSapHanaClient extends JdbcClient { protected JdbcSapHanaClient(JdbcClientConfig jdbcClientConfig) { @@ -38,7 +39,7 @@ public class JdbcSapHanaClient extends JdbcClient { @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { - String hanaType = fieldSchema.getDataTypeName(); + String hanaType = fieldSchema.getDataTypeName().orElse("unknown"); switch (hanaType) { case "TINYINT": return Type.TINYINT; @@ -50,9 +51,13 @@ public class JdbcSapHanaClient extends JdbcClient { return Type.BIGINT; case "SMALLDECIMAL": case "DECIMAL": { - int precision = fieldSchema.getColumnSize(); - int scale = fieldSchema.getDecimalDigits(); - return createDecimalOrStringType(precision, scale); + if (!fieldSchema.getDecimalDigits().isPresent()) { + return Type.DOUBLE; + } else { + int precision = fieldSchema.getColumnSize().orElse(0); + int scale = fieldSchema.getDecimalDigits().orElse(0); + return createDecimalOrStringType(precision, scale); + } } case "REAL": return Type.FLOAT; @@ -60,7 +65,7 @@ public class JdbcSapHanaClient extends JdbcClient { return Type.DOUBLE; case "TIMESTAMP": { // postgres can support microsecond - int scale = fieldSchema.getDecimalDigits(); + int scale = fieldSchema.getDecimalDigits().orElse(0); if (scale > 6) { scale = 6; } @@ -76,7 +81,7 @@ public class JdbcSapHanaClient extends JdbcClient { case "CHAR": case "NCHAR": ScalarType charType = ScalarType.createType(PrimitiveType.CHAR); - charType.setLength(fieldSchema.columnSize); + charType.setLength(fieldSchema.getColumnSize().orElse(0)); return charType; case "TIME": case "VARCHAR": diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java index e922254dbc..006bf5a44f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java @@ -21,6 +21,9 @@ import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; + +import java.util.Optional; public class JdbcTrinoClient extends JdbcClient { protected JdbcTrinoClient(JdbcClientConfig jdbcClientConfig) { @@ -29,7 +32,7 @@ public class JdbcTrinoClient extends JdbcClient { @Override protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { - String trinoType = fieldSchema.getDataTypeName(); + String trinoType = fieldSchema.getDataTypeName().orElse("unknown"); switch (trinoType) { case "integer": return Type.INT; @@ -61,12 +64,12 @@ public class JdbcTrinoClient extends JdbcClient { if (trinoType.startsWith("char")) { ScalarType charType = ScalarType.createType(PrimitiveType.CHAR); - charType.setLength(fieldSchema.columnSize); + charType.setLength(fieldSchema.getColumnSize().orElse(0)); return charType; } if (trinoType.startsWith("timestamp")) { - int scale = fieldSchema.getDecimalDigits(); + int scale = fieldSchema.getDecimalDigits().orElse(0); if (scale > 6) { scale = 6; } @@ -75,7 +78,7 @@ public class JdbcTrinoClient extends JdbcClient { if (trinoType.startsWith("array")) { String trinoArrType = trinoType.substring(6, trinoType.length() - 1); - fieldSchema.setDataTypeName(trinoArrType); + fieldSchema.setDataTypeName(Optional.of(trinoArrType)); Type type = jdbcTypeToDoris(fieldSchema); return ArrayType.create(type, true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java new file mode 100644 index 0000000000..735de93e9e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.jdbc.util; + +import lombok.Data; + +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.Map; +import java.util.Optional; + +@Data +public class JdbcFieldSchema { + protected String columnName; + // The SQL type of the corresponding java.sql.types (Type ID) + protected int dataType; + // The SQL type of the corresponding java.sql.types (Type Name) + protected Optional dataTypeName; + // For CHAR/DATA, columnSize means the maximum number of chars. + // For NUMERIC/DECIMAL, columnSize means precision. + protected Optional columnSize; + protected Optional decimalDigits; + // Base number (usually 10 or 2) + protected int numPrecRadix; + // column description + protected String remarks; + // This length is the maximum number of bytes for CHAR type + // for utf8 encoding, if columnSize=10, then charOctetLength=30 + // because for utf8 encoding, a Chinese character takes up 3 bytes + protected int charOctetLength; + protected boolean isAllowNull; + + public JdbcFieldSchema(ResultSet rs) throws SQLException { + this.columnName = rs.getString("COLUMN_NAME"); + this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null")); + this.dataTypeName = Optional.ofNullable(rs.getString("TYPE_NAME")); + this.columnSize = getInteger(rs, "COLUMN_SIZE"); + this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS"); + this.numPrecRadix = rs.getInt("NUM_PREC_RADIX"); + this.isAllowNull = rs.getInt("NULLABLE") != DatabaseMetaData.columnNoNulls; + this.remarks = rs.getString("REMARKS"); + this.charOctetLength = rs.getInt("CHAR_OCTET_LENGTH"); + } + + public JdbcFieldSchema(ResultSet rs, Map dataTypeOverrides) throws SQLException { + this.columnName = rs.getString("COLUMN_NAME"); + this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null")); + this.dataTypeName = Optional.ofNullable(dataTypeOverrides.getOrDefault(columnName, rs.getString("TYPE_NAME"))); + this.columnSize = getInteger(rs, "COLUMN_SIZE"); + this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS"); + this.numPrecRadix = rs.getInt("NUM_PREC_RADIX"); + this.isAllowNull = rs.getInt("NULLABLE") != 0; + this.remarks = rs.getString("REMARKS"); + this.charOctetLength = rs.getInt("CHAR_OCTET_LENGTH"); + } + + public JdbcFieldSchema(ResultSetMetaData metaData, int columnIndex) throws SQLException { + this.columnName = metaData.getColumnName(columnIndex); + this.dataType = metaData.getColumnType(columnIndex); + this.dataTypeName = Optional.ofNullable(metaData.getColumnTypeName(columnIndex)); + this.columnSize = Optional.of(metaData.getPrecision(columnIndex)); + this.decimalDigits = Optional.of(metaData.getScale(columnIndex)); + } + + protected static Optional getInteger(ResultSet resultSet, String columnLabel) + throws SQLException { + int value = resultSet.getInt(columnLabel); + if (resultSet.wasNull()) { + return Optional.empty(); + } + return Optional.of(value); + } +} +