database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Refactor](jdbc catalog) Enhance Field Handling in JdbcFieldSchema Using Optional for Better Null Safety (#34730)

Browse Source 
In some cases, using Resultset.getxxx cannot correctly handle null data, so I use Optional to enhance the fields of JdbcFieldSchema for better type mapping when some results are null.
This commit is contained in:
zy-kkk
2024-05-13 22:37:35 +08:00
committed by GitHub
parent 22d4543346
commit 104284bd6f
12 changed files with 165 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PostgreSQLJdbcHMSCachedClient.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.datasource.DatabaseMetadata;
import org.apache.doris.datasource.TableMetadata;
import org.apache.doris.datasource.hive.event.MetastoreNotificationFetchException;
import org.apache.doris.datasource.jdbc.client.JdbcClientConfig;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import org.apache.doris.thrift.TOdbcTableType;
import com.google.common.base.Joiner;

7
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
View File

@ -20,6 +20,9 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import java.util.Optional;
public class JdbcClickHouseClient extends JdbcClient {
@ -35,7 +38,7 @@ public class JdbcClickHouseClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
String ckType = fieldSchema.getDataTypeName();
String ckType = fieldSchema.getDataTypeName().orElse("unknown");
if (ckType.startsWith("LowCardinality")) {
fieldSchema.setAllowNull(true);
@ -81,7 +84,7 @@ public class JdbcClickHouseClient extends JdbcClient {
if (ckType.startsWith("Array")) {
String cktype = ckType.substring(6, ckType.length() - 1);
fieldSchema.setDataTypeName(cktype);
fieldSchema.setDataTypeName(Optional.of(cktype));
Type type = jdbcTypeToDoris(fieldSchema);
return ArrayType.create(type, true);
}

55
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
View File

@ -24,11 +24,11 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.jdbc.JdbcIdentifierMapping;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.zaxxer.hikari.HikariDataSource;
import lombok.Data;
import lombok.Getter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -255,14 +255,7 @@ public abstract class JdbcClient {
public List<JdbcFieldSchema> getSchemaFromResultSetMetaData(ResultSetMetaData metaData) throws SQLException {
List<JdbcFieldSchema> schemas = Lists.newArrayList();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
JdbcFieldSchema field = new JdbcFieldSchema();
field.setColumnName(metaData.getColumnName(i));
field.setDataType(metaData.getColumnType(i));
field.setDataTypeName(metaData.getColumnTypeName(i));
field.setColumnSize(metaData.getColumnDisplaySize(i));
field.setDecimalDigits(metaData.getScale(i));
field.setNumPrecRadix(metaData.getPrecision(i));
schemas.add(field);
schemas.add(new JdbcFieldSchema(metaData, i));
}
return schemas;
}
@ -347,27 +340,7 @@ public abstract class JdbcClient {
String catalogName = getCatalogName(conn);
rs = getRemoteColumns(databaseMetaData, catalogName, remoteDbName, remoteTableName);
while (rs.next()) {
JdbcFieldSchema field = new JdbcFieldSchema();
field.setColumnName(rs.getString("COLUMN_NAME"));
field.setDataType(rs.getInt("DATA_TYPE"));
field.setDataTypeName(rs.getString("TYPE_NAME"));
/*
We used this method to retrieve the key column of the JDBC table, but since we only tested mysql,
we kept the default key behavior in the parent class and only overwrite it in the mysql subclass
*/
field.setColumnSize(rs.getInt("COLUMN_SIZE"));
field.setDecimalDigits(rs.getInt("DECIMAL_DIGITS"));
field.setNumPrecRadix(rs.getInt("NUM_PREC_RADIX"));
/*
Whether it is allowed to be NULL
0 (columnNoNulls)
1 (columnNullable)
2 (columnNullableUnknown)
*/
field.setAllowNull(rs.getInt("NULLABLE") != 0);
field.setRemarks(rs.getString("REMARKS"));
field.setCharOctetLength(rs.getInt("CHAR_OCTET_LENGTH"));
tableSchema.add(field);
tableSchema.add(new JdbcFieldSchema(rs));
}
} catch (SQLException e) {
throw new JdbcClientException("failed to get jdbc columns info for remote table `%s.%s`: %s",
@ -478,28 +451,6 @@ public abstract class JdbcClient {
return jdbcLowerCaseMetaMatching.setColumnNameMapping(remoteDbName, remoteTableName, remoteColumns);
}
@Data
protected static class JdbcFieldSchema {
protected String columnName;
// The SQL type of the corresponding java.sql.types (Type ID)
protected int dataType;
// The SQL type of the corresponding java.sql.types (Type Name)
protected String dataTypeName;
// For CHAR/DATA, columnSize means the maximum number of chars.
// For NUMERIC/DECIMAL, columnSize means precision.
protected int columnSize;
protected int decimalDigits;
// Base number (usually 10 or 2)
protected int numPrecRadix;
// column description
protected String remarks;
// This length is the maximum number of bytes for CHAR type
// for utf8 encoding, if columnSize=10, then charOctetLength=30
// because for utf8 encoding, a Chinese character takes up 3 bytes
protected int charOctetLength;
protected boolean isAllowNull;
}
protected abstract Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema);
protected Type createDecimalOrStringType(int precision, int scale) {

13
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcDB2Client.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import com.google.common.collect.Lists;
@ -63,7 +64,7 @@ public class JdbcDB2Client extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
String db2Type = fieldSchema.getDataTypeName();
String db2Type = fieldSchema.getDataTypeName().orElse("unknown");
switch (db2Type) {
case "SMALLINT":
return Type.SMALLINT;
@ -73,8 +74,8 @@ public class JdbcDB2Client extends JdbcClient {
return Type.BIGINT;
case "DECFLOAT":
case "DECIMAL": {
int precision = fieldSchema.getColumnSize();
int scale = fieldSchema.getDecimalDigits();
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
return createDecimalOrStringType(precision, scale);
}
case "DOUBLE":
@ -83,18 +84,18 @@ public class JdbcDB2Client extends JdbcClient {
return Type.FLOAT;
case "CHAR":
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.columnSize);
charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
case "VARCHAR":
case "LONG VARCHAR":
ScalarType varcharType = ScalarType.createType(PrimitiveType.VARCHAR);
varcharType.setLength(fieldSchema.columnSize);
varcharType.setLength(fieldSchema.getColumnSize().orElse(0));
return varcharType;
case "DATE":
return ScalarType.createDateV2Type();
case "TIMESTAMP": {
// postgres can support microsecond
int scale = fieldSchema.getDecimalDigits();
int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}

60
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@ -122,40 +123,21 @@ public class JdbcMySQLClient extends JdbcClient {
public List<JdbcFieldSchema> getJdbcColumnsInfo(String localDbName, String localTableName) {
Connection conn = getConnection();
ResultSet rs = null;
List<JdbcFieldSchema> tableSchema = com.google.common.collect.Lists.newArrayList();
List<JdbcFieldSchema> tableSchema = Lists.newArrayList();
String remoteDbName = getRemoteDatabaseName(localDbName);
String remoteTableName = getRemoteTableName(localDbName, localTableName);
try {
DatabaseMetaData databaseMetaData = conn.getMetaData();
String catalogName = getCatalogName(conn);
rs = getRemoteColumns(databaseMetaData, catalogName, remoteDbName, remoteTableName);
Map<String, String> mapFieldtoType = null;
while (rs.next()) {
JdbcFieldSchema field = new JdbcFieldSchema();
field.setColumnName(rs.getString("COLUMN_NAME"));
field.setDataType(rs.getInt("DATA_TYPE"));
// in mysql-jdbc-connector-8.0.*, TYPE_NAME of the HLL column in doris will be "UNKNOWN"
// in mysql-jdbc-connector-5.1.*, TYPE_NAME of the HLL column in doris will be "HLL"
// in mysql-jdbc-connector-8.0.*, TYPE_NAME of BITMAP column in doris will be "BIT"
// in mysql-jdbc-connector-5.1.*, TYPE_NAME of BITMAP column in doris will be "BITMAP"
field.setDataTypeName(rs.getString("TYPE_NAME"));
if (isDoris) {
mapFieldtoType = getColumnsDataTypeUseQuery(remoteDbName, remoteTableName);
field.setDataTypeName(mapFieldtoType.get(rs.getString("COLUMN_NAME")));
}
field.setColumnSize(rs.getInt("COLUMN_SIZE"));
field.setDecimalDigits(rs.getInt("DECIMAL_DIGITS"));
field.setNumPrecRadix(rs.getInt("NUM_PREC_RADIX"));
/*
Whether it is allowed to be NULL
0 (columnNoNulls)
1 (columnNullable)
2 (columnNullableUnknown)
*/
field.setAllowNull(rs.getInt("NULLABLE") != 0);
field.setRemarks(rs.getString("REMARKS"));
field.setCharOctetLength(rs.getInt("CHAR_OCTET_LENGTH"));
Map<String, String> mapFieldtoType = Maps.newHashMap();
if (isDoris) {
mapFieldtoType = getColumnsDataTypeUseQuery(remoteDbName, remoteTableName);
}
while (rs.next()) {
JdbcFieldSchema field = new JdbcFieldSchema(rs, mapFieldtoType);
tableSchema.add(field);
}
} catch (SQLException e) {
@ -184,12 +166,12 @@ public class JdbcMySQLClient extends JdbcClient {
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
// For Doris type
if (isDoris) {
return dorisTypeToDoris(fieldSchema.getDataTypeName().toUpperCase());
return dorisTypeToDoris(fieldSchema.getDataTypeName().orElse("unknown").toUpperCase());
}
// For mysql type: "INT UNSIGNED":
// fieldSchema.getDataTypeName().split(" ")[0] == "INT"
// fieldSchema.getDataTypeName().split(" ")[1] == "UNSIGNED"
String[] typeFields = fieldSchema.getDataTypeName().split(" ");
// fieldSchema.getDataTypeName().orElse("unknown").split(" ")[0] == "INT"
// fieldSchema.getDataTypeName().orElse("unknown").split(" ")[1] == "UNSIGNED"
String[] typeFields = fieldSchema.getDataTypeName().orElse("unknown").split(" ");
String mysqlType = typeFields[0];
// For unsigned int, should extend the type.
if (typeFields.length > 1 && "UNSIGNED".equals(typeFields[1])) {
@ -204,8 +186,8 @@ public class JdbcMySQLClient extends JdbcClient {
case "BIGINT":
return Type.LARGEINT;
case "DECIMAL": {
int precision = fieldSchema.getColumnSize() + 1;
int scale = fieldSchema.getDecimalDigits();
int precision = fieldSchema.getColumnSize().orElse(0) + 1;
int scale = fieldSchema.getDecimalDigits().orElse(0);
return createDecimalOrStringType(precision, scale);
}
case "DOUBLE":
@ -242,7 +224,7 @@ public class JdbcMySQLClient extends JdbcClient {
case "DATETIME": {
// mysql can support microsecond
// use columnSize to calculate the precision of timestamp/datetime
int columnSize = fieldSchema.getColumnSize();
int columnSize = fieldSchema.getColumnSize().orElse(0);
int scale = columnSize > 19 ? columnSize - 20 : 0;
if (scale > 6) {
scale = 6;
@ -257,18 +239,18 @@ public class JdbcMySQLClient extends JdbcClient {
case "DOUBLE":
return Type.DOUBLE;
case "DECIMAL": {
int precision = fieldSchema.getColumnSize();
int scale = fieldSchema.getDecimalDigits();
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
return createDecimalOrStringType(precision, scale);
}
case "CHAR":
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.columnSize);
charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
case "VARCHAR":
return ScalarType.createVarcharType(fieldSchema.columnSize);
return ScalarType.createVarcharType(fieldSchema.getColumnSize().orElse(0));
case "BIT":
if (fieldSchema.getColumnSize() == 1) {
if (fieldSchema.getColumnSize().orElse(0) == 1) {
return Type.BOOLEAN;
} else {
return ScalarType.createStringType();

1
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOceanBaseClient.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.JdbcResource;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import java.sql.Connection;
import java.sql.ResultSet;

31
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@ -69,27 +70,7 @@ public class JdbcOracleClient extends JdbcClient {
if (isModify && isTableModified(rs.getString("TABLE_NAME"), remoteTableName)) {
continue;
}
JdbcFieldSchema field = new JdbcFieldSchema();
field.setColumnName(rs.getString("COLUMN_NAME"));
field.setDataType(rs.getInt("DATA_TYPE"));
field.setDataTypeName(rs.getString("TYPE_NAME"));
/*
We used this method to retrieve the key column of the JDBC table, but since we only tested mysql,
we kept the default key behavior in the parent class and only overwrite it in the mysql subclass
*/
field.setColumnSize(rs.getInt("COLUMN_SIZE"));
field.setDecimalDigits(rs.getInt("DECIMAL_DIGITS"));
field.setNumPrecRadix(rs.getInt("NUM_PREC_RADIX"));
/*
Whether it is allowed to be NULL
0 (columnNoNulls)
1 (columnNullable)
2 (columnNullableUnknown)
*/
field.setAllowNull(rs.getInt("NULLABLE") != 0);
field.setRemarks(rs.getString("REMARKS"));
field.setCharOctetLength(rs.getInt("CHAR_OCTET_LENGTH"));
tableSchema.add(field);
tableSchema.add(new JdbcFieldSchema(rs));
}
} catch (SQLException e) {
throw new JdbcClientException("failed to get table name list from jdbc for table %s:%s", e, remoteTableName,
@ -126,7 +107,7 @@ public class JdbcOracleClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
String oracleType = fieldSchema.getDataTypeName();
String oracleType = fieldSchema.getDataTypeName().orElse("unknown");
if (oracleType.startsWith("INTERVAL")) {
oracleType = oracleType.substring(0, 8);
} else if (oracleType.startsWith("TIMESTAMP")) {
@ -134,7 +115,7 @@ public class JdbcOracleClient extends JdbcClient {
return Type.UNSUPPORTED;
}
// oracle can support nanosecond, will lose precision
int scale = fieldSchema.getDecimalDigits();
int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
@ -160,8 +141,8 @@ public class JdbcOracleClient extends JdbcClient {
* In this case, doris can not determine p and s, so doris can not determine data type.
*/
case "NUMBER":
int precision = fieldSchema.getColumnSize();
int scale = fieldSchema.getDecimalDigits();
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale <= 0) {
precision -= scale;
if (precision < 3) {

13
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
public class JdbcPostgreSQLClient extends JdbcClient {
@ -34,7 +35,7 @@ public class JdbcPostgreSQLClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
String pgType = fieldSchema.getDataTypeName();
String pgType = fieldSchema.getDataTypeName().orElse("unknown");
switch (pgType) {
case "int2":
case "smallserial":
@ -46,8 +47,8 @@ public class JdbcPostgreSQLClient extends JdbcClient {
case "bigserial":
return Type.BIGINT;
case "numeric": {
int precision = fieldSchema.getColumnSize();
int scale = fieldSchema.getDecimalDigits();
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
return createDecimalOrStringType(precision, scale);
}
case "float4":
@ -56,12 +57,12 @@ public class JdbcPostgreSQLClient extends JdbcClient {
return Type.DOUBLE;
case "bpchar":
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.columnSize);
charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
case "timestamp":
case "timestamptz": {
// postgres can support microsecond
int scale = fieldSchema.getDecimalDigits();
int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
@ -72,7 +73,7 @@ public class JdbcPostgreSQLClient extends JdbcClient {
case "bool":
return Type.BOOLEAN;
case "bit":
if (fieldSchema.getColumnSize() == 1) {
if (fieldSchema.getColumnSize().orElse(0) == 1) {
return Type.BOOLEAN;
} else {
return ScalarType.createStringType();

9
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSQLServerClient.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
public class JdbcSQLServerClient extends JdbcClient {
@ -28,7 +29,7 @@ public class JdbcSQLServerClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
String originSqlserverType = fieldSchema.getDataTypeName();
String originSqlserverType = fieldSchema.getDataTypeName().orElse("unknown");
// For sqlserver IDENTITY type, such as 'INT IDENTITY'
// originSqlserverType is "int identity", so we only get "int".
String sqlserverType = originSqlserverType.split(" ")[0];
@ -52,8 +53,8 @@ public class JdbcSQLServerClient extends JdbcClient {
return ScalarType.createDecimalV3Type(10, 4);
case "decimal":
case "numeric": {
int precision = fieldSchema.getColumnSize();
int scale = fieldSchema.getDecimalDigits();
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
return ScalarType.createDecimalV3Type(precision, scale);
}
case "date":
@ -62,7 +63,7 @@ public class JdbcSQLServerClient extends JdbcClient {
case "datetime2":
case "smalldatetime": {
// postgres can support microsecond
int scale = fieldSchema.getDecimalDigits();
int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}

17
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcSapHanaClient.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.datasource.jdbc.client;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
public class JdbcSapHanaClient extends JdbcClient {
protected JdbcSapHanaClient(JdbcClientConfig jdbcClientConfig) {
@ -38,7 +39,7 @@ public class JdbcSapHanaClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
String hanaType = fieldSchema.getDataTypeName();
String hanaType = fieldSchema.getDataTypeName().orElse("unknown");
switch (hanaType) {
case "TINYINT":
return Type.TINYINT;
@ -50,9 +51,13 @@ public class JdbcSapHanaClient extends JdbcClient {
return Type.BIGINT;
case "SMALLDECIMAL":
case "DECIMAL": {
int precision = fieldSchema.getColumnSize();
int scale = fieldSchema.getDecimalDigits();
return createDecimalOrStringType(precision, scale);
if (!fieldSchema.getDecimalDigits().isPresent()) {
return Type.DOUBLE;
} else {
int precision = fieldSchema.getColumnSize().orElse(0);
int scale = fieldSchema.getDecimalDigits().orElse(0);
return createDecimalOrStringType(precision, scale);
}
}
case "REAL":
return Type.FLOAT;
@ -60,7 +65,7 @@ public class JdbcSapHanaClient extends JdbcClient {
return Type.DOUBLE;
case "TIMESTAMP": {
// postgres can support microsecond
int scale = fieldSchema.getDecimalDigits();
int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
@ -76,7 +81,7 @@ public class JdbcSapHanaClient extends JdbcClient {
case "CHAR":
case "NCHAR":
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.columnSize);
charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
case "TIME":
case "VARCHAR":

11
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcTrinoClient.java
View File

@ -21,6 +21,9 @@ import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
import java.util.Optional;
public class JdbcTrinoClient extends JdbcClient {
protected JdbcTrinoClient(JdbcClientConfig jdbcClientConfig) {
@ -29,7 +32,7 @@ public class JdbcTrinoClient extends JdbcClient {
@Override
protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
String trinoType = fieldSchema.getDataTypeName();
String trinoType = fieldSchema.getDataTypeName().orElse("unknown");
switch (trinoType) {
case "integer":
return Type.INT;
@ -61,12 +64,12 @@ public class JdbcTrinoClient extends JdbcClient {
if (trinoType.startsWith("char")) {
ScalarType charType = ScalarType.createType(PrimitiveType.CHAR);
charType.setLength(fieldSchema.columnSize);
charType.setLength(fieldSchema.getColumnSize().orElse(0));
return charType;
}
if (trinoType.startsWith("timestamp")) {
int scale = fieldSchema.getDecimalDigits();
int scale = fieldSchema.getDecimalDigits().orElse(0);
if (scale > 6) {
scale = 6;
}
@ -75,7 +78,7 @@ public class JdbcTrinoClient extends JdbcClient {
if (trinoType.startsWith("array")) {
String trinoArrType = trinoType.substring(6, trinoType.length() - 1);
fieldSchema.setDataTypeName(trinoArrType);
fieldSchema.setDataTypeName(Optional.of(trinoArrType));
Type type = jdbcTypeToDoris(fieldSchema);
return ArrayType.create(type, true);
}

91
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java Normal file
View File

@ -0,0 +1,91 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.datasource.jdbc.util;
import lombok.Data;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.Map;
import java.util.Optional;
@Data
public class JdbcFieldSchema {
protected String columnName;
// The SQL type of the corresponding java.sql.types (Type ID)
protected int dataType;
// The SQL type of the corresponding java.sql.types (Type Name)
protected Optional<String> dataTypeName;
// For CHAR/DATA, columnSize means the maximum number of chars.
// For NUMERIC/DECIMAL, columnSize means precision.
protected Optional<Integer> columnSize;
protected Optional<Integer> decimalDigits;
// Base number (usually 10 or 2)
protected int numPrecRadix;
// column description
protected String remarks;
// This length is the maximum number of bytes for CHAR type
// for utf8 encoding, if columnSize=10, then charOctetLength=30
// because for utf8 encoding, a Chinese character takes up 3 bytes
protected int charOctetLength;
protected boolean isAllowNull;
public JdbcFieldSchema(ResultSet rs) throws SQLException {
this.columnName = rs.getString("COLUMN_NAME");
this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null"));
this.dataTypeName = Optional.ofNullable(rs.getString("TYPE_NAME"));
this.columnSize = getInteger(rs, "COLUMN_SIZE");
this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS");
this.numPrecRadix = rs.getInt("NUM_PREC_RADIX");
this.isAllowNull = rs.getInt("NULLABLE") != DatabaseMetaData.columnNoNulls;
this.remarks = rs.getString("REMARKS");
this.charOctetLength = rs.getInt("CHAR_OCTET_LENGTH");
}
public JdbcFieldSchema(ResultSet rs, Map<String, String> dataTypeOverrides) throws SQLException {
this.columnName = rs.getString("COLUMN_NAME");
this.dataType = getInteger(rs, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null"));
this.dataTypeName = Optional.ofNullable(dataTypeOverrides.getOrDefault(columnName, rs.getString("TYPE_NAME")));
this.columnSize = getInteger(rs, "COLUMN_SIZE");
this.decimalDigits = getInteger(rs, "DECIMAL_DIGITS");
this.numPrecRadix = rs.getInt("NUM_PREC_RADIX");
this.isAllowNull = rs.getInt("NULLABLE") != 0;
this.remarks = rs.getString("REMARKS");
this.charOctetLength = rs.getInt("CHAR_OCTET_LENGTH");
}
public JdbcFieldSchema(ResultSetMetaData metaData, int columnIndex) throws SQLException {
this.columnName = metaData.getColumnName(columnIndex);
this.dataType = metaData.getColumnType(columnIndex);
this.dataTypeName = Optional.ofNullable(metaData.getColumnTypeName(columnIndex));
this.columnSize = Optional.of(metaData.getPrecision(columnIndex));
this.decimalDigits = Optional.of(metaData.getScale(columnIndex));
}
protected static Optional<Integer> getInteger(ResultSet resultSet, String columnLabel)
throws SQLException {
int value = resultSet.getInt(columnLabel);
if (resultSet.wasNull()) {
return Optional.empty();
}
return Optional.of(value);
}
}