[fix](multi-catalog) verify the precision of datetime types for each data source (#19544)

Fix threes bugs of timestampv2 precision:
1. Hive catalog doesn't set the precision of timestampv2, and can't get the precision from hive metastore, so set the largest precision for timestampv2;
2. Jdbc catalog use datetimev1 to parse timestamp, and convert to timestampv2, so the precision is lost.
3. TVF doesn't use the precision from meta data of file format.
This commit is contained in:
Ashin Gau
2023-05-17 20:50:15 +08:00
committed by GitHub
parent 73be97f8d8
commit 30c4f25cb3
20 changed files with 2309 additions and 2334 deletions

View File

@ -173,14 +173,13 @@ inline bool TextConverter::write_vec_column(const SlotDescriptor* slot_desc,
break;
}
case TYPE_DATETIMEV2: {
vectorized::VecDateTimeValue ts_slot;
if (!ts_slot.from_date_str(data, len)) {
vectorized::DateV2Value<vectorized::DateTimeV2ValueType> ts_slot;
if (!ts_slot.from_date_str(data, len, slot_desc->type().scale)) {
parse_result = StringParser::PARSE_FAILURE;
insert_after_parse_failure = false;
break;
}
ts_slot.to_datetime();
uint64_t num = ts_slot.to_datetime_v2();
uint64_t num = ts_slot.to_date_int_val();
reinterpret_cast<vectorized::ColumnVector<vectorized::UInt64>*>(col_ptr)->insert_value(num);
break;
}

View File

@ -240,6 +240,18 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::LogicalType logi
type = TypeDescriptor(TYPE_TIMEV2);
} else if (logicalType.__isset.TIMESTAMP) {
type = TypeDescriptor(TYPE_DATETIMEV2);
const auto& time_unit = logicalType.TIMESTAMP.unit;
if (time_unit.__isset.MILLIS) {
type.scale = 3;
} else if (time_unit.__isset.MICROS) {
type.scale = 6;
} else if (time_unit.__isset.NANOS) {
// will lose precision
type.scale = 6;
} else {
// default precision
type.scale = 6;
}
} else {
type = TypeDescriptor(INVALID_TYPE);
}
@ -266,9 +278,12 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(
type = TypeDescriptor(TYPE_TIMEV2);
break;
case tparquet::ConvertedType::type::TIMESTAMP_MILLIS:
[[fallthrough]];
type = TypeDescriptor(TYPE_DATETIMEV2);
type.scale = 3;
break;
case tparquet::ConvertedType::type::TIMESTAMP_MICROS:
type = TypeDescriptor(TYPE_DATETIMEV2);
type.scale = 6;
break;
case tparquet::ConvertedType::type::INT_8:
type = TypeDescriptor(TYPE_TINYINT);

View File

@ -140,6 +140,8 @@ public class ScalarType extends Type {
return createDecimalV3Type(precision, scale);
case DECIMALV2:
return createDecimalType(precision, scale);
case DATETIMEV2:
return createDatetimeV2Type(scale);
default:
return createType(type);
}

View File

@ -700,7 +700,8 @@ public class HiveMetaStoreClientHelper {
* Convert hive type to doris type.
*/
public static Type hiveTypeToDorisType(String hiveType) {
return hiveTypeToDorisType(hiveType, 0);
// use the largest scale as default time scale.
return hiveTypeToDorisType(hiveType, 6);
}
/**

View File

@ -36,7 +36,9 @@ import java.util.List;
public class IcebergExternalTable extends ExternalTable {
public static final int ICEBERG_DATETIME_SCALE_MS = 3;
// https://iceberg.apache.org/spec/#schemas-and-data-types
// All time and timestamp values are stored with microsecond precision
public static final int ICEBERG_DATETIME_SCALE_MS = 6;
public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) {
super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE);

View File

@ -51,6 +51,8 @@ public class JdbcClient {
private static final int HTTP_TIMEOUT_MS = 10000;
public static final int JDBC_DATETIME_SCALE = 6;
private String dbType;
private String jdbcUser;
@ -530,7 +532,9 @@ public class JdbcClient {
case "TIMESTAMP":
case "DATETIME":
case "DATETIMEV2": // for jdbc catalog connecting Doris database
return ScalarType.createDatetimeV2Type(0);
// mysql can support microsecond
// todo(gaoxin): Get real precision of DATETIMEV2
return ScalarType.createDatetimeV2Type(JDBC_DATETIME_SCALE);
case "FLOAT":
return Type.FLOAT;
case "DOUBLE":
@ -598,7 +602,8 @@ public class JdbcClient {
return charType;
case "timestamp":
case "timestamptz":
return ScalarType.createDatetimeV2Type(0);
// postgres can support microsecond
return ScalarType.createDatetimeV2Type(JDBC_DATETIME_SCALE);
case "date":
return ScalarType.createDateV2Type();
case "bool":
@ -649,7 +654,13 @@ public class JdbcClient {
|| ckType.startsWith("FixedString")) {
return ScalarType.createStringType();
} else if (ckType.startsWith("DateTime")) {
return ScalarType.createDatetimeV2Type(6);
// DateTime with second precision, DateTime64 with [0~9] precision
if (ckType.equals("DateTime")) {
return ScalarType.createDatetimeV2Type(0);
} else {
// will lose precision
return ScalarType.createDatetimeV2Type(JDBC_DATETIME_SCALE);
}
} else if (ckType.startsWith("Array")) {
String cktype = ckType.substring(6, ckType.length() - 1);
fieldSchema.setDataTypeName(cktype);
@ -697,7 +708,8 @@ public class JdbcClient {
if (oracleType.equals("TIMESTAMPTZ") || oracleType.equals("TIMESTAMPLTZ")) {
return Type.UNSUPPORTED;
}
return ScalarType.createDatetimeV2Type(0);
// oracle can support nanosecond, will lose precision
return ScalarType.createDatetimeV2Type(JDBC_DATETIME_SCALE);
}
switch (oracleType) {
/**
@ -746,6 +758,7 @@ public class JdbcClient {
case "FLOAT":
return Type.DOUBLE;
case "DATE":
// can save date and time with second precision
return ScalarType.createDatetimeV2Type(0);
case "VARCHAR2":
case "NVARCHAR2":
@ -796,9 +809,14 @@ public class JdbcClient {
case "date":
return ScalarType.createDateV2Type();
case "datetime":
// datetime with millisecond precision
return ScalarType.createDatetimeV2Type(3);
case "datetime2":
case "smalldatetime":
// datetime2 with 100 nanoseconds precision, will lose precision
return ScalarType.createDatetimeV2Type(6);
case "smalldatetime":
// smalldatetime with second precision
return ScalarType.createDatetimeV2Type(0);
case "char":
case "varchar":
case "nchar":
@ -838,8 +856,11 @@ public class JdbcClient {
case "DOUBLE":
return Type.DOUBLE;
case "TIMESTAMP":
case "SECONDDATE":
// TIMESTAMP with 100 nanoseconds precision, will lose precision
return ScalarType.createDatetimeV2Type(6);
case "SECONDDATE":
// SECONDDATE with second precision
return ScalarType.createDatetimeV2Type(0);
case "DATE":
return ScalarType.createDateV2Type();
case "BOOLEAN":
@ -882,7 +903,8 @@ public class JdbcClient {
charType.setLength(fieldSchema.columnSize);
return charType;
} else if (trinoType.startsWith("timestamp")) {
return ScalarType.createDatetimeV2Type(6);
// timestamp with picoseconds precision, will lose precision
return ScalarType.createDatetimeV2Type(JDBC_DATETIME_SCALE);
} else if (trinoType.startsWith("array")) {
String trinoArrType = trinoType.substring(6, trinoType.length() - 1);
fieldSchema.setDataTypeName(trinoArrType);

View File

@ -117,8 +117,8 @@ public class MetadataGenerator {
LocalDateTime committedAt = LocalDateTime.ofInstant(Instant.ofEpochMilli(
snapshot.timestampMillis()), TimeUtils.getTimeZone().toZoneId());
long encodedDatetime = convertToDateTimeV2(committedAt.getYear(), committedAt.getMonthValue(),
committedAt.getDayOfMonth(), committedAt.getHour(),
committedAt.getMinute(), committedAt.getSecond());
committedAt.getDayOfMonth(), committedAt.getHour(), committedAt.getMinute(),
committedAt.getSecond(), committedAt.getNano() / 1000);
trow.addToColumnValue(new TCell().setLongVal(encodedDatetime));
trow.addToColumnValue(new TCell().setLongVal(snapshot.snapshotId()));
@ -303,8 +303,9 @@ public class MetadataGenerator {
return hiveCatalog.loadTable(TableIdentifier.of(db, tbl));
}
private static long convertToDateTimeV2(int year, int month, int day, int hour, int minute, int second) {
return (long) second << 20 | (long) minute << 26 | (long) hour << 32
private static long convertToDateTimeV2(
int year, int month, int day, int hour, int minute, int second, int microsecond) {
return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32
| (long) day << 37 | (long) month << 42 | (long) year << 46;
}
}

View File

@ -62,11 +62,16 @@ public class TypeNativeBytes {
}
public static long convertToDateTimeV2(int year, int month, int day, int hour, int minute, int second) {
// todo: Has lost precision ? How about millisecond, microsecond ...
return (long) second << 20 | (long) minute << 26 | (long) hour << 32
| (long) day << 37 | (long) month << 42 | (long) year << 46;
}
public static long convertToDateTimeV2(int year, int month, int day, int hour, int minute, int second,
int microsecond) {
return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32
| (long) day << 37 | (long) month << 42 | (long) year << 46;
}
public static LocalDate convertToJavaDate(int date) {
int year = date >> 9;
int month = (date >> 5) & 0XF;
@ -91,10 +96,11 @@ public class TypeNativeBytes {
int hour = (int) ((time >> 32) & 0X1F);
int minute = (int) ((time >> 26) & 0X3F);
int second = (int) ((time >> 20) & 0X3F);
int microsecond = (int) (time & 0XFFFFF);
LocalDateTime value;
try {
value = LocalDateTime.of(year, month, day, hour, minute, second);
value = LocalDateTime.of(year, month, day, hour, minute, second, microsecond * 1000);
} catch (DateTimeException e) {
value = LocalDateTime.MAX;
}

View File

@ -447,7 +447,7 @@ public class VectorColumn {
private void putDateTime(int rowId, LocalDateTime v) {
long time = TypeNativeBytes.convertToDateTimeV2(v.getYear(), v.getMonthValue(), v.getDayOfMonth(), v.getHour(),
v.getMinute(), v.getSecond());
v.getMinute(), v.getSecond(), v.getNano() / 1000);
OffHeap.putLong(null, data + rowId * 8L, time);
}

View File

@ -1086,7 +1086,7 @@ public class JdbcExecutor {
UdfUtils.UNSAFE.putLong(columnAddr + (i * 8L),
UdfUtils.convertToDateTimeV2(date.getYear(), date.getMonthValue(),
date.getDayOfMonth(), date.getHour(), date.getMinute(),
date.getSecond()));
date.getSecond(), date.getNano() / 1000));
}
}
} else {
@ -1095,7 +1095,7 @@ public class JdbcExecutor {
UdfUtils.UNSAFE.putLong(columnAddr + (i * 8L),
UdfUtils.convertToDateTimeV2(date.getYear(), date.getMonthValue(),
date.getDayOfMonth(), date.getHour(), date.getMinute(),
date.getSecond()));
date.getSecond(), date.getNano() / 1000));
}
}
}
@ -1110,7 +1110,8 @@ public class JdbcExecutor {
LocalDateTime date = ((java.sql.Timestamp) column[i]).toLocalDateTime();
UdfUtils.UNSAFE.putLong(columnAddr + (i * 8L),
UdfUtils.convertToDateTimeV2(date.getYear(), date.getMonthValue(),
date.getDayOfMonth(), date.getHour(), date.getMinute(), date.getSecond()));
date.getDayOfMonth(), date.getHour(), date.getMinute(),
date.getSecond(), date.getNano() / 1000));
}
}
} else {
@ -1118,7 +1119,7 @@ public class JdbcExecutor {
LocalDateTime date = ((java.sql.Timestamp) column[i]).toLocalDateTime();
UdfUtils.UNSAFE.putLong(columnAddr + (i * 8L),
UdfUtils.convertToDateTimeV2(date.getYear(), date.getMonthValue(), date.getDayOfMonth(),
date.getHour(), date.getMinute(), date.getSecond()));
date.getHour(), date.getMinute(), date.getSecond(), date.getNano() / 1000));
}
}
}
@ -1133,7 +1134,8 @@ public class JdbcExecutor {
LocalDateTime date = ((oracle.sql.TIMESTAMP) column[i]).timestampValue().toLocalDateTime();
UdfUtils.UNSAFE.putLong(columnAddr + (i * 8L),
UdfUtils.convertToDateTimeV2(date.getYear(), date.getMonthValue(),
date.getDayOfMonth(), date.getHour(), date.getMinute(), date.getSecond()));
date.getDayOfMonth(), date.getHour(), date.getMinute(),
date.getSecond(), date.getNano() / 1000));
}
}
} else {
@ -1141,7 +1143,7 @@ public class JdbcExecutor {
LocalDateTime date = ((oracle.sql.TIMESTAMP) column[i]).timestampValue().toLocalDateTime();
UdfUtils.UNSAFE.putLong(columnAddr + (i * 8L),
UdfUtils.convertToDateTimeV2(date.getYear(), date.getMonthValue(), date.getDayOfMonth(),
date.getHour(), date.getMinute(), date.getSecond()));
date.getHour(), date.getMinute(), date.getSecond(), date.getNano() / 1000));
}
}
}
@ -1193,7 +1195,7 @@ public class JdbcExecutor {
int[] offsets = new int[numRows];
byte[][] byteRes = new byte[numRows][];
int offset = 0;
if (isNullable == true) {
if (isNullable) {
// Here can not loop from startRowForNullable,
// because byteRes will be used later
for (int i = 0; i < numRows; i++) {
@ -1201,14 +1203,36 @@ public class JdbcExecutor {
byteRes[i] = emptyBytes;
UdfUtils.UNSAFE.putByte(nullMapAddr + i, (byte) 1);
} else {
byteRes[i] = column[i].toString().getBytes(StandardCharsets.UTF_8);
String result = column[i].toString();
if (column[i] instanceof java.sql.Time) {
// the default toString() method doesn't format the milliseconds in Time.
long milliseconds = ((java.sql.Time) column[i]).getTime() % 1000L;
if (milliseconds > 0) {
result = String.format("%s.%03d", column[i].toString(), milliseconds);
}
}
byteRes[i] = result.getBytes(StandardCharsets.UTF_8);
}
offset += byteRes[i].length;
offsets[i] = offset;
}
} else {
boolean isTime = numRows > 0 && column[0] instanceof java.sql.Time;
for (int i = 0; i < numRows; i++) {
byteRes[i] = column[i].toString().getBytes(StandardCharsets.UTF_8);
String result = column[i].toString();
if (isTime) {
// Doc https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-type-conversions.html
// shows that jdbc API use java.sql.Time to hold the TIME type,
// but java.sql.Time can only have millisecond precision.
// the default toString() method doesn't format the milliseconds in Time.
// Doc https://dev.mysql.com/doc/refman/8.0/en/time.html shows that MySQL supports time[0~6],
// so time[4~6] will lose precision
long milliseconds = ((java.sql.Time) column[i]).getTime() % 1000L;
if (milliseconds > 0) {
result = String.format("%s.%03d", column[i].toString(), milliseconds);
}
}
byteRes[i] = result.getBytes(StandardCharsets.UTF_8);
offset += byteRes[i].length;
offsets[i] = offset;
}
@ -1230,7 +1254,7 @@ public class JdbcExecutor {
int[] offsets = new int[numRows];
byte[][] byteRes = new byte[numRows][];
int offset = 0;
if (isNullable == true) {
if (isNullable) {
for (int i = 0; i < numRows; i++) {
if (column[i] == null) {
byteRes[i] = emptyBytes;
@ -1388,7 +1412,7 @@ public class JdbcExecutor {
private void copyBatchDecimalResult(BigInteger[] column, boolean isNullable, int numRows,
long columnAddr, int typeLen, int startRowForNullable) {
if (isNullable == true) {
if (isNullable) {
for (int i = startRowForNullable; i < numRows; i++) {
if (column[i] != null) {
byte[] bytes = UdfUtils.convertByteOrder(column[i].toByteArray());
@ -1424,7 +1448,7 @@ public class JdbcExecutor {
int[] offsets = new int[numRows];
byte[][] byteRes = new byte[numRows][];
int offset = 0;
if (isNullable == true) {
if (isNullable) {
// Here can not loop from startRowForNullable,
// because byteRes will be used later
for (int i = 0; i < numRows; i++) {
@ -1461,7 +1485,7 @@ public class JdbcExecutor {
int[] offsets = new int[numRows];
byte[][] byteRes = new byte[numRows][];
int offset = 0;
if (isNullable == true) {
if (isNullable) {
// Here can not loop from startRowForNullable,
// because byteRes will be used later
for (int i = 0; i < numRows; i++) {

View File

@ -538,6 +538,12 @@ public class UdfUtils {
| (long) day << 37 | (long) month << 42 | (long) year << 46;
}
public static long convertToDateTimeV2(
int year, int month, int day, int hour, int minute, int second, int microsecond) {
return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32
| (long) day << 37 | (long) month << 42 | (long) year << 46;
}
public static long convertToDateTimeV2(Object obj, Class clz) {
if (LocalDateTime.class.equals(clz)) {
LocalDateTime date = (LocalDateTime) obj;
@ -546,11 +552,11 @@ public class UdfUtils {
} else if (org.joda.time.DateTime.class.equals(clz)) {
org.joda.time.DateTime date = (org.joda.time.DateTime) obj;
return convertToDateTimeV2(date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(),
date.getMinuteOfHour(), date.getSecondOfMinute());
date.getMinuteOfHour(), date.getSecondOfMinute(), date.getMillisOfSecond() * 1000);
} else if (org.joda.time.LocalDateTime.class.equals(clz)) {
org.joda.time.LocalDateTime date = (org.joda.time.LocalDateTime) obj;
return convertToDateTimeV2(date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(),
date.getMinuteOfHour(), date.getSecondOfMinute());
date.getMinuteOfHour(), date.getSecondOfMinute(), date.getMillisOfSecond() * 1000);
} else {
return 0;
}

View File

@ -1,31 +1,31 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql11 --
jdbc1_catalog CREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"password" = "*XXX",\n"driver_class" = "com.mysql.cj.jdbc.Driver",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"type" = "jdbc",\n"user" = "root",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:3336/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=false"\n);
jdbc1_catalog \nCREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"user" = "root",\n"type" = "jdbc",\n"password" = "*XXX",\n"only_specified_database" = "false",\n"oceanbase_mode" = "",\n"lower_case_table_names" = "false",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:33167/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=true&transformedBitIsBoolean=true&useUnicode=true&characterEncoding=utf-8",\n"include_database_list" = "",\n"exclude_database_list" = "",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"driver_class" = "com.mysql.cj.jdbc.Driver",\n"create_time" = "2023-05-15 22:02:06.468",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a"\n);
-- !sql12 --
jdbc1_catalog CREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"password" = "*XXX",\n"driver_class" = "com.mysql.cj.jdbc.Driver",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"type" = "jdbc",\n"user" = "root2",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:3336/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=false"\n);
jdbc1_catalog \nCREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"user" = "root2",\n"type" = "jdbc",\n"password" = "*XXX",\n"only_specified_database" = "false",\n"oceanbase_mode" = "",\n"lower_case_table_names" = "false",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:33167/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=true&transformedBitIsBoolean=true&useUnicode=true&characterEncoding=utf-8",\n"include_database_list" = "",\n"exclude_database_list" = "",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"driver_class" = "com.mysql.cj.jdbc.Driver",\n"create_time" = "2023-05-15 22:02:06.468",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a"\n);
-- !sql13 --
jdbc1_catalog CREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"password" = "*XXX",\n"driver_class" = "com.mysql.cj.jdbc.Driver",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"type" = "jdbc",\n"user" = "root2",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:3336/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=false"\n);
jdbc1_catalog \nCREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"user" = "root2",\n"type" = "jdbc",\n"password" = "*XXX",\n"only_specified_database" = "false",\n"oceanbase_mode" = "",\n"lower_case_table_names" = "false",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:33167/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=true&transformedBitIsBoolean=true&useUnicode=true&characterEncoding=utf-8",\n"include_database_list" = "",\n"exclude_database_list" = "",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"driver_class" = "com.mysql.cj.jdbc.Driver",\n"create_time" = "2023-05-15 22:02:06.468",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a"\n);
-- !sql14 --
jdbc1_catalog CREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"password" = "*XXX",\n"driver_class" = "com.mysql.jdbc.Driver",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"type" = "jdbc",\n"user" = "root2",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:3336/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=false"\n);
jdbc1_catalog \nCREATE CATALOG `jdbc1_catalog` PROPERTIES (\n"user" = "root2",\n"type" = "jdbc",\n"password" = "*XXX",\n"only_specified_database" = "false",\n"oceanbase_mode" = "",\n"lower_case_table_names" = "false",\n"jdbc_url" = "jdbc:mysql://127.0.0.1:33167/doris_test?useSSL=false&yearIsDateType=false&tinyInt1isBit=true&transformedBitIsBoolean=true&useUnicode=true&characterEncoding=utf-8",\n"include_database_list" = "",\n"exclude_database_list" = "",\n"driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/mysql-connector-java-8.0.25.jar",\n"driver_class" = "com.mysql.jdbc.Driver",\n"create_time" = "2023-05-15 22:02:06.468",\n"checksum" = "fdf55dcef04b09f2eaf42b75e61ccc9a"\n);
-- !sql21 --
hive1_catalog CREATE CATALOG `hive1_catalog` PROPERTIES (\n"type" = "hms",\n"hive.metastore.uris" = "thrift://127.0.0.1:9184"\n);
hive1_catalog \nCREATE CATALOG `hive1_catalog` PROPERTIES (\n"type" = "hms",\n"hive.metastore.uris" = "thrift://127.0.0.1:91837",\n"create_time" = "2023-05-15 22:02:06.511"\n);
-- !sql22 --
hive1_catalog CREATE CATALOG `hive1_catalog` PROPERTIES (\n"type" = "hms",\n"new_config" = "value1",\n"hive.metastore.uris" = "thrift://127.0.0.1:9184"\n);
hive1_catalog \nCREATE CATALOG `hive1_catalog` PROPERTIES (\n"type" = "hms",\n"new_config" = "value1",\n"hive.metastore.uris" = "thrift://127.0.0.1:91837",\n"create_time" = "2023-05-15 22:02:06.511"\n);
-- !sql23 --
hive1_catalog CREATE CATALOG `hive1_catalog` PROPERTIES (\n"type" = "hms",\n"new_config" = "value1",\n"hive.metastore.uris" = "thrift://127.0.0.2:9184"\n);
hive1_catalog \nCREATE CATALOG `hive1_catalog` PROPERTIES (\n"type" = "hms",\n"new_config" = "value1",\n"hive.metastore.uris" = "thrift://127.0.0.2:91837",\n"create_time" = "2023-05-15 22:02:06.511"\n);
-- !sql31 --
es1_catalog CREATE CATALOG `es1_catalog` PROPERTIES (\n"nodes_discovery" = "true",\n"enable_docvalue_scan" = "true",\n"http_ssl_enabled" = "false",\n"type" = "es",\n"enable_keyword_sniff" = "true",\n"hosts" = "http://127.0.0.1:29222"\n);
es1_catalog \nCREATE CATALOG `es1_catalog` PROPERTIES (\n"type" = "es",\n"nodes_discovery" = "true",\n"hosts" = "http://127.0.0.1:29207",\n"enable_keyword_sniff" = "true",\n"create_time" = "2023-05-15 22:02:06.522"\n);
-- !sql22 --
es1_catalog CREATE CATALOG `es1_catalog` PROPERTIES (\n"nodes_discovery" = "true",\n"enable_docvalue_scan" = "true",\n"http_ssl_enabled" = "false",\n"type" = "es",\n"enable_keyword_sniff" = "false",\n"hosts" = "http://127.0.0.1:29222"\n);
es1_catalog \nCREATE CATALOG `es1_catalog` PROPERTIES (\n"type" = "es",\n"nodes_discovery" = "true",\n"hosts" = "http://127.0.0.1:29207",\n"enable_keyword_sniff" = "false",\n"create_time" = "2023-05-15 22:02:06.522"\n);
-- !sql23 --
es1_catalog CREATE CATALOG `es1_catalog` PROPERTIES (\n"nodes_discovery" = "true",\n"enable_docvalue_scan" = "true",\n"http_ssl_enabled" = "false",\n"type" = "es",\n"enable_keyword_sniff" = "false",\n"hosts" = "http://127.0.0.2:29222"\n);
es1_catalog \nCREATE CATALOG `es1_catalog` PROPERTIES (\n"type" = "es",\n"nodes_discovery" = "true",\n"hosts" = "http://127.0.0.2:29207",\n"enable_keyword_sniff" = "false",\n"create_time" = "2023-05-15 22:02:06.522"\n);

View File

@ -765,8 +765,8 @@ cslab9e 2017-01-10T14:19:45
2012-12-23T23:28:02 229.50.247.232 //cgi-bin/feedback/ 500 410
-- !14 --
2019-12-01T09:30:13 16C448031108ABA balcony_5 door 5 temperature F 32.0
2019-12-01T09:59:58 16C448031108ABA balcony_5 door 5 temperature F 32.0
2019-12-01T09:30:13.417 16C448031108ABA balcony_5 door 5 temperature F 32.0
2019-12-01T09:59:58.874 16C448031108ABA balcony_5 door 5 temperature F 32.0
-- !15 --
balcony_3 door 3 SUMMER
@ -1547,8 +1547,8 @@ cslab9e 2017-01-10T14:19:45
2012-12-23T23:28:02 229.50.247.232 //cgi-bin/feedback/ 500 410
-- !14 --
2019-12-01T09:30:13 16C448031108ABA balcony_5 door 5 temperature F 32.0
2019-12-01T09:59:58 16C448031108ABA balcony_5 door 5 temperature F 32.0
2019-12-01T09:30:13.417 16C448031108ABA balcony_5 door 5 temperature F 32.0
2019-12-01T09:59:58.874 16C448031108ABA balcony_5 door 5 temperature F 32.0
-- !15 --
balcony_3 door 3 SUMMER
@ -2296,31 +2296,31 @@ cslab9e 2017-01-10T14:19:45
-- !12 --
2017 9 721638.3414064784 1252.8443427195807 65268.739191884546 113.31378331924401 174895.77151746236 330.61582517478706 176421.94104524454 388.5945838000981
2017 10 702508.0664338227 1256.7228379853716 66960.70685831293 120.00126677117012 131314.92843738297 270.7524297677999 212072.4254061825 379.3782207624016
2017 11 903660.5956517109 1258.5802167851127 79376.14433888883 110.55173306251926 178488.0490120137 304.0682265962755 257457.22712476418 358.5755252434042
2017 12 931357.6830229157 1251.8248427727362 76585.03687197331 102.93687751609316 112836.92815090143 198.6565636459532 241257.50714199216 324.2708429327852
2017 10 702508.0664338225 1256.7228379853711 66960.70685831291 120.0012667711701 131314.92843738303 270.75242976780004 212072.42540618256 379.3782207624017
2017 11 903660.5956517105 1258.580216785112 79376.14433888887 110.55173306251932 178488.04901201368 304.06822659627545 257457.22712476412 358.57552524340406
2017 12 931357.6830229161 1251.8248427727367 76585.03687197334 102.93687751609319 112836.92815090144 198.65656364595324 241257.5071419922 324.2708429327852
2018 1 800695.0232504869 1268.9303062606766 52913.861798515994 83.85715023536608 36392.48058457171 83.46899216644888 191628.70119383492 303.69049317564964
2018 2 788872.057316979 1188.060327284607 85621.32358955599 128.94777649029515 83922.79400835953 176.6795663333885 230227.25133465935 346.7277881546074
2018 3 896816.2190436664 1205.3981438758956 88325.94803945151 118.71767209603698 114322.79839365493 193.76745490449989 269070.6060912007 361.65404044516225
2018 4 581954.6102904766 1199.9064129700548 55146.138460788185 113.70337826966636 107909.63419773089 260.6512903326833 172977.56835847016 356.654780120557
2018 5 810269.3225697605 1246.5681885688623 69687.03583794385 107.21082436606747 83644.78159173668 143.4730387508348 248284.77976738242 381.97658425751143
2018 6 824150.0588949089 1252.507688290135 52047.118737002835 79.09896464590096 61491.79513703265 113.24455826341188 242561.98528653936 368.63523599778017
2018 7 920527.3411698382 1237.2679316798901 54001.80526436992 72.58307159189505 110136.29401019629 175.65597130812804 284977.68054289324 383.0345168587275
2018 8 934028.5503494727 1255.414718211657 56534.79015953164 75.9876211821662 18440.748190359904 37.25403674820183 281409.03078009794 378.23794459690583
2018 9 796294.7250120323 1201.0478506968814 59750.627903558845 90.12161071426674 123357.51466599082 219.10748608524125 251368.5907741443 379.1381459640186
2018 10 876930.3483784578 1178.669823089325 68192.5478930525 91.65665039388777 161682.95582566143 259.94044344961645 285017.38156064047 383.08787844172105
2018 3 896816.2190436661 1205.3981438758954 88325.94803945151 118.71767209603698 114322.79839365494 193.7674549044999 269070.6060912008 361.65404044516237
2018 4 581954.6102904766 1199.9064129700548 55146.13846078815 113.70337826966629 107909.63419773085 260.65129033268323 172977.5683584702 356.6547801205571
2018 5 810269.3225697606 1246.5681885688625 69687.03583794396 107.21082436606763 83644.78159173676 143.47303875083492 248284.77976738216 381.97658425751104
2018 6 824150.0588949074 1252.5076882901328 52047.11873700284 79.09896464590098 61491.79513703263 113.24455826341183 242561.98528653922 368.63523599777994
2018 7 920527.3411698391 1237.2679316798913 54001.805264369934 72.58307159189508 110136.29401019624 175.655971308128 284977.6805428929 383.034516858727
2018 8 934028.5503494726 1255.4147182116567 56534.79015953165 75.9876211821662 18440.748190359904 37.25403674820183 281409.0307800985 378.23794459690663
2018 9 796294.725012032 1201.0478506968807 59750.627903558845 90.12161071426674 123357.51466599084 219.10748608524128 251368.59077414437 379.13814596401863
2018 10 876930.3483784569 1178.6698230893237 68192.54789305253 91.6566503938878 161682.95582566154 259.9404434496166 285017.3815606405 383.08787844172116
2018 11 220190.80577936105 1177.4909399965832 17258.73825754189 92.29271795476946 31282.504059750245 215.74140730862237 74832.7243967542 400.17499677408665
2018 12 600604.6369429264 1213.3427008948008 40572.284658587676 81.96421143149026 45621.901088225204 103.21697078783983 174784.206063148 353.0994061881778
2019 1 875861.801519637 1177.2336041930605 62356.90803397393 83.81304843276065 38427.200289164524 68.37580122627139 266395.0245066205 358.05782863793075
2019 2 766839.5051314117 1141.1302159693628 58769.95527064857 87.45529058132227 98888.76720553766 174.40699683516343 263572.6687568533 392.22123326912697
2019 3 875531.8723175366 1176.790150964431 64134.01856961811 86.20163786238993 137670.795483173 220.9804100853499 307147.8785608932 412.8331701087274
2019 4 882405.782299406 1225.5635865269528 62498.88624107809 86.80400866816402 118655.91830720675 190.76514197300122 310192.1270515332 430.822398682685
2019 5 915804.9453381414 1230.920625454491 62733.50787805464 84.31923101889065 51908.476725653585 92.36383759013093 333347.07849929936 448.04714852056367
2019 6 743714.4735556253 1264.8205332578661 47914.37351050592 81.48702977977197 33701.15766617421 99.1210519593359 258663.9435447939 439.9046658925066
2019 7 936697.9580959912 1259.0026318494506 60426.11789318791 81.21790039406977 10876.808554053552 30.2133570945932 322797.22597334185 433.867239211481
2019 8 930640.7550409784 1250.8612298937883 60639.08568100215 81.50414742070181 13063.088331067778 44.58391921866136 336075.4530245259 451.714318581352
2019 9 862784.4318728528 1198.3117109345178 63348.64890275069 87.98423458715374 56973.39566326609 126.32681965247471 322162.84531000606 447.4483962638973
2019 10 900170.2429565524 1209.9062405330005 64953.1213626686 87.30258247670511 98168.02805722521 218.63703353502274 310577.0949153713 417.442331875499
2018 12 600604.6369429257 1213.3427008947995 40572.284658587654 81.96421143149021 45621.901088225226 103.21697078783988 174784.20606314807 353.0994061881779
2019 1 875861.801519638 1177.2336041930619 62356.908033973945 83.81304843276068 38427.200289164495 68.37580122627135 266395.0245066203 358.0578286379305
2019 2 766839.505131412 1141.130215969363 58769.95527064854 87.45529058132223 98888.76720553762 174.40699683516334 263572.66875685327 392.22123326912686
2019 3 875531.8723175367 1176.7901509644312 64134.01856961807 86.20163786238987 137670.795483173 220.9804100853499 307147.8785608931 412.8331701087273
2019 4 882405.782299407 1225.5635865269542 62498.886241078064 86.80400866816397 118655.91830720665 190.76514197300105 310192.1270515335 430.8223986826854
2019 5 915804.9453381395 1230.9206254544886 62733.507878054705 84.31923101889073 51908.476725653585 92.36383759013093 333347.07849929924 448.0471485205635
2019 6 743714.4735556262 1264.8205332578677 47914.3735105059 81.48702977977194 33701.15766617421 99.1210519593359 258663.9435447938 439.90466589250644
2019 7 936697.9580959913 1259.0026318494506 60426.11789318793 81.2179003940698 10876.808554053552 30.2133570945932 322797.225973342 433.8672392114812
2019 8 930640.7550409781 1250.8612298937878 60639.0856810021 81.50414742070174 13063.088331067778 44.58391921866136 336075.45302452584 451.7143185813519
2019 9 862784.431872853 1198.311710934518 63348.64890275075 87.98423458715382 56973.395663266085 126.32681965247468 322162.845310006 447.4483962638972
2019 10 900170.2429565525 1209.9062405330008 64953.12136266855 87.30258247670504 98168.0280572252 218.6370335350227 310577.09491537116 417.44233187549884
2019 11 881836.9061517053 1224.7734807662573 61941.828695957716 86.0303176332746 181716.3770488152 294.99411858573893 258322.18736711307 358.78081578765705
2019 12 45118.354529795746 1253.2876258276597 2884.8668441554055 80.13519011542793 13576.481448950073 377.1244846930576 12249.12485575384 340.25346821538443
@ -2329,8 +2329,8 @@ cslab9e 2017-01-10T14:19:45
2012-12-23T23:28:02 229.50.247.232 //cgi-bin/feedback/ 500 410
-- !14 --
2019-12-01T09:30:13 16C448031108ABA balcony_5 door 5 temperature F 32.0
2019-12-01T09:59:58 16C448031108ABA balcony_5 door 5 temperature F 32.0
2019-12-01T09:30:13.417 16C448031108ABA balcony_5 door 5 temperature F 32.0
2019-12-01T09:59:58.874 16C448031108ABA balcony_5 door 5 temperature F 32.0
-- !15 --
balcony_3 door 3 SUMMER

View File

@ -29,32 +29,32 @@
-- !q05 --
1937.7762425702406
992.21123681735253
56.682069922520562
940.70481552186243
992.2112368173525
56.68206992252056
940.7048155218624
1876.4831949153224
-- !q06 --
2023-03-07 20:34:59.601
2023-03-07 20:34:59.693
2023-03-07 20:34:59.708
2023-03-07 20:34:59.782
2023-03-07 20:34:59.836
2023-03-07 20:34:59.934
2023-03-07 20:34:59.950
2023-03-07 20:35:00.042
2023-03-07 20:35:00.053
2023-03-07 20:35:00.114
2023-03-07 20:35:00.134
2023-03-07 20:35:00.201
2023-03-07 20:35:00.272
2023-03-07 20:35:00.316
2023-03-07 20:35:00.337
2023-03-07 20:35:00.409
2023-03-07 20:35:00.420
2023-03-07 20:35:00.428
2023-03-07 20:35:00.500
2023-03-07 20:35:00.535
2023-03-07T20:34:59.601
2023-03-07T20:34:59.693
2023-03-07T20:34:59.708
2023-03-07T20:34:59.782
2023-03-07T20:34:59.836
2023-03-07T20:34:59.934
2023-03-07T20:34:59.950
2023-03-07T20:35:00.042
2023-03-07T20:35:00.053
2023-03-07T20:35:00.114
2023-03-07T20:35:00.134
2023-03-07T20:35:00.201
2023-03-07T20:35:00.272
2023-03-07T20:35:00.316
2023-03-07T20:35:00.337
2023-03-07T20:35:00.409
2023-03-07T20:35:00.420
2023-03-07T20:35:00.428
2023-03-07T20:35:00.500
2023-03-07T20:35:00.535
-- !q07 --
6f77a7baae184d
@ -124,15 +124,16 @@ b5e6bf2b5
5000
-- !q16 --
2023-03-07 20:35:59.064
2023-03-07 20:35:59.087
2023-03-07 20:35:59.110
2023-03-07 20:35:59.129
2023-03-07 20:35:59.224
2023-03-07T20:35:59.064
2023-03-07T20:35:59.087
2023-03-07T20:35:59.110
2023-03-07T20:35:59.129
2023-03-07T20:35:59.224
-- !q17 --
14040216 \N 2147483647 2023-03-07 20:38:02.140 81.607142423775869 b1d54a8ac60a4c8aa 66.6566 a54742979109 9a8247ed7c74 false
7847742 17740 2147483647 2023-03-07 20:36:02.376 1740.7904511543441 ff588a918be 66.8626 41c532d698024 18d9fa638cd449d893 true
9045125 27361 2147483647 2023-03-07 20:35:51.997 1245.2170379359104 b31a143e67 66.9046 52ab9d8a748f4c9 5d70ec319e true
10410585 \N 1938534851 2023-03-07 20:35:17.731 955.1760424982325 643e7c71b83d444e9261 67.0202 6a15d14103dc4 55b15adbec34 true
10055090 \N 2147483647 2023-03-07 20:38:59.078 1387.1527042831178 47 67.7351 c4c5 960637955914682b6 true
14040216 \N 2147483647 2023-03-07T20:38:02.140 81.60714242377587 b1d54a8ac60a4c8aa 66.6566 a54742979109 9a8247ed7c74 false
7847742 17740 2147483647 2023-03-07T20:36:02.376 1740.790451154344 ff588a918be 66.8626 41c532d698024 18d9fa638cd449d893 true
9045125 27361 2147483647 2023-03-07T20:35:51.997 1245.2170379359104 b31a143e67 66.9046 52ab9d8a748f4c9 5d70ec319e true
10410585 \N 1938534851 2023-03-07T20:35:17.731 955.1760424982325 643e7c71b83d444e9261 67.0202 6a15d14103dc4 55b15adbec34 true
10055090 \N 2147483647 2023-03-07T20:38:59.078 1387.1527042831178 47 67.7351 c4c5 960637955914682b6 true

View File

@ -60,3 +60,4 @@
-- !q14 --
Customer#000000004
Customer#000000007

View File

@ -81,14 +81,14 @@ b 1
c 1
-- !ex_tb13 --
张三0 11 1234567 123 321312 1999-02-13T00:00 中国 男 0
张三1 11 12345678 123 321312 1999-02-13T00:00 中国 男 0
张三2 11 12345671 123 321312 1999-02-13T00:00 中国 男 0
张三3 11 12345673 123 321312 1999-02-13T00:00 中国 男 0
张三4 11 123456711 123 321312 1999-02-13T00:00 中国 男 0
张三5 11 1232134567 123 321312 1999-02-13T00:00 中国 男 0
张三6 11 124314567 123 321312 1999-02-13T00:00 中国 男 0
张三7 11 123445167 123 321312 1998-02-13T00:00 中国 男 0
张三0 11 1234567 123 321312 1999-02-13T00:00 中国 男 false
张三1 11 12345678 123 321312 1999-02-13T00:00 中国 男 false
张三2 11 12345671 123 321312 1999-02-13T00:00 中国 男 false
张三3 11 12345673 123 321312 1999-02-13T00:00 中国 男 false
张三4 11 123456711 123 321312 1999-02-13T00:00 中国 男 false
张三5 11 1232134567 123 321312 1999-02-13T00:00 中国 男 false
张三6 11 124314567 123 321312 1999-02-13T00:00 中国 男 false
张三7 11 123445167 123 321312 1998-02-13T00:00 中国 男 false
-- !ex_tb14 --
123 2022-11-02 2022-11-02 8011 oppo
@ -179,13 +179,17 @@ doris3 20
doris3 20
-- !test_insert4 --
1 abcHa1.12345 1.123450xkalowadawd 2022-10-01 3.14159 1 2 0 100000 1.2345678 24.000 07:09:51 2022 2022-11-27T07:09:51 2022-11-27T07:09:51
true abcHa1.12345 1.123450xkalowadawd 2022-10-01 3.14159 1 2 0 100000 1.2345678 24.000 07:09:51 2022 2022-11-27T07:09:51 2022-11-27T07:09:51
-- !specified_database_1 --
doris_test
-- !specified_database_2 --
doris_test
information_schema
init_db
mysql
performance_schema
sys
-- !specified_database_3 --
information_schema
@ -195,6 +199,11 @@ performance_schema
sys
-- !specified_database_4 --
information_schema
init_db
mysql
performance_schema
sys
-- !ex_tb1 --
{"k1":"v1", "k2":"v2"}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -17,131 +17,3 @@ ex_tb1
ex_tb0
ex_tb1
-- !select --
doris_test
-- !select --
ex_tb0
ex_tb1
ex_tb10
ex_tb11
ex_tb12
ex_tb13
ex_tb14
ex_tb15
ex_tb16
ex_tb17
ex_tb18
ex_tb19
ex_tb2
ex_tb20
ex_tb3
ex_tb4
ex_tb5
ex_tb6
ex_tb7
ex_tb8
ex_tb9
test1
test_insert
test_insert2
-- !select --
ex_tb0
-- !select --
ex_tb0
ex_tb1
ex_tb10
ex_tb11
ex_tb12
ex_tb13
ex_tb14
ex_tb15
ex_tb16
ex_tb17
ex_tb18
ex_tb19
ex_tb2
ex_tb20
ex_tb3
ex_tb4
ex_tb5
ex_tb6
ex_tb7
ex_tb8
ex_tb9
test1
test_insert
test_insert2
-- !select --
ex_tb0
ex_tb1
-- !select --
ex_tb0
ex_tb1
ex_tb10
ex_tb11
ex_tb12
ex_tb13
ex_tb14
ex_tb15
ex_tb16
ex_tb17
ex_tb18
ex_tb19
ex_tb2
ex_tb20
ex_tb3
ex_tb4
ex_tb5
ex_tb6
ex_tb7
ex_tb8
ex_tb9
test1
test_insert
test_insert2
-- !select --
doris_test
-- !select --
ex_tb0
ex_tb1
-- !select --
ex_tb1
-- !select --
ex_tb0
ex_tb1
-- !select --
ex_tb0
ex_tb1
ex_tb10
ex_tb11
ex_tb12
ex_tb13
ex_tb14
ex_tb15
ex_tb16
ex_tb17
ex_tb18
ex_tb19
ex_tb2
ex_tb20
ex_tb3
ex_tb4
ex_tb5
ex_tb6
ex_tb7
ex_tb8
ex_tb9
test1
test_insert
test_insert2

View File

@ -46,7 +46,7 @@ suite("test_jdbc_query_mysql", "p0") {
sql """drop table if exists $jdbcMysql57Table1"""
sql """
CREATE EXTERNAL TABLE `$jdbcMysql57Table1` (
k1 tinyint,
k1 boolean,
k2 char(100),
k3 varchar(128),
k4 date,
@ -541,7 +541,7 @@ suite("test_jdbc_query_mysql", "p0") {
birthday DATETIME,
country varchar(128),
gender varchar(128),
covid tinyint
covid boolean
) ENGINE=JDBC
COMMENT "JDBC Mysql 外部表"
PROPERTIES (