database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bugfix](iceberg)fix datetime conversion error and data path error (#35708)

Browse Source 
## Proposed changes
Issue #31442

<!--Describe your changes.-->

1. The unit of the seventh parameter of `ZonedDateTime.of` is
nanosecond, so we should multiply the microsecond by 1000.
2. When writing to a non-partitioned iceberg table, the data path has an
extra slash
This commit is contained in:
wuwenchi
2024-06-01 00:41:56 +08:00
committed by yiguolei
parent bc062a2595
commit cb96a79d07
6 changed files with 201 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
View File

@ -1005,12 +1005,25 @@ public class DateLiteral extends LiteralExpr {
}
public long unixTimestamp(TimeZone timeZone) {
ZonedDateTime zonedDateTime = ZonedDateTime.of((int) year, (int) month, (int) day, (int) hour,
(int) minute, (int) second, (int) microsecond, ZoneId.of(timeZone.getID()));
Timestamp timestamp = Timestamp.from(zonedDateTime.toInstant());
Timestamp timestamp = getTimestamp(timeZone);
return timestamp.getTime();
}
private Timestamp getTimestamp(TimeZone timeZone) {
ZonedDateTime zonedDateTime = ZonedDateTime.of((int) year, (int) month, (int) day, (int) hour,
(int) minute, (int) second, (int) microsecond * 1000, ZoneId.of(timeZone.getID()));
return Timestamp.from(zonedDateTime.toInstant());
}
public long getUnixTimestampWithMillisecond(TimeZone timeZone) {
return unixTimestamp(timeZone);
}
public long getUnixTimestampWithMicroseconds(TimeZone timeZone) {
Timestamp timestamp = getTimestamp(timeZone);
return timestamp.getTime() * 1000 + timestamp.getNanos() / 1000 % 1000;
}
public static boolean hasTimePart(String format) {
return format.chars().anyMatch(c -> TIME_PART_SET.contains((char) c));
}

3
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
View File

@ -84,7 +84,6 @@ public class IcebergUtils {
return 0;
}
};
static long MILLIS_TO_NANO_TIME = 1000;
// https://iceberg.apache.org/spec/#schemas-and-data-types
// All time and timestamp values are stored with microsecond precision
private static final int ICEBERG_DATETIME_SCALE_MS = 6;
@ -320,7 +319,7 @@ public class IcebergUtils {
case DATE:
return dateLiteral.getStringValue();
case TIMESTAMP:
return dateLiteral.unixTimestamp(TimeUtils.getTimeZone()) * MILLIS_TO_NANO_TIME;
return dateLiteral.getUnixTimestampWithMicroseconds(TimeUtils.getTimeZone());
default:
return null;
}