database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](Nereids): datetime support microsecond overflow (#30744)

Browse Source
This commit is contained in:
jakevin
2024-02-02 23:05:52 +08:00
committed by yiguolei
parent 151735748b
commit 8a0ea4b651
11 changed files with 120 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
View File

@ -183,9 +183,9 @@ public class DateLiteral extends Literal {
// normalize leading 0 for date and time
// date and time contains 6 number part at most, so we just need normal 6 number part
int partNumber = 0;
while (i < s.length()) {
while (i < s.length() && partNumber < 6) {
char c = s.charAt(i);
if (Character.isDigit(c) && partNumber < 6) {
if (Character.isDigit(c)) {
// find consecutive digit
int j = i + 1;
while (j < s.length() && Character.isDigit(s.charAt(j))) {
@ -234,11 +234,14 @@ public class DateLiteral extends Literal {
}
// parse MicroSecond
// Keep up to 7 digits at most, 7th digit is use for overflow.
if (partNumber == 6 && i < s.length() && s.charAt(i) == '.') {
sb.append(s.charAt(i));
i += 1;
while (i < s.length() && Character.isDigit(s.charAt(i))) {
sb.append(s.charAt(i));
if (i - 19 <= 7) {
sb.append(s.charAt(i));
}
i += 1;
}
}
@ -266,11 +269,12 @@ public class DateLiteral extends Literal {
try {
TemporalAccessor dateTime;
// remove suffix ' '
// remove suffix/prefix ' '
s = s.trim();
// parse condition without '-' and ':'
boolean containsPunctuation = false;
for (int i = 0; i < s.length(); i++) {
int len = Math.min(s.length(), 11);
for (int i = 0; i < len; i++) {
if (isPunctuation(s.charAt(i))) {
containsPunctuation = true;
break;

16
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
View File

@ -136,7 +136,6 @@ public class DateTimeLiteral extends DateLiteral {
hour = DateUtils.getOrDefault(temporal, ChronoField.HOUR_OF_DAY);
minute = DateUtils.getOrDefault(temporal, ChronoField.MINUTE_OF_HOUR);
second = DateUtils.getOrDefault(temporal, ChronoField.SECOND_OF_MINUTE);
microSecond = DateUtils.getOrDefault(temporal, ChronoField.MICRO_OF_SECOND);
ZoneId zoneId = temporal.query(TemporalQueries.zone());
if (zoneId != null) {
@ -153,6 +152,21 @@ public class DateTimeLiteral extends DateLiteral {
}
}
microSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND) / 100L;
// Microseconds have 7 digits.
long sevenDigit = microSecond % 10;
microSecond = microSecond / 10;
if (sevenDigit >= 5 && this instanceof DateTimeV2Literal) {
DateTimeV2Literal result = (DateTimeV2Literal) ((DateTimeV2Literal) this).plusMicroSeconds(1);
this.second = result.second;
this.minute = result.minute;
this.hour = result.hour;
this.day = result.day;
this.month = result.month;
this.year = result.year;
this.microSecond = result.microSecond;
}
if (checkRange() || checkDate()) {
throw new AnalysisException("datetime literal [" + s + "] is out of range");
}

9
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java
View File

@ -66,7 +66,7 @@ public class DateTimeV2Literal extends DateTimeLiteral {
if (this.microSecond >= 1000000) {
LocalDateTime localDateTime = DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND,
getStringValue()).plusSeconds(1);
getStringValue()).plusSeconds(1);
this.year = localDateTime.getYear();
this.month = localDateTime.getMonthValue();
this.day = localDateTime.getDayOfMonth();
@ -77,6 +77,11 @@ public class DateTimeV2Literal extends DateTimeLiteral {
}
}
public String getFullMicroSecondValue() {
return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d",
year, month, day, hour, minute, second, microSecond);
}
@Override
public DateTimeV2Type getDataType() throws UnboundException {
return (DateTimeV2Type) super.getDataType();
@ -165,7 +170,7 @@ public class DateTimeV2Literal extends DateTimeLiteral {
public Expression plusMicroSeconds(long microSeconds) {
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getFullMicroSecondValue())
.plusNanos(microSeconds * 1000L), getDataType().getScale());
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
View File

@ -84,10 +84,13 @@ public class DateTimeV2Type extends DateLikeType {
/**
* return proper type of datetimev2 for String
* may be we need to check for validity?
* maybe we need to check for validity?
*/
public static DateTimeV2Type forTypeFromString(String s) {
int scale = DateTimeLiteral.determineScale(s);
if (scale > MAX_SCALE) {
scale = MAX_SCALE;
}
return DateTimeV2Type.of(scale);
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java
View File

@ -56,8 +56,9 @@ public class DateTimeFormatterUtils {
.appendValue(ChronoField.HOUR_OF_DAY, 2)
.appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2)
// microsecond maxWidth is 7, we may need 7th digit to judge overflow
.appendOptional(new DateTimeFormatterBuilder()
.appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter())
.appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter())
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// Time without delimiter: HHmmss[.microsecond]
private static final DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder()
@ -65,7 +66,7 @@ public class DateTimeFormatterUtils {
.appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.appendOptional(new DateTimeFormatterBuilder()
.appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter())
.appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter())
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// yyyymmdd
private static final DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder()