database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](Nereids): datetime support microsecond overflow (#30744)

Browse Source
This commit is contained in:
jakevin
2024-02-02 23:05:52 +08:00
committed by yiguolei
parent 151735748b
commit 8a0ea4b651
11 changed files with 120 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
View File

@ -183,9 +183,9 @@ public class DateLiteral extends Literal {
// normalize leading 0 for date and time
// date and time contains 6 number part at most, so we just need normal 6 number part
int partNumber = 0;
while (i < s.length()) {
while (i < s.length() && partNumber < 6) {
char c = s.charAt(i);
if (Character.isDigit(c) && partNumber < 6) {
if (Character.isDigit(c)) {
// find consecutive digit
int j = i + 1;
while (j < s.length() && Character.isDigit(s.charAt(j))) {
@ -234,11 +234,14 @@ public class DateLiteral extends Literal {
}
// parse MicroSecond
// Keep up to 7 digits at most, 7th digit is use for overflow.
if (partNumber == 6 && i < s.length() && s.charAt(i) == '.') {
sb.append(s.charAt(i));
i += 1;
while (i < s.length() && Character.isDigit(s.charAt(i))) {
sb.append(s.charAt(i));
if (i - 19 <= 7) {
sb.append(s.charAt(i));
}
i += 1;
}
}
@ -266,11 +269,12 @@ public class DateLiteral extends Literal {
try {
TemporalAccessor dateTime;
// remove suffix ' '
// remove suffix/prefix ' '
s = s.trim();
// parse condition without '-' and ':'
boolean containsPunctuation = false;
for (int i = 0; i < s.length(); i++) {
int len = Math.min(s.length(), 11);
for (int i = 0; i < len; i++) {
if (isPunctuation(s.charAt(i))) {
containsPunctuation = true;
break;

16
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
View File

@ -136,7 +136,6 @@ public class DateTimeLiteral extends DateLiteral {
hour = DateUtils.getOrDefault(temporal, ChronoField.HOUR_OF_DAY);
minute = DateUtils.getOrDefault(temporal, ChronoField.MINUTE_OF_HOUR);
second = DateUtils.getOrDefault(temporal, ChronoField.SECOND_OF_MINUTE);
microSecond = DateUtils.getOrDefault(temporal, ChronoField.MICRO_OF_SECOND);
ZoneId zoneId = temporal.query(TemporalQueries.zone());
if (zoneId != null) {
@ -153,6 +152,21 @@ public class DateTimeLiteral extends DateLiteral {
}
}
microSecond = DateUtils.getOrDefault(temporal, ChronoField.NANO_OF_SECOND) / 100L;
// Microseconds have 7 digits.
long sevenDigit = microSecond % 10;
microSecond = microSecond / 10;
if (sevenDigit >= 5 && this instanceof DateTimeV2Literal) {
DateTimeV2Literal result = (DateTimeV2Literal) ((DateTimeV2Literal) this).plusMicroSeconds(1);
this.second = result.second;
this.minute = result.minute;
this.hour = result.hour;
this.day = result.day;
this.month = result.month;
this.year = result.year;
this.microSecond = result.microSecond;
}
if (checkRange() || checkDate()) {
throw new AnalysisException("datetime literal [" + s + "] is out of range");
}

9
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java
View File

@ -66,7 +66,7 @@ public class DateTimeV2Literal extends DateTimeLiteral {
if (this.microSecond >= 1000000) {
LocalDateTime localDateTime = DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND,
getStringValue()).plusSeconds(1);
getStringValue()).plusSeconds(1);
this.year = localDateTime.getYear();
this.month = localDateTime.getMonthValue();
this.day = localDateTime.getDayOfMonth();
@ -77,6 +77,11 @@ public class DateTimeV2Literal extends DateTimeLiteral {
}
}
public String getFullMicroSecondValue() {
return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d",
year, month, day, hour, minute, second, microSecond);
}
@Override
public DateTimeV2Type getDataType() throws UnboundException {
return (DateTimeV2Type) super.getDataType();
@ -165,7 +170,7 @@ public class DateTimeV2Literal extends DateTimeLiteral {
public Expression plusMicroSeconds(long microSeconds) {
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getFullMicroSecondValue())
.plusNanos(microSeconds * 1000L), getDataType().getScale());
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
View File

@ -84,10 +84,13 @@ public class DateTimeV2Type extends DateLikeType {
/**
* return proper type of datetimev2 for String
* may be we need to check for validity?
* maybe we need to check for validity?
*/
public static DateTimeV2Type forTypeFromString(String s) {
int scale = DateTimeLiteral.determineScale(s);
if (scale > MAX_SCALE) {
scale = MAX_SCALE;
}
return DateTimeV2Type.of(scale);
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java
View File

@ -56,8 +56,9 @@ public class DateTimeFormatterUtils {
.appendValue(ChronoField.HOUR_OF_DAY, 2)
.appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2)
// microsecond maxWidth is 7, we may need 7th digit to judge overflow
.appendOptional(new DateTimeFormatterBuilder()
.appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter())
.appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter())
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// Time without delimiter: HHmmss[.microsecond]
private static final DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder()
@ -65,7 +66,7 @@ public class DateTimeFormatterUtils {
.appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.appendOptional(new DateTimeFormatterBuilder()
.appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter())
.appendFraction(ChronoField.NANO_OF_SECOND, 1, 7, true).toFormatter())
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// yyyymmdd
private static final DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder()

9
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java
View File

@ -116,15 +116,6 @@ class SimplifyComparisonPredicateSqlTest extends TestWithFeService implements Me
)
);
PlanChecker.from(connectContext)
.analyze("select CONVERT('2021-01-30 00:00:00.0000001', DATETIME(6))")
.rewrite()
.matches(
logicalResultSink(
logicalOneRowRelation().when(p -> p.getProjects().get(0).child(0).equals(new NullLiteral(DateTimeV2Type.of(6))))
)
);
PlanChecker.from(connectContext)
.analyze("select CONVERT_TZ('2021-01-32 00:00:00', '+08:00', 'America/London') = '2021-01-30'")
.rewrite()

10
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java
View File

@ -28,14 +28,6 @@ import java.util.function.Consumer;
class DateLiteralTest {
@Test
void reject() {
// TODO: reject them.
// Now parse them as date + offset.
// PG parse them as date + offset, MySQL parse them as date + time (rubbish behavior!)
// So strange! reject these strange case.
// Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-01"));
// Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-1"));
// Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01+01"));
// Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01+1"));
Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 01:00:00.000000"));
Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:01:00.000000"));
Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01 00:00:01.000000"));
@ -212,7 +204,7 @@ class DateLiteralTest {
new DateLiteral("2020.02.01 00.00.00");
new DateTimeV2Literal("2020.02.01 00.00.00.1");
new DateTimeV2Literal("2020.02.01 00.00.00.000001");
Assertions.assertThrows(AnalysisException.class, () -> new DateTimeV2Literal("2020.02.01 00.00.00.0000001"));
new DateTimeV2Literal("2020.02.01 00.00.00.0000001");
}
@Test

34
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java
View File

@ -41,13 +41,14 @@ class DateTimeLiteralTest {
Assertions.assertEquals(1, datetime.day);
Assertions.assertEquals(1, datetime.hour);
Assertions.assertEquals(1, datetime.minute);
Assertions.assertEquals(1, datetime.second);
Assertions.assertEquals(2, datetime.second);
};
assertFunc.accept(new DateTimeV2Literal("20220801010101"));
assertFunc.accept(new DateTimeV2Literal("20220801T010101"));
assertFunc.accept(new DateTimeV2Literal("220801010101"));
assertFunc.accept(new DateTimeV2Literal("220801T010101"));
assertFunc.accept(new DateTimeV2Literal("20220801010102"));
assertFunc.accept(new DateTimeV2Literal("20220801T010102"));
assertFunc.accept(new DateTimeV2Literal("220801010102"));
assertFunc.accept(new DateTimeV2Literal("220801T010102"));
assertFunc.accept(new DateTimeV2Literal("20220801010101.9999999"));
}
@Test
@ -386,6 +387,29 @@ class DateTimeLiteralTest {
// Testing with microsecond of length 6
new DateTimeV2Literal("2016-07-02 01:01:01.123456");
new DateTimeV2Literal("2016-7-02 01:01:01.123456");
// Testing with microsecond of length 7
DateTimeV2Literal literal = new DateTimeV2Literal("2016-07-02 01:01:01.12345678");
Assertions.assertEquals(123457, literal.microSecond);
literal = new DateTimeV2Literal("2016-07-02 01:01:01.44444444");
Assertions.assertEquals(444444, literal.microSecond);
literal = new DateTimeV2Literal("2016-07-02 01:01:01.44444445");
Assertions.assertEquals(444444, literal.microSecond);
literal = new DateTimeV2Literal("2016-07-02 01:01:01.4444445");
Assertions.assertEquals(444445, literal.microSecond);
literal = new DateTimeV2Literal("2016-07-02 01:01:01.9999995");
Assertions.assertEquals(0, literal.microSecond);
Assertions.assertEquals(2, literal.second);
literal = new DateTimeV2Literal("2021-01-01 23:59:59.9999995");
Assertions.assertEquals(0, literal.microSecond);
Assertions.assertEquals(0, literal.second);
Assertions.assertEquals(0, literal.minute);
Assertions.assertEquals(0, literal.hour);
}
@Test

4
fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java
View File

@ -70,6 +70,8 @@ class DateTimeFormatterUtilsTest {
assertDatePart(dateTime);
dateTime = formatter.parse("20200219T010101.1");
assertDatePart(dateTime);
dateTime = formatter.parse("20200219T010101.0000001");
assertDatePart(dateTime);
// failed case
DateTimeFormatter withT = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER;
@ -77,11 +79,9 @@ class DateTimeFormatterUtilsTest {
Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101.0000001"));
Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101.0000001"));
DateTimeFormatter withoutT = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T;
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219 010101"));
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101.0000001"));
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101.0000001"));
}

22
regression-test/data/nereids_syntax_p0/test_literal.out Normal file
View File

@ -0,0 +1,22 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !date1 --
2016-07-03
-- !date2 --
2016-07-03
-- !datetime1 --
2016-07-02T23:59:59.990
-- !datetime2 --
2016-07-03T00:00
-- !datetime3 --
2016-07-03T00:00
-- !datetime4 --
2020-02-19T01:01:01
-- !datetime5 --
2021-01-30T00:00

28
regression-test/suites/nereids_syntax_p0/test_literal.groovy
View File

@ -24,4 +24,32 @@ suite("test_literal") {
sql """
select {}, [], [[[null]], [[1, 2, 3]]], {1:[null], 3:[3]};
"""
qt_date1 """
select cast('20160702235959.9999999' as date);
"""
qt_date2 """
select cast('2016-07-02 23:59:59.9999999' as date);
"""
qt_datetime1 """
select timestamp'2016-07-02 23:59:59.99';
"""
qt_datetime2 """
select cast('20160702235959.9999999' as datetime);
"""
qt_datetime3 """
select cast('2016-07-02 23:59:59.9999999' as datetime);
"""
qt_datetime4 """
select timestamp'20200219010101.0000001';
"""
qt_datetime5 """
select CONVERT('2021-01-30 00:00:00.0000001', DATETIME(6))
"""
}