database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](Nereids): new Date/Datetime parser to support more condition (#24224)

Browse Source 
* unify all Date/Datetime use one string-parser
* support microsecond & ZoneOffset both exist
* add many UT case
* add determineScale() to get scale of datetime, original code just get length of part after .
* reject more bad condition like 2022-01-01 00:00:00., we don't allow . without microsecond.
* .....
This commit is contained in:
jakevin
2023-09-13 11:20:27 +08:00
committed by GitHub
parent f205473426
commit 7025293e17
11 changed files with 849 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java
View File

@ -27,14 +27,13 @@ import org.apache.doris.nereids.types.DateType;
import org.apache.doris.nereids.types.coercion.DateLikeType;
import org.apache.doris.nereids.util.DateTimeFormatterUtils;
import org.apache.doris.nereids.util.DateUtils;
import org.apache.doris.nereids.util.StandardDateFormat;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.time.LocalDateTime;
import java.time.Year;
import java.time.format.DateTimeFormatter;
import java.time.format.ResolverStyle;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
@ -44,10 +43,7 @@ import java.time.temporal.TemporalAccessor;
public class DateLiteral extends Literal {
public static final String JAVA_DATE_FORMAT = "yyyy-MM-dd";
protected static DateTimeFormatter DATE_FORMATTER = null;
protected static DateTimeFormatter DATE_FORMATTER_TWO_DIGIT = null;
// for cast datetime type to date type.
protected static DateTimeFormatter DATE_TIME_FORMATTER = null;
private static final LocalDateTime startOfAD = LocalDateTime.of(0, 1, 1, 0, 0, 0);
private static final LocalDateTime endOfAD = LocalDateTime.of(9999, 12, 31, 23, 59, 59);
private static final Logger LOG = LogManager.getLogger(DateLiteral.class);
@ -55,26 +51,11 @@ public class DateLiteral extends Literal {
private static final DateLiteral MIN_DATE = new DateLiteral(0000, 1, 1);
private static final DateLiteral MAX_DATE = new DateLiteral(9999, 12, 31);
private static final int[] DAYS_IN_MONTH = new int[] {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
private static final int DATEKEY_LENGTH = 8;
protected long year;
protected long month;
protected long day;
static {
try {
DATE_FORMATTER = DateUtils.formatBuilder("%Y-%m-%d").toFormatter()
.withResolverStyle(ResolverStyle.STRICT);
DATE_FORMATTER_TWO_DIGIT = DateUtils.formatBuilder("%y-%m-%d").toFormatter()
.withResolverStyle(ResolverStyle.STRICT);
DATE_TIME_FORMATTER = DateUtils.formatBuilder("%Y-%m-%d %H:%i:%s").toFormatter()
.withResolverStyle(ResolverStyle.STRICT);
} catch (AnalysisException e) {
LOG.error("invalid date format", e);
System.exit(-1);
}
}
public DateLiteral(String s) throws AnalysisException {
this(DateType.INSTANCE, s);
}
@ -115,27 +96,72 @@ public class DateLiteral extends Literal {
this.day = other.day;
}
protected void init(String s) throws AnalysisException {
// replace 'T' with ' '
private static String replaceDelimiterT(String s) {
// Matcher matcher = Pattern.compile("^(\\d{2,4}-\\d{1,2}-\\d{1,2})T").matcher(s);
// if (matcher.find()) {
// return matcher.group(1) + " " + s.substring(matcher.end());
// }
// return s;
if (s.length() <= 10) {
return s;
}
if (s.charAt(10) == 'T') {
return s.substring(0, 10) + " " + s.substring(11);
} else if (s.charAt(8) == 'T') {
return s.substring(0, 8) + " " + s.substring(9);
} else {
return s;
}
}
protected static TemporalAccessor parse(String s) {
String originalString = s;
try {
TemporalAccessor dateTime;
// parse condition without '-' and ':'
if (!s.contains("-") && !s.contains(":")) {
dateTime = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER.parse(s);
} else if (s.split("-")[0].length() == 2) {
dateTime = DATE_FORMATTER_TWO_DIGIT.parse(s);
} else if (s.length() == 19) {
dateTime = DATE_TIME_FORMATTER.parse(s);
} else {
dateTime = DATE_FORMATTER.parse(s);
// mysql reject "20200219 010101" "200219 010101", can't use ' ' spilt basic date time.
if (!s.contains("T")) {
if (s.length() == 6) {
dateTime = DateTimeFormatterUtils.BASIC_TWO_DIGIT_DATE_FORMATTER.parse(s);
} else {
dateTime = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T.parse(s);
}
} else {
dateTime = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER.parse(s);
}
return dateTime;
}
year = DateUtils.getOrDefault(dateTime, ChronoField.YEAR);
month = DateUtils.getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR);
day = DateUtils.getOrDefault(dateTime, ChronoField.DAY_OF_MONTH);
// replace first 'T' with ' '
s = replaceDelimiterT(s);
if (!s.contains(" ")) {
dateTime = DateTimeFormatterUtils.ZONE_DATE_FORMATTER.parse(s);
} else {
dateTime = DateTimeFormatterUtils.ZONE_DATE_TIME_FORMATTER.parse(s);
}
// if Year is not present, throw exception
if (!dateTime.isSupported(ChronoField.YEAR)) {
throw new AnalysisException("datetime literal [" + originalString + "] is invalid");
}
return dateTime;
} catch (Exception ex) {
throw new AnalysisException("date literal [" + s + "] is invalid");
throw new AnalysisException("datetime literal [" + originalString + "] is invalid");
}
}
protected void init(String s) throws AnalysisException {
TemporalAccessor dateTime = parse(s);
year = DateUtils.getOrDefault(dateTime, ChronoField.YEAR);
month = DateUtils.getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR);
day = DateUtils.getOrDefault(dateTime, ChronoField.DAY_OF_MONTH);
if (checkRange() || checkDate()) {
throw new AnalysisException("date literal [" + s + "] is out of range");
throw new AnalysisException("datetime literal [" + s + "] is out of range");
}
}
@ -205,15 +231,17 @@ public class DateLiteral extends Literal {
}
public Expression plusDays(long days) {
return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusDays(days));
return fromJavaDateType(DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusDays(days));
}
public Expression plusMonths(long months) {
return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusMonths(months));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusMonths(months));
}
public Expression plusYears(long years) {
return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusYears(years));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusYears(years));
}
public LocalDateTime toJavaDateType() {

205
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java
View File

@ -24,31 +24,24 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DateTimeType;
import org.apache.doris.nereids.types.coercion.DateLikeType;
import org.apache.doris.nereids.util.DateTimeFormatterUtils;
import org.apache.doris.nereids.util.DateUtils;
import org.apache.doris.nereids.util.StandardDateFormat;
import com.google.common.base.Preconditions;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.ResolverStyle;
import java.time.ZoneId;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.util.Collections;
import java.time.temporal.TemporalQueries;
import java.util.Objects;
import java.util.regex.Pattern;
/**
* date time literal.
*/
public class DateTimeLiteral extends DateLiteral {
protected static DateTimeFormatter DATE_TIME_FORMATTER_TO_HOUR = null;
protected static DateTimeFormatter DATE_TIME_FORMATTER_TO_MINUTE = null;
protected static DateTimeFormatter DATE_TIME_FORMATTER_TWO_DIGIT = null;
protected static DateTimeFormatter DATE_TIME_FORMATTER_TO_MICRO_SECOND = null;
protected static final int MAX_MICROSECOND = 999999;
private static final DateTimeLiteral MIN_DATETIME = new DateTimeLiteral(0000, 1, 1, 0, 0, 0);
@ -56,35 +49,11 @@ public class DateTimeLiteral extends DateLiteral {
private static final Logger LOG = LogManager.getLogger(DateTimeLiteral.class);
private static final Pattern HAS_OFFSET_PART = Pattern.compile("[\\+\\-]\\d{2}:\\d{2}");
protected long hour;
protected long minute;
protected long second;
protected long microSecond;
static {
try {
DATE_TIME_FORMATTER = DateUtils.formatBuilder("%Y-%m-%d %H:%i:%s")
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
DATE_TIME_FORMATTER_TO_HOUR = DateUtils.formatBuilder("%Y-%m-%d %H")
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
DATE_TIME_FORMATTER_TO_MINUTE = DateUtils.formatBuilder("%Y-%m-%d %H:%i")
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
DATE_TIME_FORMATTER_TWO_DIGIT = DateUtils.formatBuilder("%y-%m-%d %H:%i:%s")
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
DATE_TIME_FORMATTER_TO_MICRO_SECOND = new DateTimeFormatterBuilder()
.appendPattern("uuuu-MM-dd HH:mm:ss")
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true)
.toFormatter()
.withResolverStyle(ResolverStyle.STRICT);
} catch (AnalysisException e) {
LOG.error("invalid date format", e);
System.exit(-1);
}
}
public DateTimeLiteral(String s) {
this(DateTimeType.INSTANCE, s);
}
@ -124,95 +93,50 @@ public class DateTimeLiteral extends DateLiteral {
this.day = day;
}
/**
* determine scale by datetime string
*/
public static int determineScale(String s) {
TemporalAccessor dateTime = parse(s);
int microSecond = DateUtils.getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND);
if (microSecond == 0) {
return 0;
}
int scale = 6;
while (microSecond % 10 == 0) {
scale--;
microSecond /= 10;
}
return scale;
}
@Override
protected void init(String s) throws AnalysisException {
try {
TemporalAccessor dateTime = null;
// parse timezone
if (haveTimeZoneOffset(s) || haveTimeZoneName(s)) {
if (haveTimeZoneName(s)) { // GMT, UTC+8, Z[, CN, Asia/Shanghai]
int split = getTimeZoneSplitPos(s);
Preconditions.checkArgument(split > 0);
s = s.substring(0, split);
} else { // +04:30
Preconditions.checkArgument(s.charAt(s.length() - 6) == '-' || s.charAt(s.length() - 6) == '+');
s = s.substring(0, s.length() - 6);
}
}
if (!s.contains("-") && !s.contains(":")) {
dateTime = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER.parse(s);
} else {
String[] datePart = s.contains(" ") ? s.split(" ")[0].split("-") : s.split("-");
DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
if (datePart.length != 3) {
throw new AnalysisException("datetime literal [" + s + "] is invalid");
}
for (int i = 0; i < datePart.length; i++) {
switch (i) {
case 0:
if (datePart[i].length() == 2) {
// If year is represented by two digits, number bigger than 70 will be prefixed
// with 19 otherwise 20. e.g. 69 -> 2069, 70 -> 1970.
builder.appendValueReduced(ChronoField.YEAR, 2, 2, 1970);
} else {
builder.appendPattern(String.join("", Collections.nCopies(datePart[i].length(), "u")));
}
break;
case 1:
builder.appendPattern(String.join("", Collections.nCopies(datePart[i].length(), "M")));
break;
case 2:
builder.appendPattern(String.join("", Collections.nCopies(datePart[i].length(), "d")));
break;
default:
throw new AnalysisException("two many parts in date format " + s);
}
if (i < datePart.length - 1) {
builder.appendLiteral("-");
}
}
if (s.contains(" ")) {
builder.appendLiteral(" ");
}
String[] timePart = s.contains(" ") ? s.split(" ")[1].split(":") : new String[]{};
for (int i = 0; i < timePart.length; i++) {
switch (i) {
case 0:
builder.appendPattern(String.join("", Collections.nCopies(timePart[i].length(), "H")));
break;
case 1:
builder.appendPattern(String.join("", Collections.nCopies(timePart[i].length(), "m")));
break;
case 2:
builder.appendPattern(String.join("", Collections.nCopies(timePart[i].contains(".")
? timePart[i].split("\\.")[0].length() : timePart[i].length(), "s")));
if (timePart[i].contains(".")) {
builder.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true);
}
break;
default:
throw new AnalysisException("too many parts in time format " + s);
}
if (i < timePart.length - 1) {
builder.appendLiteral(":");
}
}
// The default resolver style is 'SMART', which parses "2022-06-31" as "2022-06-30"
// and does not throw an exception. 'STRICT' is used here.
DateTimeFormatter formatter = builder.toFormatter().withResolverStyle(ResolverStyle.STRICT);
dateTime = formatter.parse(s);
}
TemporalAccessor temporal = parse(s);
year = DateUtils.getOrDefault(dateTime, ChronoField.YEAR);
month = DateUtils.getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR);
day = DateUtils.getOrDefault(dateTime, ChronoField.DAY_OF_MONTH);
hour = DateUtils.getOrDefault(dateTime, ChronoField.HOUR_OF_DAY);
minute = DateUtils.getOrDefault(dateTime, ChronoField.MINUTE_OF_HOUR);
second = DateUtils.getOrDefault(dateTime, ChronoField.SECOND_OF_MINUTE);
microSecond = DateUtils.getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND);
year = DateUtils.getOrDefault(temporal, ChronoField.YEAR);
month = DateUtils.getOrDefault(temporal, ChronoField.MONTH_OF_YEAR);
day = DateUtils.getOrDefault(temporal, ChronoField.DAY_OF_MONTH);
hour = DateUtils.getOrDefault(temporal, ChronoField.HOUR_OF_DAY);
minute = DateUtils.getOrDefault(temporal, ChronoField.MINUTE_OF_HOUR);
second = DateUtils.getOrDefault(temporal, ChronoField.SECOND_OF_MINUTE);
microSecond = DateUtils.getOrDefault(temporal, ChronoField.MICRO_OF_SECOND);
} catch (Exception ex) {
throw new AnalysisException("datetime literal [" + s + "] is invalid");
ZoneId zoneId = temporal.query(TemporalQueries.zone());
if (zoneId != null) {
int offset = DateUtils.getTimeZone().getRules().getOffset(Instant.now()).getTotalSeconds()
- zoneId.getRules().getOffset(Instant.now()).getTotalSeconds();
if (offset != 0) {
DateTimeLiteral result = (DateTimeLiteral) this.plusSeconds(offset);
this.second = result.second;
this.minute = result.minute;
this.hour = result.hour;
this.day = result.day;
this.month = result.month;
this.year = result.year;
}
}
if (checkRange() || checkDate()) {
@ -262,27 +186,33 @@ public class DateTimeLiteral extends DateLiteral {
}
public Expression plusYears(long years) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusYears(years));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusYears(years));
}
public Expression plusMonths(long months) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusMonths(months));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusMonths(months));
}
public Expression plusDays(long days) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusDays(days));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusDays(days));
}
public Expression plusHours(long hours) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusHours(hours));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusHours(hours));
}
public Expression plusMinutes(long minutes) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusMinutes(minutes));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusMinutes(minutes));
}
public Expression plusSeconds(long seconds) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER, getStringValue()).plusSeconds(seconds));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER, getStringValue()).plusSeconds(seconds));
}
public long getHour() {
@ -324,27 +254,4 @@ public class DateTimeLiteral extends DateLiteral {
: new DateTimeLiteral(dateTime.getYear(), dateTime.getMonthValue(), dateTime.getDayOfMonth(),
dateTime.getHour(), dateTime.getMinute(), dateTime.getSecond());
}
private static boolean haveTimeZoneOffset(String arg) {
Preconditions.checkArgument(arg.length() > 6);
return HAS_OFFSET_PART.matcher(arg.substring(arg.length() - 6)).matches();
}
private static boolean haveTimeZoneName(String arg) {
for (char ch : arg.toCharArray()) {
if (Character.isUpperCase(ch) && ch != 'T') {
return true;
}
}
return false;
}
private static int getTimeZoneSplitPos(String arg) {
int split = arg.length() - 1;
for (; !Character.isAlphabetic(arg.charAt(split)); split--) {
} // skip +8 of UTC+8
for (; split >= 0 && (Character.isUpperCase(arg.charAt(split)) || arg.charAt(split) == '/'); split--) {
}
return split + 1;
}
}

38
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.util.DateUtils;
import org.apache.doris.nereids.util.StandardDateFormat;
import java.time.LocalDateTime;
@ -76,50 +77,57 @@ public class DateTimeV2Literal extends DateTimeLiteral {
@Override
public String getStringValue() {
return String.format("%04d-%02d-%02d %02d:%02d:%02d"
+ (getDataType().getScale() > 0 ? ".%0" + getDataType().getScale() + "d" : ""),
+ (getDataType().getScale() > 0 ? ".%0" + getDataType().getScale() + "d" : ""),
year, month, day, hour, minute, second,
(int) (microSecond / Math.pow(10, DateTimeV2Type.MAX_SCALE - getDataType().getScale())));
}
@Override
public Expression plusYears(long years) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusYears(years), getDataType().getScale());
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusYears(years), getDataType().getScale());
}
@Override
public Expression plusMonths(long months) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusMonths(months), getDataType().getScale());
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusMonths(months), getDataType().getScale());
}
@Override
public Expression plusDays(long days) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusDays(days), getDataType().getScale());
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusDays(days), getDataType().getScale());
}
@Override
public Expression plusHours(long hours) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusHours(hours), getDataType().getScale());
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusHours(hours), getDataType().getScale());
}
@Override
public Expression plusMinutes(long minutes) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusMinutes(minutes), getDataType().getScale());
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusMinutes(minutes), getDataType().getScale());
}
@Override
public Expression plusSeconds(long seconds) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusSeconds(seconds), getDataType().getScale());
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusSeconds(seconds), getDataType().getScale());
}
public Expression plusMicroSeconds(long microSeconds) {
return fromJavaDateType(DateUtils.getTime(DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusNanos(microSeconds * 1000L), getDataType().getScale());
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_TIME_FORMATTER_TO_MICRO_SECOND, getStringValue())
.plusNanos(microSeconds * 1000L), getDataType().getScale());
}
/**

9
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DateV2Type;
import org.apache.doris.nereids.util.DateUtils;
import org.apache.doris.nereids.util.StandardDateFormat;
import java.time.LocalDateTime;
@ -51,15 +52,17 @@ public class DateV2Literal extends DateLiteral {
}
public Expression plusDays(long days) {
return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusDays(days));
return fromJavaDateType(DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusDays(days));
}
public Expression plusMonths(long months) {
return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusMonths(months));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusMonths(months));
}
public Expression plusYears(long years) {
return fromJavaDateType(DateUtils.getTime(DATE_FORMATTER, getStringValue()).plusYears(years));
return fromJavaDateType(
DateUtils.getTime(StandardDateFormat.DATE_FORMATTER, getStringValue()).plusYears(years));
}
public static Expression fromJavaDateType(LocalDateTime dateTime) {

4
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DateTimeV2Type.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.nereids.types;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.types.coercion.DateLikeType;
import org.apache.doris.nereids.types.coercion.IntegralType;
@ -85,7 +86,8 @@ public class DateTimeV2Type extends DateLikeType {
if (!s.contains(".")) {
return DateTimeV2Type.SYSTEM_DEFAULT;
}
return DateTimeV2Type.of(s.length() - s.lastIndexOf(".") - 1);
int scale = DateTimeLiteral.determineScale(s);
return DateTimeV2Type.of(scale);
}
@Override

98
fe/fe-core/src/main/java/org/apache/doris/nereids/util/DateTimeFormatterUtils.java
View File

@ -39,8 +39,27 @@ import java.time.temporal.ChronoField;
* Note incomplete times 'hh:mm:ss', 'hh:mm', 'D hh:mm', 'D hh', or 'ss'
*/
public class DateTimeFormatterUtils {
// Date: %Y-%m-%d
public static DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder()
public static final DateTimeFormatter ZONE_FORMATTER = new DateTimeFormatterBuilder()
.optionalStart()
// .appendZoneText(TextStyle.FULL)
.appendZoneOrOffsetId()
.optionalEnd()
// .appendOptional(
// new DateTimeFormatterBuilder().appendOffset("+HH", "").toFormatter())
// .appendOptional(
// new DateTimeFormatterBuilder().appendOffset("+HH:MM", "").toFormatter())
// .appendOptional(
// new DateTimeFormatterBuilder().appendOffset("+HH:MM:SS", "").toFormatter())
.toFormatter()
.withResolverStyle(ResolverStyle.STRICT);
// yymmdd
public static final DateTimeFormatter BASIC_TWO_DIGIT_DATE_FORMATTER = new DateTimeFormatterBuilder()
.appendValueReduced(ChronoField.YEAR, 2, 2, 1970)
.appendValue(ChronoField.MONTH_OF_YEAR, 2)
.appendValue(ChronoField.DAY_OF_MONTH, 2)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// yyyy-mm-dd
public static final DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder()
.appendOptional(
new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR, 4).toFormatter())
.appendOptional(
@ -48,33 +67,72 @@ public class DateTimeFormatterUtils {
.appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2)
.appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// Date without delimiter: %Y%m%d
public static DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder()
.appendValue(ChronoField.YEAR, 4)
.appendValue(ChronoField.MONTH_OF_YEAR, 2)
.appendValue(ChronoField.DAY_OF_MONTH, 2)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// Time: %H:%i:%s
public static DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder()
// HH[:mm][:ss][.microsecond]
public static final DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder()
.appendValue(ChronoField.HOUR_OF_DAY, 2)
.appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.appendOptional(
new DateTimeFormatterBuilder()
.appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendOptional(
new DateTimeFormatterBuilder()
.appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.appendOptional(new DateTimeFormatterBuilder()
.appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true)
.toFormatter())
.toFormatter()
)
.toFormatter()
)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// Time without delimiter: HHmmss[microsecond]
public static DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder()
// Time without delimiter: HHmmss[.microsecond]
private static final DateTimeFormatter BASIC_TIME_FORMATTER = new DateTimeFormatterBuilder()
.appendValue(ChronoField.HOUR_OF_DAY, 2)
.appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.appendOptional(new DateTimeFormatterBuilder()
.appendFraction(ChronoField.MICRO_OF_SECOND, 1, 6, true).toFormatter())
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// yyyymmdd
private static final DateTimeFormatter BASIC_DATE_FORMATTER = new DateTimeFormatterBuilder()
.appendValue(ChronoField.YEAR, 4)
.appendValue(ChronoField.MONTH_OF_YEAR, 2)
.appendValue(ChronoField.DAY_OF_MONTH, 2)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// Date without delimiter
public static DateTimeFormatter BASIC_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
.append(BASIC_DATE_FORMATTER)
.optionalStart()
.appendOptional(new DateTimeFormatterBuilder().appendLiteral('T').toFormatter())
public static final DateTimeFormatter BASIC_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
.appendOptional(BASIC_DATE_FORMATTER)
.appendOptional(BASIC_TWO_DIGIT_DATE_FORMATTER)
.appendLiteral('T')
.append(BASIC_TIME_FORMATTER)
.optionalEnd()
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// Date without delimiter
public static final DateTimeFormatter BASIC_FORMATTER_WITHOUT_T = new DateTimeFormatterBuilder()
.append(BASIC_DATE_FORMATTER)
.appendOptional(BASIC_TIME_FORMATTER)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// Datetime
public static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
.append(DATE_FORMATTER)
.appendLiteral(' ')
.append(TIME_FORMATTER)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
public static final DateTimeFormatter ZONE_DATE_FORMATTER = new DateTimeFormatterBuilder()
.appendOptional(
new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR, 4).toFormatter())
.appendOptional(
new DateTimeFormatterBuilder().appendValueReduced(ChronoField.YEAR, 2, 2, 1970).toFormatter())
.appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2)
.appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2)
// .optionalStart()
// .appendZoneOrOffsetId()
// .optionalEnd()
.append(ZONE_FORMATTER)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
public static final DateTimeFormatter ZONE_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
.append(DATE_FORMATTER)
.appendLiteral(' ')
.append(TIME_FORMATTER)
.append(ZONE_FORMATTER)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
}

53
fe/fe-core/src/main/java/org/apache/doris/nereids/util/StandardDateFormat.java Normal file
View File

@ -0,0 +1,53 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.util;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.ResolverStyle;
import java.time.temporal.ChronoField;
/**
* Following format is *standard* format of date/datetime. It will keep format of memory/output/printf is standard.
*/
public class StandardDateFormat {
// Date: %Y-%m-%d
public static DateTimeFormatter DATE_FORMATTER = new DateTimeFormatterBuilder()
.appendValue(ChronoField.YEAR, 4)
.appendLiteral('-').appendValue(ChronoField.MONTH_OF_YEAR, 2)
.appendLiteral('-').appendValue(ChronoField.DAY_OF_MONTH, 2)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// %H:%i:%s
public static DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder()
.appendValue(ChronoField.HOUR_OF_DAY, 2)
.appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendLiteral(':').appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
public static DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
.append(DATE_FORMATTER)
.appendLiteral(' ')
.append(TIME_FORMATTER)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
// "%Y-%m-%d %H:%i:%s.%f"
public static DateTimeFormatter DATE_TIME_FORMATTER_TO_MICRO_SECOND = new DateTimeFormatterBuilder()
.append(DATE_FORMATTER)
.appendLiteral(' ')
.append(TIME_FORMATTER)
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true) // Notice: min size is 0
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
}

45
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralEqualTest.java Normal file
View File

@ -0,0 +1,45 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class LiteralEqualTest {
@Test
void testEqual() {
IntegerLiteral one = new IntegerLiteral(1);
IntegerLiteral anotherOne = new IntegerLiteral(1);
IntegerLiteral two = new IntegerLiteral(2);
Assertions.assertNotEquals(one, two);
Assertions.assertEquals(one, anotherOne);
StringLiteral str1 = new StringLiteral("hello");
Assertions.assertNotEquals(str1, one);
Assertions.assertTrue(Literal.of("world") instanceof StringLiteral);
Assertions.assertTrue(Literal.of(null) instanceof NullLiteral);
Assertions.assertTrue(Literal.of(1) instanceof IntegerLiteral);
Assertions.assertTrue(Literal.of(false) instanceof BooleanLiteral);
}
}

83
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteralTest.java Normal file
View File

@ -0,0 +1,83 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions.literal;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
class DateLiteralTest {
@Test
void testDate() {
new DateLiteral("220101");
new DateLiteral("22-01-01");
new DateLiteral("2022-01-01");
new DateLiteral("20220101");
Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("-01-01"));
}
@Test
void testZone() {
new DateLiteral("2022-01-01Z");
new DateLiteral("2022-01-01UTC");
new DateLiteral("2022-01-01GMT");
// new DateLiteral("2022-01-01UTC+08");
// new DateLiteral("2022-01-01UTC-06");
new DateLiteral("2022-01-01UTC+08:00");
new DateLiteral("2022-01-01UTC-06:00");
new DateLiteral("2022-01-01Europe/London");
}
@Test
@Disabled
void testOffset() {
new DateLiteral("2022-01-01+01:00:00");
new DateLiteral("2022-01-01+01:00");
new DateLiteral("2022-01-01+01");
new DateLiteral("2022-01-01+1:0:0");
new DateLiteral("2022-01-01+1:0");
new DateLiteral("2022-01-01+1");
new DateLiteral("2022-01-01-01:00:00");
new DateLiteral("2022-01-01-01:00");
new DateLiteral("2022-01-01-1:0:0");
new DateLiteral("2022-01-01-1:0");
Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-01"));
Assertions.assertThrows(AnalysisException.class, () -> new DateLiteral("2022-01-01-1"));
}
@Disabled
@Test
void testIrregularDate() {
new DateLiteral("2016-07-02");
new DateLiteral("2016-7-02");
new DateLiteral("2016-07-2");
new DateLiteral("2016-7-2");
new DateLiteral("2016-07-02");
new DateLiteral("2016-07-2");
new DateLiteral("2016-7-02");
new DateLiteral("2016-7-2");
}
}

369
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java Normal file
View File

@ -0,0 +1,369 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions.literal;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
class DateTimeLiteralTest {
@Test
void testWithoutZoneOrOffset() {
new DateTimeV2Literal("2022-08-01");
new DateTimeV2Literal("2022-08-01 01:01:01");
new DateTimeV2Literal("2022-08-01 01:01");
new DateTimeV2Literal("2022-08-01 01");
new DateTimeV2Literal("2022-08-01T01:01:01");
new DateTimeV2Literal("2022-08-01T01:01");
new DateTimeV2Literal("2022-08-01T01");
new DateTimeV2Literal("22-08-01T01:01:01");
new DateTimeV2Literal("22-08-01T01:01");
new DateTimeV2Literal("22-08-01T01");
}
@Test
void testDetermineScale() {
int scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.0");
Assertions.assertEquals(0, scale);
scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.00000");
Assertions.assertEquals(0, scale);
scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.000001");
Assertions.assertEquals(6, scale);
scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.123456");
Assertions.assertEquals(6, scale);
scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.0001");
Assertions.assertEquals(4, scale);
scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.00010");
Assertions.assertEquals(4, scale);
scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.12010");
Assertions.assertEquals(4, scale);
scale = DateTimeLiteral.determineScale("2022-08-01T01:01:01.02010");
Assertions.assertEquals(4, scale);
}
@Test
void testTwoDigitYear() {
new DateTimeV2Literal("22-08-01T01");
new DateTimeV2Literal("22-08-01 01");
new DateTimeV2Literal("22-08-01T01:01");
new DateTimeV2Literal("22-08-01 01:01");
new DateTimeV2Literal("22-08-01T01:01:01");
new DateTimeV2Literal("22-08-01 01:01:01");
new DateTimeV2Literal("22-08-01T01");
new DateTimeV2Literal("22-08-01 01");
new DateTimeV2Literal("22-08-01T01:01");
new DateTimeV2Literal("22-08-01 01:01");
new DateTimeV2Literal("22-08-01T01:01:01");
new DateTimeV2Literal("22-08-01 01:01:01");
}
@Test
void testZone() {
new DateTimeV2Literal("2022-08-01 01:01:01UTC");
new DateTimeV2Literal("2022-08-01 01:01:01UT");
new DateTimeV2Literal("2022-08-01 01:01:01GMT");
new DateTimeV2Literal("2022-08-01 01:01:01Z");
new DateTimeV2Literal("2022-08-01 01:01:01Europe/London");
new DateTimeV2Literal("2022-08-01 01:01:01America/New_York");
new DateTimeV2Literal("2022-08-01 01:01:01Z");
new DateTimeV2Literal("2022-08-01 01:01:01Europe/Berlin");
new DateTimeV2Literal("2022-08-01 01:01:01Europe/London");
}
@Test
void testZoneOrOffsetRight() {
java.util.function.BiConsumer<DateTimeV2Literal, Long> assertHour = (dateTimeV2Literal, expectHour) -> {
Assertions.assertSame(dateTimeV2Literal.hour, expectHour);
};
DateTimeV2Literal dateTimeV2Literal;
dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00Europe/London"); // +01:00
assertHour.accept(dateTimeV2Literal, 7L);
dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00America/New_York"); // -04:00
assertHour.accept(dateTimeV2Literal, 12L);
dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00Asia/Shanghai");
assertHour.accept(dateTimeV2Literal, 0L);
dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00+01:00");
assertHour.accept(dateTimeV2Literal, 7L);
dateTimeV2Literal = new DateTimeV2Literal("2022-08-01 00:00:00-01:00");
assertHour.accept(dateTimeV2Literal, 9L);
}
@Test
void testTwoDigitalYearZone() {
new DateTimeV2Literal("22-08-01 01:01:01UTC");
new DateTimeV2Literal("22-08-01 01:01:01UT");
new DateTimeV2Literal("22-08-01 01:01:01GMT");
new DateTimeV2Literal("22-08-01 01:01:01Z");
new DateTimeV2Literal("22-08-01 01:01:01Europe/London");
new DateTimeV2Literal("22-08-01 01:01:01UTC");
new DateTimeV2Literal("22-08-01 01:01:01America/New_York");
new DateTimeV2Literal("22-08-01 01:01:01Z");
new DateTimeV2Literal("22-08-01 01:01:01Europe/Berlin");
new DateTimeV2Literal("22-08-01 01:01:01Europe/London");
}
@Test
void testZoneOffset() {
new DateTimeV2Literal("2022-08-01 01:01:01UTC+01:01:01");
// new DateTimeV2Literal("2022-08-01 01:01:01UTC+1:1:1");
new DateTimeV2Literal("2022-08-01 01:01:01UTC+01:01");
// new DateTimeV2Literal("2022-08-01 01:01:01UTC+01");
// new DateTimeV2Literal("2022-08-01 01:01:01UTC+1");
}
@Test
void testTwoDigitalYearZoneOffset() {
new DateTimeV2Literal("22-08-01 01:01:01UTC+01:01:01");
// new DateTimeV2Literal("22-08-01 01:01:01UTC+1:1:1");
new DateTimeV2Literal("22-08-01 01:01:01UTC+01:01");
// new DateTimeV2Literal("22-08-01 01:01:01UTC+01");
// new DateTimeV2Literal("22-08-01 01:01:01UTC+1");
}
@Test
void testOffset() {
new DateTimeV2Literal("2022-08-01 01:01:01+01:01:01");
new DateTimeV2Literal("2022-08-01 01:01:01+01:01");
// new DateTimeV2Literal("2022-08-01 01:01:01+01");
// new DateTimeV2Literal("2022-08-01 01:01:01+01:1:01");
// new DateTimeV2Literal("2022-08-01 01:01:01+01:1");
// new DateTimeV2Literal("2022-08-01 01:01:01+01:01:1");
// new DateTimeV2Literal("2022-08-01 01:01:01+1:1:1");
// new DateTimeV2Literal("2022-08-01 01:01:01+1:1");
// new DateTimeV2Literal("2022-08-01 01:01:01+1");
new DateTimeV2Literal("2022-05-01 01:02:55+02:30");
new DateTimeV2Literal("2022-05-01 01:02:55.123-02:30");
new DateTimeV2Literal("2022-06-01T01:02:55+04:30");
new DateTimeV2Literal("2022-06-01 01:02:55.123-07:30");
// new DateTimeV2Literal("20220701010255+07:00");
// new DateTimeV2Literal("20220701010255-05:00");
new DateTimeV2Literal("2022-05-01 01:02:55+02:30");
new DateTimeV2Literal("2022-05-01 01:02:55.123-02:30");
new DateTimeV2Literal("2022-06-01T01:02:55+04:30");
new DateTimeV2Literal("2022-06-01 01:02:55.123-07:30");
// new DateTimeV2Literal("20220701010255+07:00");
// new DateTimeV2Literal("20220701010255-05:00");
}
@Test
void testDateTime() {
// new DateTimeV2Literal("2022-08-01 01:01:01UTC+1:1:1");
// new DateTimeV2Literal("2022-08-01 01:01:01UTC+1:1");
// new DateTimeV2Literal("2022-08-01 01:01:01UTC+1");
new DateTimeV2Literal("0001-01-01 00:01:01");
new DateTimeV2Literal("0001-01-01 00:01:01.001");
new DateTimeV2Literal("0001-01-01 00:01:01.00305");
new DateTimeV2Literal("2022-01-01 01:02:55");
new DateTimeV2Literal("2022-01-01 01:02:55.123");
new DateTimeV2Literal("2022-02-01 01:02:55Z");
new DateTimeV2Literal("2022-02-01 01:02:55.123Z");
// new DateTimeV2Literal("2022-03-01 01:02:55UTC+8");
new DateTimeV2Literal("2022-03-01 01:02:55.123UTC");
// new DateTimeV2Literal("2022-04-01 01:02:55UTC-6");
// new DateTimeV2Literal("2022-04-01T01:02:55UTC-6");
// new DateTimeV2Literal("2022-04-01T01:02:55.123UTC+6");
new DateTimeV2Literal("2022-01-01 01:02:55");
new DateTimeV2Literal("2022-01-01 01:02:55.123");
new DateTimeV2Literal("2022-02-01 01:02:55Z");
new DateTimeV2Literal("2022-02-01 01:02:55.123Z");
// new DateTimeV2Literal("2022-03-01 01:02:55UTC+8");
new DateTimeV2Literal("2022-03-01 01:02:55.123UTC");
// new DateTimeV2Literal("2022-04-01T01:02:55UTC-6");
// new DateTimeV2Literal("2022-04-01T01:02:55.123UTC+6");
new DateTimeV2Literal("0001-01-01");
// new DateTimeV2Literal("20220801GMT+5");
// new DateTimeV2Literal("20220801GMT-3");
}
@Disabled
@Test
void testIrregularDateTime() {
new DateLiteral("2016-07-02 01:01:00");
new DateLiteral("2016-7-02 01:01:00");
new DateLiteral("2016-07-2 01:01:00");
new DateLiteral("2016-7-2 01:01:00");
new DateLiteral("2016-07-02 1:01:00");
new DateLiteral("2016-07-02 01:1:00");
new DateLiteral("2016-07-02 01:01:0");
new DateLiteral("2016-07-02 1:1:00");
new DateLiteral("2016-07-02 1:01:0");
new DateLiteral("2016-07-02 10:1:0");
new DateLiteral("2016-07-02 1:1:0");
new DateLiteral("2016-7-2 1:1:0");
new DateLiteral("2016-7-02 1:01:0");
new DateLiteral("2016-07-2 1:1:0");
new DateLiteral("2016-7-02 01:01:0");
new DateLiteral("2016-7-2 01:1:0");
}
@Disabled
@Test
void testIrregularDateTimeHour() {
new DateTimeV2Literal("2016-07-02 01");
new DateTimeV2Literal("2016-07-02 1");
new DateTimeV2Literal("2016-7-02 1");
new DateTimeV2Literal("2016-7-02 01");
new DateTimeV2Literal("2016-07-2 1");
new DateTimeV2Literal("2016-07-2 01");
new DateTimeV2Literal("2016-7-2 1");
new DateTimeV2Literal("2016-7-2 01");
}
@Disabled
@Test
void testIrregularDateTimeHourMinute() {
new DateTimeV2Literal("2016-07-02 01:01");
new DateTimeV2Literal("2016-07-02 1:01");
new DateTimeV2Literal("2016-07-02 01:1");
new DateTimeV2Literal("2016-07-02 1:1");
new DateTimeV2Literal("2016-7-02 01:01");
new DateTimeV2Literal("2016-7-02 1:01");
new DateTimeV2Literal("2016-7-02 01:1");
new DateTimeV2Literal("2016-7-02 1:1");
new DateTimeV2Literal("2016-07-2 01:01");
new DateTimeV2Literal("2016-07-2 1:01");
new DateTimeV2Literal("2016-07-2 01:1");
new DateTimeV2Literal("2016-07-2 1:1");
new DateTimeV2Literal("2016-7-2 01:01");
new DateTimeV2Literal("2016-7-2 1:01");
new DateTimeV2Literal("2016-7-2 01:1");
new DateTimeV2Literal("2016-7-2 1:1");
}
@Disabled
@Test
void testIrregularDateTimeHourMinuteSecond() {
new DateTimeV2Literal("2016-07-02 01:01:01");
new DateTimeV2Literal("2016-07-02 1:01:01");
new DateTimeV2Literal("2016-07-02 01:1:01");
new DateTimeV2Literal("2016-07-02 1:1:01");
new DateTimeV2Literal("2016-07-02 01:01:1");
new DateTimeV2Literal("2016-07-02 1:01:1");
new DateTimeV2Literal("2016-07-02 01:1:1");
new DateTimeV2Literal("2016-07-02 1:1:1");
new DateTimeV2Literal("2016-7-02 01:01:01");
new DateTimeV2Literal("2016-7-02 1:01:01");
new DateTimeV2Literal("2016-7-02 01:1:01");
new DateTimeV2Literal("2016-7-02 1:1:01");
new DateTimeV2Literal("2016-7-02 01:01:1");
new DateTimeV2Literal("2016-7-02 1:01:1");
new DateTimeV2Literal("2016-7-02 01:1:1");
new DateTimeV2Literal("2016-7-02 1:1:1");
new DateTimeV2Literal("2016-07-2 01:01:01");
new DateTimeV2Literal("2016-07-2 1:01:01");
new DateTimeV2Literal("2016-07-2 01:1:01");
new DateTimeV2Literal("2016-07-2 1:1:01");
new DateTimeV2Literal("2016-07-2 01:01:1");
new DateTimeV2Literal("2016-07-2 1:01:1");
new DateTimeV2Literal("2016-07-2 01:1:1");
new DateTimeV2Literal("2016-07-2 1:1:1");
new DateTimeV2Literal("2016-7-2 01:01:01");
new DateTimeV2Literal("2016-7-2 1:01:01");
new DateTimeV2Literal("2016-7-2 01:1:01");
new DateTimeV2Literal("2016-7-2 1:1:01");
new DateTimeV2Literal("2016-7-2 01:01:1");
new DateTimeV2Literal("2016-7-2 1:01:1");
new DateTimeV2Literal("2016-7-2 01:1:1");
new DateTimeV2Literal("2016-7-2 1:1:1");
}
@Disabled
@Test
void testIrregularDateTimeHourMinuteSecondMicrosecond() {
new DateTimeV2Literal("2016-07-02 01:01:01.1");
new DateTimeV2Literal("2016-07-02 1:01:01.1");
new DateTimeV2Literal("2016-07-02 01:1:01.1");
new DateTimeV2Literal("2016-07-02 1:1:01.1");
new DateTimeV2Literal("2016-07-02 01:01:1.1");
new DateTimeV2Literal("2016-07-02 1:01:1.1");
new DateTimeV2Literal("2016-07-02 01:1:1.1");
new DateTimeV2Literal("2016-07-02 1:1:1.1");
new DateTimeV2Literal("2016-7-02 01:01:01.1");
new DateTimeV2Literal("2016-7-02 1:01:01.1");
new DateTimeV2Literal("2016-7-02 01:1:01.1");
new DateTimeV2Literal("2016-7-02 1:1:01.1");
new DateTimeV2Literal("2016-7-02 01:01:1.1");
new DateTimeV2Literal("2016-7-02 1:01:1.1");
new DateTimeV2Literal("2016-7-02 01:1:1.1");
new DateTimeV2Literal("2016-7-02 1:1:1.1");
new DateTimeV2Literal("2016-07-2 01:01:01.1");
new DateTimeV2Literal("2016-07-2 1:01:01.1");
new DateTimeV2Literal("2016-07-2 01:1:01.1");
new DateTimeV2Literal("2016-07-2 1:1:01.1");
new DateTimeV2Literal("2016-07-2 01:01:1.1");
new DateTimeV2Literal("2016-07-2 1:01:1.1");
new DateTimeV2Literal("2016-07-2 01:1:1.1");
new DateTimeV2Literal("2016-07-2 1:1:1.1");
new DateTimeV2Literal("2016-7-2 01:01:01.1");
new DateTimeV2Literal("2016-7-2 1:01:01.1");
new DateTimeV2Literal("2016-7-2 01:1:01.1");
new DateTimeV2Literal("2016-7-2 1:1:01.1");
new DateTimeV2Literal("2016-7-2 01:01:1.1");
new DateTimeV2Literal("2016-7-2 1:01:1.1");
new DateTimeV2Literal("2016-7-2 01:1:1.1");
new DateTimeV2Literal("2016-7-2 1:1:1.1");
// Testing with microsecond of length 2
new DateTimeV2Literal("2016-07-02 01:01:01.12");
new DateTimeV2Literal("2016-7-02 01:01:01.12");
// Testing with microsecond of length 3
new DateTimeV2Literal("2016-07-02 01:01:01.123");
new DateTimeV2Literal("2016-7-02 01:01:01.123");
// Testing with microsecond of length 4
new DateTimeV2Literal("2016-07-02 01:01:01.1234");
new DateTimeV2Literal("2016-7-02 01:01:01.1234");
// Testing with microsecond of length 5
new DateTimeV2Literal("2016-07-02 01:01:01.12345");
new DateTimeV2Literal("2016-7-02 01:01:01.12345");
// Testing with microsecond of length 6
new DateTimeV2Literal("2016-07-02 01:01:01.123456");
new DateTimeV2Literal("2016-7-02 01:01:01.123456");
}
}

81
fe/fe-core/src/test/java/org/apache/doris/nereids/util/DateTimeFormatterUtilsTest.java
View File

@ -24,8 +24,25 @@ import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.util.function.Consumer;
class DateTimeFormatterUtilsTest {
@Test
void test() {
DateTimeFormatter formatter = DateTimeFormatterUtils.ZONE_FORMATTER;
formatter.parse("");
// formatter.parse("UTC+01");
formatter.parse("UTC+01:00");
formatter.parse("UTC+01:00:00");
// formatter.parse("GMT+01");
formatter.parse("GMT+01:00");
formatter.parse("Asia/Shanghai");
formatter.parse("Z");
}
private void assertDatePart(TemporalAccessor dateTime) {
Assertions.assertEquals(2020, dateTime.get(ChronoField.YEAR));
Assertions.assertEquals(2, dateTime.get(ChronoField.MONTH_OF_YEAR));
@ -34,29 +51,41 @@ class DateTimeFormatterUtilsTest {
@Test
void testBasicDateTimeFormatter() {
DateTimeFormatter formatter = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER;
DateTimeFormatter formatter = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T;
TemporalAccessor dateTime = formatter.parse("20200219");
assertDatePart(dateTime);
dateTime = formatter.parse("20200219010101");
assertDatePart(dateTime);
dateTime = formatter.parse("20200219T010101");
dateTime = formatter.parse("20200219010101.1");
assertDatePart(dateTime);
// failed case
Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219 010101"));
// microsecond
dateTime = formatter.parse("20200219010101.000001");
assertDatePart(dateTime);
dateTime = formatter.parse("20200219T010101.000001");
assertDatePart(dateTime);
dateTime = formatter.parse("20200219010101.1");
assertDatePart(dateTime);
formatter = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER;
dateTime = formatter.parse("20200219T010101");
assertDatePart(dateTime);
dateTime = formatter.parse("20200219T010101.1");
assertDatePart(dateTime);
Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219010101.0000001"));
Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219T010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("20200219T010101.0000001"));
dateTime = formatter.parse("20200219T010101.000001");
assertDatePart(dateTime);
dateTime = formatter.parse("20200219T010101.1");
assertDatePart(dateTime);
// failed case
DateTimeFormatter withT = DateTimeFormatterUtils.BASIC_DATE_TIME_FORMATTER;
Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219 010101"));
Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219010101.0000001"));
Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> withT.parse("20200219T010101.0000001"));
DateTimeFormatter withoutT = DateTimeFormatterUtils.BASIC_FORMATTER_WITHOUT_T;
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219 010101"));
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219010101.0000001"));
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101."));
Assertions.assertThrows(DateTimeParseException.class, () -> withoutT.parse("20200219T010101.0000001"));
}
@Test
@ -79,4 +108,32 @@ class DateTimeFormatterUtilsTest {
}
}
}
@Test
void testDateTimeFormatter() {
DateTimeFormatter formatter = DateTimeFormatterUtils.DATE_TIME_FORMATTER;
TemporalAccessor dateTime = formatter.parse("2020-02-19 01:01:01");
assertDatePart(dateTime);
Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("2020-02-19T01:01:01"));
Assertions.assertThrows(DateTimeParseException.class, () -> formatter.parse("2020-02-1901:01:01"));
}
@Test
void testTimeFormatter() {
// use lambda function to assert time is correct.
Consumer<TemporalAccessor> assertTime = (dateTime) ->
Assertions.assertEquals(1, dateTime.get(ChronoField.HOUR_OF_DAY));
DateTimeFormatter timeFormatter = DateTimeFormatterUtils.TIME_FORMATTER;
TemporalAccessor dateTime = timeFormatter.parse("01:01:01.000001");
assertTime.accept(dateTime);
dateTime = timeFormatter.parse("01:01:01.1");
assertTime.accept(dateTime);
dateTime = timeFormatter.parse("01:01:01");
assertTime.accept(dateTime);
dateTime = timeFormatter.parse("01:01");
assertTime.accept(dateTime);
dateTime = timeFormatter.parse("01");
assertTime.accept(dateTime);
}
}