database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip] (datetimev2) support cast between datetimev2 with different scales (#11198)

Browse Source 
* [feature-wip] (datetimev2) support `cast` between datetimev2 with different scale
This commit is contained in:
Gabriel
2022-07-26 22:36:13 +08:00
committed by GitHub
parent 166b4a18ae
commit d67029c830
18 changed files with 210 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
View File

@ -31,6 +31,7 @@ import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -46,10 +47,12 @@ import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.format.TextStyle;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TimeZone;
@ -74,6 +77,7 @@ public class DateLiteral extends LiteralExpr {
private static DateTimeFormatter DATE_TIME_FORMATTER = null;
private static DateTimeFormatter DATE_TIME_FORMATTER_TO_MICRO_SECOND = null;
private static DateTimeFormatter DATE_FORMATTER = null;
private static List<DateTimeFormatter> formatterList = null;
/*
* The datekey type is widely used in data warehouses
* For example, 20121229 means '2012-12-29'
@ -99,8 +103,18 @@ public class DateLiteral extends LiteralExpr {
DATETIMEKEY_FORMATTER = formatBuilder("%Y%m%d%H%i%s").toFormatter();
DATE_TIME_FORMATTER_TO_MICRO_SECOND = new DateTimeFormatterBuilder()
.appendPattern("uuuu-MM-dd HH:mm:ss")
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 6, true)
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true)
.toFormatter();
formatterList = Lists.newArrayList(
formatBuilder("%Y%m%d").appendLiteral('T').appendPattern("HHmmss")
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true).toFormatter(),
formatBuilder("%Y%m%d").appendLiteral('T').appendPattern("HHmmss")
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, false).toFormatter(),
formatBuilder("%Y%m%d%H%i%s")
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true).toFormatter(),
formatBuilder("%Y%m%d%H%i%s")
.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, false).toFormatter(),
DATETIMEKEY_FORMATTER, DATEKEY_FORMATTER);
} catch (AnalysisException e) {
LOG.error("invalid date format", e);
System.exit(-1);
@ -321,13 +335,22 @@ public class DateLiteral extends LiteralExpr {
private void init(String s, Type type) throws AnalysisException {
try {
Preconditions.checkArgument(type.isDateType());
TemporalAccessor dateTime;
if (s.length() == DATEKEY_LENGTH && !s.contains("-")) {
TemporalAccessor dateTime = null;
boolean parsed = false;
if (!s.contains("-")) {
// handle format like 20210106, but should not handle 2021-1-6
dateTime = DATEKEY_FORMATTER.parse(s);
} else if (s.length() == DATETIMEKEY_LENGTH && !s.contains("-")) {
// handle format like 20210106, but should not handle 2021-1-6
dateTime = DATETIMEKEY_FORMATTER.parse(s);
for (DateTimeFormatter formatter : formatterList) {
try {
dateTime = formatter.parse(s);
parsed = true;
break;
} catch (DateTimeParseException ex) {
// ignore
}
}
if (!parsed) {
throw new AnalysisException("Invalid date value: " + s);
}
} else {
String[] datePart = s.contains(" ") ? s.split(" ")[0].split("-") : s.split("-");
DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
@ -380,7 +403,7 @@ public class DateLiteral extends LiteralExpr {
builder.appendPattern(String.join("", Collections.nCopies(timePart[i].contains(".")
? timePart[i].split("\\.")[0].length() : timePart[i].length(), "s")));
if (timePart[i].contains(".")) {
builder.appendFraction(ChronoField.NANO_OF_SECOND, 0, 6, true);
builder.appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true);
}
break;
default:
@ -392,8 +415,10 @@ public class DateLiteral extends LiteralExpr {
}
DateTimeFormatter formatter = builder.toFormatter();
dateTime = formatter.parse(s);
parsed = true;
}
Preconditions.checkArgument(parsed);
year = getOrDefault(dateTime, ChronoField.YEAR, 0);
month = getOrDefault(dateTime, ChronoField.MONTH_OF_YEAR, 0);
day = getOrDefault(dateTime, ChronoField.DAY_OF_MONTH, 0);
@ -402,10 +427,9 @@ public class DateLiteral extends LiteralExpr {
second = getOrDefault(dateTime, ChronoField.SECOND_OF_MINUTE, 0);
microsecond = getOrDefault(dateTime, ChronoField.MICRO_OF_SECOND, 0);
if (type.isDatetimeV2()) {
this.type = ScalarType.createDatetimeV2Type(6);
} else {
this.type = type;
this.roundFloor(((ScalarType) type).getScalarScale());
}
this.type = type;
} catch (Exception ex) {
throw new AnalysisException("date literal [" + s + "] is invalid: " + ex.getMessage());
}
@ -513,14 +537,14 @@ public class DateLiteral extends LiteralExpr {
long remain = Double.valueOf(microsecond % (Math.pow(10, 6 - newScale))).longValue();
if (remain != 0) {
microsecond = Double.valueOf((microsecond + (Math.pow(10, 6 - newScale)))
/ (Math.pow(10, 6 - newScale))).longValue();
/ (Math.pow(10, 6 - newScale)) * (Math.pow(10, 6 - newScale))).longValue();
}
type = ScalarType.createDatetimeV2Type(newScale);
}
public void roundFloor(int newScale) {
Preconditions.checkArgument(type.isDatetimeV2());
microsecond = Double.valueOf(microsecond / (Math.pow(10, 6 - newScale))).longValue();
microsecond = Double.valueOf(microsecond / (Math.pow(10, 6 - newScale))
* (Math.pow(10, 6 - newScale))).longValue();
type = ScalarType.createDatetimeV2Type(newScale);
}
@ -532,8 +556,12 @@ public class DateLiteral extends LiteralExpr {
if (type == PrimitiveType.DATE || type == PrimitiveType.DATEV2) {
return String.format("%04d-%02d-%02d", year, month, day);
} else if (type == PrimitiveType.DATETIMEV2) {
return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d",
year, month, day, hour, minute, second, microsecond);
String tmp = String.format("%04d-%02d-%02d %02d:%02d:%02d",
year, month, day, hour, minute, second);
if (microsecond == 0) {
return tmp;
}
return tmp + String.format(".%06d", microsecond);
} else {
return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second);
}
@ -673,8 +701,7 @@ public class DateLiteral extends LiteralExpr {
this.type = Type.DATE;
} else if (dateLiteralType == DateLiteralType.DATETIMEV2.value()) {
fromPackedDatetime(in.readLong());
int scale = in.readInt();
this.type = ScalarType.createDatetimeV2Type(scale);
this.type = ScalarType.createDatetimeV2Type(in.readInt());
} else if (dateLiteralType == DateLiteralType.DATEV2.value()) {
fromPackedDatetime(in.readLong());
this.type = Type.DATEV2;
@ -782,6 +809,7 @@ public class DateLiteral extends LiteralExpr {
builder.appendValue(ChronoField.ALIGNED_WEEK_OF_YEAR, 2);
break;
case 'x':
case 'Y': // %Y Year, numeric, four digits
// %x Year for the week, where Monday is the first day of the week,
// numeric, four digits; used with %v
builder.appendValue(ChronoField.YEAR, 4);
@ -789,9 +817,6 @@ public class DateLiteral extends LiteralExpr {
case 'W': // %W Weekday name (Sunday..Saturday)
builder.appendText(ChronoField.DAY_OF_WEEK, TextStyle.FULL);
break;
case 'Y': // %Y Year, numeric, four digits
builder.appendPattern("uuuu");
break;
case 'y': // %y Year, numeric (two digits)
builder.appendValueReduced(ChronoField.YEAR, 2, 2, 1970);
break;
@ -843,7 +868,7 @@ public class DateLiteral extends LiteralExpr {
final int hour = getOrDefault(accessor, ChronoField.HOUR_OF_DAY, 0);
final int minute = getOrDefault(accessor, ChronoField.MINUTE_OF_HOUR, 0);
final int second = getOrDefault(accessor, ChronoField.SECOND_OF_MINUTE, 0);
final int microSeconds = getOrDefault(accessor, ChronoField.NANO_OF_SECOND, 0);
final int microSeconds = getOrDefault(accessor, ChronoField.MICRO_OF_SECOND, 0);
return LocalDateTime.of(year, month, dayOfMonth, hour, minute, second, microSeconds);
}

1
fe/fe-core/src/main/java/org/apache/doris/catalog/ScalarType.java
View File

@ -78,6 +78,7 @@ public class ScalarType extends Type {
public static final int MAX_DECIMAL32_PRECISION = 9;
public static final int MAX_DECIMAL64_PRECISION = 18;
public static final int MAX_DECIMAL128_PRECISION = 38;
public static final int MAX_DATETIMEV2_SCALE = 6;
private static final Logger LOG = LogManager.getLogger(ScalarType.class);
@SerializedName(value = "type")

4
fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
View File

@ -104,9 +104,11 @@ public class PartitionsProcDir implements ProcDirInterface {
type = Type.DATETIME;
break;
case DATEV2:
case DATETIMEV2:
type = Type.DATETIMEV2;
break;
case DATETIMEV2:
type = subExpr.getChild(1).getType();
break;
default:
throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType());
}

4
fe/fe-core/src/main/java/org/apache/doris/common/proc/RollupProcDir.java
View File

@ -161,9 +161,11 @@ public class RollupProcDir implements ProcDirInterface {
type = Type.DATETIME;
break;
case DATEV2:
case DATETIMEV2:
type = Type.DATETIMEV2;
break;
case DATETIMEV2:
type = subExpr.getChild(1).getType();
break;
default:
throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType());
}

4
fe/fe-core/src/main/java/org/apache/doris/common/proc/SchemaChangeProcDir.java
View File

@ -80,9 +80,11 @@ public class SchemaChangeProcDir implements ProcDirInterface {
type = Type.DATETIME;
break;
case DATEV2:
case DATETIMEV2:
type = Type.DATETIMEV2;
break;
case DATETIMEV2:
type = subExpr.getChild(1).getType();
break;
default:
throw new AnalysisException("Invalid date type: " + subExpr.getChild(1).getType());
}

2
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -163,7 +163,7 @@ public class PropertyAnalyzer {
} else if (key.equalsIgnoreCase(PROPERTIES_REMOTE_STORAGE_POLICY)) {
remoteStoragePolicy = value;
} else if (key.equalsIgnoreCase(PROPERTIES_DATA_BASE_TIME)) {
DateLiteral dateLiteral = new DateLiteral(value, Type.DATETIME);
DateLiteral dateLiteral = new DateLiteral(value, DateLiteral.getDefaultDateType(Type.DATETIME));
dataBaseTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone());
} else if (!hasStoragePolicy && key.equalsIgnoreCase(PROPERTIES_STORAGE_POLICY)) {
if (!Strings.isNullOrEmpty(value)) {

10
fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java
View File

@ -37,6 +37,7 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
@ -583,12 +584,17 @@ public class DeleteHandler implements Writable {
}
} else if (column.getDataType() == PrimitiveType.DATE
|| column.getDataType() == PrimitiveType.DATETIME
|| column.getDataType() == PrimitiveType.DATEV2
|| column.getDataType() == PrimitiveType.DATETIMEV2) {
|| column.getDataType() == PrimitiveType.DATEV2) {
DateLiteral dateLiteral = new DateLiteral(value, Type.fromPrimitiveType(column.getDataType()));
value = dateLiteral.getStringValue();
binaryPredicate.setChild(1, LiteralExpr.create(value,
Type.fromPrimitiveType(column.getDataType())));
} else if (column.getDataType() == PrimitiveType.DATETIMEV2) {
DateLiteral dateLiteral = new DateLiteral(value,
ScalarType.createDecimalType(ScalarType.MAX_DATETIMEV2_SCALE));
value = dateLiteral.getStringValue();
binaryPredicate.setChild(1, LiteralExpr.create(value,
ScalarType.createDecimalType(ScalarType.MAX_DATETIMEV2_SCALE)));
}
LiteralExpr.create(value, Type.fromPrimitiveType(column.getDataType()));
} catch (AnalysisException e) {

2
fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionRange.java vendored
View File

@ -459,7 +459,7 @@ public class PartitionRange {
LiteralExpr newLiteral;
if (key.keyType == KeyType.DATE) {
try {
newLiteral = new DateLiteral(key.toString(), Type.DATE);
newLiteral = new DateLiteral(key.toString(), DateLiteral.getDefaultDateType(Type.DATE));
} catch (Exception e) {
LOG.warn("Date's format is error {},{}", key.toString(), e);
continue;

6
fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
View File

@ -254,16 +254,16 @@ public class DateLiteralTest {
public void testDateFormatForDatetimeV2() {
boolean hasException = false;
try {
DateLiteral literal = new DateLiteral("1997-10-7 00:00:00.123456", Type.DATETIMEV2);
DateLiteral literal = new DateLiteral("1997-10-7 00:00:00.123456", ScalarType.createDatetimeV2Type(6));
Assert.assertEquals(1997, literal.getYear());
Assert.assertEquals(123456, literal.getMicrosecond());
literal = new DateLiteral("2021-06-1 00:00:00.123456", Type.DATETIMEV2);
literal = new DateLiteral("2021-06-1 00:00:00.123456", ScalarType.createDatetimeV2Type(6));
Assert.assertEquals(2021, literal.getYear());
Assert.assertEquals(6, literal.getMonth());
Assert.assertEquals(1, literal.getDay());
literal = new DateLiteral("2022-6-01 00:00:00.123456", Type.DATETIMEV2);
literal = new DateLiteral("2022-6-01 00:00:00.123456", ScalarType.createDatetimeV2Type(6));
Assert.assertEquals(2022, literal.getYear());
Assert.assertEquals(6, literal.getMonth());
Assert.assertEquals(1, literal.getDay());

8
fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
View File

@ -2012,10 +2012,10 @@ public class QueryPlanTest extends TestWithFeService {
rewriteDateLiteralRuleTest.testWithIntFormatDate();
rewriteDateLiteralRuleTest.testWithInvalidFormatDate();
rewriteDateLiteralRuleTest.testWithStringFormatDate();
// rewriteDateLiteralRuleTest.testWithDoubleFormatDateV2();
// rewriteDateLiteralRuleTest.testWithIntFormatDateV2();
// rewriteDateLiteralRuleTest.testWithInvalidFormatDateV2();
// rewriteDateLiteralRuleTest.testWithStringFormatDateV2();
rewriteDateLiteralRuleTest.testWithDoubleFormatDateV2();
rewriteDateLiteralRuleTest.testWithIntFormatDateV2();
rewriteDateLiteralRuleTest.testWithInvalidFormatDateV2();
rewriteDateLiteralRuleTest.testWithStringFormatDateV2();
rewriteDateLiteralRuleTest.after();
}
// --end--

28
fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java
View File

@ -64,13 +64,13 @@ public class RewriteDateLiteralRuleTest {
public void testWithIntFormatDateV2() throws Exception {
String query = "select * from " + DB_NAME + ".tb2 where k1 > 20210301";
String planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'"));
query = "select k1 > 20210301 from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'"));
query = "select k1 > 20210301223344 from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:33:44.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:33:44'"));
}
public void testWithStringFormatDate() throws Exception {
@ -122,47 +122,47 @@ public class RewriteDateLiteralRuleTest {
public void testWithStringFormatDateV2() throws Exception {
String query = "select * from " + DB_NAME + ".tb2 where k1 > '2021030112334455'";
String planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 12:33:44.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 12:33:44.550000'"));
query = "select k1 > '20210301' from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'"));
query = "select k1 > '20210301233234.34' from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 23:32:34.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 23:32:34.340000'"));
query = "select * from " + DB_NAME + ".tb2 where k1 > '2021-03-01'";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00'"));
query = "select k1 > '2021-03-01 11:22:33' from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:33'"));
query = "select k1 > '2021-03-01 16:22:33' from " + DB_NAME + ".tb2";
query = "select k1 > '2021-03-01 16:22:33' from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 16:22:33'"));
query = "select k1 > '2021-03-01 11:22' from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:00.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:00'"));
query = "select k1 > '20210301T221133' from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:11:33.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 22:11:33'"));
query = "select k1 > '2021-03-01dd 11:22' from " + DB_NAME + ".tb2";
query = "select k1 > '2021-03-01 11:22' from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 00:00:00.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2021-03-01 11:22:00'"));
query = "select k1 > '80-03-01 11:22' from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '1980-03-01 11:22:00.000000'"));
Assert.assertTrue(planString.contains("`k1` > '1980-03-01 11:22:00'"));
query = "select k1 > '12-03-01 11:22' from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
Assert.assertTrue(planString.contains("`k1` > '2012-03-01 11:22:00.000000'"));
Assert.assertTrue(planString.contains("`k1` > '2012-03-01 11:22:00'"));
}
public void testWithDoubleFormatDate() throws Exception {