database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](nereids)create datev2 and datetimev2 literal if enable_date_conversion is true (#21065)

Browse Source
This commit is contained in:
starocean999
2023-06-21 20:29:36 +08:00
committed by GitHub
parent 442a734ef5
commit 8b561cfb03
8 changed files with 28 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -193,6 +193,7 @@ import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal;
@ -1020,7 +1021,8 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
@Override
public Expression visitCast(DorisParser.CastContext ctx) {
DataType dataType = typedVisit(ctx.dataType());
List<String> types = typedVisit(ctx.dataType());
DataType dataType = DataType.convertPrimitiveFromStrings(types, true);
Expression cast = ParserUtils.withOrigin(ctx, () ->
new Cast(getExpression(ctx.expression()), dataType));
if (dataType.isStringLikeType() && ((CharacterType) dataType).getLen() >= 0) {
@ -1182,9 +1184,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
String type = ctx.type.getText().toUpperCase();
switch (type) {
case "DATE":
return new DateLiteral(value);
return Config.enable_date_conversion ? new DateV2Literal(value) : new DateLiteral(value);
case "TIMESTAMP":
return new DateTimeLiteral(value);
return Config.enable_date_conversion ? new DateTimeV2Literal(value) : new DateTimeLiteral(value);
case "DATEV2":
return new DateV2Literal(value);
default:
@ -1880,11 +1882,11 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
}
@Override
public DataType visitPrimitiveDataType(PrimitiveDataTypeContext ctx) {
public List<String> visitPrimitiveDataType(PrimitiveDataTypeContext ctx) {
String dataType = ctx.identifier().getText().toLowerCase(Locale.ROOT);
List<String> l = Lists.newArrayList(dataType);
ctx.INTEGER_VALUE().stream().map(ParseTree::getText).forEach(l::add);
return DataType.convertPrimitiveFromStrings(l);
return l;
}
private Expression parseFunctionWithOrderKeys(String functionName, boolean isDistinct,

3
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
View File

@ -25,7 +25,6 @@ import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.types.DataType;
import com.google.common.collect.Lists;
import org.antlr.v4.runtime.CharStreams;
@ -75,7 +74,7 @@ public class NereidsParser {
return parse(expression, DorisParser::expression);
}
public DataType parseDataType(String dataType) {
public List<String> parseDataType(String dataType) {
return parse(dataType, DorisParser::dataType);
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
View File

@ -220,9 +220,9 @@ public abstract class Literal extends Expression implements LeafExpression, Comp
} else if (targetType instanceof StringType) {
return new StringLiteral(desc);
} else if (targetType.isDateType()) {
return Config.enable_date_conversion ? new DateV2Literal(desc) : new DateLiteral(desc);
return new DateLiteral(desc);
} else if (targetType.isDateTimeType()) {
return Config.enable_date_conversion ? new DateTimeV2Literal(desc) : new DateTimeLiteral(desc);
return new DateTimeLiteral(desc);
} else if (targetType.isDecimalV2Type()) {
return new DecimalLiteral((DecimalV2Type) targetType, new BigDecimal(desc));
} else if (targetType.isDecimalV3Type()) {

14
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
View File

@ -111,7 +111,7 @@ public abstract class DataType implements AbstractDataType {
* @param types data type in string representation
* @return data type in Nereids
*/
public static DataType convertPrimitiveFromStrings(List<String> types) {
public static DataType convertPrimitiveFromStrings(List<String> types, boolean tryConvert) {
String type = types.get(0).toLowerCase().trim();
switch (type) {
case "bool":
@ -133,7 +133,7 @@ public abstract class DataType implements AbstractDataType {
case "double":
return DoubleType.INSTANCE;
case "decimal":
if (Config.enable_decimal_conversion) {
if (Config.enable_decimal_conversion && tryConvert) {
switch (types.size()) {
case 1:
return DecimalV3Type.SYSTEM_DEFAULT;
@ -198,7 +198,8 @@ public abstract class DataType implements AbstractDataType {
case "null_type": // ScalarType.NULL.toSql() return "null_type", so support it
return NullType.INSTANCE;
case "date":
return DateType.INSTANCE;
return Config.enable_date_conversion && tryConvert ? DateV2Type.INSTANCE
: DateType.INSTANCE;
case "datev2":
return DateV2Type.INSTANCE;
case "time":
@ -206,7 +207,9 @@ public abstract class DataType implements AbstractDataType {
case "datetime":
switch (types.size()) {
case 1:
return DateTimeType.INSTANCE;
return Config.enable_date_conversion && tryConvert
? DateTimeV2Type.SYSTEM_DEFAULT
: DateTimeType.INSTANCE;
case 2:
return DateTimeV2Type.of(Integer.parseInt(types.get(1)));
default:
@ -243,7 +246,8 @@ public abstract class DataType implements AbstractDataType {
*/
public static DataType convertFromString(String type) {
try {
return PARSER.parseDataType(type);
List<String> types = PARSER.parseDataType(type);
return DataType.convertPrimitiveFromStrings(types, false);
} catch (Exception e) {
// TODO: remove it when Nereids parser support array
if (type.startsWith("array")) {

4
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
View File

@ -229,7 +229,9 @@ public class FoldConstantTest extends ExpressionRewriteTestHelper {
interval = "date '1991-05-01' + interval 10 / 2 + 1 day";
e7 = process((TimestampArithmetic) PARSER.parseExpression(interval));
e8 = new DateLiteral(1991, 5, 7);
e8 = Config.enable_date_conversion
? new DateV2Literal(1991, 5, 7)
: new DateLiteral(1991, 5, 7);
assertRewrite(e7, e8);
interval = "interval '1' day + '1991-05-01'";

11
fe/fe-core/src/test/java/org/apache/doris/nereids/types/DataTypeTest.java
View File

@ -18,7 +18,6 @@
package org.apache.doris.nereids.types;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@ -79,9 +78,7 @@ public class DataTypeTest {
// double
Assertions.assertEquals(DoubleType.INSTANCE, DataType.convertFromString("double"));
// decimalv2
Assertions.assertEquals(
Config.enable_decimal_conversion ? DecimalV3Type.createDecimalV3Type(13, 9)
: DecimalV2Type.createDecimalV2Type(13, 9),
Assertions.assertEquals(DecimalV2Type.createDecimalV2Type(13, 9),
DataType.convertFromString("decimal(13, 9)"));
// decimalv3
Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(13, 9),
@ -105,10 +102,10 @@ public class DataTypeTest {
Assertions.assertEquals(DateV2Type.INSTANCE, DataType.convertFromString("datev2"));
// time
Assertions.assertEquals(TimeType.INSTANCE, DataType.convertFromString("time"));
// datetime
Assertions.assertEquals(Config.enable_date_conversion ? DateTimeV2Type.of(0) : DateTimeType.INSTANCE,
Config.enable_date_conversion ? DataType.convertFromString("datetimev2(0)")
: DataType.convertFromString("datetime"));
Assertions.assertEquals(DateTimeType.INSTANCE, DataType.convertFromString("datetime"));
// datetimev2
Assertions.assertEquals(DateTimeV2Type.of(3), DataType.convertFromString("datetimev2(3)"));
// hll

2
regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query94.out
View File

@ -9,7 +9,7 @@ PhysicalTopN
------------PhysicalProject
--------------hashJoin[INNER_JOIN](ws1.ws_ship_date_sk = date_dim.d_date_sk)
----------------PhysicalProject
------------------filter((date_dim.d_date >= 2000-02-01)(cast(d_date as DATETIMEV2(0)) <= cast(days_add(cast('2000-2-01' as DATE), INTERVAL 60 DAY) as DATETIMEV2(0))))
------------------filter((date_dim.d_date >= 2000-02-01)(cast(d_date as DATETIMEV2(0)) <= cast(days_add(cast('2000-2-01' as DATEV2), INTERVAL 60 DAY) as DATETIMEV2(0))))
--------------------PhysicalOlapScan[date_dim]
----------------PhysicalDistribute
------------------PhysicalProject

2
regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
View File

@ -19,7 +19,7 @@ CteAnchor[cteId= ( CTEId#3=] )
--------------PhysicalProject
----------------hashJoin[INNER_JOIN](ws1.ws_ship_date_sk = date_dim.d_date_sk)
------------------PhysicalProject
--------------------filter((date_dim.d_date >= 1999-02-01)(cast(d_date as DATETIMEV2(0)) <= cast(days_add(cast('1999-2-01' as DATE), INTERVAL 60 DAY) as DATETIMEV2(0))))
--------------------filter((date_dim.d_date >= 1999-02-01)(cast(d_date as DATETIMEV2(0)) <= cast(days_add(cast('1999-2-01' as DATEV2), INTERVAL 60 DAY) as DATETIMEV2(0))))
----------------------PhysicalOlapScan[date_dim]
------------------PhysicalDistribute
--------------------hashJoin[RIGHT_SEMI_JOIN](ws1.ws_order_number = web_returns.wr_order_number)