From bc95902fedd9290819ee0652825b75d903f3d707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Sat, 1 Jun 2024 14:00:39 +0800 Subject: [PATCH] [fix](Nereids): Convert VarcharLiteral to StringLikeLiteral in Function Signatures (#35536) ## Proposed changes This pull request updates the function signatures where VarcharLiteral is currently used, replacing it with StringLikeLiteral. This change aims to enhance flexibility and consistency across functions that handle similar types of string data. By adopting StringLikeLiteral, we can support a broader range of string-like types beyond the basic VARCHAR type, facilitating more robust and versatile string handling capabilities in our codebase. This update ensures better type abstraction and promotes code reusability. ## Further comments If this is a relatively large or complex change, kick off the discussion at [dev@doris.apache.org](mailto:dev@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... --- .../DateTimeExtractAndTransform.java | 25 +++--- .../executable/ExecutableFunctions.java | 5 +- .../literal/StringLikeLiteral.java | 5 ++ .../rules/expression/FoldConstantTest.java | 78 +++++++++++++++++++ 4 files changed, 99 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index 8cda0c2d87..3cc46ee111 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -285,26 +286,26 @@ public class DateTimeExtractAndTransform { * datetime arithmetic function date-format */ @ExecFunction(name = "date_format", argTypes = {"DATE", "VARCHAR"}, returnType = "VARCHAR") - public static Expression dateFormat(DateLiteral date, VarcharLiteral format) { + public static Expression dateFormat(DateLiteral date, StringLikeLiteral format) { return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter().format( java.time.LocalDate.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay())))); } @ExecFunction(name = "date_format", argTypes = {"DATETIME", "VARCHAR"}, returnType = "VARCHAR") - public static Expression dateFormat(DateTimeLiteral date, VarcharLiteral format) { + public static Expression dateFormat(DateTimeLiteral date, StringLikeLiteral format) { return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter().format( java.time.LocalDateTime.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()), ((int) date.getHour()), ((int) date.getMinute()), ((int) date.getSecond())))); } @ExecFunction(name = "date_format", argTypes = {"DATEV2", "VARCHAR"}, returnType = "VARCHAR") - public static Expression dateFormat(DateV2Literal date, VarcharLiteral format) { + public static Expression dateFormat(DateV2Literal date, StringLikeLiteral format) { return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter().format( java.time.LocalDate.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay())))); } @ExecFunction(name = "date_format", argTypes = {"DATETIMEV2", "VARCHAR"}, returnType = "VARCHAR") - public static Expression dateFormat(DateTimeV2Literal date, VarcharLiteral format) { + public static Expression dateFormat(DateTimeV2Literal date, StringLikeLiteral format) { return new VarcharLiteral(DateUtils.formatBuilder(format.getValue()).toFormatter().format( java.time.LocalDateTime.of(((int) date.getYear()), ((int) date.getMonth()), ((int) date.getDay()), ((int) date.getHour()), ((int) date.getMinute()), ((int) date.getSecond())))); @@ -327,22 +328,22 @@ public class DateTimeExtractAndTransform { * datetime arithmetic function date-trunc */ @ExecFunction(name = "date_trunc", argTypes = {"DATETIME", "VARCHAR"}, returnType = "DATETIME") - public static Expression dateTrunc(DateTimeLiteral date, VarcharLiteral trunc) { + public static Expression dateTrunc(DateTimeLiteral date, StringLikeLiteral trunc) { return DateTimeLiteral.fromJavaDateType(dateTruncHelper(date.toJavaDateType(), trunc.getValue())); } @ExecFunction(name = "date_trunc", argTypes = {"DATETIMEV2", "VARCHAR"}, returnType = "DATETIMEV2") - public static Expression dateTrunc(DateTimeV2Literal date, VarcharLiteral trunc) { + public static Expression dateTrunc(DateTimeV2Literal date, StringLikeLiteral trunc) { return DateTimeV2Literal.fromJavaDateType(dateTruncHelper(date.toJavaDateType(), trunc.getValue())); } @ExecFunction(name = "date_trunc", argTypes = {"DATE", "VARCHAR"}, returnType = "DATE") - public static Expression dateTrunc(DateLiteral date, VarcharLiteral trunc) { + public static Expression dateTrunc(DateLiteral date, StringLikeLiteral trunc) { return DateLiteral.fromJavaDateType(dateTruncHelper(date.toJavaDateType(), trunc.getValue())); } @ExecFunction(name = "date_trunc", argTypes = {"DATEV2", "VARCHAR"}, returnType = "DATEV2") - public static Expression dateTrunc(DateV2Literal date, VarcharLiteral trunc) { + public static Expression dateTrunc(DateV2Literal date, StringLikeLiteral trunc) { return DateV2Literal.fromJavaDateType(dateTruncHelper(date.toJavaDateType(), trunc.getValue())); } @@ -471,7 +472,7 @@ public class DateTimeExtractAndTransform { * date transformation function: from_unixtime */ @ExecFunction(name = "from_unixtime", argTypes = {"BIGINT", "VARCHAR"}, returnType = "VARCHAR") - public static Expression fromUnixTime(BigIntLiteral second, VarcharLiteral format) { + public static Expression fromUnixTime(BigIntLiteral second, StringLikeLiteral format) { // 32536771199L is max valid timestamp of mysql from_unix_time if (second.getValue() < 0 || second.getValue() > 32536771199L) { return new NullLiteral(VarcharType.SYSTEM_DEFAULT); @@ -524,7 +525,7 @@ public class DateTimeExtractAndTransform { * date transformation function: unix_timestamp */ @ExecFunction(name = "unix_timestamp", argTypes = {"VARCHAR", "VARCHAR"}, returnType = "INT") - public static Expression unixTimestamp(VarcharLiteral date, VarcharLiteral format) { + public static Expression unixTimestamp(StringLikeLiteral date, StringLikeLiteral format) { DateTimeFormatter formatter = DateUtils.formatBuilder(format.getValue()).toFormatter(); LocalDateTime dateObj; try { @@ -603,7 +604,7 @@ public class DateTimeExtractAndTransform { * date transformation function: str_to_date */ @ExecFunction(name = "str_to_date", argTypes = {"VARCHAR", "VARCHAR"}, returnType = "DATETIMEV2") - public static Expression strToDate(VarcharLiteral str, VarcharLiteral format) { + public static Expression strToDate(StringLikeLiteral str, StringLikeLiteral format) { if (org.apache.doris.analysis.DateLiteral.hasTimePart(format.getStringValue())) { return DateTimeV2Literal.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) .toFormatter(), str.getValue())); @@ -624,7 +625,7 @@ public class DateTimeExtractAndTransform { } @ExecFunction(name = "convert_tz", argTypes = {"DATETIMEV2", "VARCHAR", "VARCHAR"}, returnType = "DATETIMEV2") - public static Expression convertTz(DateTimeV2Literal datetime, VarcharLiteral fromTz, VarcharLiteral toTz) { + public static Expression convertTz(DateTimeV2Literal datetime, StringLikeLiteral fromTz, StringLikeLiteral toTz) { LocalDateTime localDateTime = datetime.toJavaDateType(); ZonedDateTime fromDateTime = localDateTime.atZone(ZoneId.of(fromTz.getStringValue())); ZonedDateTime toDateTime = fromDateTime.withZoneSameInstant(ZoneId.of(toTz.getStringValue())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/ExecutableFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/ExecutableFunctions.java index 3e3f14b218..d11966b868 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/ExecutableFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/ExecutableFunctions.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; @@ -94,8 +95,8 @@ public class ExecutableFunctions { return new DoubleLiteral(Math.acos(literal.getValue())); } - @ExecFunction(name = "append_trailing_if_char_absent", argTypes = {"VARCHAR", "VARCHAR"}, returnType = "VARCHAR") - public static Expression appendTrailingIfCharAbsent(VarcharLiteral literal, VarcharLiteral chr) { + @ExecFunction(name = "append_trailing_char_if_absent", argTypes = {"VARCHAR", "VARCHAR"}, returnType = "VARCHAR") + public static Expression appendTrailingIfCharAbsent(StringLikeLiteral literal, StringLikeLiteral chr) { if (literal.getValue().length() != 1) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java index 815e5742d2..dca604f9db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java @@ -56,6 +56,11 @@ public abstract class StringLikeLiteral extends Literal { return (double) v; } + @Override + public String getValue() { + return value; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java index 747e72b0a9..7a27131469 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java @@ -31,6 +31,13 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; +import org.apache.doris.nereids.trees.expressions.functions.scalar.AppendTrailingCharIfAbsent; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateFormat; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.functions.scalar.FromUnixtime; +import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate; +import org.apache.doris.nereids.trees.expressions.functions.scalar.UnixTimestamp; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; @@ -39,6 +46,7 @@ import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Interval.TimeUnit; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -52,6 +60,7 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.time.LocalDateTime; import java.util.Locale; class FoldConstantTest extends ExpressionRewriteTestHelper { @@ -166,6 +175,75 @@ class FoldConstantTest extends ExpressionRewriteTestHelper { Assertions.assertEquals(rewritten, expected); } + @Test + void testFoldString() { + executor = new ExpressionRuleExecutor(ImmutableList.of( + bottomUp(FoldConstantRuleOnFE.VISITOR_INSTANCE) + )); + ConvertTz c = new ConvertTz(DateTimeV2Literal.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), + StringLiteral.of("Asia/Shanghai"), StringLiteral.of("GMT")); + Expression rewritten = executor.rewrite(c, context); + String res = "0000-12-31 16:55:18"; + Assertions.assertEquals(rewritten.toString(), res); + + DateFormat d = new DateFormat(DateTimeLiteral.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), + StringLiteral.of("%y %m %d")); + rewritten = executor.rewrite(d, context); + res = "'01 01 01'"; + Assertions.assertEquals(rewritten.toString(), res); + d = new DateFormat(DateTimeV2Literal.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), + StringLiteral.of("%y %m %d")); + rewritten = executor.rewrite(d, context); + Assertions.assertEquals(rewritten.toString(), res); + d = new DateFormat(DateLiteral.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), + StringLiteral.of("%y %m %d")); + rewritten = executor.rewrite(d, context); + Assertions.assertEquals(rewritten.toString(), res); + d = new DateFormat(DateV2Literal.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), + StringLiteral.of("%y %m %d")); + rewritten = executor.rewrite(d, context); + Assertions.assertEquals(rewritten.toString(), res); + + DateTrunc t = new DateTrunc(DateTimeLiteral.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), + StringLiteral.of("week")); + rewritten = executor.rewrite(t, context); + res = "0001-01-01 00:00:00"; + Assertions.assertEquals(rewritten.toString(), res); + t = new DateTrunc(DateTimeV2Literal.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), + StringLiteral.of("week")); + rewritten = executor.rewrite(t, context); + Assertions.assertEquals(rewritten.toString(), res); + t = new DateTrunc(DateLiteral.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), + StringLiteral.of("week")); + rewritten = executor.rewrite(t, context); + res = "0001-01-01"; + Assertions.assertEquals(rewritten.toString(), res); + t = new DateTrunc(DateV2Literal.fromJavaDateType(LocalDateTime.of(1, 1, 1, 1, 1, 1)), + StringLiteral.of("week")); + rewritten = executor.rewrite(t, context); + Assertions.assertEquals(rewritten.toString(), res); + + FromUnixtime f = new FromUnixtime(BigIntLiteral.of(123456789L), StringLiteral.of("%y %m %d")); + rewritten = executor.rewrite(f, context); + res = "'73 11 30'"; + Assertions.assertEquals(rewritten.toString(), res); + + UnixTimestamp ut = new UnixTimestamp(StringLiteral.of("2021-11-11"), StringLiteral.of("%Y-%m-%d")); + rewritten = executor.rewrite(ut, context); + res = "1636560000"; + Assertions.assertEquals(rewritten.toString(), res); + + StrToDate sd = new StrToDate(StringLiteral.of("2021-11-11"), StringLiteral.of("%Y-%m-%d")); + rewritten = executor.rewrite(sd, context); + res = "2021-11-11"; + Assertions.assertEquals(rewritten.toString(), res); + + AppendTrailingCharIfAbsent a = new AppendTrailingCharIfAbsent(StringLiteral.of("1"), StringLiteral.of("3")); + rewritten = executor.rewrite(a, context); + res = "'13'"; + Assertions.assertEquals(rewritten.toString(), res); + } + @Test void testCompareFold() { executor = new ExpressionRuleExecutor(ImmutableList.of(