From 085696744dd6fefd3b350411e11907fadc3498d3 Mon Sep 17 00:00:00 2001 From: chen Date: Wed, 20 Mar 2024 21:55:20 +0800 Subject: [PATCH] [Enhancement] when partition column is datetime, date can work in create table command (#32335) --- .../apache/doris/catalog/PartitionKey.java | 22 +++++++++- .../test_multi_column_partition.groovy | 29 ++++++------- .../test_multi_partition.groovy | 43 ++++++++++++++++++- .../test_range_partition.groovy | 28 ++++++++++++ .../test_partition_table_err_msg.groovy | 5 +-- 5 files changed, 107 insertions(+), 20 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index 5816496c93..7ad86698d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -29,6 +29,9 @@ import org.apache.doris.analysis.StringLiteral; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.qe.SessionVariable; import com.google.common.base.Joiner; @@ -90,7 +93,14 @@ public class PartitionKey implements Comparable, Writable { Preconditions.checkArgument(keys.size() <= columns.size()); int i; for (i = 0; i < keys.size(); ++i) { - partitionKey.keys.add(keys.get(i).getValue(columns.get(i).getType())); + Type keyType = columns.get(i).getType(); + // If column type is datatime and key type is date, we should convert date to datetime. + if (keyType.isDatetime() || keyType.isDatetimeV2()) { + Literal dateTimeLiteral = getDateTimeLiteral(keys.get(i).getStringValue(), keyType); + partitionKey.keys.add(dateTimeLiteral.toLegacyLiteral()); + } else { + partitionKey.keys.add(keys.get(i).getValue(keyType)); + } partitionKey.types.add(columns.get(i).getDataType()); } @@ -104,6 +114,16 @@ public class PartitionKey implements Comparable, Writable { return partitionKey; } + private static Literal getDateTimeLiteral(String value, Type type) throws AnalysisException { + if (type.isDatetime()) { + return new DateTimeLiteral(value); + } else if (type.isDatetimeV2()) { + return new DateTimeV2Literal(value); + } + throw new AnalysisException("date convert to datetime failed, " + + "value is [" + value + "], type is [" + type + "]."); + } + public static PartitionKey createListPartitionKeyWithTypes(List values, List types, boolean isHive) throws AnalysisException { diff --git a/regression-test/suites/partition_p0/multi_partition/test_multi_column_partition.groovy b/regression-test/suites/partition_p0/multi_partition/test_multi_column_partition.groovy index 6a03e72624..a42735cabf 100644 --- a/regression-test/suites/partition_p0/multi_partition/test_multi_column_partition.groovy +++ b/regression-test/suites/partition_p0/multi_partition/test_multi_column_partition.groovy @@ -77,21 +77,20 @@ suite("test_multi_partition_key", "p0") { ) // partition columns are int & datetime - test { - sql """ - CREATE TABLE err_1 ( - k1 TINYINT NOT NULL, - k5 DATETIME NOT NULL, - v1 DATE REPLACE NOT NULL) - PARTITION BY RANGE(k1,k5) ( - PARTITION partition_a VALUES LESS THAN ("-127","2010-01-01"), - PARTITION partition_e VALUES LESS THAN ("4","2010-01-01"), - PARTITION partition_f VALUES LESS THAN MAXVALUE ) - DISTRIBUTED BY HASH(k1) BUCKETS 53 - PROPERTIES("replication_allocation" = "tag.location.default: 1") - """ - exception "Invalid datetime value: 2010-01-01" - } + + sql """ + CREATE TABLE err_1 ( + k1 TINYINT NOT NULL, + k5 DATETIME NOT NULL, + v1 DATE REPLACE NOT NULL) + PARTITION BY RANGE(k1,k5) ( + PARTITION partition_a VALUES LESS THAN ("-127","2010-01-01"), + PARTITION partition_e VALUES LESS THAN ("4","2010-01-01"), + PARTITION partition_f VALUES LESS THAN MAXVALUE ) + DISTRIBUTED BY HASH(k1) BUCKETS 53 + PROPERTIES("replication_allocation" = "tag.location.default: 1") + """ + testPartitionTbl( "test_multi_partition_key_2", """ diff --git a/regression-test/suites/partition_p0/multi_partition/test_multi_partition.groovy b/regression-test/suites/partition_p0/multi_partition/test_multi_partition.groovy index 78f9359ae4..3dd698c03d 100644 --- a/regression-test/suites/partition_p0/multi_partition/test_multi_partition.groovy +++ b/regression-test/suites/partition_p0/multi_partition/test_multi_partition.groovy @@ -465,4 +465,45 @@ suite("test_multi_partition") { assertTrue(result2[1][1].startsWith("p_")) sql "drop table multi_par12" -} \ No newline at end of file + + // create one table without datetime partition, but with date string + sql """set enable_fallback_to_original_planner=false""" + sql """ + CREATE TABLE IF NOT EXISTS range_date_cast_to_datetime ( + id int, + name string, + pdate DATETIME ) + PARTITION BY RANGE(pdate)( + FROM ("2023-04-16") TO ("2023-04-20") INTERVAL 1 DAY + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 properties("replication_num" = "1") + """ + result1 = sql "show tables like 'range_date_cast_to_datetime'" + logger.info("${result1}") + assertEquals(result1.size(), 1) + result2 = sql "show partitions from range_date_cast_to_datetime" + logger.info("${result2}") + assertEquals(result2.size(), 4) + assertTrue(result2[1][1].startsWith("p_")) + sql "drop table range_date_cast_to_datetime" + + sql """set enable_fallback_to_original_planner=true""" + sql """ + CREATE TABLE IF NOT EXISTS range_date_cast_to_datetime ( + id int, + name string, + pdate DATETIME ) + PARTITION BY RANGE(pdate)( + FROM ("2023-04-16") TO ("2023-04-20") INTERVAL 1 DAY + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 properties("replication_num" = "1") + """ + result1 = sql "show tables like 'range_date_cast_to_datetime'" + logger.info("${result1}") + assertEquals(result1.size(), 1) + result2 = sql "show partitions from range_date_cast_to_datetime" + logger.info("${result2}") + assertEquals(result2.size(), 4) + assertTrue(result2[1][1].startsWith("p_")) + sql "drop table range_date_cast_to_datetime" +} diff --git a/regression-test/suites/partition_p0/multi_partition/test_range_partition.groovy b/regression-test/suites/partition_p0/multi_partition/test_range_partition.groovy index 64f5deff4a..c9e75718e3 100644 --- a/regression-test/suites/partition_p0/multi_partition/test_range_partition.groovy +++ b/regression-test/suites/partition_p0/multi_partition/test_range_partition.groovy @@ -296,4 +296,32 @@ suite("test_range_partition", "p0") { "DISTRIBUTED BY RANDOM BUCKETS 13" ) + // create one table without datetime partition, but with date string + sql """set enable_fallback_to_original_planner=false""" + sql """ + CREATE TABLE IF NOT EXISTS range_date_cast_to_datetime ( + id int, + name string, + pdate DATETIME ) + PARTITION BY RANGE(pdate)( + PARTITION pd20230418 VALUES less than ("2023-04-20") + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 properties("replication_num" = "1") + """ + sql "insert into range_date_cast_to_datetime values (1, 'name', '2023-04-19 08:08:30')" + sql "drop table range_date_cast_to_datetime" + + sql """set enable_fallback_to_original_planner=true""" + sql """ + CREATE TABLE IF NOT EXISTS range_date_cast_to_datetime ( + id int, + name string, + pdate DATETIME ) + PARTITION BY RANGE(pdate)( + PARTITION pd20230418 VALUES less than ("2023-04-20") + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 properties("replication_num" = "1") + """ + sql "insert into range_date_cast_to_datetime values (1, 'name', '2023-04-19 08:08:30')" + sql "drop table range_date_cast_to_datetime" } diff --git a/regression-test/suites/partition_p0/test_partition_table_err_msg.groovy b/regression-test/suites/partition_p0/test_partition_table_err_msg.groovy index e900c24b10..7099e96782 100644 --- a/regression-test/suites/partition_p0/test_partition_table_err_msg.groovy +++ b/regression-test/suites/partition_p0/test_partition_table_err_msg.groovy @@ -72,7 +72,7 @@ suite("test_partition_table_err_msg", "p0") { PARTITION partition_d VALUES LESS THAN ("10000-01-01 00:00:00") ) DISTRIBUTED BY HASH(k1) BUCKETS 13 """ - exception "date literal [10000-01-01 00:00:00] is invalid" + exception "date/datetime literal [10000-01-01 00:00:00] is invalid" } test { sql """ @@ -91,8 +91,7 @@ suite("test_partition_table_err_msg", "p0") { PARTITION partition_d VALUES LESS THAN ("9999-12-31 24:00:00") ) DISTRIBUTED BY HASH(k1) BUCKETS 13 """ - exception "date literal [9999-12-31 24:00:00] is invalid: Text '9999-12-31 24:00:00' could not be parsed: " + - "Invalid value for HourOfDay (valid values 0 - 23): 24" + exception "date/datetime literal [9999-12-31 24:00:00] is invalid" } test { sql """