From 776f0205f3a7a8cb4c1a98544b77570d54f6d2b6 Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Fri, 1 Dec 2023 09:58:44 +0800 Subject: [PATCH] [Fix](test) Fix an auto partition conflict and add many testcases (#27730) Fix an auto partition conflict and add many testcases --- .../doris/analysis/PartitionExprUtil.java | 2 - .../test_auto_list_partition.out | 36 +++-- .../test_auto_partition_behavior.out | 118 ++++++++++++++ .../test_auto_list_partition.groovy | 150 ++++++++++++------ .../test_auto_partition_behavior.groovy | 134 ++++++++++++++++ 5 files changed, 377 insertions(+), 63 deletions(-) create mode 100644 regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out create mode 100644 regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java index eb2af52d6b..a0f59f3fcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java @@ -206,8 +206,6 @@ public class PartitionExprUtil { char ch = value.charAt(i); if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9')) { sb.append(ch); - } else if (ch == '-' || ch == ':' || ch == ' ' || ch == '*') { - // Main user remove characters in time } else { int unicodeValue = value.codePointAt(i); String unicodeString = Integer.toHexString(unicodeValue); diff --git a/regression-test/data/partition_p0/auto_partition/test_auto_list_partition.out b/regression-test/data/partition_p0/auto_partition/test_auto_list_partition.out index 36a6418f3f..53eeb49325 100644 --- a/regression-test/data/partition_p0/auto_partition/test_auto_list_partition.out +++ b/regression-test/data/partition_p0/auto_partition/test_auto_list_partition.out @@ -20,22 +20,26 @@ xxx xxx -- !sql3 -- -Abc -Beijing -Beijing -XXX -xxx + +! +1 +? +_ +x -- !sql4 -- -Abc -Abc -Beijing -Beijing -Beijing -Beijing -XXX -XXX -new -xxx -xxx + + +! +! +1 +1 +? +? +_ +_ +_ +x +x +y diff --git a/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out b/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out new file mode 100644 index 0000000000..53cd65f5d8 --- /dev/null +++ b/regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out @@ -0,0 +1,118 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- + + + ! + ! +Xxx +xxX + +-- !sql2 -- + + + ! + ! +Xxx +xxX + +-- !sql3 -- + + ! + ! + - +- +- - +-- +Xxx +xxX + +-- !sql4 -- + +-- !sql5 -- +1 + +-- !sql6 -- + ! + - +- +- - +-- +modified +xxX + +-- !sql1 -- + + + ! + ! +Xxx +xxX + +-- !sql2 -- + + + + + ! + ! + ! + ! +Xxx +Xxx +xxX +xxX + +-- !sql3 -- + + + ! + ! + ! + ! + - +- +- - +-- +Xxx +Xxx +xxX +xxX + +-- !sql4 -- + ! + ! + - +- +- - +-- +Xxx +Xxx +xxX +xxX + +-- !sql1 -- +2009-12-12T00:00 2020-12-12T00:00 +2010-12-12T00:00 2020-12-12T12:12:12.123456 +2011-12-12T00:00 2012-12-12T00:00 +2012-12-12T00:00 2013-12-12T00:00 +2013-12-12T00:00 2013-12-12T00:00 +2020-12-12T00:00 2020-12-12T12:12:12.123456 + +-- !sql2 -- +2009-12-12T00:00 2020-12-12T00:00 +2010-12-12T00:00 2020-12-12T12:12:12.123456 +2011-12-12T00:00 2012-12-12T00:00 +2012-12-12T00:00 2013-12-12T00:00 +2013-12-12T00:00 2013-12-12T00:00 +2020-12-12T00:00 2020-12-12T12:12:12.123456 + +-- !sql3 -- +2009-12-12T00:00 2020-12-12T00:00 + +-- !sql4 -- +2010-12-12T00:00 2020-12-12T12:12:12.123456 +2011-12-12T00:00 2012-12-12T00:00 +2012-12-12T00:00 2013-12-12T00:00 +2013-12-12T00:00 2013-12-12T00:00 +2020-12-12T00:00 2020-12-12T12:12:12.123456 + diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_list_partition.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_list_partition.groovy index 20f07156da..6e420d6e9c 100644 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_list_partition.groovy +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_list_partition.groovy @@ -16,11 +16,11 @@ // under the License. suite("test_auto_list_partition") { - def tblName1 = "list_table1" - sql "drop table if exists ${tblName1}" + // varchar + sql "drop table if exists list_table1" sql """ - CREATE TABLE `${tblName1}` ( - `str` varchar not null + CREATE TABLE `list_table1` ( + `str` varchar not null ) ENGINE=OLAP DUPLICATE KEY(`str`) COMMENT 'OLAP' @@ -29,43 +29,44 @@ suite("test_auto_list_partition") { ) DISTRIBUTED BY HASH(`str`) BUCKETS 10 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_allocation" = "tag.location.default: 1" ); """ - sql """ insert into ${tblName1} values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc") """ - qt_sql1 """ select * from ${tblName1} order by `str` """ - result11 = sql "show partitions from ${tblName1}" + sql """ insert into list_table1 values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc") """ + qt_sql1 """ select * from list_table1 order by `str` """ + def result11 = sql "show partitions from list_table1" assertEquals(result11.size(), 4) - sql """ insert into ${tblName1} values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc"), ("new") """ - qt_sql2 """ select * from ${tblName1} order by `str` """ - result12 = sql "show partitions from ${tblName1}" + sql """ insert into list_table1 values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc"), ("new") """ + qt_sql2 """ select * from list_table1 order by `str` """ + def result12 = sql "show partitions from list_table1" assertEquals(result12.size(), 5) - def tblName2 = "list_table2" - sql "drop table if exists ${tblName2}" + // char + sql "drop table if exists list_table2" sql """ - CREATE TABLE `${tblName2}` ( - `str` varchar not null + CREATE TABLE `list_table2` ( + `ch` char not null ) ENGINE=OLAP - DUPLICATE KEY(`str`) + DUPLICATE KEY(`ch`) COMMENT 'OLAP' - AUTO PARTITION BY LIST (`str`) + AUTO PARTITION BY LIST (`ch`) ( ) - DISTRIBUTED BY HASH(`str`) BUCKETS 10 + DISTRIBUTED BY HASH(`ch`) BUCKETS 10 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_allocation" = "tag.location.default: 1" ); """ - sql """ insert into ${tblName2} values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc") """ - qt_sql3 """ select * from ${tblName2} order by `str` """ - result21 = sql "show partitions from ${tblName2}" - assertEquals(result21.size(), 4) - sql """ insert into ${tblName2} values ("Beijing"), ("XXX"), ("xxx"), ("Beijing"), ("Abc"), ("new") """ - qt_sql4 """ select * from ${tblName2} order by `str` """ - result22 = sql "show partitions from ${tblName2}" - assertEquals(result22.size(), 5) + sql """ insert into list_table2 values (" "), ("!"), ("?"), ("1"), ("_"), ("x") """ + qt_sql3 """ select * from list_table2 order by `ch` """ + def result21 = sql "show partitions from list_table2" + assertEquals(result21.size(), 6) + sql """ insert into list_table2 values (" "), ("!"), ("?"), ("1"), ("_"), ("x"), ("_"), ("y") """ + qt_sql4 """ select * from list_table2 order by `ch` """ + def result22 = sql "show partitions from list_table2" + assertEquals(result22.size(), 7) + // varchar upper/lower case def tblName3 = "list_table3" sql "drop table if exists ${tblName3}" sql """ @@ -81,18 +82,18 @@ suite("test_auto_list_partition") { ) DISTRIBUTED BY HASH(`k1`) BUCKETS 16 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_allocation" = "tag.location.default: 1" ); """ sql """ insert into ${tblName3} values (1, 'ABC', '2000-01-01 12:12:12.123456'), (2, 'AAA', '2000-01-01'), (3, 'aaa', '2000-01-01'), (3, 'AaA', '2000-01-01') """ - result3 = sql "show partitions from ${tblName3}" + def result3 = sql "show partitions from ${tblName3}" logger.info("${result3}") assertEquals(result3.size(), 4) - def tblName4 = "list_table4" - sql "drop table if exists ${tblName4}" + // int + sql "drop table if exists list_table4" sql """ - CREATE TABLE `${tblName4}` ( + CREATE TABLE `list_table4` ( `k1` INT not null, `k2` VARCHAR(50), `k3` DATETIMEV2(6) @@ -104,18 +105,18 @@ suite("test_auto_list_partition") { ) DISTRIBUTED BY HASH(`k1`) BUCKETS 16 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_allocation" = "tag.location.default: 1" ); """ - sql """ insert into ${tblName4} values (1, '2020-12-12 12:12:12', '2000-01-01 12:12:12.123456'), (2, '20201212 121212', '2000-01-01'), (3, '20201212121212', '2000-01-01'), (3, 'AaA', '2000-01-01') """ - result4 = sql "show partitions from ${tblName4}" + sql """ insert into list_table4 values (1, '2020-12-12 12:12:12', '2000-01-01 12:12:12.123456'), (2, '20201212 121212', '2000-01-01'), (3, '20201212121212', '2000-01-01'), (3, 'AaA', '2000-01-01') """ + def result4 = sql "show partitions from list_table4" logger.info("${result4}") assertEquals(result4.size(), 3) - def tblName5 = "list_table4" - sql "drop table if exists ${tblName5}" + // dtv2(6) + sql "drop table if exists list_table5" sql """ - CREATE TABLE `${tblName5}` ( + CREATE TABLE `list_table5` ( `k1` INT, `k2` VARCHAR(50), `k3` DATETIMEV2(6) not null @@ -127,14 +128,15 @@ suite("test_auto_list_partition") { ) DISTRIBUTED BY HASH(`k1`) BUCKETS 16 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_allocation" = "tag.location.default: 1" ); """ - sql """ insert into ${tblName5} values (1, 'ABC', '2000-01-01 12:12:12.123456'), (2, 'AAA', '2000-01-01'), (3, 'aaa', '2000-01-01'), (3, 'AaA', '2000-01-01') """ - result5 = sql "show partitions from ${tblName5}" + sql """ insert into list_table5 values (1, 'ABC', '2000-01-01 12:12:12.123456'), (2, 'AAA', '2000-01-01'), (3, 'aaa', '2000-01-01'), (3, 'AaA', '2000-01-01') """ + def result5 = sql "show partitions from list_table5" logger.info("${result5}") assertEquals(result5.size(), 2) + // largeint sql "drop table if exists test_largeint" sql """ CREATE TABLE test_largeint ( @@ -148,11 +150,12 @@ suite("test_auto_list_partition") { "replication_allocation" = "tag.location.default: 1" ); """ - sql " insert into test_largeint values (1), (-1)" - result6 = sql "show partitions from test_largeint" + sql " insert into test_largeint values (1), (-1), (170141183460469231731687303715884105727) " + def result6 = sql "show partitions from test_largeint" logger.info("${result6}") - assertEquals(result6.size(), 2) + assertEquals(result6.size(), 3) + // bool sql "drop table if exists test_bool" sql """ CREATE TABLE test_bool ( @@ -167,7 +170,64 @@ suite("test_auto_list_partition") { ); """ sql " insert into test_bool values (true), (false)" - result7 = sql "show partitions from test_bool" + def result7 = sql "show partitions from test_bool" logger.info("${result7}") assertEquals(result7.size(), 2) + + // bigint + sql "drop table if exists test_bigint" + sql """ + CREATE TABLE test_bigint ( + k bigint not null + ) + AUTO PARTITION BY LIST (`k`) + ( + ) + DISTRIBUTED BY HASH(`k`) BUCKETS 16 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql " insert into test_bigint values (1), (-1)" + def result8 = sql "show partitions from test_bigint" + logger.info("${result8}") + assertEquals(result8.size(), 2) + + // smallint + sql "drop table if exists test_smallint" + sql """ + CREATE TABLE test_smallint ( + k smallint not null + ) + AUTO PARTITION BY LIST (`k`) + ( + ) + DISTRIBUTED BY HASH(`k`) BUCKETS 16 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql " insert into test_smallint values (1), (-1)" + def result9 = sql "show partitions from test_smallint" + logger.info("${result9}") + assertEquals(result9.size(), 2) + + // tinyint + sql "drop table if exists test_tinyint" + sql """ + CREATE TABLE test_tinyint ( + k tinyint not null + ) + AUTO PARTITION BY LIST (`k`) + ( + ) + DISTRIBUTED BY HASH(`k`) BUCKETS 16 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql " insert into test_tinyint values (1), (-1)" + def result10 = sql "show partitions from test_tinyint" + logger.info("${result10}") + assertEquals(result10.size(), 2) } diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy new file mode 100644 index 0000000000..6407f9afec --- /dev/null +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_auto_partition_behavior") { + /// unique key table + sql "drop table if exists unique_table" + sql """ + CREATE TABLE `unique_table` ( + `str` varchar not null + ) ENGINE=OLAP + UNIQUE KEY(`str`) + COMMENT 'OLAP' + AUTO PARTITION BY LIST (`str`) + ( + PARTITION `partition_origin` values in (("Xxx"), ("Yyy")) + ) + DISTRIBUTED BY HASH(`str`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + // special characters + sql """ insert into unique_table values (" "), (" "), ("Xxx"), ("xxX"), (" ! "), (" ! ") """ + qt_sql1 """ select * from unique_table order by `str` """ + def result = sql "show partitions from unique_table" + assertEquals(result.size(), 6) + sql """ insert into unique_table values (" "), (" "), ("Xxx"), ("xxX"), (" ! "), (" ! ") """ + qt_sql2 """ select * from unique_table order by `str` """ + result = sql "show partitions from unique_table" + assertEquals(result.size(), 6) + sql """ insert into unique_table values ("-"), ("--"), ("- -"), (" - ") """ + result = sql "show partitions from unique_table" + assertEquals(result.size(), 10) + // drop partition + def partitions = sql "show partitions from unique_table order by PartitionName" + def partition1_name = partitions[0][1] + sql """ alter table unique_table drop partition ${partition1_name} """ // partition ' ' + result = sql "show partitions from unique_table" + assertEquals(result.size(), 9) + qt_sql3 """ select * from unique_table order by `str` """ + // modify value + sql """ update unique_table set str = "modified" where str in (" ", " ") """ // only " " + qt_sql4 """ select * from unique_table where str = ' ' order by `str` """ // modified + qt_sql5 """ select count() from unique_table where str = 'modified' """ + // crop + qt_sql6 """ select * from unique_table where ((str > ' ! ' || str = 'modified') && str != 'Xxx') order by str """ + + + /// duplicate key table + sql "drop table if exists dup_table" + sql """ + CREATE TABLE `dup_table` ( + `str` varchar not null + ) ENGINE=OLAP + DUPLICATE KEY(`str`) + COMMENT 'OLAP' + AUTO PARTITION BY LIST (`str`) + ( + PARTITION `partition_origin` values in (("Xxx"), ("Yyy")) + ) + DISTRIBUTED BY HASH(`str`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + // special characters + sql """ insert into dup_table values (" "), (" "), ("Xxx"), ("xxX"), (" ! "), (" ! ") """ + qt_sql1 """ select * from dup_table order by `str` """ + result = sql "show partitions from dup_table" + assertEquals(result.size(), 6) + sql """ insert into dup_table values (" "), (" "), ("Xxx"), ("xxX"), (" ! "), (" ! ") """ + qt_sql2 """ select * from dup_table order by `str` """ + result = sql "show partitions from dup_table" + assertEquals(result.size(), 6) + sql """ insert into dup_table values ("-"), ("--"), ("- -"), (" - ") """ + result = sql "show partitions from dup_table" + assertEquals(result.size(), 10) + // drop partition + partitions = sql "show partitions from dup_table order by PartitionName" + partition1_name = partitions[0][1] + sql """ alter table dup_table drop partition ${partition1_name} """ + result = sql "show partitions from dup_table" + assertEquals(result.size(), 9) + qt_sql3 """ select * from dup_table order by `str` """ + // crop + qt_sql4 """ select * from dup_table where str > ' ! ' order by str """ + + /// agg key table + sql "drop table if exists agg_dt6" + sql """ + CREATE TABLE `agg_dt6` ( + `k0` datetime(6) not null, + `k1` datetime(6) max not null + ) ENGINE=OLAP + AGGREGATE KEY(`k0`) + COMMENT 'OLAP' + AUTO PARTITION BY RANGE date_trunc(`k0`, 'year') + ( + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + // modify when no partition + sql """ alter table agg_dt6 add partition `p2010` values less than ('2010-01-01') """ + // insert + sql """ insert into agg_dt6 values ('2020-12-12', '2020-12-12'), ('2020-12-12', '2020-12-12 12:12:12.123456'), ('2020-12-12', '20121212'), (20131212, 20131212) """ + sql """ insert into agg_dt6 values ('2009-12-12', '2020-12-12'), ('2010-12-12', '2020-12-12 12:12:12.123456'), ('2011-12-12', '20121212'), (20121212, 20131212) """ + qt_sql1 """ select * from agg_dt6 order by k0, k1 """ + // crop + qt_sql2 """ select * from agg_dt6 where k1 <= '2020-12-12 12:12:12.123456' order by k0, k1 """ + qt_sql3 """ select * from agg_dt6 partition (p2010) order by k0, k1 """ + // modify partition + sql """ alter table agg_dt6 drop partition p2010 """ + qt_sql4 """ select * from agg_dt6 order by k0, k1 """ + sql """ insert into agg_dt6 values ('2020-12-12', '2020-12-12'), ('2020-12-12', '2020-12-12 12:12:12.123456'), ('2020-12-12', '20121212'), (20131212, 20131212) """ + result = sql "show partitions from agg_dt6" + assertEquals(result.size(), 5) +}