From 0512e0b168fb677b82faa8baabcd25bd7e1b7775 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 27 Jul 2023 15:51:01 +0800 Subject: [PATCH] [test](regression) add cases for partial update with sequence_type (#22215) --- .../partial_update/basic_with_seq.csv | 4 +- .../partial_update/basic_with_test.csv | 5 + .../partial_update/basic_with_test2.csv | 5 + .../test_partial_update_seq_col.out | 3 +- .../test_partial_update_seq_type.out | 35 ++++ .../test_partial_update_seq_col.groovy | 2 +- .../test_partial_update_seq_type.groovy | 177 ++++++++++++++++++ 7 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 regression-test/data/unique_with_mow_p0/partial_update/basic_with_test.csv create mode 100644 regression-test/data/unique_with_mow_p0/partial_update/basic_with_test2.csv create mode 100644 regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type.out create mode 100644 regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type.groovy diff --git a/regression-test/data/unique_with_mow_p0/partial_update/basic_with_seq.csv b/regression-test/data/unique_with_mow_p0/partial_update/basic_with_seq.csv index 68be488ced..27ddfd0049 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/basic_with_seq.csv +++ b/regression-test/data/unique_with_mow_p0/partial_update/basic_with_seq.csv @@ -1,3 +1,5 @@ 2,2500,2023-07-19 2,2600,2023-07-20 -1,1300,2022-07-19 \ No newline at end of file +1,1300,2022-07-19 +3,1500,2022-07-20 +3,2500,2022-07-18 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/basic_with_test.csv b/regression-test/data/unique_with_mow_p0/partial_update/basic_with_test.csv new file mode 100644 index 0000000000..f8cecdef96 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/basic_with_test.csv @@ -0,0 +1,5 @@ +2,3600,2400,2019-01-23 +2,3500,2500,2020-03-19 +1,2300,2300,2021-05-19 +3,1500,2500,2022-03-31 +3,2000,2400,2022-02-18 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/basic_with_test2.csv b/regression-test/data/unique_with_mow_p0/partial_update/basic_with_test2.csv new file mode 100644 index 0000000000..f180b9d2be --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/basic_with_test2.csv @@ -0,0 +1,5 @@ +2,3650,2450,2019-01-22 +2,3550,2550,2020-03-18 +1,2300,2350,2021-05-18 +3,1550,2550,2022-03-30 +3,2050,2450,2022-02-17 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col.out index 8246986dc1..2e10832476 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col.out @@ -10,8 +10,9 @@ -- !partial_update_with_seq -- 1 doris 200 123 1 2023-01-01 2 doris2 2600 223 1 2023-07-20 +3 unknown 1500 \N 4321 2022-07-20 -- !partial_update_with_seq_hidden_columns -- 1 doris 200 123 1 2023-01-01 0 3 2023-01-01 2 doris2 2600 223 1 2023-07-20 0 4 2023-07-20 - +3 unknown 1500 \N 4321 2022-07-20 0 4 2022-07-20 diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type.out new file mode 100644 index 0000000000..d22b34bcc1 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type.out @@ -0,0 +1,35 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 doris 1000 123 1 2023-01-01 +2 doris2 2000 223 1 2023-01-01 + +-- !partial_update_with_seq_score -- +1 doris 2300 2300 1 2021-05-19 +2 doris2 3600 2400 1 2019-01-23 +3 unknown 2000 2400 4321 2022-02-18 + +-- !partial_update_with_seq_score_hidden -- +1 doris 2300 2300 1 2021-05-19 0 3 2300 +2 doris2 3600 2400 1 2019-01-23 0 3 3600 +3 unknown 2000 2400 4321 2022-02-18 0 3 2000 + +-- !partial_update_with_seq_test -- +1 doris 2300 2300 1 2021-05-19 +2 doris2 3600 2400 1 2019-01-23 +3 unknown 1500 2500 4321 2022-03-31 + +-- !partial_update_with_seq_test_hidden -- +1 doris 2300 2300 1 2021-05-19 0 4 2300 +2 doris2 3600 2400 1 2019-01-23 0 3 3600 +3 unknown 1500 2500 4321 2022-03-31 0 4 2500 + +-- !select_no_partial_update_score -- +1 unknown 2300 2350 4321 2021-05-18 0 5 2300 +2 unknown 3650 2450 4321 2019-01-22 0 5 3650 +3 unknown 1500 2500 4321 2022-03-31 0 4 2500 + +-- !select_no_partial_update_test -- +1 unknown 2300 2350 4321 2021-05-18 0 6 2350 +2 unknown 3650 2450 4321 2019-01-22 0 5 3650 +3 unknown 1550 2550 4321 2022-03-30 0 6 2550 + diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy index aeb4cc2732..06a066fc0a 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy @@ -24,7 +24,7 @@ suite("test_primary_key_partial_update_seq_col", "p0") { sql """ CREATE TABLE ${tableName} ( `id` int(11) NOT NULL COMMENT "用户 ID", - `name` varchar(65533) NOT NULL COMMENT "用户姓名", + `name` varchar(65533) DEFAULT "unknown" COMMENT "用户姓名", `score` int(11) NOT NULL COMMENT "用户得分", `test` int(11) NULL COMMENT "null test", `dft` int(11) DEFAULT "4321", diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type.groovy new file mode 100644 index 0000000000..1ca7079c97 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_type.groovy @@ -0,0 +1,177 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_primary_key_partial_update_seq_type", "p0") { + def tableName = "test_primary_key_partial_update_seq_type" + + // create table + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `id` int(11) NOT NULL COMMENT "用户 ID", + `name` varchar(65533) DEFAULT "unknown" COMMENT "用户姓名", + `score` int(11) NOT NULL COMMENT "用户得分", + `test` int(11) NULL COMMENT "null test", + `dft` int(11) DEFAULT "4321", + `update_time` date NULL) + UNIQUE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "function_column.sequence_type" = "int" + ) + """ + // insert 2 lines + sql """ + insert into ${tableName} values + (2, "doris2", 2000, 223, 1, '2023-01-01'), + (1, "doris", 1000, 123, 1, '2023-01-01') + """ + + sql "sync" + + qt_select_default """ + select * from ${tableName} order by id; + """ + + // no sequence column header, stream load should fail + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'partial_columns', 'true' + set 'columns', 'id,score,test,update_time' + + file 'basic_with_test.csv' + time 10000 // limit inflight 10s + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("fail", json.Status.toLowerCase()) + assertTrue(json.Message.contains('need to specify the sequence column')) + } + } + + sql "sync" + + // both partial_columns and sequence column header, stream load should success + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'partial_columns', 'true' + set 'columns', 'id,score,test,update_time' + set 'function_column.sequence_col', 'score' + + file 'basic_with_test.csv' + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_partial_update_with_seq_score """ + select * from ${tableName} order by id; + """ + + sql "SET show_hidden_columns=true" + + sql "sync" + + qt_partial_update_with_seq_score_hidden """ + select * from ${tableName} order by id; + """ + + // use test as sequence column + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'partial_columns', 'true' + set 'columns', 'id,score,test,update_time' + set 'function_column.sequence_col', 'test' + + file 'basic_with_test.csv' + time 10000 // limit inflight 10s + } + + sql "SET show_hidden_columns=false" + + sql "sync" + + qt_partial_update_with_seq_test """ + select * from ${tableName} order by id; + """ + + sql "SET show_hidden_columns=true" + + sql "sync" + + qt_partial_update_with_seq_test_hidden """ + select * from ${tableName} order by id; + """ + + // no partial update header, stream load should success, + // but the missing columns will be filled with default values + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'id,score,test,update_time' + set 'function_column.sequence_col', 'score' + + file 'basic_with_test2.csv' + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_select_no_partial_update_score """ + select * from ${tableName} order by id; + """ + + // no partial update header, stream load should success, + // but the missing columns will be filled with default values + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'id,score,test,update_time' + set 'function_column.sequence_col', 'test' + + file 'basic_with_test2.csv' + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_select_no_partial_update_test """ + select * from ${tableName} order by id; + """ + + // drop table + sql """ DROP TABLE IF EXISTS ${tableName} """ +}