From 01c94a554d9c5739b21b2b565ef2ef0f2f11a04e Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 14 Dec 2023 18:02:54 +0800 Subject: [PATCH] [fix](autoinc) Fix broker load when target table has autoinc column (#28402) --- .../data_case/autoinc/auto_inc_basic.csv | 9 ++ .../data_case/autoinc/auto_inc_with_null.csv | 9 ++ .../doris/load/loadv2/LoadingTaskPlanner.java | 1 + .../hive/test_autoinc_broker_load.out | 45 +++++++ .../hive/test_autoinc_broker_load.groovy | 113 ++++++++++++++++++ 5 files changed, 177 insertions(+) create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/data_case/autoinc/auto_inc_basic.csv create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/data_case/autoinc/auto_inc_with_null.csv create mode 100644 regression-test/data/external_table_p0/hive/test_autoinc_broker_load.out create mode 100644 regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/data_case/autoinc/auto_inc_basic.csv b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/data_case/autoinc/auto_inc_basic.csv new file mode 100644 index 0000000000..d482056470 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/data_case/autoinc/auto_inc_basic.csv @@ -0,0 +1,9 @@ +Bob, 100 +Alice, 200 +Tom, 300 +Test, 400 +Carter, 500 +Smith, 600 +Beata, 700 +Doris, 800 +Nereids, 900 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/data_case/autoinc/auto_inc_with_null.csv b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/data_case/autoinc/auto_inc_with_null.csv new file mode 100644 index 0000000000..14390ca3cf --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/data_case/autoinc/auto_inc_with_null.csv @@ -0,0 +1,9 @@ +null, Bob, 100 +null, Alice, 200 +null, Tom, 300 +null, Test, 400 +4, Carter, 500 +5, Smith, 600 +6, Beata, 700 +7, Doris, 800 +8, Nereids, 900 \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java index e2f3bdcbe9..7a7687329b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -154,6 +154,7 @@ public class LoadingTaskPlanner { slotDesc.setIsMaterialized(true); slotDesc.setColumn(col); slotDesc.setIsNullable(col.isAllowNull()); + slotDesc.setAutoInc(col.isAutoInc()); SlotDescriptor scanSlotDesc = descTable.addSlotDescriptor(scanTupleDesc); scanSlotDesc.setIsMaterialized(true); scanSlotDesc.setColumn(col); diff --git a/regression-test/data/external_table_p0/hive/test_autoinc_broker_load.out b/regression-test/data/external_table_p0/hive/test_autoinc_broker_load.out new file mode 100644 index 0000000000..cf14e8ea6a --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_autoinc_broker_load.out @@ -0,0 +1,45 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +0 Bob 100 +1 Alice 200 +2 Tom 300 +3 Test 400 +4 Carter 500 +5 Smith 600 +6 Beata 700 +7 Doris 800 +8 Nereids 900 + +-- !sql -- +0 Bob 123 +1 Alice 200 +2 Tom 323 +3 Test 400 +4 Carter 523 +5 Smith 600 +6 Beata 700 +7 Doris 800 +8 Nereids 900 + +-- !sql -- +0 Bob 100 +1 Alice 200 +2 Tom 300 +3 Test 400 +4 Carter 500 +5 Smith 600 +6 Beata 700 +7 Doris 800 +8 Nereids 900 + +-- !sql -- +0 Bob 123 +1 Alice 200 +2 Tom 323 +3 Test 400 +4 Carter 523 +5 Smith 600 +6 Beata 700 +7 Doris 800 +8 Nereids 900 + diff --git a/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy b/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy new file mode 100644 index 0000000000..540714b307 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy @@ -0,0 +1,113 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_autoinc_broker_load", "p0,external,hive,external_docker,external_docker_hive") { + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + brokerName = getBrokerName() + hdfsUser = getHdfsUser() + hdfsPasswd = getHdfsPasswd() + hdfs_port = context.config.otherConfigs.get("hdfs_port") + externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + def test_dir = "user/doris/preinstalled_data/data_case/autoinc" + + def load_from_hdfs = {columns, testTable, label, testFile, format, brokerName, hdfsUser, hdfsPasswd -> + def result1= sql """ LOAD LABEL ${label} ( + DATA INFILE("hdfs://${externalEnvIp}:${hdfs_port}/${test_dir}/${testFile}") + INTO TABLE ${testTable} + COLUMNS TERMINATED BY "," + (${columns}) + ) with HDFS ( + "fs.defaultFS"="hdfs://${externalEnvIp}:${hdfs_port}", + "hadoop.username"="${hdfsUser}") + PROPERTIES ( + "timeout"="1200", + "max_filter_ratio"="0");""" + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Query OK, 0 rows affected") + } + + def wait_for_load_result = {checklabel, testTable -> + max_try_milli_secs = 10000 + while(max_try_milli_secs) { + result = sql "show load where label = '${checklabel}'" + if(result[0][2] == "FINISHED") { + break + } else { + sleep(1000) // wait 1 second every time + max_try_milli_secs -= 1000 + if(max_try_milli_secs <= 0) { + assertEquals(1, 2) + } + } + } + } + + table = "test_autoinc_broker_load" + sql "drop table if exists ${table}" + sql """ CREATE TABLE IF NOT EXISTS `${table}` ( + `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT "用户 ID", + `name` varchar(65533) NOT NULL COMMENT "用户姓名", + `value` int(11) NOT NULL COMMENT "用户得分" + ) ENGINE=OLAP + UNIQUE KEY(`id`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "enable_unique_key_merge_on_write" = "true") """ + + def test_load_label = UUID.randomUUID().toString().replaceAll("-", "") + load_from_hdfs("name, value", table, test_load_label, "auto_inc_basic.csv", "csv", brokerName, hdfsUser, hdfsPasswd) + wait_for_load_result(test_load_label, table) + qt_sql "select * from ${table};" + sql """ insert into ${table} values(0, "Bob", 123), (2, "Tom", 323), (4, "Carter", 523);""" + qt_sql "select * from ${table} order by id" + sql "drop table if exists ${table};" + + + table = "test_autoinc_broker_load2" + sql "drop table if exists ${table}" + sql """ CREATE TABLE IF NOT EXISTS `${table}` ( + `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT "用户 ID", + `name` varchar(65533) NOT NULL COMMENT "用户姓名", + `value` int(11) NOT NULL COMMENT "用户得分" + ) ENGINE=OLAP + UNIQUE KEY(`id`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "enable_unique_key_merge_on_write" = "true");""" + test_load_label = UUID.randomUUID().toString().replaceAll("-", "") + load_from_hdfs("id, name, value", table, test_load_label, "auto_inc_with_null.csv", "csv", brokerName, hdfsUser, hdfsPasswd) + wait_for_load_result(test_load_label, table) + sql "sync" + qt_sql "select * from ${table};" + sql """ insert into ${table} values(0, "Bob", 123), (2, "Tom", 323), (4, "Carter", 523);""" + qt_sql "select * from ${table} order by id" + sql "drop table if exists ${table};" + } +}