From fef627c0ba5c06e722f81352791c03071a8d5e8b Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Tue, 14 Nov 2023 10:30:10 +0800 Subject: [PATCH] [Fix](Txn) Fix transaction write to sequence column error (#26748) --- .../doris/analysis/NativeInsertStmt.java | 4 ++ .../org/apache/doris/qe/StmtExecutor.java | 28 ++++++++++++++ .../unique/test_unique_table.out | 24 ++++++++++++ .../unique/test_unique_table_new_sequence.out | 15 ++++++++ .../unique/test_unique_table_sequence.out | 15 ++++++++ .../unique/test_unique_table.groovy | 37 ++++++++++++++++++ .../test_unique_table_new_sequence.groovy | 38 +++++++++++++++++++ .../unique/test_unique_table_sequence.groovy | 38 +++++++++++++++++++ 8 files changed, 199 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java index e99a539b14..6c72bad553 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java @@ -241,6 +241,10 @@ public class NativeInsertStmt extends InsertStmt { return tblName.getTbl(); } + public List getTargetColumnNames() { + return targetColumnNames; + } + public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) throws AnalysisException { if (tableId != -1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 4ec2110156..b52890ede0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -1704,6 +1704,18 @@ public class StmtExecutor { if (selectStmt.getValueList() != null) { Table tbl = txnEntry.getTable(); int schemaSize = tbl.getBaseSchema(false).size(); + if (parsedStmt instanceof NativeInsertStmt + && ((NativeInsertStmt) parsedStmt).getTargetColumnNames() != null) { + + if (((NativeInsertStmt) parsedStmt).getTargetColumnNames() + .contains(Column.SEQUENCE_COL)) { + schemaSize++; + } + if (((NativeInsertStmt) parsedStmt).getTargetColumnNames() + .contains(Column.DELETE_SIGN)) { + schemaSize++; + } + } for (List row : selectStmt.getValueList().getRows()) { // the value columns are columns which are visible to user, so here we use // getBaseSchema(), not getFullSchema() @@ -1777,6 +1789,22 @@ public class StmtExecutor { .setMergeType(TMergeType.APPEND).setThriftRpcTimeoutMs(5000).setLoadId(context.queryId()) .setExecMemLimit(maxExecMemByte).setTimeout((int) timeoutSecond) .setTimezone(timeZone).setSendBatchParallelism(sendBatchParallelism); + if (parsedStmt instanceof NativeInsertStmt && ((NativeInsertStmt) parsedStmt).getTargetColumnNames() != null) { + List targetColumnNames = ((NativeInsertStmt) parsedStmt).getTargetColumnNames(); + if (targetColumnNames.contains(Column.SEQUENCE_COL) || targetColumnNames.contains(Column.DELETE_SIGN)) { + if (targetColumnNames.contains(Column.SEQUENCE_COL)) { + request.setSequenceCol(Column.SEQUENCE_COL); + } + StringBuilder allCols = new StringBuilder(); + for (String col : ((NativeInsertStmt) parsedStmt).getTargetColumnNames()) { + allCols.append(col); + allCols.append(","); + } + allCols.deleteCharAt(allCols.length() - 1); + request.setColumns(String.valueOf(allCols)); + request.setColumnSeparator(","); + } + } // execute begin txn InsertStreamTxnExecutor executor = new InsertStreamTxnExecutor(txnEntry); diff --git a/regression-test/data/data_model_p0/unique/test_unique_table.out b/regression-test/data/data_model_p0/unique/test_unique_table.out index ccc1edc356..34492bcbcd 100644 --- a/regression-test/data/data_model_p0/unique/test_unique_table.out +++ b/regression-test/data/data_model_p0/unique/test_unique_table.out @@ -8,3 +8,27 @@ int_value INT Yes false \N REPLACE char_value CHAR(10) Yes false \N REPLACE date_value DATE Yes false \N REPLACE +-- !0 -- +k1 INT Yes true \N +v1 TINYINT Yes false \N REPLACE +v2 INT Yes false \N REPLACE +v3 INT Yes false \N REPLACE +v4 INT Yes false \N REPLACE +__DORIS_DELETE_SIGN__ TINYINT No false 0 REPLACE +__DORIS_VERSION_COL__ BIGINT No false 0 REPLACE + +-- !1 -- +1 1 1 1 1 0 2 +2 2 2 2 2 0 2 +3 3 3 3 3 0 2 + +-- !2 -- +1 1 1 1 1 0 2 +2 20 20 20 20 0 3 +3 3 3 3 3 0 2 + +-- !3 -- +1 1 1 1 1 0 2 +2 20 20 20 20 0 3 +3 30 30 30 30 1 4 + diff --git a/regression-test/data/data_model_p0/unique/test_unique_table_new_sequence.out b/regression-test/data/data_model_p0/unique/test_unique_table_new_sequence.out index 54d3b70218..5bb6122519 100644 --- a/regression-test/data/data_model_p0/unique/test_unique_table_new_sequence.out +++ b/regression-test/data/data_model_p0/unique/test_unique_table_new_sequence.out @@ -55,3 +55,18 @@ __DORIS_DELETE_SIGN__ TINYINT No false 0 REPLACE __DORIS_VERSION_COL__ BIGINT No false 0 REPLACE __DORIS_SEQUENCE_COL__ INT Yes false \N REPLACE +-- !1 -- +1 1 1 1 1 0 2 1 +2 2 2 2 2 0 2 2 +3 3 3 3 3 0 2 3 + +-- !2 -- +1 1 1 1 1 0 2 1 +2 20 20 20 20 0 3 20 +3 3 3 3 3 0 2 3 + +-- !3 -- +1 1 1 1 1 0 2 1 +2 20 20 20 20 0 3 20 +3 3 3 3 3 0 2 3 + diff --git a/regression-test/data/data_model_p0/unique/test_unique_table_sequence.out b/regression-test/data/data_model_p0/unique/test_unique_table_sequence.out index 2be696b86e..ecd7fe83d7 100644 --- a/regression-test/data/data_model_p0/unique/test_unique_table_sequence.out +++ b/regression-test/data/data_model_p0/unique/test_unique_table_sequence.out @@ -45,3 +45,18 @@ 2 5 14 13 14 0 5 12 3 6 11 14 15 0 6 13 +-- !1 -- +1 1 1 1 1 0 2 1 +2 2 2 2 2 0 2 2 +3 3 3 3 3 0 2 3 + +-- !2 -- +1 1 1 1 1 0 2 1 +2 20 20 20 20 0 3 20 +3 3 3 3 3 0 2 3 + +-- !3 -- +1 1 1 1 1 0 2 1 +2 20 20 20 20 0 3 20 +3 3 3 3 3 0 2 3 + diff --git a/regression-test/suites/data_model_p0/unique/test_unique_table.groovy b/regression-test/suites/data_model_p0/unique/test_unique_table.groovy index eb9bd2b57d..efa4f1d340 100644 --- a/regression-test/suites/data_model_p0/unique/test_unique_table.groovy +++ b/regression-test/suites/data_model_p0/unique/test_unique_table.groovy @@ -42,4 +42,41 @@ suite("test_unique_table") { order_qt_select_uniq_table "select * from ${tbName}" qt_desc_uniq_table "desc ${tbName}" sql "DROP TABLE ${tbName}" + + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName} ( + `k1` int NULL, + `v1` tinyint NULL, + `v2` int, + `v3` int, + `v4` int + ) ENGINE=OLAP + UNIQUE KEY(k1) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql "SET show_hidden_columns=true" + qt_0 "desc ${tbName}" + sql "begin;" + sql "insert into ${tbName} (k1, v1, v2, v3, v4, __DORIS_DELETE_SIGN__) values (1,1,1,1,1,0),(2,2,2,2,2,0),(3,3,3,3,3,0);" + sql "commit;" + + qt_1 "select * from ${tbName} order by k1;" + + sql "begin;" + sql "insert into ${tbName} (k1, v1, v2, v3, v4, __DORIS_DELETE_SIGN__) values (2,20,20,20,20,0);" + sql "commit;" + + qt_2 "select * from ${tbName} order by k1;" + + sql "begin;" + sql "insert into ${tbName} (k1, v1, v2, v3, v4, __DORIS_DELETE_SIGN__) values (3,30,30,30,30,1);" + sql "commit;" + + qt_3 "select * from ${tbName} order by k1" + + sql "DROP TABLE ${tbName}" } diff --git a/regression-test/suites/data_model_p0/unique/test_unique_table_new_sequence.groovy b/regression-test/suites/data_model_p0/unique/test_unique_table_new_sequence.groovy index 82c4ec0bc1..a2646a4872 100644 --- a/regression-test/suites/data_model_p0/unique/test_unique_table_new_sequence.groovy +++ b/regression-test/suites/data_model_p0/unique/test_unique_table_new_sequence.groovy @@ -122,5 +122,43 @@ suite("test_unique_table_new_sequence") { qt_desc "desc ${tableName}" sql "DROP TABLE ${tableName}" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int NULL, + `v1` tinyint NULL, + `v2` int, + `v3` int, + `v4` int + ) ENGINE=OLAP + UNIQUE KEY(k1) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "function_column.sequence_col" = "v4", + "replication_allocation" = "tag.location.default: 1", + "light_schema_change" = "true" + ); + """ + + sql "begin;" + sql "insert into ${tableName} (k1, v1, v2, v3, v4) values (1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3);" + sql "commit;" + + qt_1 "select * from ${tableName} order by k1;" + + sql "begin;" + sql "insert into ${tableName} (k1, v1, v2, v3, v4) values (2,20,20,20,20);" + sql "commit;" + + qt_2 "select * from ${tableName} order by k1;" + + sql "begin;" + sql "insert into ${tableName} (k1, v1, v2, v3, v4) values (3,30,30,30,1);" + sql "commit;" + + qt_3 "select * from ${tableName} order by k1" + + sql "DROP TABLE ${tableName}" } diff --git a/regression-test/suites/data_model_p0/unique/test_unique_table_sequence.groovy b/regression-test/suites/data_model_p0/unique/test_unique_table_sequence.groovy index 3753391f94..d167bc457f 100644 --- a/regression-test/suites/data_model_p0/unique/test_unique_table_sequence.groovy +++ b/regression-test/suites/data_model_p0/unique/test_unique_table_sequence.groovy @@ -121,5 +121,43 @@ suite("test_unique_table_sequence") { order_qt_all "SELECT * from ${tableName}" sql "DROP TABLE ${tableName}" + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int NULL, + `v1` tinyint NULL, + `v2` int, + `v3` int, + `v4` int + ) ENGINE=OLAP + UNIQUE KEY(k1) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "function_column.sequence_type" = "int", + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql "begin;" + sql "insert into ${tableName} (k1, v1, v2, v3, v4, __DORIS_SEQUENCE_COL__) values (1,1,1,1,1,1),(2,2,2,2,2,2),(3,3,3,3,3,3);" + sql "commit;" + + qt_1 "select * from ${tableName} order by k1;" + + sql "begin;" + sql "insert into ${tableName} (k1, v1, v2, v3, v4, __DORIS_SEQUENCE_COL__) values (2,20,20,20,20,20);" + sql "commit;" + + qt_2 "select * from ${tableName} order by k1;" + + sql "begin;" + sql "insert into ${tableName} (k1, v1, v2, v3, v4, __DORIS_SEQUENCE_COL__) values (3,30,30,30,30,1);" + sql "commit;" + + qt_3 "select * from ${tableName} order by k1" + + sql "DROP TABLE ${tableName}" + }