diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index b5ee423151..38d195c6d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -554,10 +554,14 @@ public class MaterializedViewHandler extends AlterHandler { } } if (KeysType.UNIQUE_KEYS == olapTable.getKeysType() && olapTable.hasDeleteSign()) { - newMVColumns.add(new Column(olapTable.getDeleteSignColumn())); + Column newColumn = new Column(olapTable.getDeleteSignColumn()); + newColumn.setAggregationType(AggregateType.REPLACE, true); + newMVColumns.add(newColumn); } if (KeysType.UNIQUE_KEYS == olapTable.getKeysType() && olapTable.hasSequenceCol()) { - newMVColumns.add(new Column(olapTable.getSequenceCol())); + Column newColumn = new Column(olapTable.getSequenceCol()); + newColumn.setAggregationType(AggregateType.REPLACE, true); + newMVColumns.add(newColumn); } // if the column is complex type, we forbid to create materialized view for (Column column : newMVColumns) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 4a75fd086e..d7a47f9279 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -316,14 +316,6 @@ public class CreateTableStmt extends DdlStmt { analyzeEngineName(); - // `analyzeXXX` would modify `properties`, which will be used later, - // so we just clone a properties map here. - boolean enableUniqueKeyMergeOnWrite = false; - boolean enableStoreRowColumn = false; - if (properties != null) { - enableUniqueKeyMergeOnWrite = PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite(new HashMap<>(properties)); - enableStoreRowColumn = PropertyAnalyzer.analyzeStoreRowColumn(new HashMap<>(properties)); - } //pre-block creation with column type ALL for (ColumnDef columnDef : columnDefs) { if (Objects.equals(columnDef.getType(), Type.ALL)) { @@ -337,6 +329,9 @@ public class CreateTableStmt extends DdlStmt { + "please use `DECIMALV3`."); } } + + boolean enableUniqueKeyMergeOnWrite = false; + boolean enableStoreRowColumn = false; // analyze key desc if (engineName.equalsIgnoreCase("olap")) { // olap table @@ -401,6 +396,23 @@ public class CreateTableStmt extends DdlStmt { } } + if (properties != null && properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE) + && keysDesc.getKeysType() != KeysType.UNIQUE_KEYS) { + throw new AnalysisException( + PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE + " property only support unique key table"); + } + + if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) { + enableUniqueKeyMergeOnWrite = true; + if (properties != null) { + // `analyzeXXX` would modify `properties`, which will be used later, + // so we just clone a properties map here. + enableUniqueKeyMergeOnWrite = PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite( + new HashMap<>(properties)); + enableStoreRowColumn = PropertyAnalyzer.analyzeStoreRowColumn(new HashMap<>(properties)); + } + } + keysDesc.analyze(columnDefs); for (int i = 0; i < keysDesc.keysColumnSize(); ++i) { columnDefs.get(i).setIsKey(true); @@ -410,7 +422,7 @@ public class CreateTableStmt extends DdlStmt { if (keysDesc.getKeysType() == KeysType.DUP_KEYS) { type = AggregateType.NONE; } - if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite) { + if (enableUniqueKeyMergeOnWrite) { type = AggregateType.NONE; } for (int i = keysDesc.keysColumnSize(); i < columnDefs.size(); ++i) { @@ -508,7 +520,7 @@ public class CreateTableStmt extends DdlStmt { if (partitionDesc != null) { if (partitionDesc instanceof ListPartitionDesc || partitionDesc instanceof RangePartitionDesc || partitionDesc instanceof ColumnPartitionDesc) { - partitionDesc.analyze(columnDefs, properties); + partitionDesc.analyze(columnDefs, properties, keysDesc); } else { throw new AnalysisException("Currently only support range" + " and list partition with engine type olap"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java index 31a28da280..0c2c3b2922 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java @@ -17,14 +17,13 @@ package org.apache.doris.analysis; -import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; @@ -76,18 +75,12 @@ public class PartitionDesc { return partitionColNames; } - public void analyze(List columnDefs, Map otherProperties) throws AnalysisException { + public void analyze(List columnDefs, Map otherProperties, KeysDesc keysDesc) + throws AnalysisException { if (partitionColNames == null || partitionColNames.isEmpty()) { throw new AnalysisException("No partition columns."); } - // `analyzeUniqueKeyMergeOnWrite` would modify `properties`, which will be used later, - // so we just clone a properties map here. - boolean enableUniqueKeyMergeOnWrite = false; - if (otherProperties != null) { - enableUniqueKeyMergeOnWrite = - PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite(Maps.newHashMap(otherProperties)); - } Set partColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (String partitionCol : partitionColNames) { if (!partColNames.add(partitionCol)) { @@ -97,9 +90,10 @@ public class PartitionDesc { boolean found = false; for (ColumnDef columnDef : columnDefs) { if (columnDef.getName().equals(partitionCol)) { - if (!columnDef.isKey() && (columnDef.getAggregateType() != AggregateType.NONE - || enableUniqueKeyMergeOnWrite)) { - throw new AnalysisException("The partition column could not be aggregated column"); + if (!columnDef.isKey() && keysDesc.getKeysType() != KeysType.DUP_KEYS) { + throw new AnalysisException( + "The partition column must be key column in " + keysDesc.getKeysType().toSql() + + " table"); } if (columnDef.getType().isFloatingPointType()) { throw new AnalysisException("Floating point type column can not be partition column"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index afed3460ef..d9c359d6e9 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -2964,7 +2964,7 @@ public class Env { } // unique key table with merge on write - if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite()) { + if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && !olapTable.getEnableUniqueKeyMergeOnWrite()) { sb.append(",\n\"").append(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE).append("\" = \""); sb.append(olapTable.getEnableUniqueKeyMergeOnWrite()).append("\""); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 155cad1707..e42c1e3d36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -776,12 +776,10 @@ public class PropertyAnalyzer { public static boolean analyzeUniqueKeyMergeOnWrite(Map properties) throws AnalysisException { if (properties == null || properties.isEmpty()) { - return false; - } - String value = properties.get(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE); - if (value == null) { - return false; + // enable merge on write by default + return true; } + String value = properties.getOrDefault(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "true"); properties.remove(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE); if (value.equals("true")) { return true; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java index b826f5e1e9..a3f13a97ba 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java @@ -129,6 +129,7 @@ public class CreateTableStmtTest { public void testCreateTableUniqueKeyNormal() throws UserException { // setup Map properties = new HashMap<>(); + properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"); ColumnDef col3 = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.BIGINT))); col3.setIsKey(false); cols.add(col3); @@ -151,7 +152,6 @@ public class CreateTableStmtTest { public void testCreateTableUniqueKeyMoW() throws UserException { // setup Map properties = new HashMap<>(); - properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "true"); ColumnDef col3 = new ColumnDef("col3", new TypeDef(ScalarType.createType(PrimitiveType.BIGINT))); col3.setIsKey(false); cols.add(col3); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java index fe9adf9ccd..6bb8f0b170 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateTableStmtTest.java @@ -40,6 +40,14 @@ public class ShowCreateTableStmtTest extends TestWithFeService { + ") " + "distributed by hash(k1) buckets 1\n" + "properties(\"replication_num\" = \"1\");"); + createTable("create table table2\n" + + "(k1 int comment 'test column k1', k2 int comment 'test column k2', v1 string comment 'test column v1') " + + "unique key(k1, k2) distributed by hash(k1) buckets 1\n" + + "properties(\"replication_num\" = \"1\");"); + createTable("create table table3\n" + + "(k1 int comment 'test column k1', k2 int comment 'test column k2', v1 string comment 'test column v1') " + + "unique key(k1, k2) distributed by hash(k1) buckets 1\n" + + "properties(\"replication_num\" = \"1\", \"enable_unique_key_merge_on_write\" = \"false\");"); } @@ -60,4 +68,21 @@ public class ShowCreateTableStmtTest extends TestWithFeService { Assertions.assertTrue(!showSql.contains("PARTITION BY")); Assertions.assertTrue(!showSql.contains("PARTITION `p01`")); } + + @Test + public void testUniqueKeyMoW() throws Exception { + String propertyStr = "\"enable_unique_key_merge_on_write\" = \"false\""; + + String sql1 = "show create table table2"; + ShowResultSet showResultSet1 = showCreateTable(sql1); + String showSql1 = showResultSet1.getResultRows().get(0).get(1); + Assertions.assertTrue(showSql1.contains("`k1` int(11) NULL COMMENT 'test column k1'")); + Assertions.assertFalse(showSql1.contains(propertyStr)); + + String sql2 = "show create table table3"; + ShowResultSet showResultSet2 = showCreateTable(sql2); + String showSql2 = showResultSet2.getResultRows().get(0).get(1); + Assertions.assertTrue(showSql2.contains("`k1` int(11) NULL COMMENT 'test column k1'")); + Assertions.assertTrue(showSql2.contains(propertyStr)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java index 5037327005..35abb9b2e0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java @@ -233,10 +233,10 @@ public class CreateTableTest { Assert.assertTrue(tbl8.getColumn("k1").isKey()); Assert.assertTrue(tbl8.getColumn("k2").isKey()); Assert.assertFalse(tbl8.getColumn("v1").isKey()); - Assert.assertTrue(tbl8.getColumn(Column.SEQUENCE_COL).getAggregationType() == AggregateType.REPLACE); + Assert.assertTrue(tbl8.getColumn(Column.SEQUENCE_COL).getAggregationType() == AggregateType.NONE); OlapTable tbl13 = (OlapTable) db.getTableOrDdlException("tbl13"); - Assert.assertTrue(tbl13.getColumn(Column.SEQUENCE_COL).getAggregationType() == AggregateType.REPLACE); + Assert.assertTrue(tbl13.getColumn(Column.SEQUENCE_COL).getAggregationType() == AggregateType.NONE); Assert.assertTrue(tbl13.getColumn(Column.SEQUENCE_COL).getType() == Type.INT); Assert.assertEquals(tbl13.getSequenceMapCol(), "v1"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java index 2e9b53b4b1..6dde291283 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectMvIndexTest.java @@ -744,7 +744,7 @@ class SelectMvIndexTest extends BaseMaterializedIndexSelectTest implements MemoP + "k2;"; createMv(createK1MV); String query = "select * from " + TEST_TABLE_NAME + ";"; - singleTableTest(query, TEST_TABLE_NAME, false); + singleTableTest(query, TEST_TABLE_NAME, true); } /** diff --git a/regression-test/data/data_model_p0/unique/test_unique_mow_table_debug_data.out b/regression-test/data/data_model_p0/unique/test_unique_mow_table_debug_data.out new file mode 100644 index 0000000000..097eeffaed --- /dev/null +++ b/regression-test/data/data_model_p0/unique/test_unique_mow_table_debug_data.out @@ -0,0 +1,36 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_init -- +1 11 +2 11 +3 1 + +-- !select_skip_delete_bitmap -- +1 1 0 2 +1 11 0 3 +2 1 0 2 +2 11 0 3 +3 1 0 4 + +-- !select_batch_delete -- +2 11 +3 1 + +-- !select_sql_delete -- +3 1 + +-- !select_skip_delete1 -- +2 11 +3 1 + +-- !select_skip_delete_bitmap_after_batch_delete -- +1 1 0 2 +1 11 0 3 +3 1 0 4 + +-- !select_skip_delete2 -- +1 1 0 2 +1 11 0 3 +2 1 0 2 +2 11 0 3 +3 1 0 4 + diff --git a/regression-test/data/data_model_p0/unique/test_unique_mow_table_debug_data_delete.csv b/regression-test/data/data_model_p0/unique/test_unique_mow_table_debug_data_delete.csv new file mode 100644 index 0000000000..1c19a02704 --- /dev/null +++ b/regression-test/data/data_model_p0/unique/test_unique_mow_table_debug_data_delete.csv @@ -0,0 +1 @@ +1|111 diff --git a/regression-test/data/data_model_p0/unique/test_unique_table.out b/regression-test/data/data_model_p0/unique/test_unique_table.out index ccc1edc356..9745f9fc6b 100644 --- a/regression-test/data/data_model_p0/unique/test_unique_table.out +++ b/regression-test/data/data_model_p0/unique/test_unique_table.out @@ -4,7 +4,7 @@ -- !desc_uniq_table -- k INT Yes true \N -int_value INT Yes false \N REPLACE -char_value CHAR(10) Yes false \N REPLACE -date_value DATE Yes false \N REPLACE +int_value INT Yes false \N NONE +char_value CHAR(10) Yes false \N NONE +date_value DATE Yes false \N NONE diff --git a/regression-test/data/data_model_p0/unique/test_unique_table_like.out b/regression-test/data/data_model_p0/unique/test_unique_table_like.out index dd68e453ff..f6e03d26df 100644 --- a/regression-test/data/data_model_p0/unique/test_unique_table_like.out +++ b/regression-test/data/data_model_p0/unique/test_unique_table_like.out @@ -1,19 +1,19 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !desc_uniq_table -- k INT Yes true \N -int_value INT Yes false \N REPLACE -char_value CHAR(10) Yes false \N REPLACE -date_value DATE Yes false \N REPLACE -__DORIS_DELETE_SIGN__ TINYINT No false 0 REPLACE -__DORIS_VERSION_COL__ BIGINT No false 0 REPLACE -__DORIS_SEQUENCE_COL__ INT Yes false \N REPLACE +int_value INT Yes false \N NONE +char_value CHAR(10) Yes false \N NONE +date_value DATE Yes false \N NONE +__DORIS_DELETE_SIGN__ TINYINT No false 0 NONE +__DORIS_VERSION_COL__ BIGINT No false 0 NONE +__DORIS_SEQUENCE_COL__ INT Yes false \N NONE -- !desc_uniq_table -- k INT Yes true \N -int_value INT Yes false \N REPLACE -char_value CHAR(10) Yes false \N REPLACE -date_value DATE Yes false \N REPLACE -__DORIS_DELETE_SIGN__ TINYINT No false 0 REPLACE -__DORIS_VERSION_COL__ BIGINT No false 0 REPLACE -__DORIS_SEQUENCE_COL__ INT Yes false \N REPLACE +int_value INT Yes false \N NONE +char_value CHAR(10) Yes false \N NONE +date_value DATE Yes false \N NONE +__DORIS_DELETE_SIGN__ TINYINT No false 0 NONE +__DORIS_VERSION_COL__ BIGINT No false 0 NONE +__DORIS_SEQUENCE_COL__ INT Yes false \N NONE diff --git a/regression-test/data/data_model_p0/unique/test_unique_table_new_sequence.out b/regression-test/data/data_model_p0/unique/test_unique_table_new_sequence.out index 54d3b70218..5a5dd5b2be 100644 --- a/regression-test/data/data_model_p0/unique/test_unique_table_new_sequence.out +++ b/regression-test/data/data_model_p0/unique/test_unique_table_new_sequence.out @@ -47,11 +47,11 @@ -- !desc -- k1 INT Yes true \N -v1 TINYINT Yes false \N REPLACE -v2 INT Yes false \N REPLACE -v3 INT Yes false \N REPLACE -v4 INT Yes false \N REPLACE -__DORIS_DELETE_SIGN__ TINYINT No false 0 REPLACE -__DORIS_VERSION_COL__ BIGINT No false 0 REPLACE -__DORIS_SEQUENCE_COL__ INT Yes false \N REPLACE +v1 TINYINT Yes false \N NONE +v2 INT Yes false \N NONE +v3 INT Yes false \N NONE +v4 INT Yes false \N NONE +__DORIS_DELETE_SIGN__ TINYINT No false 0 NONE +__DORIS_VERSION_COL__ BIGINT No false 0 NONE +__DORIS_SEQUENCE_COL__ INT Yes false \N NONE diff --git a/regression-test/data/index_p0/test_bitmap_index.out b/regression-test/data/index_p0/test_bitmap_index.out index 5eb7f08b93..104f80acdc 100644 --- a/regression-test/data/index_p0/test_bitmap_index.out +++ b/regression-test/data/index_p0/test_bitmap_index.out @@ -86,11 +86,11 @@ k8 DATETIME Yes true \N k9 LARGEINT Yes true \N k10 DECIMAL(9, 0) Yes true \N k11 BOOLEAN Yes true \N -k12 DATE Yes false \N REPLACE -k13 DATETIME Yes false \N REPLACE -k14 DATETIME Yes false \N REPLACE -k15 DATETIME Yes false \N REPLACE -v1 INT Yes false \N REPLACE +k12 DATE Yes false \N NONE +k13 DATETIME Yes false \N NONE +k14 DATETIME Yes false \N NONE +k15 DATETIME Yes false \N NONE +v1 INT Yes false \N NONE -- !sql -- default_cluster:regression_test_index_p0.test_bitmap_index_unique index1 k1 BITMAP diff --git a/regression-test/data/inverted_index_p0/test_bitmap_index.out b/regression-test/data/inverted_index_p0/test_bitmap_index.out index fa429e6fe9..bdff99de7a 100644 --- a/regression-test/data/inverted_index_p0/test_bitmap_index.out +++ b/regression-test/data/inverted_index_p0/test_bitmap_index.out @@ -70,7 +70,7 @@ k8 DATETIME Yes true \N k9 LARGEINT Yes true \N k10 DECIMAL(9, 0) Yes true \N k11 BOOLEAN Yes true \N -v1 INT Yes false \N REPLACE +v1 INT Yes false \N NONE -- !sql -- default_cluster:regression_test_inverted_index_p0.test_bitmap_index_unique index1 k1 BITMAP diff --git a/regression-test/data/schema_change_p0/test_rename_column.out b/regression-test/data/schema_change_p0/test_rename_column.out index 3e8d56d043..bc1966690b 100644 Binary files a/regression-test/data/schema_change_p0/test_rename_column.out and b/regression-test/data/schema_change_p0/test_rename_column.out differ diff --git a/regression-test/data/schema_change_p0/test_schema_change.out b/regression-test/data/schema_change_p0/test_schema_change.out index 84ee227ec5..43857c0a39 100644 --- a/regression-test/data/schema_change_p0/test_schema_change.out +++ b/regression-test/data/schema_change_p0/test_schema_change.out @@ -2,9 +2,9 @@ -- !desc_uniq_table -- event_day DATE Yes true \N siteid INT Yes true 10 -citycode TEXT Yes false \N REPLACE -username VARCHAR(32) Yes false REPLACE -pv BIGINT Yes false 0 REPLACE +citycode TEXT Yes false \N NONE +username VARCHAR(32) Yes false NONE +pv BIGINT Yes false 0 NONE -- !sql -- 2021-11-01 1 1 用户A 3 diff --git a/regression-test/data/schema_change_p0/test_uniq_delete_sign_schema_change.out b/regression-test/data/schema_change_p0/test_uniq_delete_sign_schema_change.out index bc07e62e29..6cf1255622 100644 --- a/regression-test/data/schema_change_p0/test_uniq_delete_sign_schema_change.out +++ b/regression-test/data/schema_change_p0/test_uniq_delete_sign_schema_change.out @@ -16,12 +16,12 @@ -- !sql -- k1 INT Yes true \N -value1 INT Yes false \N REPLACE -value2 INT Yes false \N REPLACE -value3 INT Yes false \N REPLACE -value4 INT Yes false \N REPLACE -__DORIS_DELETE_SIGN__ TINYINT No false 0 REPLACE -__DORIS_VERSION_COL__ BIGINT No false 0 REPLACE +value1 INT Yes false \N NONE +value2 INT Yes false \N NONE +value3 INT Yes false \N NONE +value4 INT Yes false \N NONE +__DORIS_DELETE_SIGN__ TINYINT No false 0 NONE +__DORIS_VERSION_COL__ BIGINT No false 0 NONE -- !sql -- 1 1 1 1 \N 1 7 @@ -47,10 +47,10 @@ __DORIS_VERSION_COL__ BIGINT No false 0 REPLACE -- !sql -- k1 INT Yes true \N -value2 INT Yes false \N REPLACE -value4 INT Yes false \N REPLACE -__DORIS_DELETE_SIGN__ TINYINT No false 0 REPLACE -__DORIS_VERSION_COL__ BIGINT No false 0 REPLACE +value2 INT Yes false \N NONE +value4 INT Yes false \N NONE +__DORIS_DELETE_SIGN__ TINYINT No false 0 NONE +__DORIS_VERSION_COL__ BIGINT No false 0 NONE -- !sql -- 1 1 \N 1 7 diff --git a/regression-test/data/types_p0/unsigned/test_unsigned_int_compatibility.out b/regression-test/data/types_p0/unsigned/test_unsigned_int_compatibility.out index 0f8cd0f99b..b76de33e63 100644 --- a/regression-test/data/types_p0/unsigned/test_unsigned_int_compatibility.out +++ b/regression-test/data/types_p0/unsigned/test_unsigned_int_compatibility.out @@ -2,7 +2,7 @@ -- !desc_tb -- user_id LARGEINT No true \N city VARCHAR(20) Yes true \N -value1 BIGINT Yes false \N REPLACE +value1 BIGINT Yes false \N NONE -- !select_tb -- 1 Beijing 21474836478 @@ -10,8 +10,8 @@ value1 BIGINT Yes false \N REPLACE -- !desc_tb -- user_id LARGEINT No true \N city VARCHAR(20) Yes true \N -value1 BIGINT Yes false \N REPLACE -value2 BIGINT Yes false \N REPLACE +value1 BIGINT Yes false \N NONE +value2 BIGINT Yes false \N NONE -- !select_tb -- 1 Beijing 21474836478 \N diff --git a/regression-test/data/update/test_update_unique.out b/regression-test/data/update/test_update_unique.out index 051bd679ab..615ee7b07d 100644 --- a/regression-test/data/update/test_update_unique.out +++ b/regression-test/data/update/test_update_unique.out @@ -9,9 +9,9 @@ -- !desc_uniq_table -- k INT Yes true \N -value1 INT Yes false \N REPLACE -value2 INT Yes false \N REPLACE -date_value DATE Yes false \N REPLACE +value1 INT Yes false \N NONE +value2 INT Yes false \N NONE +date_value DATE Yes false \N NONE -- !complex_update -- 1 10 1 1000.0 2000-01-01 diff --git a/regression-test/suites/cold_heat_separation/empty_table_use_policy/alter_table_add_policy.groovy b/regression-test/suites/cold_heat_separation/empty_table_use_policy/alter_table_add_policy.groovy index c4b257953d..73fb57ecb8 100644 --- a/regression-test/suites/cold_heat_separation/empty_table_use_policy/alter_table_add_policy.groovy +++ b/regression-test/suites/cold_heat_separation/empty_table_use_policy/alter_table_add_policy.groovy @@ -26,7 +26,8 @@ suite("add_table_policy_by_alter_table") { UNIQUE KEY(k1) DISTRIBUTED BY HASH (k1) BUCKETS 3 PROPERTIES( - "replication_num" = "1" + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" ); """ assertEquals(create_table_not_have_policy_result.size(), 1); diff --git a/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_policy.groovy b/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_policy.groovy index d4e3aa1774..77e9f5a186 100644 --- a/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_policy.groovy +++ b/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_policy.groovy @@ -29,7 +29,8 @@ suite("create_table_use_policy") { DISTRIBUTED BY HASH (k1) BUCKETS 3 PROPERTIES( "storage_policy" = "not_exist_policy", - "replication_num" = "1" + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" ); """ @@ -85,7 +86,8 @@ suite("create_table_use_policy") { DISTRIBUTED BY HASH (k1) BUCKETS 3 PROPERTIES( "storage_policy" = "test_create_table_use_policy", - "replication_num" = "1" + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" ); """ diff --git a/regression-test/suites/cold_heat_separation/policy/drop.groovy b/regression-test/suites/cold_heat_separation/policy/drop.groovy index 8157df8f11..9d5a46e4a4 100644 --- a/regression-test/suites/cold_heat_separation/policy/drop.groovy +++ b/regression-test/suites/cold_heat_separation/policy/drop.groovy @@ -122,7 +122,8 @@ suite("drop_policy") { DISTRIBUTED BY HASH (k1) BUCKETS 3 PROPERTIES( "storage_policy" = "drop_policy_test_has_table_binded", - "replication_num" = "1" + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" ); """ diff --git a/regression-test/suites/data_model_p0/unique/test_unique_mow_table_debug_data.groovy b/regression-test/suites/data_model_p0/unique/test_unique_mow_table_debug_data.groovy new file mode 100644 index 0000000000..715e2e714f --- /dev/null +++ b/regression-test/suites/data_model_p0/unique/test_unique_mow_table_debug_data.groovy @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_unique_mow_table_debug_data") { + + sql "ADMIN SET FRONTEND CONFIG ('enable_batch_delete_by_default' = 'true')" + sql "SET show_hidden_columns=false" + sql "SET skip_delete_predicate=false" + sql "SET skip_delete_bitmap=false" + + def tbName = "test_unique_mow_table_debug_data" + sql "DROP TABLE IF EXISTS ${tbName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbName} ( + a int, b int + ) + unique key (a) + distributed by hash(a) buckets 16 + properties( + "replication_allocation" = "tag.location.default:1", + "disable_auto_compaction" = "true" + ); + """ + + //BackendId,Cluster,IP,HeartbeatPort,BePort,HttpPort,BrpcPort,LastStartTime,LastHeartbeat,Alive,SystemDecommissioned,ClusterDecommissioned,TabletNum,DataUsedCapacity,AvailCapacity,TotalCapacity,UsedPct,MaxDiskUsedPct,Tag,ErrMsg,Version,Status + String[][] backends = sql """ show backends; """ + assertTrue(backends.size() > 0) + StringBuilder sbCommand = new StringBuilder(); + + sql "insert into ${tbName} values(1,1),(2,1);" + sql "insert into ${tbName} values(1,11),(2,11);" + sql "insert into ${tbName} values(3,1);" + sql "sync" + + qt_select_init "select * from ${tbName} order by a, b" + + // enable skip_delete_bitmap and check select result, + // the rows that have duplicate primary key and marked delete, will be returned + sql "SET skip_delete_bitmap=true" + qt_select_skip_delete_bitmap "select * from ${tbName} order by a, b" + + // turn off skip_delete_bitmap + sql "SET skip_delete_bitmap=false" + + // batch delete and select again: + // curl --location-trusted -uroot: -H "column_separator:|" -H "columns:a, b" -H "merge_type: delete" -T delete.csv http://127.0.0.1:8030/api/test_skip/t1/_stream_load + streamLoad { + table "${tbName}" + + set 'column_separator', '|' + set 'columns', 'a, b' + set 'merge_type', 'delete' + + file 'test_unique_mow_table_debug_data_delete.csv' + + time 10000 // limit inflight 10s + } + sql "sync" + qt_select_batch_delete "select * from ${tbName} order by a, b" + + // delete rows with a = 2: + sql "delete from ${tbName} where a = 2;" + sql "sync" + qt_select_sql_delete "select * from ${tbName} order by a, b" + + // enable skip_delete_predicate, rows deleted with delete statement is returned: + sql "SET skip_delete_predicate=true" + qt_select_skip_delete1 "select * from ${tbName} order by a, b" + + sql "SET skip_delete_predicate=false" + + sql "SET skip_delete_bitmap=true" + qt_select_skip_delete_bitmap_after_batch_delete "select * from ${tbName} order by a, b" + + // enable skip_delete_predicate, rows deleted with delete statement is also returned: + sql "SET skip_delete_predicate=true" + qt_select_skip_delete2 "select * from ${tbName} order by a, b" + + sql "DROP TABLE ${tbName}" +} diff --git a/regression-test/suites/data_model_p0/unique/test_unique_table_debug_data.groovy b/regression-test/suites/data_model_p0/unique/test_unique_table_debug_data.groovy index db0c4e4ac7..e631146e5c 100644 --- a/regression-test/suites/data_model_p0/unique/test_unique_table_debug_data.groovy +++ b/regression-test/suites/data_model_p0/unique/test_unique_table_debug_data.groovy @@ -34,7 +34,8 @@ suite("test_unique_table_debug_data") { distributed by hash(a) buckets 16 properties( "replication_allocation" = "tag.location.default:1", - "disable_auto_compaction" = "true" + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false" ); """ diff --git a/regression-test/suites/unique_with_mow_p0/test_create_table.groovy b/regression-test/suites/unique_with_mow_p0/test_create_table.groovy index 9a3ad4b2a5..fdc382fbb0 100644 --- a/regression-test/suites/unique_with_mow_p0/test_create_table.groovy +++ b/regression-test/suites/unique_with_mow_p0/test_create_table.groovy @@ -35,7 +35,7 @@ suite("test_create_table") { "enable_unique_key_merge_on_write" = "true" ); """ - exception "Unknown properties" + exception "only support unique key table" } // duplicate table with enable_unique_key_merge_on_write property @@ -54,7 +54,7 @@ suite("test_create_table") { "enable_unique_key_merge_on_write" = "false" ); """ - exception "Unknown properties" + exception "only support unique key table" } // agg table with enable_unique_key_merge_on_write property @@ -73,7 +73,7 @@ suite("test_create_table") { "enable_unique_key_merge_on_write" = "true" ); """ - exception "Unknown properties" + exception "only support unique key table" } // agg table with enable_unique_key_merge_on_write property @@ -92,6 +92,6 @@ suite("test_create_table") { "enable_unique_key_merge_on_write" = "false" ); """ - exception "Unknown properties" + exception "only support unique key table" } } \ No newline at end of file