diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index b4449ae560..c4d1a4e8d8 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -1112,8 +1112,20 @@ public enum PrimitiveType { return this == MAP; } + public boolean isStructType() { + return this == STRUCT; + } + public boolean isComplexType() { - return this == HLL || this == BITMAP; + return this == ARRAY || this == MAP || this == STRUCT; + } + + public boolean isHllType() { + return this == HLL; + } + + public boolean isBitmapType() { + return this == BITMAP; } public boolean isVariantType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index 237b3b30ff..c2b654e23f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -555,10 +555,11 @@ public class MaterializedViewHandler extends AlterHandler { if (KeysType.UNIQUE_KEYS == olapTable.getKeysType() && olapTable.hasSequenceCol()) { newMVColumns.add(new Column(olapTable.getSequenceCol())); } - // if the column is array type, we forbid to create materialized view + // if the column is complex type, we forbid to create materialized view for (Column column : newMVColumns) { - if (column.getDataType() == PrimitiveType.ARRAY) { - throw new DdlException("The array column[" + column + "] not support to create materialized view"); + if (column.getDataType().isComplexType()) { + throw new DdlException("The " + column.getDataType() + " column[" + column + "] not support " + + "to create materialized view"); } if (addMVClause.getMVKeysType() != KeysType.AGG_KEYS && (column.getType().isBitmapType() || column.getType().isHllType())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 64a4ad1af1..7d9e26f502 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -378,7 +378,7 @@ public class PropertyAnalyzer { // key columns and none/replace aggregate non-key columns support if (type == PrimitiveType.TINYINT || type == PrimitiveType.FLOAT || type == PrimitiveType.DOUBLE || type == PrimitiveType.BOOLEAN - || type.isArrayType()) { + || type.isComplexType()) { throw new AnalysisException(type + " is not supported in bloom filter index. " + "invalid column: " + bfColumn); } else if (keysType != KeysType.AGG_KEYS || column.isKey()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index 167332acfc..7187a9cf52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -268,7 +268,8 @@ public class AggregateStrategies implements ImplementationRuleFactory { // over the length: https://github.com/apache/doris/pull/6293 if (mergeOp == PushDownAggOp.MIN_MAX || mergeOp == PushDownAggOp.MIX) { PrimitiveType colType = column.getType().getPrimitiveType(); - if (colType.isArrayType() || colType.isComplexType() || colType == PrimitiveType.STRING) { + if (colType.isComplexType() || colType.isHllType() || colType.isBitmapType() + || colType == PrimitiveType.STRING) { return canNotPush; } if (colType.isCharFamily() && mergeOp != PushDownAggOp.COUNT && column.getType().getLength() > 512) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 619ee2efe5..5ebb14a52e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -477,7 +477,7 @@ public class SingleNodePlanner { // over the length: https://github.com/apache/doris/pull/6293 if (aggOp == TPushAggOp.MINMAX || aggOp == TPushAggOp.MIX) { PrimitiveType colType = col.getDataType(); - if (colType.isArrayType() || colType.isComplexType() + if (colType.isComplexType() || colType.isHllType() || colType.isBitmapType() || colType == PrimitiveType.STRING) { returnColumnValidate = false; break; diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy index d00e12f200..c719eb0f64 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy @@ -19,10 +19,10 @@ suite("test_bloom_filter") { sql "SHOW ALTER TABLE COLUMN" // bloom filter index for ARRAY column - def test_tb = "test_array_bloom_filter_tb" - sql """DROP TABLE IF EXISTS ${test_tb}""" + def test_array_tb = "test_array_bloom_filter_tb" + sql """DROP TABLE IF EXISTS ${test_array_tb}""" test { - sql """CREATE TABLE IF NOT EXISTS ${test_tb} ( + sql """CREATE TABLE IF NOT EXISTS ${test_array_tb} ( `k1` int(11) NOT NULL, `a1` array NOT NULL ) ENGINE=OLAP @@ -35,7 +35,7 @@ suite("test_bloom_filter") { exception "not supported in bloom filter index" } - sql """CREATE TABLE IF NOT EXISTS ${test_tb} ( + sql """CREATE TABLE IF NOT EXISTS ${test_array_tb} ( `k1` int(11) NOT NULL, `a1` array NOT NULL ) ENGINE=OLAP @@ -46,7 +46,73 @@ suite("test_bloom_filter") { "bloom_filter_columns" = "k1" )""" test { - sql """ALTER TABLE ${test_tb} SET("bloom_filter_columns" = "k1,a1")""" + sql """ALTER TABLE ${test_array_tb} SET("bloom_filter_columns" = "k1,a1")""" + exception "not supported in bloom filter index" + } + + // bloom filter index for STRUCT column + def test_struct_tb = "test_struct_bloom_filter_tb" + sql """DROP TABLE IF EXISTS ${test_struct_tb}""" + sql "ADMIN SET FRONTEND CONFIG ('enable_struct_type' = 'true')" + test { + sql """CREATE TABLE IF NOT EXISTS ${test_struct_tb} ( + `k1` int(11) NOT NULL, + `s1` struct NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1,s1" + )""" + exception "not supported in bloom filter index" + } + + sql """CREATE TABLE IF NOT EXISTS ${test_struct_tb} ( + `k1` int(11) NOT NULL, + `s1` struct NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1" + )""" + test { + sql """ALTER TABLE ${test_struct_tb} SET("bloom_filter_columns" = "k1,s1")""" + exception "not supported in bloom filter index" + } + + // bloom filter index for MAP column + def test_map_tb = "test_map_bloom_filter_tb" + sql """DROP TABLE IF EXISTS ${test_map_tb}""" + sql "ADMIN SET FRONTEND CONFIG ('enable_map_type' = 'true')" + test { + sql """CREATE TABLE IF NOT EXISTS ${test_map_tb} ( + `k1` int(11) NOT NULL, + `m1` map NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1,m1" + )""" + exception "not supported in bloom filter index" + } + + sql """CREATE TABLE IF NOT EXISTS ${test_map_tb} ( + `k1` int(11) NOT NULL, + `m1` map NOT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 + PROPERTIES ( + "replication_num" = "1", + "bloom_filter_columns" = "k1" + )""" + test { + sql """ALTER TABLE ${test_map_tb} SET("bloom_filter_columns" = "k1,m1")""" exception "not supported in bloom filter index" } } diff --git a/regression-test/suites/rollup_p0/test_materialized_view_array.groovy b/regression-test/suites/rollup_p0/test_materialized_view_array.groovy index 2b3b6f16a3..501ff0734c 100644 --- a/regression-test/suites/rollup_p0/test_materialized_view_array.groovy +++ b/regression-test/suites/rollup_p0/test_materialized_view_array.groovy @@ -67,7 +67,7 @@ suite("test_materialized_view_array", "rollup") { create_test_table.call(tableName) test { sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from ${tableName}" - exception "errCode = 2, detailMessage = The array column[`mv_k2` array NULL] not support to create materialized view" + exception "errCode = 2, detailMessage = The ARRAY column[`mv_k2` array NULL] not support to create materialized view" } } finally { try_sql("DROP TABLE IF EXISTS ${tableName}") diff --git a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy new file mode 100644 index 0000000000..5ae06a6c26 --- /dev/null +++ b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_materialized_view_struct", "rollup") { + def tableName = "tbl_test_materialized_view_struct" + + def create_test_table = {testTable -> + + def result1 = sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` struct NULL, + `k3` struct NULL, + `k4` struct NULL, + `k5` struct NULL, + `k6` struct NULL, + `k7` struct NULL, + `k8` struct NULL, + `k9` struct NULL, + `k10` struct NULL, + `k11` struct NULL, + `k12` struct NULL, + `k13` struct NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ) + """ + def result2 = sql """ INSERT INTO ${tableName} VALUES (100, {128}, {32768}, {65535}, {'a'}, {"Doris"}, {'2022-07-13'}, + {'2022-07-13 12:30:00'}, {0.33}, {3.1415926}, {6.67}, {true}, {'how do you do'}) + """ + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 1, "Insert should update 1 rows") + } + + try { + sql "DROP TABLE IF EXISTS ${tableName}" + + create_test_table.call(tableName) + test { + sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from ${tableName}" + exception "errCode = 2, detailMessage = The STRUCT column[`mv_k2` STRUCT NULL] not support to create materialized view" + } + } finally { + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +}