From 47e33c79873f4277fd39da05a8532dcaa211a4d3 Mon Sep 17 00:00:00 2001 From: Zhengguo Yang Date: Wed, 3 Feb 2021 13:22:00 +0800 Subject: [PATCH] Support create index on unique value column (#5305) * support create index on unique table value columns --- be/src/olap/rowset/segment_v2/segment_writer.cpp | 5 +---- .../apache/doris/alter/SchemaChangeHandler.java | 3 ++- .../java/org/apache/doris/analysis/IndexDef.java | 8 ++++---- .../java/org/apache/doris/catalog/Catalog.java | 2 +- .../doris/common/util/PropertyAnalyzer.java | 15 ++++++--------- .../doris/common/PropertyAnalyzerTest.java | 16 +++++++++------- 6 files changed, 23 insertions(+), 26 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index a7fb66d40d..ad9f3146dc 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -71,12 +71,9 @@ Status SegmentWriter::init(uint32_t write_mbytes_per_sec __attribute__((unused)) // now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS // and not support zone map for array type. - opts.need_zone_map = column.is_key() || _tablet_schema->keys_type() == KeysType::DUP_KEYS; + opts.need_zone_map = column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS; if (column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { opts.need_zone_map = false; - } else { - opts.need_zone_map = - column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS; } opts.need_bloom_filter = column.is_bf_column(); opts.need_bitmap_index = column.has_bitmap_index(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 334f9b1034..2428c5809e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1032,7 +1032,8 @@ public class SchemaChangeHandler extends AlterHandler { Set bfColumns = null; double bfFpp = 0; try { - bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(propertyMap, indexSchemaMap.get(olapTable.getBaseIndexId())); + bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(propertyMap, + indexSchemaMap.get(olapTable.getBaseIndexId()), olapTable.getKeysType()); bfFpp = PropertyAnalyzer.analyzeBloomFilterFpp(propertyMap); } catch (AnalysisException e) { throw new DdlException(e.getMessage()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 19a63a9d63..e2a0042d03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -120,6 +120,7 @@ public class IndexDef { public enum IndexType { BITMAP, + } public void checkColumn(Column column, KeysType keysType) throws AnalysisException { @@ -130,11 +131,10 @@ public class IndexDef { colType.isStringType() || colType == PrimitiveType.BOOLEAN)) { throw new AnalysisException(colType + " is not supported in bitmap index. " + "invalid column: " + indexColName); - } else if (((keysType == KeysType.AGG_KEYS || keysType == KeysType.UNIQUE_KEYS) && !column.isKey()) - || keysType == KeysType.PRIMARY_KEYS) { + } else if ((keysType == KeysType.AGG_KEYS && !column.isKey())) { throw new AnalysisException( - "BITMAP index only used in columns of DUP_KEYS table or key columns of" - + " UNIQUE_KEYS/AGG_KEYS table. invalid column: " + indexColName); + "BITMAP index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of" + + " AGG_KEYS table. invalid column: " + indexColName); } } else { throw new AnalysisException("Unsupported index type: " + indexType); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 2176c05032..021664bdbe 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -3575,7 +3575,7 @@ public class Catalog { Set bfColumns = null; double bfFpp = 0; try { - bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, baseSchema); + bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, baseSchema, keysType); if (bfColumns != null && bfColumns.isEmpty()) { bfColumns = null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 9b726c6c48..5812eb7592 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -18,7 +18,6 @@ package org.apache.doris.common.util; import org.apache.doris.analysis.DateLiteral; -import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DataProperty; import org.apache.doris.catalog.KeysType; @@ -299,8 +298,8 @@ public class PropertyAnalyzer { return schemaVersion; } - public static Set analyzeBloomFilterColumns(Map properties, List columns) - throws AnalysisException { + public static Set analyzeBloomFilterColumns(Map properties, List columns, + KeysType keysType) throws AnalysisException { Set bfColumns = null; if (properties != null && properties.containsKey(PROPERTIES_BF_COLUMNS)) { bfColumns = Sets.newHashSet(); @@ -324,8 +323,7 @@ public class PropertyAnalyzer { || type == PrimitiveType.DOUBLE || type == PrimitiveType.BOOLEAN) { throw new AnalysisException(type + " is not supported in bloom filter index. " + "invalid column: " + bfColumn); - } else if (column.isKey() - || column.getAggregationType() == AggregateType.NONE) { + } else if (keysType != KeysType.AGG_KEYS || column.isKey()) { if (!bfColumnSet.add(bfColumn)) { throw new AnalysisException("Reduplicated bloom filter column: " + bfColumn); } @@ -334,10 +332,9 @@ public class PropertyAnalyzer { found = true; break; } else { - // althrough the implemention supports bf for replace non-key column, - // for simplicity and unity, we don't expose that to user. - throw new AnalysisException("Bloom filter index only used in columns of DUP_KEYS table or " - + "key columns of UNIQUE_KEYS/AGG_KEYS table. invalid column: " + bfColumn); + throw new AnalysisException("Bloom filter index only used in columns of" + + " UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table." + + " invalid column: " + bfColumn); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index 9239a711b0..913499a726 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.DateLiteral; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DataProperty; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; @@ -63,7 +64,7 @@ public class PropertyAnalyzerTest { Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1"); - Set bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns); + Set bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS); Assert.assertEquals(Sets.newHashSet("k1"), bfColumns); } @@ -84,7 +85,8 @@ public class PropertyAnalyzerTest { // no bf columns properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, ""); try { - Assert.assertEquals(Sets.newHashSet(), PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns)); + Assert.assertEquals(Sets.newHashSet(), PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, + KeysType.AGG_KEYS)); } catch (AnalysisException e) { Assert.fail(); } @@ -92,7 +94,7 @@ public class PropertyAnalyzerTest { // k4 not exist properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k4"); try { - PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns); + PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS); } catch (AnalysisException e) { Assert.assertTrue(e.getMessage().contains("column does not exist in table")); } @@ -100,7 +102,7 @@ public class PropertyAnalyzerTest { // tinyint not supported properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k2"); try { - PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns); + PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS); } catch (AnalysisException e) { Assert.assertTrue(e.getMessage().contains("TINYINT is not supported")); } @@ -108,7 +110,7 @@ public class PropertyAnalyzerTest { // bool not supported properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k3"); try { - PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns); + PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS); } catch (AnalysisException e) { Assert.assertTrue(e.getMessage().contains("BOOLEAN is not supported")); } @@ -116,7 +118,7 @@ public class PropertyAnalyzerTest { // not replace value properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "v2"); try { - PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns); + PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS); } catch (AnalysisException e) { Assert.assertTrue(e.getMessage().contains("Bloom filter index only used in")); } @@ -124,7 +126,7 @@ public class PropertyAnalyzerTest { // reduplicated column properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1,K1"); try { - PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns); + PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS); } catch (AnalysisException e) { Assert.assertTrue(e.getMessage().contains("Reduplicated bloom filter column")); }