database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Support create index on unique value column (#5305)

Browse Source 
* support create index on unique table value columns
This commit is contained in:
Zhengguo Yang
2021-02-03 13:22:00 +08:00
committed by GitHub
parent ddd85d8ae8
commit 47e33c7987
6 changed files with 23 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
be/src/olap/rowset/segment_v2/segment_writer.cpp
View File

@ -71,12 +71,9 @@ Status SegmentWriter::init(uint32_t write_mbytes_per_sec __attribute__((unused))
// now we create zone map for key columns in AGG_KEYS or all column in UNIQUE_KEYS or DUP_KEYS
// and not support zone map for array type.
opts.need_zone_map = column.is_key() || _tablet_schema->keys_type() == KeysType::DUP_KEYS;
opts.need_zone_map = column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS;
if (column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
opts.need_zone_map = false;
} else {
opts.need_zone_map =
column.is_key() || _tablet_schema->keys_type() != KeysType::AGG_KEYS;
}
opts.need_bloom_filter = column.is_bf_column();
opts.need_bitmap_index = column.has_bitmap_index();

3
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -1032,7 +1032,8 @@ public class SchemaChangeHandler extends AlterHandler {
Set<String> bfColumns = null;
double bfFpp = 0;
try {
bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(propertyMap, indexSchemaMap.get(olapTable.getBaseIndexId()));
bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(propertyMap,
indexSchemaMap.get(olapTable.getBaseIndexId()), olapTable.getKeysType());
bfFpp = PropertyAnalyzer.analyzeBloomFilterFpp(propertyMap);
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());

8
fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
View File

@ -120,6 +120,7 @@ public class IndexDef {
public enum IndexType {
BITMAP,
}
public void checkColumn(Column column, KeysType keysType) throws AnalysisException {
@ -130,11 +131,10 @@ public class IndexDef {
colType.isStringType() || colType == PrimitiveType.BOOLEAN)) {
throw new AnalysisException(colType + " is not supported in bitmap index. "
+ "invalid column: " + indexColName);
} else if (((keysType == KeysType.AGG_KEYS || keysType == KeysType.UNIQUE_KEYS) && !column.isKey())
|| keysType == KeysType.PRIMARY_KEYS) {
} else if ((keysType == KeysType.AGG_KEYS && !column.isKey())) {
throw new AnalysisException(
"BITMAP index only used in columns of DUP_KEYS table or key columns of"
+ " UNIQUE_KEYS/AGG_KEYS table. invalid column: " + indexColName);
"BITMAP index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of"
+ " AGG_KEYS table. invalid column: " + indexColName);
}
} else {
throw new AnalysisException("Unsupported index type: " + indexType);

2
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -3575,7 +3575,7 @@ public class Catalog {
Set<String> bfColumns = null;
double bfFpp = 0;
try {
bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, baseSchema);
bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, baseSchema, keysType);
if (bfColumns != null && bfColumns.isEmpty()) {
bfColumns = null;
}

15
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -18,7 +18,6 @@
package org.apache.doris.common.util;
import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.KeysType;
@ -299,8 +298,8 @@ public class PropertyAnalyzer {
return schemaVersion;
}
public static Set<String> analyzeBloomFilterColumns(Map<String, String> properties, List<Column> columns)
throws AnalysisException {
public static Set<String> analyzeBloomFilterColumns(Map<String, String> properties, List<Column> columns,
KeysType keysType) throws AnalysisException {
Set<String> bfColumns = null;
if (properties != null && properties.containsKey(PROPERTIES_BF_COLUMNS)) {
bfColumns = Sets.newHashSet();
@ -324,8 +323,7 @@ public class PropertyAnalyzer {
|| type == PrimitiveType.DOUBLE || type == PrimitiveType.BOOLEAN) {
throw new AnalysisException(type + " is not supported in bloom filter index. "
+ "invalid column: " + bfColumn);
} else if (column.isKey()
|| column.getAggregationType() == AggregateType.NONE) {
} else if (keysType != KeysType.AGG_KEYS || column.isKey()) {
if (!bfColumnSet.add(bfColumn)) {
throw new AnalysisException("Reduplicated bloom filter column: " + bfColumn);
}
@ -334,10 +332,9 @@ public class PropertyAnalyzer {
found = true;
break;
} else {
// althrough the implemention supports bf for replace non-key column,
// for simplicity and unity, we don't expose that to user.
throw new AnalysisException("Bloom filter index only used in columns of DUP_KEYS table or "
+ "key columns of UNIQUE_KEYS/AGG_KEYS table. invalid column: " + bfColumn);
throw new AnalysisException("Bloom filter index only used in columns of" +
" UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table." +
" invalid column: " + bfColumn);
}
}
}

16
fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
@ -63,7 +64,7 @@ public class PropertyAnalyzerTest {
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1");
Set<String> bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
Set<String> bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
Assert.assertEquals(Sets.newHashSet("k1"), bfColumns);
}
@ -84,7 +85,8 @@ public class PropertyAnalyzerTest {
// no bf columns
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "");
try {
Assert.assertEquals(Sets.newHashSet(), PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns));
Assert.assertEquals(Sets.newHashSet(), PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns,
KeysType.AGG_KEYS));
} catch (AnalysisException e) {
Assert.fail();
}
@ -92,7 +94,7 @@ public class PropertyAnalyzerTest {
// k4 not exist
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k4");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("column does not exist in table"));
}
@ -100,7 +102,7 @@ public class PropertyAnalyzerTest {
// tinyint not supported
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k2");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("TINYINT is not supported"));
}
@ -108,7 +110,7 @@ public class PropertyAnalyzerTest {
// bool not supported
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k3");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("BOOLEAN is not supported"));
}
@ -116,7 +118,7 @@ public class PropertyAnalyzerTest {
// not replace value
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "v2");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("Bloom filter index only used in"));
}
@ -124,7 +126,7 @@ public class PropertyAnalyzerTest {
// reduplicated column
properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1,K1");
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("Reduplicated bloom filter column"));
}