database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](index) Replace BITMAP INDEX with INVERTED INDEX (#30950)

Browse Source
This commit is contained in:
Kang
2024-02-08 22:33:13 +08:00
committed by yiguolei
parent eaaab33f0a
commit ce892d04e5
66 changed files with 102 additions and 562 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/cup/sql_parser.cup
View File

@ -3807,7 +3807,7 @@ opt_index_type ::=
:}
| KW_USING KW_BITMAP
{:
RESULT = IndexDef.IndexType.BITMAP;
RESULT = IndexDef.IndexType.INVERTED;
:}
| KW_USING KW_NGRAM_BF
{:

24
fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
View File

@ -55,7 +55,7 @@ public class IndexDef {
this.ifNotExists = ifNotExists;
this.columns = columns;
if (indexType == null) {
this.indexType = IndexType.BITMAP;
this.indexType = IndexType.INVERTED;
} else {
this.indexType = indexType;
}
@ -222,18 +222,14 @@ public class IndexDef {
|| colType.isFixedPointType() || colType.isStringType() || colType == PrimitiveType.BOOLEAN
|| colType.isVariantType())) {
throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. "
+ "invalid column: " + indexColName);
} else if (indexType == IndexType.INVERTED
&& ((keysType == KeysType.AGG_KEYS && !column.isKey())
|| (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite))) {
+ "invalid index: " + indexName);
}
if (!column.isKey()
&& ((keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite)
|| keysType == KeysType.AGG_KEYS)) {
throw new AnalysisException(indexType.toString()
+ " index only used in columns of DUP_KEYS table"
+ " or UNIQUE_KEYS table with merge_on_write enabled"
+ " or key columns of AGG_KEYS table. invalid column: " + indexColName);
} else if (keysType == KeysType.AGG_KEYS && !column.isKey() && indexType != IndexType.INVERTED) {
throw new AnalysisException(indexType.toString()
+ " index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of"
+ " AGG_KEYS table. invalid column: " + indexColName);
+ " index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table."
+ " invalid index: " + indexName);
}
if (indexType == IndexType.INVERTED) {
@ -243,10 +239,6 @@ public class IndexDef {
&& colType != PrimitiveType.STRING) {
throw new AnalysisException(colType + " is not supported in ngram_bf index. "
+ "invalid column: " + indexColName);
} else if ((keysType == KeysType.AGG_KEYS && !column.isKey())) {
throw new AnalysisException(
"ngram_bf index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of"
+ " AGG_KEYS table. invalid column: " + indexColName);
}
if (properties.size() != 2) {
throw new AnalysisException("ngram_bf index should have gram_size and bf_size properties");

4
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
View File

@ -2527,6 +2527,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
Map<String, String> properties = visitPropertyItemList(ctx.properties);
String indexType = ctx.indexType != null ? ctx.indexType.getText().toUpperCase() : null;
String comment = ctx.comment != null ? ctx.comment.getText() : "";
// change BITMAP index to INVERTED index
if (indexType.equalsIgnoreCase("BITMAP")) {
indexType = "INVERTED";
}
return new IndexDefinition(indexName, indexCols, indexType, properties, comment);
}

25
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
View File

@ -58,7 +58,7 @@ public class IndexDefinition {
Map<String, String> properties, String comment) {
this.name = name;
this.cols = Utils.copyRequiredList(cols);
this.indexType = IndexType.BITMAP;
this.indexType = IndexType.INVERTED;
if (indexTypeName != null) {
switch (indexTypeName) {
case "BITMAP": {
@ -108,19 +108,14 @@ public class IndexDefinition {
|| colType.isBooleanType())) {
// TODO add colType.isVariantType() and colType.isAggState()
throw new AnalysisException(colType + " is not supported in " + indexType.toString()
+ " index. " + "invalid column: " + indexColName);
} else if (indexType == IndexType.INVERTED && ((keysType == KeysType.AGG_KEYS
&& !column.isKey())
|| (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite))) {
+ " index. " + "invalid index: " + name);
}
if (!column.isKey()
&& ((keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite)
|| keysType == KeysType.AGG_KEYS)) {
throw new AnalysisException(indexType.toString()
+ " index only used in columns of DUP_KEYS table"
+ " or UNIQUE_KEYS table with merge_on_write enabled"
+ " or key columns of AGG_KEYS table. invalid column: " + indexColName);
} else if (keysType == KeysType.AGG_KEYS && !column.isKey()
&& indexType != IndexType.INVERTED) {
throw new AnalysisException(indexType.toString()
+ " index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of"
+ " AGG_KEYS table. invalid column: " + indexColName);
+ " index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table."
+ " invalid index: " + name);
}
if (indexType == IndexType.INVERTED) {
@ -134,10 +129,6 @@ public class IndexDefinition {
if (!colType.isStringLikeType()) {
throw new AnalysisException(colType + " is not supported in ngram_bf index. "
+ "invalid column: " + indexColName);
} else if ((keysType == KeysType.AGG_KEYS && !column.isKey())) {
throw new AnalysisException(
"ngram_bf index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of"
+ " AGG_KEYS table. invalid column: " + indexColName);
}
if (properties.size() != 2) {
throw new AnalysisException(

4
fe/fe-core/src/test/java/org/apache/doris/analysis/CreateIndexClauseTest.java
View File

@ -37,10 +37,10 @@ public class CreateIndexClauseTest {
public void testNormal() throws AnalysisException {
CreateIndexClause clause = new CreateIndexClause(
new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, "db", "table"),
new IndexDef("index1", false, Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP, null, "balabala"),
new IndexDef("index1", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "balabala"),
false);
clause.analyze(analyzer);
Assert.assertEquals("CREATE INDEX index1 ON `db`.`table` (`col1`) USING BITMAP COMMENT 'balabala'",
Assert.assertEquals("CREATE INDEX index1 ON `db`.`table` (`col1`) USING INVERTED COMMENT 'balabala'",
clause.toSql());
}

10
fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
View File

@ -29,7 +29,7 @@ public class IndexDefTest {
@Before
public void setUp() throws Exception {
def = new IndexDef("index1", false, Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP, null, "balabala");
def = new IndexDef("index1", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "balabala");
}
@Test
@ -46,7 +46,7 @@ public class IndexDefTest {
+ "x1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxx"
+ "xxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxx"
+ "xxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxx", false,
Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP, null,
Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null,
"balabala");
def.analyze();
Assert.fail("No exception throws.");
@ -54,7 +54,7 @@ public class IndexDefTest {
Assert.assertTrue(e instanceof AnalysisException);
}
try {
def = new IndexDef("", false, Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP, null, "balabala");
def = new IndexDef("", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "balabala");
def.analyze();
Assert.fail("No exception throws.");
} catch (AnalysisException e) {
@ -64,8 +64,8 @@ public class IndexDefTest {
@Test
public void toSql() {
Assert.assertEquals("INDEX index1 (`col1`) USING BITMAP COMMENT 'balabala'", def.toSql());
Assert.assertEquals("INDEX index1 ON table1 (`col1`) USING BITMAP COMMENT 'balabala'",
Assert.assertEquals("INDEX index1 (`col1`) USING INVERTED COMMENT 'balabala'", def.toSql());
Assert.assertEquals("INDEX index1 ON table1 (`col1`) USING INVERTED COMMENT 'balabala'",
def.toSql("table1"));
}
}

2
fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java
View File

@ -65,7 +65,7 @@ public class TableAddOrDropColumnsInfoTest {
indexSchemaMap.put(tableId, fullSchema);
List<Index> indexes = Lists.newArrayList(
new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.BITMAP, null, "xxxxxx"));
new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx"));
TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new TableAddOrDropColumnsInfo("", dbId, tableId,
indexSchemaMap, indexes, jobId);