database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bugfix](inverted index) Fix mv inheriting unexpectedly inverted index of base table (#19722)

Browse Source
This commit is contained in:
Kang
2023-05-17 17:18:07 +08:00
committed by GitHub
parent 16d005c7d1
commit ce12cf404c
7 changed files with 41 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
View File

@ -271,7 +271,7 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
Partition.PARTITION_INIT_VERSION,
rollupKeysType, TStorageType.COLUMN, storageMedium,
rollupSchema, tbl.getCopiedBfColumns(), tbl.getBfFpp(), countDownLatch,
tbl.getCopiedIndexes(),
null, // do not copy indexes of base tablet to ROLLUP tablet
tbl.isInMemory(),
tabletType,
null,
@ -354,7 +354,7 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
tbl.setIndexMeta(rollupIndexId, rollupIndexName, rollupSchema, 0 /* init schema version */,
rollupSchemaHash, rollupShortKeyColumnCount, TStorageType.COLUMN,
rollupKeysType, origStmt, analyzer != null ? new Analyzer(analyzer) : analyzer);
rollupKeysType, origStmt, analyzer != null ? new Analyzer(analyzer) : analyzer, null);
tbl.rebuildFullSchema();
}

2
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View File

@ -389,7 +389,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
indexSchemaVersionAndHashMap.get(shadowIdxId).schemaVersion,
indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash,
indexShortKeyMap.get(shadowIdxId), TStorageType.COLUMN,
tbl.getKeysTypeByIndexId(indexIdMap.get(shadowIdxId)));
tbl.getKeysTypeByIndexId(indexIdMap.get(shadowIdxId)), null);
}
tbl.rebuildFullSchema();

14
fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
View File

@ -68,6 +68,8 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
//for light schema change
@SerializedName(value = "maxColUniqueId")
private int maxColUniqueId = Column.COLUMN_UNIQUE_ID_INIT_VALUE;
@SerializedName(value = "indexes")
private List<Index> indexes;
private Expr whereClause;
private Map<String, Column> nameToColumn;
@ -78,6 +80,13 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
public MaterializedIndexMeta(long indexId, List<Column> schema, int schemaVersion, int schemaHash,
short shortKeyColumnCount, TStorageType storageType, KeysType keysType, OriginStatement defineStmt) {
this(indexId, schema, schemaVersion, schemaHash, shortKeyColumnCount, storageType, keysType,
defineStmt, null); // indexes is null by default
}
public MaterializedIndexMeta(long indexId, List<Column> schema, int schemaVersion, int schemaHash,
short shortKeyColumnCount, TStorageType storageType, KeysType keysType, OriginStatement defineStmt,
List<Index> indexes) {
this.indexId = indexId;
Preconditions.checkState(schema != null);
Preconditions.checkState(schema.size() != 0);
@ -90,6 +99,7 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
Preconditions.checkState(keysType != null);
this.keysType = keysType;
this.defineStmt = defineStmt;
this.indexes = indexes != null ? indexes : Lists.newArrayList();
initColumnNameMap();
}
@ -117,6 +127,10 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable {
return storageType;
}
public List<Index> getIndexes() {
return indexes != null ? indexes : Lists.newArrayList();
}
public List<Column> getSchema() {
return getSchema(true);
}

19
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -290,13 +290,28 @@ public class OlapTable extends Table {
short shortKeyColumnCount, TStorageType storageType, KeysType keysType) {
setIndexMeta(indexId, indexName, schema, schemaVersion, schemaHash, shortKeyColumnCount, storageType,
keysType,
null, null);
null, null, null); // indexes is null by default
}
public void setIndexMeta(long indexId, String indexName, List<Column> schema, int schemaVersion, int schemaHash,
short shortKeyColumnCount, TStorageType storageType, KeysType keysType, List<Index> indexes) {
setIndexMeta(indexId, indexName, schema, schemaVersion, schemaHash, shortKeyColumnCount, storageType,
keysType,
null, null, indexes);
}
public void setIndexMeta(long indexId, String indexName, List<Column> schema, int schemaVersion,
int schemaHash,
short shortKeyColumnCount, TStorageType storageType, KeysType keysType, OriginStatement origStmt,
Analyzer analyzer) {
setIndexMeta(indexId, indexName, schema, schemaVersion, schemaHash, shortKeyColumnCount, storageType,
keysType, origStmt, analyzer, null); // indexes is null by default
}
public void setIndexMeta(long indexId, String indexName, List<Column> schema, int schemaVersion,
int schemaHash,
short shortKeyColumnCount, TStorageType storageType, KeysType keysType, OriginStatement origStmt,
Analyzer analyzer, List<Index> indexes) {
// Nullable when meta comes from schema change log replay.
// The replay log only save the index id, so we need to get name by id.
if (indexName == null) {
@ -319,7 +334,7 @@ public class OlapTable extends Table {
}
MaterializedIndexMeta indexMeta = new MaterializedIndexMeta(indexId, schema, schemaVersion,
schemaHash, shortKeyColumnCount, storageType, keysType, origStmt);
schemaHash, shortKeyColumnCount, storageType, keysType, origStmt, indexes);
try {
indexMeta.parseStmt(analyzer);
} catch (Exception e) {

4
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -2087,7 +2087,7 @@ public class InternalCatalog implements CatalogIf<Database> {
}
int schemaHash = Util.generateSchemaHash();
olapTable.setIndexMeta(baseIndexId, tableName, baseSchema, schemaVersion, schemaHash, shortKeyColumnCount,
baseIndexStorageType, keysType);
baseIndexStorageType, keysType, olapTable.getIndexes());
for (AlterClause alterClause : stmt.getRollupAlterClauseList()) {
AddRollupClause addRollupClause = (AddRollupClause) alterClause;
@ -2110,7 +2110,7 @@ public class InternalCatalog implements CatalogIf<Database> {
int rollupSchemaHash = Util.generateSchemaHash();
long rollupIndexId = idGeneratorBuffer.getNextId();
olapTable.setIndexMeta(rollupIndexId, addRollupClause.getRollupName(), rollupColumns, schemaVersion,
rollupSchemaHash, rollupShortKeyColumnCount, rollupIndexStorageType, keysType);
rollupSchemaHash, rollupShortKeyColumnCount, rollupIndexStorageType, keysType, null);
}
// analyse sequence map column

2
fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
View File

@ -227,7 +227,7 @@ public class OlapTableSink extends DataSink {
column.setIndexFlag(tColumn, table);
columnsDesc.add(tColumn);
}
for (Index index : table.getIndexes()) {
for (Index index : indexMeta.getIndexes()) {
TOlapTableIndex tIndex = index.toThrift();
indexDesc.add(tIndex);
}

5
regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy
View File

@ -24,7 +24,10 @@ suite ("test_dup_mv_bitmap_hash") {
create table d_table(
k1 int null,
k2 int null,
k3 varchar(100) null
k3 varchar(100) null,
INDEX auto_idx_k1 (`k1`) USING INVERTED COMMENT 'auto added inverted index for k1',
INDEX auto_idx_k2 (`k2`) USING INVERTED COMMENT 'auto added inverted index for k2',
INDEX auto_idx_k3 (`k3`) USING INVERTED COMMENT 'auto added inverted index for k3'
)
duplicate key (k1)
distributed BY hash(k1) buckets 3