database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

branch-2.1: [chore](persist) Save old index schema map in TableAddOrDropColumnsInfo #46606 (#46741)

Browse Source 
Cherry-picked from #46606

Co-authored-by: walter <maochuan@selectdb.com>
This commit is contained in:
github-actions[bot]
2025-01-10 17:06:59 +08:00
committed by GitHub
parent 6c47898a6e
commit eb312d7160
5 changed files with 28 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -1954,16 +1954,16 @@ public class SchemaChangeHandler extends AlterHandler {
}
} else if (alterClause instanceof AddColumnsClause) {
// add columns
boolean clauseCanLigthSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable,
boolean clauseCanLightSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable,
indexSchemaMap, false, colUniqueIdSupplierMap);
if (!clauseCanLigthSchemaChange) {
if (!clauseCanLightSchemaChange) {
lightSchemaChange = false;
}
} else if (alterClause instanceof DropColumnClause) {
// drop column and drop indexes on this column
boolean clauseCanLigthSchemaChange = processDropColumn((DropColumnClause) alterClause, olapTable,
boolean clauseCanLightSchemaChange = processDropColumn((DropColumnClause) alterClause, olapTable,
indexSchemaMap, newIndexes);
if (!clauseCanLigthSchemaChange) {
if (!clauseCanLightSchemaChange) {
lightSchemaChange = false;
}
} else if (alterClause instanceof ModifyColumnClause) {
@ -2772,6 +2772,7 @@ public class SchemaChangeHandler extends AlterHandler {
}
//update base index schema
Map<Long, List<Column>> oldIndexSchemaMap = olapTable.getCopiedIndexIdToSchema(true);
try {
updateBaseIndexSchema(olapTable, indexSchemaMap, indexes);
} catch (Exception e) {
@ -2817,7 +2818,7 @@ public class SchemaChangeHandler extends AlterHandler {
} else {
if (!isReplay) {
TableAddOrDropColumnsInfo info = new TableAddOrDropColumnsInfo(rawSql, db.getId(), olapTable.getId(),
indexSchemaMap, indexes, jobId);
indexSchemaMap, oldIndexSchemaMap, indexes, jobId);
if (LOG.isDebugEnabled()) {
LOG.debug("logModifyTableAddOrDropColumns info:{}", info);
}

4
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
View File

@ -557,7 +557,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
Partition partition = tbl.getPartition(partitionId);
Preconditions.checkNotNull(partition, partitionId);
long visiableVersion = partition.getVisibleVersion();
long visibleVersion = partition.getVisibleVersion();
short expectReplicationNum = tbl.getPartitionInfo()
.getReplicaAllocation(partition.getId()).getTotalReplicaNum();
@ -570,7 +570,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
int healthyReplicaNum = 0;
for (Replica replica : replicas) {
if (!replica.isBad() && replica.getLastFailedVersion() < 0
&& replica.checkVersionCatchUp(visiableVersion, false)) {
&& replica.checkVersionCatchUp(visibleVersion, false)) {
healthyReplicaNum++;
}
}

9
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -907,6 +907,15 @@ public class OlapTable extends Table implements MTMVRelatedTableIf {
return result;
}
// get schemas with a copied column list
public Map<Long, List<Column>> getCopiedIndexIdToSchema(boolean full) {
Map<Long, List<Column>> result = Maps.newHashMap();
for (Map.Entry<Long, MaterializedIndexMeta> entry : indexIdToMeta.entrySet()) {
result.put(entry.getKey(), new ArrayList<>(entry.getValue().getSchema(full)));
}
return result;
}
public List<Column> getSchemaByIndexId(Long indexId) {
return getSchemaByIndexId(indexId, Util.showHiddenColumns());
}

8
fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java
View File

@ -42,6 +42,8 @@ public class TableAddOrDropColumnsInfo implements Writable {
private long tableId;
@SerializedName(value = "indexSchemaMap")
private Map<Long, LinkedList<Column>> indexSchemaMap;
@SerializedName(value = "oldIndexSchemaMap")
private Map<Long, List<Column>> oldIndexSchemaMap; // only used for ccr, not included in equals
@SerializedName(value = "indexes")
private List<Index> indexes;
@SerializedName(value = "jobId")
@ -50,11 +52,14 @@ public class TableAddOrDropColumnsInfo implements Writable {
private String rawSql;
public TableAddOrDropColumnsInfo(String rawSql, long dbId, long tableId,
Map<Long, LinkedList<Column>> indexSchemaMap, List<Index> indexes, long jobId) {
Map<Long, LinkedList<Column>> indexSchemaMap,
Map<Long, List<Column>> oldIndexSchemaMap,
List<Index> indexes, long jobId) {
this.rawSql = rawSql;
this.dbId = dbId;
this.tableId = tableId;
this.indexSchemaMap = indexSchemaMap;
this.oldIndexSchemaMap = oldIndexSchemaMap;
this.indexes = indexes;
this.jobId = jobId;
}
@ -111,6 +116,7 @@ public class TableAddOrDropColumnsInfo implements Writable {
sb.append(" dbId: ").append(dbId);
sb.append(" tableId: ").append(tableId);
sb.append(" indexSchemaMap: ").append(indexSchemaMap);
sb.append(" oldIndexSchemaMap: ").append(oldIndexSchemaMap);
sb.append(" indexes: ").append(indexes);
sb.append(" jobId: ").append(jobId);
return sb.toString();

5
fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java
View File

@ -64,11 +64,14 @@ public class TableAddOrDropColumnsInfoTest {
Map<Long, LinkedList<Column>> indexSchemaMap = new HashMap<>();
indexSchemaMap.put(tableId, fullSchema);
Map<Long, List<Column>> oldIndexSchemaMap = new HashMap<>();
oldIndexSchemaMap.put(tableId, fullSchema);
List<Index> indexes = Lists.newArrayList(
new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx", Lists.newArrayList(1)));
TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new TableAddOrDropColumnsInfo("", dbId, tableId,
indexSchemaMap, indexes, jobId);
indexSchemaMap, oldIndexSchemaMap, indexes, jobId);
String c1Json = GsonUtils.GSON.toJson(tableAddOrDropColumnsInfo1);
Text.writeString(out, c1Json);