From eb312d71607796137baa63263eac68302ee69f47 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 10 Jan 2025 17:06:59 +0800 Subject: [PATCH] branch-2.1: [chore](persist) Save old index schema map in TableAddOrDropColumnsInfo #46606 (#46741) Cherry-picked from #46606 Co-authored-by: walter --- .../org/apache/doris/alter/SchemaChangeHandler.java | 11 ++++++----- .../org/apache/doris/alter/SchemaChangeJobV2.java | 4 ++-- .../main/java/org/apache/doris/catalog/OlapTable.java | 9 +++++++++ .../doris/persist/TableAddOrDropColumnsInfo.java | 8 +++++++- .../doris/persist/TableAddOrDropColumnsInfoTest.java | 5 ++++- 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 15caeb55ad..2721dd49d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1954,16 +1954,16 @@ public class SchemaChangeHandler extends AlterHandler { } } else if (alterClause instanceof AddColumnsClause) { // add columns - boolean clauseCanLigthSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable, + boolean clauseCanLightSchemaChange = processAddColumns((AddColumnsClause) alterClause, olapTable, indexSchemaMap, false, colUniqueIdSupplierMap); - if (!clauseCanLigthSchemaChange) { + if (!clauseCanLightSchemaChange) { lightSchemaChange = false; } } else if (alterClause instanceof DropColumnClause) { // drop column and drop indexes on this column - boolean clauseCanLigthSchemaChange = processDropColumn((DropColumnClause) alterClause, olapTable, + boolean clauseCanLightSchemaChange = processDropColumn((DropColumnClause) alterClause, olapTable, indexSchemaMap, newIndexes); - if (!clauseCanLigthSchemaChange) { + if (!clauseCanLightSchemaChange) { lightSchemaChange = false; } } else if (alterClause instanceof ModifyColumnClause) { @@ -2772,6 +2772,7 @@ public class SchemaChangeHandler extends AlterHandler { } //update base index schema + Map> oldIndexSchemaMap = olapTable.getCopiedIndexIdToSchema(true); try { updateBaseIndexSchema(olapTable, indexSchemaMap, indexes); } catch (Exception e) { @@ -2817,7 +2818,7 @@ public class SchemaChangeHandler extends AlterHandler { } else { if (!isReplay) { TableAddOrDropColumnsInfo info = new TableAddOrDropColumnsInfo(rawSql, db.getId(), olapTable.getId(), - indexSchemaMap, indexes, jobId); + indexSchemaMap, oldIndexSchemaMap, indexes, jobId); if (LOG.isDebugEnabled()) { LOG.debug("logModifyTableAddOrDropColumns info:{}", info); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index e98d8066a1..fb6aeb3dbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -557,7 +557,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 { Partition partition = tbl.getPartition(partitionId); Preconditions.checkNotNull(partition, partitionId); - long visiableVersion = partition.getVisibleVersion(); + long visibleVersion = partition.getVisibleVersion(); short expectReplicationNum = tbl.getPartitionInfo() .getReplicaAllocation(partition.getId()).getTotalReplicaNum(); @@ -570,7 +570,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 { int healthyReplicaNum = 0; for (Replica replica : replicas) { if (!replica.isBad() && replica.getLastFailedVersion() < 0 - && replica.checkVersionCatchUp(visiableVersion, false)) { + && replica.checkVersionCatchUp(visibleVersion, false)) { healthyReplicaNum++; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index df1ad45c3a..4416a70867 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -907,6 +907,15 @@ public class OlapTable extends Table implements MTMVRelatedTableIf { return result; } + // get schemas with a copied column list + public Map> getCopiedIndexIdToSchema(boolean full) { + Map> result = Maps.newHashMap(); + for (Map.Entry entry : indexIdToMeta.entrySet()) { + result.put(entry.getKey(), new ArrayList<>(entry.getValue().getSchema(full))); + } + return result; + } + public List getSchemaByIndexId(Long indexId) { return getSchemaByIndexId(indexId, Util.showHiddenColumns()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java index 9939608067..14b7187fac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java @@ -42,6 +42,8 @@ public class TableAddOrDropColumnsInfo implements Writable { private long tableId; @SerializedName(value = "indexSchemaMap") private Map> indexSchemaMap; + @SerializedName(value = "oldIndexSchemaMap") + private Map> oldIndexSchemaMap; // only used for ccr, not included in equals @SerializedName(value = "indexes") private List indexes; @SerializedName(value = "jobId") @@ -50,11 +52,14 @@ public class TableAddOrDropColumnsInfo implements Writable { private String rawSql; public TableAddOrDropColumnsInfo(String rawSql, long dbId, long tableId, - Map> indexSchemaMap, List indexes, long jobId) { + Map> indexSchemaMap, + Map> oldIndexSchemaMap, + List indexes, long jobId) { this.rawSql = rawSql; this.dbId = dbId; this.tableId = tableId; this.indexSchemaMap = indexSchemaMap; + this.oldIndexSchemaMap = oldIndexSchemaMap; this.indexes = indexes; this.jobId = jobId; } @@ -111,6 +116,7 @@ public class TableAddOrDropColumnsInfo implements Writable { sb.append(" dbId: ").append(dbId); sb.append(" tableId: ").append(tableId); sb.append(" indexSchemaMap: ").append(indexSchemaMap); + sb.append(" oldIndexSchemaMap: ").append(oldIndexSchemaMap); sb.append(" indexes: ").append(indexes); sb.append(" jobId: ").append(jobId); return sb.toString(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java index be71998eac..4df2759e9d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java @@ -64,11 +64,14 @@ public class TableAddOrDropColumnsInfoTest { Map> indexSchemaMap = new HashMap<>(); indexSchemaMap.put(tableId, fullSchema); + Map> oldIndexSchemaMap = new HashMap<>(); + oldIndexSchemaMap.put(tableId, fullSchema); + List indexes = Lists.newArrayList( new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx", Lists.newArrayList(1))); TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new TableAddOrDropColumnsInfo("", dbId, tableId, - indexSchemaMap, indexes, jobId); + indexSchemaMap, oldIndexSchemaMap, indexes, jobId); String c1Json = GsonUtils.GSON.toJson(tableAddOrDropColumnsInfo1); Text.writeString(out, c1Json);