From e222a50c4278f4e5199e4be672de456c1998c46d Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Thu, 5 May 2022 20:44:27 +0800 Subject: [PATCH] [fix](backup) Remove colocate_with property when backing up a table (#9142) We currently not support backup table with colocation property. So that we have to remove colocate_with property from a table when backing up. --- docs/en/admin-manual/data-admin/backup.md | 2 ++ docs/zh-CN/admin-manual/data-admin/backup.md | 2 ++ .../main/java/org/apache/doris/backup/BackupJob.java | 7 +++++++ .../main/java/org/apache/doris/catalog/OlapTable.java | 3 ++- .../java/org/apache/doris/catalog/TableProperty.java | 1 + .../java/org/apache/doris/catalog/OlapTableTest.java | 11 +++++++---- 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/en/admin-manual/data-admin/backup.md b/docs/en/admin-manual/data-admin/backup.md index 68626548d4..9139d03947 100644 --- a/docs/en/admin-manual/data-admin/backup.md +++ b/docs/en/admin-manual/data-admin/backup.md @@ -52,6 +52,8 @@ The backup operation is to upload the data of the specified table or partition d ALTER TABLE tbl1 SET ("dynamic_partition.enable"="true") ``` +4. Backup and Restore operation will NOT keep the `colocate_with` property of a table. + ## Start Backup 1. Create a hdfs remote warehouse example_repo: diff --git a/docs/zh-CN/admin-manual/data-admin/backup.md b/docs/zh-CN/admin-manual/data-admin/backup.md index 7535cc7ef9..d99e4ae624 100644 --- a/docs/zh-CN/admin-manual/data-admin/backup.md +++ b/docs/zh-CN/admin-manual/data-admin/backup.md @@ -52,6 +52,8 @@ Doris 支持将当前数据以文件的形式，通过 broker 备份到远端存 ALTER TABLE tbl1 SET ("dynamic_partition.enable"="true") ``` +4. 备份和恢复操作都不会保留表的 `colocate_with` 属性。 + ## 开始备份 1. 创建一个hdfs的远程仓库example_repo： diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index f23177950e..5f07552c79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -507,6 +507,8 @@ public class BackupJob extends AbstractJob { status = new Status(ErrCode.COMMON_ERROR, "failed to copy table: " + tblName); return; } + + removeUnsupportProperties(copiedTbl); copiedTables.add(copiedTbl); } else if (table.getType() == TableType.VIEW) { View view = (View) table; @@ -543,6 +545,11 @@ public class BackupJob extends AbstractJob { backupMeta = new BackupMeta(copiedTables, copiedResources); } + private void removeUnsupportProperties(OlapTable tbl) { + // We cannot support the colocate attribute because the colocate information is not backed up + // synchronously when backing up. + tbl.setColocateGroup(null); + } private void waitingAllSnapshotsFinished() { if (unfinishedTaskIds.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index c7795d5e27..b56ea19f30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -424,10 +424,11 @@ public class OlapTable extends Table { * Reset properties to correct values. */ public void resetPropertiesForRestore() { - // disable dynamic partition if (tableProperty != null) { tableProperty.resetPropertiesForRestore(); } + // remove colocate property. + setColocateGroup(null); } public Status resetIdsForRestore(Catalog catalog, Database db, ReplicaAllocation restoreReplicaAlloc) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 31036ebb55..580fba8ae3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -110,6 +110,7 @@ public class TableProperty implements Writable { * @return this for chained */ public TableProperty resetPropertiesForRestore() { + // disable dynamic partition if (properties.containsKey(DynamicPartitionProperty.ENABLE)) { properties.put(DynamicPartitionProperty.ENABLE, "false"); executeBuildDynamicProperty(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java index cc9d7f47f5..2c57fe054d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java @@ -17,10 +17,6 @@ package org.apache.doris.catalog; -import com.google.common.collect.Maps; -import mockit.Mock; -import mockit.MockUp; - import org.apache.doris.analysis.IndexDef; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.common.FeConstants; @@ -28,6 +24,7 @@ import org.apache.doris.common.io.FastByteArrayOutputStream; import org.apache.doris.common.util.UnitTestUtil; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -38,6 +35,9 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import mockit.Mock; +import mockit.MockUp; + public class OlapTableTest { @Test @@ -88,10 +88,13 @@ public class OlapTableTest { OlapTable olapTable = new OlapTable(); olapTable.setTableProperty(tableProperty); + olapTable.setColocateGroup("test_group"); + Assert.assertTrue(olapTable.isColocateTable()); olapTable.resetPropertiesForRestore(); Assert.assertEquals(tableProperty.getProperties(), olapTable.getTableProperty().getProperties()); Assert.assertFalse(tableProperty.getDynamicPartitionProperty().isExist()); + Assert.assertFalse(olapTable.isColocateTable()); // restore with dynamic partition keys properties = Maps.newHashMap();