From b3ae7f04f9b034ea7460f78b9f0e08a7ce37ede5 Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 8 Nov 2023 22:28:49 +0800 Subject: [PATCH] [fix](backup) Add repo id to local meta/info files to avoid overwriting (#26536) The local meta/info files generated during backup are not distinguished by repo names. If two backup jobs with the same name are submitted to different repos at the same time, meta/info may be overwritten by another backup job. --- .../org/apache/doris/backup/BackupJob.java | 5 +- .../org/apache/doris/backup/Repository.java | 18 ++- ...kup_restore_diff_repo_same_snapshot.groovy | 133 ++++++++++++++++++ 3 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 regression-test/suites/backup_restore/test_backup_restore_diff_repo_same_snapshot.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index e60e9e300b..1d730db184 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -664,9 +664,10 @@ public class BackupJob extends AbstractJob { private void saveMetaInfo() { String createTimeStr = TimeUtils.longToTimeString(createTime, TimeUtils.DATETIME_FORMAT_WITH_HYPHEN); - // local job dir: backup/label__createtime/ + // local job dir: backup/repo__repo_id/label__createtime/ + // Add repo_id to isolate jobs from different repos. localJobDirPath = Paths.get(BackupHandler.BACKUP_ROOT_DIR.toString(), - label + "__" + createTimeStr).normalize(); + "repo__" + repoId, label + "__" + createTimeStr).normalize(); try { // 1. create local job dir of this backup job diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index 27ce489948..a95b6a953a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -59,6 +59,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; +import java.util.UUID; /* * Repository represents a remote storage for backup to or restore from @@ -251,7 +252,7 @@ public class Repository implements Writable { } // exist, download and parse the repo info file - String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/tmp_info_" + System.currentTimeMillis(); + String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/tmp_info_" + allocLocalFileSuffix(); try { st = fileSystem.downloadWithFileSize(repoInfoFilePath, localFilePath, remoteFile.getSize()); if (!st.ok()) { @@ -419,7 +420,7 @@ public class Repository implements Writable { public Status getSnapshotInfoFile(String label, String backupTimestamp, List infos) { String remoteInfoFilePath = assembleJobInfoFilePath(label, -1) + backupTimestamp; File localInfoFile = new File(BackupHandler.BACKUP_ROOT_DIR + PATH_DELIMITER - + "info_" + System.currentTimeMillis()); + + "info_" + allocLocalFileSuffix()); try { Status st = download(remoteInfoFilePath, localInfoFile.getPath()); if (!st.ok()) { @@ -441,7 +442,7 @@ public class Repository implements Writable { public Status getSnapshotMetaFile(String label, List backupMetas, int metaVersion) { String remoteMetaFilePath = assembleMetaInfoFilePath(label); File localMetaFile = new File(BackupHandler.BACKUP_ROOT_DIR + PATH_DELIMITER - + "meta_" + System.currentTimeMillis()); + + "meta_" + allocLocalFileSuffix()); try { Status st = download(remoteMetaFilePath, localMetaFile.getAbsolutePath()); @@ -732,9 +733,9 @@ public class Repository implements Writable { } } } else { - // get specified timestamp - // path eg: /path/to/backup/__info_2081-04-19-12-59-11 - String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/" + Repository.PREFIX_JOB_INFO + timestamp; + // get specified timestamp, different repos might have snapshots with same timestamp. + String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/" + + Repository.PREFIX_JOB_INFO + allocLocalFileSuffix(); try { String remoteInfoFilePath = assembleJobInfoFilePath(snapshotName, -1) + timestamp; Status st = download(remoteInfoFilePath, localFilePath); @@ -772,6 +773,11 @@ public class Repository implements Writable { return info; } + // Allocate an unique suffix. + private String allocLocalFileSuffix() { + return System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", "_"); + } + @Override public void write(DataOutput out) throws IOException { out.writeLong(id); diff --git a/regression-test/suites/backup_restore/test_backup_restore_diff_repo_same_snapshot.groovy b/regression-test/suites/backup_restore/test_backup_restore_diff_repo_same_snapshot.groovy new file mode 100644 index 0000000000..19e2e2fd04 --- /dev/null +++ b/regression-test/suites/backup_restore/test_backup_restore_diff_repo_same_snapshot.groovy @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_diff_repo_same_snapshot", "backup_restore") { + String suiteName = "test_backup_restore_diff_repo_same_snapshot" + String repoName = "${suiteName}_repo" + String dbName = "${suiteName}_db" + String tableName = "${suiteName}_table" + String snapshotName = "${suiteName}_snapshot" + + def syncer = getSyncer() + syncer.createS3Repository("${repoName}_1") + syncer.createS3Repository("${repoName}_2") + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}_1" + sql "CREATE DATABASE IF NOT EXISTS ${dbName}_2" + sql "DROP TABLE IF EXISTS ${dbName}_1.${tableName}_1" + sql "DROP TABLE IF EXISTS ${dbName}_2.${tableName}_2" + sql """ + CREATE TABLE ${dbName}_1.${tableName}_1 ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0") + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES ( "replication_num" = "1") + """ + sql """ + CREATE TABLE ${dbName}_2.${tableName}_2 ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0") + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES ( "replication_num" = "1" ) + """ + + List values = [] + for (int i = 1; i <= 10; ++i) { + values.add("(${i}, ${i})") + } + sql "INSERT INTO ${dbName}_1.${tableName}_1 VALUES ${values.join(",")}" + sql "INSERT INTO ${dbName}_2.${tableName}_2 VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}_1.${tableName}_1" + assertEquals(result.size(), values.size()); + result = sql "SELECT * FROM ${dbName}_2.${tableName}_2" + assertEquals(result.size(), values.size()); + + // Backup to different repo, with same snapshot name. + sql """ + BACKUP SNAPSHOT ${dbName}_1.${snapshotName} + TO `${repoName}_1` + ON (${tableName}_1) + """ + sql """ + BACKUP SNAPSHOT ${dbName}_2.${snapshotName} + TO `${repoName}_2` + ON (${tableName}_2) + """ + + while (!syncer.checkSnapshotFinish("${dbName}_1")) { + Thread.sleep(3000) + } + while (!syncer.checkSnapshotFinish("${dbName}_2")) { + Thread.sleep(3000) + } + + // Restore snapshot from repo_1 to db_1 + def snapshot = syncer.getSnapshotTimestamp("${repoName}_1", snapshotName) + assertTrue(snapshot != null) + + sql "TRUNCATE TABLE ${dbName}_1.${tableName}_1" + sql """ + RESTORE SNAPSHOT ${dbName}_1.${snapshotName} + FROM `${repoName}_1` + ON ( `${tableName}_1`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "replication_num" = "1" + ) + """ + + while (!syncer.checkAllRestoreFinish("${dbName}_1")) { + Thread.sleep(3000) + } + + result = sql "SELECT * FROM ${dbName}_1.${tableName}_1" + assertEquals(result.size(), values.size()); + + // Restore snapshot from repo_2 to db_2 + snapshot = syncer.getSnapshotTimestamp("${repoName}_2", snapshotName) + assertTrue(snapshot != null) + + sql "TRUNCATE TABLE ${dbName}_2.${tableName}_2" + sql """ + RESTORE SNAPSHOT ${dbName}_2.${snapshotName} + FROM `${repoName}_2` + ON ( `${tableName}_2`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "replication_num" = "1" + ) + """ + + while (!syncer.checkAllRestoreFinish("${dbName}_2")) { + Thread.sleep(3000) + } + + result = sql "SELECT * FROM ${dbName}_2.${tableName}_2" + assertEquals(result.size(), values.size()); + + sql "DROP TABLE ${dbName}_1.${tableName}_1 FORCE" + sql "DROP TABLE ${dbName}_2.${tableName}_2 FORCE" + sql "DROP DATABASE ${dbName}_1 FORCE" + sql "DROP DATABASE ${dbName}_2 FORCE" + sql "DROP REPOSITORY `${repoName}_1`" + sql "DROP REPOSITORY `${repoName}_2`" +} +