database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](backup) Add repo id to local meta/info files to avoid overwriting (#26536)

Browse Source 
The local meta/info files generated during backup are not distinguished
by repo names. If two backup jobs with the same name are submitted to
different repos at the same time, meta/info may be overwritten by another
backup job.
This commit is contained in:
walter
2023-11-08 22:28:49 +08:00
committed by GitHub
parent ee6e6911da
commit b3ae7f04f9
3 changed files with 148 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
View File

@ -664,9 +664,10 @@ public class BackupJob extends AbstractJob {
private void saveMetaInfo() {
String createTimeStr = TimeUtils.longToTimeString(createTime, TimeUtils.DATETIME_FORMAT_WITH_HYPHEN);
// local job dir: backup/label__createtime/
// local job dir: backup/repo__repo_id/label__createtime/
// Add repo_id to isolate jobs from different repos.
localJobDirPath = Paths.get(BackupHandler.BACKUP_ROOT_DIR.toString(),
label + "__" + createTimeStr).normalize();
"repo__" + repoId, label + "__" + createTimeStr).normalize();
try {
// 1. create local job dir of this backup job

18
fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
View File

@ -59,6 +59,7 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
import java.util.UUID;
/*
* Repository represents a remote storage for backup to or restore from
@ -251,7 +252,7 @@ public class Repository implements Writable {
}
// exist, download and parse the repo info file
String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/tmp_info_" + System.currentTimeMillis();
String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/tmp_info_" + allocLocalFileSuffix();
try {
st = fileSystem.downloadWithFileSize(repoInfoFilePath, localFilePath, remoteFile.getSize());
if (!st.ok()) {
@ -419,7 +420,7 @@ public class Repository implements Writable {
public Status getSnapshotInfoFile(String label, String backupTimestamp, List<BackupJobInfo> infos) {
String remoteInfoFilePath = assembleJobInfoFilePath(label, -1) + backupTimestamp;
File localInfoFile = new File(BackupHandler.BACKUP_ROOT_DIR + PATH_DELIMITER
+ "info_" + System.currentTimeMillis());
+ "info_" + allocLocalFileSuffix());
try {
Status st = download(remoteInfoFilePath, localInfoFile.getPath());
if (!st.ok()) {
@ -441,7 +442,7 @@ public class Repository implements Writable {
public Status getSnapshotMetaFile(String label, List<BackupMeta> backupMetas, int metaVersion) {
String remoteMetaFilePath = assembleMetaInfoFilePath(label);
File localMetaFile = new File(BackupHandler.BACKUP_ROOT_DIR + PATH_DELIMITER
+ "meta_" + System.currentTimeMillis());
+ "meta_" + allocLocalFileSuffix());
try {
Status st = download(remoteMetaFilePath, localMetaFile.getAbsolutePath());
@ -732,9 +733,9 @@ public class Repository implements Writable {
}
}
} else {
// get specified timestamp
// path eg: /path/to/backup/__info_2081-04-19-12-59-11
String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/" + Repository.PREFIX_JOB_INFO + timestamp;
// get specified timestamp, different repos might have snapshots with same timestamp.
String localFilePath = BackupHandler.BACKUP_ROOT_DIR + "/"
+ Repository.PREFIX_JOB_INFO + allocLocalFileSuffix();
try {
String remoteInfoFilePath = assembleJobInfoFilePath(snapshotName, -1) + timestamp;
Status st = download(remoteInfoFilePath, localFilePath);
@ -772,6 +773,11 @@ public class Repository implements Writable {
return info;
}
// Allocate an unique suffix.
private String allocLocalFileSuffix() {
return System.currentTimeMillis() + UUID.randomUUID().toString().replace("-", "_");
}
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(id);