database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[chore](backup) Remove delete_if_exists properties for creating repository (#38192)

Browse Source 
delete_if_exists is a temporary solution introduced in #25847, to avoid
concurrent testing conflicts.

Cherry-pick #38190
This commit is contained in:
walter
2024-07-23 11:09:02 +08:00
committed by GitHub
parent 41b355721f
commit fdc8b454d2
28 changed files with 28 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java
View File

@ -28,8 +28,6 @@ import org.apache.doris.qe.ConnectContext;
import java.util.Map;
public class CreateRepositoryStmt extends DdlStmt {
public static String PROP_DELETE_IF_EXISTS = "delete_if_exists";
private boolean isReadOnly;
private String name;
private StorageBackend storage;
@ -73,16 +71,6 @@ public class CreateRepositoryStmt extends DdlStmt {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
}
FeNameFormat.checkCommonName("repository", name);
// check delete_if_exists, this property will be used by Repository.initRepository.
Map<String, String> properties = getProperties();
String deleteIfExistsStr = properties.get(PROP_DELETE_IF_EXISTS);
if (deleteIfExistsStr != null) {
if (!deleteIfExistsStr.equalsIgnoreCase("true") && !deleteIfExistsStr.equalsIgnoreCase("false")) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
"'" + PROP_DELETE_IF_EXISTS + "' in properties, you should set it false or true");
}
}
}
@Override

21
fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
View File

@ -17,7 +17,6 @@
package org.apache.doris.backup;
import org.apache.doris.analysis.CreateRepositoryStmt;
import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.backup.Status.ErrCode;
import org.apache.doris.catalog.Env;
@ -222,26 +221,6 @@ public class Repository implements Writable {
return Status.OK;
}
// A temporary solution is to delete all stale snapshots before creating an S3 repository
// so that we can add regression tests about backup/restore.
//
// TODO: support hdfs/brokers
if (fileSystem instanceof S3FileSystem) {
String deleteStaledSnapshots = fileSystem.getProperties()
.getOrDefault(CreateRepositoryStmt.PROP_DELETE_IF_EXISTS, "false");
if (deleteStaledSnapshots.equalsIgnoreCase("true")) {
// delete with prefix:
// eg. __palo_repository_repo_name/
String snapshotPrefix = Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name));
LOG.info("property {} is set, delete snapshots with prefix: {}",
CreateRepositoryStmt.PROP_DELETE_IF_EXISTS, snapshotPrefix);
Status st = ((S3FileSystem) fileSystem).deleteDirectory(snapshotPrefix);
if (!st.ok()) {
return st;
}
}
}
String repoInfoFilePath = assembleRepoInfoFilePath();
// check if the repo is already exist in remote
List<RemoteFile> remoteFiles = Lists.newArrayList();

3
regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
View File

@ -855,8 +855,7 @@ class Syncer {
"s3.endpoint" = "http://${endpoint}",
"s3.region" = "${region}",
"s3.access_key" = "${ak}",
"s3.secret_key" = "${sk}",
"delete_if_exists" = "true"
"s3.secret_key" = "${sk}"
)
"""
}

2
regression-test/suites/backup_restore/test_backup_restore.groovy
View File

@ -17,7 +17,7 @@
suite("test_backup_restore", "backup_restore") {
String suiteName = "test_backup_restore"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "${suiteName}_db"
String tableName = "${suiteName}_table"
String snapshotName = "${suiteName}_snapshot"

2
regression-test/suites/backup_restore/test_backup_restore_alias.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_backup_restore_alias", "backup_restore") {
String repoName = "test_backup_restore_alias_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "backup_restore_alias_db"
String tableName = "test_backup_restore_alias_table"
String aliasName = "test_backup_restore_alias_table_alias"

2
regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
View File

@ -17,7 +17,7 @@
suite("test_backup_restore_backup_temp_partition", "backup_restore") {
String suiteName = "test_backup_restore_backup_temp_partition"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "${suiteName}_db"
String tableName = "${suiteName}_table"
String snapshotName = "${suiteName}_snapshot"

2
regression-test/suites/backup_restore/test_backup_restore_bloom_filter.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_backup_restore_bloom_filter", "backup_restore") {
String repoName = "test_backup_restore_bloom_filter_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "test_backup_restore_bloom_filter_db"
String tableName = "test_backup_restore_bloom_filter_table"

2
regression-test/suites/backup_restore/test_backup_restore_db.groovy
View File

@ -18,7 +18,7 @@
suite("test_backup_restore_db", "backup_restore") {
String dbName = "backup_restore_db_1"
String suiteName = "test_backup_restore_db"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String snapshotName = "${suiteName}_snapshot"
String tableNamePrefix = "${suiteName}_tables"

2
regression-test/suites/backup_restore/test_backup_restore_diff_repo_same_snapshot.groovy
View File

@ -17,7 +17,7 @@
suite("test_backup_restore_diff_repo_same_snapshot", "backup_restore") {
String suiteName = "test_backup_restore_diff_repo_same_snapshot"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "${suiteName}_db"
String tableName = "${suiteName}_table"
String snapshotName = "${suiteName}_snapshot"

2
regression-test/suites/backup_restore/test_backup_restore_dup_without_default_keys.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_backup_restore_dup_without_default_keys", "backup_restore") {
String repoName = "test_backup_restore_dup_without_default_keys_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "backup_restore_dup_without_default_keys_db"
String tableName = "dup_without_keys_table"

2
regression-test/suites/backup_restore/test_backup_restore_exclude.groovy
View File

@ -18,7 +18,7 @@
suite("test_backup_restore_exclude", "backup_restore") {
String dbName = "backup_restore_exclude"
String suiteName = "test_backup_restore_exclude"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String snapshotName = "${suiteName}_snapshot"
String tableNamePrefix = "${suiteName}_tables"

2
regression-test/suites/backup_restore/test_backup_restore_inverted_index.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_backup_restore_inverted_index", "backup_restore") {
String repoName = "test_backup_restore_inverted_index_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "test_backup_restore_inverted_index_db"
String tableName = "test_backup_restore_inverted_index_table"

2
regression-test/suites/backup_restore/test_backup_restore_mtmv.groovy
View File

@ -17,7 +17,7 @@
suite("test_backup_restore_mtmv", "backup_restore") {
String suiteName = "test_backup_restore_mtmv"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "${suiteName}_db"
String tableName = "${suiteName}_table"
String mvName = "${suiteName}_mv"

2
regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy
View File

@ -18,7 +18,7 @@
suite("test_backup_restore_multi_tables", "backup_restore") {
String dbName = "backup_restore_multi_tables_db"
String suiteName = "test_backup_restore_multi_tables"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String snapshotName = "${suiteName}_snapshot"
String tableNamePrefix = "${suiteName}_tables"

2
regression-test/suites/backup_restore/test_backup_restore_multi_tables_overwrite.groovy
View File

@ -18,7 +18,7 @@
suite("test_backup_restore_multi_tables_overwrite", "backup_restore") {
String dbName = "backup_restore_multi_tables_overwrite_db"
String suiteName = "test_backup_restore_multi_tables_overwrite"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String snapshotName = "${suiteName}_snapshot"
String tableNamePrefix = "${suiteName}_tables"

2
regression-test/suites/backup_restore/test_backup_restore_ngram_bloom_filter.groovy
View File

@ -17,7 +17,7 @@
suite("test_backup_restore_ngram_bloom_filter", "backup_restore") {
String suiteName = "test_backup_restore_ngram_bloom_filter"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "${suiteName}_db"
String tableName = "${suiteName}_table"
String snapshotName = "${suiteName}_snapshot"

2
regression-test/suites/backup_restore/test_backup_restore_overwrite_indexes.groovy
View File

@ -17,7 +17,7 @@
suite("test_backup_restore_overwrite_indexes", "backup_restore") {
String suiteName = "test_backup_restore_overwrite_indexes"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "${suiteName}_db"
String newDbName = "${suiteName}_new_db"
String tableName = "${suiteName}_table"

2
regression-test/suites/backup_restore/test_backup_restore_partition.groovy
View File

@ -18,7 +18,7 @@
suite("test_backup_restore_partition", "backup_restore") {
String suiteName = "test_backup_restore_partition"
String dbName = "${suiteName}_db"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String snapshotName = "${suiteName}_snapshot"
String tableName = "${suiteName}_table"

2
regression-test/suites/backup_restore/test_backup_restore_reserve_dynamic_partition_false.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_backup_restore_reserve_dynamic_partition_false", "backup_restore") {
String repoName = "test_backup_restore_dynamic_partition_reserve_false_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "backup_restore_dynamic_partition_reserve_false_db"
String tableName = "dynamic_partition_reserve_false_table"

2
regression-test/suites/backup_restore/test_backup_restore_reserve_dynamic_partition_true.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_backup_restore_dynamic_partition_reserve_true", "backup_restore") {
String repoName = "test_backup_restore_dynamic_partition_reserve_true_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "backup_restore_dynamic_partition_reserve_true_db"
String tableName = "dynamic_partition_reserve_true_table"

2
regression-test/suites/backup_restore/test_backup_restore_table_auto_bucket.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_backup_restore_table_auto_bucket", "backup_restore") {
String repoName = "test_backup_restore_table_auto_bucket_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "backup_restore_table_with_auto_bucket_db"
String tableName = "auto_bucket_table"

2
regression-test/suites/backup_restore/test_backup_restore_with.groovy
View File

@ -18,7 +18,7 @@
suite("test_backup_restore_with_view", "backup_restore") {
String suiteName = "backup_restore_with_view"
String dbName = "${suiteName}_db"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String snapshotName = "${suiteName}_snapshot"
String tableName = "${suiteName}_table"
String viewName = "${suiteName}_view"

2
regression-test/suites/backup_restore/test_backup_restore_with_hdfs.groovy
View File

@ -23,7 +23,7 @@ suite("test_backup_restore_with_hdfs", "backup_restore") {
return
}
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "${suiteName}_db"
String tableName = "${suiteName}_table"
String snapshotName = "${suiteName}_snapshot"

2
regression-test/suites/backup_restore/test_backup_restore_with_two_phase_fetch_opt.groovy
View File

@ -17,7 +17,7 @@
suite("test_backup_restore_with_two_phase_fetch_opt", "backup_restore") {
String suiteName = "test_backup_restore_with_two_phase_fetch_opt"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "${suiteName}_db"
String tableName = "${suiteName}_table"
String snapshotName = "${suiteName}_snapshot"

4
regression-test/suites/backup_restore/test_duplicate_backup_restore.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_duplicate_backup_restore", "backup_restore") {
String repoName = "test_duplicate_backup_restore_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "duplicate_backup_restore_db"
String tableName = "test_duplicate_backup_restore_table"
@ -81,7 +81,7 @@ suite("test_duplicate_backup_restore", "backup_restore") {
sql "DELETE FROM ${dbName}.${tableName} WHERE id = 1"
result = sql "SELECT * FROM ${dbName}.${tableName}"
assertEquals(result.size(), values.size() + 5 - 3)
repoName = "test_duplicate_delete_backup_restore_repo"
repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
syncer.createS3Repository(repoName)
snapshotName = "test_duplicate_delete_backup_restore_snapshot"
sql """

2
regression-test/suites/backup_restore/test_restore_mix_exists_and_new_table.groovy
View File

@ -18,7 +18,7 @@
suite("test_restore_mix_exists_and_new_table", "backup_restore") {
String dbName = "restore_mix_exists_and_new_table"
String suiteName = "test_restore_mix_exists_and_new_table"
String repoName = "${suiteName}_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String snapshotName = "${suiteName}_snapshot"
String tableNamePrefix = "${suiteName}_tables"

2
regression-test/suites/backup_restore/test_restore_to_new_table.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_restore_to_new_table", "backup_restore") {
String repoName = "test_restore_to_new_table_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "restore_to_new_table_db"
String tableName = "test_restore_to_new_table_table"
String snapshotName = "test_backup_restore_snapshot"

4
regression-test/suites/backup_restore/test_unique_backup_restore.groovy
View File

@ -16,7 +16,7 @@
// under the License.
suite("test_unique_backup_restore", "backup_restore") {
String repoName = "test_unique_backup_restore_repo"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String dbName = "unique_backup_restore_db"
String tableName = "test_unique_backup_restore_table"
@ -81,7 +81,7 @@ suite("test_unique_backup_restore", "backup_restore") {
sql "DELETE FROM ${dbName}.${tableName} WHERE id = 1"
result = sql "SELECT * FROM ${dbName}.${tableName}"
assertEquals(result.size(), 1)
repoName = "test_unique_delete_backup_restore_repo"
repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
syncer.createS3Repository(repoName)
snapshotName = "test_unique_delete_backup_restore_snapshot"
sql """