From 917e617df8de78ff2e02b963c68a90245fa2a40e Mon Sep 17 00:00:00 2001 From: lukeman Date: Sat, 20 Jul 2024 17:45:20 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81gs=5Fprobackup=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E5=AF=B9=E8=B1=A1=E5=AD=98=E5=82=A8=E5=A4=87=E4=BB=BD?= =?UTF-8?q?=E5=A4=87=E8=8A=82=E7=82=B9=E6=97=B6=E5=8F=AF=E4=BB=A5=E8=B7=A8?= =?UTF-8?q?=E9=9B=86=E7=BE=A4=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_probackup/backup.cpp | 8 +++++++ src/bin/pg_probackup/catalog.cpp | 4 ---- src/bin/pg_probackup/dir.cpp | 4 ++++ .../pg_probackup/oss/include/oss_operator.h | 21 +------------------ src/bin/pg_probackup/oss/include/restore.h | 2 +- src/bin/pg_probackup/oss/oss_operator.cpp | 6 ++++-- src/bin/pg_probackup/oss/restore.cpp | 12 ++++++++--- src/bin/pg_probackup/pg_probackup.cpp | 6 +++++- src/bin/pg_probackup/restore.cpp | 3 +++ src/bin/pg_probackup/util.cpp | 3 +++ 10 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/bin/pg_probackup/backup.cpp b/src/bin/pg_probackup/backup.cpp index 19655066d..76833d3c0 100644 --- a/src/bin/pg_probackup/backup.cpp +++ b/src/bin/pg_probackup/backup.cpp @@ -1859,6 +1859,10 @@ void write_table_label_and_tablespace_map(pgBackup *backup, PGresult *res, file->uncompressed_size = file->size; parray_append(backup_files_list, file); } + + if (current.media_type == MEDIA_TYPE_OSS) { + uploadConfigFile(backup_label, backup_label); + } } if (sscanf_s(PQgetvalue(res, 0, 0), XID_FMT, recovery_xid) != 1) @@ -1905,6 +1909,10 @@ void write_table_label_and_tablespace_map(pgBackup *backup, PGresult *res, parray_append(backup_files_list, file); } + + if (current.media_type == MEDIA_TYPE_OSS) { + uploadConfigFile(tablespace_map, tablespace_map); + } } } diff --git a/src/bin/pg_probackup/catalog.cpp b/src/bin/pg_probackup/catalog.cpp index f5e107ee0..f64571055 100644 --- a/src/bin/pg_probackup/catalog.cpp +++ b/src/bin/pg_probackup/catalog.cpp @@ -460,10 +460,6 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) goto err_proc; } - if (current.media_type == MEDIA_TYPE_OSS) { - restoreConfigDir(); - } - /* scan the directory and list backups */ backups = parray_new(); for (; (data_ent = fio_readdir(data_dir)) != NULL; errno = 0) diff --git a/src/bin/pg_probackup/dir.cpp b/src/bin/pg_probackup/dir.cpp index e07894a00..9a2b55a81 100644 --- a/src/bin/pg_probackup/dir.cpp +++ b/src/bin/pg_probackup/dir.cpp @@ -1407,6 +1407,10 @@ read_tablespace_map(parray *files, const char *backup_dir) join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); /* Exit if database/tablespace_map doesn't exist */ + if (current.media_type == MEDIA_TYPE_OSS) { + restoreConfigFile(map_path, true); + } + if (!fileExists(map_path, FIO_BACKUP_HOST)) { elog(LOG, "there is no file tablespace_map"); diff --git a/src/bin/pg_probackup/oss/include/oss_operator.h b/src/bin/pg_probackup/oss/include/oss_operator.h index 5fcf0b275..0b2de2e2d 100644 --- a/src/bin/pg_probackup/oss/include/oss_operator.h +++ b/src/bin/pg_probackup/oss/include/oss_operator.h @@ -23,25 +23,6 @@ #define OSS_MAX_FILE_PATH 1024 #define OSS_MAX_ETAG_LEN 256 -/* Data Structure Definition*/ - -typedef struct OssFile { - char filePath[OSS_MAX_FILE_PATH]; - /* for write */ - char uploadID[OSS_MAX_UPLOAD_ID_LEN]; - char** eTagList; - int partNum; - /* for read */ - size_t fileSize; - bool oss_eof; - bool oss_error; - void* bufDate; - int byteCout; - int actualLen; - size_t offset; - char etag[OSS_MAX_ETAG_LEN]; -} OssFile; - /* API Function */ namespace Oss { @@ -53,7 +34,7 @@ class Oss { public: Oss(const char* endpoint, const char* access_key, const char* secret_key, const char* region = NULL, bool secure = false); ~Oss(); - void GetObject(const char* bucket_name, const char* object_name, const char* file_name); + void GetObject(const char* bucket_name, const char* object_name, const char* file_name, bool errorOk = false); void GetObject(const char* from_bucket, const char* object_key, void* file); void PutObject(const char* bucket_name, const char* file_path, const char* file_name); void RemoveObject(const char* bucket_name, const char* objcet_key); diff --git a/src/bin/pg_probackup/oss/include/restore.h b/src/bin/pg_probackup/oss/include/restore.h index a7498a38a..e3803b57a 100644 --- a/src/bin/pg_probackup/oss/include/restore.h +++ b/src/bin/pg_probackup/oss/include/restore.h @@ -39,7 +39,7 @@ extern void closeRestoreFile(FileAppenderSegDescriptor* desc); extern void restoreConfigDir(); -extern void restoreConfigFile(const char* path); +extern void restoreConfigFile(const char* path, bool errorOk = false); extern void uploadConfigFile(const char* local_path, const char* object_name); diff --git a/src/bin/pg_probackup/oss/oss_operator.cpp b/src/bin/pg_probackup/oss/oss_operator.cpp index 90f3cfeea..cf0e93be8 100644 --- a/src/bin/pg_probackup/oss/oss_operator.cpp +++ b/src/bin/pg_probackup/oss/oss_operator.cpp @@ -99,7 +99,7 @@ void Oss::GetObject(const char* from_bucket, const char* object_key, void* fileP } } -void Oss::GetObject(const char* bucket_name, const char* object_name, const char* file_name) { +void Oss::GetObject(const char* bucket_name, const char* object_name, const char* file_name, bool errorOk) { auto s3_client = reinterpret_cast(s3_client_); Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket_name); @@ -107,7 +107,9 @@ void Oss::GetObject(const char* bucket_name, const char* object_name, const char auto outcome = s3_client->GetObject(request); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); - elog(ERROR, "GetObject: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str()); + int elevel = errorOk ? WARNING : ERROR; + elog(elevel, "GetObject: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str()); + return; } char* separator_pos = last_dir_separator(file_name); char* dir_path = strndup(file_name, separator_pos - file_name); diff --git a/src/bin/pg_probackup/oss/restore.cpp b/src/bin/pg_probackup/oss/restore.cpp index 40a485833..04f71ea58 100644 --- a/src/bin/pg_probackup/oss/restore.cpp +++ b/src/bin/pg_probackup/oss/restore.cpp @@ -296,9 +296,15 @@ void restoreConfigDir() Oss::Oss* oss = getOssClient(); char* bucket_name = getBucketName(); char* prefix_name = backup_instance_path + 1; + char dir_path[MAXPGPATH]; + char arclog_path[MAXPGPATH]; + int nRet = 0; + fio_mkdir(backup_instance_path, DIR_PERMISSION, location); + nRet = snprintf_s(arclog_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/%s", backup_path, "wal", instance_name); + securec_check_ss_c(nRet, "\0", "\0"); + fio_mkdir(arclog_path, DIR_PERMISSION, location); parray *obj_list = parray_new(); oss->ListObjectsWithPrefix(bucket_name, prefix_name, obj_list); - char dir_path[MAXPGPATH]; for (size_t i = 0; i < parray_num(obj_list); i++) { char* object = (char*)parray_get(obj_list, i); char* filename = last_dir_separator(object); @@ -316,14 +322,14 @@ void restoreConfigDir() parray_free(obj_list); } -void restoreConfigFile(const char* path) +void restoreConfigFile(const char* path, bool errorOk) { Oss::Oss* oss = getOssClient(); const char* object_name = NULL; const char* bucket_name = NULL; bucket_name = getBucketName(); object_name = path; - oss->GetObject(bucket_name, object_name, (char*)path); + oss->GetObject(bucket_name, object_name, (char*)path, errorOk); } void uploadConfigFile(const char* path, const char* object_name) diff --git a/src/bin/pg_probackup/pg_probackup.cpp b/src/bin/pg_probackup/pg_probackup.cpp index bfd1dc484..d10d119e2 100644 --- a/src/bin/pg_probackup/pg_probackup.cpp +++ b/src/bin/pg_probackup/pg_probackup.cpp @@ -419,6 +419,10 @@ static void parse_instance_name() { struct stat st; + if (current.media_type == MEDIA_TYPE_OSS) { + restoreConfigDir(); + } + if (fio_stat(backup_instance_path, &st, true, FIO_BACKUP_HOST) != 0) { elog(WARNING, "Failed to access directory \"%s\": %s", @@ -458,7 +462,7 @@ static void parse_instance_name() } config_read_opt(path, instance_options, ERROR, true, false); if (current.media_type == MEDIA_TYPE_OSS) { - remove(path); + uploadConfigFile(path, path); } } setMyLocation(); diff --git a/src/bin/pg_probackup/restore.cpp b/src/bin/pg_probackup/restore.cpp index a323e0985..11d109654 100644 --- a/src/bin/pg_probackup/restore.cpp +++ b/src/bin/pg_probackup/restore.cpp @@ -2268,6 +2268,9 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, { rc = snprintf_s(backup_label, MAXPGPATH, MAXPGPATH - 1, "%s/backup_label", pgdata); securec_check_ss_c(rc, "\0", "\0"); + if (current.media_type == MEDIA_TYPE_OSS) { + restoreConfigFile(backup_label, true); + } if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0) { elog(WARNING, "Destination directory contains \"backup_control\" file. " diff --git a/src/bin/pg_probackup/util.cpp b/src/bin/pg_probackup/util.cpp index 93a30d70c..d99c92c2f 100644 --- a/src/bin/pg_probackup/util.cpp +++ b/src/bin/pg_probackup/util.cpp @@ -698,6 +698,9 @@ set_min_recovery_point(pgFile *file, const char *fullpath, /* Update pg_control checksum in backup_list */ file->crc = ControlFile.crc; + if (current.media_type == MEDIA_TYPE_OSS) { + uploadConfigFile(fullpath, fullpath); + } pg_free(buffer); }