支持gs_probackup使用对象存储备份备节点时可以跨集群恢复

src/bin/pg_probackup/backup.cpp

@@ -1859,6 +1859,10 @@ void write_table_label_and_tablespace_map(pgBackup *backup, PGresult *res,
             file->uncompressed_size = file->size;
             parray_append(backup_files_list, file);
         }
+
+        if (current.media_type == MEDIA_TYPE_OSS) {
+            uploadConfigFile(backup_label, backup_label);
+        }
     }
 
     if (sscanf_s(PQgetvalue(res, 0, 0), XID_FMT, recovery_xid) != 1)
@@ -1905,6 +1909,10 @@ void write_table_label_and_tablespace_map(pgBackup *backup, PGresult *res,
 
             parray_append(backup_files_list, file);
         }
+
+        if (current.media_type == MEDIA_TYPE_OSS) {
+            uploadConfigFile(tablespace_map, tablespace_map);
+        }
     }
 
 }

src/bin/pg_probackup/catalog.cpp

@@ -460,10 +460,6 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id)
         goto err_proc;
     }
 
-    if (current.media_type == MEDIA_TYPE_OSS) {
-        restoreConfigDir();
-    }
-
     /* scan the directory and list backups */
     backups = parray_new();
     for (; (data_ent = fio_readdir(data_dir)) != NULL; errno = 0)

src/bin/pg_probackup/dir.cpp

@@ -1407,6 +1407,10 @@ read_tablespace_map(parray *files, const char *backup_dir)
     join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE);
 
     /* Exit if database/tablespace_map doesn't exist */
+    if (current.media_type == MEDIA_TYPE_OSS) {
+        restoreConfigFile(map_path, true);
+    }
+
     if (!fileExists(map_path, FIO_BACKUP_HOST))
     {
         elog(LOG, "there is no file tablespace_map");

src/bin/pg_probackup/oss/include/oss_operator.h

@@ -23,25 +23,6 @@
 #define OSS_MAX_FILE_PATH 1024
 #define OSS_MAX_ETAG_LEN 256
 
-/* Data Structure Definition*/
-
-typedef struct OssFile {
-    char filePath[OSS_MAX_FILE_PATH];
-    /* for write */
-    char uploadID[OSS_MAX_UPLOAD_ID_LEN];
-    char** eTagList;
-    int partNum;
-    /* for read */
-    size_t fileSize;
-    bool oss_eof;
-    bool oss_error;
-    void* bufDate;
-    int byteCout;
-    int actualLen;
-    size_t offset;
-    char etag[OSS_MAX_ETAG_LEN];
-} OssFile;
-
 /* API Function */
 
 namespace Oss {
@@ -53,7 +34,7 @@ class Oss {
 public:
     Oss(const char* endpoint, const char* access_key, const char* secret_key, const char* region = NULL, bool secure = false);
     ~Oss();
-    void GetObject(const char* bucket_name, const char* object_name, const char* file_name);
+    void GetObject(const char* bucket_name, const char* object_name, const char* file_name, bool errorOk = false);
     void GetObject(const char* from_bucket, const char* object_key, void* file);
     void PutObject(const char* bucket_name, const char* file_path, const char* file_name);
     void RemoveObject(const char* bucket_name, const char* objcet_key);

src/bin/pg_probackup/oss/include/restore.h

@@ -39,7 +39,7 @@ extern void closeRestoreFile(FileAppenderSegDescriptor* desc);
 
 extern void restoreConfigDir();
 
-extern void restoreConfigFile(const char* path);
+extern void restoreConfigFile(const char* path, bool errorOk = false);
 
 extern void uploadConfigFile(const char* local_path, const char* object_name);
 

src/bin/pg_probackup/oss/oss_operator.cpp

@@ -99,7 +99,7 @@ void Oss::GetObject(const char* from_bucket, const char* object_key, void* fileP
     }
 }
 
-void Oss::GetObject(const char* bucket_name, const char* object_name, const char* file_name) {
+void Oss::GetObject(const char* bucket_name, const char* object_name, const char* file_name, bool errorOk) {
     auto s3_client = reinterpret_cast<Aws::S3::S3Client *>(s3_client_);
     Aws::S3::Model::GetObjectRequest request;
     request.SetBucket(bucket_name);
@@ -107,7 +107,9 @@ void Oss::GetObject(const char* bucket_name, const char* object_name, const char
     auto outcome = s3_client->GetObject(request);
     if (!outcome.IsSuccess()) {
         auto err = outcome.GetError();
-        elog(ERROR, "GetObject: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str());
+        int elevel = errorOk ? WARNING : ERROR;
+        elog(elevel, "GetObject: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str());
+        return;
     }
     char* separator_pos = last_dir_separator(file_name);
     char* dir_path = strndup(file_name, separator_pos - file_name);

src/bin/pg_probackup/oss/restore.cpp

@@ -296,9 +296,15 @@ void restoreConfigDir()
     Oss::Oss* oss = getOssClient();
     char* bucket_name = getBucketName();
     char* prefix_name = backup_instance_path + 1;
+    char dir_path[MAXPGPATH];
+    char arclog_path[MAXPGPATH];
+    int nRet = 0;
+    fio_mkdir(backup_instance_path, DIR_PERMISSION, location);
+    nRet = snprintf_s(arclog_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/%s", backup_path, "wal", instance_name);
+    securec_check_ss_c(nRet, "\0", "\0");
+    fio_mkdir(arclog_path, DIR_PERMISSION, location);
     parray *obj_list = parray_new();
     oss->ListObjectsWithPrefix(bucket_name, prefix_name, obj_list);
-    char    dir_path[MAXPGPATH];
     for (size_t i = 0; i < parray_num(obj_list); i++) {
         char* object = (char*)parray_get(obj_list, i);
         char* filename = last_dir_separator(object);
@@ -316,14 +322,14 @@ void restoreConfigDir()
     parray_free(obj_list);
 }
 
-void restoreConfigFile(const char* path)
+void restoreConfigFile(const char* path, bool errorOk)
 {
     Oss::Oss* oss = getOssClient();
     const char* object_name = NULL;
     const char* bucket_name = NULL;
     bucket_name = getBucketName();
     object_name = path;
-    oss->GetObject(bucket_name, object_name, (char*)path);
+    oss->GetObject(bucket_name, object_name, (char*)path, errorOk);
 }
 
 void uploadConfigFile(const char* path, const char* object_name)

src/bin/pg_probackup/pg_probackup.cpp

@@ -419,6 +419,10 @@ static void parse_instance_name()
         {
             struct stat st;
 
+            if (current.media_type == MEDIA_TYPE_OSS) {
+                restoreConfigDir();
+            }
+
             if (fio_stat(backup_instance_path, &st, true, FIO_BACKUP_HOST) != 0)
             {
                 elog(WARNING, "Failed to access directory \"%s\": %s",
@@ -458,7 +462,7 @@ static void parse_instance_name()
             }
             config_read_opt(path, instance_options, ERROR, true, false);
             if (current.media_type == MEDIA_TYPE_OSS) {
-                remove(path);
+                uploadConfigFile(path, path);
             }
         }
         setMyLocation();

src/bin/pg_probackup/restore.cpp

@@ -2268,6 +2268,9 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
     {
         rc = snprintf_s(backup_label, MAXPGPATH, MAXPGPATH - 1, "%s/backup_label", pgdata);
         securec_check_ss_c(rc, "\0", "\0");
+        if (current.media_type == MEDIA_TYPE_OSS) {
+            restoreConfigFile(backup_label, true);
+        }
         if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0)
         {
             elog(WARNING, "Destination directory contains \"backup_control\" file. "

src/bin/pg_probackup/util.cpp

@@ -698,6 +698,9 @@ set_min_recovery_point(pgFile *file, const char *fullpath,
 
     /* Update pg_control checksum in backup_list */
     file->crc = ControlFile.crc;
+    if (current.media_type == MEDIA_TYPE_OSS) {
+        uploadConfigFile(fullpath, fullpath);
+    }
 
     pg_free(buffer);
 }