diff --git a/build/script/aarch64_lite_list b/build/script/aarch64_lite_list index 38b7eacdd..fdbd3de10 100644 --- a/build/script/aarch64_lite_list +++ b/build/script/aarch64_lite_list @@ -701,6 +701,8 @@ ./lib/libdcf.so ./lib/libzstd.so* ./lib/libcurl.so* +./lib/libaws-cpp-sdk-core.so +./lib/libaws-cpp-sdk-s3.so ./lib/libxgboost.so ./lib/libpagecompression.so* ./lib/postgresql/latin2_and_win1250.so diff --git a/build/script/aarch64_opengauss_list b/build/script/aarch64_opengauss_list index dc5d52fa5..102ad439e 100644 --- a/build/script/aarch64_opengauss_list +++ b/build/script/aarch64_opengauss_list @@ -876,6 +876,8 @@ ./lib/krb5/plugins/kdb/db2.so ./lib/libverto.so* ./lib/libcurl.so* +./lib/libaws-cpp-sdk-core.so +./lib/libaws-cpp-sdk-s3.so ./lib/libcrypto.so* ./lib/libssl.so* ./lib/libgcc_s.so.1 diff --git a/build/script/opengauss_release_list_ubuntu_single b/build/script/opengauss_release_list_ubuntu_single index 617221026..0632e8e03 100644 --- a/build/script/opengauss_release_list_ubuntu_single +++ b/build/script/opengauss_release_list_ubuntu_single @@ -869,6 +869,8 @@ ./lib/libcurl.so ./lib/libcurl.so.4 ./lib/libcurl.so.4.6.0 +./lib/libaws-cpp-sdk-core.so +./lib/libaws-cpp-sdk-s3.so ./lib/libcrypto.so ./lib/libcrypto.so.1.1 ./lib/libssl.so diff --git a/build/script/x86_64_lite_list b/build/script/x86_64_lite_list index ca989045f..5a1647c2d 100644 --- a/build/script/x86_64_lite_list +++ b/build/script/x86_64_lite_list @@ -700,6 +700,8 @@ ./lib/libdcf.so ./lib/libzstd.so* ./lib/libcurl.so* +./lib/libaws-cpp-sdk-core.so +./lib/libaws-cpp-sdk-s3.so ./lib/libxgboost.so ./lib/libpagecompression.so* ./lib/postgresql/latin2_and_win1250.so diff --git a/build/script/x86_64_opengauss_list b/build/script/x86_64_opengauss_list index a59ef549a..98b009cc7 100644 --- a/build/script/x86_64_opengauss_list +++ b/build/script/x86_64_opengauss_list @@ -873,6 +873,8 @@ ./lib/krb5/plugins/kdb/db2.so ./lib/libverto.so* ./lib/libcurl.so* +./lib/libaws-cpp-sdk-core.so +./lib/libaws-cpp-sdk-s3.so ./lib/libcrypto.so* ./lib/libssl.so* ./lib/libgcc_s.so.1 diff --git a/cmake/src/set_thirdparty_path.cmake b/cmake/src/set_thirdparty_path.cmake index f6eba3f04..6b0bb651f 100755 --- a/cmake/src/set_thirdparty_path.cmake +++ b/cmake/src/set_thirdparty_path.cmake @@ -61,6 +61,7 @@ set(LLVM_HOME ${DEPENDENCY_PATH}/llvm/${LIB_UNIFIED_SUPPORT}) set(LZ4_HOME ${DEPENDENCY_PATH}/lz4/${SUPPORT_LLT}) set(NANOMSG_HOME ${DEPENDENCY_PATH}/nng/${LIB_UNIFIED_SUPPORT}) set(NCURSES_HOME ${DEPENDENCY_PATH}/ncurses/${SUPPORT_LLT}) +set(AWSSDK_HOME ${DEPENDENCY_PATH}/aws-sdk-cpp/${SUPPORT_LLT}) if(($ENV{WITH_TASSL}) STREQUAL "YES") set(OPENSSL_HOME ${DEPENDENCY_PATH}/tassl/${LIB_UNIFIED_SUPPORT}) else() @@ -193,6 +194,12 @@ set(LIBCGROUP_LIB_PATH ${CGROUP_HOME}/lib) set(LIBCURL_INCLUDE_PATH ${CURL_HOME}/include) set(LIBCURL_LIB_PATH ${CURL_HOME}/lib) +############################################################################# +# awssdk component +############################################################################# +set(AWSSDK_INCLUDE_PATH ${AWSSDK_HOME}/include) +set(AWSSDK_LIB_PATH ${AWSSDK_HOME}/lib) + ############################################################################# # edit component ############################################################################# diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 7eb8bda49..b277365ce 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -300,6 +300,7 @@ ifeq ($(with_3rd), NONE) MOCKCPP_HOME = $(top_builddir)/$(BUILD_TOOLS_PATH)/mockcpp/$(LIB_NOT_SUPPORT_LLT) NUMA_HOME = $(top_builddir)/$(BINARYPATH)/numactl/$(LIB_SUPPORT_LLT) LIBCURL_HOME = $(top_builddir)/$(BINARYPATH)/libcurl/$(LIB_SUPPORT_LLT) + AWSSDK_HOME = $(top_builddir)/$(BINARYPATH)/aws-sdk-cpp/$(LIB_SUPPORT_LLT) ZSTD_HOME = $(top_builddir)/$(BINARYPATH)/zstd LIBNANOMSG_HOME = $(top_builddir)/$(BINARYPATH)/nng/comm PLJAVA_HOME = $(top_builddir)/$(BINARYPATH)/pljava/$(LIB_SUPPORT_LLT) @@ -349,6 +350,7 @@ endif MOCKCPP_HOME = $(with_3rd)/$(BUILD_TOOLS_PATH)/mockcpp/$(LIB_NOT_SUPPORT_LLT) NUMA_HOME = $(with_3rd)/$(BINARYPATH)/numactl/$(LIB_SUPPORT_LLT) LIBCURL_HOME = $(with_3rd)/$(BINARYPATH)/libcurl/$(LIB_SUPPORT_LLT) + AWSSDK_HOME = $(with_3rd)/$(BINARYPATH)/aws-sdk-cpp/$(LIB_SUPPORT_LLT) ZSTD_HOME = $(with_3rd)/$(BINARYPATH)/zstd LIBNANOMSG_HOME = $(with_3rd)/$(BINARYPATH)/nng/comm PLJAVA_HOME = $(with_3rd)/$(BINARYPATH)/pljava/$(LIB_SUPPORT_LLT) @@ -647,6 +649,12 @@ NUMA_LIB_PATH = $(NUMA_HOME)/lib LIBCURL_INCLUDE_PATH = $(LIBCURL_HOME)/include LIBCURL_LIB_PATH = $(LIBCURL_HOME)/lib +############################################################################# +# awssdk component +############################################################################# +AWSSDK_INCLUDE_PATH = $(AWSSDK_HOME)/include +AWSSDK_LIB_PATH = $(AWSSDK_HOME)/lib + ############################################################################# # masstree component ############################################################################# @@ -728,7 +736,7 @@ else # not PGXS endif endif -override CPPFLAGS := $(CPPFLAGS) -I$(LIBODBC_INCLUDE_PATH) -I$(LIBOBS_INCLUDE_PATH) -I$(LIBCGROUP_INCLUDE_PATH) -I$(LIBOPENSSL_INCLUDE_PATH) -I${BOOST_INCLUDE_PATH} -I$(LIBLLVM_INCLUDE_PATH) -I$(KERBEROS_INCLUDE_PATH) -I$(CJSON_INCLUDE_PATH) -I$(NUMA_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH) -I$(LZ4_INCLUDE_PATH) -I$(ZSTD_INCLUDE_PATH) -I$(LIBCURL_INCLUDE_PATH) -I$(DCF_INCLUDE_PATH) +override CPPFLAGS := $(CPPFLAGS) -I$(LIBODBC_INCLUDE_PATH) -I$(LIBOBS_INCLUDE_PATH) -I$(LIBCGROUP_INCLUDE_PATH) -I$(LIBOPENSSL_INCLUDE_PATH) -I${BOOST_INCLUDE_PATH} -I$(LIBLLVM_INCLUDE_PATH) -I$(KERBEROS_INCLUDE_PATH) -I$(CJSON_INCLUDE_PATH) -I$(NUMA_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH) -I$(LZ4_INCLUDE_PATH) -I$(ZSTD_INCLUDE_PATH) -I$(LIBCURL_INCLUDE_PATH) -I$(AWSSDK_INCLUDE_PATH) -I$(DCF_INCLUDE_PATH) # GDS links to libevent ifeq ($(enable_multiple_nodes), yes) @@ -894,6 +902,7 @@ endif LDFLAGS += -L$(GSTRACE_LIB_PATH) LDFLAGS += -L$(NUMA_LIB_PATH) LDFLAGS += -L$(LIBCURL_LIB_PATH) +LDFLAGS += -L$(AWSSDK_LIB_PATH) ifeq ($(enable_mot), yes) LDFLAGS += -L$(MASSTREE_LIB_PATH) endif @@ -948,6 +957,12 @@ GEN_KEYWORDLIST_DEPS = $(TOOLSDIR)/gen_keywordlist.pl $(TOOLSDIR)/PerfectHash.pm LIBCURL_INCLUDE_PATH = $(LIBCURL_HOME)/include LIBCURL_LIB_PATH = $(LIBCURL_HOME)/lib +############################################################################# +# awssdk component +############################################################################# +AWSSDK_INCLUDE_PATH = $(AWSSDK_HOME)/include +AWSSDK_LIB_PATH = $(AWSSDK_HOME)/lib + ############################################################################# # Perl diff --git a/src/Makefile.global.in_for_llt b/src/Makefile.global.in_for_llt index 83eac4245..955e120dd 100755 --- a/src/Makefile.global.in_for_llt +++ b/src/Makefile.global.in_for_llt @@ -660,6 +660,13 @@ LIBCURL_HOME = $(top_builddir)/../$(BINARYPATH)/libcurl/comm LIBCURL_INCLUDE_PATH = $(LIBCURL_HOME)/include LIBCURL_LIB_PATH = $(LIBCURL_HOME)/lib +############################################################################# +# awssdk component +############################################################################# +AWSSDK_HOME = $(top_builddir)/../$(BINARYPATH)/aws-sdk-cpp/comm +AWSSDK_INCLUDE_PATH = $(AWSSDK_HOME)/include +AWSSDK_LIB_PATH = $(AWSSDK_HOME)/lib + ############################################################################# ############################################################################# diff --git a/src/bin/pg_probackup/CMakeLists.txt b/src/bin/pg_probackup/CMakeLists.txt index 193cd5909..9d5e9a5d5 100755 --- a/src/bin/pg_probackup/CMakeLists.txt +++ b/src/bin/pg_probackup/CMakeLists.txt @@ -15,12 +15,14 @@ execute_process( AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_probackup_SRC) -set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${LZ4_INCLUDE_PATH} ${ZSTD_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression ${PROJECT_SRC_DIR}/include/storage/gs_uwal) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/oss TGT_probackup_SRC) + +set(TGT_probackup_INC ${ZLIB_INCLUDE_PATH} ${LZ4_INCLUDE_PATH} ${ZSTD_INCLUDE_PATH} ${PROJECT_SRC_DIR}/lib/page_compression ${PROJECT_SRC_DIR}/include/storage/gs_uwal ${AWSSDK_INCLUDE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/oss) set(probackup_DEF_OPTIONS ${MACRO_OPTIONS} -DFRONTEND -DHAVE_LIBZ) set(probackup_COMPILE_OPTIONS ${PROTECT_OPTIONS} ${BIN_SECURE_OPTIONS} ${OS_OPTIONS} ${WARNING_OPTIONS} ${OPTIMIZE_OPTIONS} ${CHECK_OPTIONS}) set(probackup_LINK_OPTIONS ${BIN_LINK_OPTIONS}) -set(probackup_LINK_LIBS libpgcommon.a libpgport.a -lcrypt -ldl -lm -lssl -lcrypto -l${SECURE_C_CHECK} -lrt -lz -lminiunz -llz4 -lpq -lpagecompression -lzstd) +set(probackup_LINK_LIBS libpgcommon.a libpgport.a -lcrypt -ldl -lm -lssl -lcrypto -l${SECURE_C_CHECK} -lrt -lz -lminiunz -llz4 -lpq -lpagecompression -lzstd -laws-cpp-sdk-core -laws-cpp-sdk-s3) if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON") list(APPEND probackup_LINK_LIBS -lgssapi_krb5_gauss -lgssrpc_gauss -lkrb5_gauss -lkrb5support_gauss -lk5crypto_gauss -lcom_err_gauss) endif() @@ -32,6 +34,6 @@ add_bintarget(gs_probackup TGT_probackup_SRC TGT_probackup_INC "${probackup_DEF_ add_dependencies(gs_probackup pq pgport_static pagecompression) target_link_directories(gs_probackup PUBLIC ${CMAKE_BINARY_DIR}/lib ${LIBOPENSSL_LIB_PATH} ${LIBEDIT_LIB_PATH} ${ZSTD_LIB_PATH} - ${ZLIB_LIB_PATH} ${KERBEROS_LIB_PATH} ${LZ4_LIB_PATH} ${SECURE_LIB_PATH} ${probackup_LINK_DIRS} + ${ZLIB_LIB_PATH} ${KERBEROS_LIB_PATH} ${LZ4_LIB_PATH} ${SECURE_LIB_PATH} ${probackup_LINK_DIRS} ${AWSSDK_LIB_PATH} ) install(TARGETS gs_probackup RUNTIME DESTINATION bin) diff --git a/src/bin/pg_probackup/Makefile b/src/bin/pg_probackup/Makefile index dee268dd1..5cb45cf0e 100644 --- a/src/bin/pg_probackup/Makefile +++ b/src/bin/pg_probackup/Makefile @@ -25,6 +25,8 @@ OBJS += $(top_builddir)/src/lib/pgcommon/libpgcommon.a \ $(top_builddir)/src/gausskernel/storage/dss/dss_adaptor.o \ $(top_builddir)/src/gausskernel/storage/gs_uwal/gs_uwal_adaptor.o +OBJS += oss/appender.o oss/backup.o oss/restore.o oss/buffer.o oss/oss_operator.o oss/thread.o + EXTRA_CLEAN = datapagemap.cpp datapagemap.h \ receivelog.cpp receivelog.h streamutil.cpp streamutil.h \ xlogreader.cpp instr_time.h @@ -34,6 +36,9 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global EXTRA_CLEAN += logging.h +AWS_SDK_INCLUDE_PATH = $(AWSSDK_INCLUDE_PATH) +AWS_SDK_LIB_PATH = $(AWSSDK_LIB_PATH) + LDFLAGS += -L$(LZ4_LIB_PATH) -L$(ZSTD_LIB_PATH) LIBS += -llz4 ifeq ($(enable_lite_mode), no) @@ -42,11 +47,17 @@ endif PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_builddir)/$(subdir) -I$(LZ4_INCLUDE_PATH) -I$(ZLIB_INCLUDE_PATH) -I$(ZSTD_INCLUDE_PATH) # add page_compression so .h LDFLAGS += -L../../lib/page_compression -PG_CPPFLAGS = -I../../lib/page_compression -I../../src/common/port +PG_CPPFLAGS = -I../../lib/page_compression -I../../src/common/port -I$(AWS_SDK_INCLUDE_PATH) LIBS += -lpagecompression -lzstd override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) -DHAVE_LIBZ PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} +# add oss .h +LDFLAGS += -I$(top_builddir)/$(subdir)/oss/include + +# add aws s3 so .h +LDFLAGS += -L$(AWS_SDK_LIB_PATH) -laws-cpp-sdk-core -laws-cpp-sdk-s3 -I$(AWS_SDK_INCLUDE_PATH) + all: submake-pagecompression $(PROGRAM) gs_probackup: $(OBJS) | submake-libpq submake-libpgport diff --git a/src/bin/pg_probackup/backup.cpp b/src/bin/pg_probackup/backup.cpp index d6ccfee24..19655066d 100644 --- a/src/bin/pg_probackup/backup.cpp +++ b/src/bin/pg_probackup/backup.cpp @@ -29,6 +29,8 @@ #include "common/fe_memutils.h" #include "storage/file/fio_device.h" #include "logger.h" +#include "oss/include/backup.h" +#include "oss/include/restore.h" /* list of dirs which will not to be backuped @@ -65,6 +67,9 @@ static pthread_mutex_t g_mutex = PTHREAD_MUTEX_INITIALIZER; /* list of files contained in backup */ static parray *backup_files_list = NULL; +static parray *filesinfo = NULL; +static SenderCxt *current_sender_cxt = NULL; +static ReaderCxt *current_reader_cxt = NULL; /* We need critical section for datapagemap_add() in case of using threads */ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -254,7 +259,6 @@ static void run_backup_threads(char *external_prefix, char *database_path, char for (i = 0; i < num_threads; i++) { backup_files_arg *arg = &(threads_args[i]); - elog(VERBOSE, "Start thread num: %i", i); pthread_create(&threads[i], NULL, backup_files, arg); } @@ -521,6 +525,9 @@ static void add_xlog_files_into_backup_list(const char *database_path, const cha file->name = file->rel_path; else file->name++; + if (current.media_type == MEDIA_TYPE_OSS) { + uploadConfigFile(wal_full_path, wal_full_path); + } } /* Add xlog files into the list of backed up files */ @@ -564,7 +571,6 @@ static void sync_files(parray *database_map, const char *database_path, parray * set_min_recovery_point(pg_control, fullpath, current.stop_lsn); } - /* close and sync page header map */ if (current.hdr_map.fp) { @@ -572,6 +578,10 @@ static void sync_files(parray *database_map, const char *database_path, parray * if (fio_sync(current.hdr_map.path, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync file \"%s\": %s", current.hdr_map.path, strerror(errno)); + + if (current.media_type == MEDIA_TYPE_OSS) { + uploadConfigFile(current.hdr_map.path, current.hdr_map.path); + } } /* close ssh session in main thread */ @@ -829,8 +839,36 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* * Make directories before backup and setup threads at the same time */ - run_backup_threads(external_prefix, database_path, dssdata_path, prev_backup_filelist, - external_dirs, nodeInfo, prev_backup_start_lsn); + if (current.media_type == MEDIA_TYPE_OSS) { + no_sync = true; // no need to sync file to disk + current.oss_status = OSS_STATUS_OSS; + filesinfo = parray_new(); + backup_files_arg arg; + arg.nodeInfo = nodeInfo; + arg.from_root = instance_config.pgdata; + arg.to_root = database_path; + arg.src_dss = instance_config.dss.vgdata; + arg.dst_dss = dssdata_path; + arg.external_prefix = external_prefix; + arg.external_dirs = external_dirs; + arg.files_list = backup_files_list; + arg.prev_filelist = prev_backup_filelist; + arg.prev_start_lsn = prev_backup_start_lsn; + arg.conn_arg.conn = NULL; + arg.conn_arg.cancel_conn = NULL; + arg.hdr_map = &(current.hdr_map); + arg.thread_num = -1; + current.filesinfo = filesinfo; + initBackupSenderContext(¤t_sender_cxt); + current.sender_cxt = current_sender_cxt; + initBackupReaderContexts(¤t_reader_cxt); + current.readerCxt = current_reader_cxt; + performBackup(&arg); + parray_free(filesinfo); + } else { + run_backup_threads(external_prefix, database_path, dssdata_path, prev_backup_filelist, + external_dirs, nodeInfo, prev_backup_start_lsn); + } /* clean previous backup file list */ if (prev_backup_filelist) @@ -841,7 +879,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* Notify end of backup */ pg_stop_backup(¤t, backup_conn, nodeInfo); - sync_files(database_map, database_path, external_dirs, dssdata_path, external_prefix, no_sync); /* be paranoid about instance been from the past */ @@ -1085,7 +1122,7 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, pin_backup(¤t, set_backup_params); } - if (!no_validate) + if (!no_validate && current.media_type != MEDIA_TYPE_OSS) pgBackupValidate(¤t, NULL); /* do something after backup */ diff --git a/src/bin/pg_probackup/catalog.cpp b/src/bin/pg_probackup/catalog.cpp index e275ecfda..f5e107ee0 100644 --- a/src/bin/pg_probackup/catalog.cpp +++ b/src/bin/pg_probackup/catalog.cpp @@ -20,6 +20,8 @@ #include "file.h" #include "configuration.h" #include "common/fe_memutils.h" +#include "oss/include/restore.h" +#include "oss/include/oss_operator.h" static pgBackup* get_closest_backup(timelineInfo *tlinfo); static pgBackup* get_oldest_backup(timelineInfo *tlinfo); @@ -458,6 +460,10 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) goto err_proc; } + if (current.media_type == MEDIA_TYPE_OSS) { + restoreConfigDir(); + } + /* scan the directory and list backups */ backups = parray_new(); for (; (data_ent = fio_readdir(data_dir)) != NULL; errno = 0) @@ -1724,6 +1730,12 @@ do_set_backup(const char *instance_name, time_t backup_id, if (set_backup_params->note) add_note(target_backup, set_backup_params->note); + if (set_backup_params->oss_status >= OSS_STATUS_LOCAL && set_backup_params->oss_status < OSS_STATUS_NUM) { + target_backup->oss_status = set_backup_params->oss_status; + /* Update backup.control */ + write_backup(target_backup, true); + } + parray_walk(backup_list, pgBackupFree); parray_free(backup_list); } @@ -1921,6 +1933,8 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) fio_fprintf(out, "\n#Database Storage type\n"); fio_fprintf(out, "storage-type = %s\n", dev2str(backup->storage_type)); + fio_fprintf(out, "\n#S3 Storage status\n"); + fio_fprintf(out, "s3-status = %s\n", ossStatus2str(backup->oss_status)); } /* @@ -1976,7 +1990,11 @@ write_backup(pgBackup *backup, bool strict) if (rename(path_temp, path) < 0) elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - path_temp, path, strerror(errno)); + path_temp, path, strerror(errno)); + + if (current.media_type == MEDIA_TYPE_OSS) { + uploadConfigFile(path, path); + } } void flush_and_close_file(pgBackup *backup, bool sync, FILE *out, char *control_path_temp) @@ -2154,6 +2172,10 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", control_path_temp, control_path, strerror(errno)); + if (current.media_type == MEDIA_TYPE_OSS) { + uploadConfigFile(control_path, control_path); + } + /* use extra variable to avoid reset of previous data_bytes value in case of error */ backup->data_bytes = backup_size_on_disk; backup->uncompressed_bytes = uncompressed_size_on_disk; @@ -2215,6 +2237,7 @@ readBackupControlFile(const char *path) char *recovery_name = NULL; int parsed_options; char *storage_type = NULL; + char *oss_status = NULL; errno_t rc = 0; ConfigOption options[] = @@ -2250,10 +2273,16 @@ readBackupControlFile(const char *path) {'s', 0, "recovery-name", &recovery_name, SOURCE_FILE_STRICT}, {'u', 0, "content-crc", &backup->content_crc, SOURCE_FILE_STRICT}, {'s', 0, "storage-type", &storage_type, SOURCE_FILE_STRICT}, + {'s', 0, "s3-status", &oss_status, SOURCE_FILE_STRICT}, {0} }; pgBackupInit(backup); + + if (current.media_type == MEDIA_TYPE_OSS) { + restoreConfigFile(path); + } + if (fio_access(path, F_OK, FIO_BACKUP_HOST) != 0) { elog(WARNING, "Control file \"%s\" doesn't exist", path); @@ -2351,6 +2380,14 @@ readBackupControlFile(const char *path) if (storage_type) backup->storage_type = str2dev(storage_type); + if (oss_status) { + backup->oss_status = str2ossStatus(oss_status); + } + + if (current.media_type == MEDIA_TYPE_OSS) { + remove(path); + } + return backup; } @@ -2375,6 +2412,27 @@ parse_backup_mode(const char *value) return BACKUP_MODE_INVALID; } +MediaType +parse_media_type(const char *value) +{ + const char *v = value; + size_t len; + + /* Skip all spaces detected */ + while (IsSpace(*v)) + v++; + len = strlen(v); + + if (len > 0 && pg_strncasecmp("s3", v, len) == 0) + return MEDIA_TYPE_OSS; + else if (len > 0 && pg_strncasecmp("disk", v, len) == 0) + return MEDIA_TYPE_DISK; + + /* media type is invalid, so leave with an error */ + elog(ERROR, "invalid media_type \"%s\"", value); + return MEDIA_TYPE_UNKNOWN; +} + const char * deparse_backup_mode(BackupMode mode) { @@ -2503,6 +2561,9 @@ pgBackupInit(pgBackup *backup) backup->files = NULL; backup->note = NULL; backup->content_crc = 0; + backup->dssdata_bytes = 0; + backup->oss_status = OSS_STATUS_INVALID; + backup->media_type = MEDIA_TYPE_UNKNOWN; } /* free pgBackup object */ @@ -2696,7 +2757,7 @@ scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup) while (target_backup->parent_backup_link) { if (target_backup->status != BACKUP_STATUS_OK && - target_backup->status != BACKUP_STATUS_DONE) + target_backup->status != BACKUP_STATUS_DONE) /* oldest invalid backup in parent chain */ invalid_backup = target_backup; @@ -2707,7 +2768,7 @@ scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup) /* Previous loop will skip FULL backup because his parent_backup_link is NULL */ if (target_backup->backup_mode == BACKUP_MODE_FULL && (target_backup->status != BACKUP_STATUS_OK && - target_backup->status != BACKUP_STATUS_DONE)) + target_backup->status != BACKUP_STATUS_DONE)) { invalid_backup = target_backup; } diff --git a/src/bin/pg_probackup/configure.cpp b/src/bin/pg_probackup/configure.cpp index 9210f3626..2d6d0df47 100644 --- a/src/bin/pg_probackup/configure.cpp +++ b/src/bin/pg_probackup/configure.cpp @@ -15,7 +15,8 @@ #include "configuration.h" #include "json.h" #include "catalog/pg_control.h" - +#include "oss/include/oss_operator.h" +#include "oss/include/restore.h" static void assign_log_level_console(ConfigOption *opt, const char *arg); static void assign_log_level_file(ConfigOption *opt, const char *arg); @@ -42,6 +43,7 @@ static void show_configure_json(ConfigOption *opt); #define OPTION_COMPRESS_GROUP "Compression parameters" #define OPTION_REMOTE_GROUP "Remote access parameters" #define OPTION_DSS_GROUP "DSS connect parameters" +#define OPTION_OSS_GROUP "OSS connect parameters" /* * Short name should be non-printable ASCII character. @@ -243,6 +245,32 @@ ConfigOption instance_options[] = &instance_config.dss.instance_id, SOURCE_CMD, (OptionSource)0, OPTION_DSS_GROUP, 0, option_get_value }, + /* OSS options */ + { + 's', 236, "access-id", + &instance_config.oss.access_id, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, + { + 's', 237, "access-key", + &instance_config.oss.access_key, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, + { + 's', 239, "endpoint", + &instance_config.oss.endpoint, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, + { + 's', 240, "region", + &instance_config.oss.region, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, + { + 's', 243, "access-bucket", + &instance_config.oss.access_bucket, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, { 0 } }; @@ -286,13 +314,22 @@ do_set_config(bool missing_ok) FILE *fp; int i; int nRet = 0; + char* bucket_name = NULL; + bool no_exist = false; join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); nRet = snprintf_s(path_temp, sizeof(path_temp), sizeof(path_temp) - 1, "%s.tmp", path); securec_check_ss_c(nRet, "\0", "\0"); - if (!missing_ok && !fileExists(path, FIO_LOCAL_HOST)) + if (current.media_type == MEDIA_TYPE_OSS) { + Oss::Oss* oss = getOssClient(); + bucket_name = getBucketName(); + no_exist = !oss->ObjectExists(bucket_name, path); + } + + if (!missing_ok && !fileExists(path, FIO_LOCAL_HOST) && no_exist) { elog(ERROR, "Configuration file \"%s\" doesn't exist", path); + } fp = fopen(path_temp, "wt"); if (fp == NULL) @@ -338,6 +375,13 @@ do_set_config(bool missing_ok) elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", path_temp, path, strerror(errno_temp)); } + + if (current.media_type == MEDIA_TYPE_OSS) { + Oss::Oss* oss = getOssClient(); + oss->RemoveObject(bucket_name, path); + oss->PutObject(bucket_name, path, path); + fio_unlink(path, FIO_BACKUP_HOST); + } } void @@ -558,6 +602,32 @@ readInstanceConfigFile(const char *instance_name) 'i', 235, "instance-id", &instance->dss.instance_id, SOURCE_CMD, (OptionSource)0, OPTION_DSS_GROUP, 0, option_get_value }, + /* OSS options */ + { + 's', 236, "access-id", + &instance_config.oss.access_id, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, + { + 's', 237, "access-key", + &instance_config.oss.access_key, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, + { + 's', 239, "endpoint", + &instance_config.oss.endpoint, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, + { + 's', 240, "region", + &instance_config.oss.region, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, + { + 's', 243, "access-bucket", + &instance_config.oss.access_bucket, SOURCE_CMD, (OptionSource)0, + OPTION_OSS_GROUP, 0, option_get_value + }, { 0 } }; @@ -577,6 +647,9 @@ readInstanceConfigFile(const char *instance_name) join_path_components(path, instance->backup_instance_path, BACKUP_CATALOG_CONF_FILE); + if (current.media_type == MEDIA_TYPE_OSS) { + restoreConfigFile(path); + } if (fio_access(path, F_OK, FIO_BACKUP_HOST) != 0) { elog(WARNING, "Control file \"%s\" doesn't exist", path); @@ -608,6 +681,9 @@ readInstanceConfigFile(const char *instance_name) instance->xlog_seg_size = DEFAULT_XLOG_SEG_SIZE; #endif + if (current.media_type == MEDIA_TYPE_OSS) { + remove(path); + } return instance; } diff --git a/src/bin/pg_probackup/data.cpp b/src/bin/pg_probackup/data.cpp index 36b3d488f..d386d3464 100644 --- a/src/bin/pg_probackup/data.cpp +++ b/src/bin/pg_probackup/data.cpp @@ -31,6 +31,8 @@ #include "zstd.h" #include "storage/file/fio_device.h" #include "storage/buf/bufmgr.h" +#include "oss/include/appender.h" +#include "oss/include/restore.h" typedef struct PreReadBuf { @@ -471,8 +473,8 @@ get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno) * or header corruption, * only used for checkdb */ -static int32 -prepare_page(ConnectionArgs *conn_arg, +static +int32 prepare_page(ConnectionArgs *conn_arg, pgFile *file, XLogRecPtr prev_backup_start_lsn, BlockNumber blknum, FILE *in, BackupMode backup_mode, @@ -666,7 +668,8 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, FILE *in, FILE *out, pg_crc32 *crc, int page_state, Page page, CompressAlg calg, int clevel, - const char *from_fullpath, const char *to_fullpath) + const char *from_fullpath, const char *to_fullpath, + FileAppender* appender, char** fileBuffer) { int compressed_size = 0; size_t write_buffer_size = 0; @@ -699,13 +702,29 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, bph->compressed_size = compressed_size; write_buffer_size = compressed_size + sizeof(BackupPageHeader); - /* Update CRC */ - COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); - /* write data page */ - if (fio_fwrite(out, write_buffer, write_buffer_size) != write_buffer_size) - elog(ERROR, "File: \"%s\", cannot write at block %u: %s", - to_fullpath, blknum, strerror(errno)); + if (current.media_type == MEDIA_TYPE_OSS) { + if (fileBuffer != NULL) { + rc = memcpy_s(*fileBuffer, write_buffer_size, write_buffer, write_buffer_size); + securec_check_c(rc, "\0", "\0"); + *fileBuffer += write_buffer_size; + } else { + /* Update CRC */ + COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); + file->crc = *crc; + /* write data page */ + FileAppenderSegHeader content_header; + constructHeader(&content_header, FILE_APPEND_TYPE_FILE_CONTENT, write_buffer_size, 0, file); + writeHeader(&content_header, appender); + writePayload((char*)write_buffer, write_buffer_size, appender); + } + } else { + /* Update CRC */ + COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); + if (fio_fwrite(out, write_buffer, write_buffer_size) != write_buffer_size) + elog(ERROR, "File: \"%s\", cannot write at block %u: %s", + to_fullpath, blknum, strerror(errno)); + } file->write_size += write_buffer_size; file->uncompressed_size += BLCKSZ; @@ -726,7 +745,8 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, - HeaderMap *hdr_map, bool is_merge) + HeaderMap *hdr_map, bool is_merge, + FileAppender* appender, char* fileBuffer) { int rc; bool use_pagemap; @@ -771,6 +791,14 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, file->uncompressed_size = 0; INIT_FILE_CRC32(true, file->crc); + if (fileBuffer == NULL && current.media_type == MEDIA_TYPE_OSS) { + size_t pathLen = strlen(file->rel_path); + FileAppenderSegHeader start_header; + constructHeader(&start_header, FILE_APPEND_TYPE_FILE, pathLen, 0, file); + writeHeader(&start_header, appender); + writePayload((char*)file->rel_path, pathLen, appender); + } + /* * Read each page, verify checksum and write it to backup. * If page map is empty or file is not present in previous backup @@ -794,7 +822,8 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* send pagemap if any */ use_pagemap, /* variables for error reporting */ - &err_blknum, &errmsg, &headers); + &err_blknum, &errmsg, &headers, + appender, fileBuffer ? &fileBuffer : NULL); } else { @@ -802,7 +831,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* send prev backup START_LSN */ InvalidXLogRecPtr, calg, clevel, checksum_version, use_pagemap, - &headers, backup_mode); + &headers, backup_mode, appender, fileBuffer); } /* check for errors */ @@ -866,6 +895,12 @@ cleanup: /* dump page headers */ write_page_headers(headers, file, hdr_map, is_merge); + if (fileBuffer == NULL && current.media_type == MEDIA_TYPE_OSS) { + FileAppenderSegHeader end_header; + constructHeader(&end_header, FILE_APPEND_TYPE_FILE_END, 0, file->read_size, file); + writeHeader(&end_header, appender); + } + pg_free(errmsg); pg_free(file->pagemap.bitmap); pg_free(headers); @@ -881,14 +916,14 @@ void backup_non_data_file(pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time, - bool missing_ok) + bool missing_ok, FileAppender* appender, char* fileBuffer) { fio_location from_location = is_dss_file(from_fullpath) ? FIO_DSS_HOST : FIO_DB_HOST; /* special treatment for global/pg_control */ if (file->external_dir_num == 0 && strcmp(file->name, PG_XLOG_CONTROL_FILE) == 0) { copy_pgcontrol_file(from_fullpath, from_location, - to_fullpath, FIO_BACKUP_HOST, file); + to_fullpath, FIO_BACKUP_HOST, file); return; } @@ -909,7 +944,23 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, } } - backup_non_data_file_internal(from_fullpath, from_location, to_fullpath, file, true); + if (fileBuffer == NULL && current.media_type == MEDIA_TYPE_OSS) { + // write file start header and from_fullpath + size_t pathLen = strlen(file->rel_path); + INIT_FILE_CRC32(true, file->crc); + FileAppenderSegHeader start_header; + constructHeader(&start_header, FILE_APPEND_TYPE_FILE, pathLen, 0, file); + writeHeader(&start_header, appender); + writePayload((char*)file->rel_path, pathLen, appender); + } + backup_non_data_file_internal(from_fullpath, from_location, to_fullpath, file, true, + appender, fileBuffer ? &fileBuffer : NULL); + if (fileBuffer == NULL && current.media_type == MEDIA_TYPE_OSS) { + // write file end header + FileAppenderSegHeader end_header; + constructHeader(&end_header, FILE_APPEND_TYPE_FILE_END, 0, file->read_size, file); + writeHeader(&end_header, appender); + } } /* @@ -925,7 +976,6 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, size_t total_write_len = 0; char *in_buf = (char *)pgut_malloc(STDIO_BUFSIZE); int backup_seq = 0; - /* * FULL -> INCR -> DEST * 2 1 0 @@ -1550,10 +1600,11 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, return tmp_file->write_size; } -bool backup_remote_file(const char *from_fullpath, const char *to_fullpath, pgFile *file, bool missing_ok, FILE *out) +bool backup_remote_file(const char *from_fullpath, const char *to_fullpath, pgFile *file, bool missing_ok, FILE *out, + FileAppender* appender, char** fileBuffer) { char *errmsg = NULL; - int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg); + int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg, appender, fileBuffer); /* handle errors */ if (rc == FILE_MISSING) @@ -1590,7 +1641,8 @@ bool backup_remote_file(const char *from_fullpath, const char *to_fullpath, pgFi */ void backup_non_data_file_internal(const char *from_fullpath, fio_location from_location, - const char *to_fullpath, pgFile *file, bool missing_ok) + const char *to_fullpath, pgFile *file, bool missing_ok, + FileAppender* appender, char** fileBuffer) { FILE *in = NULL; FILE *out = NULL; @@ -1605,43 +1657,45 @@ backup_non_data_file_internal(const char *from_fullpath, fio_location from_locat file->uncompressed_size = 0; /* open backup file for write */ - out = fopen(to_fullpath, PG_BINARY_W); - if (out == NULL) - { - if (file->external_dir_num) + if (current.media_type != MEDIA_TYPE_OSS) { + out = fopen(to_fullpath, PG_BINARY_W); + if (out == NULL) { - char parent[MAXPGPATH]; - errno_t rc = 0; + if (file->external_dir_num) + { + char parent[MAXPGPATH]; + errno_t rc = 0; - rc = strncpy_s(parent, MAXPGPATH, to_fullpath, MAXPGPATH - 1); - securec_check_c(rc, "", ""); - get_parent_directory(parent); + rc = strncpy_s(parent, MAXPGPATH, to_fullpath, MAXPGPATH - 1); + securec_check_c(rc, "", ""); + get_parent_directory(parent); - dir_create_dir(parent, DIR_PERMISSION); - out = fopen(to_fullpath, PG_BINARY_W); - if (out == NULL) + dir_create_dir(parent, DIR_PERMISSION); + out = fopen(to_fullpath, PG_BINARY_W); + if (out == NULL) + elog(ERROR, "Cannot open destination file \"%s\": %s", + to_fullpath, strerror(errno)); + } + else + { elog(ERROR, "Cannot open destination file \"%s\": %s", to_fullpath, strerror(errno)); + } } - else - { - elog(ERROR, "Cannot open destination file \"%s\": %s", - to_fullpath, strerror(errno)); - } - } - /* update file permission */ - if (!is_dss_type(file->type)) - { - if (chmod(to_fullpath, file->mode) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); + /* update file permission */ + if (!is_dss_type(file->type)) + { + if (chmod(to_fullpath, file->mode) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); + } } /* backup remote file */ if (fio_is_remote(FIO_DB_HOST)) { - if (!backup_remote_file(from_fullpath, to_fullpath, file, missing_ok, out)) + if (!backup_remote_file(from_fullpath, to_fullpath, file, missing_ok, out, appender, fileBuffer)) goto cleanup; } /* backup local file */ @@ -1670,7 +1724,9 @@ backup_non_data_file_internal(const char *from_fullpath, fio_location from_locat /* disable stdio buffering for local input/output files to avoid triple buffering */ setvbuf(in, NULL, _IONBF, BUFSIZ); - setvbuf(out, NULL, _IONBF, BUFSIZ); + if (current.media_type != MEDIA_TYPE_OSS) { + setvbuf(out, NULL, _IONBF, BUFSIZ); + } /* allocate 64kB buffer */ buf = (char *)pgut_malloc(CHUNK_SIZE); @@ -1686,12 +1742,28 @@ backup_non_data_file_internal(const char *from_fullpath, fio_location from_locat if (read_len > 0) { - if (fwrite(buf, 1, read_len, out) != (size_t)read_len) - elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath, - strerror(errno)); + if (current.media_type == MEDIA_TYPE_OSS) { + if (fileBuffer != NULL) { + int rc = memcpy_s(*fileBuffer, read_len, buf, read_len); + securec_check_c(rc, "\0", "\0"); + *fileBuffer += read_len; + } else { + /* Update CRC */ + COMP_FILE_CRC32(true, file->crc, buf, read_len); + /* write data page */ + FileAppenderSegHeader content_header; + constructHeader(&content_header, FILE_APPEND_TYPE_FILE_CONTENT, read_len, 0, file); + writeHeader(&content_header, appender); + writePayload((char*)buf, read_len, appender); + } + } else { + if (fwrite(buf, 1, read_len, out) != (size_t)read_len) + elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath, + strerror(errno)); - /* update CRC */ - COMP_FILE_CRC32(true, file->crc, buf, read_len); + /* update CRC */ + COMP_FILE_CRC32(true, file->crc, buf, read_len); + } file->read_size += read_len; } @@ -1712,7 +1784,7 @@ backup_non_data_file_internal(const char *from_fullpath, fio_location from_locat if (in && fclose(in)) elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno)); - if (out && fclose(out)) + if (current.media_type != MEDIA_TYPE_OSS && out && fclose(out)) elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno)); pg_free(buf); @@ -2336,7 +2408,7 @@ int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode) + BackupMode backup_mode, FileAppender* appender, char* fileBuffer) { FILE *in = NULL; FILE *out = NULL; @@ -2427,7 +2499,7 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f else if (rc == PageIsOk) { /* lazily open backup file (useful for s3) */ - if (!out) + if (current.media_type != MEDIA_TYPE_OSS && !out) out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); BackupPageHeader2 *header = pgut_new(BackupPageHeader2); @@ -2445,7 +2517,8 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), rc, curr_page, calg, clevel, - from_fullpath, to_fullpath); + from_fullpath, to_fullpath, + appender, fileBuffer ? &fileBuffer : NULL); cur_pos_out += compressed_size + sizeof(BackupPageHeader); } @@ -2486,7 +2559,7 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f elog(ERROR, "Cannot close the source file \"%s\": %s", to_fullpath, strerror(errno)); /* close local output file */ - if (out && fclose(out)) + if (current.media_type != MEDIA_TYPE_OSS && out && fclose(out)) elog(ERROR, "Cannot close the backup file \"%s\": %s", to_fullpath, strerror(errno)); pg_free(iter); @@ -2521,11 +2594,17 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b if (file->n_headers <= 0) return NULL; + if (current.media_type == MEDIA_TYPE_OSS) { + pthread_lock(&(hdr_map->mutex)); + restoreConfigFile(hdr_map->path); + pthread_mutex_unlock(&(hdr_map->mutex)); + } + in = fopen(hdr_map->path, PG_BINARY_R); if (!in) { - elog(strict ? ERROR : WARNING, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); + elog(strict ? ERROR : WARNING, "Cannot open header file1 \"%s\": %s", hdr_map->path, strerror(errno)); return NULL; } /* disable buffering for header file */ @@ -2603,6 +2682,10 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b headers = NULL; } + if (current.media_type == MEDIA_TYPE_OSS) { + remove(hdr_map->path); + } + return headers; } diff --git a/src/bin/pg_probackup/delete.cpp b/src/bin/pg_probackup/delete.cpp index 0dc96d58a..cd5541c09 100644 --- a/src/bin/pg_probackup/delete.cpp +++ b/src/bin/pg_probackup/delete.cpp @@ -15,6 +15,7 @@ #include #include #include "common/fe_memutils.h" +#include "oss/include/oss_operator.h" static void delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tli, uint32 xlog_seg_size, bool dry_run); @@ -39,6 +40,9 @@ do_delete(time_t backup_id) pgBackup *target_backup = NULL; size_t size_to_delete = 0; char size_to_delete_pretty[20]; + Oss::Oss* oss = getOssClient(); + char* bucket_name = NULL; + /* Get complete list of backups */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); @@ -93,6 +97,12 @@ do_delete(time_t backup_id) if (!dry_run) { + if (current.media_type == MEDIA_TYPE_OSS) { + bucket_name = getBucketName(); + if (!oss->BucketExists(bucket_name)) { + elog(ERROR, "bucket %s not found.", bucket_name); + } + } /* Lock marked for delete backups */ catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false); @@ -105,6 +115,17 @@ do_delete(time_t backup_id) elog(ERROR, "interrupted during delete backup"); delete_backup_files(backup); + if (current.media_type == MEDIA_TYPE_OSS) { + char* prefix_name = getPrefixName(backup); + parray *delete_obj_list = parray_new(); + oss->ListObjectsWithPrefix(bucket_name, prefix_name, delete_obj_list); + for (size_t j = 0; j < parray_num(delete_obj_list); j++) { + char* object = (char*)parray_get(delete_obj_list, j); + oss->RemoveObject(bucket_name, object); + elog(INFO, "Object '%s' successfully deleted from S3", object); + } + parray_free(delete_obj_list); + } } } @@ -165,6 +186,10 @@ void do_retention(void) backup_deleted = false; backup_merged = false; + if (current.media_type == MEDIA_TYPE_OSS) { + elog(ERROR, "Not supported when specifying OSS options"); + } + /* Get a complete list of backups. */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); @@ -1018,7 +1043,6 @@ do_delete_instance(void) size_t i = 0; char instance_config_path[MAXPGPATH]; - /* Delete all backups. */ backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); @@ -1039,7 +1063,7 @@ do_delete_instance(void) /* Delete backup instance config file */ join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - if (remove(instance_config_path)) + if (current.media_type != MEDIA_TYPE_OSS && remove(instance_config_path)) { elog(ERROR, "Can't remove \"%s\": %s", instance_config_path, strerror(errno)); @@ -1054,6 +1078,25 @@ do_delete_instance(void) elog(ERROR, "Can't remove \"%s\": %s", arclog_path, strerror(errno)); + if (current.media_type == MEDIA_TYPE_OSS) { + Oss::Oss* oss = getOssClient(); + char* bucket_name = getBucketName(); + if (!oss->BucketExists(bucket_name)) { + elog(ERROR, "bucket %s not found.", bucket_name); + } else { + // delete backups first + parray *delete_list = parray_new(); + char* prefix_name = backup_instance_path + 1; + oss->ListObjectsWithPrefix(bucket_name, prefix_name, delete_list); + for (i = 0; i < parray_num(delete_list); i++) { + char* object = (char*)parray_get(delete_list, i); + oss->RemoveObject(bucket_name, object); + elog(INFO, "Object '%s' successfully deleted from S3", object); + } + parray_free(delete_list); + } + } + elog(INFO, "Instance '%s' successfully deleted", instance_name); return 0; } @@ -1069,6 +1112,8 @@ do_delete_status(InstanceConfig *instance_config, const char *status) size_t size_to_delete = 0; char size_to_delete_pretty[20]; pgBackup *backup; + Oss::Oss* oss = getOssClient(); + char* bucket_name = NULL; BackupStatus status_for_delete = str2status(status); delete_list = parray_new(); @@ -1126,6 +1171,13 @@ do_delete_status(InstanceConfig *instance_config, const char *status) catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false); } + if (current.media_type == MEDIA_TYPE_OSS) { + bucket_name = getBucketName(); + if (!oss->BucketExists(bucket_name)) { + elog(ERROR, "bucket %s not found.", bucket_name); + } + } + /* delete and calculate free size from delete_list */ for (i = 0; i < parray_num(delete_list); i++) { @@ -1140,6 +1192,17 @@ do_delete_status(InstanceConfig *instance_config, const char *status) if (!dry_run) { delete_backup_files(backup); + if (current.media_type == MEDIA_TYPE_OSS) { + char* prefix_name = getPrefixName(backup); + parray *delete_list = parray_new(); + oss->ListObjectsWithPrefix(bucket_name, prefix_name, delete_list); + for (size_t j = 0; j < parray_num(delete_list); j++) { + char* object = (char*)parray_get(delete_list, j); + oss->RemoveObject(bucket_name, object); + elog(INFO, "Object '%s' successfully deleted from S3", object); + } + parray_free(delete_list); + } } n_deleted++; diff --git a/src/bin/pg_probackup/dir.cpp b/src/bin/pg_probackup/dir.cpp index b2f21264d..e07894a00 100644 --- a/src/bin/pg_probackup/dir.cpp +++ b/src/bin/pg_probackup/dir.cpp @@ -26,6 +26,7 @@ #include "common/fe_memutils.h" #include "PageCompression.h" #include "storage/file/fio_device.h" +#include "oss/include/restore.h" /* * The contents of these directories are removed or recreated during server @@ -1760,6 +1761,10 @@ dir_read_file_list(const char *root, const char *external_prefix, char stdio_buf[STDIO_BUFSIZE]; pg_crc32 content_crc = 0; + if (current.media_type == MEDIA_TYPE_OSS) { + restoreConfigFile(file_txt); + } + fp = fio_open_stream(file_txt, location); if (fp == NULL) elog(ERROR, "cannot open \"%s\": %s", file_txt, strerror(errno)); @@ -1879,6 +1884,10 @@ dir_read_file_list(const char *root, const char *external_prefix, return NULL; } + if (current.media_type == MEDIA_TYPE_OSS) { + remove(file_txt); + } + return files; } @@ -2084,6 +2093,9 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ parray_append(backup_files_list, file); } + if (current.media_type == MEDIA_TYPE_OSS) { + uploadConfigFile(database_map_path, database_map_path); + } } /* diff --git a/src/bin/pg_probackup/file.cpp b/src/bin/pg_probackup/file.cpp index 29c218433..f5ccf7e39 100644 --- a/src/bin/pg_probackup/file.cpp +++ b/src/bin/pg_probackup/file.cpp @@ -17,6 +17,7 @@ #include "storage/checksum.h" #include "storage/file/fio_device.h" #include "common/fe_memutils.h" +#include "oss/include/appender.h" #define PRINTF_BUF_SIZE 1024 @@ -1206,7 +1207,7 @@ static void fio_load_file(int out, char const* path) int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, bool use_pagemap, BlockNumber* err_blknum, char **errormsg, - BackupPageHeader2 **headers) + BackupPageHeader2 **headers, FileAppender* appender, char** fileBuffer) { FILE *out = NULL; char *out_buf = NULL; @@ -1330,15 +1331,29 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f COMP_FILE_CRC32(true, file->crc, buf, hdr.size); /* lazily open backup file */ - if (!out) + if (current.media_type != MEDIA_TYPE_OSS && !out) out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); - if (fio_fwrite(out, buf, hdr.size) != hdr.size) - { - fio_fclose(out); - *err_blknum = blknum; - return WRITE_FAILED; + if (current.media_type == MEDIA_TYPE_OSS) { + if (fileBuffer != NULL) { + int rc = memcpy_s(*fileBuffer, hdr.size, buf, hdr.size); + securec_check_c(rc, "\0", "\0"); + *fileBuffer += hdr.size; + } else { + /* write data page */ + FileAppenderSegHeader content_header; + constructHeader(&content_header, FILE_APPEND_TYPE_FILE_CONTENT, hdr.size, 0, file); + writeHeader(&content_header, appender); + writePayload((char*)buf, hdr.size, appender); + } + } else { + if (fio_fwrite(out, buf, hdr.size) != hdr.size) { + fio_fclose(out); + *err_blknum = blknum; + return WRITE_FAILED; + } } + file->write_size += hdr.size; file->uncompressed_size += BLCKSZ; } @@ -1346,7 +1361,7 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); } - if (out) + if (current.media_type != MEDIA_TYPE_OSS && out) fclose(out); pg_free(out_buf); @@ -1681,7 +1696,8 @@ static char *ProcessErrorIn(int out, fio_header &hdr, const char *fromFullpath) * If pgFile is not NULL then we must calculate crc and read_size for it. */ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, - pgFile *file, char **errormsg) + pgFile *file, char **errormsg, + FileAppender* appender, char** fileBuffer) { fio_header hdr; int exit_code = SEND_OK; @@ -1727,17 +1743,32 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, } IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - /* We have received a chunk of data data, lets write it out */ - if (fwrite(buf, 1, hdr.size, out) != hdr.size) - { - exit_code = WRITE_FAILED; - break; - } - - if (file) - { + if (current.media_type == MEDIA_TYPE_OSS) { + if (fileBuffer != NULL) { + int rc = memcpy_s(*fileBuffer, hdr.size, buf, hdr.size); + securec_check_c(rc, "\0", "\0"); + *fileBuffer += hdr.size; + } else { + /* Update CRC */ + COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + /* write data page */ + FileAppenderSegHeader content_header; + constructHeader(&content_header, FILE_APPEND_TYPE_FILE_CONTENT, hdr.size, 0, file); + writeHeader(&content_header, appender); + writePayload((char*)buf, hdr.size, appender); + } file->read_size += hdr.size; - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + } else { + /* We have received a chunk of data data, lets write it out */ + if (fwrite(buf, 1, hdr.size, out) != hdr.size) { + exit_code = WRITE_FAILED; + break; + } + + if (file) { + file->read_size += hdr.size; + COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + } } } else diff --git a/src/bin/pg_probackup/init.cpp b/src/bin/pg_probackup/init.cpp index 70f43a085..ba76730b6 100644 --- a/src/bin/pg_probackup/init.cpp +++ b/src/bin/pg_probackup/init.cpp @@ -9,6 +9,8 @@ */ #include "pg_probackup.h" +#include "oss/include/oss_operator.h" +#include "oss/include/restore.h" #include #include @@ -77,6 +79,14 @@ do_add_instance(InstanceConfig *instance) if (access(arclog_path_dir, F_OK) != 0) elog(ERROR, "Directory does not exist: '%s'", arclog_path_dir); + if (current.media_type == MEDIA_TYPE_OSS) { + Oss::Oss* oss = getOssClient(); + char* bucket_name = getBucketName(); + if (!oss->BucketExists(bucket_name)) { + elog(ERROR, "Bucket '%s' does not exist on OSS, please create it first.", bucket_name); + } + } + if (stat(instance->backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) elog(ERROR, "Instance '%s' backup directory already exists: '%s'", instance->name, instance->backup_instance_path); @@ -125,6 +135,17 @@ do_add_instance(InstanceConfig *instance) config_set_opt(instance_options, &instance_config.remote.ssh_config, SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.oss.access_id, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.oss.access_key, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.oss.endpoint, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.oss.region, + SOURCE_DEFAULT); + config_set_opt(instance_options, &instance_config.oss.access_bucket, + SOURCE_DEFAULT); + /* pgdata and vgname were set through command line */ do_set_config(true); diff --git a/src/bin/pg_probackup/merge.cpp b/src/bin/pg_probackup/merge.cpp index 17088e66d..b29c13067 100644 --- a/src/bin/pg_probackup/merge.cpp +++ b/src/bin/pg_probackup/merge.cpp @@ -121,6 +121,11 @@ do_merge(time_t backup_id) pgBackup *dest_backup = NULL; pgBackup *full_backup = NULL; + + if (current.media_type == MEDIA_TYPE_OSS) { + elog(ERROR, "Not supported when specifying OSS options"); + } + if (backup_id == INVALID_BACKUP_ID) elog(ERROR, "required parameter is not specified: --backup-id"); diff --git a/src/bin/pg_probackup/oss/appender.cpp b/src/bin/pg_probackup/oss/appender.cpp new file mode 100644 index 000000000..67d1c4bee --- /dev/null +++ b/src/bin/pg_probackup/oss/appender.cpp @@ -0,0 +1,227 @@ +/*------------------------------------------------------------------------- + * + * appender.cpp: Appender used by Backup/Recovery manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "include/appender.h" +#include "include/buffer.h" +#include "workload/gscgroup.h" +#include "include/restore.h" +#include "common/fe_memutils.h" + +void initFileAppender(FileAppender* appender, FILE_APPEND_SEG_TYPE type, uint32 minFileNo, uint32 maxFileNo) +{ + appender->fileNo = maxFileNo; + appender->minFileNo = minFileNo; + appender->maxFileNo = maxFileNo; + appender->type = type; + appender->currFileSize = 0; + appender->filePtr = NULL; + appender->currFileName = getAppendFileName(appender->baseFileName, appender->fileNo); + appender->filePtr = openWriteBufferFile(appender->currFileName, "wb"); + FileAppenderSegHeader header; + header.type = type; + header.size = 0; + header.permission = 0; + header.filesize = 0; + header.crc = 0; + header.external_dir_num = 0; + header.file_type = DEV_TYPE_INVALID; + writeHeader(&header,appender); +} + +void initSegDescriptor(FileAppenderSegDescriptor** segDesc) +{ + FileAppenderSegDescriptor* desc = (FileAppenderSegDescriptor*)palloc(sizeof(FileAppenderSegDescriptor)); + if (desc == NULL) { + elog(ERROR, "Failed to allocate memory for seg descriptor."); + return; + } + desc->header.type = FILE_APPEND_TYPE_UNKNOWN; + desc->header_offset = -1; + desc->payload_offset = -1; + desc->crc = 0; + desc->payload = NULL; + desc->outputFile = NULL; + desc->inputFile = NULL; + *segDesc = desc; +} + +void getSegDescriptor(FileAppenderSegDescriptor* desc, char** buffOffset, size_t* remainBuffLen, BufferCxt* cxt) +{ + errno_t rc; + *remainBuffLen = *remainBuffLen - sizeof(FileAppenderSegHeader); + /* The header may span across two buffs. + * So, we cannot directly copy the header from the buffer. + */ + if (likely(desc->header_offset == -1)) { + rc = memcpy_s(&desc->header, sizeof(FileAppenderSegHeader), *buffOffset, sizeof(FileAppenderSegHeader)); + securec_check(rc, "\0", "\0"); + desc->payload = *buffOffset + sizeof(FileAppenderSegHeader); + *buffOffset = desc->payload; + } else { + rc = memcpy_s(&desc->header, desc->header_offset, *buffOffset, desc->header_offset); + securec_check(rc, "\0", "\0"); + rc = memcpy_s(&desc->header + desc->header_offset, (sizeof(FileAppenderSegHeader) - desc->header_offset), + cxt->bufData, (sizeof(FileAppenderSegHeader) - desc->header_offset)); + securec_check(rc, "\0", "\0"); + desc->payload = cxt->bufData + (sizeof(FileAppenderSegHeader) - desc->header_offset); + *buffOffset = desc->payload; + } +} + +void parseSegDescriptor(FileAppenderSegDescriptor* desc, char** buffOffset, size_t* remainBuffLen, char* tempBuffer, + BufferCxt* cxt, pgBackup* dest_backup, bool isValidate, validate_files_arg* arg) { + error_t rc = 0; + parray* files = dest_backup->files; + *remainBuffLen = *remainBuffLen - desc->header.size; + + if (desc->payload_offset == -1) { + rc = memcpy_s(tempBuffer, desc->header.size, desc->payload, desc->header.size); + securec_check(rc, "\0", "\0"); + *buffOffset = desc->payload + desc->header.size; + } else { + rc = memcpy_s(tempBuffer, desc->payload_offset, desc->payload, desc->payload_offset); + securec_check(rc, "\0", "\0"); + rc = memcpy_s(tempBuffer + desc->payload_offset, (desc->header.size - desc->payload_offset), + cxt->bufData, (desc->header.size - desc->payload_offset)); + securec_check(rc, "\0", "\0"); + *buffOffset = cxt->bufData + (desc->header.size - desc->payload_offset); + desc->payload_offset = -1; + } + + if (desc->header.type == FILE_APPEND_TYPE_FILES_END || desc->header.type == FILE_APPEND_TYPE_FILES) { + return; + } else if (desc->header.type == FILE_APPEND_TYPE_DIR) { + restoreDir(tempBuffer, desc, dest_backup, files, isValidate); + } else if (desc->header.type == FILE_APPEND_TYPE_FILE) { + openRestoreFile(tempBuffer, desc, dest_backup, files, isValidate, arg); + } else if (desc->header.type == FILE_APPEND_TYPE_FILE_CONTENT) { + writeOrValidateRestoreFile(tempBuffer, desc, isValidate, arg); + } else if (desc->header.type == FILE_APPEND_TYPE_FILE_END) { + closeRestoreFile(desc); + } else { + if (isValidate) { + arg->corrupted = true; + } else { + elog(ERROR, "Unknown file type: %d, when restore file: %s", desc->header.type, desc->inputFile->rel_path); + } + } +} + +void destorySegDescriptor(FileAppenderSegDescriptor** descriptor) +{ + FileAppenderSegDescriptor* desc = *descriptor; + pfree_ext(desc); +} + +void closeFileAppender(FileAppender* appender) +{ + if (!appender) { + return; + } + FileAppenderSegHeader header; + header.type = FILE_APPEND_TYPE_FILES_END; + header.size = 0; + header.permission = 0; + header.filesize = 0; + header.crc = 0; + header.external_dir_num = 0; + header.file_type = DEV_TYPE_INVALID; + ((BufferCxt *)appender->filePtr)->fileEnd = true; + writeHeader(&header, appender); +} + +void destoryFileAppender(FileAppender** retAppender) +{ + FileAppender* appender = *retAppender; + if (appender != NULL) { + if (appender->baseFileName != NULL) { + pfree_ext(appender->baseFileName); + } + if (appender->currFileName != NULL) { + pfree_ext(appender->currFileName); + } + pfree_ext(appender); + appender = NULL; + } +} + +char* getAppendFileName(const char* baseFileName, uint32 fileNo) +{ + char* fileName = NULL; + if (baseFileName == NULL) { + return NULL; + } + size_t nameLen = strlen(baseFileName) + APPEND_FILENAME_END_SIZE + APPEND_FILENAME_END_DIGIT; + fileName = (char*)palloc(nameLen); + if (fileName == NULL) { + elog(ERROR, "Failed to allocate memory for file name"); + } + errno_t rc = snprintf_s(fileName, nameLen, (nameLen - 1), "%s/file-%u.pbk", baseFileName, fileNo); + securec_check_ss_c(rc, "\0", "\0"); + return fileName; +} + +void constructHeader(FileAppenderSegHeader* header, FILE_APPEND_SEG_TYPE type, + uint32 size, off_t filesize, pgFile* file) +{ + header->type = type; + header->size = size; + header->permission = file->mode; + header->filesize = filesize; + header->crc = file->crc; + header->external_dir_num = file->external_dir_num; + header->file_type = file->type; +} + +void writeHeader(FileAppenderSegHeader* header, FileAppender* appender) +{ + size_t writeLen = 0; + if (!appender || ((appender->currFileSize + APPEND_FILE_HEADER_SIZE) > APPEND_FILE_MAX_SIZE)) { + elog(ERROR, "Write header failed."); + } + if (header->type != FILE_APPEND_TYPE_FILES_END && (appender->currFileSize + APPEND_FILE_HEADER_SIZE + header->size) > + (APPEND_FILE_MAX_SIZE - APPEND_FILE_HEADER_SIZE)) { + uint32 minFileNo = appender->minFileNo; + uint32 maxFileNo = appender->maxFileNo; + closeFileAppender(appender); + initFileAppender(appender, FILE_APPEND_TYPE_FILES, minFileNo, maxFileNo + 1); + } + /* filePtr is a buffer context*/ + writeLen = writeToCompFile((char*)header, sizeof(FileAppenderSegHeader), appender->filePtr); + if (writeLen != sizeof(FileAppenderSegHeader)) { + elog(ERROR, "Write header failed, write length: %lu.", writeLen); + } + appender->currFileSize += writeLen; +} + + +size_t writeToCompFile(const char* data, size_t len, void* file) +{ + if (writeToBuffer(data, len, file) != len) { + return 0; + } + return len; +} + +void writePayload(const char* data, size_t len, FileAppender* appender) +{ + if (appender->currFileSize + len > (APPEND_FILE_MAX_SIZE - APPEND_FILE_HEADER_SIZE)) { + uint32 minFileNo = appender->minFileNo; + uint32 maxFileNo = appender->maxFileNo; + closeFileAppender(appender); + initFileAppender(appender, FILE_APPEND_TYPE_FILES, minFileNo, maxFileNo + 1); + } + size_t writeLen = writeToCompFile(data, len, appender->filePtr); + if (writeLen != len) { + elog(ERROR, "Write payload data failed, write length: %lu.", writeLen); + } + appender->currFileSize += writeLen; +} \ No newline at end of file diff --git a/src/bin/pg_probackup/oss/backup.cpp b/src/bin/pg_probackup/oss/backup.cpp new file mode 100644 index 000000000..5ce177198 --- /dev/null +++ b/src/bin/pg_probackup/oss/backup.cpp @@ -0,0 +1,317 @@ +/*------------------------------------------------------------------------- + * + * backup.cpp: Backup api used by Backup/Recovery manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "include/backup.h" +#include "storage/file/fio_device.h" +#include "common/fe_memutils.h" + +/* Progress Counter */ +static int g_doneFiles = 0; +static int g_totalFiles = 0; +static volatile bool g_progressFlag = false; +static pthread_cond_t g_cond = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t g_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void handleZeroSizeFile(FileAppender *appender, pgFile* file); +static void *ProgressReportProbackup(void *arg); + +void performBackup(backup_files_arg* arg) +{ + backupReaderThreadArgs* thread_args = (backupReaderThreadArgs*)palloc(sizeof(backupReaderThreadArgs) * current.readerThreadCount); + initPerformBackup(arg, thread_args); + backupDataFiles(arg); + pfree(thread_args); +} + +void initPerformBackup(backup_files_arg* arg, backupReaderThreadArgs* thread_args) +{ + startBackupSender(); + startBackupReaders(arg, thread_args); +} + +void backupDataFiles(backup_files_arg* arg) +{ + FileAppender* appender = NULL; + /* bucket name: instance_name; object name: /instance_name/backupset_name/file-%d.pbk + * instance_name complies with the naming rules of object storage service + */ + appender = (FileAppender*)palloc(sizeof(FileAppender)); + if (appender == NULL) { + elog(ERROR, "Failed to allocate memory for appender"); + return; + } + appender->baseFileName = pg_strdup(current.root_dir); + initFileAppender(appender, FILE_APPEND_TYPE_FILES, 0, 0); + /* backup starts */ + backupDirectories(appender, arg); + backupFiles(appender, arg); + /* clean up FileAppender and reader threads*/ + flushReaderContexts(arg); + closeFileAppender(appender); + destoryFileAppender(&appender); + stopBackupReaders(); + destoryBackupReaderContexts(); + /* no more new data to be produced, backup ends */ + setSenderState(current.sender_cxt, SENDER_THREAD_STATE_FINISH); + waitForSenderThread(); + stopBackupSender(); + destoryBackupSenderContext(); +} + +void backupDirectories(FileAppender* appender, backup_files_arg* arg) +{ + int totalFiles = (int)parray_num(arg->files_list); + g_totalFiles = totalFiles; + for (int i = 0; i < totalFiles; i++) { + pgFile* dir = (pgFile*) parray_get(arg->files_list, i); + /* if the entry was a directory, create it in the backup file */ + if (S_ISDIR(dir->mode)) { + char dirpath[MAXPGPATH]; + int nRet = snprintf_s(dirpath, MAXPGPATH, MAXPGPATH - 1, "%s", dir->rel_path); + securec_check_ss_c(nRet, "\0", "\0"); + // write into backup file + appendDir(appender, dirpath, DIR_PERMISSION, dir->external_dir_num, dir->type); + g_doneFiles++; + } + } +} + +void appendDir(FileAppender* appender, const char* dirPath, uint32 permission, + int external_dir_num, device_type_t type) +{ + FileAppenderSegHeader header; + size_t pathLen = strlen(dirPath); + header.type = FILE_APPEND_TYPE_DIR; + header.size = pathLen; + header.permission = permission; + header.filesize = 0; + header.crc = 0; + header.external_dir_num = external_dir_num; + header.file_type = type; + writeHeader(&header, appender); + writePayload((char*)dirPath, pathLen, appender); +} + +void backupFiles(FileAppender* appender, backup_files_arg* arg) +{ + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + static time_t prev_time; + time_t start_time, end_time; + char pretty_time[20]; + int n_backup_files_list = parray_num(arg->files_list); + prev_time = current.start_time; + + /* Sort by size for load balancing */ + parray_qsort(arg->files_list, pgFileCompareSize); + /* Sort the array for binary search */ + if (arg->prev_filelist) + parray_qsort(arg->prev_filelist, pgFileCompareRelPathWithExternal); + /* write initial backup_content.control file and update backup.control */ + write_backup_filelist(¤t, arg->files_list, + instance_config.pgdata, arg->external_dirs, true); + write_backup(¤t, true); + /* Init backup page header map */ + init_header_map(¤t); + /* Run threads */ + thread_interrupted = false; + elog(INFO, "Start backing up files"); + time(&start_time); + /* Create the thread for progress report */ + pthread_t progressThread; + pthread_create(&progressThread, nullptr, ProgressReportProbackup, nullptr); + + /* backup a file */ + for (int i = 0; i < n_backup_files_list; i++) { + pgFile *prev_file = NULL; + pgFile *file = (pgFile *) parray_get(arg->files_list, i); + /* We have already copied all directories */ + if (S_ISDIR(file->mode)) { + continue; + } + /* check for interrupt */ + if (interrupted || thread_interrupted) { + elog(ERROR, "interrupted during backup"); + } + if (progress) + elog_file(INFO, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_backup_files_list, file->rel_path); + /* update done_files */ + pg_atomic_add_fetch_u32((volatile uint32*) &g_doneFiles, 1); + + /* Handle zero sized files */ + if (file->size == 0) { + file->write_size = 0; + handleZeroSizeFile(appender, file); + continue; + } + + /* construct filepath */ + if (file->external_dir_num != 0) { + char external_dst[MAXPGPATH]; + char *external_path = (char *)parray_get(arg->external_dirs, + file->external_dir_num - 1); + + makeExternalDirPathByNum(external_dst, + arg->external_prefix, + file->external_dir_num); + + join_path_components(to_fullpath, external_dst, file->rel_path); + join_path_components(from_fullpath, external_path, file->rel_path); + } else if (is_dss_type(file->type)) { + join_path_components(from_fullpath, arg->src_dss, file->rel_path); + join_path_components(to_fullpath, arg->dst_dss, file->rel_path); + } else { + join_path_components(from_fullpath, arg->from_root, file->rel_path); + join_path_components(to_fullpath, arg->to_root, file->rel_path); + } + + /* Encountered some strange beast */ + if (!S_ISREG(file->mode)) { + elog(WARNING, "Unexpected type %d of file \"%s\", skipping", + file->mode, from_fullpath); + } + + /* Check that file exist in previous backup */ + if (current.backup_mode != BACKUP_MODE_FULL) { + pgFile **prev_file_tmp = NULL; + prev_file_tmp = (pgFile **) parray_bsearch(arg->prev_filelist, + file, pgFileCompareRelPathWithExternal); + if (prev_file_tmp) { + /* File exists in previous backup */ + file->exists_in_prev = true; + prev_file = *prev_file_tmp; + } + } + + /* special treatment for global/pg_control */ + if (file->external_dir_num == 0 && strcmp(file->name, PG_XLOG_CONTROL_FILE) == 0) { + char* filename = last_dir_separator(to_fullpath); + char* dirpath = strndup(to_fullpath, filename - to_fullpath + 1); + fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); + pg_free(dirpath); + } + + /* If the file size is less than 8MB, + * a load-balancing reason prevents the direct writing of the appender file + */ + if (file->size <= FILE_BUFFER_SIZE && current.readerThreadCount > 0) { + int thread_slot = getFreeReaderThread(); + while (thread_slot == -1) { + flushReaderContexts(arg); + thread_slot = getFreeReaderThread(); + } + ReaderCxt* reader_cxt = ¤t.readerCxt[thread_slot]; + int current_fileidx = reader_cxt->fileCount; + reader_cxt->file[current_fileidx] = file; + reader_cxt->prefile[current_fileidx] = prev_file; + reader_cxt->fromPath[current_fileidx] = pgut_strdup(from_fullpath); + reader_cxt->toPath[current_fileidx] = pgut_strdup(to_fullpath); + reader_cxt->appender = appender; + reader_cxt->segType[current_fileidx] = FILE_APPEND_TYPE_FILE; + reader_cxt->fileRemoved[current_fileidx] = false; + reader_cxt->fileCount++; + if (reader_cxt->fileCount == READER_THREAD_FILE_COUNT) { + setReaderState(reader_cxt, READER_THREAD_STATE_START); + } + } else { + if (file->is_datafile && !file->is_cfs) { + backup_data_file(&(arg->conn_arg), file, from_fullpath, to_fullpath, + arg->prev_start_lsn, + current.backup_mode, + instance_config.compress_alg, + instance_config.compress_level, + arg->nodeInfo->checksum_version, + arg->hdr_map, false, appender, NULL); + } else { + backup_non_data_file(file, prev_file, from_fullpath, to_fullpath, + current.backup_mode, current.parent_backup, true, appender, NULL); + } + } + + if (file->write_size == FILE_NOT_FOUND) { + continue; + } + + if (file->write_size == BYTES_INVALID) { + elog(VERBOSE, "Skipping the unchanged file: \"%s\"", from_fullpath); + continue; + } + } + g_progressFlag = true; + pthread_mutex_lock(&g_mutex); + pthread_cond_signal(&g_cond); + pthread_mutex_unlock(&g_mutex); + pthread_join(progressThread, nullptr); + + /* ssh connection to longer needed */ + fio_disconnect(); + /* Close connection */ + if (arg->conn_arg.conn) { + pgut_disconnect(arg->conn_arg.conn); + } + /* Data files transferring is successful */ + arg->ret = 0; + elog(INFO, "Finish backuping file"); + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + elog(INFO, "Backup files are backuped to oss, time elapsed: %s", pretty_time); +} + + +/* static function*/ + +static void handleZeroSizeFile(FileAppender *appender, pgFile* file) +{ + size_t pathLen = strlen(file->rel_path); + FileAppenderSegHeader start_header; + constructHeader(&start_header, FILE_APPEND_TYPE_FILE, pathLen, 0, file); + writeHeader(&start_header, appender); + writePayload((char*)file->rel_path, pathLen, appender); + FileAppenderSegHeader end_header; + constructHeader(&end_header, FILE_APPEND_TYPE_FILE_END, 0, 0, file); + writeHeader(&end_header, appender); +} + +/* copy from backup.c for static variables*/ +static void *ProgressReportProbackup(void *arg) +{ + if (g_totalFiles == 0) { + return nullptr; + } + char progressBar[53]; + int percent; + do { + /* progress report */ + percent = (int)(g_doneFiles * 100 / g_totalFiles); + GenerateProgressBar(percent, progressBar); + fprintf(stdout, "Progress: %s %d%% (%d/%d, done_files/total_files). backup file \r", + progressBar, percent, g_doneFiles, g_totalFiles); + pthread_mutex_lock(&g_mutex); + timespec timeout; + timeval now; + gettimeofday(&now, nullptr); + timeout.tv_sec = now.tv_sec + 1; + timeout.tv_nsec = 0; + int ret = pthread_cond_timedwait(&g_cond, &g_mutex, &timeout); + pthread_mutex_unlock(&g_mutex); + if (ret == ETIMEDOUT) { + continue; + } else { + break; + } + } while ((g_doneFiles < g_totalFiles) && !g_progressFlag); + percent = 100; + GenerateProgressBar(percent, progressBar); + fprintf(stdout, "Progress: %s %d%% (%d/%d, done_files/total_files). backup file \n", + progressBar, percent, g_doneFiles, g_totalFiles); + return nullptr; +} \ No newline at end of file diff --git a/src/bin/pg_probackup/oss/buffer.cpp b/src/bin/pg_probackup/oss/buffer.cpp new file mode 100644 index 000000000..f2f0a0f92 --- /dev/null +++ b/src/bin/pg_probackup/oss/buffer.cpp @@ -0,0 +1,176 @@ +/*------------------------------------------------------------------------- + * + * buffer.cpp: Buffer used by Backup/Recovery manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "include/buffer.h" +#include "include/thread.h" + +void initBufferCxt(BufferCxt* cxt, size_t bufferSize) +{ + size_t bufnum = (bufferSize + BUFSIZE -1) / BUFSIZE; + cxt->bufNum = bufnum; + cxt->bufHeader = (BufferDesc*)palloc(sizeof(BufferDesc) * bufnum); + cxt->bufData = (char*)palloc(BUFSIZE * bufnum); + cxt->fileEnd = false; + cxt->fileNum = -1; + cxt->producerIdx = 0; + cxt->producerIdxCache = 0; + cxt->consumerIdx = 0; + cxt->consumerIdxCache = 0; + cxt->earlyExit = false; + if (cxt->bufHeader == NULL || cxt->bufData == NULL) { + pfree_ext(cxt->bufHeader); + pfree_ext(cxt->bufData); + elog(ERROR, "buffer context allocate failed: out of memory"); + } + for(size_t i = 0; i < bufnum; i++) { + cxt->bufHeader[i].bufId = i; + cxt->bufHeader[i].fileId = -1; + cxt->bufHeader[i].usedLen = 0; + cxt->bufHeader[i].flags = 0; + pthread_spin_init(&cxt->bufHeader[i].lock, PTHREAD_PROCESS_PRIVATE); + } +} + +void destroyBufferCxt(BufferCxt* cxt) +{ + for(size_t i = 0; i < cxt->bufNum; i++) { + pthread_spin_destroy(&cxt->bufHeader[i].lock); + } + pfree_ext(cxt->bufHeader); + pfree_ext(cxt->bufData); +} + +BufferDesc* getNextFreeWriteBuffer(BufferCxt* cxt) +{ + BufferDesc* buff = NULL; + const size_t producerIdx = cxt->producerIdx.load(std::memory_order_relaxed); + const size_t nextIdx = (producerIdx + 1) % buffNum(cxt); + /* check whether the buffer queue is full */ + if (nextIdx == cxt->consumerIdxCache) { + cxt->consumerIdxCache = cxt->consumerIdx.load(std::memory_order_acquire); + if (nextIdx == cxt->consumerIdxCache) { + return NULL; + } + } + buff = &(cxt->bufHeader[producerIdx]); + if (testBufferFlag(buff, BUFF_FLAG_FILE_FINISHED | BUFF_FLAG_FILE_CLOSED)) { + cxt->producerIdx.store(nextIdx, std::memory_order_release); + return NULL; + } + return buff; +} + +BufferDesc* tryGetNextFreeWriteBuffer(BufferCxt* cxt) +{ + BufferDesc* buff = NULL; + while (!(buff = getNextFreeWriteBuffer(cxt))) { + pg_usleep(WAIT_FOR_BUFF_SLEEP_TIME); + continue; + } + if (buffFreeLen(buff) != 0) { + return buff; + } + return tryGetNextFreeWriteBuffer(cxt); +} + +BufferDesc* getNextFreeReadBuffer(BufferCxt* cxt) +{ + BufferDesc* buff = NULL; + const size_t consumerIdx = cxt->consumerIdx.load(std::memory_order_relaxed); + /* check whether the buffer queue is empty */ + if (consumerIdx == cxt->producerIdxCache) { + cxt->producerIdxCache = cxt->producerIdx.load(std::memory_order_acquire); + buff = &(cxt->bufHeader[consumerIdx]); + if (!testBufferFlag(buff, BUFF_FLAG_FILE_FINISHED | BUFF_FLAG_FILE_CLOSED) && + consumerIdx == cxt->producerIdxCache) { + return NULL; + } + } + buff = &(cxt->bufHeader[consumerIdx]); + // buffer read finished + if (!testBufferFlag(buff, BUFF_FLAG_FILE_FINISHED | BUFF_FLAG_FILE_CLOSED)) { + return NULL; + } + const size_t next = (consumerIdx + 1) % buffNum(cxt); + cxt->consumerIdx.store(next, std::memory_order_release); + return buff; +} + +BufferDesc* tryGetNextFreeReadBuffer(BufferCxt* cxt) +{ + BufferDesc* buff = NULL; + while (!(buff = getNextFreeReadBuffer(cxt))) { + pg_usleep(WAIT_FOR_BUFF_SLEEP_TIME); + } + /* the buffer is ready */ + if (buff->usedLen != 0) { + return buff; + } + return tryGetNextFreeReadBuffer(cxt); +} + +size_t writeToBuffer(const char* data, size_t len, void* fp) +{ + BufferCxt* cxt = (BufferCxt*)fp; + BufferDesc* buff = NULL; + int64 writeLen = 0; + int64 freeLen = 0; + int64 remainingLen = (int64)len; + errno_t rc; + while (remainingLen > 0) { + buff = tryGetNextFreeWriteBuffer(cxt); + if (buff == NULL) { + return 0; + } + freeLen = buffFreeLen(buff); + writeLen = (remainingLen > freeLen) ? freeLen : remainingLen; + rc = memcpy_s(buffFreeLoc(buff, cxt), writeLen, data, writeLen); + securec_check_c(rc, "\0", "\0"); + addBuffLen(buff, writeLen); + data = data + writeLen; + remainingLen = remainingLen - writeLen; + if (buffFreeLen(buff) == 0) { + markBufferFlag(buff, BUFF_FLAG_FILE_FINISHED); + } + if ((remainingLen == 0 && cxt->fileEnd)) { + markBufferFlag(buff, BUFF_FLAG_FILE_CLOSED); + } + } + return len; +} + +bool hasBufferForRead(BufferCxt* cxt) +{ + const size_t consumerIdx = cxt->consumerIdx.load(std::memory_order_acquire); + BufferDesc* buff = &(cxt->bufHeader[consumerIdx]); + return testBufferFlag(buff, BUFF_FLAG_FILE_FINISHED | BUFF_FLAG_FILE_CLOSED); +} + +void* openWriteBufferFile(const char* filename, const char* mode) +{ + BufferCxt* buffCxt = current.sender_cxt->bufferCxt; + BufferDesc* buff = NULL; + SendFileInfo* fileInfo = NULL; + fileInfo = (SendFileInfo*)palloc(sizeof(SendFileInfo)); + if (fileInfo == NULL) { + elog(ERROR, "file info allocate failed: out of memory"); + } + fileInfo->filename = pgut_strdup(filename); + parray_append(current.filesinfo, fileInfo); + buffCxt->fileEnd = false; + buff = tryGetNextFreeWriteBuffer(buffCxt); + if (buff == NULL) { + elog(ERROR, "Failed to open buff file: %s", fileInfo->filename); + } + markBufferFlag(buff, BUFF_FLAG_FILE_OPENED); + buff->fileId = parray_num(current.filesinfo) - 1; + return buffCxt; +} diff --git a/src/bin/pg_probackup/oss/include/appender.h b/src/bin/pg_probackup/oss/include/appender.h new file mode 100644 index 000000000..83b6c2358 --- /dev/null +++ b/src/bin/pg_probackup/oss/include/appender.h @@ -0,0 +1,80 @@ +/*------------------------------------------------------------------------- + * + * appender.h: File appender used by Backup/Restore manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef FILE_APPEND_H +#define FILE_APPEND_H + +#include "../../pg_probackup.h" +#include "buffer.h" + +/* Data Structure Definition*/ + +typedef struct FileAppenderSegHeader +{ + FILE_APPEND_SEG_TYPE type; /* seg type */ + uint32 size; /* payload size */ + uint32 permission; + off_t filesize; + pg_crc32 crc; + int external_dir_num; + device_type_t file_type; +} FileAppenderSegHeader; + +typedef struct FileAppenderSegDescriptor +{ + FileAppenderSegHeader header; + char* payload; + int header_offset; /* set value only when header spans across two buffs and a rewind exists. */ + int payload_offset; /* set value only when payload spans across two buffs and a rewind exists. */ + FILE* outputFile; + pgFile* inputFile; + pg_crc32 crc; + pg_crc32 size; +} FileAppenderSegDescriptor; + +/* Constants Definition */ + +#define READ_BUFFER_BLOCK_COUNT 2 +#define APPEND_FILENAME_END_SIZE 10 +#define APPEND_FILENAME_END_DIGIT 6 +#define APPEND_FILE_MAX_SIZE 536870912 // 536870912, 512MB; 1073741824, 1GB +#define APPEND_FILE_HEADER_SIZE (sizeof(FileAppenderSegHeader)) + +/* API Function */ + +extern void initFileAppender(FileAppender* appender, FILE_APPEND_SEG_TYPE type, uint32 minFileNo, uint32 maxFileNo); + +extern void initSegDescriptor(FileAppenderSegDescriptor** segDesc); + +extern void getSegDescriptor(FileAppenderSegDescriptor* desc, char** buffOffset, size_t* remainBuffLen, BufferCxt* cxt); + +extern void parseSegDescriptor(FileAppenderSegDescriptor* segDesc, char** buffOffset, size_t* remainBuffLen, char* tempBuffer, + BufferCxt* cxt, pgBackup* dest_backup, + bool isValidate = false, validate_files_arg* arg = NULL); + +extern void destorySegDescriptor(FileAppenderSegDescriptor** descriptor); + +extern void closeFileAppender(FileAppender* retAppender); + +extern void destoryFileAppender(FileAppender** retAppender); + +extern char* getAppendFileName(const char* baseFileName, uint32 fileNo); + +extern void constructHeader(FileAppenderSegHeader* header, FILE_APPEND_SEG_TYPE type, + uint32 size, off_t filesize, pgFile* file); + +extern void writeHeader(FileAppenderSegHeader* header, FileAppender* appender); + +extern size_t writeToCompFile(const char* data, size_t len, void* file); + +extern void writePayload(const char* data, size_t len, FileAppender* appender); + +#endif /* FILE_APPEND_H */ + diff --git a/src/bin/pg_probackup/oss/include/backup.h b/src/bin/pg_probackup/oss/include/backup.h new file mode 100644 index 000000000..6ce46afab --- /dev/null +++ b/src/bin/pg_probackup/oss/include/backup.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * backup.h: Backup utils used by Backup/Restore manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef BACKUP_H +#define BACKUP_H + +#include "../../pg_probackup.h" +#include "appender.h" +#include "thread.h" + +/* API Function */ + +extern void performBackup(backup_files_arg* arg); + +extern void initPerformBackup(backup_files_arg* arg, backupReaderThreadArgs* thread_args); + +extern void backupDataFiles(backup_files_arg* arg); + +extern void backupDirectories(FileAppender* appender, backup_files_arg* arg); + +extern void backupFiles(FileAppender* appender, backup_files_arg* arg); + +extern void appendDir(FileAppender* appender, const char* dirPath, uint32 permission, + int external_dir_num, device_type_t type); + +extern void appendDataFile(FileAppender* appender, char* fileBuffer, pgFile* file, + FILE_APPEND_SEG_TYPE type, char* from_fullpath, backup_files_arg* arg); + +extern void appendNonDataFile(FileAppender* appender, char* fileBuffer, pgFile* file, + FILE_APPEND_SEG_TYPE type, char* from_fullpath, backup_files_arg* arg); + +extern void appendPgControlFile(FileAppender* appender, const char *from_fullpath, fio_location from_location, pgFile *file); + +extern int writeDataFile(FileAppender* appender, char* fileBuffer, pgFile* file, char* from_fullpath, backup_files_arg* arg); + +#endif /* BACKUP_H */ + diff --git a/src/bin/pg_probackup/oss/include/buffer.h b/src/bin/pg_probackup/oss/include/buffer.h new file mode 100644 index 000000000..81f88f307 --- /dev/null +++ b/src/bin/pg_probackup/oss/include/buffer.h @@ -0,0 +1,136 @@ +/*------------------------------------------------------------------------- + * + * buffer.h: Buffer used by Backup/Recovery manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef BUFFER_H +#define BUFFER_H + +#include + +#include "../../pg_probackup.h" + + +/* Constants Definition */ + +#define BUFSIZE 2097152 /* 2 * 1024 * 1024, 2MB */ +#define WAIT_FOR_BUFF_SLEEP_TIME 100000 /* 100 ms*/ +#define BUFF_FLAG_FILE_OPENED 0x1 +#define BUFF_FLAG_FILE_CLOSED 0x2 +#define BUFF_FLAG_FILE_FINISHED 0x4 + +#ifdef __cpp_lib_hardware_interference_size + static constexpr size_t CacheLineSize = + std::hardware_constructive_interference_size; +#else + static constexpr size_t CacheLineSize = 64; +#endif + +/* Data Structure Definition*/ + +typedef struct SendFileInfo { + char* filename; +} SendFileInfo; + +/* each buffer's description */ +typedef struct BufferDesc +{ + uint32 bufId; + int32 fileId; /* the buffer belong to which file */ + uint32 usedLen; + uint32 flags; + pthread_spinlock_t lock; /* for lock schema */ +} BufferDesc; + +/* the context of buffers */ +typedef struct BufferCxt +{ + BufferDesc* bufHeader; + char* bufData; + uint32 bufNum; + bool fileEnd; + int fileNum; /* for restore */ + volatile bool earlyExit; + alignas(CacheLineSize) std::atomic producerIdx = {0}; + alignas(CacheLineSize) size_t producerIdxCache = 0; + alignas(CacheLineSize) std::atomic consumerIdx = {0}; + alignas(CacheLineSize) size_t consumerIdxCache = 0; +} BufferCxt; + +/* API Function */ + +extern void initBufferCxt(BufferCxt* cxt, size_t bufferSize); + +extern void destroyBufferCxt(BufferCxt* cxt); + +extern BufferDesc* tryGetNextFreeWriteBuffer(BufferCxt* cxt); + +extern BufferDesc* getNextFreeWriteBuffer(BufferCxt* cxt); + +extern BufferDesc* tryGetNextFreeReadBuffer(BufferCxt* cxt); + +extern BufferDesc* getNextFreeReadBuffer(BufferCxt* cxt); + +extern void* openWriteBufferFile(const char* filename, const char* mode); + +extern void closeWriteBufferFile(void* fp); + +extern size_t writeToBuffer(const char* data, size_t len, void* fp); + +extern bool hasBufferForRead(BufferCxt* cxt); + +extern bool hasNextBufferForRead(BufferCxt* cxt, const size_t buffIdx); + +/* inline function */ +inline uint32 buffFreeLen(BufferDesc* buff) +{ + return BUFSIZE - buff->usedLen; +} + +inline uint32 buffNum(BufferCxt* cxt) +{ + return cxt->bufNum; +} + +inline char* buffLoc(BufferDesc* buff, BufferCxt* cxt) +{ + return cxt->bufData + buff->bufId * BUFSIZE; +} + +inline char* buffFreeLoc(BufferDesc* buff, BufferCxt* cxt) +{ + return buffLoc(buff, cxt) + buff->usedLen; +} + +inline void addBuffLen(BufferDesc* buff, uint32 len) +{ + pthread_spin_lock(&buff->lock); + buff->usedLen += len; + pthread_spin_unlock(&buff->lock); +} + +inline void markBufferFlag(BufferDesc* buff, uint32 flag) +{ + (*((volatile uint32*)&(buff->flags))) |= flag; +} + +inline bool testBufferFlag(BufferDesc* buff, uint32 flag) +{ + return ((*((volatile uint32*)&(buff->flags))) & flag); +} + +inline void clearBuff(BufferDesc* buff) +{ + pthread_spin_lock(&buff->lock); + buff->usedLen = 0; + buff->flags = 0; + buff->fileId = -1; + pthread_spin_unlock(&buff->lock); +} + +#endif /* BUFFER_H */ \ No newline at end of file diff --git a/src/bin/pg_probackup/oss/include/oss_operator.h b/src/bin/pg_probackup/oss/include/oss_operator.h new file mode 100644 index 000000000..5fcf0b275 --- /dev/null +++ b/src/bin/pg_probackup/oss/include/oss_operator.h @@ -0,0 +1,91 @@ +/*------------------------------------------------------------------------- + * + * oss_operator.h: OSS Operator used by Backup/Restore manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef OSS_OPERATOR_H +#define OSS_OPERATOR_H + +#include "stddef.h" +#include +#include +#include +#include "../../parray.h" + +/* Constants Definition */ + +#define OSS_MAX_UPLOAD_ID_LEN 256 +#define OSS_MAX_FILE_PATH 1024 +#define OSS_MAX_ETAG_LEN 256 + +/* Data Structure Definition*/ + +typedef struct OssFile { + char filePath[OSS_MAX_FILE_PATH]; + /* for write */ + char uploadID[OSS_MAX_UPLOAD_ID_LEN]; + char** eTagList; + int partNum; + /* for read */ + size_t fileSize; + bool oss_eof; + bool oss_error; + void* bufDate; + int byteCout; + int actualLen; + size_t offset; + char etag[OSS_MAX_ETAG_LEN]; +} OssFile; + +/* API Function */ + +namespace Oss { +using namespace std; +using SDKOptions = void *; +using S3Client = void *; + +class Oss { +public: + Oss(const char* endpoint, const char* access_key, const char* secret_key, const char* region = NULL, bool secure = false); + ~Oss(); + void GetObject(const char* bucket_name, const char* object_name, const char* file_name); + void GetObject(const char* from_bucket, const char* object_key, void* file); + void PutObject(const char* bucket_name, const char* file_path, const char* file_name); + void RemoveObject(const char* bucket_name, const char* objcet_key); + void ListObjects(char* bucket_name, parray* objects); + void ListObjectsWithPrefix(char* bucket_name, char* prefix, parray* objects); + void MakeBucket(char* bucket_name); + void ListBuckets(parray* buckets); + bool ObjectExists(char* bucket_name, char* object_name); + bool BucketExists(char* bucket_name); + void RemoveBucket(char* bucket_name); + void StartMultipartUpload(char* bucket_name, char* object_name); + void MultipartUpload(char* bucket_name, char* object_name, char* data, size_t data_size); + void CompleteMultipartUploadRequest(char* bucket_name, char* object_name); + +private: + const string kEndpoint; + const string kAccessKey; + const string kSecretKey; + const bool kSecure; + string kRegion; + void* completePartVector; + string UploadId; + int partNumber; + SDKOptions options_; + S3Client s3_client_; +}; +} // namespace Oss + +extern void parseBackupControlFilePath(const char* path, char** bucket_name, char** object_name); +extern char* getBucketName(); +extern char* getPrefixName(void* backup); +extern Oss::Oss* getOssClient(); + +#endif /* OSS_OPERATOR_H */ + diff --git a/src/bin/pg_probackup/oss/include/restore.h b/src/bin/pg_probackup/oss/include/restore.h new file mode 100644 index 000000000..a7498a38a --- /dev/null +++ b/src/bin/pg_probackup/oss/include/restore.h @@ -0,0 +1,46 @@ +/*------------------------------------------------------------------------- + * + * restore.h: Restore utils used by Backup/Restore manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef RESTORE_H +#define RESTORE_H + +#include "../../pg_probackup.h" +#include "appender.h" + +/* API Function */ + +extern void performValidate(pgBackup *backup, pgRestoreParams *params); + +extern void performRestoreOrValidate(pgBackup *dest_backup, bool isValidate = false); + +extern void restoreDir(const char* path, FileAppenderSegDescriptor* desc, pgBackup* dest_backup, + parray* files, bool isValidate); + +extern void restoreDataFile(const char* data, FileAppenderSegDescriptor* desc, parray *parent_chain, bool use_bitmap, + bool use_headers); + +extern void restoreNonDataFile(const char* data, FileAppenderSegDescriptor* desc, parray *parent_chain, pgBackup *dest_backup, + bool isValidate, validate_files_arg* arg); + +extern void openRestoreFile(const char* path, FileAppenderSegDescriptor* desc, pgBackup* dest_backup, + parray* files, bool isValidate = false, validate_files_arg* arg = NULL); + +extern void writeOrValidateRestoreFile(const char* data, FileAppenderSegDescriptor* desc, + bool isValidate, validate_files_arg* arg); + +extern void closeRestoreFile(FileAppenderSegDescriptor* desc); + +extern void restoreConfigDir(); + +extern void restoreConfigFile(const char* path); + +extern void uploadConfigFile(const char* local_path, const char* object_name); + +#endif /* RESTORE_H */ \ No newline at end of file diff --git a/src/bin/pg_probackup/oss/include/thread.h b/src/bin/pg_probackup/oss/include/thread.h new file mode 100644 index 000000000..1dfbdc936 --- /dev/null +++ b/src/bin/pg_probackup/oss/include/thread.h @@ -0,0 +1,124 @@ +/*------------------------------------------------------------------------- + * + * thread.h: Thread utils used by Backup/Restore manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef THREAD_H +#define THREAD_H + +#include "../../pg_probackup.h" +#include "appender.h" +#include "buffer.h" +#include "../../parray.h" + +/* Constants Definition */ + +#define SENDER_BUFFER_SIZE 268435456 /* 256 * 1024 * 1024 Bytes, 256MB*/ +#define READER_THREAD_FILE_COUNT 8 +#define FILE_BUFFER_SIZE 8388608 /* 8 * 1024 * 1024, 8MB */ +#define WAIT_FOR_STATE_CHANGE_TIME 100000 /* 100 ms*/ + +/* Data Structure Definition*/ +typedef enum readerThreadState { + READER_THREAD_STATE_INIT = 0, + READER_THREAD_STATE_START, + READER_THREAD_STATE_FLUSHING, + READER_THREAD_STATE_FLUSHED, + READER_THREAD_STATE_ERROR, + READER_THREAD_STATE_STOP +} ReaderThreadState; + +typedef enum SenderThreadState { + SENDER_THREAD_STATE_INIT = 0, + SENDER_THREAD_STATE_START, + SENDER_THREAD_STATE_FINISH, + SENDER_THREAD_STATE_FINISHED, + SENDER_THREAD_STATE_ERROR, + SENDER_THREAD_STATE_STOP +} SenderThreadState; + +typedef struct ReaderCxt { + pgFile** file; + pgFile** prefile; + char** fromPath; + char** toPath; + char* fileBuffer; + uint32 fileCount; + FileAppender* appender; + FILE_APPEND_SEG_TYPE* segType; + bool* fileRemoved; + pthread_t readerThreadId; + ReaderThreadState state; + pthread_spinlock_t lock; +} ReaderCxt; + +typedef struct SenderCxt { + BufferCxt* bufferCxt; + pthread_t senderThreadId; + SenderThreadState state; + pthread_spinlock_t lock; +} SenderCxt; + +typedef struct restoreReaderThreadArgs { + BufferCxt* bufferCxt; + pgBackup* dest_backup; +} restoreReaderThreadArgs; + +typedef struct backupReaderThreadArgs +{ + backup_files_arg* arg; + ReaderCxt* readerCxt; +} backupReaderThreadArgs; + +/* API Function */ + +int getFreeReaderThread(); + +extern ReaderThreadState getReaderState(ReaderCxt* readerCxt); + +extern void setReaderState(ReaderCxt* readerCxt, ReaderThreadState state); + +extern SenderThreadState getSenderState(SenderCxt* senderCxt); + +extern void setSenderState(SenderCxt* senderCxt, SenderThreadState state); + +extern void startBackupReaders(backup_files_arg* arg, backupReaderThreadArgs* thread_args); + +extern void startBackupSender(); + +extern void* backupReaderThreadMain(void* arg); + +extern void* backupSenderThreadMain(void* arg); + +extern void initBackupSenderContext(SenderCxt** cxt); + +extern void initBackupReaderContexts(ReaderCxt** cxt); + +extern bool isSenderThreadStopped(SenderCxt* senderCxt); + +extern void destoryBackupSenderContext(); + +extern bool isReaderThreadStopped(ReaderCxt* readerCxt); + +extern void destoryBackupReaderContexts(); + +extern void copyFileToFileBuffer(ReaderCxt* readerCxt, int fileIndex, backup_files_arg* arg); + +extern void flushReaderContexts(void* arg); + +extern void waitForSenderThread(); + +extern void stopBackupReaders(); + +extern void stopBackupSender(); + +extern void waitForReadersCopyComplete(); + +extern void* restoreReaderThreadMain(void* arg); + +#endif /* THREAD_H */ \ No newline at end of file diff --git a/src/bin/pg_probackup/oss/oss_operator.cpp b/src/bin/pg_probackup/oss/oss_operator.cpp new file mode 100644 index 000000000..90f3cfeea --- /dev/null +++ b/src/bin/pg_probackup/oss/oss_operator.cpp @@ -0,0 +1,288 @@ +/*------------------------------------------------------------------------- + * + * oss_operator.cpp: OSS operator used by Backup/Recovery manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "include/oss_operator.h" +#include "utils/elog.h" +#include "include/buffer.h" +#include "include/restore.h" + +namespace Oss { +Oss::Oss(const char* endpoint, const char* access_key, const char* secret_key, const char* region, bool secure) + : kEndpoint(endpoint), kAccessKey(access_key), kSecretKey(secret_key), kSecure(secure) { + options_ = new Aws::SDKOptions; + auto options = reinterpret_cast(options_); + Aws::InitAPI(*options); + Aws::Client::ClientConfiguration config; + if (region != NULL) { + kRegion.assign(region); + config.region = kRegion; + } + config.endpointOverride = kEndpoint; + if (kSecure) { + config.scheme = Aws::Http::Scheme::HTTPS; + config.verifySSL = true; + } else { + config.scheme = Aws::Http::Scheme::HTTP; + config.verifySSL = false; + } + completePartVector = new Aws::Vector(); + s3_client_ = new Aws::S3::S3Client(Aws::Auth::AWSCredentials(kAccessKey, kSecretKey), config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false); +} + +Oss::~Oss() { + auto options = reinterpret_cast(options_); + // Before the application the applications terminates, the SDK must be shut down. + Aws::ShutdownAPI(*options); + // Clean up + delete reinterpret_cast(options_); + delete reinterpret_cast(s3_client_); + delete reinterpret_cast *>(completePartVector); +} + +void Oss::GetObject(const char* from_bucket, const char* object_key, void* filePtr) { + BufferCxt* file = (BufferCxt*)(filePtr); + auto s3_client = reinterpret_cast(s3_client_); + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(from_bucket); + request.SetKey(object_key); + + auto outcome = s3_client->GetObject(request); + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + elog(ERROR, "GetObject: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str()); + } + char buffer[BUFSIZE]; + std::stringstream ss; + ss << outcome.GetResultWithOwnership().GetBody().rdbuf(); + while (!ss.eof()) { + size_t readlen = ss.read(buffer, BUFSIZE).gcount(); + if (readlen < BUFSIZE) { + file->fileEnd = true; + } + if (writeToBuffer(buffer, readlen, file) != readlen) { + elog(ERROR, "GetObject: write to buffer failed."); + } + if (file->earlyExit) { + break; + } + } +} + +void Oss::GetObject(const char* bucket_name, const char* object_name, const char* file_name) { + auto s3_client = reinterpret_cast(s3_client_); + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket_name); + request.SetKey(object_name); + auto outcome = s3_client->GetObject(request); + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + elog(ERROR, "GetObject: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str()); + } + char* separator_pos = last_dir_separator(file_name); + char* dir_path = strndup(file_name, separator_pos - file_name); + fio_mkdir(dir_path, DIR_PERMISSION, FIO_BACKUP_HOST); + free(dir_path); + Aws::OFStream local_file; + local_file.open(file_name, ios::out | ios::binary); + if (!local_file) { + elog(ERROR, "Error GetObject : dst path's directory: %s is not existed.", file_name); + } + local_file << outcome.GetResultWithOwnership().GetBody().rdbuf(); +} + +void Oss::PutObject(const char* bucket_name, const char* file_path, const char* file_name) { + struct stat buffer; + if (stat(file_path, &buffer) == -1) { + elog(ERROR, "PutObject: File %s does not exist", file_name); + } + auto s3_client = reinterpret_cast(s3_client_); + Aws::S3::Model::PutObjectRequest request; + request.SetBucket(bucket_name); + request.SetKey(file_name); + shared_ptr input_data = + Aws::MakeShared("sSampleAllocationtTag", file_path, ios_base::in | ios_base::binary); + request.SetBody(input_data); + auto outcome = s3_client->PutObject(request); + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + elog(ERROR, "PutObject: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str()); + } +} + +void Oss::StartMultipartUpload(char* bucket_name, char* object_name) +{ + auto s3_client = reinterpret_cast(s3_client_); + auto completeParts = (Aws::Vector*)completePartVector; + // set the target bucket and file path for uploading + const Aws::String bucket = bucket_name; + const Aws::String key = object_name; + // initialize parts upload task + Aws::S3::Model::CreateMultipartUploadRequest create_request; + create_request.SetBucket(bucket_name); + create_request.SetKey(object_name); + Aws::S3::Model::CreateMultipartUploadOutcome outcome = s3_client->CreateMultipartUpload(create_request); + // obtain upload ID and part number + UploadId = std::string(outcome.GetResult().GetUploadId().c_str()); + partNumber = 1; + completeParts->clear(); + // create part upload output + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + elog(ERROR, "StartMultipartUpload: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str()); + } +} + +void Oss::MultipartUpload(char* bucket_name, char* object_name, char* data, size_t data_size) +{ + if (data_size == 0 || data == NULL) { + return; + } + auto s3_client = reinterpret_cast(s3_client_); + auto completeParts = (Aws::Vector*)completePartVector; + const Aws::String bucket = bucket_name; + const Aws::String key = object_name; + Aws::S3::Model::UploadPartRequest request1; + request1.SetBucket(bucket_name); + request1.SetKey(object_name); + // start uploading + Aws::S3::Model::UploadPartRequest uploadPartRequest; + uploadPartRequest.WithBucket(bucket).WithKey(key).WithUploadId(Aws::String(UploadId.c_str())).WithPartNumber(partNumber).WithContentLength(data_size); + Aws::String str(data, data_size); + auto input_data = Aws::MakeShared("UploadPartStream", str); + uploadPartRequest.SetBody(input_data); + auto uploadPartResult = s3_client->UploadPart(uploadPartRequest); + completeParts->push_back(Aws::S3::Model::CompletedPart().WithETag(uploadPartResult.GetResult().GetETag()).WithPartNumber(partNumber)); + ++partNumber; +} + +void Oss::CompleteMultipartUploadRequest(char* bucket_name, char* object_name) +{ + auto s3_client = reinterpret_cast(s3_client_); + auto completeParts = (Aws::Vector*)completePartVector; + const Aws::String bucket = bucket_name; + const Aws::String key = object_name; + // Complete parts upload + Aws::S3::Model::CompleteMultipartUploadRequest request; + request.SetBucket(bucket); + request.SetKey(key); + request.SetUploadId(Aws::String(UploadId.c_str())); + Aws::S3::Model::CompletedMultipartUpload completed_multipart_upload; + completed_multipart_upload.SetParts(*completeParts); + request.SetMultipartUpload(completed_multipart_upload); + Aws::S3::Model::CompleteMultipartUploadOutcome outcome = s3_client->CompleteMultipartUpload(request); + if (!outcome.IsSuccess()) { + elog(ERROR, "CompleteMultipartUploadRequest: %s", outcome.GetError().GetMessage().c_str()); + } +} + + +void Oss::RemoveObject(const char* bucket_name, const char* objcet_key) { + auto s3_client = reinterpret_cast(s3_client_); + Aws::S3::Model::DeleteObjectRequest request; + request.WithKey(objcet_key).WithBucket(bucket_name); + auto outcome = s3_client->DeleteObject(request); + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + elog(WARNING, "RemoveObject: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str()); + } +} + +void Oss::ListObjectsWithPrefix(char* bucket_name, char* prefix, parray* objects) +{ + auto s3_client = reinterpret_cast(s3_client_); + Aws::S3::Model::ListObjectsRequest request; + request.WithBucket(bucket_name); + request.SetPrefix(prefix); + auto outcome = s3_client->ListObjects(request); + if (!outcome.IsSuccess()) { + auto err = outcome.GetError(); + elog(ERROR, "ListObjectsWithPrefix: %s, %s", err.GetExceptionName().c_str(), err.GetMessage().c_str()); + } + Aws::Vector resp = outcome.GetResult().GetContents(); + for (auto &bucket : resp) { + char* key = pg_strdup(bucket.GetKey().c_str()); + parray_append(objects, key); + } +} + +bool Oss::BucketExists(char* bucket_name) { + auto s3_client = reinterpret_cast(s3_client_); + Aws::S3::Model::HeadBucketRequest request; + request.SetBucket(bucket_name); + auto outcome = s3_client->HeadBucket(request); + if (!outcome.IsSuccess()) { + return false; + } + return true; +} + +bool Oss::ObjectExists(char* bucket_name, char* object_name) { + auto s3_client = reinterpret_cast(s3_client_); + Aws::S3::Model::HeadObjectRequest headObjectRequest; + headObjectRequest.WithBucket(bucket_name).WithKey(object_name); + auto result = s3_client->HeadObject(headObjectRequest); + return result.IsSuccess(); +} +} // namespace Oss + +char* getBucketName() +{ + char* bucket_name = instance_config.oss.access_bucket; + if (bucket_name == NULL) { + elog(ERROR, "Required parameter not specified: OSS(--bucket_name)"); + } + return bucket_name; +} + +char* getPrefixName(void* backup) +{ + return ((pgBackup*)backup)->root_dir + 1; +} + +Oss::Oss* getOssClient() +{ + if (oss_client == NULL) { + const char* endpoint = instance_config.oss.endpoint; + const char* access_key = instance_config.oss.access_id; + const char* secret_key = instance_config.oss.access_key; + const char* region = instance_config.oss.region; + const char* access_bucket = instance_config.oss.access_bucket; + if (!endpoint || !access_key || !secret_key || !access_bucket) { + elog(ERROR, + "Required parameter not specified: OSS(--endpoint, --access_bucket, --access_id or --access_key)"); + } + oss_client = new Oss::Oss(endpoint, access_key, secret_key, region); + } + return (Oss::Oss*)oss_client; +} diff --git a/src/bin/pg_probackup/oss/restore.cpp b/src/bin/pg_probackup/oss/restore.cpp new file mode 100644 index 000000000..40a485833 --- /dev/null +++ b/src/bin/pg_probackup/oss/restore.cpp @@ -0,0 +1,350 @@ +#include "include/restore.h" +#include "include/oss_operator.h" +#include "include/thread.h" + +#include "storage/file/fio_device.h" +#include "common/fe_memutils.h" + +#define IsIllegalCharacter(c) ((c) != '/' && !isdigit((c)) && !isalpha((c)) && (c) != '_' && (c) != '-' && (c) != '.') +static fio_location location = FIO_BACKUP_HOST; +static pgFile* findpgFile(parray* files, char* to_path, int external_dir_num, device_type_t type); + +void performRestoreOrValidate(pgBackup *dest_backup, bool isValidate) +{ + /* for validate */ + char base_path[MAXPGPATH]; + char dss_path[MAXPGPATH]; + char external_prefix[MAXPGPATH]; + parray *files = NULL; + bool corrupted = false; + validate_files_arg arg; + if (dest_backup->files == NULL) { + files = get_backup_filelist(dest_backup, true); + dest_backup->files = files; + parray_qsort(dest_backup->files, pgFileCompareRelPathWithExternal); + } else { + files = dest_backup->files; + parray_qsort(dest_backup->files, pgFileCompareRelPathWithExternal); + } + if (isValidate) { + if (!pre_check_backup(dest_backup)) { + return; + } + join_path_components(base_path, dest_backup->root_dir, DATABASE_DIR); + join_path_components(dss_path, dest_backup->root_dir, DSSDATA_DIR); + join_path_components(external_prefix, dest_backup->root_dir, EXTERNAL_DIR); + if (!files) { + elog(WARNING, "Backup %s file list is corrupted", base36enc(dest_backup->start_time)); + dest_backup->status = BACKUP_STATUS_CORRUPT; + write_backup_status(dest_backup, BACKUP_STATUS_CORRUPT, instance_name, true); + return; + } + arg.base_path = base_path; + arg.dss_path = dss_path; + arg.files = files; + arg.corrupted = false; + arg.backup_mode = dest_backup->backup_mode; + arg.stop_lsn = dest_backup->stop_lsn; + arg.checksum_version = dest_backup->checksum_version; + arg.backup_version = parse_program_version(dest_backup->program_version); + arg.external_prefix = external_prefix; + arg.hdr_map = &(dest_backup->hdr_map); + arg.ret = 1; + } + /* Initialize the buffer context */ + BufferCxt* bufferCxt = (BufferCxt*)palloc(sizeof(BufferCxt)); + if (bufferCxt == NULL) { + elog(ERROR, "buffer context allocate failed: out of memory"); + } + initBufferCxt(bufferCxt, SENDER_BUFFER_SIZE); + restoreReaderThreadArgs args; + args.bufferCxt = bufferCxt; + args.dest_backup = dest_backup; + pthread_t restoreReaderThread; + pthread_create(&restoreReaderThread, nullptr, restoreReaderThreadMain, &args); + + char tempBuffer[BUFSIZE]; + BufferDesc* buff = NULL; + BufferDesc* nextBuff = NULL; + BufferDesc* prevBuff = NULL; + FileAppenderSegDescriptor* desc = NULL; + FileAppenderSegDescriptor* predesc = NULL; + initSegDescriptor(&desc); + initSegDescriptor(&predesc); + const size_t segHdrLen = sizeof(FileAppenderSegHeader); + char* buffOffset = NULL; + size_t remainBuffLen = 0; + int filenum = 0; + while(true) { + buff = tryGetNextFreeReadBuffer(bufferCxt); + char* buffEnd = buffLoc(buff, bufferCxt) + buff->usedLen; + if (remainBuffLen == 0) { + buffOffset = buffLoc(buff, bufferCxt); + } + if (unlikely(prevBuff != NULL)) { + remainBuffLen = remainBuffLen + buff->usedLen; + } else { + remainBuffLen = buffEnd - buffOffset; + } + while (segHdrLen <= remainBuffLen) { + memcpy_s(predesc, sizeof(FileAppenderSegDescriptor), desc, sizeof(FileAppenderSegDescriptor)); + getSegDescriptor(desc, &buffOffset, &remainBuffLen, bufferCxt); + if (prevBuff != NULL) { + clearBuff(prevBuff); + prevBuff = NULL; + } + // The payload spans across two buffs. + if (desc->header.size > 0 && desc->header.size > remainBuffLen) { + nextBuff = tryGetNextFreeReadBuffer(bufferCxt); + while (nextBuff->bufId == buff->bufId) { + pg_usleep(WAIT_FOR_BUFF_SLEEP_TIME); + nextBuff = tryGetNextFreeReadBuffer(bufferCxt); + } + // rewind + if (nextBuff->bufId == 0 && remainBuffLen > 0) { + desc->payload_offset = remainBuffLen; + } + remainBuffLen = remainBuffLen + nextBuff->usedLen; + } + parseSegDescriptor(desc, &buffOffset, &remainBuffLen, tempBuffer, bufferCxt, dest_backup, isValidate, &arg); + if (isValidate && arg.corrupted) { + remainBuffLen = 0; + corrupted = true; + break; + } + if (desc->header.type == FILE_APPEND_TYPE_FILES_END) { + filenum++; + if (filenum == bufferCxt->fileNum) { + break; + } + nextBuff = tryGetNextFreeReadBuffer(bufferCxt); + while (nextBuff->bufId == buff->bufId) { + pg_usleep(WAIT_FOR_BUFF_SLEEP_TIME); + nextBuff = tryGetNextFreeReadBuffer(bufferCxt); + } + remainBuffLen = nextBuff->usedLen; + buffOffset = buffLoc(nextBuff, bufferCxt); + } + if (nextBuff != NULL) { + clearBuff(buff); + buff = nextBuff; + nextBuff = NULL; + } + } + // The header spans across two buffs. + if (remainBuffLen > 0) { + prevBuff = buff; + // rewind + if (prevBuff->bufId == (buffNum(bufferCxt) - 1)) { + desc->header_offset = remainBuffLen; + } + continue; + } + // reuse the buffer + clearBuff(buff); + if (filenum == bufferCxt->fileNum || corrupted) { + break; + } + } + + args.bufferCxt->earlyExit = true; + pthread_join(restoreReaderThread, nullptr); + destorySegDescriptor(&desc); + destroyBufferCxt(bufferCxt); + if (isValidate) { + elog(INFO, "Finish validate file."); + if (corrupted) { + dest_backup->status = BACKUP_STATUS_CORRUPT; + } + write_backup_status(dest_backup, corrupted ? BACKUP_STATUS_CORRUPT : + BACKUP_STATUS_OK, instance_name, true); + if (corrupted) { + elog(WARNING, "Backup %s data files are corrupted", base36enc(dest_backup->start_time)); + } else { + elog(INFO, "Backup %s data files are valid", base36enc(dest_backup->start_time)); + } + } +} + +void restoreDir(const char* path, FileAppenderSegDescriptor* desc, pgBackup* dest_backup, + parray* files, bool isValidate) +{ + if (isValidate) { + return; + } + /* create directories */ + char to_path[MAXPGPATH]; + char from_root[MAXPGPATH]; + char dir_path[MAXPGPATH]; + errno_t rc; + size_t pathlen = desc->header.size; + rc = strncpy_s(to_path, pathlen + 1, path, pathlen); + pgFile* dir = findpgFile(files, to_path, desc->header.external_dir_num, desc->header.file_type); + if (dir == NULL) { + elog(ERROR, "Cannot find dir \"%s\"", to_path); + } + + if (dir->external_dir_num != 0) { + char external_prefix[MAXPGPATH]; + join_path_components(external_prefix, dest_backup->root_dir, EXTERNAL_DIR); + makeExternalDirPathByNum(from_root, external_prefix, desc->inputFile->external_dir_num); + } + else if (is_dss_type(dir->type)) { + join_path_components(from_root, dest_backup->root_dir, DSSDATA_DIR); + } else { + join_path_components(from_root, dest_backup->root_dir, DATABASE_DIR); + } + join_path_components(dir_path, from_root, to_path); + fio_mkdir(dir_path, desc->header.permission, location); +} + +void openRestoreFile(const char* path, FileAppenderSegDescriptor* desc, pgBackup* dest_backup, + parray* files, bool isValidate, validate_files_arg* arg) +{ + char to_path[MAXPGPATH]; + char from_root[MAXPGPATH]; + char filepath[MAXPGPATH]; + errno_t rc; + rc = strncpy_s(to_path, desc->header.size + 1, path, desc->header.size); + securec_check_c(rc, "\0", "\0"); + desc->inputFile = findpgFile(files, to_path, desc->header.external_dir_num, desc->header.file_type); + if (desc->inputFile == NULL) { + elog(ERROR, "Cannot find file \"%s\"", to_path); + } + if (isValidate) { + if (desc->inputFile->write_size == BYTES_INVALID) { + if (arg->backup_mode == BACKUP_MODE_FULL) { + /* It is illegal for file in FULL backup to have BYTES_INVALID */ + elog(WARNING, "Backup file \"%s\" has invalid size. Possible metadata corruption.", + desc->inputFile->rel_path); + arg->corrupted = true; + } + return; + } + INIT_FILE_CRC32(true, desc->crc); + } else { + if (desc->inputFile->external_dir_num != 0) { + char external_prefix[MAXPGPATH]; + join_path_components(external_prefix, dest_backup->root_dir, EXTERNAL_DIR); + makeExternalDirPathByNum(from_root, external_prefix, desc->inputFile->external_dir_num); + } + else if (is_dss_type(desc->inputFile->type)) { + join_path_components(from_root, dest_backup->root_dir, DSSDATA_DIR); + } else { + join_path_components(from_root, dest_backup->root_dir, DATABASE_DIR); + } + join_path_components(filepath, from_root, desc->inputFile->rel_path); + + if (desc->outputFile == NULL) { + desc->outputFile = fio_fopen(filepath, PG_BINARY_W, location); + } else if (desc->outputFile != NULL) { + desc->outputFile = fio_fopen(filepath, PG_BINARY_R "+", location); + } + if (desc->outputFile == NULL) { + elog(ERROR, "Cannot open restore file \"%s\": %s", + filepath, strerror(errno)); + } + setvbuf(desc->outputFile, NULL, _IONBF, BUFSIZ); + } +} + +void closeRestoreFile(FileAppenderSegDescriptor* desc) +{ + if (desc->outputFile && fio_fclose(desc->outputFile) != 0) { + elog(ERROR, "Cannot close file!", strerror(errno)); + } + desc->outputFile = NULL; + desc->inputFile = NULL; +} + +void writeOrValidateRestoreFile(const char* data, FileAppenderSegDescriptor* desc, + bool isValidate, validate_files_arg* arg) +{ + pgFile* dest_file = desc->inputFile; + if (dest_file == NULL) { + return; + } + /* Restore or Validate destination file */ + if (isValidate) { + if (!S_ISREG(dest_file->mode) || dest_file->write_size == 0 || + strcmp(dest_file->name, PG_XLOG_CONTROL_FILE) == 0) { + return; + } + if (dest_file->write_size == BYTES_INVALID) { + if (arg->backup_mode == BACKUP_MODE_FULL) { + elog(WARNING, "Backup file \"%s\" has invalid size. Possible metadata corruption.", + dest_file->rel_path); + arg->corrupted = true; + return; + } + return; + } + COMP_FILE_CRC32(true, desc->crc, data, desc->header.size); + if (desc->crc != desc->header.crc) { + arg->corrupted = true; + return; + } + } else if (desc->header.size > 0) { + if (fio_fwrite(desc->outputFile, data, desc->header.size) != desc->header.size) { + elog(ERROR, "Cannot write blocks of \"%s\": %s", desc->inputFile->rel_path, strerror(errno)); + } + } +} + +void restoreConfigDir() +{ + Oss::Oss* oss = getOssClient(); + char* bucket_name = getBucketName(); + char* prefix_name = backup_instance_path + 1; + parray *obj_list = parray_new(); + oss->ListObjectsWithPrefix(bucket_name, prefix_name, obj_list); + char dir_path[MAXPGPATH]; + for (size_t i = 0; i < parray_num(obj_list); i++) { + char* object = (char*)parray_get(obj_list, i); + char* filename = last_dir_separator(object); + char* dir_name = strndup(object, filename - object); + if (strcmp(filename + 1, BACKUP_CATALOG_CONF_FILE) == 0) { + pg_free(dir_name); + continue; + } + if (strcmp(filename + 1, BACKUP_CONTROL_FILE) == 0) { + join_path_components(dir_path, "/", dir_name); + fio_mkdir(dir_path, DIR_PERMISSION, location); + pg_free(dir_name); + } + } + parray_free(obj_list); +} + +void restoreConfigFile(const char* path) +{ + Oss::Oss* oss = getOssClient(); + const char* object_name = NULL; + const char* bucket_name = NULL; + bucket_name = getBucketName(); + object_name = path; + oss->GetObject(bucket_name, object_name, (char*)path); +} + +void uploadConfigFile(const char* path, const char* object_name) +{ + Oss::Oss* oss = getOssClient(); + const char* bucket_name = getBucketName(); + oss->RemoveObject(bucket_name, object_name); + oss->PutObject(bucket_name, path, object_name); + fio_unlink(path, location); +} + +static pgFile* findpgFile(parray* files, char* to_path, int external_dir_num, device_type_t type) +{ + pgFile* resfile = NULL; + pgFile tempfile; + tempfile.rel_path = to_path; + tempfile.external_dir_num = external_dir_num; + tempfile.type = type; + void* res = parray_bsearch(files, &tempfile, pgFileCompareRelPathWithExternal); + if (res != NULL) { + resfile = *(pgFile **)res; + } + return resfile; +} \ No newline at end of file diff --git a/src/bin/pg_probackup/oss/thread.cpp b/src/bin/pg_probackup/oss/thread.cpp new file mode 100644 index 000000000..ea06bdfa9 --- /dev/null +++ b/src/bin/pg_probackup/oss/thread.cpp @@ -0,0 +1,453 @@ +/*------------------------------------------------------------------------- + * + * thread.cpp: Thread api used by Backup/Recovery manager. + * + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION + * Portions Copyright (c) 2015-2018, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#include "include/thread.h" +#include "include/oss_operator.h" +#include "include/backup.h" +#include "include/appender.h" + +#include "utils/palloc.h" +#include "common/fe_memutils.h" +#include "storage/file/fio_device.h" + +void initBackupSenderContext(SenderCxt** cxt) +{ + SenderCxt* senderCxt = NULL; + senderCxt = (SenderCxt*)palloc(sizeof(SenderCxt)); + if (senderCxt == NULL) { + elog(ERROR, "sender context allocate failed: out of memory"); + } + pthread_spin_init(&senderCxt->lock, PTHREAD_PROCESS_PRIVATE); + senderCxt->state = SENDER_THREAD_STATE_INIT; + senderCxt->bufferCxt = (BufferCxt*)palloc(sizeof(BufferCxt)); + if (senderCxt->bufferCxt == NULL) { + pfree_ext(senderCxt); + elog(ERROR, "buffer context allocate failed: out of memory"); + } + /* Initialize the buffer context */ + initBufferCxt(senderCxt->bufferCxt, SENDER_BUFFER_SIZE); + *cxt = senderCxt; +} + +void startBackupSender() +{ + pthread_create(¤t.sender_cxt->senderThreadId, nullptr, backupSenderThreadMain, (void*)current.sender_cxt); +} + +bool isSenderThreadStopped(SenderCxt* senderCxt) +{ + pthread_spin_lock(&senderCxt->lock); + bool isStopped = senderCxt->state == SENDER_THREAD_STATE_STOP; + pthread_spin_unlock(&senderCxt->lock); + return isStopped; +} + +bool isReaderThreadStopped(ReaderCxt* readerCxt) +{ + pthread_spin_lock(&readerCxt->lock); + bool isStopped = (readerCxt->state == READER_THREAD_STATE_STOP); + pthread_spin_unlock(&readerCxt->lock); + return isStopped; +} + +void destoryBackupReaderContexts() +{ + ReaderCxt* readerCxt = NULL; + for (uint i = 0; i < current.readerThreadCount; i++) { + readerCxt = ¤t.readerCxt[i]; + pthread_spin_lock(&readerCxt->lock); + readerCxt->state = READER_THREAD_STATE_STOP; + pthread_spin_unlock(&readerCxt->lock); + pthread_join(readerCxt->readerThreadId, NULL); + pthread_spin_destroy(&readerCxt->lock); + pfree_ext(readerCxt->file); + pfree_ext(readerCxt->prefile); + pfree_ext(readerCxt->fromPath); + pfree_ext(readerCxt->fileBuffer); + pfree_ext(readerCxt->segType); + pfree_ext(readerCxt->fileRemoved); + } + pfree_ext(current.readerCxt); +} + +void destoryBackupSenderContext() +{ + SenderCxt* senderCxt = current.sender_cxt; + pthread_spin_lock(&senderCxt->lock); + senderCxt->state = SENDER_THREAD_STATE_STOP; + pthread_spin_unlock(&senderCxt->lock); + pthread_join(senderCxt->senderThreadId, NULL); + pthread_spin_destroy(&senderCxt->lock); + pfree_ext(senderCxt->bufferCxt); + pfree_ext(senderCxt); +} + +void* backupSenderThreadMain(void* arg) +{ + SenderCxt* senderCxt = (SenderCxt*)arg; + BufferCxt* bufferCxt = senderCxt->bufferCxt; + BufferDesc* buff = NULL; + Oss::Oss* oss = NULL; + const uint32 partSize = 6 * 1024 * 1024; // 6MB + uint32 partLeftSize = partSize; + char* buffer = (char*)palloc(partSize * sizeof(char)); + char* bufferEndPtr = buffer + partSize; + // Open OSS connection + oss = getOssClient(); + // find OSS bucket + char* bucket_name = getBucketName(); + char* object_name = NULL; + SendFileInfo* fileinfo = NULL; + errno_t rc; + while (true) { + // wait for the buffer to be consumed + if (isSenderThreadStopped(senderCxt)) { + break; + } else if (getSenderState(senderCxt) != SENDER_THREAD_STATE_FINISH && !hasBufferForRead(bufferCxt)) { + continue; + } else if (getSenderState(senderCxt) == SENDER_THREAD_STATE_FINISH && !hasBufferForRead(bufferCxt)) { + setSenderState(senderCxt, SENDER_THREAD_STATE_FINISHED); + continue; + } + buff = tryGetNextFreeReadBuffer(bufferCxt); + // write the buffer to OSS + if (buff->usedLen != 0) { + if (buff->fileId != -1) { + pthread_spin_lock(&senderCxt->lock); + fileinfo = (SendFileInfo*)parray_get(current.filesinfo, buff->fileId); + pthread_spin_unlock(&senderCxt->lock); + } + if (fileinfo != NULL && object_name == NULL) { + object_name = fileinfo->filename; + oss->StartMultipartUpload(bucket_name, object_name); + partLeftSize = partSize; + } else if (fileinfo != NULL && strcmp(object_name, fileinfo->filename) != 0) { + oss->MultipartUpload(bucket_name, object_name, buffer, (partSize - partLeftSize)); + oss->CompleteMultipartUploadRequest(bucket_name, object_name); + object_name = fileinfo->filename; + oss->StartMultipartUpload(bucket_name, object_name); + partLeftSize = partSize; + } else if (fileinfo == NULL) { + elog(ERROR, "get file info failed."); + } + if (buff->usedLen < partLeftSize) { + rc = memcpy_s(bufferEndPtr - partLeftSize, buff->usedLen, buffLoc(buff, bufferCxt), buff->usedLen); + securec_check(rc, "\0", "\0"); + partLeftSize = partLeftSize - buff->usedLen; + } else { + uint32 buff_off = buff->usedLen - partLeftSize; + rc = memcpy_s(bufferEndPtr - partLeftSize, partLeftSize, buffLoc(buff, bufferCxt), partLeftSize); + securec_check(rc, "\0", "\0"); + oss->MultipartUpload(bucket_name, object_name, buffer, partSize); + memset_s(buffer, partSize, 0, partSize); + securec_check(rc, "\0", "\0"); + if (buff_off > 0) { + rc = memcpy_s(buffer, buff_off, buffLoc(buff, bufferCxt) + partLeftSize, buff_off); + securec_check(rc, "\0", "\0"); + } + partLeftSize = partSize - buff_off; + } + } + // reuse the buffer + clearBuff(buff); + } + if (bucket_name != NULL && object_name != NULL) { + oss->MultipartUpload(bucket_name, object_name, buffer, (partSize - partLeftSize)); + oss->CompleteMultipartUploadRequest(bucket_name, object_name); + } + if (bufferCxt != NULL) { + destroyBufferCxt((BufferCxt *)bufferCxt); + } + pfree_ext(buffer); + return NULL; +} + +void initBackupReaderContexts(ReaderCxt** cxt) +{ + ReaderCxt* readerCxt= NULL; + current.readerThreadCount = num_threads - 1; + /* alloc memmory */ + ReaderCxt* current_readerCxt = (ReaderCxt*)palloc(sizeof(ReaderCxt) * current.readerThreadCount); + if (current_readerCxt == NULL) { + elog(ERROR, "reader thread allocate failed: out of memory"); + } + /* Initialize the reader context */ + for (uint i = 0; i < current.readerThreadCount; i++) { + readerCxt = ¤t_readerCxt[i]; + pthread_spin_init(&readerCxt->lock, PTHREAD_PROCESS_PRIVATE); + readerCxt->state = READER_THREAD_STATE_INIT; + readerCxt->file = (pgFile**)palloc(sizeof(pgFile*) * READER_THREAD_FILE_COUNT); + if (readerCxt->file == NULL) { + elog(ERROR, "file list allocate failed: out of memory"); + } + readerCxt->prefile = (pgFile**)palloc(sizeof(pgFile*) * READER_THREAD_FILE_COUNT); + if (readerCxt->prefile == NULL) { + elog(ERROR, "prefile list allocate failed: out of memory"); + } + readerCxt->fromPath = (char**)palloc(sizeof(char*) * READER_THREAD_FILE_COUNT); + if (readerCxt->fromPath == NULL) { + elog(ERROR, "fromPath list allocate failed: out of memory"); + } + readerCxt->toPath = (char**)palloc(sizeof(char*) * READER_THREAD_FILE_COUNT); + if (readerCxt->toPath == NULL) { + elog(ERROR, "toPath list allocate failed: out of memory"); + } + readerCxt->fileBuffer = (char *)palloc(FILE_BUFFER_SIZE * READER_THREAD_FILE_COUNT); + if (readerCxt->fileBuffer == NULL) { + elog(ERROR, "file buffer allocate failed: out of memory"); + } + readerCxt->segType = (FILE_APPEND_SEG_TYPE *)palloc(sizeof(FILE_APPEND_SEG_TYPE) * READER_THREAD_FILE_COUNT); + if (readerCxt->segType == NULL) { + elog(ERROR, "segment type allocate failed: out of memory"); + } + readerCxt->fileRemoved = (bool *)palloc(sizeof(bool) * READER_THREAD_FILE_COUNT); + if (readerCxt->fileRemoved == NULL) { + elog(ERROR, "file removed flag allocate failed: out of memory"); + } + } + for (uint i = 0; i < current.readerThreadCount; i++) { + readerCxt = ¤t_readerCxt[i]; + readerCxt->fileCount = 0; + readerCxt->readerThreadId = 0; + readerCxt->appender = NULL; + for (uint j = 0; j < READER_THREAD_FILE_COUNT; j++) { + readerCxt->file[j] = NULL; + readerCxt->prefile[j] = NULL; + readerCxt->fromPath[j] = NULL; + readerCxt->toPath[j] = NULL; + readerCxt->segType[j] = FILE_APPEND_TYPE_UNKNOWN; + readerCxt->fileRemoved[j] = false; + } + } + *cxt = current_readerCxt; +} + +void startBackupReaders(backup_files_arg* arg, backupReaderThreadArgs* thread_args) +{ + for (uint i = 0; i < current.readerThreadCount; i++) { + backupReaderThreadArgs* args = &thread_args[i]; + args->arg = arg; + args->readerCxt = ¤t.readerCxt[i]; + pthread_create(&(args->readerCxt->readerThreadId), nullptr, backupReaderThreadMain, (void*)args); + } +} + +void* backupReaderThreadMain(void* thread_args) +{ + backupReaderThreadArgs* args = (backupReaderThreadArgs*)thread_args; + backup_files_arg* arg = (backup_files_arg*)(args->arg); + ReaderCxt* readerCxt = (ReaderCxt*)(args->readerCxt); + while (!isReaderThreadStopped(readerCxt)) { + if (getReaderState(readerCxt) != READER_THREAD_STATE_START) { + pg_usleep(WAIT_FOR_STATE_CHANGE_TIME); + continue; + } + Assert(readerCxt->fileCount <= READER_THREAD_FILE_COUNT); + for (uint i = 0; i < readerCxt->fileCount; i++) { + copyFileToFileBuffer(readerCxt, i, arg); + } + setReaderState(readerCxt, READER_THREAD_STATE_FLUSHING); + } + return NULL; +} + +void copyFileToFileBuffer(ReaderCxt* readerCxt, int fileIndex, backup_files_arg* arg) +{ + pgFile* file = readerCxt->file[fileIndex]; + pgFile* prev_file = readerCxt->prefile[fileIndex]; + char* fileBuffer = readerCxt->fileBuffer + fileIndex * FILE_BUFFER_SIZE; + char* from_fullpath = readerCxt->fromPath[fileIndex]; + char* to_fullpath = readerCxt->toPath[fileIndex]; + if (file->is_datafile && !file->is_cfs) { + backup_data_file(&(arg->conn_arg), file, from_fullpath, to_fullpath, + arg->prev_start_lsn, + current.backup_mode, + instance_config.compress_alg, + instance_config.compress_level, + arg->nodeInfo->checksum_version, + arg->hdr_map, false, NULL, fileBuffer); + } else { + backup_non_data_file(file, prev_file, from_fullpath, to_fullpath, + current.backup_mode, current.parent_backup, true, NULL, fileBuffer); + } +} + +ReaderThreadState getReaderState(ReaderCxt* readerCxt) +{ + pthread_spin_lock(&readerCxt->lock); + ReaderThreadState state = readerCxt->state; + pthread_spin_unlock(&readerCxt->lock); + return state; +} + +void setReaderState(ReaderCxt* readerCxt, ReaderThreadState state) +{ + pthread_spin_lock(&readerCxt->lock); + readerCxt->state = state; + pthread_spin_unlock(&readerCxt->lock); +} + +void setSenderState(SenderCxt* senderCxt, SenderThreadState state) +{ + pthread_spin_lock(&senderCxt->lock); + senderCxt->state = state; + pthread_spin_unlock(&senderCxt->lock); +} + +SenderThreadState getSenderState(SenderCxt* senderCxt) +{ + pthread_spin_lock(&senderCxt->lock); + SenderThreadState state = senderCxt->state; + pthread_spin_unlock(&senderCxt->lock); + return state; +} + +void flushReaderContexts(void* arg) +{ + ReaderCxt* readerCxt = NULL; + char* fileBuffer = NULL; + backup_files_arg* args = (backup_files_arg*)arg; + waitForReadersCopyComplete(); + for (uint i = 0; i < current.readerThreadCount; i++) { + readerCxt = ¤t.readerCxt[i]; + for (uint j = 0; j < readerCxt->fileCount; j++) { + if (!readerCxt->fileRemoved[j]) { + fileBuffer = readerCxt->fileBuffer + j * FILE_BUFFER_SIZE; + if (readerCxt->file[j]->is_datafile && !readerCxt->file[j]->is_cfs) { + backup_data_file(&(args->conn_arg), readerCxt->file[j], readerCxt->fromPath[j], readerCxt->toPath[j], + args->prev_start_lsn, + current.backup_mode, + instance_config.compress_alg, + instance_config.compress_level, + args->nodeInfo->checksum_version, + args->hdr_map, false, readerCxt->appender, NULL); + } else { + backup_non_data_file(readerCxt->file[j], readerCxt->prefile[j], readerCxt->fromPath[j], readerCxt->toPath[j], + current.backup_mode, current.parent_backup, true, readerCxt->appender, NULL); + } + pg_free(readerCxt->fromPath[j]); + pg_free(readerCxt->toPath[j]); + readerCxt->fileRemoved[j] = true; + readerCxt->file[j] = NULL; + readerCxt->prefile[j] = NULL; + } + } + readerCxt->fileCount = 0; + setReaderState(readerCxt, READER_THREAD_STATE_FLUSHED); + } +} + +void waitForReadersCopyComplete() +{ + ReaderCxt* readerCxt = NULL; + for (uint i = 0; i < current.readerThreadCount; i++) { + readerCxt = ¤t.readerCxt[i]; + if (getReaderState(readerCxt) == READER_THREAD_STATE_INIT || + getReaderState(readerCxt) == READER_THREAD_STATE_FLUSHED) { + setReaderState(readerCxt, READER_THREAD_STATE_START); + } + } + for (uint i = 0; i < current.readerThreadCount; i++) { + readerCxt = ¤t.readerCxt[i]; + if (getReaderState(readerCxt) == READER_THREAD_STATE_START) { + pg_usleep(WAIT_FOR_STATE_CHANGE_TIME); + i = i - 1; + continue; + } + } +} + +void waitForSenderThread() +{ + SenderCxt* senderCxt = current.sender_cxt; + while (getSenderState(senderCxt) != SENDER_THREAD_STATE_FINISHED) { + pg_usleep(WAIT_FOR_STATE_CHANGE_TIME); + } +} + +void stopBackupReaders() +{ + ReaderCxt* readerCxt = NULL; + for (uint i = 0; i < current.readerThreadCount; i++) { + readerCxt = ¤t.readerCxt[i]; + setReaderState(readerCxt, READER_THREAD_STATE_STOP); + } + for (uint i = 0; i < current.readerThreadCount; i++) { + readerCxt = ¤t.readerCxt[i]; + pthread_join(readerCxt->readerThreadId, NULL); + } +} + +void stopBackupSender() +{ + SenderCxt* senderCxt = current.sender_cxt; + setSenderState(senderCxt, SENDER_THREAD_STATE_STOP); + pthread_join(senderCxt->senderThreadId, NULL); +} + +int getFreeReaderThread() +{ + int slot = -1; + ReaderCxt* readerCxt = NULL; + for (size_t i = 0; i < current.readerThreadCount; i++) { + readerCxt = ¤t.readerCxt[i]; + if (getReaderState(readerCxt) == READER_THREAD_STATE_INIT || + getReaderState(readerCxt) == READER_THREAD_STATE_FLUSHED) { + slot = i; + break; + } + } + return slot; +} + +void* restoreReaderThreadMain(void* arg) +{ + restoreReaderThreadArgs* args = (restoreReaderThreadArgs*)arg; + /* get pbk file from oss server */ + Oss::Oss* oss = getOssClient(); + const int object_suffix_len = 4; + char* object_name = NULL; + char* prefix_name = getPrefixName(args->dest_backup); + char* bucket_name = getBucketName(); + if (bucket_name == NULL || !oss->BucketExists(bucket_name)) { + elog(ERROR, "bucket %s not found, please create it first", bucket_name ? bucket_name : "null"); + } + parray* objects = parray_new(); + oss->ListObjectsWithPrefix(bucket_name, prefix_name, objects); + size_t objects_num = parray_num(objects); + size_t pbk_objects_num = 0; + for(size_t i = 0; i < objects_num; ++i) { + object_name = (char*)parray_get(objects, i); + if (strncmp(object_name + strlen(object_name) - object_suffix_len, ".pbk", object_suffix_len) == 0) { + pbk_objects_num++; + } + } + args->bufferCxt->fileNum = pbk_objects_num; + elog(INFO, "the total number of backup %s's file objects is %d, and pbk file objects is %d", + base36enc(args->dest_backup->start_time), objects_num, pbk_objects_num); + for(size_t i = 0; i < objects_num; ++i) { + if (args->bufferCxt->earlyExit) { + break; + } + object_name = (char*)parray_get(objects, i); + elog(INFO, "download object: %s from s3", object_name); + if (strncmp(object_name + strlen(object_name) - object_suffix_len, ".pbk", object_suffix_len) == 0) { + args->bufferCxt->fileEnd = false; + oss->GetObject(bucket_name, object_name, (void*)args->bufferCxt); + } else { + char file_name[MAXPGPATH]; + int rc = snprintf_s(file_name, MAXPGPATH, MAXPGPATH - 1, "/%s", object_name); + securec_check_ss_c(rc, "\0", "\0"); + oss->GetObject(bucket_name, object_name, (char*)file_name); + } + } + parray_free(objects); + return NULL; +} \ No newline at end of file diff --git a/src/bin/pg_probackup/pg_probackup.cpp b/src/bin/pg_probackup/pg_probackup.cpp index 362aff48a..bfd1dc484 100644 --- a/src/bin/pg_probackup/pg_probackup.cpp +++ b/src/bin/pg_probackup/pg_probackup.cpp @@ -25,6 +25,7 @@ #include "storage/file/fio_device.h" #include "storage/dss/dss_adaptor.h" #include +#include "oss/include/restore.h" #define MIN_ULIMIT_STACK_SIZE 8388608 // 1024 * 1024 * 8 @@ -87,6 +88,7 @@ bool backup_replslots = false; bool smooth_checkpoint; char *remote_agent; static char *backup_note = NULL; +static char *oss_status_string = NULL; /* restore options */ static char *target_time = NULL; static char *target_xid = NULL; @@ -148,11 +150,15 @@ static pgSetBackupParams *set_backup_params = NULL; pgBackup current; static ProbackupSubcmd backup_subcmd = NO_CMD; +/* Oss Client*/ +void* oss_client = NULL; + static bool help_opt = false; static void opt_incr_restore_mode(ConfigOption *opt, const char *arg); static void opt_backup_mode(ConfigOption *opt, const char *arg); static void opt_show_format(ConfigOption *opt, const char *arg); +static void opt_media_type(ConfigOption *opt, const char *arg); static void compress_init(void); static void dss_init(void); @@ -222,6 +228,8 @@ static ConfigOption cmd_options[] = { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, + { 'f', 'M', "media-type", (void *)opt_media_type, SOURCE_CMD_STRICT }, + { 's', 241, "s3-status", &oss_status_string, SOURCE_CMD_STRICT }, { 0 } }; @@ -445,7 +453,13 @@ static void parse_instance_name() { join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); + if (current.media_type == MEDIA_TYPE_OSS) { + restoreConfigFile(path); + } config_read_opt(path, instance_options, ERROR, true, false); + if (current.media_type == MEDIA_TYPE_OSS) { + remove(path); + } } setMyLocation(); } @@ -610,6 +624,7 @@ static void parse_backup_option_to_params(char *command, char *command_name) if (backup_subcmd == SET_BACKUP_CMD || backup_subcmd == BACKUP_CMD) { time_t expire_time = 0; + oss_status_t oss_status = OSS_STATUS_INVALID; if (expire_time_string && ttl >= 0) elog(ERROR, "You cannot specify '--expire-time' and '--ttl' options together"); @@ -622,12 +637,17 @@ static void parse_backup_option_to_params(char *command, char *command_name) expire_time_string); } - if (expire_time > 0 || ttl >= 0 || backup_note) + if (oss_status_string) { + oss_status = str2ossStatus(oss_status_string); + } + + if (expire_time > 0 || ttl >= 0 || backup_note || oss_status_string) { set_backup_params = pgut_new(pgSetBackupParams); set_backup_params->ttl = ttl; set_backup_params->expire_time = expire_time; set_backup_params->note = backup_note; + set_backup_params->oss_status = oss_status; if (backup_note && strlen(backup_note) > MAX_NOTE_SIZE) elog(ERROR, "Backup note cannot exceed %u bytes", MAX_NOTE_SIZE); @@ -918,6 +938,12 @@ opt_backup_mode(ConfigOption *opt, const char *arg) current.backup_mode = parse_backup_mode(arg); } +static void +opt_media_type(ConfigOption *opt, const char *arg) +{ + current.media_type = parse_media_type(arg); +} + static void opt_show_format(ConfigOption *opt, const char *arg) { diff --git a/src/bin/pg_probackup/pg_probackupb.h b/src/bin/pg_probackup/pg_probackupb.h index 56d19a24f..9144f8fee 100644 --- a/src/bin/pg_probackup/pg_probackupb.h +++ b/src/bin/pg_probackup/pg_probackupb.h @@ -101,6 +101,12 @@ typedef enum ShowFormat SHOW_JSON } ShowFormat; +typedef enum MediaType { + MEDIA_TYPE_UNKNOWN = 0, + MEDIA_TYPE_DISK, + MEDIA_TYPE_OSS +} MediaType; + /* special values of pgBackup fields */ #define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ @@ -174,6 +180,9 @@ typedef struct InstanceConfig /* DSS conntct parameters */ DssOptions dss; + + /* OSS parameters*/ + OssOptions oss; } InstanceConfig; extern ConfigOption instance_options[]; @@ -204,6 +213,9 @@ typedef struct HeaderMap } HeaderMap; +struct SenderCxt; +struct ReaderCxt; + typedef struct pgBackup pgBackup; /* Information about single backup stored in backup.conf */ @@ -291,6 +303,17 @@ struct pgBackup /* device type */ device_type_t storage_type; + + /* media type */ + MediaType media_type; + /* local or oss */ + oss_status_t oss_status; + /* sender context */ + SenderCxt* sender_cxt; + parray* filesinfo; + /* reader count and context */ + uint32 readerThreadCount; + ReaderCxt* readerCxt; }; /* Recovery target for restore and validate subcommands */ @@ -339,6 +362,7 @@ typedef struct pgSetBackupParams * must be pinned. */ char *note; + oss_status_t oss_status; } pgSetBackupParams; typedef struct @@ -444,6 +468,28 @@ typedef struct BackupPageHeader2 uint16 checksum; } BackupPageHeader2; +typedef enum FILE_APPEND_SEG_TYPE +{ + FILE_APPEND_TYPE_UNKNOWN = 0, + FILE_APPEND_TYPE_FILES, + FILE_APPEND_TYPE_DIR, + FILE_APPEND_TYPE_FILE, + FILE_APPEND_TYPE_FILE_CONTENT, + FILE_APPEND_TYPE_FILE_END, + FILE_APPEND_TYPE_FILES_END +} FILE_APPEND_SEG_TYPE; + +typedef struct FileAppender { + void* filePtr; /* hold the buffer context handle */ + char* baseFileName; + char* currFileName; + uint32 fileNo; + uint32 minFileNo; + uint32 maxFileNo; + uint64 currFileSize; + FILE_APPEND_SEG_TYPE type; +} FileAppender; + /* Special value for compressed_size field */ #define PageIsOk 0 #define SkipCurrentPage -1 diff --git a/src/bin/pg_probackup/pg_probackupc.h b/src/bin/pg_probackup/pg_probackupc.h index 93408264c..095c73ec5 100644 --- a/src/bin/pg_probackup/pg_probackupc.h +++ b/src/bin/pg_probackup/pg_probackupc.h @@ -88,6 +88,9 @@ extern bool skip_block_validation; /* current settings */ extern pgBackup current; +/* Oss Client*/ +extern void* oss_client; + /* argv of the process */ extern char** commands_args; @@ -100,6 +103,7 @@ extern int do_backup(time_t start_time, pgSetBackupParams *set_backup_params, bool no_validate, bool no_sync, bool backup_logs, bool backup_replslots); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); +extern MediaType parse_media_type(const char *value); extern void process_block_change(ForkNumber forknum, const RelFileNode rnode, BlockNumber blkno); @@ -166,6 +170,7 @@ extern int do_validate_all(void); extern int validate_one_page(Page page, BlockNumber absolute_blkno, XLogRecPtr stop_lsn, PageState *page_st, uint32 checksum_version); +extern bool pre_check_backup(pgBackup *backup); /* return codes for validate_one_page */ /* TODO: use enum */ @@ -315,15 +320,16 @@ extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, - HeaderMap *hdr_map, bool missing_ok); + HeaderMap *hdr_map, bool missing_ok, + FileAppender* appender = NULL, char* fileBuffer = NULL); extern void backup_non_data_file(pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time, - bool missing_ok); + bool missing_ok, FileAppender* appender = NULL, char* fileBuffer = NULL); extern void backup_non_data_file_internal(const char *from_fullpath, fio_location from_location, const char *to_fullpath, pgFile *file, - bool missing_ok); + bool missing_ok, FileAppender* appender = NULL, char** fileBuffer = NULL); extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath, bool use_bitmap, PageState *checksum_map, @@ -404,7 +410,9 @@ extern void time2iso(char *buf, size_t len, time_t time); extern const char *status2str(BackupStatus status); extern BackupStatus str2status(const char *status); extern const char *dev2str(device_type_t type); +extern const char *ossStatus2str(oss_status_t status); extern device_type_t str2dev(const char *dev); +extern oss_status_t str2ossStatus(const char *status); extern const char *base36enc(long unsigned int value); extern char *base36enc_dup(long unsigned int value); extern long unsigned int base36dec(const char *text); @@ -436,18 +444,19 @@ extern FILE* open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 extern int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode); + BackupMode backup_mode, FileAppender* appender = NULL, char* fileBuffer = NULL); /* FIO */ extern void fio_delete(mode_t mode, const char *fullpath, fio_location location); extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, bool use_pagemap, BlockNumber *err_blknum, char **errormsg, - BackupPageHeader2 **headers); + BackupPageHeader2 **headers, FileAppender* appender = NULL, char** fileBuffer = NULL); /* return codes for fio_send_pages */ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, - pgFile *file, char **errormsg); + pgFile *file, char **errormsg, + FileAppender* appender = NULL, char** fileBuffer = NULL); extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num, @@ -496,4 +505,25 @@ void *gs_palloc0(Size size); char *gs_pstrdup(const char *in); void *gs_repalloc(void *pointer, Size size); +typedef struct +{ + const char *base_path; + const char *dss_path; + parray *files; + bool corrupted; + XLogRecPtr stop_lsn; + uint32 checksum_version; + uint32 backup_version; + BackupMode backup_mode; + const char *external_prefix; + HeaderMap *hdr_map; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; +} validate_files_arg; + + #endif /* PG_PROBACKUPC_H */ diff --git a/src/bin/pg_probackup/restore.cpp b/src/bin/pg_probackup/restore.cpp index 683252dcb..a323e0985 100644 --- a/src/bin/pg_probackup/restore.cpp +++ b/src/bin/pg_probackup/restore.cpp @@ -21,6 +21,7 @@ #include "catalog/catalog.h" #include "storage/file/fio_device.h" #include "logger.h" +#include "oss/include/restore.h" #define RESTORE_ARRAY_LEN 100 @@ -380,39 +381,45 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } /* validate datafiles only */ - pgBackupValidate(tmp_backup, params); - - /* After pgBackupValidate() only following backup - * states are possible: ERROR, RUNNING, CORRUPT and OK. - * Validate WAL only for OK, because there is no point - * in WAL validation for corrupted, errored or running backups. - */ - if (tmp_backup->status != BACKUP_STATUS_OK) - { - corrupted_backup = tmp_backup; - break; + if (current.media_type == MEDIA_TYPE_OSS && !params->is_restore && + tmp_backup->oss_status != OSS_STATUS_LOCAL) { + performRestoreOrValidate(tmp_backup, true); + } else if (current.media_type != MEDIA_TYPE_OSS || tmp_backup->oss_status == OSS_STATUS_LOCAL) { + pgBackupValidate(tmp_backup, params); + /* After pgBackupValidate() only following backup + * states are possible: ERROR, RUNNING, CORRUPT and OK. + * Validate WAL only for OK, because there is no point + * in WAL validation for corrupted, errored or running backups. + */ + if (tmp_backup->status != BACKUP_STATUS_OK) + { + corrupted_backup = tmp_backup; + break; + } + /* We do not validate WAL files of intermediate backups + * It`s done to speed up restore + */ } - /* We do not validate WAL files of intermediate backups - * It`s done to speed up restore - */ } /* There is no point in wal validation of corrupted backups */ // TODO: there should be a way for a user to request only(!) WAL validation - if (!corrupted_backup) - { - /* - * Validate corresponding WAL files. - * We pass base_full_backup timeline as last argument to this function, - * because it's needed to form the name of xlog file. - */ - validate_wal(dest_backup, arclog_path, rt->target_time, - rt->target_xid, rt->target_lsn, - dest_backup->tli, instance_config.xlog_seg_size); + if(current.media_type != MEDIA_TYPE_OSS || tmp_backup->oss_status == OSS_STATUS_LOCAL) { + if (!corrupted_backup) + { + /* + * Validate corresponding WAL files. + * We pass base_full_backup timeline as last argument to this function, + * because it's needed to form the name of xlog file. + */ + validate_wal(dest_backup, arclog_path, rt->target_time, + rt->target_xid, rt->target_lsn, + dest_backup->tli, instance_config.xlog_seg_size); + } + /* Orphanize every OK descendant of corrupted backup */ + else + set_orphan_status(backups, corrupted_backup); } - /* Orphanize every OK descendant of corrupted backup */ - else - set_orphan_status(backups, corrupted_backup); } /* @@ -449,6 +456,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->is_restore) { + restore_chain(dest_backup, parent_chain, params, instance_config.pgdata, instance_config.dss.vgdata, no_sync); @@ -1230,6 +1238,41 @@ static void threads_handle(pthread_t *threads, pthread_create(&progressThread, nullptr, ProgressReportRestore, nullptr); /* Restore files into target directory */ + if (current.media_type == MEDIA_TYPE_OSS) { + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + if (!lock_backup(backup, true)) { + elog(ERROR, "Cannot lock backup %s", base36enc(backup->start_time)); + } + if (backup->oss_status == OSS_STATUS_LOCAL) { + continue; + } + if (backup->status != BACKUP_STATUS_OK && + backup->status != BACKUP_STATUS_DONE) { + if (params->force) + elog(WARNING, "Backup %s is not valid, restore is forced", + base36enc(backup->start_time)); + else + elog(ERROR, "Backup %s cannot be restored because it is not valid", + base36enc(backup->start_time)); + } + /* confirm block size compatibility */ + if (backup->block_size != BLCKSZ) + elog(ERROR, + "BLCKSZ(%d) is not compatible(%d expected)", + backup->block_size, BLCKSZ); + if (backup->wal_block_size != XLOG_BLCKSZ) + elog(ERROR, + "XLOG_BLCKSZ(%d) is not compatible(%d expected)", + backup->wal_block_size, XLOG_BLCKSZ); + performRestoreOrValidate(backup, false); + /* Backup is downloaded. Update backup status */ + backup->end_time = time(NULL); + backup->oss_status = OSS_STATUS_LOCAL; + write_backup(backup, true); + } + } + for (i = 0; i < num_threads; i++) { restore_files_arg *arg = &(threads_args[i]); @@ -1397,7 +1440,6 @@ restore_files(void *arg) PageState *checksum_map = NULL; /* it should take ~1.5MB at most */ datapagemap_t *lsn_map = NULL; /* it should take 16kB at most */ pgFile *dest_file = (pgFile *)parray_get(arguments->dest_files, i); - /* Directories were created before */ if (S_ISDIR(dest_file->mode)) { directoryFilesLocal++; @@ -1504,8 +1546,6 @@ restore_files(void *arg) elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, strerror(errno)); - - // If destination file is 0 sized, then just close it and go for the next if (dest_file->write_size == 0) goto done; diff --git a/src/bin/pg_probackup/show.cpp b/src/bin/pg_probackup/show.cpp index d7fa02e91..249eec150 100644 --- a/src/bin/pg_probackup/show.cpp +++ b/src/bin/pg_probackup/show.cpp @@ -37,6 +37,7 @@ typedef struct ShowBackendRow char start_lsn[20]; char stop_lsn[20]; char type[20]; + char oss_status[20]; const char *status; } ShowBackendRow; @@ -572,16 +573,16 @@ static void process_time(pgBackup *backup, ShowBackendRow *row) static void show_instance_plain(const char *instance_name, device_type_t instance_type, parray *backup_list, bool show_name) { -#define SHOW_FIELDS_COUNT 15 +#define SHOW_FIELDS_COUNT 16 int i; const char *names[SHOW_FIELDS_COUNT] = { "Instance", "Version", "ID", "Recovery Time", "Mode", "WAL Mode", "TLI", "Time", "Data", "WAL", - "Zratio", "Start LSN", "Stop LSN", "Type", "Status" }; + "Zratio", "Start LSN", "Stop LSN", "Type", "S3 Status", "Status" }; const char *field_formats[SHOW_FIELDS_COUNT] = { " %-*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s ", " %*s ", " %*s ", " %*s ", - " %*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s "}; + " %*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s "}; uint32 widths[SHOW_FIELDS_COUNT]; uint32 widths_sum = 0; ShowBackendRow *rows = NULL; @@ -713,6 +714,12 @@ show_instance_plain(const char *instance_name, device_type_t instance_type, par widths[cur] = Max(widths[cur], (uint32)strlen(row->type)); cur++; + /* S3 Status (LOCAL OR S3) */ + rc = snprintf_s(row->oss_status, lengthof(row->oss_status), lengthof(row->oss_status) - 1, "%s", ossStatus2str(backup->oss_status)); + securec_check_ss_c(rc, "\0", "\0"); + widths[cur] = Max(widths[cur], (uint32)strlen(row->oss_status)); + cur++; + /* Status */ row->status = status2str(backup->status); widths[cur] = Max(widths[cur], strlen(row->status)); @@ -805,6 +812,10 @@ show_instance_plain(const char *instance_name, device_type_t instance_type, par row->type); cur++; + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + row->oss_status); + cur++; + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], row->status); cur++; diff --git a/src/bin/pg_probackup/util.cpp b/src/bin/pg_probackup/util.cpp index 1951c0e31..2cd684c6d 100644 --- a/src/bin/pg_probackup/util.cpp +++ b/src/bin/pg_probackup/util.cpp @@ -22,6 +22,7 @@ #include "tool_common.h" #include "common/fe_memutils.h" #include "storage/file/fio_device.h" +#include "oss/include/restore.h" static const char *statusName[] = { @@ -46,6 +47,14 @@ static const char *devTypeName[] = "UNKNOWN" }; +static const char *ossStatusName[] = +{ + "LOCAL", + "S3", + "UNKNOWN", + "UNKNOWN" +}; + uint32 NUM_65536 = 65536; uint32 NUM_10000 = 10000; @@ -733,6 +742,9 @@ copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, } else { writeControlFile(&ControlFile, to_fullpath, to_location); } + if (current.media_type == MEDIA_TYPE_OSS) { + uploadConfigFile(to_fullpath, to_fullpath); + } pg_free(buffer); } @@ -817,6 +829,11 @@ const char *dev2str(device_type_t type) return devTypeName[type]; } +const char *ossStatus2str(oss_status_t type) +{ + return ossStatusName[type]; +} + device_type_t str2dev(const char *dev) { for (int i = 0; i < (int)DEV_TYPE_NUM; i++) { @@ -826,6 +843,15 @@ device_type_t str2dev(const char *dev) return DEV_TYPE_INVALID; } +oss_status_t str2ossStatus(const char *status) +{ + for (int i = 0; i < (int)OSS_STATUS_NUM; i++) { + if (pg_strcasecmp(status, ossStatusName[i]) == 0) + return (oss_status_t)i; + } + return OSS_STATUS_INVALID; +} + bool datapagemap_is_set(datapagemap_t *map, BlockNumber blkno) { diff --git a/src/bin/pg_probackup/validate.cpp b/src/bin/pg_probackup/validate.cpp index 8faaa0d31..df24982c5 100644 --- a/src/bin/pg_probackup/validate.cpp +++ b/src/bin/pg_probackup/validate.cpp @@ -18,6 +18,7 @@ #include "common/fe_memutils.h" #include "storage/file/fio_device.h" #include "logger.h" +#include "oss/include/restore.h" static void *pgBackupValidateFiles(void *arg); static void do_validate_instance(void); @@ -25,26 +26,6 @@ static void do_validate_instance(void); static bool corrupted_backup_found = false; static bool skipped_due_to_lock = false; -typedef struct -{ - const char *base_path; - const char *dss_path; - parray *files; - bool corrupted; - XLogRecPtr stop_lsn; - uint32 checksum_version; - uint32 backup_version; - BackupMode backup_mode; - const char *external_prefix; - HeaderMap *hdr_map; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; -} validate_files_arg; - /* Progress Counter */ static int g_inregularFiles = 0; static int g_doneFiles = 0; @@ -111,7 +92,7 @@ bool pre_check_backup(pgBackup *backup) return false; } - /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ + /* Revalidation is attempted for DONE, ORPHAN, LOCAL and CORRUPT backups */ if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE && backup->status != BACKUP_STATUS_ORPHAN && @@ -669,7 +650,11 @@ do_validate_instance(void) continue; } /* Valiate backup files*/ - pgBackupValidate(current_backup, NULL); + if (current.media_type == MEDIA_TYPE_OSS && current.oss_status != OSS_STATUS_LOCAL) { + performRestoreOrValidate(current_backup, true); + } else if (current.media_type != MEDIA_TYPE_OSS || current.oss_status == OSS_STATUS_LOCAL) { + pgBackupValidate(current_backup, NULL); + } /* Validate corresponding WAL files */ if (current_backup->status == BACKUP_STATUS_OK) diff --git a/src/include/storage/file/fio_device_com.h b/src/include/storage/file/fio_device_com.h index 47faaf5d2..c5159fc91 100644 --- a/src/include/storage/file/fio_device_com.h +++ b/src/include/storage/file/fio_device_com.h @@ -38,6 +38,13 @@ typedef enum en_device_type { DEV_TYPE_INVALID } device_type_t; +typedef enum en_status_type { + OSS_STATUS_LOCAL = 0, + OSS_STATUS_OSS, + OSS_STATUS_NUM, + OSS_STATUS_INVALID +} oss_status_t; + extern bool g_enable_dss; extern uint64 XLogSegmentSize; diff --git a/src/include/tool_common.h b/src/include/tool_common.h index 5a2354cd4..a6cd75f1f 100644 --- a/src/include/tool_common.h +++ b/src/include/tool_common.h @@ -132,6 +132,15 @@ typedef struct DssOptions { char *socketpath; } DssOptions; +/* OSS conntct parameters */ +typedef struct OssOptions { + char *access_id; + char *access_key; + char *endpoint; + char *region; + char *access_bucket; +} OssOptions; + typedef struct SSInstanceConfig { DssOptions dss; } SSInstanceConfig;