From 3f7a909887cde77644572812f128da6c9a6f94a7 Mon Sep 17 00:00:00 2001 From: chenxiaobin19 <1025221611@qq.com> Date: Mon, 18 Apr 2022 16:54:09 +0800 Subject: [PATCH] =?UTF-8?q?gs=5Fprobackup=E6=94=AF=E6=8C=81=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=8F=91=E5=B8=83=E8=AE=A2=E9=98=85=E7=9A=84=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E5=A4=8D=E5=88=B6=E6=A7=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_probackup/backup.cpp | 57 +++++++++++++++---- src/bin/pg_probackup/dir.cpp | 81 +++++++++++++++++++++------ src/bin/pg_probackup/file.cpp | 15 ++++- src/bin/pg_probackup/file.h | 2 + src/bin/pg_probackup/help.cpp | 3 + src/bin/pg_probackup/pg_probackup.cpp | 4 +- src/bin/pg_probackup/pg_probackupa.h | 1 + src/bin/pg_probackup/pg_probackupc.h | 11 +++- 8 files changed, 141 insertions(+), 33 deletions(-) diff --git a/src/bin/pg_probackup/backup.cpp b/src/bin/pg_probackup/backup.cpp index d37c57c13..b92d7ccb0 100644 --- a/src/bin/pg_probackup/backup.cpp +++ b/src/bin/pg_probackup/backup.cpp @@ -31,6 +31,9 @@ it will be backuped up in external dirs */ parray *pgdata_nobackup_dir = NULL; +/* list of logical replication slots */ +parray *logical_replslot = NULL; + static int standby_message_timeout_local = 10 ; /* 10 sec = default */ static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; @@ -89,10 +92,11 @@ static void backup_cleanup(bool fatal, void *userdata); static void *backup_files(void *arg); -static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); +static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs, + bool backup_replslots); static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, - PGNodeInfo *nodeInfo, PGconn *conn); + PGNodeInfo *nodeInfo, PGconn *conn, bool backup_replslots); static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); static int checkpoint_timeout(PGconn *backup_conn); @@ -558,7 +562,7 @@ static void sync_files(parray *database_map, const char *database_path, parray * * Move files from 'pgdata' to a subdirectory in 'backup_path'. */ static void -do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) +do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs, bool backup_replslots) { int i; char database_path[MAXPGPATH]; @@ -591,7 +595,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool securec_check_c(rc, "\0", "\0"); /* Call pg_start_backup function in openGauss connect */ - pg_start_backup(label, smooth_checkpoint, ¤t, nodeInfo, backup_conn); + pg_start_backup(label, smooth_checkpoint, ¤t, nodeInfo, backup_conn, backup_replslots); /* Obtain current timeline */ #if PG_VERSION_NUM >= 90600 @@ -624,10 +628,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* list files with the logical path. omit $PGDATA */ if (fio_is_remote(FIO_DB_HOST)) fio_list_dir(backup_files_list, instance_config.pgdata, - true, true, false, backup_logs, true, 0); + true, true, false, backup_logs, true, 0, backup_replslots); else dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); + true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST, backup_replslots); /* * Get database_map (name to oid) for use in partial restore feature. @@ -749,6 +753,11 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool } pgdata_nobackup_dir = NULL; + if (logical_replslot) { + free_dir_list(logical_replslot); + } + logical_replslot = NULL; + /* Cleanup */ if (backup_list) { @@ -849,7 +858,7 @@ static void do_after_backup() */ int do_backup(time_t start_time, pgSetBackupParams *set_backup_params, - bool no_validate, bool no_sync, bool backup_logs) + bool no_validate, bool no_sync, bool backup_logs, bool backup_replslots) { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; @@ -925,7 +934,7 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, add_note(¤t, set_backup_params->note); /* backup data */ - do_backup_instance(backup_conn, &nodeInfo, no_sync, backup_logs); + do_backup_instance(backup_conn, &nodeInfo, no_sync, backup_logs, backup_replslots); pgut_atexit_pop(backup_cleanup, NULL); /* compute size of wal files of this backup stored in the archive */ @@ -1034,13 +1043,15 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz) */ static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, - PGNodeInfo *nodeInfo, PGconn *conn) + PGNodeInfo *nodeInfo, PGconn *conn, bool backup_replslots) { PGresult *res; const char *params[2]; uint32 lsn_hi; uint32 lsn_lo; int ret; + int i; + XLogRecPtr startLsn; params[0] = label; @@ -1068,7 +1079,33 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, XLogDataFromLSN(ret, PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); securec_check_for_sscanf_s(ret, 2, "\0", "\0"); /* Calculate LSN */ - backup->start_lsn = ((uint64) lsn_hi )<< 32 | lsn_lo; + startLsn = ((uint64) lsn_hi )<< 32 | lsn_lo; + + if (backup_replslots) { + logical_replslot = parray_new(); + /* query for logical replication slots of subscriptions */ + res = pgut_execute(conn, + "SELECT slot_name, restart_lsn FROM pg_catalog.pg_get_replication_slots()" + "WHERE slot_type = 'logical' AND plugin = 'pgoutput'", 0, NULL); + if (PQntuples(res) == 0) { + elog(LOG, "logical replication slots for subscriptions not found"); + } else { + XLogRecPtr repslotLsn; + + for (i = 0; i < PQntuples(res); i++) { + XLogDataFromLSN(ret, PQgetvalue(res, i, 1), &lsn_hi, &lsn_lo); + securec_check_for_sscanf_s(ret, 2, "\0", "\0"); + repslotLsn = ((uint64) lsn_hi )<< 32 | lsn_lo; + startLsn = Min(startLsn, repslotLsn); + + char* slotname = pg_strdup(PQgetvalue(res, i, 0)); + parray_append(logical_replslot, slotname); + } + elog(WARNING, "logical replication slots for subscriptions will be backed up. " + "If don't use them after restoring, please drop them to avoid affecting xlog recycling."); + } + } + backup->start_lsn = startLsn; PQclear(res); } diff --git a/src/bin/pg_probackup/dir.cpp b/src/bin/pg_probackup/dir.cpp index 00079dd1d..ad259f860 100644 --- a/src/bin/pg_probackup/dir.cpp +++ b/src/bin/pg_probackup/dir.cpp @@ -42,13 +42,6 @@ const char *pgdata_exclude_dir[] = (const char *)"pg_stat_tmp", (const char *)"pgsql_tmp", - /* - * It is generally not useful to backup the contents of this directory even - * if the intention is to restore to another master. See backup.sgml for a - * more detailed description. - */ - (const char *)"pg_replslot", - /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ (const char *)"pg_dynshmem", @@ -68,7 +61,7 @@ const char *pgdata_exclude_dir[] = (const char *)"pg_subtrans", /* end of list */ - NULL, /* pg_log will be set later */ + NULL, /* pg_log and pg_replslot will be set later */ NULL }; @@ -128,17 +121,20 @@ may be removed int the future */ static int pgCompareString(const void *str1, const void *str2); -static char dir_check_file(pgFile *file, bool backup_logs); +static char dir_check_file(pgFile *file, bool backup_logs, bool backup_replslots); static char check_in_tablespace(pgFile *file, bool in_tablespace); static char check_db_dir(pgFile *file); static char check_digit_file(pgFile *file); static char check_nobackup_dir(pgFile *file); static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, bool exclude, bool follow_symlink, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location); + bool skip_hidden, int external_dir_num, fio_location location, + bool backup_replslots); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); +char check_logical_replslot_dir(const char *rel_path); + /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; /* Extra directories mapping */ @@ -538,7 +534,7 @@ db_map_entry_free(void *entry) void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num, - fio_location location) + fio_location location, bool backup_replslots) { pgFile *file; @@ -565,7 +561,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink parray_append(files, file); dir_list_file_internal(files, file, root, exclude, follow_symlink, - backup_logs, skip_hidden, external_dir_num, location); + backup_logs, skip_hidden, external_dir_num, location, backup_replslots); if (!add_root) pgFileFree(file); @@ -589,7 +585,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink * - datafiles */ static char -dir_check_file(pgFile *file, bool backup_logs) +dir_check_file(pgFile *file, bool backup_logs, bool backup_replslots) { int i; int sscanf_res; @@ -652,6 +648,29 @@ dir_check_file(pgFile *file, bool backup_logs) } } + /* + * Backup pg_replslot if it is specified. + * It is generally not useful to backup the contents of this directory even + * if the intention is to restore to another master. See backup.sgml for a + * more detailed description. + */ + if (!backup_replslots) { + if (strcmp(file->rel_path, PG_REPLSLOT_DIR) == 0) { + /* Skip */ + elog(VERBOSE, "Excluding directory content: %s", file->rel_path); + return CHECK_EXCLUDE_FALSE; + } + } else { + /* + * Check file that under pg_replslot and judge whether it + * belonged to logical replication slots for subscriptions. + */ + if (strcmp(file->rel_path, PG_REPLSLOT_DIR) != 0 && + path_is_prefix_of_path(PG_REPLSLOT_DIR, file->rel_path)) { + return check_logical_replslot_dir(file->rel_path); + } + } + ret = check_nobackup_dir(file); if (ret != -1) { /* -1 means need backup */ return ret; @@ -749,6 +768,35 @@ static char check_nobackup_dir(pgFile *file) return ret; } +char check_logical_replslot_dir(const char *rel_path) +{ + char ret = CHECK_FALSE; + int i = 0; + char *tmp = pg_strdup(rel_path); + char *p; +#define DIRECTORY_DELIMITER "/" + + if (logical_replslot) { + /* extract slot name from rel_path, such as sub1 from pg_replslot/sub1/snap */ + p = strtok(tmp, DIRECTORY_DELIMITER); + if (p != NULL) { + p = strtok(NULL, DIRECTORY_DELIMITER); + } + + for (i = 0; p != NULL && i < (int)parray_num(logical_replslot); i++) { + char *slotName = (char *)parray_get(logical_replslot, i); + if (strcmp(p, slotName) == 0) { + pfree(tmp); + return CHECK_TRUE; + } + } + } else { + ret = CHECK_TRUE; + } + pfree(tmp); + return ret; +} + static char check_db_dir(pgFile *file) { char ret = -1; @@ -889,7 +937,8 @@ bool SkipSomeDirFile(pgFile *file, struct dirent *dent, bool skipHidden) static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, bool exclude, bool follow_symlink, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location) + bool skip_hidden, int external_dir_num, fio_location location, + bool backup_replslots) { DIR *dir; struct dirent *dent; @@ -937,7 +986,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, if (exclude) { - check_res = dir_check_file(file, backup_logs); + check_res = dir_check_file(file, backup_logs, backup_replslots); if (check_res == CHECK_FALSE) { /* Skip */ @@ -963,7 +1012,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, */ if (S_ISDIR(file->mode)) dir_list_file_internal(files, file, child, exclude, follow_symlink, - backup_logs, skip_hidden, external_dir_num, location); + backup_logs, skip_hidden, external_dir_num, location, backup_replslots); } if (errno && errno != ENOENT) diff --git a/src/bin/pg_probackup/file.cpp b/src/bin/pg_probackup/file.cpp index 959619e36..2bf6bb2f4 100644 --- a/src/bin/pg_probackup/file.cpp +++ b/src/bin/pg_probackup/file.cpp @@ -51,6 +51,7 @@ typedef struct bool exclusive_backup; bool skip_hidden; int external_dir_num; + bool backup_replslots; } fio_list_dir_request; typedef struct @@ -1794,7 +1795,7 @@ cleanup: /* Compile the array of files located on remote machine in directory root */ void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num) + bool skip_hidden, int external_dir_num, bool backup_replslots) { fio_header hdr; fio_list_dir_request req; @@ -1811,6 +1812,7 @@ void fio_list_dir(parray *files, const char *root, bool exclude, req.exclusive_backup = exclusive_backup; req.skip_hidden = skip_hidden; req.external_dir_num = external_dir_num; + req.backup_replslots = backup_replslots; hdr.cop = FIO_LIST_DIR; hdr.size = sizeof(req); @@ -1870,7 +1872,14 @@ void fio_list_dir(parray *files, const char *root, bool exclude, securec_check_ss_c(nRet, "\0", "\0"); } - + /* + * Check file that under pg_replslot and judge whether it + * belonged to logical replication slots for subscriptions. + */ + if (backup_replslots && strcmp(buf, PG_REPLSLOT_DIR) != 0 && + path_is_prefix_of_path(PG_REPLSLOT_DIR, buf) && check_logical_replslot_dir(file->rel_path) != 1) { + continue; + } parray_append(files, file); } @@ -1914,7 +1923,7 @@ static void fio_list_dir_impl(int out, char* buf) dir_list_file(file_files, req->path, req->exclude, req->follow_symlink, req->add_root, req->backup_logs, req->skip_hidden, - req->external_dir_num, FIO_LOCAL_HOST); + req->external_dir_num, FIO_LOCAL_HOST, req->backup_replslots); /* send information about files to the main process */ for (i = 0; i < (int)parray_num(file_files); i++) diff --git a/src/bin/pg_probackup/file.h b/src/bin/pg_probackup/file.h index 47e6eae14..fe633e812 100644 --- a/src/bin/pg_probackup/file.h +++ b/src/bin/pg_probackup/file.h @@ -163,5 +163,7 @@ extern z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence); extern const char* fio_gzerror(gzFile file, int *errnum); #endif +extern char check_logical_replslot_dir(const char *rel_path); + #endif diff --git a/src/bin/pg_probackup/help.cpp b/src/bin/pg_probackup/help.cpp index 9d00020a3..1b4279188 100644 --- a/src/bin/pg_probackup/help.cpp +++ b/src/bin/pg_probackup/help.cpp @@ -154,6 +154,7 @@ void help_pg_probackup(void) printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n")); printf(_(" [--remote-libpath=libpath]\n")); printf(_(" [--ttl=interval] [--expire-time=time]\n")); + printf(_(" [--backup-pg-replslot]\n")); printf(_(" [--help]\n")); printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); @@ -420,6 +421,7 @@ static void help_backup(void) printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n")); printf(_(" [--remote-libpath=libpath]\n")); printf(_(" [--ttl=interval] [--expire-time=time]\n\n")); + printf(_(" [--backup-pg-replslot]\n")); printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); printf(_(" --instance=instance_name name of the instance\n")); @@ -441,6 +443,7 @@ static void help_backup(void) printf(_(" --note=text add note to backup\n")); printf(_(" (example: --note='backup before app update to v13.1')\n")); printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); + printf(_(" --backup-pg-replslot] backup of '%s' directory\n"), PG_REPLSLOT_DIR); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); diff --git a/src/bin/pg_probackup/pg_probackup.cpp b/src/bin/pg_probackup/pg_probackup.cpp index 0eebf801b..bbbe094ba 100644 --- a/src/bin/pg_probackup/pg_probackup.cpp +++ b/src/bin/pg_probackup/pg_probackup.cpp @@ -77,6 +77,7 @@ int rw_timeout = 0; /* backup options */ bool backup_logs = false; +bool backup_replslots = false; bool smooth_checkpoint; char *remote_agent; static char *backup_note = NULL; @@ -186,6 +187,7 @@ static ConfigOption cmd_options[] = { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, { 's', 172, "status", &delete_status, SOURCE_CMD_STRICT }, + { 'b', 186, "backup-pg-replslot", &backup_replslots, SOURCE_CMD_STRICT}, { 'b', 147, "force", &force, SOURCE_CMD_STRICT }, { 'b', 148, "compress", &compress_shortcut, SOURCE_CMD_STRICT }, @@ -550,7 +552,7 @@ static int do_actual_operate() elog(ERROR, "required parameter not specified: BACKUP_MODE " "(-b, --backup-mode)"); - return do_backup(start_time, set_backup_params, no_validate, no_sync, backup_logs); + return do_backup(start_time, set_backup_params, no_validate, no_sync, backup_logs, backup_replslots); } case RESTORE_CMD: return do_restore_or_validate(current.backup_id, diff --git a/src/bin/pg_probackup/pg_probackupa.h b/src/bin/pg_probackup/pg_probackupa.h index 7fe53ffca..d9def2cb8 100644 --- a/src/bin/pg_probackup/pg_probackupa.h +++ b/src/bin/pg_probackup/pg_probackupa.h @@ -69,6 +69,7 @@ extern const char *PROGRAM_FULL_PATH; #define HEADER_MAP "page_header_map" #define HEADER_MAP_TMP "page_header_map_tmp" #define PG_RELATIVE_TBLSPC_DIR "pg_location" +#define PG_REPLSLOT_DIR "pg_replslot" /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 diff --git a/src/bin/pg_probackup/pg_probackupc.h b/src/bin/pg_probackup/pg_probackupc.h index 70ffaa239..af44209b5 100644 --- a/src/bin/pg_probackup/pg_probackupc.h +++ b/src/bin/pg_probackup/pg_probackupc.h @@ -54,6 +54,9 @@ extern bool smooth_checkpoint; it will be backuped up in external dirs */ extern parray *pgdata_nobackup_dir; +/* list of logical replication slots */ +extern parray *logical_replslot; + /* remote probackup options */ extern char* remote_agent; @@ -89,7 +92,7 @@ extern const char *pgdata_exclude_dir[]; /* in backup.c */ extern int do_backup(time_t start_time, pgSetBackupParams *set_backup_params, - bool no_validate, bool no_sync, bool backup_logs); + bool no_validate, bool no_sync, bool backup_logs, bool backup_replslots); extern BackupMode parse_backup_mode(const char *value); extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, const RelFileNode rnode, @@ -239,7 +242,8 @@ extern const char* deparse_compress_alg(int alg); /* in dir.c */ extern void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location); + bool skip_hidden, int external_dir_num, fio_location location, + bool backup_replslots = false); extern void create_data_directories(parray *dest_files, const char *data_dir, @@ -432,7 +436,8 @@ extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FIL pgFile *file, char **errormsg); extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, - bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num); + bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num, + bool backup_replslots = false); extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict);