database/openGauss-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

!1678 openGauss 发布订阅支持gs_probackup备份恢复后连接不断开

Browse Source 
Merge pull request !1678 from chenxiaobin/probackup
This commit is contained in:
opengauss-bot
2022-04-22 02:42:08 +00:00
committed by Gitee
parent 2250adfd4b 3f7a909887
commit eb4e54d4eb
8 changed files with 141 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/bin/pg_probackup/backup.cpp
View File

@ -31,6 +31,9 @@
it will be backuped up in external dirs */
parray *pgdata_nobackup_dir = NULL;
/* list of logical replication slots */
parray *logical_replslot = NULL;
static int standby_message_timeout_local = 10 ; /* 10 sec = default */
static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr;
static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr;
@ -89,10 +92,11 @@ static void backup_cleanup(bool fatal, void *userdata);
static void *backup_files(void *arg);
static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs);
static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs,
bool backup_replslots);
static void pg_start_backup(const char *label, bool smooth, pgBackup *backup,
PGNodeInfo *nodeInfo, PGconn *conn);
PGNodeInfo *nodeInfo, PGconn *conn, bool backup_replslots);
static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo);
static int checkpoint_timeout(PGconn *backup_conn);
@ -558,7 +562,7 @@ static void sync_files(parray *database_map, const char *database_path, parray *
* Move files from 'pgdata' to a subdirectory in 'backup_path'.
*/
static void
do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs)
do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs, bool backup_replslots)
{
int i;
char database_path[MAXPGPATH];
@ -591,7 +595,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
securec_check_c(rc, "\0", "\0");
/* Call pg_start_backup function in openGauss connect */
pg_start_backup(label, smooth_checkpoint, &current, nodeInfo, backup_conn);
pg_start_backup(label, smooth_checkpoint, &current, nodeInfo, backup_conn, backup_replslots);
/* Obtain current timeline */
#if PG_VERSION_NUM >= 90600
@ -624,10 +628,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
/* list files with the logical path. omit $PGDATA */
if (fio_is_remote(FIO_DB_HOST))
fio_list_dir(backup_files_list, instance_config.pgdata,
true, true, false, backup_logs, true, 0);
true, true, false, backup_logs, true, 0, backup_replslots);
else
dir_list_file(backup_files_list, instance_config.pgdata,
true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST);
true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST, backup_replslots);
/*
* Get database_map (name to oid) for use in partial restore feature.
@ -749,6 +753,11 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
}
pgdata_nobackup_dir = NULL;
if (logical_replslot) {
free_dir_list(logical_replslot);
}
logical_replslot = NULL;
/* Cleanup */
if (backup_list)
{
@ -849,7 +858,7 @@ static void do_after_backup()
*/
int
do_backup(time_t start_time, pgSetBackupParams *set_backup_params,
bool no_validate, bool no_sync, bool backup_logs)
bool no_validate, bool no_sync, bool backup_logs, bool backup_replslots)
{
PGconn *backup_conn = NULL;
PGNodeInfo nodeInfo;
@ -925,7 +934,7 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params,
add_note(&current, set_backup_params->note);
/* backup data */
do_backup_instance(backup_conn, &nodeInfo, no_sync, backup_logs);
do_backup_instance(backup_conn, &nodeInfo, no_sync, backup_logs, backup_replslots);
pgut_atexit_pop(backup_cleanup, NULL);
/* compute size of wal files of this backup stored in the archive */
@ -1034,13 +1043,15 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz)
*/
static void
pg_start_backup(const char *label, bool smooth, pgBackup *backup,
PGNodeInfo *nodeInfo, PGconn *conn)
PGNodeInfo *nodeInfo, PGconn *conn, bool backup_replslots)
{
PGresult *res;
const char *params[2];
uint32 lsn_hi;
uint32 lsn_lo;
int ret;
int i;
XLogRecPtr startLsn;
params[0] = label;
@ -1068,7 +1079,33 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
XLogDataFromLSN(ret, PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo);
securec_check_for_sscanf_s(ret, 2, "\0", "\0");
/* Calculate LSN */
backup->start_lsn = ((uint64) lsn_hi )<< 32 | lsn_lo;
startLsn = ((uint64) lsn_hi )<< 32 | lsn_lo;
if (backup_replslots) {
logical_replslot = parray_new();
/* query for logical replication slots of subscriptions */
res = pgut_execute(conn,
"SELECT slot_name, restart_lsn FROM pg_catalog.pg_get_replication_slots()"
"WHERE slot_type = 'logical' AND plugin = 'pgoutput'", 0, NULL);
if (PQntuples(res) == 0) {
elog(LOG, "logical replication slots for subscriptions not found");
} else {
XLogRecPtr repslotLsn;
for (i = 0; i < PQntuples(res); i++) {
XLogDataFromLSN(ret, PQgetvalue(res, i, 1), &lsn_hi, &lsn_lo);
securec_check_for_sscanf_s(ret, 2, "\0", "\0");
repslotLsn = ((uint64) lsn_hi )<< 32 | lsn_lo;
startLsn = Min(startLsn, repslotLsn);
char* slotname = pg_strdup(PQgetvalue(res, i, 0));
parray_append(logical_replslot, slotname);
}
elog(WARNING, "logical replication slots for subscriptions will be backed up. "
"If don't use them after restoring, please drop them to avoid affecting xlog recycling.");
}
}
backup->start_lsn = startLsn;
PQclear(res);
}

81
src/bin/pg_probackup/dir.cpp
View File

@ -42,13 +42,6 @@ const char *pgdata_exclude_dir[] =
(const char *)"pg_stat_tmp",
(const char *)"pgsql_tmp",
/*
* It is generally not useful to backup the contents of this directory even
* if the intention is to restore to another master. See backup.sgml for a
* more detailed description.
*/
(const char *)"pg_replslot",
/* Contents removed on startup, see dsm_cleanup_for_mmap(). */
(const char *)"pg_dynshmem",
@ -68,7 +61,7 @@ const char *pgdata_exclude_dir[] =
(const char *)"pg_subtrans",
/* end of list */
NULL, /* pg_log will be set later */
NULL, /* pg_log and pg_replslot will be set later */
NULL
};
@ -128,17 +121,20 @@ may be removed int the future */
static int pgCompareString(const void *str1, const void *str2);
static char dir_check_file(pgFile *file, bool backup_logs);
static char dir_check_file(pgFile *file, bool backup_logs, bool backup_replslots);
static char check_in_tablespace(pgFile *file, bool in_tablespace);
static char check_db_dir(pgFile *file);
static char check_digit_file(pgFile *file);
static char check_nobackup_dir(pgFile *file);
static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
bool exclude, bool follow_symlink, bool backup_logs,
bool skip_hidden, int external_dir_num, fio_location location);
bool skip_hidden, int external_dir_num, fio_location location,
bool backup_replslots);
static void opt_path_map(ConfigOption *opt, const char *arg,
TablespaceList *list, const char *type);
char check_logical_replslot_dir(const char *rel_path);
/* Tablespace mapping */
static TablespaceList tablespace_dirs = {NULL, NULL};
/* Extra directories mapping */
@ -538,7 +534,7 @@ db_map_entry_free(void *entry)
void
dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink,
bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num,
fio_location location)
fio_location location, bool backup_replslots)
{
pgFile *file;
@ -565,7 +561,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink
parray_append(files, file);
dir_list_file_internal(files, file, root, exclude, follow_symlink,
backup_logs, skip_hidden, external_dir_num, location);
backup_logs, skip_hidden, external_dir_num, location, backup_replslots);
if (!add_root)
pgFileFree(file);
@ -589,7 +585,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink
* - datafiles
*/
static char
dir_check_file(pgFile *file, bool backup_logs)
dir_check_file(pgFile *file, bool backup_logs, bool backup_replslots)
{
int i;
int sscanf_res;
@ -652,6 +648,29 @@ dir_check_file(pgFile *file, bool backup_logs)
}
}
/*
* Backup pg_replslot if it is specified.
* It is generally not useful to backup the contents of this directory even
* if the intention is to restore to another master. See backup.sgml for a
* more detailed description.
*/
if (!backup_replslots) {
if (strcmp(file->rel_path, PG_REPLSLOT_DIR) == 0) {
/* Skip */
elog(VERBOSE, "Excluding directory content: %s", file->rel_path);
return CHECK_EXCLUDE_FALSE;
}
} else {
/*
* Check file that under pg_replslot and judge whether it
* belonged to logical replication slots for subscriptions.
*/
if (strcmp(file->rel_path, PG_REPLSLOT_DIR) != 0 &&
path_is_prefix_of_path(PG_REPLSLOT_DIR, file->rel_path)) {
return check_logical_replslot_dir(file->rel_path);
}
}
ret = check_nobackup_dir(file);
if (ret != -1) { /* -1 means need backup */
return ret;
@ -749,6 +768,35 @@ static char check_nobackup_dir(pgFile *file)
return ret;
}
char check_logical_replslot_dir(const char *rel_path)
{
char ret = CHECK_FALSE;
int i = 0;
char *tmp = pg_strdup(rel_path);
char *p;
#define DIRECTORY_DELIMITER "/"
if (logical_replslot) {
/* extract slot name from rel_path, such as sub1 from pg_replslot/sub1/snap */
p = strtok(tmp, DIRECTORY_DELIMITER);
if (p != NULL) {
p = strtok(NULL, DIRECTORY_DELIMITER);
}
for (i = 0; p != NULL && i < (int)parray_num(logical_replslot); i++) {
char *slotName = (char *)parray_get(logical_replslot, i);
if (strcmp(p, slotName) == 0) {
pfree(tmp);
return CHECK_TRUE;
}
}
} else {
ret = CHECK_TRUE;
}
pfree(tmp);
return ret;
}
static char check_db_dir(pgFile *file)
{
char ret = -1;
@ -889,7 +937,8 @@ bool SkipSomeDirFile(pgFile *file, struct dirent *dent, bool skipHidden)
static void
dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
bool exclude, bool follow_symlink, bool backup_logs,
bool skip_hidden, int external_dir_num, fio_location location)
bool skip_hidden, int external_dir_num, fio_location location,
bool backup_replslots)
{
DIR *dir;
struct dirent *dent;
@ -937,7 +986,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
if (exclude)
{
check_res = dir_check_file(file, backup_logs);
check_res = dir_check_file(file, backup_logs, backup_replslots);
if (check_res == CHECK_FALSE)
{
/* Skip */
@ -963,7 +1012,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
*/
if (S_ISDIR(file->mode))
dir_list_file_internal(files, file, child, exclude, follow_symlink,
backup_logs, skip_hidden, external_dir_num, location);
backup_logs, skip_hidden, external_dir_num, location, backup_replslots);
}
if (errno && errno != ENOENT)

15
src/bin/pg_probackup/file.cpp
View File

@ -51,6 +51,7 @@ typedef struct
bool exclusive_backup;
bool skip_hidden;
int external_dir_num;
bool backup_replslots;
} fio_list_dir_request;
typedef struct
@ -1794,7 +1795,7 @@ cleanup:
/* Compile the array of files located on remote machine in directory root */
void fio_list_dir(parray *files, const char *root, bool exclude,
bool follow_symlink, bool add_root, bool backup_logs,
bool skip_hidden, int external_dir_num)
bool skip_hidden, int external_dir_num, bool backup_replslots)
{
fio_header hdr;
fio_list_dir_request req;
@ -1811,6 +1812,7 @@ void fio_list_dir(parray *files, const char *root, bool exclude,
req.exclusive_backup = exclusive_backup;
req.skip_hidden = skip_hidden;
req.external_dir_num = external_dir_num;
req.backup_replslots = backup_replslots;
hdr.cop = FIO_LIST_DIR;
hdr.size = sizeof(req);
@ -1870,7 +1872,14 @@ void fio_list_dir(parray *files, const char *root, bool exclude,
securec_check_ss_c(nRet, "\0", "\0");
}
/*
* Check file that under pg_replslot and judge whether it
* belonged to logical replication slots for subscriptions.
*/
if (backup_replslots && strcmp(buf, PG_REPLSLOT_DIR) != 0 &&
path_is_prefix_of_path(PG_REPLSLOT_DIR, buf) && check_logical_replslot_dir(file->rel_path) != 1) {
continue;
}
parray_append(files, file);
}
@ -1914,7 +1923,7 @@ static void fio_list_dir_impl(int out, char* buf)
dir_list_file(file_files, req->path, req->exclude, req->follow_symlink,
req->add_root, req->backup_logs, req->skip_hidden,
req->external_dir_num, FIO_LOCAL_HOST);
req->external_dir_num, FIO_LOCAL_HOST, req->backup_replslots);
/* send information about files to the main process */
for (i = 0; i < (int)parray_num(file_files); i++)

2
src/bin/pg_probackup/file.h
View File

@ -163,5 +163,7 @@ extern z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence);
extern const char* fio_gzerror(gzFile file, int *errnum);
#endif
extern char check_logical_replslot_dir(const char *rel_path);
#endif

3
src/bin/pg_probackup/help.cpp
View File

@ -154,6 +154,7 @@ void help_pg_probackup(void)
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--ttl=interval] [--expire-time=time]\n"));
printf(_(" [--backup-pg-replslot]\n"));
printf(_(" [--help]\n"));
printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
@ -420,6 +421,7 @@ static void help_backup(void)
printf(_(" [--remote-port=port] [--ssh-options=ssh_options]\n"));
printf(_(" [--remote-libpath=libpath]\n"));
printf(_(" [--ttl=interval] [--expire-time=time]\n\n"));
printf(_(" [--backup-pg-replslot]\n"));
printf(_(" -B, --backup-path=backup-path location of the backup storage area\n"));
printf(_(" --instance=instance_name name of the instance\n"));
@ -441,6 +443,7 @@ static void help_backup(void)
printf(_(" --note=text add note to backup\n"));
printf(_(" (example: --note='backup before app update to v13.1')\n"));
printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n"));
printf(_(" --backup-pg-replslot] backup of '%s' directory\n"), PG_REPLSLOT_DIR);
printf(_("\n Logging options:\n"));
printf(_(" --log-level-console=log-level-console\n"));

4
src/bin/pg_probackup/pg_probackup.cpp
View File

@ -77,6 +77,7 @@ int rw_timeout = 0;
/* backup options */
bool backup_logs = false;
bool backup_replslots = false;
bool smooth_checkpoint;
char *remote_agent;
static char *backup_note = NULL;
@ -186,6 +187,7 @@ static ConfigOption cmd_options[] =
{ 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT },
{ 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT },
{ 's', 172, "status", &delete_status, SOURCE_CMD_STRICT },
{ 'b', 186, "backup-pg-replslot", &backup_replslots, SOURCE_CMD_STRICT},
{ 'b', 147, "force", &force, SOURCE_CMD_STRICT },
{ 'b', 148, "compress", &compress_shortcut, SOURCE_CMD_STRICT },
@ -550,7 +552,7 @@ static int do_actual_operate()
elog(ERROR, "required parameter not specified: BACKUP_MODE "
"(-b, --backup-mode)");
return do_backup(start_time, set_backup_params, no_validate, no_sync, backup_logs);
return do_backup(start_time, set_backup_params, no_validate, no_sync, backup_logs, backup_replslots);
}
case RESTORE_CMD:
return do_restore_or_validate(current.backup_id,

1
src/bin/pg_probackup/pg_probackupa.h
View File

@ -69,6 +69,7 @@ extern const char *PROGRAM_FULL_PATH;
#define HEADER_MAP "page_header_map"
#define HEADER_MAP_TMP "page_header_map_tmp"
#define PG_RELATIVE_TBLSPC_DIR "pg_location"
#define PG_REPLSLOT_DIR "pg_replslot"
/* Timeout defaults */
#define ARCHIVE_TIMEOUT_DEFAULT 300

11
src/bin/pg_probackup/pg_probackupc.h
View File

@ -54,6 +54,9 @@ extern bool smooth_checkpoint;
it will be backuped up in external dirs */
extern parray *pgdata_nobackup_dir;
/* list of logical replication slots */
extern parray *logical_replslot;
/* remote probackup options */
extern char* remote_agent;
@ -89,7 +92,7 @@ extern const char *pgdata_exclude_dir[];
/* in backup.c */
extern int do_backup(time_t start_time, pgSetBackupParams *set_backup_params,
bool no_validate, bool no_sync, bool backup_logs);
bool no_validate, bool no_sync, bool backup_logs, bool backup_replslots);
extern BackupMode parse_backup_mode(const char *value);
extern const char *deparse_backup_mode(BackupMode mode);
extern void process_block_change(ForkNumber forknum, const RelFileNode rnode,
@ -239,7 +242,8 @@ extern const char* deparse_compress_alg(int alg);
/* in dir.c */
extern void dir_list_file(parray *files, const char *root, bool exclude,
bool follow_symlink, bool add_root, bool backup_logs,
bool skip_hidden, int external_dir_num, fio_location location);
bool skip_hidden, int external_dir_num, fio_location location,
bool backup_replslots = false);
extern void create_data_directories(parray *dest_files,
const char *data_dir,
@ -432,7 +436,8 @@ extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FIL
pgFile *file, char **errormsg);
extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink,
bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num);
bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num,
bool backup_replslots = false);
extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict);