From 54325d9faedb5d5c539c3e6a3bdd3c4faac8b425 Mon Sep 17 00:00:00 2001 From: mujinqiang <1165845907@qq.com> Date: Wed, 4 Aug 2021 09:51:04 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=88=E5=85=A5=E5=A2=9E=E5=8A=A0build?= =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=90=8E=E6=95=B0=E6=8D=AEflush=E8=90=BD?= =?UTF-8?q?=E7=9B=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/pg_ctl/backup.cpp | 7 ++ src/bin/pg_ctl/backup.h | 1 + src/bin/pg_ctl/pg_build.cpp | 177 ++++++++++++++++++++++++++++++++++++ src/bin/pg_ctl/pg_build.h | 2 + src/bin/pg_ctl/pg_ctl.cpp | 5 + 5 files changed, 192 insertions(+) diff --git a/src/bin/pg_ctl/backup.cpp b/src/bin/pg_ctl/backup.cpp index 52268fd9b..8ff699329 100644 --- a/src/bin/pg_ctl/backup.cpp +++ b/src/bin/pg_ctl/backup.cpp @@ -1436,6 +1436,13 @@ static void BaseBackup(const char* dirname, uint32 term) PQfinish(streamConn); streamConn = NULL; + /* fsync all data come from source */ + if (!no_need_fsync) { + show_full_build_process("starting fsync all files come from source."); + (void) fsync_pgdata(basedir); + show_full_build_process("finish fsync all files."); + } + /* delete dw file if exists, recreate it and write a page of zero */ backup_dw_file(dirname); show_full_build_process("build dummy dw file success"); diff --git a/src/bin/pg_ctl/backup.h b/src/bin/pg_ctl/backup.h index 101f5b2d9..8cc0075d9 100644 --- a/src/bin/pg_ctl/backup.h +++ b/src/bin/pg_ctl/backup.h @@ -9,6 +9,7 @@ extern int standby_connect_timeout; extern int standby_message_timeout; extern char* conn_str; +extern bool no_need_fsync; extern pid_t process_id; extern char* basedir; extern int bgpipe[2]; diff --git a/src/bin/pg_ctl/pg_build.cpp b/src/bin/pg_ctl/pg_build.cpp index 99b163882..f955f4e23 100644 --- a/src/bin/pg_ctl/pg_build.cpp +++ b/src/bin/pg_ctl/pg_build.cpp @@ -57,6 +57,8 @@ int g_replication_type = -1; #define RT_WITH_DUMMY_STANDBY 0 #define RT_WITH_MULTI_STANDBY 1 +static void walkdir(const char *path, int (*action) (const char *fname, bool isdir), bool process_symlinks); + int32 pg_atoi(const char* s, int size, int c) { long l; @@ -1452,3 +1454,178 @@ bool libpqRotateCbmFile(PGconn* connObj, XLogRecPtr lsn) return ec; } +/* + * Issue fsync recursively on PGDATA and all its contents. + * + * We fsync regular files and directories wherever they are, but we follow + * symlinks only for pg_wal (or pg_xlog) and immediately under pg_tblspc. + * Other symlinks are presumed to point at files we're not responsible for + * fsyncing, and might not have privileges to write at all. + * + */ +void fsync_pgdata(const char *pg_data) +{ + bool xlog_is_symlink = false; + char pg_xlog[MAXPGPATH] = {0}; + char pg_tblspc[MAXPGPATH] = {0}; + errno_t errorno = EOK; + + errorno = snprintf_s(pg_xlog, MAXPGPATH, MAXPGPATH - 1, "%s/pg_xlog", pg_data); + securec_check_ss_c(errorno, "\0", "\0"); + errorno = snprintf_s(pg_tblspc, MAXPGPATH, MAXPGPATH - 1, "%s/pg_tblspc", pg_data); + securec_check_ss_c(errorno, "\0", "\0"); + +#ifndef WIN32 + { + struct stat st; + + if (lstat(pg_xlog, &st) < 0) { + pg_log(PG_WARNING, _("could not stat file \"%s\": %m\n"), pg_xlog); + exit(1); + } + else if (S_ISLNK(st.st_mode)) + xlog_is_symlink = true; + } +#else + if (pgwin32_is_junction(pg_xlog)) + xlog_is_symlink = true; +#endif + + /* + * Now we do the fsync()s in the same order. + * + * The main call ignores symlinks, so in addition to specially processing + * pg_wal if it's a symlink, pg_tblspc has to be visited separately with + * process_symlinks = true. Note that if there are any plain directories + * in pg_tblspc, they'll get fsync'd twice. That's not an expected case + * so we don't worry about optimizing it. + */ + walkdir(pg_data, fsync_fname, false); + if (xlog_is_symlink) + walkdir(pg_xlog, fsync_fname, false); + walkdir(pg_tblspc, fsync_fname, true); +} + +/* + * walkdir: recursively walk a directory, applying the action to each + * regular file and directory (including the named directory itself). + * + * If process_symlinks is true, the action and recursion are also applied + * to regular files and directories that are pointed to by symlinks in the + * given directory; otherwise symlinks are ignored. Symlinks are always + * ignored in subdirectories, ie we intentionally don't pass down the + * process_symlinks flag to recursive calls. + * + * Errors are reported but not considered fatal. + * + * See also walkdir in fd.cpp, which is a backend version of this logic. + */ +static void walkdir(const char *path, int (*action) (const char *fname, bool isdir), bool process_symlinks) +{ + DIR *dir; + struct dirent *de = NULL; + errno_t errorno = EOK; + + dir = opendir(path); + if (dir == NULL) { + pg_log(PG_WARNING, _("could not open directory \"%s\": %m\n"), path); + return; + } + + while (errno = 0, (de = readdir(dir)) != NULL) { + char subpath[MAXPGPATH * 2] = {0}; + struct stat fst; + int sret; + + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + + if (strcmp(de->d_name, "pg_ctl.lock") == 0) { + continue; + } + errorno = snprintf_s(subpath, sizeof(subpath), sizeof(subpath) - 1, "%s/%s", path, de->d_name); + securec_check_ss_c(errorno, "\0", "\0"); + + if (process_symlinks) + sret = stat(subpath, &fst); + else + sret = lstat(subpath, &fst); + if (sret < 0) { + pg_log(PG_WARNING, _("could not stat file \"%s\": %m\n"), subpath); + continue; + } + + if (S_ISREG(fst.st_mode)) + (*action) (subpath, false); + else if (S_ISDIR(fst.st_mode)) + walkdir(subpath, action, false); + } + + if (errno) + pg_log(PG_WARNING, _("could not read directory \"%s\": %m\n"), path); + + (void)closedir(dir); + + /* + * It's important to fsync the destination directory itself as individual + * file fsyncs don't guarantee that the directory entry for the file is + * synced. Recent versions of ext4 have made the window much wider but + * it's been an issue for ext3 and other filesystems in the past. + */ + (*action) (path, true); +} + +/* + * fsync_fname -- Try to fsync a file or directory + * + * Ignores errors trying to open unreadable files, or trying to fsync + * directories on systems where that isn't allowed/required. All other errors + * are fatal. + */ +int fsync_fname(const char *fname, bool isdir) +{ + int fd = -1; + int flags; + int returncode; + + /* + * Some OSs require directories to be opened read-only whereas other + * systems don't allow us to fsync files opened read-only; so we need both + * cases here. Using O_RDWR will cause us to fail to fsync files that are + * not writable by our userid, but we assume that's OK. + */ + flags = PG_BINARY; + if (!isdir) + flags |= O_RDWR; + else + flags |= O_RDONLY; + + /* + * Open the file, silently ignoring errors about unreadable files (or + * unsupported operations, e.g. opening a directory under Windows), and + * logging others. + */ + fd = open(fname, flags, 0); + if (fd < 0) { + if (errno == EACCES || (isdir && errno == EISDIR)) + return 0; + pg_log(PG_WARNING, _("could not open file \"%s\": %m\n"), fname); + return -1; + } + + returncode = fsync(fd); + + /* + * Some OSes don't allow us to fsync directories at all, so we can ignore + * those errors. Anything else needs to be reported. + */ + if (returncode != 0 && !(isdir && (errno == EBADF || errno == EINVAL))) { + pg_log(PG_WARNING, _("could not fsync file \"%s\": %m\n"), fname); + (void) close(fd); + exit(EXIT_FAILURE); + } + + (void) close(fd); + return 0; +} + diff --git a/src/bin/pg_ctl/pg_build.h b/src/bin/pg_ctl/pg_build.h index e87704143..40a67319e 100644 --- a/src/bin/pg_ctl/pg_build.h +++ b/src/bin/pg_ctl/pg_build.h @@ -60,5 +60,7 @@ extern char* pg_strdup(const char* in); extern void pg_free(void* ptr); extern void get_slot_name(char* slotname, size_t len); extern bool libpqRotateCbmFile(PGconn* connObj, XLogRecPtr lsn); +extern int fsync_fname(const char *fname, bool isdir); +extern void fsync_pgdata(const char *pg_data); #endif /* PG_BUILD_H */ diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 4d7ff06ef..69b9f1095 100644 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -184,6 +184,7 @@ char gaussdb_state_file[MAXPGPATH] = {0}; static char postport_lock_file[MAXPGPATH]; static PGconn* dbConn = NULL; +bool no_need_fsync = false; pid_t process_id = 0; const int g_length_stop_char = 2; @@ -4637,6 +4638,7 @@ int main(int argc, char** argv) {"connect-string", required_argument, NULL, 'C'}, {"remove-backup", no_argument, NULL, 1}, {"action", required_argument, NULL, 'a'}, + {"no-fsync", no_argument, NULL, 3}, {NULL, 0, NULL, 0}}; int option_index; @@ -4923,6 +4925,9 @@ int main(int argc, char** argv) case 1: clear_backup_dir = true; break; + case 3: + no_need_fsync = true; + break; default: /* getopt_long already issued a suitable error message */ do_advice();