diff --git a/src/bin/pg_basebackup/pg_basebackup.cpp b/src/bin/pg_basebackup/pg_basebackup.cpp index 52217cf09..06a67edd5 100644 --- a/src/bin/pg_basebackup/pg_basebackup.cpp +++ b/src/bin/pg_basebackup/pg_basebackup.cpp @@ -52,7 +52,6 @@ typedef struct TablespaceList { /* Global options */ char* basedir = NULL; static TablespaceList tablespacee_dirs = {NULL, NULL}; -char* g_xlogOption = NULL; char format = 'p'; /* p(lain)/t(ar) */ char* label = "gs_basebackup base backup"; bool showprogress = false; @@ -627,7 +626,8 @@ static void ReceiveTarFile(PGconn* conn, PGresult* res, int rownum) } else #endif tarfile = stdout; - strcpy_s(filename, 1, "-"); + errorno = strcpy_s(filename, 1, "-"); + securec_check_c(errorno, "\0", "\0"); } else { #ifdef HAVE_LIBZ if (compresslevel != 0) { @@ -783,16 +783,20 @@ static void ReceiveTarFile(PGconn* conn, PGresult* res, int rownum) */ static const char* get_tablespace_mapping(const char* dir) { - TablespaceListCell* cell; + TablespaceListCell* cell = NULL; char canon_dir[MAXPGPATH]; + errno_t errorno = EOK; /* Canonicalize path for comparison consistency */ - strlcpy(canon_dir, dir, sizeof(canon_dir)); + errorno = strcpy_s(canon_dir, MAXPGPATH, dir); + securec_check_c(errorno, "\0", "\0"); canonicalize_path(canon_dir); - for (cell = tablespacee_dirs.head; cell; cell = cell->next) - if (strcmp(canon_dir, cell->old_dir) == 0) + for (cell = tablespacee_dirs.head; cell; cell = cell->next) { + if (strcmp(canon_dir, cell->old_dir) == 0) { return cell->new_dir; + } + } return dir; } @@ -928,7 +932,7 @@ static void ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum) /* * All files are padded up to 512 bytes */ - current_padding = ((current_len_left + TAR_FILE_PADDING) & ~TAR_FILE_PADDING) - current_len_left; + current_padding = PADDING_LEFT(current_len_left); /* * First part of header is zero terminated filename @@ -1134,21 +1138,14 @@ static void BaseBackup(void) */ res = PQexec(conn, "IDENTIFY_SYSTEM"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { - fprintf(stderr, - _("%s: could not send replication command \"%s\": %s"), - progname, - "IDENTIFY_SYSTEM", + fprintf(stderr, _("%s: could not send replication command \"%s\": %s"), progname, "IDENTIFY_SYSTEM", PQerrorMessage(conn)); disconnect_and_exit(1); } if (PQntuples(res) != 1 || PQnfields(res) < 3) { fprintf(stderr, _("%s: could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields\n"), - progname, - PQntuples(res), - PQnfields(res), - 1, - 3); + progname, PQntuples(res), PQnfields(res), 1, 3); disconnect_and_exit(1); } sysidentifier = strdup(PQgetvalue(res, 0, 0)); @@ -1163,23 +1160,14 @@ static void BaseBackup(void) * Start the actual backup */ PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i); - rc = snprintf_s(current_path, - sizeof(current_path), - sizeof(current_path) - 1, - "BASE_BACKUP LABEL '%s' %s %s %s %s %s", - escaped_label, - showprogress ? "PROGRESS" : "", - includewal && !streamwal ? "WAL" : "", - fastcheckpoint ? "FAST" : "", - includewal ? "NOWAIT" : "", + rc = snprintf_s(current_path, sizeof(current_path), sizeof(current_path) - 1, + "BASE_BACKUP LABEL '%s' %s %s %s %s %s", escaped_label, showprogress ? "PROGRESS" : "", + includewal && !streamwal ? "WAL" : "", fastcheckpoint ? "FAST" : "", includewal ? "NOWAIT" : "", format == 't' ? "TABLESPACE_MAP" : ""); securec_check_ss_c(rc, "", ""); if (PQsendQuery(conn, current_path) == 0) { - fprintf(stderr, - _("%s: could not send replication command \"%s\": %s"), - progname, - "BASE_BACKUP", + fprintf(stderr, _("%s: could not send replication command \"%s\": %s"), progname, "BASE_BACKUP", PQerrorMessage(conn)); free(sysidentifier); disconnect_and_exit(1); @@ -1276,8 +1264,8 @@ static void BaseBackup(void) rc = snprintf_s(prefix, MAXPGPATH, strlen(basedir) + 1, "%s/", basedir); securec_check_ss_c(rc, "\0", "\0"); } - rc = snprintf_s( - nodetablespacepath, MAXPGPATH, sizeof(nodetablespacepath) - 1, "%s%s", prefix, tablespacepath); + rc = snprintf_s(nodetablespacepath, MAXPGPATH, sizeof(nodetablespacepath) - 1, "%s%s", prefix, + tablespacepath); securec_check_ss_c(rc, "\0", "\0"); verify_dir_is_empty_or_create(nodetablespacepath); @@ -1543,7 +1531,7 @@ int main(int argc, char** argv) } else if (!strcmp(progname, "gs_tar")) { return GsTar(argc, argv); } else { - fprintf(stderr, _("unsupport progname: %s"), progname); + fprintf(stderr, _("unsupported progname: %s"), progname); return 0; } } @@ -1628,14 +1616,15 @@ static int GsTar(int argc, char** argv) uint64 current_padding = 0; errno_t errorno = EOK; - strncpy_s(current_path, MAXPGPATH, basedir, strlen(basedir)); + errorno = strncpy_s(current_path, MAXPGPATH, basedir, strlen(basedir)); + securec_check_c(errorno, "\0", "\0"); tarfile = fopen(tarfilename, "rb"); if (tarfile == NULL) { fprintf(stderr, _("WARNING: %s not found!\n"), filename); exit(1); } while (true) { - int r; + size_t r; if (copybuf != NULL) { PQfreemem(copybuf); @@ -1649,16 +1638,8 @@ static int GsTar(int argc, char** argv) } else { copybuf = (char*)xmalloc0(current_len_left); r = fread(copybuf, 1, current_len_left, tarfile); - // end of file - if ((uint64)r < current_len_left) { - if (file != NULL) { - fclose(file); - file = NULL; - } - break; - } } - if (r == -1) { + if (r == 0) { /* * End of chunk */ @@ -1667,9 +1648,6 @@ static int GsTar(int argc, char** argv) file = NULL; } break; - } else if (r == -2) { - fprintf(stderr, "%s: could not read file data: %s", progname, tarfilename); - return -1; } if (file == NULL) { /* new file */ @@ -1677,7 +1655,7 @@ static int GsTar(int argc, char** argv) /* No current file, so this must be the header for a new file */ if (r != TAR_BLOCK_SIZE) { - fprintf(stderr, "%s: invalid tar block header size: %d\n", progname, r); + fprintf(stderr, "%s: invalid tar block header size: %zu\n", progname, r); fclose(tarfile); return -1; } @@ -1685,19 +1663,21 @@ static int GsTar(int argc, char** argv) if (sscanf_s(copybuf + 1048, "%201o", ¤t_len_left) != 1) { fprintf(stderr, "%s: could not parse file size\n", progname); + fclose(tarfile); return -1; } /* Set permissions on the file */ if (sscanf_s(©buf[1024], "%07o ", (unsigned int*)&filemode) != 1) { fprintf(stderr, "%s: could not parse file mode\n", progname); + fclose(tarfile); return -1; } /* * All files are padded up to 512 bytes */ - current_padding = ((current_len_left + 511) & ~511) - current_len_left; + current_padding = PADDING_LEFT(current_len_left); /* * First part of header is zero terminated filename @@ -1707,6 +1687,7 @@ static int GsTar(int argc, char** argv) "%s: the copybuf/current_path file path including .. is unallowed: %s\n", progname, strerror(errno)); + fclose(tarfile); return -1; } errorno = snprintf_s(filename, sizeof(filename), sizeof(filename) - 1, "%s/%s", current_path, copybuf); @@ -1788,11 +1769,13 @@ static int GsTar(int argc, char** argv) filename, ©buf[1081], strerror(errno)); + fclose(tarfile); return -1; } } } else { pg_log(PG_WARNING, "unrecognized link indicator \"%c\"\n", copybuf[TAR_FILE_TYPE]); + fclose(tarfile); return -1; } continue; /* directory or link handled */ @@ -1828,21 +1811,22 @@ static int GsTar(int argc, char** argv) /* * Continuing blocks in existing file */ - if (current_len_left == 0 && (uint64)r == current_padding) { + if (current_len_left == 0 && r == current_padding) { /* * Received the padding block for this file, ignore it and * close the file, then move on to the next tar header. */ fclose(file); file = NULL; - totaldone += (uint64)r; - current_padding -= (uint64)r; + totaldone += r; + current_padding -= r; continue; } if (fwrite(copybuf, r, 1, file) != 1) { fprintf(stderr, "%s: could not write to file \"%s\": %s\n", progname, filename, strerror(errno)); fclose(file); + fclose(tarfile); file = NULL; return -1; } @@ -1863,16 +1847,16 @@ static int GsTar(int argc, char** argv) } /* loop over all data blocks */ /* delete dw file if exists, recreate it and write a page of zero */ backup_dw_file(basedir); + if (tarfile != NULL) { + fclose(tarfile); + tarfile = NULL; + } if (file != NULL) { fprintf(stderr, "%s: COPY stream ended before last file was finished\n", progname); fclose(file); file = NULL; return -1; } - if (tarfile != NULL) { - fclose(tarfile); - tarfile = NULL; - } return 0; } @@ -1959,20 +1943,17 @@ static int GsBaseBackup(int argc, char** argv) includewal = true; check_env_value_c(optarg); - g_xlogOption = xstrdup(optarg); - if (strcmp(g_xlogOption, "f") == 0 || strcmp(g_xlogOption, "fetch") == 0) + if (strcmp(optarg, "f") == 0 || strcmp(optarg, "fetch") == 0) streamwal = false; - else if (strcmp(g_xlogOption, "s") == 0 || strcmp(g_xlogOption, "stream") == 0) + else if (strcmp(optarg, "s") == 0 || strcmp(optarg, "stream") == 0) streamwal = true; else { fprintf(stderr, _("%s: invalid xlog-method option \"%s\", must be \"fetch\" or \"stream\"\n"), progname, optarg); - GS_FREE(g_xlogOption); exit(1); } - GS_FREE(g_xlogOption); break; case 'l': if (label != NULL && strcmp(label, "gs_basebackup base backup") != 0) { diff --git a/src/bin/pg_ctl/fetchmot.cpp b/src/bin/pg_ctl/fetchmot.cpp index f70910fcc..be512206d 100644 --- a/src/bin/pg_ctl/fetchmot.cpp +++ b/src/bin/pg_ctl/fetchmot.cpp @@ -66,7 +66,7 @@ static void MotReceiveAndAppendTarFile( int current_len_left = 0; int current_padding = 0; - errorno = strncpy_s(current_path, sizeof(current_path), basedir, sizeof(current_path) - 1); + errorno = strncpy_s(current_path, sizeof(current_path), basedir, strlen(basedir)); securec_check_c(errorno, "", ""); #ifdef HAVE_LIBZ @@ -98,7 +98,8 @@ static void MotReceiveAndAppendTarFile( } else #endif tarfile = stdout; - strcpy_s(filename, 1, "-"); + errorno = strcpy_s(filename, 1, "-"); + securec_check_c(errorno, "\0", "\0"); } else { #ifdef HAVE_LIBZ if (compresslevel != 0) { @@ -118,15 +119,11 @@ static void MotReceiveAndAppendTarFile( /* chkptName header */ copybuf = (char*)xmalloc0(TAR_BLOCK_SIZE); - int chkptDirLen = strlen(chkptName); - errorno = strcpy_s(copybuf + 2, chkptDirLen + 1, chkptName); - securec_check_ss_c(errorno, "", ""); - - copybuf[0] = '.'; - copybuf[1] = '/'; - copybuf[chkptDirLen + 2] = '/'; + errorno = snprintf_s(copybuf, TAR_BLOCK_SIZE, TAR_BLOCK_SIZE - 1, "./%s/", chkptName); copybuf[TAR_FILE_TYPE] = TAR_TYPE_DICTORY; + securec_check_ss_c(errorno, "", ""); + for (int i = 0; i < 11; i++) { copybuf[TAR_LEN_LEFT + i] = '0'; } @@ -210,7 +207,7 @@ static void MotReceiveAndAppendTarFile( /* * All files are padded up to 512 bytes */ - current_padding = ((current_len_left + TAR_FILE_PADDING) & ~TAR_FILE_PADDING) - current_len_left; + current_padding = PADDING_LEFT(current_len_left); /* * First part of header is zero terminated filename. * when getting a checkpoint, file name can be either @@ -236,7 +233,8 @@ static void MotReceiveAndAppendTarFile( if (filename[strlen(filename) - 1] == '/') { filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */ - strcpy_s(copybuf, strlen(filename), filename); + errorno = strcpy_s(copybuf, r, filename); + securec_check_c(errorno, "\0", "\0"); continue; /* directory or link handled */ } #ifdef HAVE_LIBZ @@ -405,7 +403,7 @@ static void MotReceiveAndUnpackTarFile(const char* basedir, const char* chkptNam /* * All files are padded up to 512 bytes */ - current_padding = ((current_len_left + 511) & ~511) - current_len_left; + current_padding = PADDING_LEFT(current_len_left); /* * First part of header is zero terminated filename. diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index a5d9cd964..3502d592d 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -167,7 +167,8 @@ THR_LOCAL bool redo_oldversion_xlog = false; /* * GUC support */ -struct config_enum_entry sync_method_options[] = {{"fsync", SYNC_METHOD_FSYNC, false}, +struct config_enum_entry sync_method_options[] = { + {"fsync", SYNC_METHOD_FSYNC, false}, #ifdef HAVE_FSYNC_WRITETHROUGH {"fsync_writethrough", SYNC_METHOD_FSYNC_WRITETHROUGH, false}, #endif @@ -180,7 +181,8 @@ struct config_enum_entry sync_method_options[] = {{"fsync", SYNC_METHOD_FSYNC, f #ifdef OPEN_DATASYNC_FLAG {"open_datasync", SYNC_METHOD_OPEN_DSYNC, false}, #endif - {NULL, 0, false}}; + {NULL, 0, false} +}; XLogRecPtr latestValidRecord = InvalidXLogRecPtr; XLogSegNo XlogRemoveSegPrimary = InvalidXLogSegPtr; @@ -12272,7 +12274,7 @@ XLogRecPtr do_pg_start_backup(const char* backupidstr, bool fast, char** labelfi int rllen; errno_t errorno = EOK; - errorno = memset_s(fullpath, MAXPGPATH, '\0', MAXPGPATH); + errorno = memset_s(fullpath, MAXPGPATH + PG_TBLSPCS, '\0', MAXPGPATH + PG_TBLSPCS); securec_check(errorno, "", ""); errorno = memset_s(linkpath, MAXPGPATH, '\0', MAXPGPATH); @@ -12282,7 +12284,9 @@ XLogRecPtr do_pg_start_backup(const char* backupidstr, bool fast, char** labelfi if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; - snprintf(fullpath, sizeof(fullpath), "pg_tblspc/%s", de->d_name); + errorno = + snprintf_s(fullpath, MAXPGPATH + PG_TBLSPCS, MAXPGPATH + PG_TBLSPCS - 1, "pg_tblspc/%s", de->d_name); + securec_check_ss(errorno, "\0", "\0"); #if defined(HAVE_READLINK) || defined(WIN32) rllen = readlink(fullpath, linkpath, sizeof(linkpath)); @@ -13792,7 +13796,7 @@ static bool read_tablespace_map(List** tablespaces) char tbsoid[MAXPGPATH]; char* tbslinkpath; char str[MAXPGPATH]; - int ch, prev_ch = -1, i = 0, n; + int ch, prevCh = -1, i = 0, n; /* * See if tablespace_map file is present @@ -13814,7 +13818,7 @@ static bool read_tablespace_map(List** tablespaces) * escape character that has been added in tablespace path during backup. */ while ((ch = fgetc(lfp)) != EOF) { - if ((ch == '\n' || ch == '\r') && prev_ch != '\\') { + if ((ch == '\n' || ch == '\r') && prevCh != '\\') { str[i] = '\0'; if (sscanf(str, "%s %n", tbsoid, &n) != 1) ereport(FATAL, @@ -13829,11 +13833,11 @@ static bool read_tablespace_map(List** tablespaces) *tablespaces = lappend(*tablespaces, ti); continue; - } else if ((ch == '\n' || ch == '\r') && prev_ch == '\\') + } else if ((ch == '\n' || ch == '\r') && prevCh == '\\') str[i - 1] = ch; else str[i++] = ch; - prev_ch = ch; + prevCh = ch; } if (ferror(lfp) || FreeFile(lfp)) diff --git a/src/gausskernel/storage/access/transam/xlogfuncs.cpp b/src/gausskernel/storage/access/transam/xlogfuncs.cpp index 95b77edcf..894d315ba 100755 --- a/src/gausskernel/storage/access/transam/xlogfuncs.cpp +++ b/src/gausskernel/storage/access/transam/xlogfuncs.cpp @@ -76,6 +76,9 @@ Datum pg_start_backup(PG_FUNCTION_ARGS) backupidstr = text_to_cstring(backupid); dir = AllocateDir("pg_tblspc"); + if (!dir) { + ereport(ERROR, (errmsg("could not open directory \"%s\": %m", "pg_tblspc"))); + } startpoint = do_pg_start_backup(backupidstr, fast, NULL, dir, NULL, NULL, false, true); errorno = snprintf_s(startxlogstr, diff --git a/src/gausskernel/storage/replication/basebackup.cpp b/src/gausskernel/storage/replication/basebackup.cpp index 81817f246..aa86d747c 100755 --- a/src/gausskernel/storage/replication/basebackup.cpp +++ b/src/gausskernel/storage/replication/basebackup.cpp @@ -68,6 +68,12 @@ const int MATCH_SIX = 6; * Size of each block sent into the tar stream for larger files. */ #define TAR_SEND_SIZE (32 * 1024) /* data send unit 32KB */ +#define EREPORT_WAL_NOT_FOUND(segno) \ + do { \ + char walErrorName[MAXFNAMELEN]; \ + XLogFileName(walErrorName, t_thrd.xlog_cxt.ThisTimeLineID, segno); \ + ereport(ERROR, (errmsg("could not find WAL file \"%s\"", walErrorName))); \ + } while (0) XLogRecPtr XlogCopyStartPtr = InvalidXLogRecPtr; @@ -409,6 +415,14 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir) * recycled before we get a chance to send it over. */ nWalFiles = list_length(walFileList); + /* + * There must be at least one xlog file in the pg_xlog directory, + * since we are doing backup-including-xlog. + */ + if (nWalFiles < 1) { + ereport(ERROR, (errmsg("could not find any WAL files"))); + } + walFiles = (char**)palloc0(nWalFiles * sizeof(char*)); i = 0; foreach (lc, walFileList) { @@ -416,24 +430,13 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir) } qsort(walFiles, nWalFiles, sizeof(char*), CompareWalFileNames); - /* - * There must be at least one xlog file in the pg_xlog directory, - * since we are doing backup-including-xlog. - */ - if (nWalFiles < 1) { - ereport(ERROR, (errmsg("could not find any WAL files"))); - } - /* * Sanity check: the first and last segment should cover startptr and * endptr, with no gaps in between. */ XLogFromFileName(walFiles[0], &tli, &segno); if (segno != startsegno) { - char startfname[MAXFNAMELEN]; - - XLogFileName(startfname, t_thrd.xlog_cxt.ThisTimeLineID, startsegno); - ereport(ERROR, (errmsg("could not find WAL file \"%s\"", startfname))); + EREPORT_WAL_NOT_FOUND(startsegno); } for (i = 0; i < nWalFiles; i++) { XLogSegNo currsegno = segno; @@ -441,17 +444,11 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir) XLogFromFileName(walFiles[i], &tli, &segno); if (!(nextsegno == segno || currsegno == segno)) { - char nextfname[MAXFNAMELEN]; - - XLogFileName(nextfname, t_thrd.xlog_cxt.ThisTimeLineID, nextsegno); - ereport(ERROR, (errmsg("could not find WAL file \"%s\"", nextfname))); + EREPORT_WAL_NOT_FOUND(nextsegno); } } if (segno != endsegno) { - char endfname[MAXFNAMELEN]; - - XLogFileName(endfname, t_thrd.xlog_cxt.ThisTimeLineID, endsegno); - ereport(ERROR, (errmsg("could not find WAL file \"%s\"", endfname))); + EREPORT_WAL_NOT_FOUND(endsegno); } /* Ok, we have everything we need. Send the WAL files. */ @@ -461,7 +458,8 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir) size_t cnt; pgoff_t len = 0; - snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]); + int rt = snprintf_s(pathbuf, MAXPGPATH,MAXPGPATH -1, XLOGDIR "/%s", walFiles[i]); + securec_check_ss_c(rt, "\0", "\0"); XLogFromFileName(walFiles[i], &tli, &segno); fp = AllocateFile(pathbuf, "rb"); @@ -533,8 +531,8 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir) foreach (lc, historyFileList) { char* fname = (char*)lfirst(lc); - snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname); - + int rt = snprintf_s(pathbuf, MAXPGPATH, MAXPGPATH-1, XLOGDIR "/%s", fname); + securec_check_ss_c(rt, "\0", "\0"); if (lstat(pathbuf, &statbuf) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", pathbuf))); @@ -705,8 +703,9 @@ static void parse_basebackup_options(List* options, basebackup_options* opt) opt->includewal = true; o_wal = true; } else if (strcmp(defel->defname, "tablespace_map") == 0) { - if (o_tablespace_map) + if (o_tablespace_map) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname))); + } opt->sendtblspcmapfile = true; o_tablespace_map = true; } else @@ -969,7 +968,7 @@ static void sendFileWithContent(const char* filename, const char* content) /* Pad to 512 byte boundary, per tar format requirements */ pad = ((len + 511) & ~511) - len; if (pad > 0) { - char buf[512]; + char buf[2560]; errno_t rc = 0; rc = memset_s(buf, sizeof(buf), 0, pad); @@ -1021,7 +1020,7 @@ int64 sendTablespace(const char* path, bool sizeonly) } if (!sizeonly) _tarWriteHeader(relativedirname, NULL, &statbuf); - size = 512; /* Size of the header just added */ + size = 2560; /* Size of the header just added */ /* Send all the files in the tablespace version directory */ size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true); @@ -1237,7 +1236,7 @@ static int64 sendDir( _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf); } } - size += 512; /* Size of the header just added */ + size += 2560; /* Size of the header just added */ if (!sizeonly) { #ifndef WIN32 if (S_ISLNK(statbuf.st_mode)) { @@ -1279,7 +1278,7 @@ static int64 sendDir( _tarWriteHeader("pg_xlog/archive_status", NULL, &statbuf); } } - size += 512; /* Size of the header just added */ + size += 2560; /* Size of the header just added */ continue; /* don't recurse into pg_xlog */ } /* Allow symbolic links in pg_tblspc only */ diff --git a/src/include/gs_tar_const.h b/src/include/gs_tar_const.h index ca610c579..c0de07eac 100644 --- a/src/include/gs_tar_const.h +++ b/src/include/gs_tar_const.h @@ -28,6 +28,7 @@ #include "zlib.h" #endif + /* gs_tar offset */ static const int TAR_LEN_LEFT = 1048; static const int TAR_FILE_PADDING = 511; /* All files are padded up to 512 bytes */ @@ -42,6 +43,8 @@ static const int TAR_READ_ERROR = -2; static const char TAR_TYPE_DICTORY = '5'; static const int FILE_PERMISSION = 16832; +#define PADDING_LEFT(current_len_left) (((current_len_left + TAR_FILE_PADDING) & ~TAR_FILE_PADDING) - current_len_left); + #ifdef HAVE_LIBZ static const char* get_gz_error(gzFile gzf) { diff --git a/src/test/regress/input/gs_basebackup.source b/src/test/regress/input/gs_basebackup.source new file mode 100644 index 000000000..450a59e68 --- /dev/null +++ b/src/test/regress/input/gs_basebackup.source @@ -0,0 +1,21 @@ +--restart_node +\! @abs_bindir@/gs_guc set -D @abs_srcdir@/tmp_check/datanode1/ -c "enable_incremental_checkpoint=off" +\! @abs_bindir@/gs_ctl restart -D @abs_srcdir@/tmp_check/datanode1/ > @abs_bindir@/../datanode1.restart.log 2>&1 + +---prepare data +\! @abs_bindir@/gsql -dpostgres -p @portstring@ -c "create database gs_basebackup;" +\! @abs_bindir@/gsql -dgs_basebackup -p @portstring@ -f "@abs_srcdir@/sql/gs_basebackup/init/tablespace.sql"; +\! @abs_bindir@/gsql -dgs_basebackup -p @portstring@ -f "@abs_srcdir@/sql/gs_basebackup/init/mot.sql"; + +--pre +\! mkdir @abs_bindir@/../gs_basebackup_node_stream_p +\! chmod 700 @abs_bindir@/../gs_basebackup_node_stream_p +\! mkdir @abs_bindir@/../gs_basebackup_node_fetch_t +\! chmod 700 @abs_bindir@/../gs_basebackup_node_fetch_t + +--run +\! chmod +x @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh +\! @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh @abs_bindir@ @abs_srcdir@ @portstring@ gs_basebackup_node_stream_p stream p +\! @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh @abs_bindir@ @abs_srcdir@ @portstring@ gs_basebackup_node_fetch_t fetch t + + diff --git a/src/test/regress/output/gs_basebackup.source b/src/test/regress/output/gs_basebackup.source new file mode 100644 index 000000000..b7ef8a60f --- /dev/null +++ b/src/test/regress/output/gs_basebackup.source @@ -0,0 +1,88 @@ +--restart_node +\! @abs_bindir@/gs_guc set -D @abs_srcdir@/tmp_check/datanode1/ -c "enable_incremental_checkpoint=off" +--?expected instance path: [.*/postgresql.conf] +--?gs_guc set: enable_incremental_checkpoint=off: .*postgresql.conf.* + +Total instances: 1. Failed instances: 0. +Success to perform gs_guc! + +\! @abs_bindir@/gs_ctl restart -D @abs_srcdir@/tmp_check/datanode1/ > @abs_bindir@/../datanode1.restart.log 2>&1 +---prepare data +\! @abs_bindir@/gsql -dpostgres -p @portstring@ -c "create database gs_basebackup;" +CREATE DATABASE +\! @abs_bindir@/gsql -dgs_basebackup -p @portstring@ -f "@abs_srcdir@/sql/gs_basebackup/init/tablespace.sql"; +CREATE TABLESPACE +CREATE TABLE +INSERT 0 1 +--?total time: .* ms +\! @abs_bindir@/gsql -dgs_basebackup -p @portstring@ -f "@abs_srcdir@/sql/gs_basebackup/init/mot.sql"; +CREATE FOREIGN TABLE +INSERT 0 1 +--?total time: .* ms +--pre +\! mkdir @abs_bindir@/../gs_basebackup_node_stream_p +\! chmod 700 @abs_bindir@/../gs_basebackup_node_stream_p +\! mkdir @abs_bindir@/../gs_basebackup_node_fetch_t +\! chmod 700 @abs_bindir@/../gs_basebackup_node_fetch_t +--run +\! chmod +x @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh +\! @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh @abs_bindir@ @abs_srcdir@ @portstring@ gs_basebackup_node_stream_p stream p +--?! [.*][.*][][gs_ctl]: gs_ctl status,datadir is -D ".*/gs_basebackup_node_stream_p" +--?! gs_ctl: server is running (PID: .*) +--?!.* +--?.*List of tablespaces.* +--?.*Name.*|.*Owner.*|.*Location.* +--?--------------------------+.*+-------------------------- +--? gs_basebackup_tablespace |.*| gs_basebackup_tablespace +(1 row) + + tablespace +-------------------------- + gs_basebackup_tablespace +(1 row) + + a +--- + 1 +(1 row) + +--?total time: .* ms + a +--- + 1 +(1 row) + +--?total time: .* ms +--?.*[gs_ctl]: gs_ctl stopped ,datadir is -D ".*/gs_basebackup_node_stream_p" +--?waiting for server to shut down.* done +server stopped +\! @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh @abs_bindir@ @abs_srcdir@ @portstring@ gs_basebackup_node_fetch_t fetch t +--?! [.*][.*][][gs_ctl]: gs_ctl status,datadir is -D ".*/gs_basebackup_node_fetch_t" +--?! gs_ctl: server is running (PID: .*) +--?!.* +--?.*List of tablespaces.* +--?.*Name.*|.*Owner.*|.*Location.* +--?--------------------------+.*+------------------------------------------------------------------------------------------------- +--? gs_basebackup_tablespace |.*| .*/gs_basebackup_tablespace +(1 row) + + tablespace +-------------------------- + gs_basebackup_tablespace +(1 row) + + a +--- + 1 +(1 row) + +--?total time: .* ms + a +--- + 1 +(1 row) + +--?total time: * ms +--?.*[gs_ctl]: gs_ctl stopped ,datadir is -D ".*/gs_basebackup_node_fetch_t" +--?waiting for server to shut down.* done +server stopped \ No newline at end of file diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 9674a16fb..fb483ea7d 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -591,3 +591,6 @@ test: create_procedure create_function pg_compatibility postgres_fdw # autonomous transaction Test test: autonomous_transaction + +# gs_basebackup +test: gs_basebackup diff --git a/src/test/regress/script/gs_basebackup/gs_basebackup.sh b/src/test/regress/script/gs_basebackup/gs_basebackup.sh new file mode 100644 index 000000000..1f778d328 --- /dev/null +++ b/src/test/regress/script/gs_basebackup/gs_basebackup.sh @@ -0,0 +1,55 @@ +abs_bindir=$1 +abs_srcdir=$2 +abs_port=$3 +dataNode=$4 +x_option=$5 +format=$6 +# backup +$abs_bindir/gs_basebackup -D $abs_bindir/../$dataNode -p $abs_port -X$x_option -F$format > $abs_bindir/../$dataNode.log 2>&1 +for gs_basebackup_port in {40015..60000}; +do + if [ 'x'`netstat -an | grep -v STREAM | grep -v DGRAM | grep $gs_basebackup_port | head -n1 | awk '{print $1}'` == 'x' ]; + then + break; + fi; +done; + +if [ $format == 't' ] +then + tmp_dir="$abs_bindir/../$dataNode/../tmp" + mv $abs_bindir/../$dataNode/* $abs_bindir/../$dataNode/../ + $abs_bindir/gs_tar -F $abs_bindir/../base.tar -D $abs_bindir/../$dataNode/ + mkdir $abs_bindir/../$dataNode/pg_location + count='0'; + tablespace=""; + mkdir $tmp_dir + absolute_path=`cd $abs_bindir; pwd` + for i in `cat $abs_bindir/../$dataNode/tablespace_map`; + do + if [ $count == '0' ]; + then + tablespace=$i; + count='1'; + else + mkdir "$abs_bindir/../$dataNode/pg_location/${i##/*/}" + $abs_bindir/gs_tar -F $abs_bindir/../$tablespace.tar -D $tmp_dir + mv $tmp_dir/* "$abs_bindir/../$dataNode/pg_location/${i##/*/}" + count='0'; + fi + done +fi + + +$abs_bindir/gs_ctl start -o "-p ${gs_basebackup_port} -c listen_addresses=*" -D $abs_bindir/../$dataNode >> $abs_bindir/../$dataNode.log 2>&1 + + +# ----check start or not +$abs_bindir/gs_ctl status -D $abs_bindir/../$dataNode + +#validate +$abs_bindir/gsql -dgs_basebackup -p$gs_basebackup_port -f "$abs_srcdir/sql/gs_basebackup/validate/tablespace.sql"; +$abs_bindir/gsql -dgs_basebackup -p$gs_basebackup_port -f "$abs_srcdir/sql/gs_basebackup/validate/mot.sql"; + + +#stop node +$abs_bindir/gs_ctl stop -D $abs_bindir/../$dataNode \ No newline at end of file diff --git a/src/test/regress/sql/gs_basebackup/init/mot.sql b/src/test/regress/sql/gs_basebackup/init/mot.sql new file mode 100644 index 000000000..cca1e77a3 --- /dev/null +++ b/src/test/regress/sql/gs_basebackup/init/mot.sql @@ -0,0 +1,2 @@ +create foreign table gs_basebackupmot_table(a int); +INSERT INTO gs_basebackupmot_table VALUES(1); \ No newline at end of file diff --git a/src/test/regress/sql/gs_basebackup/init/tablespace.sql b/src/test/regress/sql/gs_basebackup/init/tablespace.sql new file mode 100644 index 000000000..a89a48c3d --- /dev/null +++ b/src/test/regress/sql/gs_basebackup/init/tablespace.sql @@ -0,0 +1,3 @@ +CREATE TABLESPACE gs_basebackup_tablespace relative LOCATION 'gs_basebackup_tablespace'; +CREATE TABLE tablespace_table_test(a int) TABLESPACE gs_basebackup_tablespace; +INSERT INTO tablespace_table_test VALUES(1); \ No newline at end of file diff --git a/src/test/regress/sql/gs_basebackup/validate/mot.sql b/src/test/regress/sql/gs_basebackup/validate/mot.sql new file mode 100644 index 000000000..a75d23d31 --- /dev/null +++ b/src/test/regress/sql/gs_basebackup/validate/mot.sql @@ -0,0 +1 @@ +select * from gs_basebackupmot_table; \ No newline at end of file diff --git a/src/test/regress/sql/gs_basebackup/validate/tablespace.sql b/src/test/regress/sql/gs_basebackup/validate/tablespace.sql new file mode 100644 index 000000000..3e814b415 --- /dev/null +++ b/src/test/regress/sql/gs_basebackup/validate/tablespace.sql @@ -0,0 +1,3 @@ +\db gs_basebackup_tablespace +select tablespace from pg_tables where tablename = 'tablespace_table_test'; +select * from tablespace_table_test; \ No newline at end of file