hcq/openGauss-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

!166 add gs_basebackup testcases

Browse Source 
Merge pull request !166 from 吴岳川/gs_basebackup
This commit is contained in:
opengauss-bot
2020-09-01 19:26:39 +08:00
committed by Gitee
parent 1b17e6ab6d 7e27d28848
commit 141bb77173
14 changed files with 272 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
src/bin/pg_basebackup/pg_basebackup.cpp
View File

@ -52,7 +52,6 @@ typedef struct TablespaceList {
/* Global options */
char* basedir = NULL;
static TablespaceList tablespacee_dirs = {NULL, NULL};
char* g_xlogOption = NULL;
char format = 'p'; /* p(lain)/t(ar) */
char* label = "gs_basebackup base backup";
bool showprogress = false;
@ -627,7 +626,8 @@ static void ReceiveTarFile(PGconn* conn, PGresult* res, int rownum)
} else
#endif
tarfile = stdout;
strcpy_s(filename, 1, "-");
errorno = strcpy_s(filename, 1, "-");
securec_check_c(errorno, "\0", "\0");
} else {
#ifdef HAVE_LIBZ
if (compresslevel != 0) {
@ -783,16 +783,20 @@ static void ReceiveTarFile(PGconn* conn, PGresult* res, int rownum)
*/
static const char* get_tablespace_mapping(const char* dir)
{
TablespaceListCell* cell;
TablespaceListCell* cell = NULL;
char canon_dir[MAXPGPATH];
errno_t errorno = EOK;
/* Canonicalize path for comparison consistency */
strlcpy(canon_dir, dir, sizeof(canon_dir));
errorno = strcpy_s(canon_dir, MAXPGPATH, dir);
securec_check_c(errorno, "\0", "\0");
canonicalize_path(canon_dir);
for (cell = tablespacee_dirs.head; cell; cell = cell->next)
if (strcmp(canon_dir, cell->old_dir) == 0)
for (cell = tablespacee_dirs.head; cell; cell = cell->next) {
if (strcmp(canon_dir, cell->old_dir) == 0) {
return cell->new_dir;
}
}
return dir;
}
@ -928,7 +932,7 @@ static void ReceiveAndUnpackTarFile(PGconn* conn, PGresult* res, int rownum)
/*
* All files are padded up to 512 bytes
*/
current_padding = ((current_len_left + TAR_FILE_PADDING) & ~TAR_FILE_PADDING) - current_len_left;
current_padding = PADDING_LEFT(current_len_left);
/*
* First part of header is zero terminated filename
@ -1134,21 +1138,14 @@ static void BaseBackup(void)
*/
res = PQexec(conn, "IDENTIFY_SYSTEM");
if (PQresultStatus(res) != PGRES_TUPLES_OK) {
fprintf(stderr,
_("%s: could not send replication command \"%s\": %s"),
progname,
"IDENTIFY_SYSTEM",
fprintf(stderr, _("%s: could not send replication command \"%s\": %s"), progname, "IDENTIFY_SYSTEM",
PQerrorMessage(conn));
disconnect_and_exit(1);
}
if (PQntuples(res) != 1 || PQnfields(res) < 3) {
fprintf(stderr,
_("%s: could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields\n"),
progname,
PQntuples(res),
PQnfields(res),
1,
3);
progname, PQntuples(res), PQnfields(res), 1, 3);
disconnect_and_exit(1);
}
sysidentifier = strdup(PQgetvalue(res, 0, 0));
@ -1163,23 +1160,14 @@ static void BaseBackup(void)
* Start the actual backup
*/
PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
rc = snprintf_s(current_path,
sizeof(current_path),
sizeof(current_path) - 1,
"BASE_BACKUP LABEL '%s' %s %s %s %s %s",
escaped_label,
showprogress ? "PROGRESS" : "",
includewal && !streamwal ? "WAL" : "",
fastcheckpoint ? "FAST" : "",
includewal ? "NOWAIT" : "",
rc = snprintf_s(current_path, sizeof(current_path), sizeof(current_path) - 1,
"BASE_BACKUP LABEL '%s' %s %s %s %s %s", escaped_label, showprogress ? "PROGRESS" : "",
includewal && !streamwal ? "WAL" : "", fastcheckpoint ? "FAST" : "", includewal ? "NOWAIT" : "",
format == 't' ? "TABLESPACE_MAP" : "");
securec_check_ss_c(rc, "", "");
if (PQsendQuery(conn, current_path) == 0) {
fprintf(stderr,
_("%s: could not send replication command \"%s\": %s"),
progname,
"BASE_BACKUP",
fprintf(stderr, _("%s: could not send replication command \"%s\": %s"), progname, "BASE_BACKUP",
PQerrorMessage(conn));
free(sysidentifier);
disconnect_and_exit(1);
@ -1276,8 +1264,8 @@ static void BaseBackup(void)
rc = snprintf_s(prefix, MAXPGPATH, strlen(basedir) + 1, "%s/", basedir);
securec_check_ss_c(rc, "\0", "\0");
}
rc = snprintf_s(
nodetablespacepath, MAXPGPATH, sizeof(nodetablespacepath) - 1, "%s%s", prefix, tablespacepath);
rc = snprintf_s(nodetablespacepath, MAXPGPATH, sizeof(nodetablespacepath) - 1, "%s%s", prefix,
tablespacepath);
securec_check_ss_c(rc, "\0", "\0");
verify_dir_is_empty_or_create(nodetablespacepath);
@ -1543,7 +1531,7 @@ int main(int argc, char** argv)
} else if (!strcmp(progname, "gs_tar")) {
return GsTar(argc, argv);
} else {
fprintf(stderr, _("unsupport progname: %s"), progname);
fprintf(stderr, _("unsupported progname: %s"), progname);
return 0;
}
}
@ -1628,14 +1616,15 @@ static int GsTar(int argc, char** argv)
uint64 current_padding = 0;
errno_t errorno = EOK;
strncpy_s(current_path, MAXPGPATH, basedir, strlen(basedir));
errorno = strncpy_s(current_path, MAXPGPATH, basedir, strlen(basedir));
securec_check_c(errorno, "\0", "\0");
tarfile = fopen(tarfilename, "rb");
if (tarfile == NULL) {
fprintf(stderr, _("WARNING: %s not found!\n"), filename);
exit(1);
}
while (true) {
int r;
size_t r;
if (copybuf != NULL) {
PQfreemem(copybuf);
@ -1649,16 +1638,8 @@ static int GsTar(int argc, char** argv)
} else {
copybuf = (char*)xmalloc0(current_len_left);
r = fread(copybuf, 1, current_len_left, tarfile);
// end of file
if ((uint64)r < current_len_left) {
if (file != NULL) {
fclose(file);
file = NULL;
}
break;
}
}
if (r == -1) {
if (r == 0) {
/*
* End of chunk
*/
@ -1667,9 +1648,6 @@ static int GsTar(int argc, char** argv)
file = NULL;
}
break;
} else if (r == -2) {
fprintf(stderr, "%s: could not read file data: %s", progname, tarfilename);
return -1;
}
if (file == NULL) {
/* new file */
@ -1677,7 +1655,7 @@ static int GsTar(int argc, char** argv)
/* No current file, so this must be the header for a new file */
if (r != TAR_BLOCK_SIZE) {
fprintf(stderr, "%s: invalid tar block header size: %d\n", progname, r);
fprintf(stderr, "%s: invalid tar block header size: %zu\n", progname, r);
fclose(tarfile);
return -1;
}
@ -1685,19 +1663,21 @@ static int GsTar(int argc, char** argv)
if (sscanf_s(copybuf + 1048, "%201o", &current_len_left) != 1) {
fprintf(stderr, "%s: could not parse file size\n", progname);
fclose(tarfile);
return -1;
}
/* Set permissions on the file */
if (sscanf_s(&copybuf[1024], "%07o ", (unsigned int*)&filemode) != 1) {
fprintf(stderr, "%s: could not parse file mode\n", progname);
fclose(tarfile);
return -1;
}
/*
* All files are padded up to 512 bytes
*/
current_padding = ((current_len_left + 511) & ~511) - current_len_left;
current_padding = PADDING_LEFT(current_len_left);
/*
* First part of header is zero terminated filename
@ -1707,6 +1687,7 @@ static int GsTar(int argc, char** argv)
"%s: the copybuf/current_path file path including .. is unallowed: %s\n",
progname,
strerror(errno));
fclose(tarfile);
return -1;
}
errorno = snprintf_s(filename, sizeof(filename), sizeof(filename) - 1, "%s/%s", current_path, copybuf);
@ -1788,11 +1769,13 @@ static int GsTar(int argc, char** argv)
filename,
&copybuf[1081],
strerror(errno));
fclose(tarfile);
return -1;
}
}
} else {
pg_log(PG_WARNING, "unrecognized link indicator \"%c\"\n", copybuf[TAR_FILE_TYPE]);
fclose(tarfile);
return -1;
}
continue; /* directory or link handled */
@ -1828,21 +1811,22 @@ static int GsTar(int argc, char** argv)
/*
* Continuing blocks in existing file
*/
if (current_len_left == 0 && (uint64)r == current_padding) {
if (current_len_left == 0 && r == current_padding) {
/*
* Received the padding block for this file, ignore it and
* close the file, then move on to the next tar header.
*/
fclose(file);
file = NULL;
totaldone += (uint64)r;
current_padding -= (uint64)r;
totaldone += r;
current_padding -= r;
continue;
}
if (fwrite(copybuf, r, 1, file) != 1) {
fprintf(stderr, "%s: could not write to file \"%s\": %s\n", progname, filename, strerror(errno));
fclose(file);
fclose(tarfile);
file = NULL;
return -1;
}
@ -1863,16 +1847,16 @@ static int GsTar(int argc, char** argv)
} /* loop over all data blocks */
/* delete dw file if exists, recreate it and write a page of zero */
backup_dw_file(basedir);
if (tarfile != NULL) {
fclose(tarfile);
tarfile = NULL;
}
if (file != NULL) {
fprintf(stderr, "%s: COPY stream ended before last file was finished\n", progname);
fclose(file);
file = NULL;
return -1;
}
if (tarfile != NULL) {
fclose(tarfile);
tarfile = NULL;
}
return 0;
}
@ -1959,20 +1943,17 @@ static int GsBaseBackup(int argc, char** argv)
includewal = true;
check_env_value_c(optarg);
g_xlogOption = xstrdup(optarg);
if (strcmp(g_xlogOption, "f") == 0 || strcmp(g_xlogOption, "fetch") == 0)
if (strcmp(optarg, "f") == 0 || strcmp(optarg, "fetch") == 0)
streamwal = false;
else if (strcmp(g_xlogOption, "s") == 0 || strcmp(g_xlogOption, "stream") == 0)
else if (strcmp(optarg, "s") == 0 || strcmp(optarg, "stream") == 0)
streamwal = true;
else {
fprintf(stderr,
_("%s: invalid xlog-method option \"%s\", must be \"fetch\" or \"stream\"\n"),
progname,
optarg);
GS_FREE(g_xlogOption);
exit(1);
}
GS_FREE(g_xlogOption);
break;
case 'l':
if (label != NULL && strcmp(label, "gs_basebackup base backup") != 0) {

22
src/bin/pg_ctl/fetchmot.cpp
View File

@ -66,7 +66,7 @@ static void MotReceiveAndAppendTarFile(
int current_len_left = 0;
int current_padding = 0;
errorno = strncpy_s(current_path, sizeof(current_path), basedir, sizeof(current_path) - 1);
errorno = strncpy_s(current_path, sizeof(current_path), basedir, strlen(basedir));
securec_check_c(errorno, "", "");
#ifdef HAVE_LIBZ
@ -98,7 +98,8 @@ static void MotReceiveAndAppendTarFile(
} else
#endif
tarfile = stdout;
strcpy_s(filename, 1, "-");
errorno = strcpy_s(filename, 1, "-");
securec_check_c(errorno, "\0", "\0");
} else {
#ifdef HAVE_LIBZ
if (compresslevel != 0) {
@ -118,15 +119,11 @@ static void MotReceiveAndAppendTarFile(
/* chkptName header */
copybuf = (char*)xmalloc0(TAR_BLOCK_SIZE);
int chkptDirLen = strlen(chkptName);
errorno = strcpy_s(copybuf + 2, chkptDirLen + 1, chkptName);
securec_check_ss_c(errorno, "", "");
copybuf[0] = '.';
copybuf[1] = '/';
copybuf[chkptDirLen + 2] = '/';
errorno = snprintf_s(copybuf, TAR_BLOCK_SIZE, TAR_BLOCK_SIZE - 1, "./%s/", chkptName);
copybuf[TAR_FILE_TYPE] = TAR_TYPE_DICTORY;
securec_check_ss_c(errorno, "", "");
for (int i = 0; i < 11; i++) {
copybuf[TAR_LEN_LEFT + i] = '0';
}
@ -210,7 +207,7 @@ static void MotReceiveAndAppendTarFile(
/*
* All files are padded up to 512 bytes
*/
current_padding = ((current_len_left + TAR_FILE_PADDING) & ~TAR_FILE_PADDING) - current_len_left;
current_padding = PADDING_LEFT(current_len_left);
/*
* First part of header is zero terminated filename.
* when getting a checkpoint, file name can be either
@ -236,7 +233,8 @@ static void MotReceiveAndAppendTarFile(
if (filename[strlen(filename) - 1] == '/') {
filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
strcpy_s(copybuf, strlen(filename), filename);
errorno = strcpy_s(copybuf, r, filename);
securec_check_c(errorno, "\0", "\0");
continue; /* directory or link handled */
}
#ifdef HAVE_LIBZ
@ -405,7 +403,7 @@ static void MotReceiveAndUnpackTarFile(const char* basedir, const char* chkptNam
/*
* All files are padded up to 512 bytes
*/
current_padding = ((current_len_left + 511) & ~511) - current_len_left;
current_padding = PADDING_LEFT(current_len_left);
/*
* First part of header is zero terminated filename.

20
src/gausskernel/storage/access/transam/xlog.cpp
View File

@ -167,7 +167,8 @@ THR_LOCAL bool redo_oldversion_xlog = false;
/*
* GUC support
*/
struct config_enum_entry sync_method_options[] = {{"fsync", SYNC_METHOD_FSYNC, false},
struct config_enum_entry sync_method_options[] = {
{"fsync", SYNC_METHOD_FSYNC, false},
#ifdef HAVE_FSYNC_WRITETHROUGH
{"fsync_writethrough", SYNC_METHOD_FSYNC_WRITETHROUGH, false},
#endif
@ -180,7 +181,8 @@ struct config_enum_entry sync_method_options[] = {{"fsync", SYNC_METHOD_FSYNC, f
#ifdef OPEN_DATASYNC_FLAG
{"open_datasync", SYNC_METHOD_OPEN_DSYNC, false},
#endif
{NULL, 0, false}};
{NULL, 0, false}
};
XLogRecPtr latestValidRecord = InvalidXLogRecPtr;
XLogSegNo XlogRemoveSegPrimary = InvalidXLogSegPtr;
@ -12272,7 +12274,7 @@ XLogRecPtr do_pg_start_backup(const char* backupidstr, bool fast, char** labelfi
int rllen;
errno_t errorno = EOK;
errorno = memset_s(fullpath, MAXPGPATH, '\0', MAXPGPATH);
errorno = memset_s(fullpath, MAXPGPATH + PG_TBLSPCS, '\0', MAXPGPATH + PG_TBLSPCS);
securec_check(errorno, "", "");
errorno = memset_s(linkpath, MAXPGPATH, '\0', MAXPGPATH);
@ -12282,7 +12284,9 @@ XLogRecPtr do_pg_start_backup(const char* backupidstr, bool fast, char** labelfi
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
continue;
snprintf(fullpath, sizeof(fullpath), "pg_tblspc/%s", de->d_name);
errorno =
snprintf_s(fullpath, MAXPGPATH + PG_TBLSPCS, MAXPGPATH + PG_TBLSPCS - 1, "pg_tblspc/%s", de->d_name);
securec_check_ss(errorno, "\0", "\0");
#if defined(HAVE_READLINK) || defined(WIN32)
rllen = readlink(fullpath, linkpath, sizeof(linkpath));
@ -13792,7 +13796,7 @@ static bool read_tablespace_map(List** tablespaces)
char tbsoid[MAXPGPATH];
char* tbslinkpath;
char str[MAXPGPATH];
int ch, prev_ch = -1, i = 0, n;
int ch, prevCh = -1, i = 0, n;
/*
* See if tablespace_map file is present
@ -13814,7 +13818,7 @@ static bool read_tablespace_map(List** tablespaces)
* escape character that has been added in tablespace path during backup.
*/
while ((ch = fgetc(lfp)) != EOF) {
if ((ch == '\n' || ch == '\r') && prev_ch != '\\') {
if ((ch == '\n' || ch == '\r') && prevCh != '\\') {
str[i] = '\0';
if (sscanf(str, "%s %n", tbsoid, &n) != 1)
ereport(FATAL,
@ -13829,11 +13833,11 @@ static bool read_tablespace_map(List** tablespaces)
*tablespaces = lappend(*tablespaces, ti);
continue;
} else if ((ch == '\n' || ch == '\r') && prev_ch == '\\')
} else if ((ch == '\n' || ch == '\r') && prevCh == '\\')
str[i - 1] = ch;
else
str[i++] = ch;
prev_ch = ch;
prevCh = ch;
}
if (ferror(lfp) || FreeFile(lfp))

3
src/gausskernel/storage/access/transam/xlogfuncs.cpp
View File

@ -76,6 +76,9 @@ Datum pg_start_backup(PG_FUNCTION_ARGS)
backupidstr = text_to_cstring(backupid);
dir = AllocateDir("pg_tblspc");
if (!dir) {
ereport(ERROR, (errmsg("could not open directory \"%s\": %m", "pg_tblspc")));
}
startpoint = do_pg_start_backup(backupidstr, fast, NULL, dir, NULL, NULL, false, true);
errorno = snprintf_s(startxlogstr,

55
src/gausskernel/storage/replication/basebackup.cpp
View File

@ -68,6 +68,12 @@ const int MATCH_SIX = 6;
* Size of each block sent into the tar stream for larger files.
*/
#define TAR_SEND_SIZE (32 * 1024) /* data send unit 32KB */
#define EREPORT_WAL_NOT_FOUND(segno) \
do { \
char walErrorName[MAXFNAMELEN]; \
XLogFileName(walErrorName, t_thrd.xlog_cxt.ThisTimeLineID, segno); \
ereport(ERROR, (errmsg("could not find WAL file \"%s\"", walErrorName))); \
} while (0)
XLogRecPtr XlogCopyStartPtr = InvalidXLogRecPtr;
@ -409,6 +415,14 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir)
* recycled before we get a chance to send it over.
*/
nWalFiles = list_length(walFileList);
/*
* There must be at least one xlog file in the pg_xlog directory,
* since we are doing backup-including-xlog.
*/
if (nWalFiles < 1) {
ereport(ERROR, (errmsg("could not find any WAL files")));
}
walFiles = (char**)palloc0(nWalFiles * sizeof(char*));
i = 0;
foreach (lc, walFileList) {
@ -416,24 +430,13 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir)
}
qsort(walFiles, nWalFiles, sizeof(char*), CompareWalFileNames);
/*
* There must be at least one xlog file in the pg_xlog directory,
* since we are doing backup-including-xlog.
*/
if (nWalFiles < 1) {
ereport(ERROR, (errmsg("could not find any WAL files")));
}
/*
* Sanity check: the first and last segment should cover startptr and
* endptr, with no gaps in between.
*/
XLogFromFileName(walFiles[0], &tli, &segno);
if (segno != startsegno) {
char startfname[MAXFNAMELEN];
XLogFileName(startfname, t_thrd.xlog_cxt.ThisTimeLineID, startsegno);
ereport(ERROR, (errmsg("could not find WAL file \"%s\"", startfname)));
EREPORT_WAL_NOT_FOUND(startsegno);
}
for (i = 0; i < nWalFiles; i++) {
XLogSegNo currsegno = segno;
@ -441,17 +444,11 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir)
XLogFromFileName(walFiles[i], &tli, &segno);
if (!(nextsegno == segno || currsegno == segno)) {
char nextfname[MAXFNAMELEN];
XLogFileName(nextfname, t_thrd.xlog_cxt.ThisTimeLineID, nextsegno);
ereport(ERROR, (errmsg("could not find WAL file \"%s\"", nextfname)));
EREPORT_WAL_NOT_FOUND(nextsegno);
}
}
if (segno != endsegno) {
char endfname[MAXFNAMELEN];
XLogFileName(endfname, t_thrd.xlog_cxt.ThisTimeLineID, endsegno);
ereport(ERROR, (errmsg("could not find WAL file \"%s\"", endfname)));
EREPORT_WAL_NOT_FOUND(endsegno);
}
/* Ok, we have everything we need. Send the WAL files. */
@ -461,7 +458,8 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir)
size_t cnt;
pgoff_t len = 0;
snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]);
int rt = snprintf_s(pathbuf, MAXPGPATH,MAXPGPATH -1, XLOGDIR "/%s", walFiles[i]);
securec_check_ss_c(rt, "\0", "\0");
XLogFromFileName(walFiles[i], &tli, &segno);
fp = AllocateFile(pathbuf, "rb");
@ -533,8 +531,8 @@ static void perform_base_backup(basebackup_options* opt, DIR* tblspcdir)
foreach (lc, historyFileList) {
char* fname = (char*)lfirst(lc);
snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
int rt = snprintf_s(pathbuf, MAXPGPATH, MAXPGPATH-1, XLOGDIR "/%s", fname);
securec_check_ss_c(rt, "\0", "\0");
if (lstat(pathbuf, &statbuf) != 0)
ereport(ERROR, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", pathbuf)));
@ -705,8 +703,9 @@ static void parse_basebackup_options(List* options, basebackup_options* opt)
opt->includewal = true;
o_wal = true;
} else if (strcmp(defel->defname, "tablespace_map") == 0) {
if (o_tablespace_map)
if (o_tablespace_map) {
ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname)));
}
opt->sendtblspcmapfile = true;
o_tablespace_map = true;
} else
@ -969,7 +968,7 @@ static void sendFileWithContent(const char* filename, const char* content)
/* Pad to 512 byte boundary, per tar format requirements */
pad = ((len + 511) & ~511) - len;
if (pad > 0) {
char buf[512];
char buf[2560];
errno_t rc = 0;
rc = memset_s(buf, sizeof(buf), 0, pad);
@ -1021,7 +1020,7 @@ int64 sendTablespace(const char* path, bool sizeonly)
}
if (!sizeonly)
_tarWriteHeader(relativedirname, NULL, &statbuf);
size = 512; /* Size of the header just added */
size = 2560; /* Size of the header just added */
/* Send all the files in the tablespace version directory */
size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true);
@ -1237,7 +1236,7 @@ static int64 sendDir(
_tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf);
}
}
size += 512; /* Size of the header just added */
size += 2560; /* Size of the header just added */
if (!sizeonly) {
#ifndef WIN32
if (S_ISLNK(statbuf.st_mode)) {
@ -1279,7 +1278,7 @@ static int64 sendDir(
_tarWriteHeader("pg_xlog/archive_status", NULL, &statbuf);
}
}
size += 512; /* Size of the header just added */
size += 2560; /* Size of the header just added */
continue; /* don't recurse into pg_xlog */
}
/* Allow symbolic links in pg_tblspc only */

3
src/include/gs_tar_const.h
View File

@ -28,6 +28,7 @@
#include "zlib.h"
#endif
/* gs_tar offset */
static const int TAR_LEN_LEFT = 1048;
static const int TAR_FILE_PADDING = 511; /* All files are padded up to 512 bytes */
@ -42,6 +43,8 @@ static const int TAR_READ_ERROR = -2;
static const char TAR_TYPE_DICTORY = '5';
static const int FILE_PERMISSION = 16832;
#define PADDING_LEFT(current_len_left) (((current_len_left + TAR_FILE_PADDING) & ~TAR_FILE_PADDING) - current_len_left);
#ifdef HAVE_LIBZ
static const char* get_gz_error(gzFile gzf)
{

21
src/test/regress/input/gs_basebackup.source Normal file
View File

@ -0,0 +1,21 @@
--restart_node
\! @abs_bindir@/gs_guc set -D @abs_srcdir@/tmp_check/datanode1/ -c "enable_incremental_checkpoint=off"
\! @abs_bindir@/gs_ctl restart -D @abs_srcdir@/tmp_check/datanode1/ > @abs_bindir@/../datanode1.restart.log 2>&1
---prepare data
\! @abs_bindir@/gsql -dpostgres -p @portstring@ -c "create database gs_basebackup;"
\! @abs_bindir@/gsql -dgs_basebackup -p @portstring@ -f "@abs_srcdir@/sql/gs_basebackup/init/tablespace.sql";
\! @abs_bindir@/gsql -dgs_basebackup -p @portstring@ -f "@abs_srcdir@/sql/gs_basebackup/init/mot.sql";
--pre
\! mkdir @abs_bindir@/../gs_basebackup_node_stream_p
\! chmod 700 @abs_bindir@/../gs_basebackup_node_stream_p
\! mkdir @abs_bindir@/../gs_basebackup_node_fetch_t
\! chmod 700 @abs_bindir@/../gs_basebackup_node_fetch_t
--run
\! chmod +x @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh
\! @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh @abs_bindir@ @abs_srcdir@ @portstring@ gs_basebackup_node_stream_p stream p
\! @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh @abs_bindir@ @abs_srcdir@ @portstring@ gs_basebackup_node_fetch_t fetch t

88
src/test/regress/output/gs_basebackup.source Normal file
View File

@ -0,0 +1,88 @@
--restart_node
\! @abs_bindir@/gs_guc set -D @abs_srcdir@/tmp_check/datanode1/ -c "enable_incremental_checkpoint=off"
--?expected instance path: [.*/postgresql.conf]
--?gs_guc set: enable_incremental_checkpoint=off: .*postgresql.conf.*
Total instances: 1. Failed instances: 0.
Success to perform gs_guc!
\! @abs_bindir@/gs_ctl restart -D @abs_srcdir@/tmp_check/datanode1/ > @abs_bindir@/../datanode1.restart.log 2>&1
---prepare data
\! @abs_bindir@/gsql -dpostgres -p @portstring@ -c "create database gs_basebackup;"
CREATE DATABASE
\! @abs_bindir@/gsql -dgs_basebackup -p @portstring@ -f "@abs_srcdir@/sql/gs_basebackup/init/tablespace.sql";
CREATE TABLESPACE
CREATE TABLE
INSERT 0 1
--?total time: .* ms
\! @abs_bindir@/gsql -dgs_basebackup -p @portstring@ -f "@abs_srcdir@/sql/gs_basebackup/init/mot.sql";
CREATE FOREIGN TABLE
INSERT 0 1
--?total time: .* ms
--pre
\! mkdir @abs_bindir@/../gs_basebackup_node_stream_p
\! chmod 700 @abs_bindir@/../gs_basebackup_node_stream_p
\! mkdir @abs_bindir@/../gs_basebackup_node_fetch_t
\! chmod 700 @abs_bindir@/../gs_basebackup_node_fetch_t
--run
\! chmod +x @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh
\! @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh @abs_bindir@ @abs_srcdir@ @portstring@ gs_basebackup_node_stream_p stream p
--?! [.*][.*][][gs_ctl]: gs_ctl status,datadir is -D ".*/gs_basebackup_node_stream_p"
--?! gs_ctl: server is running (PID: .*)
--?!.*
--?.*List of tablespaces.*
--?.*Name.*|.*Owner.*|.*Location.*
--?--------------------------+.*+--------------------------
--? gs_basebackup_tablespace |.*| gs_basebackup_tablespace
(1 row)
tablespace
--------------------------
gs_basebackup_tablespace
(1 row)
a
---
1
(1 row)
--?total time: .* ms
a
---
1
(1 row)
--?total time: .* ms
--?.*[gs_ctl]: gs_ctl stopped ,datadir is -D ".*/gs_basebackup_node_stream_p"
--?waiting for server to shut down.* done
server stopped
\! @abs_srcdir@/script/gs_basebackup/gs_basebackup.sh @abs_bindir@ @abs_srcdir@ @portstring@ gs_basebackup_node_fetch_t fetch t
--?! [.*][.*][][gs_ctl]: gs_ctl status,datadir is -D ".*/gs_basebackup_node_fetch_t"
--?! gs_ctl: server is running (PID: .*)
--?!.*
--?.*List of tablespaces.*
--?.*Name.*|.*Owner.*|.*Location.*
--?--------------------------+.*+-------------------------------------------------------------------------------------------------
--? gs_basebackup_tablespace |.*| .*/gs_basebackup_tablespace
(1 row)
tablespace
--------------------------
gs_basebackup_tablespace
(1 row)
a
---
1
(1 row)
--?total time: .* ms
a
---
1
(1 row)
--?total time: * ms
--?.*[gs_ctl]: gs_ctl stopped ,datadir is -D ".*/gs_basebackup_node_fetch_t"
--?waiting for server to shut down.* done
server stopped

3
src/test/regress/parallel_schedule
View File

@ -591,3 +591,6 @@ test: create_procedure create_function pg_compatibility postgres_fdw
# autonomous transaction Test
test: autonomous_transaction
# gs_basebackup
test: gs_basebackup

55
src/test/regress/script/gs_basebackup/gs_basebackup.sh Normal file
View File

@ -0,0 +1,55 @@
abs_bindir=$1
abs_srcdir=$2
abs_port=$3
dataNode=$4
x_option=$5
format=$6
# backup
$abs_bindir/gs_basebackup -D $abs_bindir/../$dataNode -p $abs_port -X$x_option -F$format > $abs_bindir/../$dataNode.log 2>&1
for gs_basebackup_port in {40015..60000};
do
if [ 'x'`netstat -an | grep -v STREAM | grep -v DGRAM | grep $gs_basebackup_port | head -n1 | awk '{print $1}'` == 'x' ];
then
break;
fi;
done;
if [ $format == 't' ]
then
tmp_dir="$abs_bindir/../$dataNode/../tmp"
mv $abs_bindir/../$dataNode/* $abs_bindir/../$dataNode/../
$abs_bindir/gs_tar -F $abs_bindir/../base.tar -D $abs_bindir/../$dataNode/
mkdir $abs_bindir/../$dataNode/pg_location
count='0';
tablespace="";
mkdir $tmp_dir
absolute_path=`cd $abs_bindir; pwd`
for i in `cat $abs_bindir/../$dataNode/tablespace_map`;
do
if [ $count == '0' ];
then
tablespace=$i;
count='1';
else
mkdir "$abs_bindir/../$dataNode/pg_location/${i##/*/}"
$abs_bindir/gs_tar -F $abs_bindir/../$tablespace.tar -D $tmp_dir
mv $tmp_dir/* "$abs_bindir/../$dataNode/pg_location/${i##/*/}"
count='0';
fi
done
fi
$abs_bindir/gs_ctl start -o "-p ${gs_basebackup_port} -c listen_addresses=*" -D $abs_bindir/../$dataNode >> $abs_bindir/../$dataNode.log 2>&1
# ----check start or not
$abs_bindir/gs_ctl status -D $abs_bindir/../$dataNode
#validate
$abs_bindir/gsql -dgs_basebackup -p$gs_basebackup_port -f "$abs_srcdir/sql/gs_basebackup/validate/tablespace.sql";
$abs_bindir/gsql -dgs_basebackup -p$gs_basebackup_port -f "$abs_srcdir/sql/gs_basebackup/validate/mot.sql";
#stop node
$abs_bindir/gs_ctl stop -D $abs_bindir/../$dataNode

2
src/test/regress/sql/gs_basebackup/init/mot.sql Normal file
View File

@ -0,0 +1,2 @@
create foreign table gs_basebackupmot_table(a int);
INSERT INTO gs_basebackupmot_table VALUES(1);

3
src/test/regress/sql/gs_basebackup/init/tablespace.sql Normal file
View File

@ -0,0 +1,3 @@
CREATE TABLESPACE gs_basebackup_tablespace relative LOCATION 'gs_basebackup_tablespace';
CREATE TABLE tablespace_table_test(a int) TABLESPACE gs_basebackup_tablespace;
INSERT INTO tablespace_table_test VALUES(1);

1
src/test/regress/sql/gs_basebackup/validate/mot.sql Normal file
View File

@ -0,0 +1 @@
select * from gs_basebackupmot_table;

3
src/test/regress/sql/gs_basebackup/validate/tablespace.sql Normal file
View File

@ -0,0 +1,3 @@
\db gs_basebackup_tablespace
select tablespace from pg_tables where tablename = 'tablespace_table_test';
select * from tablespace_table_test;