hcq/openGauss-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

!407 add new way to archive xlog to local machine without invoking popen

Browse Source 
Merge pull request !407 from LiHeng/master
This commit is contained in:
opengauss-bot
2020-11-16 09:02:00 +08:00
committed by Gitee
parent 92d941f325 4fc1e30ca9
commit 1e6f0c3818
6 changed files with 110 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/bin/gs_guc/cluster_guc.conf
View File

@ -34,6 +34,7 @@ allow_concurrent_tuple_update|bool|0,0|NULL|NULL|
allow_system_table_mods|bool|0,0|NULL|NULL|
application_name|string|0,0|NULL|NULL|
archive_command|string|0,0|NULL|NULL|
archive_dest|string|0,0|NULL|NULL|
archive_mode|bool|0,0|NULL|When wal_level set to minimal, parameters archive_mode can not be used.|
archive_timeout|int|0,1073741823|s|Forced to switch WAL segment exceeds the parameter setting time. Since forced to switch off prematurely archive remains intact archive the same length. Therefore, archive_timeout to occupy a very small value will result in a huge archive storage space, it is recommended archive_timeout set to 60 seconds.|
array_nulls|bool|0,0|NULL|NULL|

14
src/common/backend/utils/misc/guc.cpp
View File

@ -9926,6 +9926,20 @@ static void init_configure_names_string()
NULL,
show_archive_command
},
{
{
"archive_dest",
PGC_SIGHUP,
WAL_ARCHIVING,
gettext_noop("Sets the path that will be used to archive a WAL file."),
NULL
},
&u_sess->attr.attr_storage.XLogArchiveDest,
"",
NULL,
NULL,
NULL
},
{
{
"client_encoding",

2
src/common/backend/utils/misc/postgresql.conf.sample
View File

@ -239,7 +239,7 @@ incremental_checkpoint_timeout = 60s # range 1s-1h
# e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
#archive_timeout = 0 # force a logfile segment switch after this
# number of seconds; 0 disables
#archive_dest = '' # path to use to archive a logfile segment
#------------------------------------------------------------------------------
# REPLICATION

94
src/gausskernel/process/postmaster/pgarch.cpp
View File

@ -80,6 +80,8 @@
#define NUM_ARCHIVE_RETRIES 3
#define ARCHIVE_BUF_SIZE (1024 * 1024)
NON_EXEC_STATIC void PgArchiverMain();
static void pgarch_exit(SIGNAL_ARGS);
static void ArchSigHupHandler(SIGNAL_ARGS);
@ -269,6 +271,24 @@ static void pgarch_waken_stop(SIGNAL_ARGS)
errno = save_errno;
}
static void VerifyDestDirIsEmptyOrCreate(char* dirname)
{
switch (pg_check_dir(dirname)) {
case 0:
/* Does not exist, so create */
if (pg_mkdir_p(dirname, S_IRWXU) == -1) {
ereport(FATAL, (errmsg_internal("could not create directory \"%s\": %s\n", dirname, strerror(errno))));
}
case -1:
/* Access problem */
ereport(FATAL, (errmsg_internal("could not access directory \"%s\": %s\n", dirname, strerror(errno))));
default: /* Nothing */
break;
}
return;
}
/*
* pgarch_MainLoop
*
@ -287,6 +307,9 @@ static void pgarch_MainLoop(void)
*/
t_thrd.arch.wakened = true;
if (XLogArchiveDestSet()) {
VerifyDestDirIsEmptyOrCreate(u_sess->attr.attr_storage.XLogArchiveDest);
}
/*
* There shouldn't be anything for the archiver to do except to wait for a
* signal ... however, the archiver exists to protect our data, so she
@ -402,8 +425,8 @@ static void pgarch_ArchiverCopyLoop(void)
}
/* can't do anything if no command ... */
if (!XLogArchiveCommandSet()) {
ereport(WARNING, (errmsg("archive_mode enabled, yet archive_command is not set")));
if (!XLogArchiveCommandSet() && !XLogArchiveDestSet()) {
ereport(WARNING, (errmsg("archive_mode enabled, yet archive_command or archive_dest is not set")));
return;
}
@ -423,6 +446,70 @@ static void pgarch_ArchiverCopyLoop(void)
}
}
/*
* PgarchArchiveXlogToDest
*
* Invokes read/write to copy one archive file to wherever it should go
*
* Returns true if successful
*/
static bool PgarchArchiveXlogToDest(const char* xlog)
{
int fdSrc = -1;
int fdDest = -1;
char srcPath[MAXPGPATH] = {0};
char destPath[MAXPGPATH] = {0};
char activitymsg[MAXFNAMELEN + 16];
unsigned long int fileBytes = 0;
int rc = 0;
if (xlog == NULL) {
return false;
}
rc = snprintf_s(srcPath, MAXPGPATH, MAXPGPATH - 1, XLOGDIR "/%s", xlog);
securec_check_ss(rc, "\0", "\0");
rc = snprintf_s(destPath, MAXPGPATH, MAXPGPATH - 1, "%s/%s", u_sess->attr.attr_storage.XLogArchiveDest, xlog);
securec_check_ss(rc, "\0", "\0");
if ((fdSrc = open(srcPath, O_RDONLY)) >= 0) {
if ((fdDest = open(destPath, O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR)) >= 0) {
char pbuff[ARCHIVE_BUF_SIZE] = {0};
while ((fileBytes = read(fdSrc, pbuff, sizeof(pbuff))) > 0) {
if (write(fdDest, pbuff, fileBytes) != fileBytes) {
close(fdSrc);
ereport(FATAL, (errmsg_internal("could not write file\"%s\":%m\n", srcPath)));
}
(void)memset_s(pbuff, sizeof(pbuff), 0, sizeof(pbuff));
}
close(fdSrc);
close(fdDest);
if (fileBytes < 0) {
ereport(FATAL, (errmsg_internal("could not read file\"%s\":%m\n", xlog)));
}
g_instance.WalSegmentArchSucceed = true;
ereport(DEBUG1, (errmsg("archived transaction log file \"%s\"", xlog)));
rc = snprintf_s(activitymsg, sizeof(activitymsg), sizeof(activitymsg) - 1, "last was %s", xlog);
securec_check_ss(rc, "\0", "\0");
set_ps_display(activitymsg, false);
return true;
} else {
close(fdSrc);
ereport(FATAL, (errmsg_internal("could not open dest file \"%s\":%m\n", destPath)));
}
} else {
ereport(FATAL, (errmsg_internal("could not open src file \"%s\":%m\n", srcPath, strerror(errno))));
}
return false;
}
/*
* pgarch_archiveXlog
*
@ -443,6 +530,9 @@ static bool pgarch_archiveXlog(char* xlog)
rc = snprintf_s(pathname, MAXPGPATH, MAXPGPATH - 1, XLOGDIR "/%s", xlog);
securec_check_ss(rc, "\0", "\0");
if (XLogArchiveDestSet()) {
return PgarchArchiveXlogToDest(xlog);
}
/*
* construct the command to be executed
*/

1
src/include/access/xlog.h
View File

@ -106,6 +106,7 @@ typedef enum WalLevel {
#define XLogArchivingActive() \
(u_sess->attr.attr_common.XLogArchiveMode && g_instance.attr.attr_storage.wal_level >= WAL_LEVEL_ARCHIVE)
#define XLogArchiveCommandSet() (u_sess->attr.attr_storage.XLogArchiveCommand[0] != '\0')
#define XLogArchiveDestSet() (u_sess->attr.attr_storage.XLogArchiveDest[0] != '\0')
/*
* Is WAL-logging necessary for archival or log-shipping, or can we skip

1
src/include/knl/knl_guc/knl_session_attr_storage.h
View File

@ -156,6 +156,7 @@ typedef struct knl_session_attr_storage {
double candidate_buf_percent_target;
double dirty_page_percent_max;
char* XLogArchiveCommand;
char* XLogArchiveDest;
char* default_tablespace;
char* temp_tablespaces;
char* XactIsoLevel_string;