From d5f29ec6daab769bbc0f4019d6aaf2ea50cf098a Mon Sep 17 00:00:00 2001 From: luozihao <1165977584@qq.com> Date: Thu, 9 Nov 2023 16:20:59 +0800 Subject: [PATCH] =?UTF-8?q?=E5=86=85=E6=A0=B8=E5=AE=9E=E7=8E=B0=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E7=A9=BA=E9=97=B4=E8=BD=AE=E8=BD=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 1 + src/common/backend/utils/misc/guc.cpp | 15 ++ .../process/postmaster/syslogger.cpp | 216 +++++++++++++++++- .../process/threadpool/knl_thread.cpp | 1 + .../knl/knl_guc/knl_session_attr_common.h | 1 + src/include/knl/knl_thread.h | 1 + 6 files changed, 233 insertions(+), 2 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 384e2612f..af83b3064 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -368,6 +368,7 @@ log_parser_stats|bool|0,0|NULL|NULL| log_planner_stats|bool|0,0|NULL|NULL| log_rotation_age|int|0,35791394|min|NULL| log_rotation_size|int|0,2097151|kB|NULL| +log_max_size|int|0,2147483647|kB|NULL| log_statement|enum|none,ddl,mod,all|NULL|Even if log_statement set to all, simple statement contains a syntax error will not be recorded. Unless log_min_error_statement set to error or lower to records that contain simple syntax errors in the statement.| log_statement_stats|bool|0,0|NULL|NULL| log_temp_files|int|-1,2147483647|kB|NULL| diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index 3f85c967a..4f846707e 100755 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -2474,6 +2474,21 @@ static void InitConfigureNamesInt() NULL, NULL}, + {{"log_max_size", + PGC_SIGHUP, + NODE_SINGLENODE, + LOGGING_WHERE, + gettext_noop("Automatic log file elimination will occur after N kilobytes."), + NULL, + GUC_UNIT_KB}, + &u_sess->attr.attr_common.LogMaxSize, + 0, + 0, + INT_MAX, + NULL, + NULL, + NULL}, + {{"max_function_args", PGC_INTERNAL, NODE_ALL, diff --git a/src/gausskernel/process/postmaster/syslogger.cpp b/src/gausskernel/process/postmaster/syslogger.cpp index beeb06aa0..2d379c706 100644 --- a/src/gausskernel/process/postmaster/syslogger.cpp +++ b/src/gausskernel/process/postmaster/syslogger.cpp @@ -42,12 +42,16 @@ #include "storage/ipc.h" #include "storage/latch.h" #include "storage/pg_shmem.h" +#include "storage/smgr/fd.h" #include "utils/guc.h" #include "utils/ps_status.h" #include "utils/timestamp.h" #include "gssignal/gs_signal.h" #define PROFILE_LOG_ROTATE_SIZE ((long)20 * 1024 * 1024L) +#define LOG_SIZE_INIT (1024L * INT_MAX + 1) +#define LOG_MAX_SIZE (u_sess->attr.attr_common.LogMaxSize * 1024L) +#define NEED_ELIMINATE_LOG(total) (total > LOG_MAX_SIZE && LOG_MAX_SIZE > 0) /* * We really want line-buffered mode for logfile output, but Windows does @@ -101,6 +105,13 @@ typedef struct { StringInfoData data; /* accumulated data, as a StringInfo */ } save_buffer; +typedef int (*cmp_func)(const void*, const void*); +typedef struct { + int64 fileSize; + time_t mTime; + char fileName[FLEXIBLE_ARRAY_MEMBER]; +} LogInfo; + /* These must be exported for EXEC_BACKEND case ... annoying */ #ifndef WIN32 @@ -153,6 +164,11 @@ static void LogCtlProcessInput(LogControlData* logctl, const char* msg, int len) static void PLogCtlInit(void); static void slow_query_logfile_rotate(bool time_based_rotation, int size_rotation_for); static void asp_logfile_rotate(bool time_based_rotation, int size_rotation_for); +static void RemoveOldestLog(const char* path, int64* totalSize, int64 curSize); +static int LogInfoCmpByModifyTime(const void* v1, const void* v2); +static void MakeSortedLogInfoList(List* list, cmp_func cmp); +static inline int64 GetUpdateSize(FILE* fh, const char* filename); +static inline void CheckTotalLogSize(FILE* fh, bool hasCsvFormat, char* currentLogDir, int64* totalSize); /* * Main entry point for syslogger process @@ -305,6 +321,13 @@ NON_EXEC_STATIC void SysLoggerMain(int fd) currentLogDir = pstrdup(u_sess->attr.attr_common.Log_directory); currentLogFilename = pstrdup(u_sess->attr.attr_common.Log_filename); currentLogRotationAge = u_sess->attr.attr_common.Log_RotationAge; + + /* + * Set it to 1 more than the maximum possible LOG_MAX_SIZE to ensure that the initial value can stably trigger + * the RemoveOldestLog. In the RemoveOldestLog, the size of the latest log folder is obtained by traversing + * the log file. + */ + t_thrd.logger.total_syslogs_size = LOG_SIZE_INIT; /* set next planned rotation time */ set_next_rotation_time(); @@ -338,6 +361,9 @@ NON_EXEC_STATIC void SysLoggerMain(int fd) if (strcmp(u_sess->attr.attr_common.Log_directory, currentLogDir) != 0) { pfree(currentLogDir); currentLogDir = pstrdup(u_sess->attr.attr_common.Log_directory); + + /* reinitialize the log size and total size */ + t_thrd.logger.total_syslogs_size = LOG_SIZE_INIT; t_thrd.logger.rotation_requested = true; /* not affect pLogCtl's Log_directory */ /* @@ -431,6 +457,18 @@ NON_EXEC_STATIC void SysLoggerMain(int fd) } } + if (!t_thrd.logger.rotation_requested && LOG_MAX_SIZE > 0) { + if (t_thrd.logger.syslogFile != NULL && ftell(t_thrd.logger.syslogFile) >= LOG_MAX_SIZE) { + t_thrd.logger.rotation_requested = true; + size_rotation_for |= LOG_DESTINATION_STDERR; + } else { + CheckTotalLogSize(t_thrd.logger.syslogFile, + true, + currentLogDir, + &t_thrd.logger.total_syslogs_size); + } + } + /* * check the other log types' file rotation before error log type. * logfile_rotate() will change t_thrd.logger.next_rotation_time value by calling @@ -1181,6 +1219,8 @@ static void logfile_rotate(bool time_based_rotation, int size_rotation_for, bool pg_time_t fntime; FILE* fh = NULL; bool is_ow_oldlog = false; + int64 update_syslog_size = 0; + int64 update_csvlog_size = 0; t_thrd.logger.rotation_requested = false; @@ -1211,14 +1251,29 @@ static void logfile_rotate(bool time_based_rotation, int size_rotation_for, bool if (u_sess->attr.attr_common.Log_truncate_on_rotation && time_based_rotation && t_thrd.logger.last_file_name != NULL) { if (strcmp(filename, t_thrd.logger.last_file_name) != 0) { + /* + * This indicates that the log is switched to a new file for overwriting. + * Therefore, the size of the file being written must be added to the total size. + * If the file to be overwritten is an existing file, the size of the existing + * file is subtracted from the total size. + */ + update_syslog_size += GetUpdateSize(t_thrd.logger.syslogFile, filename); fh = logfile_open(filename, "w", true); } else { fclose(t_thrd.logger.syslogFile); fh = logfile_open(filename, "w", true); is_ow_oldlog = true; } - } else + } else { + /* + * This indicates that logs are written in append mode. At least, the writing of the current log file + * is complete. Therefore, you need to add the size of the current log file. The file that is being written + * becomes a new log file that is being written. Therefore, the size of the file that is being written + * needs to be subtracted. + */ + update_syslog_size += GetUpdateSize(t_thrd.logger.syslogFile, filename); fh = logfile_open(filename, "a", true); + } if (fh == NULL) { /* @@ -1257,14 +1312,17 @@ static void logfile_rotate(bool time_based_rotation, int size_rotation_for, bool if (u_sess->attr.attr_common.Log_truncate_on_rotation && time_based_rotation && t_thrd.logger.last_csv_file_name != NULL) { if (strcmp(csvfilename, t_thrd.logger.last_csv_file_name) != 0) { + update_csvlog_size += GetUpdateSize(t_thrd.logger.csvlogFile, csvfilename); fh = logfile_open(csvfilename, "w", true); } else { fclose(t_thrd.logger.csvlogFile); fh = logfile_open(csvfilename, "w", true); is_ow_oldlog = true; } - } else + } else { + update_csvlog_size += GetUpdateSize(t_thrd.logger.csvlogFile, csvfilename); fh = logfile_open(csvfilename, "a", true); + } if (fh == NULL) { /* @@ -1301,6 +1359,7 @@ static void logfile_rotate(bool time_based_rotation, int size_rotation_for, bool if (csvfilename != NULL) pfree(csvfilename); + t_thrd.logger.total_syslogs_size += (update_syslog_size + update_csvlog_size); set_next_rotation_time(); } @@ -2115,3 +2174,156 @@ void init_instr_log_directory(bool include_nodename, const char* logid) pfree(logdir); } } + +static int LogInfoCmpByModifyTime(const void* v1, const void* v2) +{ + const LogInfo* l1 = *(LogInfo**)v1; + const LogInfo* l2 = *(LogInfo**)v2; + return l1->mTime - l2->mTime; +} + +static void MakeSortedLogInfoList(List* list, cmp_func cmp) +{ + int len = list_length(list); + if (0 == len || 1 == len) { + return; /* empty or single list */ + } + + void** ptr_array = (void**)palloc(len * sizeof(LogInfo)); + ListCell* cell = NULL; + + /* contruct a temp list to sort */ + int idx = 0; + foreach (cell, list) { + ptr_array[idx++] = (void*)lfirst(cell); + } + + /* do sorting */ + qsort(ptr_array, len, sizeof(void*), cmp); + + /* write back to make this list sorted */ + idx = 0; + foreach (cell, list) { + lfirst(cell) = ptr_array[idx++]; + } + + /* free this temp array */ + pfree_ext(ptr_array); +} + +static void RemoveOldestLog(const char* path, int64* totalSize, int64 curSize) +{ + char pathname[MAXPGPATH] = {'\0'}; + DIR* dirDesc = NULL; + struct dirent* dirEntry = NULL; + errno_t rc = EOK; + List* logInfoList = NIL; + int64 totalSizeBak = *totalSize; + *totalSize = 0; + bool isInterrupt = false; + + dirDesc = AllocateDir(path); + if (dirDesc == NULL) { + ereport(ERROR, (errmsg("can not open directory %s", path))); + } + + while ((dirEntry = ReadDir(dirDesc, path)) != NULL) { + if (InterruptPending) { + isInterrupt = true; + *totalSize = totalSizeBak; + FreeDir(dirDesc); + CHECK_FOR_INTERRUPTS(); + break; + } + struct stat fst; + if (unlikely(strcmp(dirEntry->d_name, ".") == 0 || strcmp(dirEntry->d_name, "..") == 0)) { + continue; + } + rc = snprintf_s(pathname, MAXPGPATH, MAXPGPATH - 1, "%s/%s", path, dirEntry->d_name); + securec_check_ss(rc, "\0", "\0"); + if (unlikely(stat(pathname, &fst) < 0)) { + FreeDir(dirDesc); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not %s file \"%s\"", errno == ENOENT ? "find" : "stat", pathname))); + } + /* just check the file */ + if (S_ISDIR(fst.st_mode)) { + continue; + } + + if ((t_thrd.logger.last_file_name != NULL && strcmp(pathname, t_thrd.logger.last_file_name) == 0) || + (t_thrd.logger.last_csv_file_name != NULL && strcmp(pathname, t_thrd.logger.last_csv_file_name) == 0)) { + continue; + } + + /* + * The feature of flexible arrays is used here to facilitate memory release of the logInfoList at the end + */ + Size dirLen = strlen(dirEntry->d_name); + LogInfo* logInfo = (LogInfo*)palloc(sizeof(LogInfo) + dirLen + 1); + logInfo->mTime = fst.st_mtime; + rc = snprintf_s(logInfo->fileName, dirLen + 1, dirLen, "%s", dirEntry->d_name); + securec_check_ss(rc, "\0", "\0"); + logInfo->fileSize = fst.st_size; + + /* Get the newest totalSize */ + *totalSize += fst.st_size; + logInfoList = lappend(logInfoList, logInfo); + } + if (!isInterrupt) { + FreeDir(dirDesc); + } else { + list_free_deep(logInfoList); + return; + } + + /* + * Perform a secondary judgment on the total number of logs. If the latest log space size does not meet + * the conditions for deleting logs, there is no need to delete them, which means there is no need to sort + * the log files in chronological order + */ + if (NEED_ELIMINATE_LOG(*totalSize + curSize)) { + MakeSortedLogInfoList(logInfoList, LogInfoCmpByModifyTime); + } + + ListCell* cell = NULL; + foreach (cell, logInfoList) { + if (!NEED_ELIMINATE_LOG(*totalSize + curSize)) { + break; + } + LogInfo* logInfo = (LogInfo*)lfirst(cell); + rc = snprintf_s(pathname, MAXPGPATH, MAXPGPATH - 1, "%s/%s", path, logInfo->fileName); + securec_check_ss(rc, "\0", "\0"); + if (remove(pathname) < 0) { + ereport(ERROR, (errmsg("remove log file \"%s\" failed.", pathname))); + } else { + *totalSize -= logInfo->fileSize; + } + } + list_free_deep(logInfoList); +} + +static inline void CheckTotalLogSize(FILE* fh, bool hasCsvFormat, char* currentLogDir, int64* totalSize) +{ + if (fh == NULL) { + return; + } + + int64 curSysLogSize = ftell(fh); + int64 curCsvLogSize = (hasCsvFormat && t_thrd.logger.csvlogFile != NULL) ? ftell(t_thrd.logger.csvlogFile) : 0; + if (NEED_ELIMINATE_LOG(*totalSize + curSysLogSize + curCsvLogSize)) { + RemoveOldestLog(currentLogDir, totalSize, curSysLogSize + curCsvLogSize); + } +} + +static inline int64 GetUpdateSize(FILE* fh, const char* filename) +{ + int64 result = 0; + result += (fh == NULL) ? 0 : ftell(fh); + struct stat fst; + if (stat(filename, &fst) >= 0) { + result -= fst.st_size; + } + return result; +} diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index b7e5f5de6..ab07980af 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -678,6 +678,7 @@ static void knl_t_logger_init(knl_t_logger_context* logger) logger->last_asp_file_name = NULL; logger->got_SIGHUP = false; logger->rotation_requested = false; + logger->total_syslogs_size = 0; } static void knl_t_bulkload_init(knl_t_bulkload_context* bulk_cxt) diff --git a/src/include/knl/knl_guc/knl_session_attr_common.h b/src/include/knl/knl_guc/knl_session_attr_common.h index fa0cb7ee6..f2c0753de 100644 --- a/src/include/knl/knl_guc/knl_session_attr_common.h +++ b/src/include/knl/knl_guc/knl_session_attr_common.h @@ -93,6 +93,7 @@ typedef struct knl_session_attr_common { int backend_flush_after; int Log_RotationAge; int Log_RotationSize; + int LogMaxSize; int max_function_args; int max_user_defined_exception; int tcp_keepalives_idle; diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index e6ebc3d29..16b39500e 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -1364,6 +1364,7 @@ typedef struct knl_t_logger_context { List* buffer_lists[NBUFFER_LISTS]; volatile sig_atomic_t got_SIGHUP; volatile sig_atomic_t rotation_requested; + int64 total_syslogs_size; } knl_t_logger_context; /*****************************************************************************