database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity
Files
0318da1df3661663cd0affdf6c073e686eb99dfe
doris/thirdparty/patches/glog-0.6.0.patch
Xinyi Zou ba351af452 [enhancement](thirdparty) upgrade thirdparty libs - again (#23414) 
submit again #23290 (not upgrade brpc, because bthread local has error)

protobuf 3.15.0 -> 21.11
glog 0.4.0 -> 0.6.0
lz4 1.9.3 -> 1.9.4
curl 7.79.0 -> 8.2.1
zstd 1.5.2 -> 1.5.5
arrow 7.0.0 -> 13.0.0
abseil 20220623.1 -> 20230125.3
orc 1.7.2 -> 1.9.0
jemalloc for arrow 5.2.1 -> 5.3.0
xsimd 7.0.0 -> 13.0.0
opentelemetry-proto 0.19.0 -> 1.0.0
opentelemetry 1.8.3 -> 1.10.0

new:
c-ares -> 1.19.1
grpc -> 1.54.3
2023-08-26 22:59:10 +08:00

354 lines
12 KiB
Diff
Raw Blame History

diff --git a/src/glog/logging.h.in b/src/glog/logging.h.in
index 95a573b..57d2e9b 100644
--- a/src/glog/logging.h.in
+++ b/src/glog/logging.h.in
@@ -425,6 +425,12 @@ typedef void(*CustomPrefixCallback)(std::ostream& s, const LogMessageInfo& l, vo
using fLS::FLAGS_##name
#endif
+// Set max log file num
+DECLARE_int32(log_filenum_quota);
+
+// Set log file split method
+DECLARE_string(log_split_method);
+
// Set whether appending a timestamp to the log file name
DECLARE_bool(timestamp_in_logfile_name);
diff --git a/src/logging.cc b/src/logging.cc
index 4028ccc..fd60fc3 100644
--- a/src/logging.cc
+++ b/src/logging.cc
@@ -34,6 +34,7 @@
#include <algorithm>
#include <cassert>
#include <iomanip>
+#include <list>
#include <string>
#ifdef HAVE_UNISTD_H
# include <unistd.h> // For _exit.
@@ -50,6 +51,7 @@
#include <iostream>
#include <cstdarg>
#include <cstdlib>
+#include <dirent.h>
#ifdef HAVE_PWD_H
# include <pwd.h>
#endif
@@ -204,6 +206,12 @@ GLOG_DEFINE_string(log_backtrace_at, "",
GLOG_DEFINE_bool(log_utc_time, false,
"Use UTC time for logging.");
+GLOG_DEFINE_string(log_split_method, "day",
+ "split log by size, day, hour");
+
+GLOG_DEFINE_int32(log_filenum_quota, 10,
+ "max log file num in log dir");
+
// TODO(hamaji): consider windows
#define PATH_SEPARATOR '/'
@@ -429,6 +437,15 @@ namespace {
namespace {
+typedef struct filetime {
+ std::string name;
+ time_t time;
+
+ bool operator < (const struct filetime& o) const {
+ return o.time > time;
+ }
+} Filetime;
+
// Encapsulates all file-system related state
class LogFileObject : public base::Logger {
public:
@@ -459,6 +476,7 @@ class LogFileObject : public base::Logger {
// can avoid grabbing a lock. Usually Flush() calls it after
// acquiring lock_.
void FlushUnlocked();
+ void CheckFileNumQuota();
private:
static const uint32 kRolloverAttemptFrequency = 0x20;
@@ -476,6 +494,9 @@ class LogFileObject : public base::Logger {
unsigned int rollover_attempt_;
int64 next_flush_time_; // cycle count at which to flush log
WallTime start_time_;
+ std::list<Filetime> file_list_;
+ bool inited_;
+ struct ::tm tm_time_;
// Actually create a logfile using the value of base_filename_ and the
// optional argument time_pid_string
@@ -665,7 +686,7 @@ inline void LogDestination::FlushLogFiles(int min_severity) {
// all this stuff.
MutexLock l(&log_mutex);
for (int i = min_severity; i < NUM_SEVERITIES; i++) {
- LogDestination* log = log_destination(i);
+ LogDestination* log = log_destinations_[i];
if (log != NULL) {
log->logger_->Flush();
}
@@ -876,9 +897,12 @@ inline void LogDestination::LogToAllLogfiles(LogSeverity severity,
} else if (FLAGS_logtostderr) { // global flag: never log to file
ColoredWriteToStderr(severity, message, len);
} else {
- for (int i = severity; i >= 0; --i) {
- LogDestination::MaybeLogToLogfile(i, timestamp, message, len);
- }
+ if (severity >= 1) {
+ LogDestination::MaybeLogToLogfile(1, timestamp, message, len);
+ LogDestination::MaybeLogToLogfile(0, timestamp, message, len);
+ } else if (severity == 0) {
+ LogDestination::MaybeLogToLogfile(0, timestamp, message, len);
+ } else {}
}
}
@@ -976,6 +1000,7 @@ LogFileObject::LogFileObject(LogSeverity severity,
file_length_(0),
rollover_attempt_(kRolloverAttemptFrequency-1),
next_flush_time_(0),
+ inited_(false),
start_time_(WallTime_Now()) {
assert(severity >= 0);
assert(severity < NUM_SEVERITIES);
@@ -1045,11 +1070,11 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) {
string_filename += filename_extension_;
const char* filename = string_filename.c_str();
//only write to files, create if non-existant.
- int flags = O_WRONLY | O_CREAT;
- if (FLAGS_timestamp_in_logfile_name) {
- //demand that the file is unique for our timestamp (fail if it exists).
- flags = flags | O_EXCL;
- }
+ int flags = O_WRONLY | O_CREAT | O_APPEND;
+ // if (FLAGS_timestamp_in_logfile_name) {
+ // //demand that the file is unique for our timestamp (fail if it exists).
+ // flags = flags | O_EXCL;
+ // }
int fd = open(filename, flags, FLAGS_logfile_mode);
if (fd == -1) return false;
#ifdef HAVE_FCNTL
@@ -1097,6 +1122,10 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) {
}
}
#endif
+ Filetime ft;
+ ft.name = string_filename;
+ file_list_.push_back(ft);
+
// We try to create a symlink called <program_name>.<severity>,
// which is easier to use. (Every time we create a new logfile,
// we destroy the old symlink and create a new one, so it always
@@ -1138,6 +1167,59 @@ bool LogFileObject::CreateLogfile(const string& time_pid_string) {
return true; // Everything worked
}
+void LogFileObject::CheckFileNumQuota() {
+ struct dirent *entry;
+ DIR *dp;
+
+ const vector<string> & log_dirs = GetLoggingDirectories();
+ if (log_dirs.size() < 1) return;
+
+ //fprintf(stderr, "log dir: %s\n", log_dirs[0].c_str());
+
+ // list file in log dir
+ dp = opendir(log_dirs[0].c_str());
+ if (dp == NULL) {
+ fprintf(stderr, "open log dir %s fail\n", log_dirs[0].c_str());
+ return;
+ }
+
+ file_list_.clear();
+ while ((entry = readdir(dp)) != NULL) {
+ if (DT_DIR == entry->d_type ||
+ DT_LNK == entry->d_type) {
+ continue;
+ }
+ std::string filename = std::string(entry->d_name);
+ //fprintf(stderr, "filename: %s\n", filename.c_str());
+
+ if (filename.find(symlink_basename_ + '.' + LogSeverityNames[severity_]) == 0) {
+ std::string filepath = log_dirs[0] + "/" + filename;
+
+ struct stat fstat;
+ if (::stat(filepath.c_str(), &fstat) < 0) {
+ fprintf(stderr, "state %s fail\n", filepath.c_str());
+ closedir(dp);
+ return;
+ }
+ //fprintf(stderr, "filepath: %s\n", filepath.c_str());
+
+ Filetime file_time;
+ file_time.time = fstat.st_mtime;
+ file_time.name = filepath;
+ file_list_.push_back(file_time);
+ }
+ }
+ closedir(dp);
+
+ file_list_.sort();
+
+ while (FLAGS_log_filenum_quota > 0 && file_list_.size() >= FLAGS_log_filenum_quota) {
+ // fprintf(stderr, "delete %s\n", file_list_.front().name.c_str());
+ unlink(file_list_.front().name.c_str());
+ file_list_.pop_front();
+ }
+}
+
void LogFileObject::Write(bool force_flush,
time_t timestamp,
const char* message,
@@ -1149,13 +1231,55 @@ void LogFileObject::Write(bool force_flush,
return;
}
- if (file_length_ >> 20U >= MaxLogSize() || PidHasChanged()) {
+ struct ::tm tm_time;
+
+ bool is_split = false;
+ if ("day" == FLAGS_log_split_method) {
+ localtime_r(&timestamp, &tm_time);
+ if (tm_time.tm_year != tm_time_.tm_year
+ || tm_time.tm_mon != tm_time_.tm_mon
+ || tm_time.tm_mday != tm_time_.tm_mday) {
+ is_split = true;
+ }
+ } else if ("hour" == FLAGS_log_split_method) {
+ localtime_r(&timestamp, &tm_time);
+ if (tm_time.tm_year != tm_time_.tm_year
+ || tm_time.tm_mon != tm_time_.tm_mon
+ || tm_time.tm_mday != tm_time_.tm_mday
+ || tm_time.tm_hour != tm_time_.tm_hour) {
+ is_split = true;
+ }
+ } else if (file_length_ >> 20U >= MaxLogSize()) {
+ // PidHasChanged()) {
+ is_split = true;
+ }
+
+ if (is_split) {
if (file_ != NULL) fclose(file_);
file_ = NULL;
file_length_ = bytes_since_flush_ = dropped_mem_length_ = 0;
rollover_attempt_ = kRolloverAttemptFrequency - 1;
}
+ if ((file_ == NULL) && (!inited_) && (FLAGS_log_split_method == "size")) {
+ CheckFileNumQuota();
+ const char* filename = file_list_.back().name.c_str();
+ int fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, FLAGS_logfile_mode);
+ if (fd != -1) {
+#ifdef HAVE_FCNTL
+ // Mark the file close-on-exec. We don't really care if this fails
+ fcntl(fd, F_SETFD, FD_CLOEXEC);
+#endif
+ file_ = fopen(filename, "a+"); // Read and append a FILE*.
+ if (file_ == NULL) { // Man, we're screwed!, try to create new log file
+ close(fd);
+ }
+ fseek(file_, 0, SEEK_END);
+ file_length_ = bytes_since_flush_ = ftell(file_);
+ inited_ = true;
+ }
+ }
+
// If there's no destination file, make one before outputting
if (file_ == NULL) {
// Try to rollover the log file every 32 log messages. The only time
@@ -1164,7 +1288,15 @@ void LogFileObject::Write(bool force_flush,
if (++rollover_attempt_ != kRolloverAttemptFrequency) return;
rollover_attempt_ = 0;
- struct ::tm tm_time;
+ if (!inited_) {
+ CheckFileNumQuota();
+ inited_ = true;
+ } else {
+ while (FLAGS_log_filenum_quota > 0 && file_list_.size() >= FLAGS_log_filenum_quota) {
+ unlink(file_list_.front().name.c_str());
+ file_list_.pop_front();
+ }
+ }
if (FLAGS_log_utc_time) {
gmtime_r(&timestamp, &tm_time);
} else {
@@ -1176,13 +1308,19 @@ void LogFileObject::Write(bool force_flush,
time_pid_stream.fill('0');
time_pid_stream << 1900+tm_time.tm_year
<< setw(2) << 1+tm_time.tm_mon
- << setw(2) << tm_time.tm_mday
- << '-'
- << setw(2) << tm_time.tm_hour
- << setw(2) << tm_time.tm_min
- << setw(2) << tm_time.tm_sec
- << '.'
- << GetMainThreadPid();
+ << setw(2) << tm_time.tm_mday;
+
+ if ("hour" == FLAGS_log_split_method) {
+ time_pid_stream << setw(2) << tm_time.tm_hour;
+ } else if ("day" != FLAGS_log_split_method) {
+ time_pid_stream << '-'
+ << setw(2) << tm_time.tm_hour
+ << setw(2) << tm_time.tm_min
+ << setw(2) << tm_time.tm_sec;
+ }
+
+ tm_time_ = tm_time;
+
const string& time_pid_string = time_pid_stream.str();
if (base_filename_selected_) {
@@ -1216,9 +1354,7 @@ void LogFileObject::Write(bool force_flush,
// deadlock. Simply use a name like invalid-user.
if (uidname.empty()) uidname = "invalid-user";
- stripped_filename = stripped_filename+'.'+hostname+'.'
- +uidname+".log."
- +LogSeverityNames[severity_]+'.';
+ stripped_filename = stripped_filename + "." + LogSeverityNames[severity_] + ".log.";
// We're going to (potentially) try to put logs in several different dirs
const vector<string> & log_dirs = GetLoggingDirectories();
@@ -1243,36 +1379,6 @@ void LogFileObject::Write(bool force_flush,
}
}
- // Write a header message into the log file
- ostringstream file_header_stream;
- file_header_stream.fill('0');
- file_header_stream << "Log file created at: "
- << 1900+tm_time.tm_year << '/'
- << setw(2) << 1+tm_time.tm_mon << '/'
- << setw(2) << tm_time.tm_mday
- << ' '
- << setw(2) << tm_time.tm_hour << ':'
- << setw(2) << tm_time.tm_min << ':'
- << setw(2) << tm_time.tm_sec << (FLAGS_log_utc_time ? " UTC\n" : "\n")
- << "Running on machine: "
- << LogDestination::hostname() << '\n';
-
- if(!g_application_fingerprint.empty()) {
- file_header_stream << "Application fingerprint: " << g_application_fingerprint << '\n';
- }
- const char* const date_time_format = FLAGS_log_year_in_prefix
- ? "yyyymmdd hh:mm:ss.uuuuuu"
- : "mmdd hh:mm:ss.uuuuuu";
- file_header_stream << "Running duration (h:mm:ss): "
- << PrettyDuration(static_cast<int>(WallTime_Now() - start_time_)) << '\n'
- << "Log line format: [IWEF]" << date_time_format << " "
- << "threadid file:line] msg" << '\n';
- const string& file_header_string = file_header_stream.str();
-
- const size_t header_len = file_header_string.size();
- fwrite(file_header_string.data(), 1, header_len, file_);
- file_length_ += header_len;
- bytes_since_flush_ += header_len;
}
// Write to LOG file
Reference in New Issue View Git Blame Copy Permalink