database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](log) refine the BE logger (#35942) (#35988)

Browse Source 
bp #35942
This commit is contained in:
Mingyu Chen
2024-06-06 22:25:22 +08:00
committed by GitHub
parent af779f5cd8
commit fbc82e0253
6 changed files with 75 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/CMakeLists.txt
View File

@ -66,6 +66,9 @@ else()
add_definitions(-DBOOST_STACKTRACE_USE_NOOP)
endif()
# enable glog custom prefix
add_definitions(-DGLOG_CUSTOM_PREFIX_SUPPORT)
# Options
option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
option(USE_LIBCPP "Use libc++" OFF)

2
be/src/common/config.cpp
View File

@ -1228,6 +1228,8 @@ DEFINE_Bool(enable_jvm_monitor, "false");
// Skip loading stale rowset meta when initializing `TabletMeta` from protobuf
DEFINE_mBool(skip_loading_stale_rowset_meta, "false");
DEFINE_Bool(enable_file_logger, "true");
// clang-format off
#ifdef BE_TEST
// test s3

5
be/src/common/config.h
View File

@ -1308,6 +1308,11 @@ DECLARE_Bool(enable_jvm_monitor);
// Skip loading stale rowset meta when initializing `TabletMeta` from protobuf
DECLARE_mBool(skip_loading_stale_rowset_meta);
// Whether to use file to record log. When starting BE with --console,
// all logs will be written to both standard output and file.
// Disable this option will no longer use file to record log.
// Only works when starting BE with --console.
DECLARE_Bool(enable_file_logger);
#ifdef BE_TEST
// test s3

39
be/src/common/logconfig.cpp
View File

@ -21,6 +21,7 @@
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <iomanip>
#include <iostream>
#include <mutex>
#include <string>
@ -48,6 +49,28 @@ static bool iequals(const std::string& a, const std::string& b) {
return true;
}
void custom_prefix(std::ostream& s, const google::LogMessageInfo& l, void*) {
// Add prefix "RuntimeLogger ".
s << "RuntimeLogger ";
// Same as in fe.log
// The following is same as default log format. eg:
// I20240605 15:25:15.677153 1763151 wal_manager.cpp:481] msg...
s << l.severity[0];
s << std::setw(4) << 1900 + l.time.year();
s << std::setw(2) << 1 + l.time.month();
s << std::setw(2) << l.time.day();
s << ' ';
s << std::setw(2) << l.time.hour() << ':';
s << std::setw(2) << l.time.min() << ':';
s << std::setw(2) << l.time.sec() << ".";
s << std::setw(6) << l.time.usec();
s << ' ';
s << std::setfill(' ') << std::setw(5);
s << l.thread_id << std::setfill('0');
s << ' ';
s << l.filename << ':' << l.line_number << "]";
}
bool init_glog(const char* basename) {
std::lock_guard<std::mutex> logging_lock(logging_mutex);
@ -55,8 +78,13 @@ bool init_glog(const char* basename) {
return true;
}
if (getenv("DORIS_LOG_TO_STDERR") != nullptr) {
FLAGS_alsologtostderr = true;
bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr);
if (log_to_console) {
if (config::enable_file_logger) {
FLAGS_alsologtostderr = true;
} else {
FLAGS_logtostderr = true;
}
}
// don't log to stderr except fatal level
@ -144,7 +172,12 @@ bool init_glog(const char* basename) {
}
}
google::InitGoogleLogging(basename);
if (log_to_console) {
// Only add prefix if log output to stderr
google::InitGoogleLogging(basename, &custom_prefix);
} else {
google::InitGoogleLogging(basename);
}
logging_initialized = true;

45
bin/start_be.sh
View File

@ -140,7 +140,7 @@ export CLASSPATH="${DORIS_HOME}/conf/:${DORIS_CLASSPATH}:${CLASSPATH}"
# DORIS_CLASSPATH is for self-managed jni
export DORIS_CLASSPATH="-Djava.class.path=${DORIS_CLASSPATH}"
#echo ${DORIS_CLASSPATH}
# log ${DORIS_CLASSPATH}
export LD_LIBRARY_PATH="${DORIS_HOME}/lib/hadoop_hdfs/native:${LD_LIBRARY_PATH}"
@ -181,16 +181,6 @@ PID_DIR="$(
)"
export PID_DIR
# set odbc conf path
export ODBCSYSINI="${DORIS_HOME}/conf"
# support utf8 for oracle database
export NLS_LANG='AMERICAN_AMERICA.AL32UTF8'
# filter known leak.
export LSAN_OPTIONS="suppressions=${DORIS_HOME}/conf/lsan_suppr.conf"
export ASAN_OPTIONS="suppressions=${DORIS_HOME}/conf/asan_suppr.conf"
while read -r line; do
envline="$(echo "${line}" |
sed 's/[[:blank:]]*=[[:blank:]]*/=/g' |
@ -203,6 +193,27 @@ while read -r line; do
fi
done <"${DORIS_HOME}/conf/be.conf"
STDOUT_LOGGER="${LOG_DIR}/be.out"
log() {
# same datetime format as in fe.log: 2024-06-03 14:54:41,478
cur_date=$(date +"%Y-%m-%d %H:%M:%S,$(date +%3N)")
if [[ "${RUN_CONSOLE}" -eq 1 ]]; then
echo "StdoutLogger ${cur_date} $1"
else
echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}"
fi
}
# set odbc conf path
export ODBCSYSINI="${DORIS_HOME}/conf"
# support utf8 for oracle database
export NLS_LANG='AMERICAN_AMERICA.AL32UTF8'
# filter known leak.
export LSAN_OPTIONS="suppressions=${DORIS_HOME}/conf/lsan_suppr.conf"
export ASAN_OPTIONS="suppressions=${DORIS_HOME}/conf/asan_suppr.conf"
if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then
# shellcheck disable=1091
source "${DORIS_HOME}/bin/palo_env.sh"
@ -220,7 +231,7 @@ fi
for var in http_proxy HTTP_PROXY https_proxy HTTPS_PROXY; do
if [[ -n ${!var} ]]; then
echo "env '${var}' = '${!var}', need unset it using 'unset ${var}'"
log "env '${var}' = '${!var}', need unset it using 'unset ${var}'"
exit 1
fi
done
@ -241,7 +252,7 @@ if [[ -f "${pidfile}" ]]; then
fi
chmod 550 "${DORIS_HOME}/lib/doris_be"
echo "start time: $(date)" >>"${LOG_DIR}/be.out"
log "Start time: $(date)"
if [[ ! -f '/bin/limit3' ]]; then
LIMIT=''
@ -287,7 +298,7 @@ set_tcmalloc_heap_limit() {
fi
if [[ "${mem_limit_mb}" -gt "${total_mem_mb}" ]]; then
echo "mem_limit is larger than whole memory of the server. ${mem_limit_mb} > ${total_mem_mb}."
log "mem_limit is larger than the total memory of the server. ${mem_limit_mb} > ${total_mem_mb}"
return 1
fi
export TCMALLOC_HEAP_LIMIT_MB=${mem_limit_mb}
@ -342,9 +353,9 @@ fi
# set LIBHDFS_OPTS for hadoop libhdfs
export LIBHDFS_OPTS="${final_java_opt}"
#echo "CLASSPATH: ${CLASSPATH}"
#echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
#echo "LIBHDFS_OPTS: ${LIBHDFS_OPTS}"
# log "CLASSPATH: ${CLASSPATH}"
# log "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
# log "LIBHDFS_OPTS: ${LIBHDFS_OPTS}"
if [[ -z ${JEMALLOC_CONF} ]]; then
JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"

2
regression-test/framework/pom.xml
View File

@ -230,7 +230,7 @@ under the License.
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.0.1-jre</version>
<version>32.1.2-jre</version>
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>