[opt](log) refine the FE logger (#35679)
Previously, FE logs were written to files. The main FE logs include fe.log, fe.warn.log, fe.audit.log, fe.out, and fe.gc.log. In a K8s deployment environment, logs usually need to be output to standard output, and then other components process the log stream. This PR made the following changes: 1. Modified the log4j configuration template - When started with `--daemon`, logs are still written to various files, and the format remains unchanged. - When started with `--console`, all logs are output to standard output and marked with different prefixes: - `StdoutLogger`: logs for standard output - `StderrLogger`: logs for standard error output - `RuntimeLogger`: logs for fe.log or fe.warn.log - `AuditLogger:` logs for fe.audit.log - No prefix: logs for fe.gc.log Examples are as follows: ``` RuntimeLogger 2024-06-03 14:54:51,229 INFO (binlog-gcer|62) [BinlogManager.gc():359] begin gc binlog ``` 2. Added a new FE config: `enable_file_logger` Defaults to true. Indicates that logs will be recorded to files regardless of the startup method. For example, if it is started with `--console`, the log will be output to both the file and the standard output. If it is `false`, the log will not be recorded in the file regardless of the startup method. 3. Optimized the log format of standard output The byte streams of stdout and stderr are captured. The logs previously outputted using `System.out` will be captured in fe.log for unified management.
This commit is contained in:
Mingyu Chen
@ -116,6 +116,18 @@ under the License.
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-s3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-iostreams</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<finalName>doris-fe-common</finalName>
|
||||
|
||||
@ -127,6 +127,13 @@ public class Config extends ConfigBase {
|
||||
@ConfField(description = {"是否压缩 FE 的 Audit 日志", "enable compression for FE audit log file"})
|
||||
public static boolean audit_log_enable_compress = false;
|
||||
|
||||
@ConfField(description = {"是否使用文件记录日志。当使用 --console 启动 FE 时，全部日志同时写入到标准输出和文件。"
|
||||
+ "如果关闭这个选项，不再使用文件记录日志。",
|
||||
"Whether to use file to record log. When starting FE with --console, "
|
||||
+ "all logs will be written to both standard output and file. "
|
||||
+ "Close this option will no longer use file to record log."})
|
||||
public static boolean enable_file_logger = true;
|
||||
|
||||
@ConfField(mutable = false, masterOnly = false,
|
||||
description = {"是否检查table锁泄漏", "Whether to check table lock leaky"})
|
||||
public static boolean check_table_lock_leaky = false;
|
||||
|
||||
@ -0,0 +1,97 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.common;
|
||||
|
||||
import org.apache.logging.log4j.core.Layout;
|
||||
import org.apache.logging.log4j.core.LogEvent;
|
||||
import org.apache.logging.log4j.core.config.Node;
|
||||
import org.apache.logging.log4j.core.config.plugins.Plugin;
|
||||
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
|
||||
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
|
||||
import org.apache.logging.log4j.core.layout.AbstractStringLayout;
|
||||
import org.apache.logging.log4j.core.layout.PatternLayout;
|
||||
import org.apache.logging.log4j.core.util.StringBuilderWriter;
|
||||
|
||||
import java.io.PrintWriter;
|
||||
import java.nio.charset.Charset;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
public class LogUtils {
|
||||
|
||||
public static final String STDOUT_LOG_MARKER = "StdoutLogger ";
|
||||
public static final String STDERR_LOG_MARKER = "StderrLogger ";
|
||||
|
||||
private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS");
|
||||
|
||||
private static String formattedTime() {
|
||||
LocalDateTime dateTime = LocalDateTime.now();
|
||||
return dateTime.format(TIME_FORMATTER);
|
||||
}
|
||||
|
||||
// Developer should use `LogUtils.stdout` or `LogUtils.stderr`
|
||||
// instead of `System.out` and `System.err`.
|
||||
public static void stdout(String message) {
|
||||
System.out.println(STDOUT_LOG_MARKER + formattedTime() + " " + message);
|
||||
}
|
||||
|
||||
public static void stderr(String message) {
|
||||
System.err.println(STDERR_LOG_MARKER + formattedTime() + " " + message);
|
||||
}
|
||||
|
||||
// TODO: this custom layout is not used in the codebase, but it is a good example of how to create a custom layout
|
||||
// 1. Add log4j2.component.properties in fe/conf with content:
|
||||
// log4j.layoutFactory=org.apache.doris.common.LogUtils$SingleLineExceptionLayout
|
||||
// 2. Change PatternLayout in Log4jConfig.java to SingleLineExceptionLayout
|
||||
@Plugin(name = "SingleLineExceptionLayout", category = Node.CATEGORY,
|
||||
elementType = Layout.ELEMENT_TYPE, printObject = true)
|
||||
public static class SingleLineExceptionLayout extends AbstractStringLayout {
|
||||
|
||||
private final PatternLayout patternLayout;
|
||||
|
||||
protected SingleLineExceptionLayout(PatternLayout patternLayout, Charset charset) {
|
||||
super(charset);
|
||||
this.patternLayout = patternLayout;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toSerializable(LogEvent event) {
|
||||
StringBuilder result = new StringBuilder(patternLayout.toSerializable(event));
|
||||
|
||||
if (event.getThrown() != null) {
|
||||
StringBuilderWriter sw = new StringBuilderWriter();
|
||||
event.getThrown().printStackTrace(new PrintWriter(sw));
|
||||
String stackTrace = sw.toString().replace("\n", " ").replace("\r", " ");
|
||||
result.append(stackTrace);
|
||||
}
|
||||
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
@PluginFactory
|
||||
public static Layout<String> createLayout(
|
||||
@PluginAttribute(value = "pattern") String pattern,
|
||||
@PluginAttribute(value = "charset", defaultString = "UTF-8") Charset charset) {
|
||||
PatternLayout patternLayout = PatternLayout.newBuilder()
|
||||
.withPattern(pattern)
|
||||
.withCharset(charset)
|
||||
.build();
|
||||
return new SingleLineExceptionLayout(patternLayout, charset);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user