diff --git a/bin/start_fe.sh b/bin/start_fe.sh index dc8d785784..d4648c805d 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -175,6 +175,17 @@ if [[ ! -d "${LOG_DIR}" ]]; then mkdir -p "${LOG_DIR}" fi +STDOUT_LOGGER="${LOG_DIR}/fe.out" +log() { + # same datetime format as in fe.log: 2024-06-03 14:54:41,478 + cur_date=$(date +"%Y-%m-%d %H:%M:%S,$(date +%3N)") + if [[ "${RUN_CONSOLE}" -eq 1 ]]; then + echo "StdoutLogger ${cur_date} $1" + else + echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}" + fi +} + # check java version and choose correct JAVA_OPTS java_version="$( set -e @@ -183,19 +194,19 @@ java_version="$( final_java_opt="${JAVA_OPTS}" if [[ "${java_version}" -gt 16 ]]; then if [[ -z "${JAVA_OPTS_FOR_JDK_17}" ]]; then - echo "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf" >>"${LOG_DIR}/fe.out" + log "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf" exit 1 fi final_java_opt="${JAVA_OPTS_FOR_JDK_17}" elif [[ "${java_version}" -gt 8 ]]; then if [[ -z "${JAVA_OPTS_FOR_JDK_9}" ]]; then - echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >>"${LOG_DIR}/fe.out" + log "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" exit 1 fi final_java_opt="${JAVA_OPTS_FOR_JDK_9}" fi -echo "using java version ${java_version}" >>"${LOG_DIR}/fe.out" -echo "${final_java_opt}" >>"${LOG_DIR}/fe.out" +log "using java version ${java_version}" +log "${final_java_opt}" # add libs to CLASSPATH DORIS_FE_JAR= @@ -242,7 +253,8 @@ if [[ -n "${JACOCO_COVERAGE_OPT}" ]]; then coverage_opt="${JACOCO_COVERAGE_OPT}" fi -date >>"${LOG_DIR}/fe.out" +CUR_DATE=$(date) +log "start time: ${CUR_DATE}" if [[ "${HELPER}" != "" ]]; then # change it to '-helper' to be compatible with code in Frontend @@ -256,12 +268,12 @@ if [[ "${IMAGE_TOOL}" -eq 1 ]]; then echo "Internal Error. USE IMAGE_TOOL like : ./start_fe.sh --image image_path" fi elif [[ "${RUN_DAEMON}" -eq 1 ]]; then - nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE ${HELPER:+${HELPER}} "${METADATA_FAILURE_RECOVERY}" "$@" >>"${LOG_DIR}/fe.out" 2>&1 >"${STDOUT_LOGGER}" 2>&1 >"${LOG_DIR}/fe.out" 2>&1 >"${STDOUT_LOGGER}" 2>&1 com.amazonaws aws-java-sdk-s3 + + org.apache.logging.log4j + log4j-web + + + org.apache.logging.log4j + log4j-iostreams + + + org.apache.logging.log4j + log4j-core + doris-fe-common diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 5f1d104781..43c5889ccf 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -127,6 +127,13 @@ public class Config extends ConfigBase { @ConfField(description = {"是否压缩 FE 的 Audit 日志", "enable compression for FE audit log file"}) public static boolean audit_log_enable_compress = false; + @ConfField(description = {"是否使用文件记录日志。当使用 --console 启动 FE 时，全部日志同时写入到标准输出和文件。" + + "如果关闭这个选项，不再使用文件记录日志。", + "Whether to use file to record log. When starting FE with --console, " + + "all logs will be written to both standard output and file. " + + "Close this option will no longer use file to record log."}) + public static boolean enable_file_logger = true; + @ConfField(mutable = false, masterOnly = false, description = {"是否检查table锁泄漏", "Whether to check table lock leaky"}) public static boolean check_table_lock_leaky = false; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java new file mode 100644 index 0000000000..2f93cdb885 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.layout.AbstractStringLayout; +import org.apache.logging.log4j.core.layout.PatternLayout; +import org.apache.logging.log4j.core.util.StringBuilderWriter; + +import java.io.PrintWriter; +import java.nio.charset.Charset; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +public class LogUtils { + + public static final String STDOUT_LOG_MARKER = "StdoutLogger "; + public static final String STDERR_LOG_MARKER = "StderrLogger "; + + private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS"); + + private static String formattedTime() { + LocalDateTime dateTime = LocalDateTime.now(); + return dateTime.format(TIME_FORMATTER); + } + + // Developer should use `LogUtils.stdout` or `LogUtils.stderr` + // instead of `System.out` and `System.err`. + public static void stdout(String message) { + System.out.println(STDOUT_LOG_MARKER + formattedTime() + " " + message); + } + + public static void stderr(String message) { + System.err.println(STDERR_LOG_MARKER + formattedTime() + " " + message); + } + + // TODO: this custom layout is not used in the codebase, but it is a good example of how to create a custom layout + // 1. Add log4j2.component.properties in fe/conf with content: + // log4j.layoutFactory=org.apache.doris.common.LogUtils$SingleLineExceptionLayout + // 2. Change PatternLayout in Log4jConfig.java to SingleLineExceptionLayout + @Plugin(name = "SingleLineExceptionLayout", category = Node.CATEGORY, + elementType = Layout.ELEMENT_TYPE, printObject = true) + public static class SingleLineExceptionLayout extends AbstractStringLayout { + + private final PatternLayout patternLayout; + + protected SingleLineExceptionLayout(PatternLayout patternLayout, Charset charset) { + super(charset); + this.patternLayout = patternLayout; + } + + @Override + public String toSerializable(LogEvent event) { + StringBuilder result = new StringBuilder(patternLayout.toSerializable(event)); + + if (event.getThrown() != null) { + StringBuilderWriter sw = new StringBuilderWriter(); + event.getThrown().printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString().replace("\n", " ").replace("\r", " "); + result.append(stackTrace); + } + + return result.toString(); + } + + @PluginFactory + public static Layout createLayout( + @PluginAttribute(value = "pattern") String pattern, + @PluginAttribute(value = "charset", defaultString = "UTF-8") Charset charset) { + PatternLayout patternLayout = PatternLayout.newBuilder() + .withPattern(pattern) + .withCharset(charset) + .build(); + return new SingleLineExceptionLayout(patternLayout, charset); + } + } +} diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index ef5cd1f5d0..22b60de41e 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -90,11 +90,6 @@ under the License. fe-common ${project.version} - - org.apache.logging.log4j - log4j-web - ${log4j2.version} - org.springframework.boot spring-boot-devtools diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index bc6a95e6db..d1d7ab1f21 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -24,6 +24,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.LdapConfig; import org.apache.doris.common.Log4jConfig; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.Version; import org.apache.doris.common.util.JdkUtils; @@ -388,11 +389,11 @@ public class DorisFE { } private static void printVersion() { - System.out.println("Build version: " + Version.DORIS_BUILD_VERSION); - System.out.println("Build time: " + Version.DORIS_BUILD_TIME); - System.out.println("Build info: " + Version.DORIS_BUILD_INFO); - System.out.println("Build hash: " + Version.DORIS_BUILD_HASH); - System.out.println("Java compile version: " + Version.DORIS_JAVA_COMPILE_VERSION); + LogUtils.stdout("Build version: " + Version.DORIS_BUILD_VERSION); + LogUtils.stdout("Build time: " + Version.DORIS_BUILD_TIME); + LogUtils.stdout("Build info: " + Version.DORIS_BUILD_INFO); + LogUtils.stdout("Build hash: " + Version.DORIS_BUILD_HASH); + LogUtils.stdout("Java compile version: " + Version.DORIS_JAVA_COMPILE_VERSION); } private static void checkCommandLineOptions(CommandLineOptions cmdLineOpts) { @@ -409,16 +410,16 @@ public class DorisFE { } else if (cmdLineOpts.runImageTool()) { File imageFile = new File(cmdLineOpts.getImagePath()); if (!imageFile.exists()) { - System.out.println("image does not exist: " + imageFile.getAbsolutePath() + LogUtils.stderr("image does not exist: " + imageFile.getAbsolutePath() + " . Please put an absolute path instead"); System.exit(-1); } else { - System.out.println("Start to load image: "); + LogUtils.stdout("Start to load image: "); try { MetaReader.read(imageFile, Env.getCurrentEnv()); - System.out.println("Load image success. Image file " + cmdLineOpts.getImagePath() + " is valid"); + LogUtils.stdout("Load image success. Image file " + cmdLineOpts.getImagePath() + " is valid"); } catch (Exception e) { - System.out.println("Load image failed. Image file " + cmdLineOpts.getImagePath() + " is invalid"); + LogUtils.stderr("Load image failed. Image file " + cmdLineOpts.getImagePath() + " is invalid"); LOG.warn("", e); } finally { System.exit(0); @@ -476,9 +477,7 @@ public class DorisFE { + "same time"); } - private static void releaseFileLockAndCloseFileChannel() { - if (processFileLock != null && processFileLock.isValid()) { try { processFileLock.release(); @@ -493,7 +492,6 @@ public class DorisFE { LOG.warn("release process lock file failed", ignored); } } - } public static class StartupOptions { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 93edb9dce5..4209eeaa53 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -103,6 +103,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.NereidsSqlCacheManager; import org.apache.doris.common.Pair; @@ -1581,7 +1582,7 @@ public class Env { checkLowerCaseTableNames(); String msg = "master finished to replay journal, can write now."; - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); LOG.info(msg); // for master, there are some new thread pools need to register metric ThreadPoolManager.registerAllThreadPoolMetric(); @@ -2655,7 +2656,7 @@ public class Env { try { String msg = "notify new FE type transfer: " + newType; LOG.warn(msg); - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); this.typeTransferQueue.put(newType); } catch (InterruptedException e) { LOG.error("failed to put new FE type: {}", newType, e); @@ -2673,7 +2674,7 @@ public class Env { newType = typeTransferQueue.take(); } catch (InterruptedException e) { LOG.error("got exception when take FE type from queue", e); - Util.stdoutWithTime("got exception when take FE type from queue. " + e.getMessage()); + LogUtils.stdout("got exception when take FE type from queue. " + e.getMessage()); System.exit(-1); } Preconditions.checkNotNull(newType); @@ -2755,7 +2756,7 @@ public class Env { // exit if master changed to any other type String msg = "transfer FE type from MASTER to " + newType.name() + ". exit"; LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); System.exit(-1); break; } @@ -6189,3 +6190,4 @@ public class Env { } } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 8c840a876b..5395de076a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -21,15 +21,18 @@ import org.apache.doris.httpv2.config.SpringLog4j2Config; import com.google.common.base.Strings; import com.google.common.collect.Maps; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.ConfigurationSource; import org.apache.logging.log4j.core.config.xml.XmlConfiguration; import org.apache.logging.log4j.core.lookup.Interpolator; import org.apache.logging.log4j.core.lookup.StrSubstitutor; +import org.apache.logging.log4j.io.IoBuilder; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -54,20 +57,55 @@ public class Log4jConfig extends XmlConfiguration { } } + // Placeholders + private static final String RUNTIME_LOG_FORMAT_PLACEHOLDER = ""; + private static final String VERBOSE_MODULE_PLACEHOLDER = ""; + private static final String CONSOLE_APPENDER_PLACEHOLDER = ""; + private static final String RUNTIME_LOG_FILE_APPENDER_PLACEHOLDER = ""; + private static final String RUNTIME_LOG_WARN_FILE_APPENDER_PLACEHOLDER = ""; + private static final String AUDIT_CONSOLE_LOGGER_PLACEHOLDER = ""; + private static final String AUDIT_FILE_LOGGER_PLACEHOLDER = ""; + private static final String RUNTIME_LOG_MARKER_PLACEHOLDER = ""; + private static final String AUDIT_LOG_MARKER_PLACEHOLDER = ""; + + // Appender names + private static final String RUNTIME_LOG_CONSOLE_APPENDER = "Console"; + private static final String RUNTIME_LOG_FILE_APPENDER = "Sys"; + private static final String RUNTIME_LOG_WARN_FILE_APPENDER = "SysWF"; + private static final String AUDIT_LOG_CONSOLE_APPENDER = "AuditConsole"; + private static final String AUDIT_LOG_FILE_APPENDER = "AuditFile"; + + // Log patterns + private static final String RUNTIME_LOG_PATTERN + = RUNTIME_LOG_MARKER_PLACEHOLDER + "%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)" + + RUNTIME_LOG_FORMAT_PLACEHOLDER + "%m%n"; + private static final String AUDIT_LOG_PATTERN + = AUDIT_LOG_MARKER_PLACEHOLDER + "%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n"; + + // Log markers + private static final String RUNTIME_LOG_MARKER = "RuntimeLogger "; + private static final String AUDIT_LOG_MARKER = "AuditLogger "; + + // @formatter:off static { // CHECKSTYLE OFF xmlConfTemplateBuilder.append("\n") .append("\n\n") .append("\n") .append(" \n") - .append(" ") + .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") + .append(" " + RUNTIME_LOG_PATTERN + "\n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") + .append(" " + AUDIT_LOG_PATTERN + "\n") + .append(" \n") + .append(" \n") + .append(" \n") + .append(" \n") + .append(" " + RUNTIME_LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -83,9 +121,9 @@ public class Log4jConfig extends XmlConfiguration { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") + .append(" " + RUNTIME_LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -101,9 +139,9 @@ public class Log4jConfig extends XmlConfiguration { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n\n") + .append(" " + AUDIT_LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -122,18 +160,20 @@ public class Log4jConfig extends XmlConfiguration { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") - .append(" \n") - .append(" \n") + .append(" " + RUNTIME_LOG_FILE_APPENDER_PLACEHOLDER + "\n") + .append(" " + RUNTIME_LOG_WARN_FILE_APPENDER_PLACEHOLDER + "\n") + .append(" " + CONSOLE_APPENDER_PLACEHOLDER + "\n") .append(" \n") .append(" \n") - .append(" \n") + .append(" " + AUDIT_FILE_LOGGER_PLACEHOLDER + "\n") + .append(" " + AUDIT_CONSOLE_LOGGER_PLACEHOLDER + "\n") .append(" \n") - .append(" \n") + .append(" " + VERBOSE_MODULE_PLACEHOLDER + "\n") .append(" \n") .append(""); // CHECKSTYLE ON } + // @formatter:on private static StrSubstitutor strSub; private static String sysLogLevel; @@ -210,24 +250,33 @@ public class Log4jConfig extends XmlConfiguration { for (String s : auditModules) { sb.append(""); } - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - sb.toString()); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(VERBOSE_MODULE_PLACEHOLDER, sb.toString()); if (sysLogMode.equalsIgnoreCase("NORMAL")) { - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - " [%C{1}.%M():%L] "); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_FORMAT_PLACEHOLDER, " [%C{1}.%M():%L] "); } else { - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", " "); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_FORMAT_PLACEHOLDER, " "); if (sysLogMode.equalsIgnoreCase("ASYNC")) { newXmlConfTemplate = newXmlConfTemplate.replaceAll("Root", "AsyncRoot"); } } + if (Config.enable_file_logger) { + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_FILE_APPENDER_PLACEHOLDER, + "\n"); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_WARN_FILE_APPENDER_PLACEHOLDER, + "\n"); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_FILE_LOGGER_PLACEHOLDER, + "\n"); + } + if (foreground) { - StringBuilder consoleLogger = new StringBuilder(); - consoleLogger.append("\n"); - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - consoleLogger.toString()); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_MARKER_PLACEHOLDER, RUNTIME_LOG_MARKER); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_LOG_MARKER_PLACEHOLDER, AUDIT_LOG_MARKER); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(CONSOLE_APPENDER_PLACEHOLDER, + "\n"); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_CONSOLE_LOGGER_PLACEHOLDER, + "\n"); } Map properties = Maps.newHashMap(); @@ -256,9 +305,7 @@ public class Log4jConfig extends XmlConfiguration { strSub = new StrSubstitutor(new Interpolator(properties)); newXmlConfTemplate = strSub.replace(newXmlConfTemplate); - System.out.println("====="); - System.out.println(newXmlConfTemplate); - System.out.println("====="); + LogUtils.stdout("=====\n" + newXmlConfTemplate + "\n====="); logXmlConfTemplate = newXmlConfTemplate; SpringLog4j2Config.writeSpringLogConf(customConfDir); @@ -277,6 +324,17 @@ public class Log4jConfig extends XmlConfiguration { } catch (Exception e) { throw new IOException("Error occurred while configuring Log4j", e); } + + redirectStd(); + } + + private static void redirectStd() { + PrintStream logPrintStream = IoBuilder.forLogger(LogManager.getLogger("system.out")).setLevel(Level.INFO) + .buildPrintStream(); + System.setOut(logPrintStream); + PrintStream errorPrintStream = IoBuilder.forLogger(LogManager.getLogger("system.err")).setLevel(Level.ERROR) + .buildPrintStream(); + System.setErr(errorPrintStream); } public static String getLogXmlConfTemplate() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java index 0a207925fd..c0a544d825 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java @@ -19,6 +19,7 @@ package org.apache.doris.common.util; import org.apache.doris.common.Config; import org.apache.doris.common.ConfigBase.ConfField; +import org.apache.doris.common.LogUtils; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.VariableMgr; @@ -148,7 +149,7 @@ public class DocGenerator { sortedDoc.put(field.getName(), res); } } catch (Exception e) { - System.out.println("Failed to generate doc for field: " + field.getName()); + LogUtils.stderr("Failed to generate doc for field: " + field.getName()); throw e; } } @@ -204,7 +205,7 @@ public class DocGenerator { sortedDoc.put(field.getAnnotation(VariableMgr.VarAttr.class).name(), res); } } catch (Exception e) { - System.out.println("Failed to generate doc for " + field.getName()); + LogUtils.stderr("Failed to generate doc for " + field.getName()); throw e; } } @@ -217,7 +218,7 @@ public class DocGenerator { sortedDoc.put(field.getAnnotation(VariableMgr.VarAttr.class).name(), res); } } catch (Exception e) { - System.out.println("Failed to generate doc for field: " + field.getName()); + LogUtils.stderr("Failed to generate doc for field: " + field.getName()); throw e; } } @@ -288,7 +289,7 @@ public class DocGenerator { sessionVariableDocOutputPath, sessionVariableDocOutputPathCN); try { docGenerator.generate(); - System.out.println("Done!"); + LogUtils.stdout("Done!"); } catch (Exception e) { log.info("failed to generate doc", e); System.exit(-1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java index 34a604eb0d..7d3a0402bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java @@ -18,6 +18,7 @@ package org.apache.doris.common.util; import org.apache.doris.common.Config; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.Version; import com.google.common.annotations.VisibleForTesting; @@ -42,7 +43,7 @@ public class JdkUtils { int runtimeVersion = JdkUtils.getJavaVersionAsInteger(javaRuntimeVersionStr); if (runtimeVersion < compileVersion) { - System.out.println("The runtime java version " + javaRuntimeVersionStr + " is less than " + LogUtils.stdout("The runtime java version " + javaRuntimeVersionStr + " is less than " + "compile version " + javaCompileVersionStr); return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java deleted file mode 100644 index 0953c37196..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.util; - -public class PlatformName { - - private static final String platformName = System.getProperty("os.name") + "-" - + System.getProperty("os.arch") + "-" - + System.getProperty("sun.arch.data.model"); - - public static String getPlatformName() { - return platformName; - } - - public static void main(String[] args) { - System.out.println(platformName); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index be3e711f45..79b5450877 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -421,10 +421,6 @@ public class Util { } } - public static void stdoutWithTime(String msg) { - System.out.println("[" + TimeUtils.longToTimeString(System.currentTimeMillis()) + "] " + msg); - } - // not support encode negative value now public static void encodeVarint64(long source, DataOutput out) throws IOException { assert source >= 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java b/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java index 2ff66a5aad..302717c605 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java +++ b/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java @@ -18,7 +18,7 @@ package org.apache.doris.ha; import org.apache.doris.catalog.Env; -import org.apache.doris.common.util.Util; +import org.apache.doris.common.LogUtils; import com.google.common.base.Preconditions; import com.sleepycat.je.rep.StateChangeEvent; @@ -57,7 +57,7 @@ public class BDBStateChangeListener implements StateChangeListener { default: { String msg = "this node is " + sce.getState().name(); LOG.warn(msg); - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); return; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java index f80f5e7dd2..11bea966cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java @@ -105,7 +105,7 @@ public class BDBEnvironment { DbResetRepGroup resetUtility = new DbResetRepGroup( envHome, PALO_JOURNAL_GROUP, selfNodeName, selfNodeHostPort); resetUtility.reset(); - LOG.info("group has been reset."); + LOG.warn("metadata recovery mode, group has been reset."); } // set replication config diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java index 603a9def72..55b63e18d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java @@ -19,10 +19,10 @@ package org.apache.doris.journal.bdbje; import org.apache.doris.catalog.Env; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.io.DataOutputBuffer; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.NetUtils; -import org.apache.doris.common.util.Util; import org.apache.doris.journal.Journal; import org.apache.doris.journal.JournalBatch; import org.apache.doris.journal.JournalCursor; @@ -119,7 +119,7 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B + "journal id: %d, current db: %s, expected db count: %d", newName, currentDbName, newNameVerify); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } } @@ -192,7 +192,7 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B String msg = "write bdb failed. will exit. the first journalId: " + firstId + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } catch (DatabaseException e) { LOG.error("catch an exception when writing to database. sleep and retry. the first journal id {}", @@ -217,7 +217,7 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B String msg = "write bdb failed. will exit. the first journalId: " + firstId + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); return 0; // unreachable! } @@ -284,7 +284,7 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B String msg = "write bdb failed. will exit. journalId: " + id + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } catch (DatabaseException e) { LOG.error("catch an exception when writing to database. sleep and retry. journal id {}", id, e); @@ -300,7 +300,7 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B String msg = "write bdb failed. will exit. journalId: " + id + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } return id; @@ -356,7 +356,7 @@ public class BDBJEJournal implements Journal { // CHECKSTYLE IGNORE THIS LINE: B LOG.warn("", e); } } else { - System.out.println("No record found for key '" + journalId + "'."); + LOG.warn("No record found for key '{}'.", journalId); } } catch (Exception e) { LOG.warn("catch an exception when get JournalEntity. key:{}", journalId, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java index 5466df71dd..8c48cab42c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java @@ -46,7 +46,7 @@ public class BDBJournalCursor implements JournalCursor { public static BDBJournalCursor getJournalCursor(BDBEnvironment env, long fromKey, long toKey) { if (toKey < fromKey || fromKey < 0) { - System.out.println("Invalid key range!"); + LOG.warn("Invalid key range! fromKey:{} toKey:{}", fromKey, toKey); return null; } BDBJournalCursor cursor = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java index cc9defa378..55c1547528 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java @@ -18,6 +18,7 @@ package org.apache.doris.journal.bdbje; import org.apache.doris.catalog.Env; +import org.apache.doris.common.LogUtils; import org.apache.doris.journal.JournalEntity; import org.apache.doris.meta.MetaContext; @@ -66,7 +67,7 @@ public class BDBTool { env = new Environment(new File(metaPath), envConfig); } catch (DatabaseException e) { LOG.warn("", e); - System.err.println("Failed to open BDBJE env: " + Env.getCurrentEnv().getBdbDir() + ". exit"); + LogUtils.stderr("Failed to open BDBJE env: " + Env.getCurrentEnv().getBdbDir() + ". exit"); return false; } Preconditions.checkNotNull(env); @@ -75,7 +76,7 @@ public class BDBTool { if (options.isListDbs()) { // list all databases List dbNames = env.getDatabaseNames(); - System.out.println(JSONArray.toJSONString(dbNames)); + LogUtils.stdout(JSONArray.toJSONString(dbNames)); return true; } else { // db operations @@ -90,7 +91,7 @@ public class BDBTool { // get db stat Map statMap = Maps.newHashMap(); statMap.put("count", String.valueOf(db.count())); - System.out.println(JSONObject.toJSONString(statMap)); + LogUtils.stdout(JSONObject.toJSONString(statMap)); return true; } else { // set from key @@ -99,7 +100,7 @@ public class BDBTool { try { fromKey = Long.valueOf(fromKeyStr); } catch (NumberFormatException e) { - System.err.println("Not a valid from key: " + fromKeyStr); + LogUtils.stderr("Not a valid from key: " + fromKeyStr); return false; } @@ -109,13 +110,13 @@ public class BDBTool { try { endKey = Long.valueOf(options.getEndKey()); } catch (NumberFormatException e) { - System.err.println("Not a valid end key: " + options.getEndKey()); + LogUtils.stderr("Not a valid end key: " + options.getEndKey()); return false; } } if (fromKey > endKey) { - System.err.println("from key should less than or equal to end key[" + LogUtils.stderr("from key should less than or equal to end key[" + fromKey + " vs. " + endKey + "]"); return false; } @@ -132,7 +133,7 @@ public class BDBTool { } } catch (Exception e) { LOG.warn("", e); - System.err.println("Failed to run bdb tools"); + LogUtils.stderr("Failed to run bdb tools"); return false; } return true; @@ -155,15 +156,15 @@ public class BDBTool { entity.readFields(in); } catch (Exception e) { LOG.warn("", e); - System.err.println("Fail to read journal entity for key: " + key + ". reason: " + e.getMessage()); + LogUtils.stderr("Fail to read journal entity for key: " + key + ". reason: " + e.getMessage()); System.exit(-1); } - System.out.println("key: " + key); - System.out.println("op code: " + entity.getOpCode()); - System.out.println("value: " + entity.getData().toString()); + LogUtils.stdout("key: " + key); + LogUtils.stdout("op code: " + entity.getOpCode()); + LogUtils.stdout("value: " + entity.getData().toString()); } else if (status == OperationStatus.NOTFOUND) { - System.out.println("key: " + key); - System.out.println("value: NOT FOUND"); + LogUtils.stdout("key: " + key); + LogUtils.stdout("value: NOT FOUND"); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java index 45ae377a40..8e39f8a6a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java @@ -84,7 +84,7 @@ public class LocalJournal implements Journal { try { storage = new Storage(imageDir); if (journalId.get() == storage.getEditsSeq()) { - System.out.println("Does not need to roll!"); + LOG.warn("Does not need to roll! journalId: {}, editsSeq: {}", journalId.get(), storage.getEditsSeq()); return; } if (outputStream != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java index d570719dd3..d24e394927 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java @@ -50,7 +50,7 @@ public final class LocalJournalCursor implements JournalCursor { public static LocalJournalCursor getJournalCursor(String imageDir, long fromKey, long toKey) { if (toKey < fromKey && toKey != -1 || fromKey < 0) { - System.out.println("Invalid key range!"); + LOG.warn("Invalid fromKey:{} toKey:{}", fromKey, toKey); return null; } long newToKey = toKey; @@ -95,7 +95,7 @@ public final class LocalJournalCursor implements JournalCursor { } if (fileName == null) { - System.out.println("Can not find the key:" + fromKey); + LOG.warn("Can not find the key:{}", fromKey); throw new IOException(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java index 0e58f1bc97..ae3aa52c4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java @@ -79,9 +79,4 @@ public class CustomRewriteJob implements RewriteJob { public boolean isOnce() { return false; } - - private void printTraceLog(RuleType ruleType, String traceBefore, String traceAfter) { - System.out.println("========== " + getClass().getSimpleName() + " " + ruleType - + " ==========\nbefore:\n" + traceBefore + "\n\nafter:\n" + traceAfter + "\n"); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 5aa10946db..5abda5dc45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -438,8 +438,7 @@ public class EditLog { Frontend fe = (Frontend) journal.getData(); env.replayDropFrontend(fe); if (fe.getNodeName().equals(Env.getCurrentEnv().getNodeName())) { - System.out.println("current fe " + fe + " is removed. will exit"); - LOG.info("current fe " + fe + " is removed. will exit"); + LOG.warn("current fe {} is removed. will exit", fe); System.exit(-1); } break; diff --git a/fe/pom.xml b/fe/pom.xml index 7108abfad9..43fbf896ef 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -815,6 +815,11 @@ under the License. log4j-core ${log4j2.version} + + org.apache.logging.log4j + log4j-iostreams + ${log4j2.version} + org.apache.logging.log4j log4j-slf4j-impl