database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Limit the FE log file number (#2163)

Browse Source 
1. upgrade log4j to 2.12.1
2. Add 2 new FE config:
        'sys_log_delete_age' and default is '7d', for sys log.
        'audit_log_delete_age' and default is '30d', for audit log.

   it means if a log's last modification time is 7/30 days ago, it will be deleted.
This commit is contained in:
Mingyu Chen
2019-11-11 09:12:57 +08:00
committed by GitHub
parent c92de36bec
commit 9eaba67606
6 changed files with 52 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/documentation/cn/getting-started/data-partition.md
View File

@ -61,7 +61,7 @@ CREATE TABLE IF NOT EXISTS example_db.expamle_tbl
`last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间",
`cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费",
`max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间",
`min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间",
`min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间"
)
ENGINE=olap
AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`)

4
docs/documentation/en/getting-started/data-partition_EN.md
View File

@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS example_db.expamle_tbl
    `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "User last visit time",
    `cost` BIGINT SUM DEFAULT "0" COMMENT "Total user consumption",
    `max_dwell_time` INT MAX DEFAULT "0" COMMENT "User maximum dwell time",
    `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "User minimum dwell time",
    `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "User minimum dwell time"
)
ENGINE=olap
AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`)
@ -289,4 +289,4 @@ In this example, the type of ENGINE is olap, the default ENGINE type. In Doris,
    Doris's table creation command is a synchronous command. The timeout of this command is currently set to be relatively simple, ie (tablet num * replication num) seconds. If you create more data fragments and have fragment creation failed, it may cause an error to be returned after waiting for a long timeout.
    
    Under normal circumstances, the statement will return in a few seconds or ten seconds. If it is more than one minute, it is recommended to cancel this operation directly and go to the FE or BE log to view the related errors.
    Under normal circumstances, the statement will return in a few seconds or ten seconds. If it is more than one minute, it is recommended to cancel this operation directly and go to the FE or BE log to view the related errors.

8
fe/pom.xml
View File

@ -295,21 +295,21 @@ under the License.
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.2</version>
<version>2.12.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.2</version>
<version>2.12.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-slf4j-impl -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.2</version>
<version>2.12.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
@ -509,7 +509,7 @@ under the License.
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.7</version>
<version>1.2.17</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->

25
fe/src/main/java/org/apache/doris/common/Config.java
View File

@ -34,7 +34,8 @@ public class Config extends ConfigBase {
* INFO, WARNING, ERROR, FATAL
*
* sys_log_roll_num:
* Maximal FE log files to be kept.
* Maximal FE log files to be kept within an sys_log_roll_interval.
* default is 10, which means there will be at most 10 log files in a day
*
* sys_log_verbose_modules:
* Verbose modules. VERBOSE level is implemented by log4j DEBUG level.
@ -45,12 +46,21 @@ public class Config extends ConfigBase {
* sys_log_roll_interval:
* DAY: log suffix is yyyyMMdd
* HOUR: log suffix is yyyyMMddHH
*
* sys_log_delete_age:
* default is 7 days, if log's last modify time is 7 days ago, it will be deleted.
* support format:
* 7d 7 days
* 10h 10 hours
* 60m 60 mins
* 120s 120 seconds
*/
@ConfField public static String sys_log_dir = System.getenv("DORIS_HOME") + "/log";
@ConfField public static String sys_log_level = "INFO";
@ConfField public static int sys_log_roll_num = 10;
@ConfField public static String[] sys_log_verbose_modules = {};
@ConfField public static String sys_log_roll_interval = "DAY";
@ConfField public static String sys_log_delete_age = "7d";
@Deprecated
@ConfField public static String sys_log_roll_mode = "SIZE-MB-1024";
@ -60,7 +70,7 @@ public class Config extends ConfigBase {
* Audit log fe.audit.log contains all requests with related infos such as user, host, cost, status, etc.
*
* audit_log_roll_num:
* Maximal FE audit log files to be kept.
* Maximal FE audit log files to be kept within an audit_log_roll_interval.
*
* audit_log_modules:
* Slow query contains all queries which cost exceed *qe_slow_log_ms*
@ -71,12 +81,21 @@ public class Config extends ConfigBase {
* audit_log_roll_interval:
* DAY: log suffix is yyyyMMdd
* HOUR: log suffix is yyyyMMddHH
*
* audit_log_delete_age:
* default is 30 days, if log's last modify time is 30 days ago, it will be deleted.
* support format:
* 7d 7 days
* 10h 10 hours
* 60m 60 mins
* 120s 120 seconds
*/
@ConfField public static String audit_log_dir = System.getenv("DORIS_HOME") + "/log";
@ConfField public static int audit_log_roll_num = 90; // nearly 3 months
@ConfField public static int audit_log_roll_num = 90;
@ConfField public static String[] audit_log_modules = {"slow_query", "query"};
@ConfField(mutable = true) public static long qe_slow_log_ms = 5000;
@ConfField public static String audit_log_roll_interval = "DAY";
@ConfField public static String audit_log_delete_age = "30d";
@Deprecated
@ConfField public static String audit_log_roll_mode = "TIME-DAY";

27
fe/src/main/java/org/apache/doris/common/Log4jConfig.java
View File

@ -48,7 +48,12 @@ public class Log4jConfig extends XmlConfiguration {
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${sys_roll_maxsize}MB\"/>\n" +
" </Policies>\n" +
" <DefaultRolloverStrategy max=\"${sys_roll_num}\" fileIndex=\"min\"/>\n" +
" <DefaultRolloverStrategy max=\"${sys_roll_num}\" fileIndex=\"min\">\n" +
" <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n" +
" <IfFileName glob=\"fe.log.*\" />\n" +
" <IfLastModified age=\"${sys_log_delete_age}\" />\n" +
" </Delete>\n" +
" </DefaultRolloverStrategy>\n" +
" </RollingFile>\n" +
" <RollingFile name=\"SysWF\" fileName=\"${sys_log_dir}/fe.warn.log\" filePattern=\"${sys_log_dir}/fe.warn.log.${sys_file_pattern}-%i\">\n" +
" <PatternLayout charset=\"UTF-8\">\n" +
@ -58,7 +63,12 @@ public class Log4jConfig extends XmlConfiguration {
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${sys_roll_maxsize}MB\"/>\n" +
" </Policies>\n" +
" <DefaultRolloverStrategy max=\"${sys_roll_num}\" fileIndex=\"min\"/>\n" +
" <DefaultRolloverStrategy max=\"${sys_roll_num}\" fileIndex=\"min\">\n" +
" <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n" +
" <IfFileName glob=\"fe.warn.log.*\" />\n" +
" <IfLastModified age=\"${sys_log_delete_age}\" />\n" +
" </Delete>\n" +
" </DefaultRolloverStrategy>\n" +
" </RollingFile>\n" +
" <RollingFile name=\"Auditfile\" fileName=\"${audit_log_dir}/fe.audit.log\" filePattern=\"${audit_log_dir}/fe.audit.log.${audit_file_pattern}-%i\">\n" +
" <PatternLayout charset=\"UTF-8\">\n" +
@ -68,7 +78,12 @@ public class Log4jConfig extends XmlConfiguration {
" <TimeBasedTriggeringPolicy/>\n" +
" <SizeBasedTriggeringPolicy size=\"${audit_roll_maxsize}MB\"/>\n" +
" </Policies>\n" +
" <DefaultRolloverStrategy max=\"${audit_roll_num}\" fileIndex=\"min\"/>\n" +
" <DefaultRolloverStrategy max=\"${sys_roll_num}\" fileIndex=\"min\">\n" +
" <Delete basePath=\"${audit_log_dir}/\" maxDepth=\"1\">\n" +
" <IfFileName glob=\"fe.audit.log.*\" />\n" +
" <IfLastModified age=\"${audit_log_delete_age}\" />\n" +
" </Delete>\n" +
" </DefaultRolloverStrategy>\n" +
" </RollingFile>\n" +
" </Appenders>\n" +
" <Loggers>\n" +
@ -103,6 +118,7 @@ public class Log4jConfig extends XmlConfiguration {
// sys log config
String sysLogDir = Config.sys_log_dir;
String sysRollNum = String.valueOf(Config.sys_log_roll_num);
String sysDeleteAge = String.valueOf(Config.sys_log_delete_age);
if (!(sysLogLevel.equalsIgnoreCase("INFO") ||
sysLogLevel.equalsIgnoreCase("WARN") ||
@ -126,6 +142,7 @@ public class Log4jConfig extends XmlConfiguration {
String auditLogRollPattern = "%d{yyyyMMdd}";
String auditRollNum = String.valueOf(Config.audit_log_roll_num);
String auditRollMaxSize = String.valueOf(Config.log_roll_size_mb);
String auditDeleteAge = String.valueOf(Config.audit_log_delete_age);
if (Config.audit_log_roll_interval.equals("HOUR")) {
auditLogRollPattern = "%d{yyyyMMddHH}";
} else if (Config.audit_log_roll_interval.equals("DAY")) {
@ -150,12 +167,14 @@ public class Log4jConfig extends XmlConfiguration {
properties.put("sys_file_pattern", sysLogRollPattern);
properties.put("sys_roll_maxsize", sysRollMaxSize);
properties.put("sys_roll_num", sysRollNum);
properties.put("sys_log_delete_age", sysDeleteAge);
properties.put("sys_log_level", sysLogLevel);
properties.put("audit_log_dir", auditLogDir);
properties.put("audit_file_pattern", auditLogRollPattern);
properties.put("audit_roll_maxsize", auditRollMaxSize);
properties.put("audit_roll_num", auditRollNum);
properties.put("audit_log_delete_age", auditDeleteAge);
strSub = new StrSubstitutor(new Interpolator(properties));
newXmlConfTemplate = strSub.replace(newXmlConfTemplate);
@ -191,7 +210,7 @@ public class Log4jConfig extends XmlConfiguration {
}
public Log4jConfig(final ConfigurationSource configSource) {
super(configSource);
super(LoggerContext.getContext(), configSource);
}
public synchronized static void initLogging() throws IOException {

40
fe/src/main/java/org/apache/doris/common/ThreadIdConverter.java
View File

@ -1,40 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.pattern.ConverterKeys;
import org.apache.logging.log4j.core.pattern.LogEventPatternConverter;
@Plugin(name = "threadIdConverter", category = "Converter")
@ConverterKeys({ "i", "tid" })
public class ThreadIdConverter extends LogEventPatternConverter {
protected ThreadIdConverter(final String[] options) {
super("threadIdConverter", "");
}
public static ThreadIdConverter newInstance(final String[] options) {
return new ThreadIdConverter(options);
}
@Override
public void format(LogEvent arg0, StringBuilder arg1) {
arg1.append(Thread.currentThread().getId());
}
}