database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](audit) add timeout for audit log load and modify the label format (#35535)

Browse Source 
## Proposed changes

1. Add a new global variable: `audit_plugin_load_timeout`, default is
600 sec
Avoid using default stream load, which is 4 hours, too long for audit
log load.

2. Modify the label of audit log load
   Add millisecond to avoid same label from different FE.
This commit is contained in:
Mingyu Chen
2024-05-30 11:05:30 +08:00
committed by yiguolei
parent 0ed7dc1081
commit 14de0974a5
2 changed files with 16 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.plugin.audit;
import org.apache.doris.catalog.InternalSchema;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.GlobalVariable;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -61,6 +62,7 @@ public class AuditStreamLoader {
conn.addRequestProperty("Expect", "100-continue");
conn.addRequestProperty("Content-Type", "text/plain; charset=UTF-8");
conn.addRequestProperty("label", label);
conn.setRequestProperty("timeout", String.valueOf(GlobalVariable.auditPluginLoadTimeoutS));
conn.addRequestProperty("max_filter_ratio", "1.0");
conn.addRequestProperty("columns",
InternalSchema.AUDIT_SCHEMA.stream().map(c -> c.getName()).collect(
@ -105,11 +107,7 @@ public class AuditStreamLoader {
}
public LoadResponse loadBatch(StringBuilder sb, String clusterToken) {
Calendar calendar = Calendar.getInstance();
String label = String.format("_log_%s%02d%02d_%02d%02d%02d_%s",
calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar.get(Calendar.DAY_OF_MONTH),
calendar.get(Calendar.HOUR_OF_DAY), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND),
feIdentity);
String label = genLabel();
HttpURLConnection feConn = null;
HttpURLConnection beConn = null;
@ -159,6 +157,15 @@ public class AuditStreamLoader {
}
}
private String genLabel() {
Calendar calendar = Calendar.getInstance();
return String.format("_log_%s%02d%02d_%02d%02d%02d_%s_%s",
calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar.get(Calendar.DAY_OF_MONTH),
calendar.get(Calendar.HOUR_OF_DAY), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND),
calendar.get(Calendar.MILLISECOND),
feIdentity);
}
public static class LoadResponse {
public int status;
public String respMsg;

4
fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
View File

@ -55,6 +55,7 @@ public final class GlobalVariable {
public static final String AUDIT_PLUGIN_MAX_BATCH_BYTES = "audit_plugin_max_batch_bytes";
public static final String AUDIT_PLUGIN_MAX_BATCH_INTERVAL_SEC = "audit_plugin_max_batch_interval_sec";
public static final String AUDIT_PLUGIN_MAX_SQL_LENGTH = "audit_plugin_max_sql_length";
public static final String AUDIT_PLUGIN_LOAD_TIMEOUT = "audit_plugin_load_timeout";
public static final String ENABLE_GET_ROW_COUNT_FROM_FILE_LIST = "enable_get_row_count_from_file_list";
public static final String READ_ONLY = "read_only";
@ -131,6 +132,9 @@ public final class GlobalVariable {
@VariableMgr.VarAttr(name = AUDIT_PLUGIN_MAX_SQL_LENGTH, flag = VariableMgr.GLOBAL)
public static int auditPluginMaxSqlLength = 4096;
@VariableMgr.VarAttr(name = AUDIT_PLUGIN_LOAD_TIMEOUT, flag = VariableMgr.GLOBAL)
public static int auditPluginLoadTimeoutS = 600;
@VariableMgr.VarAttr(name = ENABLE_GET_ROW_COUNT_FROM_FILE_LIST, flag = VariableMgr.GLOBAL,
description = {
"针对外表，是否允许根据文件列表估算表行数。获取文件列表可能是一个耗时的操作，"