database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](audit) add errorCode and errorMessage in audit log (#14925)

Browse Source 
* [feat] add errorCode and errorMessage in audit log.

* [Feature](audit) add errorCode and errorMessage in audit log

Co-authored-by: wangxiangyu@360shuke.com <wangxiangyu@360shuke.com>
This commit is contained in:
wxy
2023-01-09 08:47:57 +08:00
committed by GitHub
parent c57fa7c930
commit 6829d361cb
7 changed files with 36 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/en/docs/ecosystem/audit-plugin.md
View File

@ -65,6 +65,8 @@ create table doris_audit_db__.doris_audit_log_tbl__
user varchar(64) comment "User name",
db varchar(96) comment "Database of this query",
state varchar(8) comment "Query result state. EOF, ERR, OK",
error_code int comment "Error code of failing query.",
error_message string comment "Error message of failing query.",
query_time bigint comment "Query execution time in millisecond",
scan_bytes bigint comment "Total scan bytes of this query",
scan_rows bigint comment "Total scan rows of this query",
@ -99,6 +101,8 @@ create table doris_audit_db__.doris_slow_log_tbl__
user varchar(64) comment "User name",
db varchar(96) comment "Database of this query",
state varchar(8) comment "Query result state. EOF, ERR, OK",
error_code int comment "Error code of failing query.",
error_message string comment "Error message of failing query.",
query_time bigint comment "Query execution time in millisecond",
scan_bytes bigint comment "Total scan bytes of this query",
scan_rows bigint comment "Total scan rows of this query",

4
docs/zh-CN/docs/ecosystem/audit-plugin.md
View File

@ -65,6 +65,8 @@ create table doris_audit_db__.doris_audit_log_tbl__
user varchar(64) comment "User name",
db varchar(96) comment "Database of this query",
state varchar(8) comment "Query result state. EOF, ERR, OK",
error_code int comment "Error code of failing query.",
error_message string comment "Error message of failing query.",
query_time bigint comment "Query execution time in millisecond",
scan_bytes bigint comment "Total scan bytes of this query",
scan_rows bigint comment "Total scan rows of this query",
@ -99,6 +101,8 @@ create table doris_audit_db__.doris_slow_log_tbl__
user varchar(64) comment "User name",
db varchar(96) comment "Database of this query",
state varchar(8) comment "Query result state. EOF, ERR, OK",
error_code int comment "Error code of failing query.",
error_message string comment "Error message of failing query.",
query_time bigint comment "Query execution time in millisecond",
scan_bytes bigint comment "Total scan bytes of this query",
scan_rows bigint comment "Total scan rows of this query",

15
fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java
View File

@ -60,6 +60,10 @@ public class AuditEvent {
public String db = "";
@AuditField(value = "State")
public String state = "";
@AuditField(value = "ErrorCode")
public int errorCode = 0;
@AuditField(value = "ErrorMessage")
public String errorMessage = "";
@AuditField(value = "Time")
public long queryTime = -1;
@AuditField(value = "ScanBytes")
@ -86,7 +90,6 @@ public class AuditEvent {
public long peakMemoryBytes = -1;
@AuditField(value = "SqlDigest")
public String sqlDigest = "";
@AuditField(value = "TraceId")
public String traceId = "";
@ -131,6 +134,16 @@ public class AuditEvent {
return this;
}
public AuditEventBuilder setErrorCode(int errorCode) {
auditEvent.errorCode = errorCode;
return this;
}
public AuditEventBuilder setErrorMessage(String errorMessage) {
auditEvent.errorMessage = errorMessage;
return this;
}
public AuditEventBuilder setQueryTime(long queryTime) {
auditEvent.queryTime = queryTime;
return this;

6
fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
View File

@ -178,7 +178,11 @@ public class ConnectProcessor {
ctx.getAuditEventBuilder().setEventType(EventType.AFTER_QUERY)
.setDb(ClusterNamespace.getNameFromFullName(ctx.getDatabase()))
.setState(ctx.getState().toString()).setQueryTime(elapseMs)
.setState(ctx.getState().toString())
.setErrorCode(ctx.getState().getErrorCode() == null ? 0 : ctx.getState().getErrorCode().getCode())
.setErrorMessage((ctx.getState().getErrorMessage() == null ? "" :
ctx.getState().getErrorMessage().replace("\n", " ").replace("\t", " ")))
.setQueryTime(elapseMs)
.setScanBytes(statistics == null ? 0 : statistics.getScanBytes())
.setScanRows(statistics == null ? 0 : statistics.getScanRows())
.setCpuTimeMs(statistics == null ? 0 : statistics.getCpuMs())

1
fe/fe-core/src/main/java/org/apache/doris/qe/QueryState.java
View File

@ -56,6 +56,7 @@ public class QueryState {
stateType = MysqlStateType.OK;
errorCode = null;
infoMessage = null;
errorMessage = "";
serverStatus = 0;
isQuery = false;
affectedRows = 0;

2
fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
View File

@ -163,6 +163,8 @@ public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
logBuffer.append(event.user).append("\t");
logBuffer.append(event.db).append("\t");
logBuffer.append(event.state).append("\t");
logBuffer.append(event.errorCode).append("\t");
logBuffer.append(event.errorMessage).append("\t");
logBuffer.append(event.queryTime).append("\t");
logBuffer.append(event.scanBytes).append("\t");
logBuffer.append(event.scanRows).append("\t");

14
fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
View File

@ -23,15 +23,12 @@ import org.apache.logging.log4j.Logger;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Calendar;
import java.util.List;
import java.util.Map;
public class DorisStreamLoader {
private final static Logger LOG = LogManager.getLogger(DorisStreamLoader.class);
@ -73,8 +70,9 @@ public class DorisStreamLoader {
conn.addRequestProperty("label", label);
conn.addRequestProperty("max_filter_ratio", "1.0");
conn.addRequestProperty("columns", "query_id, `time`, client_ip, user, db, state, query_time, scan_bytes," +
" scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash, sql_digest, peak_memory_bytes, stmt");
conn.addRequestProperty("columns", "query_id, `time`, client_ip, user, db, state, error_code, error_message, " +
"query_time, scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash, " +
"sql_digest, peak_memory_bytes, stmt");
conn.setDoOutput(true);
conn.setDoInput(true);
@ -89,9 +87,9 @@ public class DorisStreamLoader {
sb.append("-H \"").append("Expect\":").append("\"100-continue\" \\\n ");
sb.append("-H \"").append("Content-Type\":").append("\"text/plain; charset=UTF-8\" \\\n ");
sb.append("-H \"").append("max_filter_ratio\":").append("\"1.0\" \\\n ");
sb.append("-H \"").append("columns\":").append("\"query_id, time, client_ip, user, db, state, query_time," +
" scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash," +
" sql_digest, peak_memory_bytes, stmt\" \\\n ");
sb.append("-H \"").append("columns\":").append("\"query_id, time, client_ip, user, db, state, error_code, " +
"error_message, query_time, scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, " +
"cpu_time_ms, sql_hash, sql_digest, peak_memory_bytes, stmt\" \\\n ");
sb.append("\"").append(conn.getURL()).append("\"");
return sb.toString();
}