From daea751a986823bf5858704663d58f49fd5dfb39 Mon Sep 17 00:00:00 2001 From: Yulei-Yang Date: Wed, 8 Nov 2023 10:25:15 +0800 Subject: [PATCH] [Improvement](auditlog) add column catalog for audit log and audit log table (#26403) --- docs/en/docs/ecosystem/audit-plugin.md | 2 ++ docs/zh-CN/docs/ecosystem/audit-plugin.md | 2 ++ .../src/main/java/org/apache/doris/plugin/AuditEvent.java | 7 +++++++ .../src/main/java/org/apache/doris/qe/AuditLogHelper.java | 5 +++++ .../org/apache/doris/plugin/audit/AuditLoaderPlugin.java | 1 + .../org/apache/doris/plugin/audit/DorisStreamLoader.java | 4 ++-- pytest/deploy/start.py | 1 + 7 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/en/docs/ecosystem/audit-plugin.md b/docs/en/docs/ecosystem/audit-plugin.md index a355a75e59..d63c15f8b9 100644 --- a/docs/en/docs/ecosystem/audit-plugin.md +++ b/docs/en/docs/ecosystem/audit-plugin.md @@ -87,6 +87,7 @@ create table doris_audit_db__.doris_audit_log_tbl__ `time` datetime not null comment "Query start time", client_ip varchar(32) comment "Client IP", user varchar(64) comment "User name", + catalog varchar(128) comment "Catalog of this query", db varchar(96) comment "Database of this query", state varchar(8) comment "Query result state. EOF, ERR, OK", error_code int comment "Error code of failing query.", @@ -123,6 +124,7 @@ create table doris_audit_db__.doris_slow_log_tbl__ `time` datetime not null comment "Query start time", client_ip varchar(32) comment "Client IP", user varchar(64) comment "User name", + catalog varchar(128) comment "Catalog of this query", db varchar(96) comment "Database of this query", state varchar(8) comment "Query result state. EOF, ERR, OK", error_code int comment "Error code of failing query.", diff --git a/docs/zh-CN/docs/ecosystem/audit-plugin.md b/docs/zh-CN/docs/ecosystem/audit-plugin.md index 7fe4549926..f700ce28d0 100644 --- a/docs/zh-CN/docs/ecosystem/audit-plugin.md +++ b/docs/zh-CN/docs/ecosystem/audit-plugin.md @@ -93,6 +93,7 @@ create table doris_audit_db__.doris_audit_log_tbl__ `time` datetime not null comment "Query start time", client_ip varchar(32) comment "Client IP", user varchar(64) comment "User name", + catalog varchar(128) comment "Catalog of this query", db varchar(96) comment "Database of this query", state varchar(8) comment "Query result state. EOF, ERR, OK", error_code int comment "Error code of failing query.", @@ -129,6 +130,7 @@ create table doris_audit_db__.doris_slow_log_tbl__ `time` datetime not null comment "Query start time", client_ip varchar(32) comment "Client IP", user varchar(64) comment "User name", + catalog varchar(128) comment "Catalog of this query", db varchar(96) comment "Database of this query", state varchar(8) comment "Query result state. EOF, ERR, OK", error_code int comment "Error code of failing query.", diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java index dbef8321b5..8eb539706c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java @@ -56,6 +56,8 @@ public class AuditEvent { public String clientIp = ""; @AuditField(value = "User") public String user = ""; + @AuditField(value = "Catalog") + public String catalog = ""; @AuditField(value = "Db") public String db = ""; @AuditField(value = "State") @@ -131,6 +133,11 @@ public class AuditEvent { return this; } + public AuditEventBuilder setCatalog(String catalog) { + auditEvent.catalog = catalog; + return this; + } + public AuditEventBuilder setDb(String db) { auditEvent.db = db; return this; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index 40f870eee1..0ed0e90025 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -62,6 +62,11 @@ public class AuditLogHelper { .setWorkloadGroup(ctx.getWorkloadGroupName()) .setFuzzyVariables(!printFuzzyVariables ? "" : ctx.getSessionVariable().printFuzzyVariables()); + // when doric fe is booting, current catalog may not be set + if (ctx.getCurrentCatalog() != null) { + ctx.getAuditEventBuilder().setCatalog(ctx.getCurrentCatalog().getName()); + } + if (ctx.getState().isQuery()) { MetricRepo.COUNTER_QUERY_ALL.increase(1L); MetricRepo.USER_COUNTER_QUERY_ALL.getOrAdd(ctx.getQualifiedUser()).increase(1L); diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java index 3cfb0eeeae..75c45f56f6 100755 --- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java +++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java @@ -163,6 +163,7 @@ public class AuditLoaderPlugin extends Plugin implements AuditPlugin { logBuffer.append(longToTimeString(event.timestamp)).append("\t"); logBuffer.append(event.clientIp).append("\t"); logBuffer.append(event.user).append("\t"); + logBuffer.append(event.catalog).append("\t"); logBuffer.append(event.db).append("\t"); logBuffer.append(event.state).append("\t"); logBuffer.append(event.errorCode).append("\t"); diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java index d389f0dfa8..7b568f0a04 100644 --- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java +++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java @@ -71,7 +71,7 @@ public class DorisStreamLoader { conn.addRequestProperty("label", label); conn.addRequestProperty("max_filter_ratio", "1.0"); - conn.addRequestProperty("columns", "query_id, `time`, client_ip, user, db, state, error_code, error_message, " + + conn.addRequestProperty("columns", "query_id, `time`, client_ip, user, catalog, db, state, error_code, error_message, " + "query_time, scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash, " + "sql_digest, peak_memory_bytes, stmt"); @@ -88,7 +88,7 @@ public class DorisStreamLoader { sb.append("-H \"").append("Expect\":").append("\"100-continue\" \\\n "); sb.append("-H \"").append("Content-Type\":").append("\"text/plain; charset=UTF-8\" \\\n "); sb.append("-H \"").append("max_filter_ratio\":").append("\"1.0\" \\\n "); - sb.append("-H \"").append("columns\":").append("\"query_id, time, client_ip, user, db, state, error_code, " + + sb.append("-H \"").append("columns\":").append("\"query_id, time, client_ip, user, catalog, db, state, error_code, " + "error_message, query_time, scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, " + "cpu_time_ms, sql_hash, sql_digest, peak_memory_bytes, stmt\" \\\n "); sb.append("\"").append(conn.getURL()).append("\""); diff --git a/pytest/deploy/start.py b/pytest/deploy/start.py index 608d477fe9..dc6498b9e9 100644 --- a/pytest/deploy/start.py +++ b/pytest/deploy/start.py @@ -181,6 +181,7 @@ def add_auditload_plugin(): \`time\` datetime not null comment 'Query start time', \ client_ip varchar(32) comment 'Client IP', \ user varchar(64) comment 'User name', \ + catalog varchar(128) comment 'Catalog of this query', \ db varchar(96) comment 'Database of this query', \ state varchar(8) comment 'Query result state. EOF, ERR, OK', \ query_time bigint comment 'Query execution time in millisecond', \