[Plugin] Making FE audit module pluggable (#3219)
Currently we have implemented the plugin framework in FE.
This CL make the original audit log logic pluggable.
The following classes are mainly implemented:
1. AuditPlugin
The interface of audit plugin
2. AuditEvent
An AuditEvent contains all information about an audit event, such as a query, or a connection.
3. AuditEventProcessor
Audit event processor receive all audit events and deliver them to all installed audit plugins.
This CL implements two audit module plugins:
1. The builtin plugin `AuditLogBuilder`, which act same as the previous logic, to save the
audit log to the `fe.audit.log`
2. An optional plugin `AuditLoader`, which will periodically inserts the audit log into a Doris table
specified by the user. In this way, users can conveniently use SQL to query and analyze this
audit log table.
Some documents are added:
1. HELP docs of install/uninstall/show plugin.
2. Rename the `README.md` in `fe_plugins/` dir to `plugin-development-manual.md` and move
it to the `docs/` dir
3. `audit-plugin.md` to introduce the usage of `AuditLoader` plugin.
ISSUE： #3226
This commit is contained in:
Mingyu Chen
@ -25,18 +25,15 @@ export DORIS_HOME=${ROOT}
|
||||
|
||||
. ${DORIS_HOME}/env.sh
|
||||
|
||||
|
||||
# Check args
|
||||
usage() {
|
||||
echo "
|
||||
Usage: $0 <options>
|
||||
Optional options:
|
||||
--fe build Frontend
|
||||
--p build special plugin
|
||||
--clean clean and build Frontend
|
||||
--plugin build special plugin
|
||||
Eg.
|
||||
$0 --fe build Frontend and all plugin
|
||||
$0 --p xxx build xxx plugin
|
||||
$0 --plugin xxx build xxx plugin
|
||||
$0 build all plugins
|
||||
"
|
||||
exit 1
|
||||
}
|
||||
@ -45,8 +42,7 @@ OPTS=$(getopt \
|
||||
-n $0 \
|
||||
-o '' \
|
||||
-o 'h' \
|
||||
-l 'fe' \
|
||||
-l 'p' \
|
||||
-l 'plugin' \
|
||||
-l 'clean' \
|
||||
-l 'help' \
|
||||
-- "$@")
|
||||
@ -57,23 +53,17 @@ fi
|
||||
|
||||
eval set -- "$OPTS"
|
||||
|
||||
BUILD_CLEAN=
|
||||
BUILD_FE=
|
||||
ALL_PLUGIN=
|
||||
ALL_PLUGIN=1
|
||||
CLEAN=0
|
||||
if [ $# == 1 ] ; then
|
||||
# defuat
|
||||
BUILD_CLEAN=0
|
||||
BUILD_FE=0
|
||||
ALL_PLUGIN=1
|
||||
CLEAN=0
|
||||
else
|
||||
BUILD_FE=0
|
||||
ALL_PLUGIN=1
|
||||
BUILD_CLEAN=0
|
||||
while true; do
|
||||
case "$1" in
|
||||
--fe) BUILD_FE=1 ; shift ;;
|
||||
--p) ALL_PLUGIN=0 ; shift ;;
|
||||
--clean) BUILD_CLEAN=1 ; shift ;;
|
||||
--plugin) ALL_PLUGIN=0 ; shift ;;
|
||||
--clean) CLEAN=1 ; shift ;;
|
||||
-h) HELP=1; shift ;;
|
||||
--help) HELP=1; shift ;;
|
||||
--) shift ; break ;;
|
||||
@ -87,35 +77,33 @@ if [[ ${HELP} -eq 1 ]]; then
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ ${CLEAN} -eq 1 -a ${BUILD_FE} -eq 0 ]; then
|
||||
echo "--clean can not be specified without --fe"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Get params:
|
||||
BUILD_FE -- $BUILD_FE
|
||||
BUILD_CLEAN -- $BUILD_CLEAN
|
||||
BUILD_ALL_PLUGIN -- $ALL_PLUGIN
|
||||
CLEAN -- $CLEAN
|
||||
"
|
||||
|
||||
cd ${DORIS_HOME}
|
||||
|
||||
# Clean and build Frontend
|
||||
if [ ${BUILD_FE} -eq 1 ] ; then
|
||||
if [ ${CLEAN} -eq 1 ] ; then
|
||||
sh build.sh --fe --clean
|
||||
else
|
||||
sh build.sh --fe
|
||||
fi
|
||||
# check if palo-fe.jar exist
|
||||
if [ ! -f "$DORIS_HOME/fe/target/palo-fe.jar" ]; then
|
||||
echo "ERROR: palo-fe.jar does not exist. Please build FE first"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
cd ${DORIS_HOME}
|
||||
PLUGIN_MODULE=
|
||||
if [ ${ALL_PLUGIN} -eq 1 ] ; then
|
||||
cd ${DORIS_HOME}/fe_plugins
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
${MVN_CMD} clean
|
||||
fi
|
||||
echo "build all plugins"
|
||||
${MVN_CMD} package -DskipTests
|
||||
${MVN_CMD} package -DskipTests
|
||||
else
|
||||
cd ${DORIS_HOME}/fe_plugins/$1
|
||||
echo "build plugin $1"
|
||||
PLUGIN_MODULE=$1
|
||||
cd ${DORIS_HOME}/fe_plugins/$PLUGIN_MODULE
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
${MVN_CMD} clean
|
||||
fi
|
||||
echo "build plugin $PLUGIN_MODULE"
|
||||
${MVN_CMD} package -DskipTests
|
||||
fi
|
||||
|
||||
@ -124,10 +112,15 @@ cd ${DORIS_HOME}
|
||||
DORIS_OUTPUT=${DORIS_HOME}/fe_plugins/output/
|
||||
mkdir -p ${DORIS_OUTPUT}
|
||||
|
||||
cp -p ${DORIS_HOME}/fe_plugins/*/target/*.zip ${DORIS_OUTPUT}/
|
||||
if [ ${ALL_PLUGIN} -eq 1 ] ; then
|
||||
cp -p ${DORIS_HOME}/fe_plugins/*/target/*.zip ${DORIS_OUTPUT}/
|
||||
else
|
||||
cp -p ${DORIS_HOME}/fe_plugins/$PLUGIN_MODULE/target/*.zip ${DORIS_OUTPUT}/
|
||||
fi
|
||||
|
||||
echo "***************************************"
|
||||
echo "Successfully build Doris Plugin"
|
||||
echo "Successfully build Doris FE Plugin"
|
||||
echo "***************************************"
|
||||
|
||||
exit 0
|
||||
|
||||
|
||||
108
docs/documentation/cn/extending-doris/audit-plugin.md
Normal file
108
docs/documentation/cn/extending-doris/audit-plugin.md
Normal file
@ -0,0 +1,108 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# 审计日志插件
|
||||
|
||||
Doris 的审计日志插件是在 FE 的插件框架基础上开发的。是一个可选插件。用户可以在运行时安装或卸载这个插件。
|
||||
|
||||
该插件可以将 FE 的审计日志定期的导入到指定 Doris 集群中，以方便用户通过 SQL 对审计日志进行查看和分析。
|
||||
|
||||
## 编译、配置和部署
|
||||
|
||||
### 编译
|
||||
|
||||
在 Doris 代码目录下执行 `sh build_plugin.sh` 后，会在 `fe_plugins/output` 目录下得到 `auditloader.zip` 文件。
|
||||
|
||||
### 配置
|
||||
|
||||
解压 `auditloader.zip` 可以看到三个文件：
|
||||
|
||||
```
|
||||
auditloader.jar
|
||||
plugin.properties
|
||||
plugin.conf
|
||||
```
|
||||
|
||||
打开 `plugin.conf` 进行配置。配置项说明参见注释。
|
||||
|
||||
配置完成后，重新将三个文件打包为 `auditloader.zip`.
|
||||
|
||||
### 部署
|
||||
|
||||
您可以将这个文件放置在一个 http 下载服务器上，或者拷贝到所有 FE 的指定目录下。这里我们使用后者。
|
||||
|
||||
### 安装
|
||||
|
||||
部署完成后，安装插件前，需要创建之前在 `plugin.conf` 中指定的审计数据库和表。其中建表语句如下：
|
||||
|
||||
```
|
||||
create table doris_audit_tbl__
|
||||
(
|
||||
query_id varchar(48) comment "Unique query id",
|
||||
time datetime not null comment "Query start time",
|
||||
client_ip varchar(32) comment "Client IP",
|
||||
user varchar(64) comment "User name",
|
||||
db varchar(96) comment "Database of this query",
|
||||
state varchar(8) comment "Query result state. EOF, ERR, OK",
|
||||
query_time bigint comment "Query execution time in millisecond",
|
||||
scan_bytes bigint comment "Total scan bytes of this query",
|
||||
scan_rows bigint comment "Total scan rows of this query",
|
||||
return_rows bigint comment "Returned rows of this query",
|
||||
stmt_id int comment "An incremental id of statement",
|
||||
is_query tinyint comment "Is this statemt a query. 1 or 0",
|
||||
frontend_ip varchar(32) comment "Frontend ip of executing this statement",
|
||||
stmt varchar(2048) comment "The original statement, trimed if longer than 2048 bytes"
|
||||
)
|
||||
partition by range(time) ()
|
||||
distributed by hash(query_id) buckets 1
|
||||
properties(
|
||||
"dynamic_partition.time_unit" = "DAY",
|
||||
"dynamic_partition.start" = "-30",
|
||||
"dynamic_partition.end" = "3",
|
||||
"dynamic_partition.prefix" = "p",
|
||||
"dynamic_partition.buckets" = "1",
|
||||
"dynamic_partition.enable" = "true",
|
||||
"replication_num" = "1"
|
||||
);
|
||||
```
|
||||
|
||||
其中 `dynamic_partition` 属性根据自己的需要，选择审计日志安保留的天数。
|
||||
|
||||
之后，连接到 Doris 后使用 `INSTALL PLUGIN` 命令完成安装。安装成功后，可以通过 `SHOW PLUGINS` 看到已经安装的插件，并且状态为 `INSTALLED`。
|
||||
|
||||
完成后，插件会不断的以指定的时间间隔将审计日志插入到这个表中。
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,291 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# 插件开发手册
|
||||
|
||||
## 介绍
|
||||
|
||||
Doris 支持动态加载插件。用户可以通过开发自己的插件来扩展Doris的功能。这个手册主要介绍如何开发、编译和部署 Frontend 端的插件。
|
||||
|
||||
`fe_plugins` 目录是 FE 插件的根模块。这个根模块统一管理插件所需的依赖。添加一个新的插件，相当于在这个根模块添加一个子模块。
|
||||
|
||||
## 插件组成
|
||||
|
||||
一个FE的插件可以使一个**zip压缩包**或者是一个**目录**。其内容至少包含两个文件：`plugin.properties` 和 `.jar` 文件。`plugin.properties`用于描述插件信息。
|
||||
|
||||
文件结构如下：
|
||||
|
||||
```
|
||||
# plugin .zip
|
||||
auditodemo.zip:
|
||||
-plugin.properties
|
||||
-auditdemo.jar
|
||||
-xxx.config
|
||||
-data/
|
||||
-test_data/
|
||||
|
||||
# plugin local directory
|
||||
auditodemo/:
|
||||
-plugin.properties
|
||||
-auditdemo.jar
|
||||
-xxx.config
|
||||
-data/
|
||||
-test_data/
|
||||
```
|
||||
|
||||
`plugin.properties` 内容示例:
|
||||
|
||||
```
|
||||
### required:
|
||||
#
|
||||
# the plugin name
|
||||
name = audit_plugin_demo
|
||||
#
|
||||
# the plugin type
|
||||
type = AUDIT
|
||||
#
|
||||
# simple summary of the plugin
|
||||
description = just for test
|
||||
#
|
||||
# Doris's version, like: 0.11.0
|
||||
version = 0.11.0
|
||||
|
||||
### FE-Plugin optional:
|
||||
#
|
||||
# version of java the code is built against
|
||||
# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4
|
||||
java.version = 1.8.31
|
||||
#
|
||||
# the name of the class to load, fully-qualified.
|
||||
classname = AuditPluginDemo
|
||||
|
||||
### BE-Plugin optional:
|
||||
# the name of the so to load
|
||||
soName = example.so
|
||||
```
|
||||
|
||||
## 编写一个插件
|
||||
|
||||
插件的开发环境依赖Doris的开发编译环境。所以请先确保Doris的编译开发环境运行正常。
|
||||
|
||||
### 创建一个模块
|
||||
|
||||
我们可以通过以下命令在 `fe_plugins` 目录创建一个子模块用户实现创建和创建工程。其中 `doris-fe-test` 为插件名称。
|
||||
|
||||
```
|
||||
mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false
|
||||
```
|
||||
|
||||
这个命令会创建一个新的 maven 工程，并且自动向 `fe_plugins/pom.xml` 中添加一个子模块：
|
||||
|
||||
```
|
||||
.....
|
||||
<groupId>org.apache</groupId>
|
||||
<artifactId>doris-fe-plugins</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<modules>
|
||||
<module>auditdemo</module>
|
||||
# new plugin module
|
||||
<module>doris-fe-test</module>
|
||||
</modules>
|
||||
.....
|
||||
```
|
||||
|
||||
新的工程目录结构如下：
|
||||
|
||||
```
|
||||
-doris-fe-test/
|
||||
-pom.xml
|
||||
-src/
|
||||
---- main/java/org/apache/
|
||||
------- App.java # mvn auto generate, ignore
|
||||
---- test/java/org/apache
|
||||
```
|
||||
|
||||
接下来我们在 `main` 目录下添加一个 `assembly` 目录来存放 `plugin.properties` 和 `zip.xml`。最终的工程目录结构如下：
|
||||
|
||||
```
|
||||
-doris-fe-test/
|
||||
-pom.xml
|
||||
-src/
|
||||
---- main/
|
||||
------ assembly/
|
||||
-------- plugin.properties
|
||||
-------- zip.xml
|
||||
------ java/org/apache/
|
||||
--------App.java # mvn auto generate, ignore
|
||||
---- test/java/org/apache
|
||||
```
|
||||
|
||||
### 添加 zip.xml
|
||||
|
||||
`zip.xml` 用于描述最终生成的 zip 压缩包中的文件内容。（如 .jar file, plugin.properties 等等）
|
||||
|
||||
```
|
||||
<assembly>
|
||||
<id>plugin</id>
|
||||
<formats>
|
||||
<format>zip</format>
|
||||
</formats>
|
||||
<!-IMPORTANT: must be false->
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>target</directory>
|
||||
<includes>
|
||||
<include>*.jar</include>
|
||||
</ ncludes>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
</fileSet>
|
||||
|
||||
<fileSet>
|
||||
<directory>src/main/assembly</directory>
|
||||
<includes>
|
||||
<include>plugin.properties</include>
|
||||
</includes>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
</assembly>
|
||||
```
|
||||
|
||||
### 更新 pom.xml
|
||||
|
||||
接下来我们需要更新子模块的 `pom.xml` 文件，添加 doris-fe 依赖：
|
||||
|
||||
```
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<groupId>org.apache</groupId>
|
||||
<artifactId>doris-fe-plugins</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>auditloader</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache</groupId>
|
||||
<artifactId>doris-fe</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- other dependencies -->
|
||||
<dependency>
|
||||
...
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<build>
|
||||
<finalName>auditloader</finalName>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.4.1</version>
|
||||
<configuration>
|
||||
<appendAssemblyId>false</appendAssemblyId>
|
||||
<descriptors>
|
||||
<descriptor>src/main/assembly/zip.xml</descriptor>
|
||||
</descriptors>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
```
|
||||
|
||||
### 实现插件
|
||||
|
||||
之后我们就可以开始愉快的进行插件功能的开发啦。插件需要实现 `Plugin` 接口。具体可以参阅 Doris 自带的 `auditdemo` 插件示例代码。
|
||||
|
||||
### 编译
|
||||
|
||||
在编译插件之前，需要先执行 `sh build.sh --fe` 进行 Doris FE 代码的编译，并确保编译成功。
|
||||
|
||||
之后，执行 `sh build_plugin.sh` 编译所有插件。最终的产出会存放在 `fe_plugins/output` 目录中。
|
||||
|
||||
或者也可以执行 `sh build_plugin.sh --plugin your_plugin_name` 来仅编译指定的插件。
|
||||
|
||||
### 另一种开发方式
|
||||
|
||||
您可以直接通过修改自带的 `auditdemo` 插件示例代码进行开发。
|
||||
|
||||
## 部署
|
||||
|
||||
插件可以通过以下三种方式部署。
|
||||
|
||||
* 将 `.zip` 文件放在 Http 或 Https 服务器上。如：`http://xxx.xxxxxx.com/data/plugin.zip`, Doris 会下载这个文件。
|
||||
* 本地 `.zip` 文件。 如：`/home/work/data/plugin.zip`。需要在所有 FE 和 BE 节点部署。
|
||||
* 本地目录。如：`/home/work/data/plugin/`。这个相当于 `.zip` 文件解压后的目录。需要在所有 FE 和 BE 节点部署。
|
||||
|
||||
注意：需保证部署路径在整个插件生命周期内有效。
|
||||
|
||||
## 安装和卸载插件
|
||||
|
||||
通过如下命令安装和卸载插件。更多帮助请参阅 `HELP INSTALL PLUGIN;` `HELP IUNNSTALL PLUGIN;` `HELP SHOW PLUGINS;`
|
||||
|
||||
```
|
||||
mysql> install plugin from "/home/users/seaven/auditdemo.zip";
|
||||
Query OK, 0 rows affected (0.09 sec)
|
||||
|
||||
mysql> mysql> show plugins\G
|
||||
*************************** 1. row ***************************
|
||||
Name: auditloader
|
||||
Type: AUDIT
|
||||
Description: load audit log to olap load, and user can view the statistic of queries
|
||||
Version: 0.12.0
|
||||
JavaVersion: 1.8.31
|
||||
ClassName: AuditLoaderPlugin
|
||||
SoName: NULL
|
||||
Sources: /home/cmy/git/doris/core/fe_plugins/output/auditloader.zip
|
||||
Status: INSTALLED
|
||||
*************************** 2. row ***************************
|
||||
Name: AuditLogBuilder
|
||||
Type: AUDIT
|
||||
Description: builtin audit logger
|
||||
Version: 0.12.0
|
||||
JavaVersion: 1.8.31
|
||||
ClassName: org.apache.doris.qe.AuditLogBuilder
|
||||
SoName: NULL
|
||||
Sources: Builtin
|
||||
Status: INSTALLED
|
||||
2 rows in set (0.00 sec)
|
||||
|
||||
mysql> uninstall plugin auditloader;
|
||||
Query OK, 0 rows affected (0.05 sec)
|
||||
|
||||
mysql> show plugins;
|
||||
Empty set (0.00 sec)
|
||||
```
|
||||
@ -0,0 +1,50 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# INSTALL PLUGIN
|
||||
## description
|
||||
|
||||
该语句用于安装一个插件。
|
||||
|
||||
语法
|
||||
|
||||
INSTALL PLUGIN FROM [source]
|
||||
|
||||
source 支持三种类型：
|
||||
|
||||
1. 指向一个 zip 文件的绝对路径。
|
||||
2. 指向一个插件目录的绝对路径。
|
||||
3. 指向一个 http 或 https 协议的 zip 文件下载路径
|
||||
|
||||
## example
|
||||
|
||||
1. 安装一个本地 zip 文件插件：
|
||||
|
||||
INSTALL PLUGIN FROM "/home/users/seaven/auditdemo.zip";
|
||||
|
||||
2. 安装一个本地目录中的插件：
|
||||
|
||||
INSTALL PLUGIN FROM "/home/users/seaven/auditdemo/";
|
||||
|
||||
2. 下载并安装一个插件：
|
||||
|
||||
INSTALL PLUGIN FROM "http://mywebsite.com/plugin.zip";
|
||||
|
||||
## keyword
|
||||
INSTALL,PLUGIN
|
||||
@ -0,0 +1,38 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# SHOW PLUGINS
|
||||
## description
|
||||
|
||||
该语句用于展示已安装的插件。
|
||||
|
||||
语法
|
||||
|
||||
SHOW PLUGINS;
|
||||
|
||||
该命令会展示所有用户安装的和系统内置的插件。
|
||||
|
||||
## example
|
||||
|
||||
1. 展示已安装的插件：
|
||||
|
||||
SHOW PLUGINS;
|
||||
|
||||
## keyword
|
||||
SHOW PLUGINS
|
||||
@ -0,0 +1,40 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# UNINTALL PLUGIN
|
||||
## description
|
||||
|
||||
该语句用于卸载一个插件。
|
||||
|
||||
语法
|
||||
|
||||
UNINSTALL PLUGIN plugin_name;
|
||||
|
||||
plugin_name 可以通过 `SHOW PLUGINS;` 命令查看。
|
||||
|
||||
只能卸载非 builtin 的插件。
|
||||
|
||||
## example
|
||||
|
||||
1. 卸载一个插件：
|
||||
|
||||
UNINSTALL PLUGIN auditdemo;
|
||||
|
||||
## keyword
|
||||
UNINSTALL,PLUGIN
|
||||
89
docs/documentation/en/extending-doris/audit-plugin_EN.md
Normal file
89
docs/documentation/en/extending-doris/audit-plugin_EN.md
Normal file
@ -0,0 +1,89 @@
|
||||
<!-
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
->
|
||||
|
||||
# Audit log plugin
|
||||
|
||||
Doris's audit log plugin was developed based on FE's plugin framework. Is an optional plugin. Users can install or uninstall this plugin at runtime.
|
||||
|
||||
This plugin can periodically import the FE audit log into the specified Doris cluster, so that users can easily view and analyze the audit log through SQL.
|
||||
|
||||
## Compile, Configure and Deploy
|
||||
|
||||
### Compile
|
||||
|
||||
After executing `sh build_plugin.sh` in the Doris code directory, you will get the `auditloader.zip` file in the `fe_plugins/output` directory.
|
||||
|
||||
### Configuration
|
||||
|
||||
Unzip `auditloader.zip` and you will see three files:
|
||||
|
||||
```
|
||||
auditloader.jar
|
||||
plugin.properties
|
||||
plugin.conf
|
||||
```
|
||||
|
||||
Open `plugin.conf` for configuration. See the comments of the configuration items.
|
||||
|
||||
After the configuration is complete, repackage the three files as `auditloader.zip`.
|
||||
|
||||
### Deployment
|
||||
|
||||
You can place this file on an http download server or copy it to the specified directory of all FEs. Here we use the latter.
|
||||
|
||||
### Installation
|
||||
|
||||
After deployment is complete, and before installing the plugin, you need to create the audit database and tables previously specified in `plugin.conf`. The table creation statement is as follows:
|
||||
|
||||
```
|
||||
create table doris_audit_tbl__
|
||||
(
|
||||
query_id varchar (48) comment "Unique query id",
|
||||
time datetime not null comment "Query start time",
|
||||
client_ip varchar (32) comment "Client IP",
|
||||
user varchar (64) comment "User name",
|
||||
db varchar (96) comment "Database of this query",
|
||||
state varchar (8) comment "Query result state. EOF, ERR, OK",
|
||||
query_time bigint comment "Query execution time in millisecond",
|
||||
scan_bytes bigint comment "Total scan bytes of this query",
|
||||
scan_rows bigint comment "Total scan rows of this query",
|
||||
return_rows bigint comment "Returned rows of this query",
|
||||
stmt_id int comment "An incremental id of statement",
|
||||
is_query tinyint comment "Is this statemt a query. 1 or 0",
|
||||
frontend_ip varchar (32) comment "Frontend ip of executing this statement",
|
||||
stmt varchar (2048) comment "The original statement, trimed if longer than 2048 bytes"
|
||||
)
|
||||
partition by range (time) ()
|
||||
distributed by hash (query_id) buckets 1
|
||||
properties (
|
||||
"dynamic_partition.time_unit" = "DAY",
|
||||
"dynamic_partition.start" = "-30",
|
||||
"dynamic_partition.end" = "3",
|
||||
"dynamic_partition.prefix" = "p",
|
||||
"dynamic_partition.buckets" = "1",
|
||||
"dynamic_partition.enable" = "true",
|
||||
"replication_num" = "1"
|
||||
);
|
||||
```
|
||||
|
||||
The `dynamic_partition` attribute selects the number of days to keep the audit log based on your needs.
|
||||
|
||||
After that, connect to Doris and use the `INSTALL PLUGIN` command to complete the installation. After successful installation, you can see the installed plug-ins through `SHOW PLUGINS`, and the status is `INSTALLED`.
|
||||
|
||||
Upon completion, the plug-in will continuously import audit date into this table at specified intervals.
|
||||
@ -0,0 +1,293 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Plugin Development Manual
|
||||
|
||||
## Introduction
|
||||
|
||||
Doris supports dynamic loading of plug-ins. Users can develop their own plug-ins to implement some extended functions. This manual mainly introduces the development, compilation and deployment methods of Frontend-side plug-ins.
|
||||
|
||||
|
||||
`fe_plugins` is the parent module of the fe plugins. It can uniformly manage the third-party library information that the plugin depends on. Adding a plugin can add a submodule implementation under `fe_plugins`.
|
||||
|
||||
## Plugin
|
||||
|
||||
A FE Plugin can be a **.zip package** or a **directory**, which contains at least two parts: the `plugin.properties` and `.jar` files. The `plugin.properties` file is used to describe the plugin information.
|
||||
|
||||
The file structure of a Plugin looks like this:
|
||||
|
||||
```
|
||||
# plugin .zip
|
||||
auditodemo.zip:
|
||||
-plugin.properties
|
||||
-auditdemo.jar
|
||||
-xxx.config
|
||||
-data/
|
||||
-test_data/
|
||||
|
||||
# plugin local directory
|
||||
auditodemo/:
|
||||
-plugin.properties
|
||||
-auditdemo.jar
|
||||
-xxx.config
|
||||
-data/
|
||||
-test_data/
|
||||
```
|
||||
|
||||
`plugin.properties` example:
|
||||
|
||||
```
|
||||
### required:
|
||||
#
|
||||
# the plugin name
|
||||
name = audit_plugin_demo
|
||||
#
|
||||
# the plugin type
|
||||
type = AUDIT
|
||||
#
|
||||
# simple summary of the plugin
|
||||
description = just for test
|
||||
#
|
||||
# Doris's version, like: 0.11.0
|
||||
version = 0.11.0
|
||||
|
||||
### FE-Plugin optional:
|
||||
#
|
||||
# version of java the code is built against
|
||||
# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4
|
||||
java.version = 1.8.31
|
||||
#
|
||||
# the name of the class to load, fully-qualified.
|
||||
classname = AuditPluginDemo
|
||||
|
||||
### BE-Plugin optional:
|
||||
# the name of the so to load
|
||||
soName = example.so
|
||||
```
|
||||
|
||||
## Write A Plugin
|
||||
|
||||
The development environment of the FE plugin depends on the development environment of Doris. So please make sure Doris's compilation and development environment works normally.
|
||||
|
||||
### Create module
|
||||
|
||||
We can add a submodule in the `fe_plugins` directory to implement Plugin and create a project:
|
||||
|
||||
```
|
||||
mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false
|
||||
```
|
||||
|
||||
The command produces a new mvn project, and a new submodule is automatically added to `fe_plugins/pom.xml`:
|
||||
|
||||
```
|
||||
.....
|
||||
<groupId>org.apache</groupId>
|
||||
<artifactId>doris-fe-plugins</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<modules>
|
||||
<module>auditdemo</module>
|
||||
# new plugin module
|
||||
<module>doris-fe-test</module>
|
||||
</modules>
|
||||
.....
|
||||
```
|
||||
|
||||
The new plugin project file structure is as follows:
|
||||
|
||||
```
|
||||
-doris-fe-test/
|
||||
-pom.xml
|
||||
-src/
|
||||
---- main/java/org/apache/
|
||||
------- App.java # mvn auto generate, ignore
|
||||
---- test/java/org/apache
|
||||
```
|
||||
|
||||
We will add an assembly folder under main to store `plugin.properties` and `zip.xml`. After completion, the file structure is as follows:
|
||||
|
||||
```
|
||||
-doris-fe-test/
|
||||
-pom.xml
|
||||
-src/
|
||||
---- main/
|
||||
------ assembly/
|
||||
-------- plugin.properties
|
||||
-------- zip.xml
|
||||
------ java/org/apache/
|
||||
--------App.java # mvn auto generate, ignore
|
||||
---- test/java/org/apache
|
||||
```
|
||||
|
||||
### Add zip.xml
|
||||
|
||||
`zip.xml`, used to describe the content of the final package of the plugin (.jar file, plugin.properties):
|
||||
|
||||
```
|
||||
<assembly>
|
||||
<id>plugin</id>
|
||||
<formats>
|
||||
<format>zip</format>
|
||||
</formats>
|
||||
<!-IMPORTANT: must be false->
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>target</directory>
|
||||
<includes>
|
||||
<include>*.jar</include>
|
||||
</ ncludes>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
</fileSet>
|
||||
|
||||
<fileSet>
|
||||
<directory>src/main/assembly</directory>
|
||||
<includes>
|
||||
<include>plugin.properties</include>
|
||||
</includes>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
</assembly>
|
||||
```
|
||||
|
||||
|
||||
### Update pom.xml
|
||||
|
||||
Then we need to update `pom.xml`, add doris-fe dependency, and modify maven packaging way:
|
||||
|
||||
```
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<groupId>org.apache</groupId>
|
||||
<artifactId>doris-fe-plugins</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>auditloader</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache</groupId>
|
||||
<artifactId>doris-fe</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- other dependencies -->
|
||||
<dependency>
|
||||
...
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<build>
|
||||
<finalName>auditloader</finalName>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.4.1</version>
|
||||
<configuration>
|
||||
<appendAssemblyId>false</appendAssemblyId>
|
||||
<descriptors>
|
||||
<descriptor>src/main/assembly/zip.xml</descriptor>
|
||||
</descriptors>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
```
|
||||
|
||||
### Implement plugin
|
||||
|
||||
Then we can happily implement Plugin according to the needs. Plugins need to implement the `Plugin` interface. For details, please refer to the `auditdemo` plugin sample code that comes with Doris.
|
||||
|
||||
### Compile
|
||||
|
||||
Before compiling the plugin, you must first execute `sh build.sh --fe` of Doris to complete the compilation of Doris FE.
|
||||
|
||||
Finally, execute `sh build_plugin.sh` in the ${DORIS_HOME} path and you will find the `your_plugin_name.zip` file in `fe_plugins/output`
|
||||
|
||||
Or you can execute `sh build_plugin.sh --plugin your_plugin_name` to only build your plugin.
|
||||
|
||||
### Other way
|
||||
|
||||
The easiest way, you can implement your plugin by modifying the example `auditdemo`
|
||||
|
||||
## Deploy
|
||||
|
||||
Doris's plugin can be deployed in three ways:
|
||||
|
||||
* Http or Https .zip, like `http://xxx.xxxxxx.com/data/plugin.zip`, Doris will download this .zip file
|
||||
* Local .zip, like `/home/work/data/plugin.zip`, need to be deployed on all FE and BE nodes
|
||||
* Local directory, like `/home/work/data/plugin`, .zip decompressed folder, need to be deployed on all FE, BE nodes
|
||||
|
||||
Note: Need to ensure that the plugin .zip file is available in the life cycle of doris!
|
||||
|
||||
## Install and Uninstall
|
||||
|
||||
Install and uninstall the plugin through the install/uninstall statements. More details, see `HELP INSTALL PLUGIN;` `HELP IUNNSTALL PLUGIN;` `HELP SHOW PLUGINS;`
|
||||
|
||||
```
|
||||
mysql> install plugin from "/home/users/seaven/auditdemo.zip";
|
||||
Query OK, 0 rows affected (0.09 sec)
|
||||
|
||||
mysql> mysql> show plugins\G
|
||||
*************************** 1. row ***************************
|
||||
Name: auditloader
|
||||
Type: AUDIT
|
||||
Description: load audit log to olap load, and user can view the statistic of queries
|
||||
Version: 0.12.0
|
||||
JavaVersion: 1.8.31
|
||||
ClassName: AuditLoaderPlugin
|
||||
SoName: NULL
|
||||
Sources: /home/cmy/git/doris/core/fe_plugins/output/auditloader.zip
|
||||
Status: INSTALLED
|
||||
*************************** 2. row ***************************
|
||||
Name: AuditLogBuilder
|
||||
Type: AUDIT
|
||||
Description: builtin audit logger
|
||||
Version: 0.12.0
|
||||
JavaVersion: 1.8.31
|
||||
ClassName: org.apache.doris.qe.AuditLogBuilder
|
||||
SoName: NULL
|
||||
Sources: Builtin
|
||||
Status: INSTALLED
|
||||
2 rows in set (0.00 sec)
|
||||
|
||||
mysql> uninstall plugin auditloader;
|
||||
Query OK, 0 rows affected (0.05 sec)
|
||||
|
||||
mysql> show plugins;
|
||||
Empty set (0.00 sec)
|
||||
```
|
||||
@ -0,0 +1,50 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# INSTALL PLUGIN
|
||||
## description
|
||||
|
||||
To install a plugin
|
||||
|
||||
Syntax
|
||||
|
||||
INSTALL PLUGIN FROM [source]
|
||||
|
||||
source supports 3 kinds:
|
||||
|
||||
1. Point to a zip file with absolute path.
|
||||
2. Point to a plugin dir with absolute path.
|
||||
3. Point to a http/https download link of zip file.
|
||||
|
||||
## example
|
||||
|
||||
1. Intall a plugin with a local zip file:
|
||||
|
||||
INSTALL PLUGIN FROM "/home/users/seaven/auditdemo.zip";
|
||||
|
||||
2. Intall a plugin with a local dir:
|
||||
|
||||
INSTALL PLUGIN FROM "/home/users/seaven/auditdemo/";
|
||||
|
||||
2. Download and install a plugin:
|
||||
|
||||
INSTALL PLUGIN FROM "http://mywebsite.com/plugin.zip";
|
||||
|
||||
## keyword
|
||||
INSTALL,PLUGIN
|
||||
@ -0,0 +1,38 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# SHOW PLUGINS
|
||||
## description
|
||||
|
||||
To view the installed plugins.
|
||||
|
||||
Syntax
|
||||
|
||||
SHOW PLUGINS;
|
||||
|
||||
This command will show all builtin and custom plugins.
|
||||
|
||||
## example
|
||||
|
||||
1. To view the installed plugins:
|
||||
|
||||
SHOW PLUGINS;
|
||||
|
||||
## keyword
|
||||
SHOW PLUGINS
|
||||
@ -0,0 +1,40 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# UNINTALL PLUGIN
|
||||
## description
|
||||
|
||||
To uninstall a plugin.
|
||||
|
||||
Syntax
|
||||
|
||||
UNINSTALL PLUGIN plugin_name;
|
||||
|
||||
plugin_name can be found by `SHOW PLUGINS;`.
|
||||
|
||||
Can only uninstall non-builtin plugins.
|
||||
|
||||
## example
|
||||
|
||||
1. Uninstall a plugin:
|
||||
|
||||
UNINSTALL PLUGIN auditdemo;
|
||||
|
||||
## keyword
|
||||
UNINSTALL,PLUGIN
|
||||
@ -179,10 +179,6 @@ public class Alter {
|
||||
}
|
||||
OlapTable olapTable = (OlapTable) table;
|
||||
|
||||
if (olapTable.getPartitions().size() == 0 && !currentAlterOps.hasPartitionOp()) {
|
||||
throw new DdlException("Table with empty parition cannot do schema change. [" + tableName + "]");
|
||||
}
|
||||
|
||||
if (olapTable.getState() != OlapTableState.NORMAL) {
|
||||
throw new DdlException(
|
||||
"Table[" + table.getName() + "]'s state is not NORMAL. Do not allow doing ALTER ops");
|
||||
|
||||
@ -180,6 +180,7 @@ import org.apache.doris.persist.TablePropertyInfo;
|
||||
import org.apache.doris.persist.TruncateTableInfo;
|
||||
import org.apache.doris.plugin.PluginInfo;
|
||||
import org.apache.doris.plugin.PluginMgr;
|
||||
import org.apache.doris.qe.AuditEventProcessor;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.qe.JournalObservable;
|
||||
import org.apache.doris.qe.SessionVariable;
|
||||
@ -387,6 +388,8 @@ public class Catalog {
|
||||
|
||||
private PluginMgr pluginMgr;
|
||||
|
||||
private AuditEventProcessor auditEventProcessor;
|
||||
|
||||
public List<Frontend> getFrontends(FrontendNodeType nodeType) {
|
||||
if (nodeType == null) {
|
||||
// get all
|
||||
@ -521,6 +524,7 @@ public class Catalog {
|
||||
this.imageDir = this.metaDir + IMAGE_DIR;
|
||||
|
||||
this.pluginMgr = new PluginMgr();
|
||||
this.auditEventProcessor = new AuditEventProcessor(this.pluginMgr);
|
||||
}
|
||||
|
||||
public static void destroyCheckpoint() {
|
||||
@ -586,6 +590,10 @@ public class Catalog {
|
||||
return fullNameToDb;
|
||||
}
|
||||
|
||||
public AuditEventProcessor getAuditEventProcessor() {
|
||||
return auditEventProcessor;
|
||||
}
|
||||
|
||||
// use this to get correct ClusterInfoService instance
|
||||
public static SystemInfoService getCurrentSystemInfo() {
|
||||
return getCurrentCatalog().getClusterInfo();
|
||||
@ -622,6 +630,10 @@ public class Catalog {
|
||||
return getCurrentCatalog().getPluginMgr();
|
||||
}
|
||||
|
||||
public static AuditEventProcessor getCurrentAuditEventProcessor() {
|
||||
return getCurrentCatalog().getAuditEventProcessor();
|
||||
}
|
||||
|
||||
// Use tryLock to avoid potential dead lock
|
||||
private boolean tryLock(boolean mustLock) {
|
||||
while (true) {
|
||||
@ -704,6 +716,7 @@ public class Catalog {
|
||||
|
||||
// init plugin manager
|
||||
pluginMgr.init();
|
||||
auditEventProcessor.start();
|
||||
|
||||
// 2. get cluster id and role (Observer or Follower)
|
||||
getClusterIdAndRole();
|
||||
@ -3612,7 +3625,6 @@ public class Catalog {
|
||||
olapTable.setIndexMeta(baseIndexId, tableName, baseSchema, schemaVersion, schemaHash,
|
||||
shortKeyColumnCount, baseIndexStorageType, keysType);
|
||||
|
||||
|
||||
for (AlterClause alterClause : stmt.getRollupAlterClauseList()) {
|
||||
AddRollupClause addRollupClause = (AddRollupClause)alterClause;
|
||||
|
||||
@ -6405,9 +6417,7 @@ public class Catalog {
|
||||
}
|
||||
|
||||
public void installPlugin(InstallPluginStmt stmt) throws UserException, IOException {
|
||||
PluginInfo pluginInfo = pluginMgr.installPlugin(stmt);
|
||||
editLog.logInstallPlugin(pluginInfo);
|
||||
LOG.info("install plugin = " + pluginInfo.getName());
|
||||
pluginMgr.installPlugin(stmt);
|
||||
}
|
||||
|
||||
public long savePlugins(DataOutputStream dos, long checksum) throws IOException {
|
||||
|
||||
@ -101,11 +101,11 @@ public class TableProperty implements Writable {
|
||||
return this;
|
||||
}
|
||||
|
||||
void modifyTableProperties(Map<String, String> modifyProperties) {
|
||||
public void modifyTableProperties(Map<String, String> modifyProperties) {
|
||||
properties.putAll(modifyProperties);
|
||||
}
|
||||
|
||||
void modifyTableProperties(String key, String value) {
|
||||
public void modifyTableProperties(String key, String value) {
|
||||
properties.put(key, value);
|
||||
}
|
||||
|
||||
|
||||
@ -250,7 +250,13 @@ public class DynamicPartitionUtil {
|
||||
public static void checkAndSetDynamicPartitionProperty(OlapTable olapTable, Map<String, String> properties) throws DdlException {
|
||||
if (DynamicPartitionUtil.checkInputDynamicPartitionProperties(properties, olapTable.getPartitionInfo())) {
|
||||
Map<String, String> dynamicPartitionProperties = DynamicPartitionUtil.analyzeDynamicPartition(properties);
|
||||
olapTable.setTableProperty(new TableProperty(dynamicPartitionProperties).buildDynamicProperty());
|
||||
TableProperty tableProperty = olapTable.getTableProperty();
|
||||
if (tableProperty != null) {
|
||||
tableProperty.modifyTableProperties(dynamicPartitionProperties);
|
||||
tableProperty.buildDynamicProperty();
|
||||
} else {
|
||||
olapTable.setTableProperty(new TableProperty(dynamicPartitionProperties).buildDynamicProperty());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -17,83 +17,153 @@
|
||||
|
||||
package org.apache.doris.plugin;
|
||||
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
|
||||
/*
|
||||
* AuditEvent contains all information about audit log info.
|
||||
* It should be created by AuditEventBuilder. For example:
|
||||
*
|
||||
* AuditEvent event = new AuditEventBuilder()
|
||||
* .setEventType(AFTER_QUERY)
|
||||
* .setClientIp(xxx)
|
||||
* ...
|
||||
* .build();
|
||||
*/
|
||||
public class AuditEvent {
|
||||
/**
|
||||
* Audit plugin event type
|
||||
*/
|
||||
public final static short AUDIT_CONNECTION = 1;
|
||||
public final static short AUDIT_QUERY = 2;
|
||||
|
||||
/**
|
||||
* Connection sub-event masks
|
||||
*/
|
||||
public final static short AUDIT_CONNECTION_CONNECT = 1;
|
||||
public final static short AUDIT_CONNECTION_DISCONNECT = 1 << 1;
|
||||
|
||||
/**
|
||||
* Query sub-event masks
|
||||
*/
|
||||
public final static short AUDIT_QUERY_START = 1;
|
||||
public final static short AUDIT_QUERY_END = 1 << 1;
|
||||
|
||||
protected String query;
|
||||
|
||||
protected short type;
|
||||
|
||||
protected short masks;
|
||||
|
||||
protected ConnectContext context;
|
||||
|
||||
protected AuditEvent(ConnectContext context) {
|
||||
this.context = context;
|
||||
public enum EventType {
|
||||
CONNECTION,
|
||||
DISCONNECTION,
|
||||
BEFORE_QUERY,
|
||||
AFTER_QUERY
|
||||
}
|
||||
|
||||
protected AuditEvent(ConnectContext context, String query) {
|
||||
this.context = context;
|
||||
this.query = query;
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
public static @interface AuditField {
|
||||
String value() default "";
|
||||
}
|
||||
|
||||
public void setEventTypeAndMasks(short type, short masks) {
|
||||
this.type = type;
|
||||
this.masks = masks;
|
||||
}
|
||||
public EventType type;
|
||||
|
||||
public short getEventType() {
|
||||
return type;
|
||||
}
|
||||
// all fields which is about to be audit should be annotated by "@AuditField"
|
||||
// make them all "public" so that easy to visit.
|
||||
@AuditField(value = "Timestamp")
|
||||
public long timestamp = -1;
|
||||
@AuditField(value = "Client")
|
||||
public String clientIp = "";
|
||||
@AuditField(value = "User")
|
||||
public String user = "";
|
||||
@AuditField(value = "Db")
|
||||
public String db = "";
|
||||
@AuditField(value = "State")
|
||||
public String state = "";
|
||||
@AuditField(value = "Time")
|
||||
public long queryTime = -1;
|
||||
@AuditField(value = "ScanBytes")
|
||||
public long scanBytes = -1;
|
||||
@AuditField(value = "ScanRows")
|
||||
public long scanRows = -1;
|
||||
@AuditField(value = "ReturnRows")
|
||||
public long returnRows = -1;
|
||||
@AuditField(value = "StmtId")
|
||||
public long stmtId = -1;
|
||||
@AuditField(value = "QueryId")
|
||||
public String queryId = "";
|
||||
@AuditField(value = "IsQuery")
|
||||
public boolean isQuery = false;
|
||||
@AuditField(value = "fe_ip")
|
||||
public String feIp = "";
|
||||
@AuditField(value = "Stmt")
|
||||
public String stmt = "";
|
||||
|
||||
public short getEventMasks() {
|
||||
return masks;
|
||||
}
|
||||
public static class AuditEventBuilder {
|
||||
|
||||
public int getConnectId() {
|
||||
return context.getConnectionId();
|
||||
}
|
||||
private AuditEvent auditEvent = new AuditEvent();
|
||||
|
||||
public String getIp() {
|
||||
return context.getRemoteIP();
|
||||
}
|
||||
public AuditEventBuilder() {
|
||||
}
|
||||
|
||||
public String getUser() {
|
||||
return context.getQualifiedUser();
|
||||
}
|
||||
public void reset() {
|
||||
auditEvent = new AuditEvent();
|
||||
}
|
||||
|
||||
public long getStmtId() {
|
||||
return context.getStmtId();
|
||||
}
|
||||
public AuditEventBuilder setEventType(EventType eventType) {
|
||||
auditEvent.type = eventType;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getQuery() {
|
||||
return query;
|
||||
}
|
||||
public AuditEventBuilder setTimestamp(long timestamp) {
|
||||
auditEvent.timestamp = timestamp;
|
||||
return this;
|
||||
}
|
||||
|
||||
public static AuditEvent createConnectEvent(ConnectContext ctx) {
|
||||
return new AuditEvent(ctx);
|
||||
}
|
||||
public AuditEventBuilder setClientIp(String clientIp) {
|
||||
auditEvent.clientIp = clientIp;
|
||||
return this;
|
||||
}
|
||||
|
||||
public static AuditEvent createQueryEvent(ConnectContext ctx, String query) {
|
||||
return new AuditEvent(ctx, query);
|
||||
}
|
||||
public AuditEventBuilder setUser(String user) {
|
||||
auditEvent.user = user;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setDb(String db) {
|
||||
auditEvent.db = db;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setState(String state) {
|
||||
auditEvent.state = state;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setQueryTime(long queryTime) {
|
||||
auditEvent.queryTime = queryTime;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setScanBytes(long scanBytes) {
|
||||
auditEvent.scanBytes = scanBytes;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setScanRows(long scanRows) {
|
||||
auditEvent.scanRows = scanRows;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setReturnRows(long returnRows) {
|
||||
auditEvent.returnRows = returnRows;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setStmtId(long stmtId) {
|
||||
auditEvent.stmtId = stmtId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setQueryId(String queryId) {
|
||||
auditEvent.queryId = queryId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setIsQuery(boolean isQuery) {
|
||||
auditEvent.isQuery = isQuery;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setFeIp(String feIp) {
|
||||
auditEvent.feIp = feIp;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEventBuilder setStmt(String stmt) {
|
||||
auditEvent.stmt = stmt;
|
||||
return this;
|
||||
}
|
||||
|
||||
public AuditEvent build() {
|
||||
return this.auditEvent;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -22,16 +22,14 @@ package org.apache.doris.plugin;
|
||||
*/
|
||||
public interface AuditPlugin {
|
||||
/**
|
||||
*
|
||||
* use for check audit event type and masks, the event will skip the plugin if return false
|
||||
* use for check audit event type, the event will skip the plugin if return false
|
||||
*/
|
||||
public boolean eventFilter(short type, short masks);
|
||||
public boolean eventFilter(AuditEvent.EventType type);
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
* process the event.
|
||||
* This method should be implemented as a non-blocking or lightweight method.
|
||||
* Because it will be called after each query. So it must be efficient.
|
||||
*/
|
||||
public void exec(AuditEvent event);
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -17,10 +17,10 @@
|
||||
|
||||
package org.apache.doris.plugin;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.doris.common.UserException;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class BuiltinPluginLoader extends PluginLoader {
|
||||
|
||||
BuiltinPluginLoader(String path, PluginInfo info, Plugin plugin) {
|
||||
@ -39,7 +39,6 @@ public class BuiltinPluginLoader extends PluginLoader {
|
||||
public void uninstall() throws IOException, UserException {
|
||||
if (plugin != null) {
|
||||
pluginUninstallValid();
|
||||
|
||||
plugin.close();
|
||||
}
|
||||
}
|
||||
|
||||
@ -17,6 +17,12 @@
|
||||
|
||||
package org.apache.doris.plugin;
|
||||
|
||||
import org.apache.doris.common.UserException;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.net.URL;
|
||||
@ -30,20 +36,20 @@ import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
public class DynamicPluginLoader extends PluginLoader {
|
||||
private final static Logger LOG = LogManager.getLogger(DynamicPluginLoader.class);
|
||||
|
||||
// the final dir which contains all plugin files.
|
||||
// eg:
|
||||
// Config.plugin_dir/plugin_name/
|
||||
protected Path installPath;
|
||||
|
||||
DynamicPluginLoader(String pluginPath, String source) {
|
||||
super(pluginPath, source);
|
||||
// for processing install stmt
|
||||
DynamicPluginLoader(String pluginDir, String source) {
|
||||
super(pluginDir, source);
|
||||
}
|
||||
|
||||
// for test and replay
|
||||
DynamicPluginLoader(String pluginPath, PluginInfo info) {
|
||||
super(pluginPath, info);
|
||||
this.installPath = FileSystems.getDefault().getPath(pluginDir.toString(), pluginInfo.getName());
|
||||
@ -67,9 +73,10 @@ public class DynamicPluginLoader extends PluginLoader {
|
||||
if (installPath == null) {
|
||||
// download plugin and extract
|
||||
PluginZip zip = new PluginZip(source);
|
||||
Path target = Files.createTempDirectory(pluginDir, ".install_");
|
||||
|
||||
installPath = zip.extract(target);
|
||||
// generation a tmp dir to extract the zip
|
||||
Path tmpTarget = Files.createTempDirectory(pluginDir, ".install_");
|
||||
// for now, installPath point to the temp dir which contains all extracted files from zip file.
|
||||
installPath = zip.extract(tmpTarget);
|
||||
}
|
||||
|
||||
pluginInfo = PluginInfo.readFromProperties(installPath, source);
|
||||
@ -87,12 +94,14 @@ public class DynamicPluginLoader extends PluginLoader {
|
||||
|
||||
getPluginInfo();
|
||||
|
||||
Path realPath = movePlugin();
|
||||
movePlugin();
|
||||
|
||||
plugin = dynamicLoadPlugin(realPath);
|
||||
plugin = dynamicLoadPlugin();
|
||||
|
||||
pluginInstallValid();
|
||||
|
||||
pluginContext.setPluginPath(installPath.toString());
|
||||
|
||||
plugin.init(pluginInfo, pluginContext);
|
||||
}
|
||||
|
||||
@ -104,7 +113,6 @@ public class DynamicPluginLoader extends PluginLoader {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -125,24 +133,25 @@ public class DynamicPluginLoader extends PluginLoader {
|
||||
|
||||
/**
|
||||
* reload plugin if plugin has already been installed, else will re-install
|
||||
*
|
||||
* @throws PluginException
|
||||
*/
|
||||
public void reload() throws IOException, UserException {
|
||||
if (hasInstalled()) {
|
||||
plugin = dynamicLoadPlugin(installPath);
|
||||
plugin = dynamicLoadPlugin();
|
||||
pluginInstallValid();
|
||||
pluginContext.setPluginPath(installPath.toString());
|
||||
plugin.init(pluginInfo, pluginContext);
|
||||
|
||||
} else {
|
||||
// re-install
|
||||
this.pluginInfo = null;
|
||||
this.installPath = null;
|
||||
|
||||
install();
|
||||
}
|
||||
}
|
||||
|
||||
Plugin dynamicLoadPlugin(Path path) throws IOException, UserException {
|
||||
Set<URL> jarList = getJarUrl(path);
|
||||
Plugin dynamicLoadPlugin() throws IOException, UserException {
|
||||
Set<URL> jarList = getJarUrl(installPath);
|
||||
|
||||
// create a child to load the plugin in this bundle
|
||||
ClassLoader parentLoader = PluginClassLoader.createLoader(getClass().getClassLoader(), Collections.EMPTY_LIST);
|
||||
@ -198,19 +207,18 @@ public class DynamicPluginLoader extends PluginLoader {
|
||||
}
|
||||
|
||||
/**
|
||||
* move plugin's temp install directory to Doris's PLUGIN_DIR
|
||||
* move plugin's temp install directory to Doris's PLUGIN_DIR/plugin_name
|
||||
*/
|
||||
Path movePlugin() throws UserException, IOException {
|
||||
|
||||
public void movePlugin() throws UserException, IOException {
|
||||
if (installPath == null || !Files.exists(installPath)) {
|
||||
throw new UserException("Install plugin " + pluginInfo.getName() + " failed, because install path isn't "
|
||||
throw new PluginException("Install plugin " + pluginInfo.getName() + " failed, because install path isn't "
|
||||
+ "exists.");
|
||||
}
|
||||
|
||||
Path targetPath = FileSystems.getDefault().getPath(pluginDir.toString(), pluginInfo.getName());
|
||||
if (Files.exists(targetPath)) {
|
||||
if (!Files.isSameFile(installPath, targetPath)) {
|
||||
throw new UserException(
|
||||
throw new PluginException(
|
||||
"Install plugin " + pluginInfo.getName() + " failed. because " + installPath.toString()
|
||||
+ " exists");
|
||||
}
|
||||
@ -220,8 +228,6 @@ public class DynamicPluginLoader extends PluginLoader {
|
||||
|
||||
// move success
|
||||
installPath = targetPath;
|
||||
|
||||
return installPath;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -33,15 +33,20 @@ public abstract class Plugin implements Closeable {
|
||||
public static final int PLUGIN_NOT_DYNAMIC_UNINSTALL = 1 << 1;
|
||||
|
||||
/**
|
||||
* invoke when the plugin install
|
||||
* invoke when the plugin install.
|
||||
* the plugin should only be initialized once.
|
||||
* So this method must be idempotent
|
||||
*/
|
||||
public void init(PluginInfo info, PluginContext ctx) { }
|
||||
public void init(PluginInfo info, PluginContext ctx) throws PluginException {
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* invoke when the plugin uninstall
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException { }
|
||||
public void close() throws IOException {
|
||||
}
|
||||
|
||||
public int flags() {
|
||||
return PLUGIN_DEFAULT_FLAGS;
|
||||
|
||||
@ -18,4 +18,15 @@
|
||||
package org.apache.doris.plugin;
|
||||
|
||||
public class PluginContext {
|
||||
|
||||
// the dir path where the plugin's files saved
|
||||
private String pluginPath;
|
||||
|
||||
protected void setPluginPath(String pluginPath) {
|
||||
this.pluginPath = pluginPath;
|
||||
}
|
||||
|
||||
public String getPluginPath() {
|
||||
return pluginPath;
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,27 +15,16 @@
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.qe;
|
||||
package org.apache.doris.plugin;
|
||||
|
||||
// Helper class used to build audit log.
|
||||
// Now, implemented with StringBuilder
|
||||
public class AuditBuilder {
|
||||
private StringBuilder sb;
|
||||
import org.apache.doris.common.UserException;
|
||||
|
||||
public AuditBuilder() {
|
||||
sb = new StringBuilder();
|
||||
public class PluginException extends UserException {
|
||||
public PluginException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
sb = new StringBuilder();
|
||||
}
|
||||
|
||||
public void put(String key, Object value) {
|
||||
sb.append("|").append(key).append("=").append(value.toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return sb.toString();
|
||||
public PluginException(String msg, Throwable e) {
|
||||
super(msg, e);
|
||||
}
|
||||
}
|
||||
@ -17,13 +17,13 @@
|
||||
|
||||
package org.apache.doris.plugin;
|
||||
|
||||
import org.apache.doris.common.UserException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.doris.common.UserException;
|
||||
|
||||
public abstract class PluginLoader {
|
||||
|
||||
public enum PluginStatus {
|
||||
@ -34,8 +34,9 @@ public abstract class PluginLoader {
|
||||
ERROR,
|
||||
}
|
||||
|
||||
// the root dir of Frontend plugin, should always be Config.plugin_dir
|
||||
protected final Path pluginDir;
|
||||
|
||||
// source of plugin, eg, a remote download link of zip file, or a local zip file.
|
||||
protected String source;
|
||||
|
||||
protected Plugin plugin;
|
||||
@ -92,7 +93,7 @@ public abstract class PluginLoader {
|
||||
|
||||
public void pluginUninstallValid() throws UserException {
|
||||
// check plugin flags
|
||||
if ((plugin.flags() & Plugin.PLUGIN_NOT_DYNAMIC_UNINSTALL) > 0) {
|
||||
if (plugin != null && (plugin.flags() & Plugin.PLUGIN_NOT_DYNAMIC_UNINSTALL) > 0) {
|
||||
throw new UserException("plugin " + pluginInfo + " not allow dynamic uninstall");
|
||||
}
|
||||
}
|
||||
|
||||
@ -18,11 +18,13 @@
|
||||
package org.apache.doris.plugin;
|
||||
|
||||
import org.apache.doris.analysis.InstallPluginStmt;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.doris.common.io.Writable;
|
||||
import org.apache.doris.plugin.PluginInfo.PluginType;
|
||||
import org.apache.doris.plugin.PluginLoader.PluginStatus;
|
||||
import org.apache.doris.qe.AuditLogBuilder;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.Lists;
|
||||
@ -47,26 +49,35 @@ public class PluginMgr implements Writable {
|
||||
|
||||
public PluginMgr() {
|
||||
plugins = new Map[PluginType.MAX_PLUGIN_SIZE];
|
||||
|
||||
for (int i = 0; i < PluginType.MAX_PLUGIN_SIZE; i++) {
|
||||
plugins[i] = Maps.newConcurrentMap();
|
||||
}
|
||||
}
|
||||
|
||||
// create the plugin dir if missing
|
||||
public void init() {
|
||||
public void init() throws PluginException {
|
||||
File file = new File(Config.plugin_dir);
|
||||
if (file.exists() && !file.isDirectory()) {
|
||||
LOG.error("FE plugin dir {} is not a directory", Config.plugin_dir);
|
||||
System.exit(-1);
|
||||
throw new PluginException("FE plugin dir " + Config.plugin_dir + " is not a directory");
|
||||
}
|
||||
|
||||
if (!file.exists()) {
|
||||
if (!file.mkdir()) {
|
||||
LOG.error("failed to create FE plugin dir {}", Config.plugin_dir);
|
||||
System.exit(-1);
|
||||
throw new PluginException("failed to create FE plugin dir " + Config.plugin_dir);
|
||||
}
|
||||
}
|
||||
|
||||
registerBuiltinPlugins();
|
||||
}
|
||||
|
||||
private void registerBuiltinPlugins() {
|
||||
// AuditLog
|
||||
AuditLogBuilder auditLogBuilder = new AuditLogBuilder();
|
||||
if (!registerPlugin(auditLogBuilder.getPluginInfo(), auditLogBuilder)) {
|
||||
LOG.warn("failed to register audit log builder");
|
||||
}
|
||||
|
||||
// other builtin plugins
|
||||
}
|
||||
|
||||
public PluginInfo installPlugin(InstallPluginStmt stmt) throws IOException, UserException {
|
||||
@ -76,14 +87,21 @@ public class PluginMgr implements Writable {
|
||||
try {
|
||||
PluginInfo info = pluginLoader.getPluginInfo();
|
||||
|
||||
if (plugins[info.getTypeId()].containsKey(info.getName())) {
|
||||
throw new UserException("plugin " + info.getName() + " has already been installed.");
|
||||
}
|
||||
|
||||
// install plugin
|
||||
pluginLoader.install();
|
||||
pluginLoader.setStatus(PluginStatus.INSTALLED);
|
||||
|
||||
if (plugins[info.getTypeId()].putIfAbsent(info.getName(), pluginLoader) != null) {
|
||||
pluginLoader.uninstall();
|
||||
throw new UserException("plugin " + info.getName() + " has already been installed.");
|
||||
}
|
||||
|
||||
// install plugin
|
||||
pluginLoader.install();
|
||||
pluginLoader.setStatus(PluginStatus.INSTALLED);
|
||||
Catalog.getCurrentCatalog().getEditLog().logInstallPlugin(info);
|
||||
LOG.info("install plugin = " + info.getName());
|
||||
return info;
|
||||
} catch (IOException | UserException e) {
|
||||
pluginLoader.setStatus(PluginStatus.ERROR);
|
||||
@ -120,14 +138,12 @@ public class PluginMgr implements Writable {
|
||||
*/
|
||||
public boolean registerPlugin(PluginInfo pluginInfo, Plugin plugin) {
|
||||
if (Objects.isNull(pluginInfo) || Objects.isNull(plugin) || Objects.isNull(pluginInfo.getType())
|
||||
|| Strings
|
||||
.isNullOrEmpty(pluginInfo.getName())) {
|
||||
|| Strings.isNullOrEmpty(pluginInfo.getName())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
PluginLoader loader = new BuiltinPluginLoader(Config.plugin_dir, pluginInfo, plugin);
|
||||
PluginLoader checkLoader =
|
||||
plugins[pluginInfo.getTypeId()].putIfAbsent(pluginInfo.getName(), loader);
|
||||
PluginLoader checkLoader = plugins[pluginInfo.getTypeId()].putIfAbsent(pluginInfo.getName(), loader);
|
||||
|
||||
return checkLoader == null;
|
||||
}
|
||||
@ -170,7 +186,6 @@ public class PluginMgr implements Writable {
|
||||
|
||||
public final List<Plugin> getActivePluginList(PluginType type) {
|
||||
Map<String, PluginLoader> m = plugins[type.ordinal()];
|
||||
|
||||
List<Plugin> l = Lists.newArrayListWithCapacity(m.size());
|
||||
|
||||
m.values().forEach(d -> {
|
||||
|
||||
@ -17,6 +17,18 @@
|
||||
|
||||
package org.apache.doris.plugin;
|
||||
|
||||
import org.apache.doris.common.UserException;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
@ -31,17 +43,6 @@ import java.util.List;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
/**
|
||||
* Describe plugin install file(.zip)
|
||||
* Support remote(http/https) source and local source
|
||||
@ -61,6 +62,10 @@ class PluginZip {
|
||||
cleanPathList = Lists.newLinkedList();
|
||||
}
|
||||
|
||||
/*
|
||||
* download and extract the zip file to the target path.
|
||||
* return the path dir which contains all extracted files.
|
||||
*/
|
||||
public Path extract(Path targetPath) throws IOException, UserException {
|
||||
try {
|
||||
Path zipPath = downloadZip(targetPath);
|
||||
@ -75,11 +80,12 @@ class PluginZip {
|
||||
|
||||
/**
|
||||
* download zip if the source in remote,
|
||||
* or return if the source in local
|
||||
* return the zip file path.
|
||||
* This zip file is currently in a temp directory, such ash
|
||||
**/
|
||||
Path downloadZip(Path targetPath) throws IOException, UserException {
|
||||
if (Strings.isNullOrEmpty(source)) {
|
||||
throw new IllegalArgumentException("Plugin library path: " + source);
|
||||
throw new PluginException("empty plugin source path: " + source);
|
||||
}
|
||||
|
||||
boolean isLocal = true;
|
||||
@ -135,11 +141,11 @@ class PluginZip {
|
||||
}
|
||||
|
||||
/**
|
||||
* unzip .zip file
|
||||
* unzip the specified .zip file "zip" to the target path
|
||||
*/
|
||||
Path extractZip(Path zip, Path targetPath) throws IOException, UserException {
|
||||
if (!Files.exists(zip)) {
|
||||
throw new UserException("Download plugin zip failed, source: " + source);
|
||||
throw new PluginException("Download plugin zip failed. zip file does not exist. source: " + source);
|
||||
}
|
||||
|
||||
if (Files.isDirectory(zip)) {
|
||||
|
||||
117
fe/src/main/java/org/apache/doris/qe/AuditEventProcessor.java
Normal file
117
fe/src/main/java/org/apache/doris/qe/AuditEventProcessor.java
Normal file
@ -0,0 +1,117 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.qe;
|
||||
|
||||
import org.apache.doris.plugin.AuditEvent;
|
||||
import org.apache.doris.plugin.AuditPlugin;
|
||||
import org.apache.doris.plugin.Plugin;
|
||||
import org.apache.doris.plugin.PluginInfo.PluginType;
|
||||
import org.apache.doris.plugin.PluginMgr;
|
||||
|
||||
import com.google.common.collect.Queues;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/*
|
||||
* Class for processing all audit events.
|
||||
* It will receive audit events and handle them to all AUDIT type plugins.
|
||||
*/
|
||||
public class AuditEventProcessor {
|
||||
private static final Logger LOG = LogManager.getLogger(AuditEventProcessor.class);
|
||||
private static final long UPDATE_PLUGIN_INTERVAL_MS = 60 * 1000; // 1min
|
||||
|
||||
private PluginMgr pluginMgr;
|
||||
|
||||
private List<Plugin> auditPlugins;
|
||||
private long lastUpdateTime = 0;
|
||||
|
||||
private BlockingQueue<AuditEvent> eventQueue = Queues.newLinkedBlockingDeque(10000);
|
||||
private Thread workerThread;
|
||||
|
||||
private volatile boolean isStopped = false;
|
||||
|
||||
public AuditEventProcessor(PluginMgr pluginMgr) {
|
||||
this.pluginMgr = pluginMgr;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
workerThread = new Thread(new Worker(), "AuditEventProcessor");
|
||||
workerThread.start();
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
isStopped = true;
|
||||
if (workerThread != null) {
|
||||
try {
|
||||
workerThread.join();
|
||||
} catch (InterruptedException e) {
|
||||
LOG.warn("join worker join failed.", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void handleAuditEvent(AuditEvent auditEvent) {
|
||||
try {
|
||||
eventQueue.put(auditEvent);
|
||||
} catch (InterruptedException e) {
|
||||
LOG.debug("encounter exception when handle audit event, ignore", e);
|
||||
}
|
||||
}
|
||||
|
||||
public class Worker implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
AuditEvent auditEvent = null;
|
||||
while (!isStopped) {
|
||||
// update audit plugin list every UPDATE_PLUGIN_INTERVAL_MS.
|
||||
// because some of plugins may be installed or uninstalled at runtime.
|
||||
if (auditPlugins == null || System.currentTimeMillis() - lastUpdateTime > UPDATE_PLUGIN_INTERVAL_MS) {
|
||||
auditPlugins = pluginMgr.getActivePluginList(PluginType.AUDIT);
|
||||
lastUpdateTime = System.currentTimeMillis();
|
||||
LOG.debug("update audit plugins. num: {}", auditPlugins.size());
|
||||
}
|
||||
|
||||
try {
|
||||
auditEvent = eventQueue.poll(5, TimeUnit.SECONDS);
|
||||
if (auditEvent == null) {
|
||||
continue;
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
LOG.debug("encounter exception when getting audit event from queue, ignore", e);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
for (Plugin plugin : auditPlugins) {
|
||||
if (((AuditPlugin) plugin).eventFilter(auditEvent.type)) {
|
||||
((AuditPlugin) plugin).exec(auditEvent);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.debug("encounter exception when processing audit event.", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
92
fe/src/main/java/org/apache/doris/qe/AuditLogBuilder.java
Normal file
92
fe/src/main/java/org/apache/doris/qe/AuditLogBuilder.java
Normal file
@ -0,0 +1,92 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.qe;
|
||||
|
||||
import org.apache.doris.common.AuditLog;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.util.DigitalVersion;
|
||||
import org.apache.doris.plugin.AuditEvent;
|
||||
import org.apache.doris.plugin.AuditEvent.AuditField;
|
||||
import org.apache.doris.plugin.AuditEvent.EventType;
|
||||
import org.apache.doris.plugin.AuditPlugin;
|
||||
import org.apache.doris.plugin.Plugin;
|
||||
import org.apache.doris.plugin.PluginInfo;
|
||||
import org.apache.doris.plugin.PluginInfo.PluginType;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
|
||||
// A builtin Audit plugin, registered when FE start.
|
||||
// it will receive "AFTER_QUERY" AuditEventy and print it as a log in fe.audit.log
|
||||
public class AuditLogBuilder extends Plugin implements AuditPlugin {
|
||||
private static final Logger LOG = LogManager.getLogger(AuditLogBuilder.class);
|
||||
|
||||
private PluginInfo pluginInfo;
|
||||
|
||||
public AuditLogBuilder() {
|
||||
pluginInfo = new PluginInfo("AuditLogBuilder", PluginType.AUDIT,
|
||||
"builtin audit logger", DigitalVersion.fromString("0.12.0"),
|
||||
DigitalVersion.fromString("1.8.31"), AuditLogBuilder.class.getName(), null, null);
|
||||
}
|
||||
|
||||
public PluginInfo getPluginInfo() {
|
||||
return pluginInfo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean eventFilter(EventType type) {
|
||||
return type == EventType.AFTER_QUERY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exec(AuditEvent event) {
|
||||
try {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
long queryTime = 0;
|
||||
// get each field with annotation "AuditField" in AuditEvent
|
||||
// and assemble them into a string.
|
||||
Field[] fields = event.getClass().getFields();
|
||||
for (Field f : fields) {
|
||||
AuditField af = f.getAnnotation(AuditField.class);
|
||||
if (af == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (af.value().equals("Timestamp")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (af.value().equals("Time")) {
|
||||
queryTime = (long) f.get(event);
|
||||
}
|
||||
sb.append("|").append(af.value()).append("=").append(String.valueOf(f.get(event)));
|
||||
}
|
||||
|
||||
String auditLog = sb.toString();
|
||||
AuditLog.getQueryAudit().log(auditLog);
|
||||
// slow query
|
||||
if (queryTime > Config.qe_slow_log_ms) {
|
||||
AuditLog.getSlowAudit().log(auditLog);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.debug("failed to process audit event", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -24,6 +24,7 @@ import org.apache.doris.mysql.MysqlCapability;
|
||||
import org.apache.doris.mysql.MysqlChannel;
|
||||
import org.apache.doris.mysql.MysqlCommand;
|
||||
import org.apache.doris.mysql.MysqlSerializer;
|
||||
import org.apache.doris.plugin.AuditEvent.AuditEventBuilder;
|
||||
import org.apache.doris.thrift.TResourceInfo;
|
||||
import org.apache.doris.thrift.TUniqueId;
|
||||
|
||||
@ -92,7 +93,7 @@ public class ConnectContext {
|
||||
protected Catalog catalog;
|
||||
protected boolean isSend;
|
||||
|
||||
protected AuditBuilder auditBuilder;
|
||||
protected AuditEventBuilder auditEventBuilder = new AuditEventBuilder();;
|
||||
|
||||
protected String remoteIP;
|
||||
|
||||
@ -119,7 +120,6 @@ public class ConnectContext {
|
||||
isKilled = false;
|
||||
serializer = MysqlSerializer.newInstance();
|
||||
sessionVariable = VariableMgr.newSessionVariable();
|
||||
auditBuilder = new AuditBuilder();
|
||||
command = MysqlCommand.COM_SLEEP;
|
||||
}
|
||||
|
||||
@ -131,7 +131,6 @@ public class ConnectContext {
|
||||
mysqlChannel = new MysqlChannel(channel);
|
||||
serializer = MysqlSerializer.newInstance();
|
||||
sessionVariable = VariableMgr.newSessionVariable();
|
||||
auditBuilder = new AuditBuilder();
|
||||
command = MysqlCommand.COM_SLEEP;
|
||||
if (channel != null) {
|
||||
remoteIP = mysqlChannel.getRemoteIp();
|
||||
@ -162,8 +161,8 @@ public class ConnectContext {
|
||||
this.remoteIP = remoteIP;
|
||||
}
|
||||
|
||||
public AuditBuilder getAuditBuilder() {
|
||||
return auditBuilder;
|
||||
public AuditEventBuilder getAuditEventBuilder() {
|
||||
return auditEventBuilder;
|
||||
}
|
||||
|
||||
public void setThreadLocalInfo() {
|
||||
|
||||
@ -28,8 +28,6 @@ import org.apache.doris.catalog.Database;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.AuditLog;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.ErrorCode;
|
||||
import org.apache.doris.common.ErrorReport;
|
||||
@ -43,7 +41,9 @@ import org.apache.doris.mysql.MysqlPacket;
|
||||
import org.apache.doris.mysql.MysqlProto;
|
||||
import org.apache.doris.mysql.MysqlSerializer;
|
||||
import org.apache.doris.mysql.MysqlServerStatusFlag;
|
||||
import org.apache.doris.plugin.AuditEvent.EventType;
|
||||
import org.apache.doris.proto.PQueryStatistics;
|
||||
import org.apache.doris.service.FrontendOptions;
|
||||
import org.apache.doris.thrift.TMasterOpRequest;
|
||||
import org.apache.doris.thrift.TMasterOpResult;
|
||||
|
||||
@ -106,14 +106,14 @@ public class ConnectProcessor {
|
||||
private void auditAfterExec(String origStmt, StatementBase parsedStmt, PQueryStatistics statistics) {
|
||||
// slow query
|
||||
long elapseMs = System.currentTimeMillis() - ctx.getStartTime();
|
||||
// query state log
|
||||
ctx.getAuditBuilder().put("State", ctx.getState());
|
||||
ctx.getAuditBuilder().put("Time", elapseMs);
|
||||
ctx.getAuditBuilder().put("ScanBytes", statistics == null ? 0 : statistics.scan_bytes);
|
||||
ctx.getAuditBuilder().put("ScanRows", statistics == null ? 0 : statistics.scan_rows);
|
||||
ctx.getAuditBuilder().put("ReturnRows", ctx.getReturnRows());
|
||||
ctx.getAuditBuilder().put("StmtId", ctx.getStmtId());
|
||||
ctx.getAuditBuilder().put("QueryId", ctx.queryId() == null ? "NaN" : DebugUtil.printId(ctx.queryId()));
|
||||
|
||||
ctx.getAuditEventBuilder().setEventType(EventType.AFTER_QUERY)
|
||||
.setState(ctx.getState().toString()).setQueryTime(elapseMs)
|
||||
.setScanBytes(statistics == null ? 0 : statistics.scan_bytes)
|
||||
.setScanRows(statistics == null ? 0 : statistics.scan_rows)
|
||||
.setReturnRows(ctx.getReturnRows())
|
||||
.setStmtId(ctx.getStmtId())
|
||||
.setQueryId(ctx.queryId() == null ? "NaN" : DebugUtil.printId(ctx.queryId()));
|
||||
|
||||
if (ctx.getState().isQuery()) {
|
||||
MetricRepo.COUNTER_QUERY_ALL.increase(1L);
|
||||
@ -125,23 +125,21 @@ public class ConnectProcessor {
|
||||
// ok query
|
||||
MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs);
|
||||
}
|
||||
ctx.getAuditBuilder().put("IsQuery", 1);
|
||||
ctx.getAuditEventBuilder().setIsQuery(true);
|
||||
} else {
|
||||
ctx.getAuditBuilder().put("IsQuery", 0);
|
||||
ctx.getAuditEventBuilder().setIsQuery(false);
|
||||
}
|
||||
|
||||
ctx.getAuditEventBuilder().setFeIp(FrontendOptions.getLocalHostAddress());
|
||||
|
||||
// We put origin query stmt at the end of audit log, for parsing the log more convenient.
|
||||
if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) {
|
||||
ctx.getAuditBuilder().put("Stmt", parsedStmt.toSql());
|
||||
ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql());
|
||||
} else {
|
||||
ctx.getAuditBuilder().put("Stmt", origStmt);
|
||||
}
|
||||
|
||||
AuditLog.getQueryAudit().log(ctx.getAuditBuilder().toString());
|
||||
|
||||
// slow query
|
||||
if (elapseMs > Config.qe_slow_log_ms) {
|
||||
AuditLog.getSlowAudit().log(ctx.getAuditBuilder().toString());
|
||||
ctx.getAuditEventBuilder().setStmt(origStmt);
|
||||
}
|
||||
|
||||
Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build());
|
||||
}
|
||||
|
||||
// process COM_QUERY statement,
|
||||
@ -163,10 +161,12 @@ public class ConnectProcessor {
|
||||
ctx.getState().setError("Unsupported character set(UTF-8)");
|
||||
return;
|
||||
}
|
||||
ctx.getAuditBuilder().reset();
|
||||
ctx.getAuditBuilder().put("Client", ctx.getMysqlChannel().getRemoteHostPortString());
|
||||
ctx.getAuditBuilder().put("User", ctx.getQualifiedUser());
|
||||
ctx.getAuditBuilder().put("Db", ctx.getDatabase());
|
||||
ctx.getAuditEventBuilder().reset();
|
||||
ctx.getAuditEventBuilder()
|
||||
.setTimestamp(System.currentTimeMillis())
|
||||
.setClientIp(ctx.getMysqlChannel().getRemoteHostPortString())
|
||||
.setUser(ctx.getQualifiedUser())
|
||||
.setDb(ctx.getDatabase());
|
||||
|
||||
// execute this query.
|
||||
StatementBase parsedStmt = null;
|
||||
|
||||
@ -28,7 +28,6 @@ import org.apache.doris.catalog.OlapTable;
|
||||
import org.apache.doris.catalog.Table;
|
||||
import org.apache.doris.cluster.ClusterNamespace;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.AuditLog;
|
||||
import org.apache.doris.common.AuthenticationException;
|
||||
import org.apache.doris.common.CaseSensibility;
|
||||
import org.apache.doris.common.Config;
|
||||
@ -45,7 +44,9 @@ import org.apache.doris.load.MiniEtlTaskInfo;
|
||||
import org.apache.doris.master.MasterImpl;
|
||||
import org.apache.doris.mysql.privilege.PrivPredicate;
|
||||
import org.apache.doris.planner.StreamLoadPlanner;
|
||||
import org.apache.doris.qe.AuditBuilder;
|
||||
import org.apache.doris.plugin.AuditEvent;
|
||||
import org.apache.doris.plugin.AuditEvent.AuditEventBuilder;
|
||||
import org.apache.doris.plugin.AuditEvent.EventType;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.qe.ConnectProcessor;
|
||||
import org.apache.doris.qe.QeProcessorImpl;
|
||||
@ -418,15 +419,15 @@ public class FrontendServiceImpl implements FrontendService.Iface {
|
||||
|
||||
private void logMiniLoadStmt(TMiniLoadRequest request) throws UnknownHostException {
|
||||
String stmt = getMiniLoadStmt(request);
|
||||
AuditBuilder auditBuilder = new AuditBuilder();
|
||||
auditBuilder.put("client", request.user_ip + ":0");
|
||||
auditBuilder.put("user", request.user);
|
||||
auditBuilder.put("db", request.db);
|
||||
auditBuilder.put("state", TStatusCode.OK);
|
||||
auditBuilder.put("time", "0");
|
||||
auditBuilder.put("stmt", stmt);
|
||||
|
||||
AuditLog.getQueryAudit().log(auditBuilder.toString());
|
||||
AuditEvent auditEvent = new AuditEventBuilder().setEventType(EventType.AFTER_QUERY)
|
||||
.setClientIp(request.user_ip + ":0")
|
||||
.setUser(request.user)
|
||||
.setDb(request.db)
|
||||
.setState(TStatusCode.OK.name())
|
||||
.setQueryTime(0)
|
||||
.setStmt(stmt).build();
|
||||
|
||||
Catalog.getCurrentAuditEventProcessor().handleAuditEvent(auditEvent);
|
||||
}
|
||||
|
||||
private String getMiniLoadStmt(TMiniLoadRequest request) throws UnknownHostException {
|
||||
|
||||
@ -55,6 +55,35 @@ public class QueryPlanTest {
|
||||
String createDbStmtStr = "create database test;";
|
||||
CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
|
||||
Catalog.getCurrentCatalog().createDb(createDbStmt);
|
||||
|
||||
createTable("create table test.test1\n" +
|
||||
"(\n" +
|
||||
" query_id varchar(48) comment \"Unique query id\",\n" +
|
||||
" time datetime not null comment \"Query start time\",\n" +
|
||||
" client_ip varchar(32) comment \"Client IP\",\n" +
|
||||
" user varchar(64) comment \"User name\",\n" +
|
||||
" db varchar(96) comment \"Database of this query\",\n" +
|
||||
" state varchar(8) comment \"Query result state. EOF, ERR, OK\",\n" +
|
||||
" query_time bigint comment \"Query execution time in millisecond\",\n" +
|
||||
" scan_bytes bigint comment \"Total scan bytes of this query\",\n" +
|
||||
" scan_rows bigint comment \"Total scan rows of this query\",\n" +
|
||||
" return_rows bigint comment \"Returned rows of this query\",\n" +
|
||||
" stmt_id int comment \"An incremental id of statement\",\n" +
|
||||
" is_query tinyint comment \"Is this statemt a query. 1 or 0\",\n" +
|
||||
" frontend_ip varchar(32) comment \"Frontend ip of executing this statement\",\n" +
|
||||
" stmt varchar(2048) comment \"The original statement, trimed if longer than 2048 bytes\"\n" +
|
||||
")\n" +
|
||||
"partition by range(time) ()\n" +
|
||||
"distributed by hash(query_id) buckets 1\n" +
|
||||
"properties(\n" +
|
||||
" \"dynamic_partition.time_unit\" = \"DAY\",\n" +
|
||||
" \"dynamic_partition.start\" = \"-30\",\n" +
|
||||
" \"dynamic_partition.end\" = \"3\",\n" +
|
||||
" \"dynamic_partition.prefix\" = \"p\",\n" +
|
||||
" \"dynamic_partition.buckets\" = \"1\",\n" +
|
||||
" \"dynamic_partition.enable\" = \"true\",\n" +
|
||||
" \"replication_num\" = \"1\"\n" +
|
||||
");");
|
||||
|
||||
createTable("CREATE TABLE test.bitmap_table (\n" +
|
||||
" `id` int(11) NULL COMMENT \"\",\n" +
|
||||
|
||||
@ -21,17 +21,18 @@ import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.doris.common.util.DigitalVersion;
|
||||
import org.apache.doris.plugin.PluginInfo.PluginType;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.Map;
|
||||
|
||||
public class PluginLoaderTest {
|
||||
|
||||
@Before
|
||||
@ -71,7 +72,7 @@ public class PluginLoaderTest {
|
||||
DigitalVersion.JDK_1_8_0, "plugin.PluginTest", "libtest.so", "plugin_test.jar");
|
||||
|
||||
DynamicPluginLoader util = new DynamicPluginLoader(PluginTestUtil.getTestPathString(""), info);
|
||||
Plugin p = util.dynamicLoadPlugin(PluginTestUtil.getTestPath(""));
|
||||
Plugin p = util.dynamicLoadPlugin();
|
||||
|
||||
p.init(null, null);
|
||||
p.close();
|
||||
|
||||
@ -20,71 +20,51 @@ package org.apache.doris.plugin;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import org.apache.doris.analysis.InstallPluginStmt;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.doris.common.io.DataOutputBuffer;
|
||||
import org.apache.doris.utframe.UtFrameUtils;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.doris.analysis.InstallPluginStmt;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.catalog.FakeCatalog;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.FeConstants;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.doris.common.io.DataOutputBuffer;
|
||||
import org.apache.doris.common.jmockit.Deencapsulation;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import java.util.UUID;
|
||||
|
||||
public class PluginMgrTest {
|
||||
|
||||
class TestPlugin extends Plugin implements AuditPlugin {
|
||||
private static String runningDir = "fe/mocked/PluginMgrTest/" + UUID.randomUUID().toString() + "/";
|
||||
|
||||
@Override
|
||||
public boolean eventFilter(short type, short masks) {
|
||||
return type == AuditEvent.AUDIT_CONNECTION;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exec(AuditEvent event) {
|
||||
}
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
UtFrameUtils.createMinDorisCluster(runningDir);
|
||||
}
|
||||
|
||||
|
||||
private Catalog catalog;
|
||||
|
||||
private FakeCatalog fakeCatalog;
|
||||
|
||||
private PluginMgr mgr;
|
||||
@AfterClass
|
||||
public static void tearDown() {
|
||||
File file = new File(runningDir);
|
||||
file.delete();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
try {
|
||||
fakeCatalog = new FakeCatalog();
|
||||
catalog = Deencapsulation.newInstance(Catalog.class);
|
||||
|
||||
FakeCatalog.setCatalog(catalog);
|
||||
FakeCatalog.setMetaVersion(FeConstants.meta_version);
|
||||
|
||||
FileUtils.deleteQuietly(PluginTestUtil.getTestFile("target"));
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target")));
|
||||
Files.createDirectory(PluginTestUtil.getTestPath("target"));
|
||||
assertTrue(Files.exists(PluginTestUtil.getTestPath("target")));
|
||||
|
||||
Config.plugin_dir = PluginTestUtil.getTestPathString("target");
|
||||
|
||||
mgr = new PluginMgr();
|
||||
PluginInfo info = new PluginInfo("TestPlugin", PluginInfo.PluginType.AUDIT, "use for test");
|
||||
assertTrue(mgr.registerPlugin(info, new TestPlugin()));
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
public void setUp() throws IOException {
|
||||
FileUtils.deleteQuietly(PluginTestUtil.getTestFile("target"));
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target")));
|
||||
Files.createDirectory(PluginTestUtil.getTestPath("target"));
|
||||
assertTrue(Files.exists(PluginTestUtil.getTestPath("target")));
|
||||
Config.plugin_dir = PluginTestUtil.getTestPathString("target");
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -94,30 +74,31 @@ public class PluginMgrTest {
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar")));
|
||||
|
||||
InstallPluginStmt stmt = new InstallPluginStmt(PluginTestUtil.getTestPathString("auditdemo.zip"));
|
||||
mgr.installPlugin(stmt);
|
||||
Catalog.getCurrentCatalog().installPlugin(stmt);
|
||||
|
||||
assertEquals(2, mgr.getActivePluginList(PluginInfo.PluginType.AUDIT).size());
|
||||
PluginMgr pluginMgr = Catalog.getCurrentPluginMgr();
|
||||
|
||||
Plugin p = mgr.getActivePlugin("audit_plugin_demo", PluginInfo.PluginType.AUDIT);
|
||||
assertEquals(2, pluginMgr.getActivePluginList(PluginInfo.PluginType.AUDIT).size());
|
||||
|
||||
Plugin p = pluginMgr.getActivePlugin("audit_plugin_demo", PluginInfo.PluginType.AUDIT);
|
||||
|
||||
assertNotNull(p);
|
||||
assertTrue(p instanceof AuditPlugin);
|
||||
assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_QUERY, AuditEvent.AUDIT_QUERY_START));
|
||||
assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_QUERY, AuditEvent.AUDIT_QUERY_END));
|
||||
assertFalse(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_CONNECTION, AuditEvent.AUDIT_QUERY_END));
|
||||
assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.EventType.AFTER_QUERY));
|
||||
assertFalse(((AuditPlugin) p).eventFilter(AuditEvent.EventType.BEFORE_QUERY));
|
||||
|
||||
assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo")));
|
||||
assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar")));
|
||||
|
||||
assertEquals(1, mgr.getAllDynamicPluginInfo().size());
|
||||
PluginInfo info = mgr.getAllDynamicPluginInfo().get(0);
|
||||
assertEquals(1, pluginMgr.getAllDynamicPluginInfo().size());
|
||||
PluginInfo info = pluginMgr.getAllDynamicPluginInfo().get(0);
|
||||
|
||||
assertEquals("audit_plugin_demo", info.getName());
|
||||
assertEquals(PluginInfo.PluginType.AUDIT, info.getType());
|
||||
assertEquals("just for test", info.getDescription());
|
||||
assertEquals("plugin.AuditPluginDemo", info.getClassName());
|
||||
|
||||
mgr.uninstallPlugin("audit_plugin_demo");
|
||||
pluginMgr.uninstallPlugin("audit_plugin_demo");
|
||||
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo")));
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar")));
|
||||
@ -139,25 +120,27 @@ public class PluginMgrTest {
|
||||
PluginTestUtil.getTestPath("test_plugin").toFile());
|
||||
|
||||
InstallPluginStmt stmt = new InstallPluginStmt(PluginTestUtil.getTestPathString("test_plugin"));
|
||||
mgr.installPlugin(stmt);
|
||||
Catalog.getCurrentCatalog().installPlugin(stmt);
|
||||
|
||||
PluginMgr pluginMgr = Catalog.getCurrentPluginMgr();
|
||||
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("test_plugin")));
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("test_plugin/auditdemo.jar")));
|
||||
|
||||
Plugin p = mgr.getActivePlugin("audit_plugin_demo", PluginInfo.PluginType.AUDIT);
|
||||
Plugin p = pluginMgr.getActivePlugin("audit_plugin_demo", PluginInfo.PluginType.AUDIT);
|
||||
|
||||
assertEquals(2, mgr.getActivePluginList(PluginInfo.PluginType.AUDIT).size());
|
||||
assertEquals(2, pluginMgr.getActivePluginList(PluginInfo.PluginType.AUDIT).size());
|
||||
|
||||
assertNotNull(p);
|
||||
assertTrue(p instanceof AuditPlugin);
|
||||
assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_QUERY, AuditEvent.AUDIT_QUERY_START));
|
||||
assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_QUERY, AuditEvent.AUDIT_QUERY_END));
|
||||
assertFalse(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_CONNECTION, AuditEvent.AUDIT_QUERY_END));
|
||||
assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.EventType.AFTER_QUERY));
|
||||
assertFalse(((AuditPlugin) p).eventFilter(AuditEvent.EventType.BEFORE_QUERY));
|
||||
|
||||
assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo")));
|
||||
assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar")));
|
||||
|
||||
mgr.uninstallPlugin("audit_plugin_demo");
|
||||
testSerializeBuiltinPlugin(pluginMgr);
|
||||
pluginMgr.uninstallPlugin("audit_plugin_demo");
|
||||
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo")));
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar")));
|
||||
@ -168,10 +151,7 @@ public class PluginMgrTest {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSerializeBuiltinPlugin() {
|
||||
|
||||
private void testSerializeBuiltinPlugin(PluginMgr mgr) {
|
||||
try {
|
||||
DataOutputBuffer dob = new DataOutputBuffer();
|
||||
DataOutputStream dos = new DataOutputStream(dob);
|
||||
@ -180,47 +160,10 @@ public class PluginMgrTest {
|
||||
PluginMgr test = new PluginMgr();
|
||||
|
||||
test.readFields(new DataInputStream(new ByteArrayInputStream(dob.getData())));
|
||||
assertEquals(0, test.getAllDynamicPluginInfo().size());
|
||||
assertEquals(1, test.getAllDynamicPluginInfo().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSerializeDynamicPlugin() {
|
||||
try {
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo")));
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar")));
|
||||
|
||||
InstallPluginStmt stmt = new InstallPluginStmt(PluginTestUtil.getTestPathString("auditdemo.zip"));
|
||||
mgr.installPlugin(stmt);
|
||||
|
||||
assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo")));
|
||||
assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar")));
|
||||
|
||||
assertEquals(1, mgr.getAllDynamicPluginInfo().size());
|
||||
|
||||
DataOutputBuffer dob = new DataOutputBuffer();
|
||||
DataOutputStream dos = new DataOutputStream(dob);
|
||||
mgr.write(dos);
|
||||
|
||||
PluginMgr test = new PluginMgr();
|
||||
|
||||
assertNotNull(dob);
|
||||
assertNotNull(test);
|
||||
DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dob.getData()));
|
||||
test.readFields(dis);
|
||||
assertEquals(1, test.getAllDynamicPluginInfo().size());
|
||||
|
||||
mgr.uninstallPlugin("audit_plugin_demo");
|
||||
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo")));
|
||||
assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar")));
|
||||
} catch (IOException | UserException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,124 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.qe;
|
||||
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.util.DigitalVersion;
|
||||
import org.apache.doris.plugin.AuditEvent;
|
||||
import org.apache.doris.plugin.PluginInfo;
|
||||
import org.apache.doris.plugin.AuditEvent.AuditEventBuilder;
|
||||
import org.apache.doris.plugin.AuditEvent.EventType;
|
||||
import org.apache.doris.qe.AuditEventProcessor;
|
||||
import org.apache.doris.qe.AuditLogBuilder;
|
||||
import org.apache.doris.utframe.UtFrameUtils;
|
||||
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.UUID;
|
||||
|
||||
public class AuditEventProcessorTest {
|
||||
|
||||
private static String runningDir = "fe/mocked/AuditProcessorTest/" + UUID.randomUUID().toString() + "/";
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
UtFrameUtils.createMinDorisCluster(runningDir);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() {
|
||||
File file = new File(runningDir);
|
||||
file.delete();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAuditEvent() {
|
||||
AuditEvent event = new AuditEvent.AuditEventBuilder().setEventType(EventType.AFTER_QUERY)
|
||||
.setTimestamp(System.currentTimeMillis())
|
||||
.setClientIp("127.0.0.1")
|
||||
.setUser("user1")
|
||||
.setDb("db1")
|
||||
.setState("EOF")
|
||||
.setQueryTime(2000)
|
||||
.setScanBytes(100000)
|
||||
.setScanRows(200000)
|
||||
.setReturnRows(1)
|
||||
.setStmtId(1234)
|
||||
.setStmt("select * from tbl1").build();
|
||||
|
||||
Assert.assertEquals("127.0.0.1", event.clientIp);
|
||||
Assert.assertEquals(200000, event.scanRows);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAuditLogBuilder() throws IOException {
|
||||
try (AuditLogBuilder auditLogBuilder = new AuditLogBuilder()) {
|
||||
PluginInfo pluginInfo = auditLogBuilder.getPluginInfo();
|
||||
Assert.assertEquals(DigitalVersion.fromString("0.12.0"), pluginInfo.getVersion());
|
||||
Assert.assertEquals(DigitalVersion.fromString("1.8.31"), pluginInfo.getJavaVersion());
|
||||
long start = System.currentTimeMillis();
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
AuditEvent event = new AuditEvent.AuditEventBuilder().setEventType(EventType.AFTER_QUERY)
|
||||
.setTimestamp(System.currentTimeMillis())
|
||||
.setClientIp("127.0.0.1")
|
||||
.setUser("user1")
|
||||
.setDb("db1")
|
||||
.setState("EOF")
|
||||
.setQueryTime(2000)
|
||||
.setScanBytes(100000)
|
||||
.setScanRows(200000)
|
||||
.setReturnRows(i)
|
||||
.setStmtId(1234)
|
||||
.setStmt("select * from tbl1").build();
|
||||
if (auditLogBuilder.eventFilter(event.type)) {
|
||||
auditLogBuilder.exec(event);
|
||||
}
|
||||
}
|
||||
long total = System.currentTimeMillis() - start;
|
||||
System.out.println("total(ms): " + total + ", avg: " + total / 10000.0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAuditEventProcessor() throws IOException {
|
||||
AuditEventProcessor processor = Catalog.getCurrentAuditEventProcessor();
|
||||
long start = System.currentTimeMillis();
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
AuditEvent event = new AuditEvent.AuditEventBuilder().setEventType(EventType.AFTER_QUERY)
|
||||
.setTimestamp(System.currentTimeMillis())
|
||||
.setClientIp("127.0.0.1")
|
||||
.setUser("user1")
|
||||
.setDb("db1")
|
||||
.setState("EOF")
|
||||
.setQueryTime(2000)
|
||||
.setScanBytes(100000)
|
||||
.setScanRows(200000)
|
||||
.setReturnRows(i)
|
||||
.setStmtId(1234)
|
||||
.setStmt("select * from tbl1").build();
|
||||
processor.handleAuditEvent(event);
|
||||
}
|
||||
long total = System.currentTimeMillis() - start;
|
||||
System.out.println("total(ms): " + total + ", avg: " + total / 10000.0);
|
||||
}
|
||||
}
|
||||
@ -17,8 +17,6 @@
|
||||
|
||||
package org.apache.doris.qe;
|
||||
|
||||
import mockit.Expectations;
|
||||
import mockit.Mocked;
|
||||
import org.apache.doris.analysis.AccessTestUtil;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.jmockit.Deencapsulation;
|
||||
@ -29,6 +27,7 @@ import org.apache.doris.mysql.MysqlEofPacket;
|
||||
import org.apache.doris.mysql.MysqlErrPacket;
|
||||
import org.apache.doris.mysql.MysqlOkPacket;
|
||||
import org.apache.doris.mysql.MysqlSerializer;
|
||||
import org.apache.doris.plugin.AuditEvent.AuditEventBuilder;
|
||||
import org.apache.doris.proto.PQueryStatistics;
|
||||
import org.apache.doris.thrift.TUniqueId;
|
||||
|
||||
@ -41,13 +40,16 @@ import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.SocketChannel;
|
||||
|
||||
import mockit.Expectations;
|
||||
import mockit.Mocked;
|
||||
|
||||
public class ConnectProcessorTest {
|
||||
private static ByteBuffer initDbPacket;
|
||||
private static ByteBuffer pingPacket;
|
||||
private static ByteBuffer quitPacket;
|
||||
private static ByteBuffer queryPacket;
|
||||
private static ByteBuffer fieldListPacket;
|
||||
private static AuditBuilder auditBuilder = new AuditBuilder();
|
||||
private static AuditEventBuilder auditBuilder = new AuditEventBuilder();
|
||||
private static ConnectContext myContext;
|
||||
|
||||
@Mocked
|
||||
@ -202,7 +204,7 @@ public class ConnectProcessorTest {
|
||||
minTimes = 0;
|
||||
result = catalog;
|
||||
|
||||
context.getAuditBuilder();
|
||||
context.getAuditEventBuilder();
|
||||
minTimes = 0;
|
||||
result = auditBuilder;
|
||||
|
||||
|
||||
@ -24,6 +24,7 @@ import org.apache.doris.analysis.StatementBase;
|
||||
import org.apache.doris.analysis.UserIdentity;
|
||||
import org.apache.doris.catalog.Catalog;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.Config;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.Pair;
|
||||
import org.apache.doris.common.util.SqlParserUtils;
|
||||
@ -126,6 +127,7 @@ public class UtFrameUtils {
|
||||
if (Strings.isNullOrEmpty(dorisHome)) {
|
||||
dorisHome = Files.createTempDirectory("DORIS_HOME").toAbsolutePath().toString();
|
||||
}
|
||||
Config.plugin_dir = dorisHome + "/plugins";
|
||||
|
||||
int fe_http_port = findValidPort();
|
||||
int fe_rpc_port = findValidPort();
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@ -1,245 +0,0 @@
|
||||
# README
|
||||
[TOC]
|
||||
## Introduction
|
||||
fe_plugins is the parent module of the fe plugins. It can uniformly manage the third-party library information that
|
||||
the plugin depends on. Adding a plugin can add a submodule implementation under fe_plugins
|
||||
|
||||
## Plugin
|
||||
|
||||
A FE Plugin can be a .zip package or a directory, which contains at least two parts: the plugin.properties and .jar files. The plugin.properties file is used to describe the plugin information.
|
||||
|
||||
The file structure of a Plugin looks like this:
|
||||
```
|
||||
# plugin .zip
|
||||
auditodemo.zip:
|
||||
-plugin.properties
|
||||
-auditdemo.jar
|
||||
-xxx.config
|
||||
-data \
|
||||
-test_data
|
||||
|
||||
# plugin local directory
|
||||
auditodemo /:
|
||||
-plugin.properties
|
||||
-auditdemo.jar
|
||||
-xxx.config
|
||||
-data \
|
||||
-test_data
|
||||
```
|
||||
|
||||
plugin.properties example:
|
||||
|
||||
```
|
||||
### required:
|
||||
#
|
||||
# the plugin name
|
||||
name = audit_plugin_demo
|
||||
#
|
||||
# the plugin type
|
||||
type = AUDIT
|
||||
#
|
||||
# simple summary of the plugin
|
||||
description = just for test
|
||||
#
|
||||
# Doris's version, like: 0.11.0
|
||||
version = 0.11.0
|
||||
|
||||
### FE-Plugin optional:
|
||||
#
|
||||
# version of java the code is built against
|
||||
# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4
|
||||
java.version = 1.8.31
|
||||
#
|
||||
# the name of the class to load, fully-qualified.
|
||||
classname = plugin.AuditPluginDemo
|
||||
|
||||
### BE-Plugin optional:
|
||||
# the name of the so to load
|
||||
soName = example.so
|
||||
```
|
||||
|
||||
## Write A Plugin
|
||||
### create module
|
||||
We can add a submodule in the fe_plugins directory to implement Plugin and create a project:
|
||||
```
|
||||
mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false
|
||||
```
|
||||
|
||||
The command produces a new mvn project, and a new submodule is automatically added to fe_plugins / pom.xml:
|
||||
```
|
||||
.....
|
||||
<groupId> org.apache </ groupId>
|
||||
<artifactId> doris-fe-plugins </ artifactId>
|
||||
<packaging> pom </ packaging>
|
||||
<version> 1.0-SNAPSHOT </ version>
|
||||
<modules>
|
||||
<module> auditdemo </ module>
|
||||
# new plugin module
|
||||
<module> doris-fe-test </ module>
|
||||
</ modules>
|
||||
|
||||
<properties>
|
||||
<doris.home> $ {basedir} /../../ </ doris.home>
|
||||
</ properties>
|
||||
.....
|
||||
```
|
||||
|
||||
The new plugin project file structure is as follows:
|
||||
```
|
||||
-doris-fe-test /
|
||||
-pom.xml
|
||||
-src /
|
||||
---- main / java / org / apache /
|
||||
------App.java # mvn auto generate, ignore
|
||||
---- test / java / org / apache
|
||||
```
|
||||
|
||||
We will add an assembly folder under main to store plugin.properties and zip.xml. After completion, the file structure is as follows:
|
||||
```
|
||||
-doris-fe-test /
|
||||
-pom.xml
|
||||
-src /
|
||||
---- main /
|
||||
------ assembly /
|
||||
-------- plugin.properties
|
||||
-------- zip.xml
|
||||
------ java / org / apache /
|
||||
--------App.java # mvn auto generate, ignore
|
||||
---- test / java / org / apache
|
||||
```
|
||||
|
||||
### add zip.xml
|
||||
zip.xml, used to describe the content of the final package of the plugin (.jar file, plugin.properties):
|
||||
|
||||
```
|
||||
<assembly>
|
||||
<id> plugin </ id>
|
||||
<formats>
|
||||
<format> zip </ format>
|
||||
</ formats>
|
||||
<!-IMPORTANT: must be false->
|
||||
<includeBaseDirectory> false </ includeBaseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory> target </ directory>
|
||||
<includes>
|
||||
<include> *. jar </ include>
|
||||
</ includes>
|
||||
<outputDirectory> / </ outputDirectory>
|
||||
</ fileSet>
|
||||
|
||||
<fileSet>
|
||||
<directory> src / main / assembly </ directory>
|
||||
<includes>
|
||||
<include> plugin.properties </ include>
|
||||
</ includes>
|
||||
<outputDirectory> / </ outputDirectory>
|
||||
</ fileSet>
|
||||
</ fileSets>
|
||||
</ assembly>
|
||||
```
|
||||
|
||||
|
||||
### update pom.xml
|
||||
Then we need to update pom.xml, add doris-fe dependency, and modify maven packaging way:
|
||||
```
|
||||
<? xml version = "1.0" encoding = "UTF-8"?>
|
||||
<project xmlns = "http://maven.apache.org/POM/4.0.0"
|
||||
xmlns: xsi = "http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi: schemaLocation = "http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<groupId> org.apache </ groupId>
|
||||
<artifactId> doris-fe-test </ artifactId>
|
||||
<version> 1.0-SNAPSHOT </ version>
|
||||
</ parent>
|
||||
<modelVersion> 4.0.0 </ modelVersion>
|
||||
|
||||
<! --- IMPORTANT UPDATE! --->
|
||||
<artifactId> doris-fe-test </ artifactId>
|
||||
<packaging> jar </ packaging>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId> org.apache </ groupId>
|
||||
<artifactId> doris-fe </ artifactId>
|
||||
</ dependency>
|
||||
</ dependencies>
|
||||
<build>
|
||||
<finalName> doris-fe-test </ finalName>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId> maven-assembly-plugin </ artifactId>
|
||||
<version> 2.4.1 </ version>
|
||||
<configuration>
|
||||
<appendAssemblyId> false </ appendAssemblyId>
|
||||
<descriptors>
|
||||
<descriptor> src / main / assembly / zip.xml </ descriptor>
|
||||
</ descriptors>
|
||||
</ configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id> make-assembly </ id>
|
||||
<phase> package </ phase>
|
||||
<goals>
|
||||
<goal> single </ goal>
|
||||
</ goals>
|
||||
</ execution>
|
||||
</ executions>
|
||||
</ plugin>
|
||||
</ plugins>
|
||||
</ build>
|
||||
<! --- IMPORTANT UPDATE! --->
|
||||
</ project>
|
||||
```
|
||||
|
||||
### implement plugin
|
||||
Then we can happily implement Plugin according to the needs
|
||||
|
||||
### compile
|
||||
Before compiling the plugin, you must first execute `sh build.sh --fe` of Doris to complete the compilation of Doris FE.
|
||||
|
||||
Finally, execute `sh build_plugin.sh` in the ${DORIS_HOME} path and you will find the plugin .zip file in fe_plugins/output
|
||||
|
||||
### other way
|
||||
The easiest way, you can implement your plugin by modifying an example
|
||||
|
||||
## Deploy
|
||||
Doris's plugin can be deployed in three ways:
|
||||
|
||||
* Http or Https .zip, like http://xxx.xxxxxx.com/data/plugin.zip, Doris will download this .zip file
|
||||
* Local .zip, like /home/work/data/plugin.zip, need to be deployed on all FE and BE nodes
|
||||
* Local directory, like /home/work/data/plugin, .zip decompressed folder, need to be deployed on all FE, BE nodes
|
||||
|
||||
Note: Need to ensure that the plugin .zip file is available in the life cycle of doris!
|
||||
|
||||
## Use
|
||||
|
||||
Install and uninstall the plugin through the install/uninstall statements:
|
||||
|
||||
```
|
||||
mysql>
|
||||
mysql>
|
||||
mysql> install plugin from "/home/users/seaven/auditdemo.zip";
|
||||
Query OK, 0 rows affected (0.09 sec)
|
||||
|
||||
mysql>
|
||||
mysql>
|
||||
mysql>
|
||||
mysql> show plugins;
|
||||
+ ------------------- + ------- + --------------- + ----- ---- + ------------- + ------------------------ + ------ -+ --------------------------------- + ----------- +
|
||||
Name | Type | Description | Version | JavaVersion | ClassName | SoName | Sources |
|
||||
+ ------------------- + ------- + --------------- + ----- ---- + ------------- + ------------------------ + ------ -+ --------------------------------- + ----------- +
|
||||
audit_plugin_demo | AUDIT | just for test | 0.11.0 | 1.8.31 | plugin.AuditPluginDemo | NULL | /home/users/hekai/auditdemo.zip | INSTALLED |
|
||||
+ ------------------- + ------- + --------------- + ----- ---- + ------------- + ------------------------ + ------ -+ --------------------------------- + ----------- +
|
||||
1 row in set (0.02 sec)
|
||||
mysql>
|
||||
mysql>
|
||||
mysql>
|
||||
mysql> uninstall plugin audit_plugin_demo;
|
||||
Query OK, 0 rows affected (0.05 sec)
|
||||
|
||||
mysql> show plugins;
|
||||
Empty set (0.00 sec)
|
||||
|
||||
mysql>
|
||||
|
||||
```
|
||||
@ -1,6 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<groupId>org.apache</groupId>
|
||||
@ -18,7 +18,6 @@
|
||||
<artifactId>doris-fe</artifactId>
|
||||
</dependency>
|
||||
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
@ -71,4 +70,4 @@
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
</project>
|
||||
|
||||
@ -11,7 +11,6 @@ description=just for test
|
||||
#
|
||||
# Doris's version, like: 0.11.0
|
||||
version=0.11.0
|
||||
|
||||
### FE-Plugin optional:
|
||||
#
|
||||
# version of java the code is built against
|
||||
@ -19,8 +18,7 @@ version=0.11.0
|
||||
java.version=1.8.31
|
||||
#
|
||||
# the name of the class to load, fully-qualified.
|
||||
classname=plugin.AuditPluginDemo
|
||||
|
||||
classname=org.apache.doris.plugin.audit.AuditPluginDemo
|
||||
### BE-Plugin optional:
|
||||
#
|
||||
#
|
||||
@ -15,25 +15,25 @@
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package plugin;
|
||||
|
||||
import java.io.IOException;
|
||||
package org.apache.doris.plugin.audit;
|
||||
|
||||
import org.apache.doris.plugin.AuditEvent;
|
||||
import org.apache.doris.plugin.AuditPlugin;
|
||||
import org.apache.doris.plugin.Plugin;
|
||||
import org.apache.doris.plugin.PluginContext;
|
||||
import org.apache.doris.plugin.PluginException;
|
||||
import org.apache.doris.plugin.PluginInfo;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class AuditPluginDemo extends Plugin implements AuditPlugin {
|
||||
private final static Logger LOG = LogManager.getLogger(AuditPluginDemo.class);
|
||||
|
||||
private final static short EVENT_MASKS = AuditEvent.AUDIT_QUERY_START | AuditEvent.AUDIT_QUERY_END;
|
||||
|
||||
@Override
|
||||
public void init(PluginInfo info, PluginContext ctx) {
|
||||
public void init(PluginInfo info, PluginContext ctx) throws PluginException {
|
||||
super.init(info, ctx);
|
||||
LOG.info("this is audit plugin demo init");
|
||||
}
|
||||
@ -44,17 +44,11 @@ public class AuditPluginDemo extends Plugin implements AuditPlugin {
|
||||
LOG.info("this is audit plugin demo close");
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean eventFilter(short type, short masks) {
|
||||
return type == AuditEvent.AUDIT_QUERY && (masks & EVENT_MASKS) != 0;
|
||||
public boolean eventFilter(AuditEvent.EventType type) {
|
||||
return type == AuditEvent.EventType.AFTER_QUERY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exec(AuditEvent event) {
|
||||
LOG.info("audit demo plugin log: event={} masks={} user={}, ip={}, query={}", event.getEventType(),
|
||||
event.getEventMasks(),
|
||||
event.getUser(),
|
||||
event.getIp(),
|
||||
event.getQuery());
|
||||
LOG.info("audit demo plugin log exec");
|
||||
}
|
||||
}
|
||||
73
fe_plugins/auditloader/pom.xml
Normal file
73
fe_plugins/auditloader/pom.xml
Normal file
@ -0,0 +1,73 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<groupId>org.apache</groupId>
|
||||
<artifactId>doris-fe-plugins</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>auditloader</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache</groupId>
|
||||
<artifactId>doris-fe</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-core</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-slf4j-impl -->
|
||||
<dependency>
|
||||
<groupId>org.apache.logging.log4j</groupId>
|
||||
<artifactId>log4j-slf4j-impl</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/log4j/log4j -->
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<build>
|
||||
<finalName>auditloader</finalName>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.4.1</version>
|
||||
<configuration>
|
||||
<appendAssemblyId>false</appendAssemblyId>
|
||||
<descriptors>
|
||||
<descriptor>src/main/assembly/zip.xml</descriptor>
|
||||
</descriptors>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
24
fe_plugins/auditloader/src/main/assembly/plugin.conf
Executable file
24
fe_plugins/auditloader/src/main/assembly/plugin.conf
Executable file
@ -0,0 +1,24 @@
|
||||
### plugin configuration
|
||||
|
||||
# The max size of a batch, default is 50MB
|
||||
max_batch_size=52428800
|
||||
|
||||
# The max interval of batch loaded, default is 60 seconds
|
||||
max_batch_interval_sec=60
|
||||
|
||||
# Doris FE host for loading the audit, default is 127.0.0.1:8030.
|
||||
# this should be the host port for stream load
|
||||
frontend_host_port=127.0.0.1:8030
|
||||
|
||||
# Database of the audit table
|
||||
database=doris_audit_db__
|
||||
|
||||
# Audit table name, to save the audit data.
|
||||
table=doris_audit_tbl__
|
||||
|
||||
# Doris user. This user must have LOAD_PRIV to the audit table.
|
||||
user=root
|
||||
|
||||
# Doris user's password
|
||||
password=
|
||||
|
||||
6
fe_plugins/auditloader/src/main/assembly/plugin.properties
Executable file
6
fe_plugins/auditloader/src/main/assembly/plugin.properties
Executable file
@ -0,0 +1,6 @@
|
||||
name=AuditLoader
|
||||
type=AUDIT
|
||||
description=load audit log to olap load, and user can view the statistic of queries
|
||||
version=0.12.0
|
||||
java.version=1.8.0
|
||||
classname=org.apache.doris.plugin.audit.AuditLoaderPlugin
|
||||
25
fe_plugins/auditloader/src/main/assembly/zip.xml
Normal file
25
fe_plugins/auditloader/src/main/assembly/zip.xml
Normal file
@ -0,0 +1,25 @@
|
||||
<assembly>
|
||||
<id>plugin</id>
|
||||
<formats>
|
||||
<format>zip</format>
|
||||
</formats>
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>target</directory>
|
||||
<includes>
|
||||
<include>*.jar</include>
|
||||
</includes>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
</fileSet>
|
||||
|
||||
<fileSet>
|
||||
<directory>src/main/assembly</directory>
|
||||
<includes>
|
||||
<include>plugin.properties</include>
|
||||
<include>plugin.conf</include>
|
||||
</includes>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
</assembly>
|
||||
@ -0,0 +1,254 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.plugin.audit;
|
||||
|
||||
import org.apache.doris.plugin.AuditEvent;
|
||||
import org.apache.doris.plugin.AuditPlugin;
|
||||
import org.apache.doris.plugin.Plugin;
|
||||
import org.apache.doris.plugin.PluginContext;
|
||||
import org.apache.doris.plugin.PluginException;
|
||||
import org.apache.doris.plugin.PluginInfo;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.TimeZone;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/*
|
||||
* This plugin will load audit log to specified doris table at specified interval
|
||||
*/
|
||||
public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
|
||||
private final static Logger LOG = LogManager.getLogger(AuditLoaderPlugin.class);
|
||||
|
||||
private static SimpleDateFormat DATETIME_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
private StringBuilder auditBuffer = new StringBuilder();
|
||||
private long lastLoadTime = 0;
|
||||
|
||||
private BlockingQueue<StringBuilder> batchQueue = new LinkedBlockingDeque<StringBuilder>(1);
|
||||
private DorisStreamLoader streamLoader;
|
||||
private Thread loadThread;
|
||||
|
||||
private AuditLoaderConf conf;
|
||||
private volatile boolean isClosed = false;
|
||||
private volatile boolean isInit = false;
|
||||
|
||||
@Override
|
||||
public void init(PluginInfo info, PluginContext ctx) throws PluginException {
|
||||
super.init(info, ctx);
|
||||
|
||||
synchronized (this) {
|
||||
if (isInit) {
|
||||
return;
|
||||
}
|
||||
this.lastLoadTime = System.currentTimeMillis();
|
||||
|
||||
loadConfig(ctx);
|
||||
|
||||
this.streamLoader = new DorisStreamLoader(conf);
|
||||
this.loadThread = new Thread(new LoadWorker(this.streamLoader), "audit loader thread");
|
||||
this.loadThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
private void loadConfig(PluginContext ctx) throws PluginException {
|
||||
Path pluginPath = FileSystems.getDefault().getPath(ctx.getPluginPath());
|
||||
if (!Files.exists(pluginPath)) {
|
||||
throw new PluginException("plugin path does not exist: " + pluginPath);
|
||||
}
|
||||
|
||||
Path confFile = pluginPath.resolve("plugin.conf");
|
||||
if (!Files.exists(confFile)) {
|
||||
throw new PluginException("plugin conf file does not exist: " + confFile);
|
||||
}
|
||||
|
||||
final Properties props = new Properties();
|
||||
try (InputStream stream = Files.newInputStream(confFile)) {
|
||||
props.load(stream);
|
||||
} catch (IOException e) {
|
||||
throw new PluginException(e.getMessage());
|
||||
}
|
||||
final Map<String, String> properties = props.stringPropertyNames().stream()
|
||||
.collect(Collectors.toMap(Function.identity(), props::getProperty));
|
||||
|
||||
conf = new AuditLoaderConf();
|
||||
conf.init(properties);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
super.close();
|
||||
isClosed = true;
|
||||
if (loadThread != null) {
|
||||
try {
|
||||
loadThread.join();
|
||||
} catch (InterruptedException e) {
|
||||
LOG.debug("encounter exception when closing the audit loader", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public boolean eventFilter(AuditEvent.EventType type) {
|
||||
return type == AuditEvent.EventType.AFTER_QUERY;
|
||||
}
|
||||
|
||||
public void exec(AuditEvent event) {
|
||||
assembleAudit(event);
|
||||
loadIfNecessary();
|
||||
}
|
||||
|
||||
private void assembleAudit(AuditEvent event) {
|
||||
auditBuffer.append(event.queryId).append("\t");
|
||||
auditBuffer.append(longToTimeString(event.timestamp)).append("\t");
|
||||
auditBuffer.append(event.clientIp).append("\t");
|
||||
auditBuffer.append(event.user).append("\t");
|
||||
auditBuffer.append(event.db).append("\t");
|
||||
auditBuffer.append(event.state).append("\t");
|
||||
auditBuffer.append(event.queryTime).append("\t");
|
||||
auditBuffer.append(event.scanBytes).append("\t");
|
||||
auditBuffer.append(event.scanRows).append("\t");
|
||||
auditBuffer.append(event.returnRows).append("\t");
|
||||
auditBuffer.append(event.stmtId).append("\t");
|
||||
auditBuffer.append(event.isQuery ? 1 : 0).append("\t");
|
||||
auditBuffer.append(event.feIp).append("\t");
|
||||
// trim the query to avoid too long
|
||||
int maxLen = Math.min(2048, event.stmt.length());
|
||||
String stmt = event.stmt.substring(0, maxLen).replace("\t", " ");
|
||||
LOG.debug("receive audit event with stmt: {}", stmt);
|
||||
auditBuffer.append(stmt).append("\n");
|
||||
}
|
||||
|
||||
private void loadIfNecessary() {
|
||||
if (auditBuffer.length() < conf.maxBatchSize && System.currentTimeMillis() - lastLoadTime < conf.maxBatchIntervalSec * 1000) {
|
||||
return;
|
||||
}
|
||||
|
||||
lastLoadTime = System.currentTimeMillis();
|
||||
// begin to load
|
||||
try {
|
||||
if (!batchQueue.isEmpty()) {
|
||||
// TODO(cmy): if queue is not empty, which means the last batch is not processed.
|
||||
// In order to ensure that the system can run normally, here we directly
|
||||
// discard the current batch. If this problem occurs frequently,
|
||||
// improvement can be considered.
|
||||
throw new PluginException("The previous batch is not processed, and the current batch is discarded.");
|
||||
}
|
||||
|
||||
batchQueue.put(this.auditBuffer);
|
||||
} catch (Exception e) {
|
||||
LOG.debug("encounter exception when putting current audit batch, discard current batch", e);
|
||||
} finally {
|
||||
// make a new string builder to receive following events.
|
||||
this.auditBuffer = new StringBuilder();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
public static class AuditLoaderConf {
|
||||
public static final String PROP_MAX_BATCH_SIZE = "max_batch_size";
|
||||
public static final String PROP_MAX_BATCH_INTERVAL_SEC = "max_batch_interval_sec";
|
||||
public static final String PROP_FRONTEND_HOST_PORT = "frontend_host_port";
|
||||
public static final String PROP_USER = "user";
|
||||
public static final String PROP_PASSWORD = "password";
|
||||
public static final String PROP_DATABASE = "database";
|
||||
public static final String PROP_TABLE = "table";
|
||||
|
||||
public long maxBatchSize = 50 * 1024 * 1024;
|
||||
public long maxBatchIntervalSec = 60;
|
||||
public String frontendHostPort = "127.0.0.1:9030";
|
||||
public String user = "root";
|
||||
public String password = "";
|
||||
public String database = "__doris_audit_db";
|
||||
public String table = "__doris_audit_tbl";
|
||||
|
||||
public void init(Map<String, String> properties) throws PluginException {
|
||||
try {
|
||||
if (properties.containsKey(PROP_MAX_BATCH_SIZE)) {
|
||||
maxBatchSize = Long.valueOf(properties.get(PROP_MAX_BATCH_SIZE));
|
||||
}
|
||||
if (properties.containsKey(PROP_MAX_BATCH_INTERVAL_SEC)) {
|
||||
maxBatchIntervalSec = Long.valueOf(properties.get(PROP_MAX_BATCH_INTERVAL_SEC));
|
||||
}
|
||||
if (properties.containsKey(PROP_FRONTEND_HOST_PORT)) {
|
||||
frontendHostPort = properties.get(PROP_FRONTEND_HOST_PORT);
|
||||
}
|
||||
if (properties.containsKey(PROP_USER)) {
|
||||
user = properties.get(PROP_USER);
|
||||
}
|
||||
if (properties.containsKey(PROP_PASSWORD)) {
|
||||
password = properties.get(PROP_PASSWORD);
|
||||
}
|
||||
if (properties.containsKey(PROP_DATABASE)) {
|
||||
database = properties.get(PROP_DATABASE);
|
||||
}
|
||||
if (properties.containsKey(PROP_TABLE)) {
|
||||
table = properties.get(PROP_TABLE);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new PluginException(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class LoadWorker implements Runnable {
|
||||
private DorisStreamLoader loader;
|
||||
|
||||
public LoadWorker(DorisStreamLoader loader) {
|
||||
this.loader = loader;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
while (!isClosed) {
|
||||
try {
|
||||
StringBuilder batch = batchQueue.poll(5, TimeUnit.SECONDS);
|
||||
if (batch == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
DorisStreamLoader.LoadResponse response = loader.loadBatch(batch);
|
||||
LOG.debug("audit loader response: {}", response);
|
||||
} catch (InterruptedException e) {
|
||||
LOG.debug("encounter exception when loading current audit batch", e);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static synchronized String longToTimeString(long timeStamp) {
|
||||
if (timeStamp <= 0L) {
|
||||
return "1900-01-01 00:00:00";
|
||||
}
|
||||
return DATETIME_FORMAT.format(new Date(timeStamp));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,151 @@
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package org.apache.doris.plugin.audit;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Base64;
|
||||
import java.util.Calendar;
|
||||
|
||||
public class DorisStreamLoader {
|
||||
private final static Logger LOG = LogManager.getLogger(DorisStreamLoader.class);
|
||||
private static String loadUrlPattern = "http://%s/api/%s/%s/_stream_load?";
|
||||
private String hostPort;
|
||||
private String db;
|
||||
private String tbl;
|
||||
private String user;
|
||||
private String passwd;
|
||||
private String loadUrlStr;
|
||||
private String authEncoding;
|
||||
|
||||
public DorisStreamLoader(AuditLoaderPlugin.AuditLoaderConf conf) {
|
||||
this.hostPort = conf.frontendHostPort;
|
||||
this.db = conf.database;
|
||||
this.tbl = conf.table;
|
||||
this.user = conf.user;
|
||||
this.passwd = conf.password;
|
||||
|
||||
this.loadUrlStr = String.format(loadUrlPattern, hostPort, db, tbl);
|
||||
this.authEncoding = Base64.getEncoder().encodeToString(String.format("%s:%s", user, passwd).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
AuditLoaderPlugin.AuditLoaderConf conf = new AuditLoaderPlugin.AuditLoaderConf();
|
||||
conf.frontendHostPort = "fe_host";
|
||||
conf.database = "db1";
|
||||
conf.table = "tbl1";
|
||||
conf.user = "root";
|
||||
conf.password = "";
|
||||
|
||||
DorisStreamLoader loader = new DorisStreamLoader(conf);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("1\t2\n3\t4\n");
|
||||
|
||||
System.out.println("before load");
|
||||
LoadResponse loadResponse = loader.loadBatch(sb);
|
||||
|
||||
System.out.println(loadResponse);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public LoadResponse loadBatch(StringBuilder sb) {
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
String label = String.format("audit_%s%02d%02d_%02d%02d%02d",
|
||||
calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH), calendar.get(Calendar.DAY_OF_MONTH),
|
||||
calendar.get(Calendar.HOUR), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND));
|
||||
|
||||
|
||||
try {
|
||||
// build requst
|
||||
URL loadUrl = new URL(loadUrlStr);
|
||||
HttpURLConnection conn = (HttpURLConnection) loadUrl.openConnection();
|
||||
conn.setRequestMethod("PUT");
|
||||
conn.setRequestProperty("Authorization", "Basic " + authEncoding);
|
||||
conn.addRequestProperty("Expect", "100-continue");
|
||||
conn.addRequestProperty("Content-Type", "text/plain; charset=UTF-8");
|
||||
|
||||
conn.addRequestProperty("label", label);
|
||||
conn.addRequestProperty("max_fiter_ratio", "1.0");
|
||||
|
||||
conn.setDoOutput(true);
|
||||
conn.setDoInput(true);
|
||||
|
||||
// send data
|
||||
BufferedOutputStream bos = new BufferedOutputStream(conn.getOutputStream());
|
||||
bos.write(sb.toString().getBytes());
|
||||
bos.close();
|
||||
|
||||
// get respond
|
||||
int status = conn.getResponseCode();
|
||||
String respMsg = conn.getResponseMessage();
|
||||
|
||||
InputStream stream = (InputStream) conn.getContent();
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(stream));
|
||||
StringBuilder response = new StringBuilder();
|
||||
String line;
|
||||
while ((line = br.readLine()) != null) {
|
||||
response.append(line);
|
||||
}
|
||||
|
||||
LOG.info("AuditLoader plugin load with label: {}, response code: {}, msg: {}, content: {}",
|
||||
label, status, respMsg, response.toString());
|
||||
|
||||
return new LoadResponse(status, respMsg, response.toString());
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
String err = "failed to load audit via AuditLoader plugin with label: " + label;
|
||||
LOG.warn(err, e);
|
||||
return new LoadResponse(-1, e.getMessage(), err);
|
||||
}
|
||||
}
|
||||
|
||||
public static class LoadResponse {
|
||||
public int status;
|
||||
public String respMsg;
|
||||
public String respContent;
|
||||
|
||||
public LoadResponse(int status, String respMsg, String respContent) {
|
||||
this.status = status;
|
||||
this.respMsg = respMsg;
|
||||
this.respContent = respContent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("status: ").append(status);
|
||||
sb.append(", resp msg: ").append(respMsg);
|
||||
sb.append(", resp content: ").append(respContent);
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,5 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.apache</groupId>
|
||||
@ -8,10 +9,11 @@
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<modules>
|
||||
<module>auditdemo</module>
|
||||
<module>auditloader</module>
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<doris.home>${basedir}/../../</doris.home>
|
||||
<doris.home>${env.DORIS_HOME}</doris.home>
|
||||
</properties>
|
||||
<profiles>
|
||||
<!-- for custom internal repository -->
|
||||
@ -79,4 +81,18 @@
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
||||
Reference in New Issue
Block a user