diff --git a/build_plugin.sh b/build_plugin.sh index 7ecc6be8a0..05044fd0d8 100755 --- a/build_plugin.sh +++ b/build_plugin.sh @@ -25,18 +25,15 @@ export DORIS_HOME=${ROOT} . ${DORIS_HOME}/env.sh - # Check args usage() { echo " Usage: $0 Optional options: - --fe build Frontend - --p build special plugin - --clean clean and build Frontend + --plugin build special plugin Eg. - $0 --fe build Frontend and all plugin - $0 --p xxx build xxx plugin + $0 --plugin xxx build xxx plugin + $0 build all plugins " exit 1 } @@ -45,8 +42,7 @@ OPTS=$(getopt \ -n $0 \ -o '' \ -o 'h' \ - -l 'fe' \ - -l 'p' \ + -l 'plugin' \ -l 'clean' \ -l 'help' \ -- "$@") @@ -57,23 +53,17 @@ fi eval set -- "$OPTS" -BUILD_CLEAN= -BUILD_FE= -ALL_PLUGIN= +ALL_PLUGIN=1 +CLEAN=0 if [ $# == 1 ] ; then # defuat - BUILD_CLEAN=0 - BUILD_FE=0 ALL_PLUGIN=1 + CLEAN=0 else - BUILD_FE=0 - ALL_PLUGIN=1 - BUILD_CLEAN=0 while true; do case "$1" in - --fe) BUILD_FE=1 ; shift ;; - --p) ALL_PLUGIN=0 ; shift ;; - --clean) BUILD_CLEAN=1 ; shift ;; + --plugin) ALL_PLUGIN=0 ; shift ;; + --clean) CLEAN=1 ; shift ;; -h) HELP=1; shift ;; --help) HELP=1; shift ;; --) shift ; break ;; @@ -87,35 +77,33 @@ if [[ ${HELP} -eq 1 ]]; then exit fi -if [ ${CLEAN} -eq 1 -a ${BUILD_FE} -eq 0 ]; then - echo "--clean can not be specified without --fe" - exit 1 -fi - echo "Get params: - BUILD_FE -- $BUILD_FE - BUILD_CLEAN -- $BUILD_CLEAN BUILD_ALL_PLUGIN -- $ALL_PLUGIN + CLEAN -- $CLEAN " -cd ${DORIS_HOME} - -# Clean and build Frontend -if [ ${BUILD_FE} -eq 1 ] ; then - if [ ${CLEAN} -eq 1 ] ; then - sh build.sh --fe --clean - else - sh build.sh --fe - fi +# check if palo-fe.jar exist +if [ ! -f "$DORIS_HOME/fe/target/palo-fe.jar" ]; then + echo "ERROR: palo-fe.jar does not exist. Please build FE first" + exit -1 fi +cd ${DORIS_HOME} +PLUGIN_MODULE= if [ ${ALL_PLUGIN} -eq 1 ] ; then cd ${DORIS_HOME}/fe_plugins + if [ ${CLEAN} -eq 1 ]; then + ${MVN_CMD} clean + fi echo "build all plugins" - ${MVN_CMD} package -DskipTests + ${MVN_CMD} package -DskipTests else - cd ${DORIS_HOME}/fe_plugins/$1 - echo "build plugin $1" + PLUGIN_MODULE=$1 + cd ${DORIS_HOME}/fe_plugins/$PLUGIN_MODULE + if [ ${CLEAN} -eq 1 ]; then + ${MVN_CMD} clean + fi + echo "build plugin $PLUGIN_MODULE" ${MVN_CMD} package -DskipTests fi @@ -124,10 +112,15 @@ cd ${DORIS_HOME} DORIS_OUTPUT=${DORIS_HOME}/fe_plugins/output/ mkdir -p ${DORIS_OUTPUT} -cp -p ${DORIS_HOME}/fe_plugins/*/target/*.zip ${DORIS_OUTPUT}/ +if [ ${ALL_PLUGIN} -eq 1 ] ; then + cp -p ${DORIS_HOME}/fe_plugins/*/target/*.zip ${DORIS_OUTPUT}/ +else + cp -p ${DORIS_HOME}/fe_plugins/$PLUGIN_MODULE/target/*.zip ${DORIS_OUTPUT}/ +fi echo "***************************************" -echo "Successfully build Doris Plugin" +echo "Successfully build Doris FE Plugin" echo "***************************************" exit 0 + diff --git a/docs/documentation/cn/extending-doris/audit-plugin.md b/docs/documentation/cn/extending-doris/audit-plugin.md new file mode 100644 index 0000000000..79bf5c44f5 --- /dev/null +++ b/docs/documentation/cn/extending-doris/audit-plugin.md @@ -0,0 +1,108 @@ + + +# 审计日志插件 + +Doris 的审计日志插件是在 FE 的插件框架基础上开发的。是一个可选插件。用户可以在运行时安装或卸载这个插件。 + +该插件可以将 FE 的审计日志定期的导入到指定 Doris 集群中，以方便用户通过 SQL 对审计日志进行查看和分析。 + +## 编译、配置和部署 + +### 编译 + +在 Doris 代码目录下执行 `sh build_plugin.sh` 后，会在 `fe_plugins/output` 目录下得到 `auditloader.zip` 文件。 + +### 配置 + +解压 `auditloader.zip` 可以看到三个文件： + +``` +auditloader.jar +plugin.properties +plugin.conf +``` + +打开 `plugin.conf` 进行配置。配置项说明参见注释。 + +配置完成后，重新将三个文件打包为 `auditloader.zip`. + +### 部署 + +您可以将这个文件放置在一个 http 下载服务器上，或者拷贝到所有 FE 的指定目录下。这里我们使用后者。 + +### 安装 + +部署完成后，安装插件前，需要创建之前在 `plugin.conf` 中指定的审计数据库和表。其中建表语句如下： + +``` +create table doris_audit_tbl__ +( + query_id varchar(48) comment "Unique query id", + time datetime not null comment "Query start time", + client_ip varchar(32) comment "Client IP", + user varchar(64) comment "User name", + db varchar(96) comment "Database of this query", + state varchar(8) comment "Query result state. EOF, ERR, OK", + query_time bigint comment "Query execution time in millisecond", + scan_bytes bigint comment "Total scan bytes of this query", + scan_rows bigint comment "Total scan rows of this query", + return_rows bigint comment "Returned rows of this query", + stmt_id int comment "An incremental id of statement", + is_query tinyint comment "Is this statemt a query. 1 or 0", + frontend_ip varchar(32) comment "Frontend ip of executing this statement", + stmt varchar(2048) comment "The original statement, trimed if longer than 2048 bytes" +) +partition by range(time) () +distributed by hash(query_id) buckets 1 +properties( + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.start" = "-30", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "1", + "dynamic_partition.enable" = "true", + "replication_num" = "1" +); +``` + +其中 `dynamic_partition` 属性根据自己的需要，选择审计日志安保留的天数。 + +之后，连接到 Doris 后使用 `INSTALL PLUGIN` 命令完成安装。安装成功后，可以通过 `SHOW PLUGINS` 看到已经安装的插件，并且状态为 `INSTALLED`。 + +完成后，插件会不断的以指定的时间间隔将审计日志插入到这个表中。 + + + + + + + + + + + + + + + + + + + diff --git a/docs/documentation/cn/extending-doris/plugin-development-manual.md b/docs/documentation/cn/extending-doris/plugin-development-manual.md new file mode 100644 index 0000000000..63e12c67e8 --- /dev/null +++ b/docs/documentation/cn/extending-doris/plugin-development-manual.md @@ -0,0 +1,291 @@ + + +# 插件开发手册 + +## 介绍 + +Doris 支持动态加载插件。用户可以通过开发自己的插件来扩展Doris的功能。这个手册主要介绍如何开发、编译和部署 Frontend 端的插件。 + +`fe_plugins` 目录是 FE 插件的根模块。这个根模块统一管理插件所需的依赖。添加一个新的插件，相当于在这个根模块添加一个子模块。 + +## 插件组成 + +一个FE的插件可以使一个**zip压缩包**或者是一个**目录**。其内容至少包含两个文件：`plugin.properties` 和 `.jar` 文件。`plugin.properties`用于描述插件信息。 + +文件结构如下： + +``` +# plugin .zip +auditodemo.zip: + -plugin.properties + -auditdemo.jar + -xxx.config + -data/ + -test_data/ + +# plugin local directory +auditodemo/: + -plugin.properties + -auditdemo.jar + -xxx.config + -data/ + -test_data/ +``` + +`plugin.properties` 内容示例: + +``` +### required: +# +# the plugin name +name = audit_plugin_demo +# +# the plugin type +type = AUDIT +# +# simple summary of the plugin +description = just for test +# +# Doris's version, like: 0.11.0 +version = 0.11.0 + +### FE-Plugin optional: +# +# version of java the code is built against +# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4 +java.version = 1.8.31 +# +# the name of the class to load, fully-qualified. +classname = AuditPluginDemo + +### BE-Plugin optional: +# the name of the so to load +soName = example.so +``` + +## 编写一个插件 + +插件的开发环境依赖Doris的开发编译环境。所以请先确保Doris的编译开发环境运行正常。 + +### 创建一个模块 + +我们可以通过以下命令在 `fe_plugins` 目录创建一个子模块用户实现创建和创建工程。其中 `doris-fe-test` 为插件名称。 + +``` +mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false +``` + +这个命令会创建一个新的 maven 工程，并且自动向 `fe_plugins/pom.xml` 中添加一个子模块： + +``` +    ..... +    org.apache +    doris-fe-plugins +    pom +    1.0-SNAPSHOT +     +        auditdemo +        # new plugin module +        doris-fe-test +     +    ..... +``` + +新的工程目录结构如下： + +``` +-doris-fe-test/ +-pom.xml +-src/ + ---- main/java/org/apache/ + ------- App.java # mvn auto generate, ignore + ---- test/java/org/apache +``` + +接下来我们在 `main` 目录下添加一个 `assembly` 目录来存放 `plugin.properties` 和 `zip.xml`。最终的工程目录结构如下： + +``` +-doris-fe-test/ +-pom.xml +-src/ +---- main/ +------ assembly/ +-------- plugin.properties +-------- zip.xml +------ java/org/apache/ +--------App.java # mvn auto generate, ignore +---- test/java/org/apache +``` + +### 添加 zip.xml + +`zip.xml` 用于描述最终生成的 zip 压缩包中的文件内容。（如 .jar file, plugin.properties 等等） + +``` + +    plugin +     +        zip +     +     +    false +     +         +            target +             +                *.jar +             +            / +         + +         +            src/main/assembly +             +                plugin.properties +             +            / +         +     + +``` + +### 更新 pom.xml + +接下来我们需要更新子模块的 `pom.xml` 文件，添加 doris-fe 依赖： + +``` + + + + org.apache + doris-fe-plugins + 1.0-SNAPSHOT + + 4.0.0 + + auditloader + jar + + + + org.apache + doris-fe + + + + + ... + + + + + + auditloader + + + maven-assembly-plugin + 2.4.1 + + false + + src/main/assembly/zip.xml + + + + + make-assembly + package + + single + + + + + + + + +``` + +### 实现插件 + +之后我们就可以开始愉快的进行插件功能的开发啦。插件需要实现 `Plugin` 接口。具体可以参阅 Doris 自带的 `auditdemo` 插件示例代码。 + +### 编译 + +在编译插件之前，需要先执行 `sh build.sh --fe` 进行 Doris FE 代码的编译，并确保编译成功。 + +之后，执行 `sh build_plugin.sh` 编译所有插件。最终的产出会存放在 `fe_plugins/output` 目录中。 + +或者也可以执行 `sh build_plugin.sh --plugin your_plugin_name` 来仅编译指定的插件。 + +### 另一种开发方式 + +您可以直接通过修改自带的 `auditdemo` 插件示例代码进行开发。 + +## 部署 + +插件可以通过以下三种方式部署。 + +* 将 `.zip` 文件放在 Http 或 Https 服务器上。如：`http://xxx.xxxxxx.com/data/plugin.zip`, Doris 会下载这个文件。 +* 本地 `.zip` 文件。 如：`/home/work/data/plugin.zip`。需要在所有 FE 和 BE 节点部署。 +* 本地目录。如：`/home/work/data/plugin/`。这个相当于 `.zip` 文件解压后的目录。需要在所有 FE 和 BE 节点部署。 + +注意：需保证部署路径在整个插件生命周期内有效。 + +## 安装和卸载插件 + +通过如下命令安装和卸载插件。更多帮助请参阅 `HELP INSTALL PLUGIN;` `HELP IUNNSTALL PLUGIN;` `HELP SHOW PLUGINS;` + +``` +mysql> install plugin from "/home/users/seaven/auditdemo.zip"; +Query OK, 0 rows affected (0.09 sec) + +mysql> mysql> show plugins\G +*************************** 1. row *************************** + Name: auditloader + Type: AUDIT +Description: load audit log to olap load, and user can view the statistic of queries + Version: 0.12.0 +JavaVersion: 1.8.31 + ClassName: AuditLoaderPlugin + SoName: NULL + Sources: /home/cmy/git/doris/core/fe_plugins/output/auditloader.zip + Status: INSTALLED +*************************** 2. row *************************** + Name: AuditLogBuilder + Type: AUDIT +Description: builtin audit logger + Version: 0.12.0 +JavaVersion: 1.8.31 + ClassName: org.apache.doris.qe.AuditLogBuilder + SoName: NULL + Sources: Builtin + Status: INSTALLED +2 rows in set (0.00 sec) + +mysql> uninstall plugin auditloader; +Query OK, 0 rows affected (0.05 sec) + +mysql> show plugins; +Empty set (0.00 sec) +``` diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md new file mode 100644 index 0000000000..7233189f1f --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-statements/Administration/INSTALL PLUGIN.md @@ -0,0 +1,50 @@ + + +# INSTALL PLUGIN +## description + + 该语句用于安装一个插件。 + + 语法 + + INSTALL PLUGIN FROM [source] + + source 支持三种类型： + + 1. 指向一个 zip 文件的绝对路径。 + 2. 指向一个插件目录的绝对路径。 + 3. 指向一个 http 或 https 协议的 zip 文件下载路径 + +## example + + 1. 安装一个本地 zip 文件插件： + + INSTALL PLUGIN FROM "/home/users/seaven/auditdemo.zip"; + + 2. 安装一个本地目录中的插件： + + INSTALL PLUGIN FROM "/home/users/seaven/auditdemo/"; + + 2. 下载并安装一个插件： + + INSTALL PLUGIN FROM "http://mywebsite.com/plugin.zip"; + +## keyword + INSTALL,PLUGIN diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW PLUGINS.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW PLUGINS.md new file mode 100644 index 0000000000..aceeb9cf86 --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-statements/Administration/SHOW PLUGINS.md @@ -0,0 +1,38 @@ + + +# SHOW PLUGINS +## description + + 该语句用于展示已安装的插件。 + + 语法 + + SHOW PLUGINS; + + 该命令会展示所有用户安装的和系统内置的插件。 + +## example + + 1. 展示已安装的插件： + + SHOW PLUGINS; + +## keyword + SHOW PLUGINS \ No newline at end of file diff --git a/docs/documentation/cn/sql-reference/sql-statements/Administration/UNINSTALL PLUGIN.md b/docs/documentation/cn/sql-reference/sql-statements/Administration/UNINSTALL PLUGIN.md new file mode 100644 index 0000000000..06abe10554 --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-statements/Administration/UNINSTALL PLUGIN.md @@ -0,0 +1,40 @@ + + +# UNINTALL PLUGIN +## description + + 该语句用于卸载一个插件。 + + 语法 + + UNINSTALL PLUGIN plugin_name; + + plugin_name 可以通过 `SHOW PLUGINS;` 命令查看。 + + 只能卸载非 builtin 的插件。 + +## example + + 1. 卸载一个插件： + + UNINSTALL PLUGIN auditdemo; + +## keyword + UNINSTALL,PLUGIN \ No newline at end of file diff --git a/docs/documentation/en/extending-doris/audit-plugin_EN.md b/docs/documentation/en/extending-doris/audit-plugin_EN.md new file mode 100644 index 0000000000..84143cca5f --- /dev/null +++ b/docs/documentation/en/extending-doris/audit-plugin_EN.md @@ -0,0 +1,89 @@ + + +# Audit log plugin + +Doris's audit log plugin was developed based on FE's plugin framework. Is an optional plugin. Users can install or uninstall this plugin at runtime. + +This plugin can periodically import the FE audit log into the specified Doris cluster, so that users can easily view and analyze the audit log through SQL. + +## Compile, Configure and Deploy + +### Compile + +After executing `sh build_plugin.sh` in the Doris code directory, you will get the `auditloader.zip` file in the `fe_plugins/output` directory. + +### Configuration + +Unzip `auditloader.zip` and you will see three files: + +``` +auditloader.jar +plugin.properties +plugin.conf +``` + +Open `plugin.conf` for configuration. See the comments of the configuration items. + +After the configuration is complete, repackage the three files as `auditloader.zip`. + +### Deployment + +You can place this file on an http download server or copy it to the specified directory of all FEs. Here we use the latter. + +### Installation + +After deployment is complete, and before installing the plugin, you need to create the audit database and tables previously specified in `plugin.conf`. The table creation statement is as follows: + +``` +create table doris_audit_tbl__ +( +    query_id varchar (48) comment "Unique query id", +    time datetime not null comment "Query start time", +    client_ip varchar (32) comment "Client IP", +    user varchar (64) comment "User name", +    db varchar (96) comment "Database of this query", +    state varchar (8) comment "Query result state. EOF, ERR, OK", +    query_time bigint comment "Query execution time in millisecond", +    scan_bytes bigint comment "Total scan bytes of this query", +    scan_rows bigint comment "Total scan rows of this query", +    return_rows bigint comment "Returned rows of this query", +    stmt_id int comment "An incremental id of statement", +    is_query tinyint comment "Is this statemt a query. 1 or 0", +    frontend_ip varchar (32) comment "Frontend ip of executing this statement", +    stmt varchar (2048) comment "The original statement, trimed if longer than 2048 bytes" +) +partition by range (time) () +distributed by hash (query_id) buckets 1 +properties ( +    "dynamic_partition.time_unit" = "DAY", +    "dynamic_partition.start" = "-30", +    "dynamic_partition.end" = "3", +    "dynamic_partition.prefix" = "p", +    "dynamic_partition.buckets" = "1", +    "dynamic_partition.enable" = "true", +    "replication_num" = "1" +); +``` + +The `dynamic_partition` attribute selects the number of days to keep the audit log based on your needs. + +After that, connect to Doris and use the `INSTALL PLUGIN` command to complete the installation. After successful installation, you can see the installed plug-ins through `SHOW PLUGINS`, and the status is `INSTALLED`. + +Upon completion, the plug-in will continuously import audit date into this table at specified intervals. \ No newline at end of file diff --git a/docs/documentation/en/extending-doris/plugin-development-manual_EN.md b/docs/documentation/en/extending-doris/plugin-development-manual_EN.md new file mode 100644 index 0000000000..3ca7b0ef22 --- /dev/null +++ b/docs/documentation/en/extending-doris/plugin-development-manual_EN.md @@ -0,0 +1,293 @@ + + +# Plugin Development Manual + +## Introduction + +Doris supports dynamic loading of plug-ins. Users can develop their own plug-ins to implement some extended functions. This manual mainly introduces the development, compilation and deployment methods of Frontend-side plug-ins. + + +`fe_plugins` is the parent module of the fe plugins. It can uniformly manage the third-party library information that the plugin depends on. Adding a plugin can add a submodule implementation under `fe_plugins`. + +## Plugin + +A FE Plugin can be a **.zip package** or a **directory**, which contains at least two parts: the `plugin.properties` and `.jar` files. The `plugin.properties` file is used to describe the plugin information. + +The file structure of a Plugin looks like this: + +``` +# plugin .zip +auditodemo.zip: + -plugin.properties + -auditdemo.jar + -xxx.config + -data/ + -test_data/ + +# plugin local directory +auditodemo/: + -plugin.properties + -auditdemo.jar + -xxx.config + -data/ + -test_data/ +``` + +`plugin.properties` example: + +``` +### required: +# +# the plugin name +name = audit_plugin_demo +# +# the plugin type +type = AUDIT +# +# simple summary of the plugin +description = just for test +# +# Doris's version, like: 0.11.0 +version = 0.11.0 + +### FE-Plugin optional: +# +# version of java the code is built against +# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4 +java.version = 1.8.31 +# +# the name of the class to load, fully-qualified. +classname = AuditPluginDemo + +### BE-Plugin optional: +# the name of the so to load +soName = example.so +``` + +## Write A Plugin + +The development environment of the FE plugin depends on the development environment of Doris. So please make sure Doris's compilation and development environment works normally. + +### Create module + +We can add a submodule in the `fe_plugins` directory to implement Plugin and create a project: + +``` +mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false +``` + +The command produces a new mvn project, and a new submodule is automatically added to `fe_plugins/pom.xml`: + +``` +    ..... +    org.apache +    doris-fe-plugins +    pom +    1.0-SNAPSHOT +     +        auditdemo +        # new plugin module +        doris-fe-test +     +    ..... +``` + +The new plugin project file structure is as follows: + +``` +-doris-fe-test/ +-pom.xml +-src/ + ---- main/java/org/apache/ + ------- App.java # mvn auto generate, ignore + ---- test/java/org/apache +``` + +We will add an assembly folder under main to store `plugin.properties` and `zip.xml`. After completion, the file structure is as follows: + +``` +-doris-fe-test/ +-pom.xml +-src/ +---- main/ +------ assembly/ +-------- plugin.properties +-------- zip.xml +------ java/org/apache/ +--------App.java # mvn auto generate, ignore +---- test/java/org/apache +``` + +### Add zip.xml + +`zip.xml`, used to describe the content of the final package of the plugin (.jar file, plugin.properties): + +``` + +    plugin +     +        zip +     +     +    false +     +         +            target +             +                *.jar +             +            / +         + +         +            src/main/assembly +             +                plugin.properties +             +            / +         +     + +``` + + +### Update pom.xml + +Then we need to update `pom.xml`, add doris-fe dependency, and modify maven packaging way: + +``` + + + + org.apache + doris-fe-plugins + 1.0-SNAPSHOT + + 4.0.0 + + auditloader + jar + + + + org.apache + doris-fe + + + + + ... + + + + + + auditloader + + + maven-assembly-plugin + 2.4.1 + + false + + src/main/assembly/zip.xml + + + + + make-assembly + package + + single + + + + + + + + +``` + +### Implement plugin + +Then we can happily implement Plugin according to the needs. Plugins need to implement the `Plugin` interface. For details, please refer to the `auditdemo` plugin sample code that comes with Doris. + +### Compile + +Before compiling the plugin, you must first execute `sh build.sh --fe` of Doris to complete the compilation of Doris FE. + +Finally, execute `sh build_plugin.sh` in the ${DORIS_HOME} path and you will find the `your_plugin_name.zip` file in `fe_plugins/output` + +Or you can execute `sh build_plugin.sh --plugin your_plugin_name` to only build your plugin. + +### Other way + +The easiest way, you can implement your plugin by modifying the example `auditdemo` + +## Deploy + +Doris's plugin can be deployed in three ways: + +* Http or Https .zip, like `http://xxx.xxxxxx.com/data/plugin.zip`, Doris will download this .zip file +* Local .zip, like `/home/work/data/plugin.zip`, need to be deployed on all FE and BE nodes +* Local directory, like `/home/work/data/plugin`, .zip decompressed folder, need to be deployed on all FE, BE nodes + +Note: Need to ensure that the plugin .zip file is available in the life cycle of doris! + +## Install and Uninstall + +Install and uninstall the plugin through the install/uninstall statements. More details, see `HELP INSTALL PLUGIN;` `HELP IUNNSTALL PLUGIN;` `HELP SHOW PLUGINS;` + +``` +mysql> install plugin from "/home/users/seaven/auditdemo.zip"; +Query OK, 0 rows affected (0.09 sec) + +mysql> mysql> show plugins\G +*************************** 1. row *************************** + Name: auditloader + Type: AUDIT +Description: load audit log to olap load, and user can view the statistic of queries + Version: 0.12.0 +JavaVersion: 1.8.31 + ClassName: AuditLoaderPlugin + SoName: NULL + Sources: /home/cmy/git/doris/core/fe_plugins/output/auditloader.zip + Status: INSTALLED +*************************** 2. row *************************** + Name: AuditLogBuilder + Type: AUDIT +Description: builtin audit logger + Version: 0.12.0 +JavaVersion: 1.8.31 + ClassName: org.apache.doris.qe.AuditLogBuilder + SoName: NULL + Sources: Builtin + Status: INSTALLED +2 rows in set (0.00 sec) + +mysql> uninstall plugin auditloader; +Query OK, 0 rows affected (0.05 sec) + +mysql> show plugins; +Empty set (0.00 sec) +``` diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/INSTALL PLUGIN_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/INSTALL PLUGIN_EN.md new file mode 100644 index 0000000000..f4744302cf --- /dev/null +++ b/docs/documentation/en/sql-reference/sql-statements/Administration/INSTALL PLUGIN_EN.md @@ -0,0 +1,50 @@ + + +# INSTALL PLUGIN +## description + + To install a plugin + + Syntax + + INSTALL PLUGIN FROM [source] + + source supports 3 kinds: + + 1. Point to a zip file with absolute path. + 2. Point to a plugin dir with absolute path. + 3. Point to a http/https download link of zip file. + +## example + + 1. Intall a plugin with a local zip file: + + INSTALL PLUGIN FROM "/home/users/seaven/auditdemo.zip"; + + 2. Intall a plugin with a local dir: + + INSTALL PLUGIN FROM "/home/users/seaven/auditdemo/"; + + 2. Download and install a plugin: + + INSTALL PLUGIN FROM "http://mywebsite.com/plugin.zip"; + +## keyword + INSTALL,PLUGIN diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW PLUGINS_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW PLUGINS_EN.md new file mode 100644 index 0000000000..1284ab7f90 --- /dev/null +++ b/docs/documentation/en/sql-reference/sql-statements/Administration/SHOW PLUGINS_EN.md @@ -0,0 +1,38 @@ + + +# SHOW PLUGINS +## description + + To view the installed plugins. + + Syntax + + SHOW PLUGINS; + + This command will show all builtin and custom plugins. + +## example + + 1. To view the installed plugins: + + SHOW PLUGINS; + +## keyword + SHOW PLUGINS \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-statements/Administration/UNINTALL PLUGIN_EN.md b/docs/documentation/en/sql-reference/sql-statements/Administration/UNINTALL PLUGIN_EN.md new file mode 100644 index 0000000000..48c52eb273 --- /dev/null +++ b/docs/documentation/en/sql-reference/sql-statements/Administration/UNINTALL PLUGIN_EN.md @@ -0,0 +1,40 @@ + + +# UNINTALL PLUGIN +## description + + To uninstall a plugin. + + Syntax + + UNINSTALL PLUGIN plugin_name; + + plugin_name can be found by `SHOW PLUGINS;`. + + Can only uninstall non-builtin plugins. + +## example + + 1. Uninstall a plugin: + + UNINSTALL PLUGIN auditdemo; + +## keyword + UNINSTALL,PLUGIN \ No newline at end of file diff --git a/fe/src/main/java/org/apache/doris/alter/Alter.java b/fe/src/main/java/org/apache/doris/alter/Alter.java index 8028023e98..4678d50927 100644 --- a/fe/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/src/main/java/org/apache/doris/alter/Alter.java @@ -179,10 +179,6 @@ public class Alter { } OlapTable olapTable = (OlapTable) table; - if (olapTable.getPartitions().size() == 0 && !currentAlterOps.hasPartitionOp()) { - throw new DdlException("Table with empty parition cannot do schema change. [" + tableName + "]"); - } - if (olapTable.getState() != OlapTableState.NORMAL) { throw new DdlException( "Table[" + table.getName() + "]'s state is not NORMAL. Do not allow doing ALTER ops"); diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index 2cbcee202a..603cbcaf32 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -180,6 +180,7 @@ import org.apache.doris.persist.TablePropertyInfo; import org.apache.doris.persist.TruncateTableInfo; import org.apache.doris.plugin.PluginInfo; import org.apache.doris.plugin.PluginMgr; +import org.apache.doris.qe.AuditEventProcessor; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.JournalObservable; import org.apache.doris.qe.SessionVariable; @@ -387,6 +388,8 @@ public class Catalog { private PluginMgr pluginMgr; + private AuditEventProcessor auditEventProcessor; + public List getFrontends(FrontendNodeType nodeType) { if (nodeType == null) { // get all @@ -521,6 +524,7 @@ public class Catalog { this.imageDir = this.metaDir + IMAGE_DIR; this.pluginMgr = new PluginMgr(); + this.auditEventProcessor = new AuditEventProcessor(this.pluginMgr); } public static void destroyCheckpoint() { @@ -586,6 +590,10 @@ public class Catalog { return fullNameToDb; } + public AuditEventProcessor getAuditEventProcessor() { + return auditEventProcessor; + } + // use this to get correct ClusterInfoService instance public static SystemInfoService getCurrentSystemInfo() { return getCurrentCatalog().getClusterInfo(); @@ -622,6 +630,10 @@ public class Catalog { return getCurrentCatalog().getPluginMgr(); } + public static AuditEventProcessor getCurrentAuditEventProcessor() { + return getCurrentCatalog().getAuditEventProcessor(); + } + // Use tryLock to avoid potential dead lock private boolean tryLock(boolean mustLock) { while (true) { @@ -704,6 +716,7 @@ public class Catalog { // init plugin manager pluginMgr.init(); + auditEventProcessor.start(); // 2. get cluster id and role (Observer or Follower) getClusterIdAndRole(); @@ -3612,7 +3625,6 @@ public class Catalog { olapTable.setIndexMeta(baseIndexId, tableName, baseSchema, schemaVersion, schemaHash, shortKeyColumnCount, baseIndexStorageType, keysType); - for (AlterClause alterClause : stmt.getRollupAlterClauseList()) { AddRollupClause addRollupClause = (AddRollupClause)alterClause; @@ -6405,9 +6417,7 @@ public class Catalog { } public void installPlugin(InstallPluginStmt stmt) throws UserException, IOException { - PluginInfo pluginInfo = pluginMgr.installPlugin(stmt); - editLog.logInstallPlugin(pluginInfo); - LOG.info("install plugin = " + pluginInfo.getName()); + pluginMgr.installPlugin(stmt); } public long savePlugins(DataOutputStream dos, long checksum) throws IOException { diff --git a/fe/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/src/main/java/org/apache/doris/catalog/TableProperty.java index 6260d3a7d9..64d5564c1c 100644 --- a/fe/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -101,11 +101,11 @@ public class TableProperty implements Writable { return this; } - void modifyTableProperties(Map modifyProperties) { + public void modifyTableProperties(Map modifyProperties) { properties.putAll(modifyProperties); } - void modifyTableProperties(String key, String value) { + public void modifyTableProperties(String key, String value) { properties.put(key, value); } diff --git a/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 03c41d987b..806d5ebe10 100644 --- a/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -250,7 +250,13 @@ public class DynamicPartitionUtil { public static void checkAndSetDynamicPartitionProperty(OlapTable olapTable, Map properties) throws DdlException { if (DynamicPartitionUtil.checkInputDynamicPartitionProperties(properties, olapTable.getPartitionInfo())) { Map dynamicPartitionProperties = DynamicPartitionUtil.analyzeDynamicPartition(properties); - olapTable.setTableProperty(new TableProperty(dynamicPartitionProperties).buildDynamicProperty()); + TableProperty tableProperty = olapTable.getTableProperty(); + if (tableProperty != null) { + tableProperty.modifyTableProperties(dynamicPartitionProperties); + tableProperty.buildDynamicProperty(); + } else { + olapTable.setTableProperty(new TableProperty(dynamicPartitionProperties).buildDynamicProperty()); + } } } diff --git a/fe/src/main/java/org/apache/doris/plugin/AuditEvent.java b/fe/src/main/java/org/apache/doris/plugin/AuditEvent.java index 87fc24f7de..302134e0a4 100644 --- a/fe/src/main/java/org/apache/doris/plugin/AuditEvent.java +++ b/fe/src/main/java/org/apache/doris/plugin/AuditEvent.java @@ -17,83 +17,153 @@ package org.apache.doris.plugin; -import org.apache.doris.qe.ConnectContext; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +/* + * AuditEvent contains all information about audit log info. + * It should be created by AuditEventBuilder. For example: + * + * AuditEvent event = new AuditEventBuilder() + * .setEventType(AFTER_QUERY) + * .setClientIp(xxx) + * ... + * .build(); + */ public class AuditEvent { - /** - * Audit plugin event type - */ - public final static short AUDIT_CONNECTION = 1; - public final static short AUDIT_QUERY = 2; - - /** - * Connection sub-event masks - */ - public final static short AUDIT_CONNECTION_CONNECT = 1; - public final static short AUDIT_CONNECTION_DISCONNECT = 1 << 1; - - /** - * Query sub-event masks - */ - public final static short AUDIT_QUERY_START = 1; - public final static short AUDIT_QUERY_END = 1 << 1; - - protected String query; - - protected short type; - - protected short masks; - - protected ConnectContext context; - - protected AuditEvent(ConnectContext context) { - this.context = context; + public enum EventType { + CONNECTION, + DISCONNECTION, + BEFORE_QUERY, + AFTER_QUERY } - protected AuditEvent(ConnectContext context, String query) { - this.context = context; - this.query = query; + @Retention(RetentionPolicy.RUNTIME) + public static @interface AuditField { + String value() default ""; } - public void setEventTypeAndMasks(short type, short masks) { - this.type = type; - this.masks = masks; - } + public EventType type; - public short getEventType() { - return type; - } + // all fields which is about to be audit should be annotated by "@AuditField" + // make them all "public" so that easy to visit. + @AuditField(value = "Timestamp") + public long timestamp = -1; + @AuditField(value = "Client") + public String clientIp = ""; + @AuditField(value = "User") + public String user = ""; + @AuditField(value = "Db") + public String db = ""; + @AuditField(value = "State") + public String state = ""; + @AuditField(value = "Time") + public long queryTime = -1; + @AuditField(value = "ScanBytes") + public long scanBytes = -1; + @AuditField(value = "ScanRows") + public long scanRows = -1; + @AuditField(value = "ReturnRows") + public long returnRows = -1; + @AuditField(value = "StmtId") + public long stmtId = -1; + @AuditField(value = "QueryId") + public String queryId = ""; + @AuditField(value = "IsQuery") + public boolean isQuery = false; + @AuditField(value = "fe_ip") + public String feIp = ""; + @AuditField(value = "Stmt") + public String stmt = ""; - public short getEventMasks() { - return masks; - } + public static class AuditEventBuilder { - public int getConnectId() { - return context.getConnectionId(); - } + private AuditEvent auditEvent = new AuditEvent(); - public String getIp() { - return context.getRemoteIP(); - } + public AuditEventBuilder() { + } - public String getUser() { - return context.getQualifiedUser(); - } + public void reset() { + auditEvent = new AuditEvent(); + } - public long getStmtId() { - return context.getStmtId(); - } + public AuditEventBuilder setEventType(EventType eventType) { + auditEvent.type = eventType; + return this; + } - public String getQuery() { - return query; - } + public AuditEventBuilder setTimestamp(long timestamp) { + auditEvent.timestamp = timestamp; + return this; + } - public static AuditEvent createConnectEvent(ConnectContext ctx) { - return new AuditEvent(ctx); - } + public AuditEventBuilder setClientIp(String clientIp) { + auditEvent.clientIp = clientIp; + return this; + } - public static AuditEvent createQueryEvent(ConnectContext ctx, String query) { - return new AuditEvent(ctx, query); - } + public AuditEventBuilder setUser(String user) { + auditEvent.user = user; + return this; + } + public AuditEventBuilder setDb(String db) { + auditEvent.db = db; + return this; + } + + public AuditEventBuilder setState(String state) { + auditEvent.state = state; + return this; + } + + public AuditEventBuilder setQueryTime(long queryTime) { + auditEvent.queryTime = queryTime; + return this; + } + + public AuditEventBuilder setScanBytes(long scanBytes) { + auditEvent.scanBytes = scanBytes; + return this; + } + + public AuditEventBuilder setScanRows(long scanRows) { + auditEvent.scanRows = scanRows; + return this; + } + + public AuditEventBuilder setReturnRows(long returnRows) { + auditEvent.returnRows = returnRows; + return this; + } + + public AuditEventBuilder setStmtId(long stmtId) { + auditEvent.stmtId = stmtId; + return this; + } + + public AuditEventBuilder setQueryId(String queryId) { + auditEvent.queryId = queryId; + return this; + } + + public AuditEventBuilder setIsQuery(boolean isQuery) { + auditEvent.isQuery = isQuery; + return this; + } + + public AuditEventBuilder setFeIp(String feIp) { + auditEvent.feIp = feIp; + return this; + } + + public AuditEventBuilder setStmt(String stmt) { + auditEvent.stmt = stmt; + return this; + } + + public AuditEvent build() { + return this.auditEvent; + } + } } diff --git a/fe/src/main/java/org/apache/doris/plugin/AuditPlugin.java b/fe/src/main/java/org/apache/doris/plugin/AuditPlugin.java index b27c9480f6..d9c9ec8469 100644 --- a/fe/src/main/java/org/apache/doris/plugin/AuditPlugin.java +++ b/fe/src/main/java/org/apache/doris/plugin/AuditPlugin.java @@ -22,16 +22,14 @@ package org.apache.doris.plugin; */ public interface AuditPlugin { /** - * - * use for check audit event type and masks, the event will skip the plugin if return false + * use for check audit event type, the event will skip the plugin if return false */ - public boolean eventFilter(short type, short masks); + public boolean eventFilter(AuditEvent.EventType type); /** - * - * + * process the event. + * This method should be implemented as a non-blocking or lightweight method. + * Because it will be called after each query. So it must be efficient. */ public void exec(AuditEvent event); - - } diff --git a/fe/src/main/java/org/apache/doris/plugin/BuiltinPluginLoader.java b/fe/src/main/java/org/apache/doris/plugin/BuiltinPluginLoader.java index 52ed38e5e6..dbec0c249c 100644 --- a/fe/src/main/java/org/apache/doris/plugin/BuiltinPluginLoader.java +++ b/fe/src/main/java/org/apache/doris/plugin/BuiltinPluginLoader.java @@ -17,10 +17,10 @@ package org.apache.doris.plugin; -import java.io.IOException; - import org.apache.doris.common.UserException; +import java.io.IOException; + public class BuiltinPluginLoader extends PluginLoader { BuiltinPluginLoader(String path, PluginInfo info, Plugin plugin) { @@ -39,7 +39,6 @@ public class BuiltinPluginLoader extends PluginLoader { public void uninstall() throws IOException, UserException { if (plugin != null) { pluginUninstallValid(); - plugin.close(); } } diff --git a/fe/src/main/java/org/apache/doris/plugin/DynamicPluginLoader.java b/fe/src/main/java/org/apache/doris/plugin/DynamicPluginLoader.java index 3b2f78e5a9..03c0970b52 100644 --- a/fe/src/main/java/org/apache/doris/plugin/DynamicPluginLoader.java +++ b/fe/src/main/java/org/apache/doris/plugin/DynamicPluginLoader.java @@ -17,6 +17,12 @@ package org.apache.doris.plugin; +import org.apache.doris.common.UserException; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.IOException; import java.lang.reflect.Constructor; import java.net.URL; @@ -30,20 +36,20 @@ import java.util.Collections; import java.util.LinkedHashSet; import java.util.Set; -import org.apache.commons.io.FileUtils; -import org.apache.doris.common.UserException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - public class DynamicPluginLoader extends PluginLoader { private final static Logger LOG = LogManager.getLogger(DynamicPluginLoader.class); + // the final dir which contains all plugin files. + // eg: + // Config.plugin_dir/plugin_name/ protected Path installPath; - DynamicPluginLoader(String pluginPath, String source) { - super(pluginPath, source); + // for processing install stmt + DynamicPluginLoader(String pluginDir, String source) { + super(pluginDir, source); } + // for test and replay DynamicPluginLoader(String pluginPath, PluginInfo info) { super(pluginPath, info); this.installPath = FileSystems.getDefault().getPath(pluginDir.toString(), pluginInfo.getName()); @@ -67,9 +73,10 @@ public class DynamicPluginLoader extends PluginLoader { if (installPath == null) { // download plugin and extract PluginZip zip = new PluginZip(source); - Path target = Files.createTempDirectory(pluginDir, ".install_"); - - installPath = zip.extract(target); + // generation a tmp dir to extract the zip + Path tmpTarget = Files.createTempDirectory(pluginDir, ".install_"); + // for now, installPath point to the temp dir which contains all extracted files from zip file. + installPath = zip.extract(tmpTarget); } pluginInfo = PluginInfo.readFromProperties(installPath, source); @@ -87,12 +94,14 @@ public class DynamicPluginLoader extends PluginLoader { getPluginInfo(); - Path realPath = movePlugin(); + movePlugin(); - plugin = dynamicLoadPlugin(realPath); + plugin = dynamicLoadPlugin(); pluginInstallValid(); + pluginContext.setPluginPath(installPath.toString()); + plugin.init(pluginInfo, pluginContext); } @@ -104,7 +113,6 @@ public class DynamicPluginLoader extends PluginLoader { return true; } } - return false; } @@ -125,24 +133,25 @@ public class DynamicPluginLoader extends PluginLoader { /** * reload plugin if plugin has already been installed, else will re-install + * + * @throws PluginException */ public void reload() throws IOException, UserException { if (hasInstalled()) { - plugin = dynamicLoadPlugin(installPath); + plugin = dynamicLoadPlugin(); pluginInstallValid(); + pluginContext.setPluginPath(installPath.toString()); plugin.init(pluginInfo, pluginContext); - } else { // re-install this.pluginInfo = null; this.installPath = null; - install(); } } - Plugin dynamicLoadPlugin(Path path) throws IOException, UserException { - Set jarList = getJarUrl(path); + Plugin dynamicLoadPlugin() throws IOException, UserException { + Set jarList = getJarUrl(installPath); // create a child to load the plugin in this bundle ClassLoader parentLoader = PluginClassLoader.createLoader(getClass().getClassLoader(), Collections.EMPTY_LIST); @@ -198,19 +207,18 @@ public class DynamicPluginLoader extends PluginLoader { } /** - * move plugin's temp install directory to Doris's PLUGIN_DIR + * move plugin's temp install directory to Doris's PLUGIN_DIR/plugin_name */ - Path movePlugin() throws UserException, IOException { - + public void movePlugin() throws UserException, IOException { if (installPath == null || !Files.exists(installPath)) { - throw new UserException("Install plugin " + pluginInfo.getName() + " failed, because install path isn't " + throw new PluginException("Install plugin " + pluginInfo.getName() + " failed, because install path isn't " + "exists."); } Path targetPath = FileSystems.getDefault().getPath(pluginDir.toString(), pluginInfo.getName()); if (Files.exists(targetPath)) { if (!Files.isSameFile(installPath, targetPath)) { - throw new UserException( + throw new PluginException( "Install plugin " + pluginInfo.getName() + " failed. because " + installPath.toString() + " exists"); } @@ -220,8 +228,6 @@ public class DynamicPluginLoader extends PluginLoader { // move success installPath = targetPath; - - return installPath; } } diff --git a/fe/src/main/java/org/apache/doris/plugin/Plugin.java b/fe/src/main/java/org/apache/doris/plugin/Plugin.java index 27488909b4..b9487a75c1 100644 --- a/fe/src/main/java/org/apache/doris/plugin/Plugin.java +++ b/fe/src/main/java/org/apache/doris/plugin/Plugin.java @@ -33,15 +33,20 @@ public abstract class Plugin implements Closeable { public static final int PLUGIN_NOT_DYNAMIC_UNINSTALL = 1 << 1; /** - * invoke when the plugin install + * invoke when the plugin install. + * the plugin should only be initialized once. + * So this method must be idempotent */ - public void init(PluginInfo info, PluginContext ctx) { } + public void init(PluginInfo info, PluginContext ctx) throws PluginException { + + } /** * invoke when the plugin uninstall */ @Override - public void close() throws IOException { } + public void close() throws IOException { + } public int flags() { return PLUGIN_DEFAULT_FLAGS; diff --git a/fe/src/main/java/org/apache/doris/plugin/PluginContext.java b/fe/src/main/java/org/apache/doris/plugin/PluginContext.java index cbd5807f26..1ba6137a81 100644 --- a/fe/src/main/java/org/apache/doris/plugin/PluginContext.java +++ b/fe/src/main/java/org/apache/doris/plugin/PluginContext.java @@ -18,4 +18,15 @@ package org.apache.doris.plugin; public class PluginContext { + + // the dir path where the plugin's files saved + private String pluginPath; + + protected void setPluginPath(String pluginPath) { + this.pluginPath = pluginPath; + } + + public String getPluginPath() { + return pluginPath; + } } diff --git a/fe/src/main/java/org/apache/doris/qe/AuditBuilder.java b/fe/src/main/java/org/apache/doris/plugin/PluginException.java similarity index 62% rename from fe/src/main/java/org/apache/doris/qe/AuditBuilder.java rename to fe/src/main/java/org/apache/doris/plugin/PluginException.java index 8e5ae36912..1c35c86c3e 100644 --- a/fe/src/main/java/org/apache/doris/qe/AuditBuilder.java +++ b/fe/src/main/java/org/apache/doris/plugin/PluginException.java @@ -15,27 +15,16 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.qe; +package org.apache.doris.plugin; -// Helper class used to build audit log. -// Now, implemented with StringBuilder -public class AuditBuilder { - private StringBuilder sb; +import org.apache.doris.common.UserException; - public AuditBuilder() { - sb = new StringBuilder(); +public class PluginException extends UserException { + public PluginException(String msg) { + super(msg); } - public void reset() { - sb = new StringBuilder(); - } - - public void put(String key, Object value) { - sb.append("|").append(key).append("=").append(value.toString()); - } - - @Override - public String toString() { - return sb.toString(); + public PluginException(String msg, Throwable e) { + super(msg, e); } } diff --git a/fe/src/main/java/org/apache/doris/plugin/PluginLoader.java b/fe/src/main/java/org/apache/doris/plugin/PluginLoader.java index 2273f24306..e72d8781ca 100644 --- a/fe/src/main/java/org/apache/doris/plugin/PluginLoader.java +++ b/fe/src/main/java/org/apache/doris/plugin/PluginLoader.java @@ -17,13 +17,13 @@ package org.apache.doris.plugin; +import org.apache.doris.common.UserException; + import java.io.IOException; import java.nio.file.FileSystems; import java.nio.file.Path; import java.util.Objects; -import org.apache.doris.common.UserException; - public abstract class PluginLoader { public enum PluginStatus { @@ -34,8 +34,9 @@ public abstract class PluginLoader { ERROR, } + // the root dir of Frontend plugin, should always be Config.plugin_dir protected final Path pluginDir; - + // source of plugin, eg, a remote download link of zip file, or a local zip file. protected String source; protected Plugin plugin; @@ -92,7 +93,7 @@ public abstract class PluginLoader { public void pluginUninstallValid() throws UserException { // check plugin flags - if ((plugin.flags() & Plugin.PLUGIN_NOT_DYNAMIC_UNINSTALL) > 0) { + if (plugin != null && (plugin.flags() & Plugin.PLUGIN_NOT_DYNAMIC_UNINSTALL) > 0) { throw new UserException("plugin " + pluginInfo + " not allow dynamic uninstall"); } } diff --git a/fe/src/main/java/org/apache/doris/plugin/PluginMgr.java b/fe/src/main/java/org/apache/doris/plugin/PluginMgr.java index f40a74bdfa..722a5a90d4 100644 --- a/fe/src/main/java/org/apache/doris/plugin/PluginMgr.java +++ b/fe/src/main/java/org/apache/doris/plugin/PluginMgr.java @@ -18,11 +18,13 @@ package org.apache.doris.plugin; import org.apache.doris.analysis.InstallPluginStmt; +import org.apache.doris.catalog.Catalog; import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; import org.apache.doris.plugin.PluginInfo.PluginType; import org.apache.doris.plugin.PluginLoader.PluginStatus; +import org.apache.doris.qe.AuditLogBuilder; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -47,26 +49,35 @@ public class PluginMgr implements Writable { public PluginMgr() { plugins = new Map[PluginType.MAX_PLUGIN_SIZE]; - for (int i = 0; i < PluginType.MAX_PLUGIN_SIZE; i++) { plugins[i] = Maps.newConcurrentMap(); } } // create the plugin dir if missing - public void init() { + public void init() throws PluginException { File file = new File(Config.plugin_dir); if (file.exists() && !file.isDirectory()) { - LOG.error("FE plugin dir {} is not a directory", Config.plugin_dir); - System.exit(-1); + throw new PluginException("FE plugin dir " + Config.plugin_dir + " is not a directory"); } if (!file.exists()) { if (!file.mkdir()) { - LOG.error("failed to create FE plugin dir {}", Config.plugin_dir); - System.exit(-1); + throw new PluginException("failed to create FE plugin dir " + Config.plugin_dir); } } + + registerBuiltinPlugins(); + } + + private void registerBuiltinPlugins() { + // AuditLog + AuditLogBuilder auditLogBuilder = new AuditLogBuilder(); + if (!registerPlugin(auditLogBuilder.getPluginInfo(), auditLogBuilder)) { + LOG.warn("failed to register audit log builder"); + } + + // other builtin plugins } public PluginInfo installPlugin(InstallPluginStmt stmt) throws IOException, UserException { @@ -76,14 +87,21 @@ public class PluginMgr implements Writable { try { PluginInfo info = pluginLoader.getPluginInfo(); + if (plugins[info.getTypeId()].containsKey(info.getName())) { + throw new UserException("plugin " + info.getName() + " has already been installed."); + } + + // install plugin + pluginLoader.install(); + pluginLoader.setStatus(PluginStatus.INSTALLED); + if (plugins[info.getTypeId()].putIfAbsent(info.getName(), pluginLoader) != null) { pluginLoader.uninstall(); throw new UserException("plugin " + info.getName() + " has already been installed."); } - // install plugin - pluginLoader.install(); - pluginLoader.setStatus(PluginStatus.INSTALLED); + Catalog.getCurrentCatalog().getEditLog().logInstallPlugin(info); + LOG.info("install plugin = " + info.getName()); return info; } catch (IOException | UserException e) { pluginLoader.setStatus(PluginStatus.ERROR); @@ -120,14 +138,12 @@ public class PluginMgr implements Writable { */ public boolean registerPlugin(PluginInfo pluginInfo, Plugin plugin) { if (Objects.isNull(pluginInfo) || Objects.isNull(plugin) || Objects.isNull(pluginInfo.getType()) - || Strings - .isNullOrEmpty(pluginInfo.getName())) { + || Strings.isNullOrEmpty(pluginInfo.getName())) { return false; } PluginLoader loader = new BuiltinPluginLoader(Config.plugin_dir, pluginInfo, plugin); - PluginLoader checkLoader = - plugins[pluginInfo.getTypeId()].putIfAbsent(pluginInfo.getName(), loader); + PluginLoader checkLoader = plugins[pluginInfo.getTypeId()].putIfAbsent(pluginInfo.getName(), loader); return checkLoader == null; } @@ -170,7 +186,6 @@ public class PluginMgr implements Writable { public final List getActivePluginList(PluginType type) { Map m = plugins[type.ordinal()]; - List l = Lists.newArrayListWithCapacity(m.size()); m.values().forEach(d -> { diff --git a/fe/src/main/java/org/apache/doris/plugin/PluginZip.java b/fe/src/main/java/org/apache/doris/plugin/PluginZip.java index 78cde04556..b40ea23f22 100644 --- a/fe/src/main/java/org/apache/doris/plugin/PluginZip.java +++ b/fe/src/main/java/org/apache/doris/plugin/PluginZip.java @@ -17,6 +17,18 @@ package org.apache.doris.plugin; +import org.apache.doris.common.UserException; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -31,17 +43,6 @@ import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.doris.common.UserException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - /** * Describe plugin install file(.zip) * Support remote(http/https) source and local source @@ -61,6 +62,10 @@ class PluginZip { cleanPathList = Lists.newLinkedList(); } + /* + * download and extract the zip file to the target path. + * return the path dir which contains all extracted files. + */ public Path extract(Path targetPath) throws IOException, UserException { try { Path zipPath = downloadZip(targetPath); @@ -75,11 +80,12 @@ class PluginZip { /** * download zip if the source in remote, - * or return if the source in local + * return the zip file path. + * This zip file is currently in a temp directory, such ash **/ Path downloadZip(Path targetPath) throws IOException, UserException { if (Strings.isNullOrEmpty(source)) { - throw new IllegalArgumentException("Plugin library path: " + source); + throw new PluginException("empty plugin source path: " + source); } boolean isLocal = true; @@ -135,11 +141,11 @@ class PluginZip { } /** - * unzip .zip file + * unzip the specified .zip file "zip" to the target path */ Path extractZip(Path zip, Path targetPath) throws IOException, UserException { if (!Files.exists(zip)) { - throw new UserException("Download plugin zip failed, source: " + source); + throw new PluginException("Download plugin zip failed. zip file does not exist. source: " + source); } if (Files.isDirectory(zip)) { diff --git a/fe/src/main/java/org/apache/doris/qe/AuditEventProcessor.java b/fe/src/main/java/org/apache/doris/qe/AuditEventProcessor.java new file mode 100644 index 0000000000..2ed8fb2ce9 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/qe/AuditEventProcessor.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.plugin.AuditPlugin; +import org.apache.doris.plugin.Plugin; +import org.apache.doris.plugin.PluginInfo.PluginType; +import org.apache.doris.plugin.PluginMgr; + +import com.google.common.collect.Queues; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +/* + * Class for processing all audit events. + * It will receive audit events and handle them to all AUDIT type plugins. + */ +public class AuditEventProcessor { + private static final Logger LOG = LogManager.getLogger(AuditEventProcessor.class); + private static final long UPDATE_PLUGIN_INTERVAL_MS = 60 * 1000; // 1min + + private PluginMgr pluginMgr; + + private List auditPlugins; + private long lastUpdateTime = 0; + + private BlockingQueue eventQueue = Queues.newLinkedBlockingDeque(10000); + private Thread workerThread; + + private volatile boolean isStopped = false; + + public AuditEventProcessor(PluginMgr pluginMgr) { + this.pluginMgr = pluginMgr; + } + + public void start() { + workerThread = new Thread(new Worker(), "AuditEventProcessor"); + workerThread.start(); + } + + public void stop() { + isStopped = true; + if (workerThread != null) { + try { + workerThread.join(); + } catch (InterruptedException e) { + LOG.warn("join worker join failed.", e); + } + } + } + + public void handleAuditEvent(AuditEvent auditEvent) { + try { + eventQueue.put(auditEvent); + } catch (InterruptedException e) { + LOG.debug("encounter exception when handle audit event, ignore", e); + } + } + + public class Worker implements Runnable { + @Override + public void run() { + AuditEvent auditEvent = null; + while (!isStopped) { + // update audit plugin list every UPDATE_PLUGIN_INTERVAL_MS. + // because some of plugins may be installed or uninstalled at runtime. + if (auditPlugins == null || System.currentTimeMillis() - lastUpdateTime > UPDATE_PLUGIN_INTERVAL_MS) { + auditPlugins = pluginMgr.getActivePluginList(PluginType.AUDIT); + lastUpdateTime = System.currentTimeMillis(); + LOG.debug("update audit plugins. num: {}", auditPlugins.size()); + } + + try { + auditEvent = eventQueue.poll(5, TimeUnit.SECONDS); + if (auditEvent == null) { + continue; + } + } catch (InterruptedException e) { + LOG.debug("encounter exception when getting audit event from queue, ignore", e); + continue; + } + + try { + for (Plugin plugin : auditPlugins) { + if (((AuditPlugin) plugin).eventFilter(auditEvent.type)) { + ((AuditPlugin) plugin).exec(auditEvent); + } + } + } catch (Exception e) { + LOG.debug("encounter exception when processing audit event.", e); + } + } + } + + } +} diff --git a/fe/src/main/java/org/apache/doris/qe/AuditLogBuilder.java b/fe/src/main/java/org/apache/doris/qe/AuditLogBuilder.java new file mode 100644 index 0000000000..f85e26a9cd --- /dev/null +++ b/fe/src/main/java/org/apache/doris/qe/AuditLogBuilder.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.common.AuditLog; +import org.apache.doris.common.Config; +import org.apache.doris.common.util.DigitalVersion; +import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.plugin.AuditEvent.AuditField; +import org.apache.doris.plugin.AuditEvent.EventType; +import org.apache.doris.plugin.AuditPlugin; +import org.apache.doris.plugin.Plugin; +import org.apache.doris.plugin.PluginInfo; +import org.apache.doris.plugin.PluginInfo.PluginType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.reflect.Field; + +// A builtin Audit plugin, registered when FE start. +// it will receive "AFTER_QUERY" AuditEventy and print it as a log in fe.audit.log +public class AuditLogBuilder extends Plugin implements AuditPlugin { + private static final Logger LOG = LogManager.getLogger(AuditLogBuilder.class); + + private PluginInfo pluginInfo; + + public AuditLogBuilder() { + pluginInfo = new PluginInfo("AuditLogBuilder", PluginType.AUDIT, + "builtin audit logger", DigitalVersion.fromString("0.12.0"), + DigitalVersion.fromString("1.8.31"), AuditLogBuilder.class.getName(), null, null); + } + + public PluginInfo getPluginInfo() { + return pluginInfo; + } + + @Override + public boolean eventFilter(EventType type) { + return type == EventType.AFTER_QUERY; + } + + @Override + public void exec(AuditEvent event) { + try { + StringBuilder sb = new StringBuilder(); + long queryTime = 0; + // get each field with annotation "AuditField" in AuditEvent + // and assemble them into a string. + Field[] fields = event.getClass().getFields(); + for (Field f : fields) { + AuditField af = f.getAnnotation(AuditField.class); + if (af == null) { + continue; + } + + if (af.value().equals("Timestamp")) { + continue; + } + + if (af.value().equals("Time")) { + queryTime = (long) f.get(event); + } + sb.append("|").append(af.value()).append("=").append(String.valueOf(f.get(event))); + } + + String auditLog = sb.toString(); + AuditLog.getQueryAudit().log(auditLog); + // slow query + if (queryTime > Config.qe_slow_log_ms) { + AuditLog.getSlowAudit().log(auditLog); + } + } catch (Exception e) { + LOG.debug("failed to process audit event", e); + } + } +} diff --git a/fe/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/src/main/java/org/apache/doris/qe/ConnectContext.java index fff09a8a29..ef672d750a 100644 --- a/fe/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -24,6 +24,7 @@ import org.apache.doris.mysql.MysqlCapability; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.MysqlSerializer; +import org.apache.doris.plugin.AuditEvent.AuditEventBuilder; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TUniqueId; @@ -92,7 +93,7 @@ public class ConnectContext { protected Catalog catalog; protected boolean isSend; - protected AuditBuilder auditBuilder; + protected AuditEventBuilder auditEventBuilder = new AuditEventBuilder();; protected String remoteIP; @@ -119,7 +120,6 @@ public class ConnectContext { isKilled = false; serializer = MysqlSerializer.newInstance(); sessionVariable = VariableMgr.newSessionVariable(); - auditBuilder = new AuditBuilder(); command = MysqlCommand.COM_SLEEP; } @@ -131,7 +131,6 @@ public class ConnectContext { mysqlChannel = new MysqlChannel(channel); serializer = MysqlSerializer.newInstance(); sessionVariable = VariableMgr.newSessionVariable(); - auditBuilder = new AuditBuilder(); command = MysqlCommand.COM_SLEEP; if (channel != null) { remoteIP = mysqlChannel.getRemoteIp(); @@ -162,8 +161,8 @@ public class ConnectContext { this.remoteIP = remoteIP; } - public AuditBuilder getAuditBuilder() { - return auditBuilder; + public AuditEventBuilder getAuditEventBuilder() { + return auditEventBuilder; } public void setThreadLocalInfo() { diff --git a/fe/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/src/main/java/org/apache/doris/qe/ConnectProcessor.java index bb7f0dbce9..fcbf6f3a24 100644 --- a/fe/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -28,8 +28,6 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Table; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.AuditLog; -import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; @@ -43,7 +41,9 @@ import org.apache.doris.mysql.MysqlPacket; import org.apache.doris.mysql.MysqlProto; import org.apache.doris.mysql.MysqlSerializer; import org.apache.doris.mysql.MysqlServerStatusFlag; +import org.apache.doris.plugin.AuditEvent.EventType; import org.apache.doris.proto.PQueryStatistics; +import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TMasterOpRequest; import org.apache.doris.thrift.TMasterOpResult; @@ -106,14 +106,14 @@ public class ConnectProcessor { private void auditAfterExec(String origStmt, StatementBase parsedStmt, PQueryStatistics statistics) { // slow query long elapseMs = System.currentTimeMillis() - ctx.getStartTime(); - // query state log - ctx.getAuditBuilder().put("State", ctx.getState()); - ctx.getAuditBuilder().put("Time", elapseMs); - ctx.getAuditBuilder().put("ScanBytes", statistics == null ? 0 : statistics.scan_bytes); - ctx.getAuditBuilder().put("ScanRows", statistics == null ? 0 : statistics.scan_rows); - ctx.getAuditBuilder().put("ReturnRows", ctx.getReturnRows()); - ctx.getAuditBuilder().put("StmtId", ctx.getStmtId()); - ctx.getAuditBuilder().put("QueryId", ctx.queryId() == null ? "NaN" : DebugUtil.printId(ctx.queryId())); + + ctx.getAuditEventBuilder().setEventType(EventType.AFTER_QUERY) + .setState(ctx.getState().toString()).setQueryTime(elapseMs) + .setScanBytes(statistics == null ? 0 : statistics.scan_bytes) + .setScanRows(statistics == null ? 0 : statistics.scan_rows) + .setReturnRows(ctx.getReturnRows()) + .setStmtId(ctx.getStmtId()) + .setQueryId(ctx.queryId() == null ? "NaN" : DebugUtil.printId(ctx.queryId())); if (ctx.getState().isQuery()) { MetricRepo.COUNTER_QUERY_ALL.increase(1L); @@ -125,23 +125,21 @@ public class ConnectProcessor { // ok query MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs); } - ctx.getAuditBuilder().put("IsQuery", 1); + ctx.getAuditEventBuilder().setIsQuery(true); } else { - ctx.getAuditBuilder().put("IsQuery", 0); + ctx.getAuditEventBuilder().setIsQuery(false); } + + ctx.getAuditEventBuilder().setFeIp(FrontendOptions.getLocalHostAddress()); + // We put origin query stmt at the end of audit log, for parsing the log more convenient. if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { - ctx.getAuditBuilder().put("Stmt", parsedStmt.toSql()); + ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql()); } else { - ctx.getAuditBuilder().put("Stmt", origStmt); - } - - AuditLog.getQueryAudit().log(ctx.getAuditBuilder().toString()); - - // slow query - if (elapseMs > Config.qe_slow_log_ms) { - AuditLog.getSlowAudit().log(ctx.getAuditBuilder().toString()); + ctx.getAuditEventBuilder().setStmt(origStmt); } + + Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build()); } // process COM_QUERY statement, @@ -163,10 +161,12 @@ public class ConnectProcessor { ctx.getState().setError("Unsupported character set(UTF-8)"); return; } - ctx.getAuditBuilder().reset(); - ctx.getAuditBuilder().put("Client", ctx.getMysqlChannel().getRemoteHostPortString()); - ctx.getAuditBuilder().put("User", ctx.getQualifiedUser()); - ctx.getAuditBuilder().put("Db", ctx.getDatabase()); + ctx.getAuditEventBuilder().reset(); + ctx.getAuditEventBuilder() + .setTimestamp(System.currentTimeMillis()) + .setClientIp(ctx.getMysqlChannel().getRemoteHostPortString()) + .setUser(ctx.getQualifiedUser()) + .setDb(ctx.getDatabase()); // execute this query. StatementBase parsedStmt = null; diff --git a/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 78b2138647..7a4f4460d5 100644 --- a/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -28,7 +28,6 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.AuditLog; import org.apache.doris.common.AuthenticationException; import org.apache.doris.common.CaseSensibility; import org.apache.doris.common.Config; @@ -45,7 +44,9 @@ import org.apache.doris.load.MiniEtlTaskInfo; import org.apache.doris.master.MasterImpl; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.planner.StreamLoadPlanner; -import org.apache.doris.qe.AuditBuilder; +import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.plugin.AuditEvent.AuditEventBuilder; +import org.apache.doris.plugin.AuditEvent.EventType; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectProcessor; import org.apache.doris.qe.QeProcessorImpl; @@ -418,15 +419,15 @@ public class FrontendServiceImpl implements FrontendService.Iface { private void logMiniLoadStmt(TMiniLoadRequest request) throws UnknownHostException { String stmt = getMiniLoadStmt(request); - AuditBuilder auditBuilder = new AuditBuilder(); - auditBuilder.put("client", request.user_ip + ":0"); - auditBuilder.put("user", request.user); - auditBuilder.put("db", request.db); - auditBuilder.put("state", TStatusCode.OK); - auditBuilder.put("time", "0"); - auditBuilder.put("stmt", stmt); - - AuditLog.getQueryAudit().log(auditBuilder.toString()); + AuditEvent auditEvent = new AuditEventBuilder().setEventType(EventType.AFTER_QUERY) + .setClientIp(request.user_ip + ":0") + .setUser(request.user) + .setDb(request.db) + .setState(TStatusCode.OK.name()) + .setQueryTime(0) + .setStmt(stmt).build(); + + Catalog.getCurrentAuditEventProcessor().handleAuditEvent(auditEvent); } private String getMiniLoadStmt(TMiniLoadRequest request) throws UnknownHostException { diff --git a/fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 11d2553fd0..170c96e9df 100644 --- a/fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -55,6 +55,35 @@ public class QueryPlanTest { String createDbStmtStr = "create database test;"; CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext); Catalog.getCurrentCatalog().createDb(createDbStmt); + + createTable("create table test.test1\n" + + "(\n" + + " query_id varchar(48) comment \"Unique query id\",\n" + + " time datetime not null comment \"Query start time\",\n" + + " client_ip varchar(32) comment \"Client IP\",\n" + + " user varchar(64) comment \"User name\",\n" + + " db varchar(96) comment \"Database of this query\",\n" + + " state varchar(8) comment \"Query result state. EOF, ERR, OK\",\n" + + " query_time bigint comment \"Query execution time in millisecond\",\n" + + " scan_bytes bigint comment \"Total scan bytes of this query\",\n" + + " scan_rows bigint comment \"Total scan rows of this query\",\n" + + " return_rows bigint comment \"Returned rows of this query\",\n" + + " stmt_id int comment \"An incremental id of statement\",\n" + + " is_query tinyint comment \"Is this statemt a query. 1 or 0\",\n" + + " frontend_ip varchar(32) comment \"Frontend ip of executing this statement\",\n" + + " stmt varchar(2048) comment \"The original statement, trimed if longer than 2048 bytes\"\n" + + ")\n" + + "partition by range(time) ()\n" + + "distributed by hash(query_id) buckets 1\n" + + "properties(\n" + + " \"dynamic_partition.time_unit\" = \"DAY\",\n" + + " \"dynamic_partition.start\" = \"-30\",\n" + + " \"dynamic_partition.end\" = \"3\",\n" + + " \"dynamic_partition.prefix\" = \"p\",\n" + + " \"dynamic_partition.buckets\" = \"1\",\n" + + " \"dynamic_partition.enable\" = \"true\",\n" + + " \"replication_num\" = \"1\"\n" + + ");"); createTable("CREATE TABLE test.bitmap_table (\n" + " `id` int(11) NULL COMMENT \"\",\n" + diff --git a/fe/src/test/java/org/apache/doris/plugin/PluginLoaderTest.java b/fe/src/test/java/org/apache/doris/plugin/PluginLoaderTest.java index 11b5865688..67527fa083 100644 --- a/fe/src/test/java/org/apache/doris/plugin/PluginLoaderTest.java +++ b/fe/src/test/java/org/apache/doris/plugin/PluginLoaderTest.java @@ -21,17 +21,18 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.IOException; -import java.nio.file.Files; -import java.util.Map; - -import org.apache.commons.io.FileUtils; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DigitalVersion; import org.apache.doris.plugin.PluginInfo.PluginType; + +import org.apache.commons.io.FileUtils; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Map; + public class PluginLoaderTest { @Before @@ -71,7 +72,7 @@ public class PluginLoaderTest { DigitalVersion.JDK_1_8_0, "plugin.PluginTest", "libtest.so", "plugin_test.jar"); DynamicPluginLoader util = new DynamicPluginLoader(PluginTestUtil.getTestPathString(""), info); - Plugin p = util.dynamicLoadPlugin(PluginTestUtil.getTestPath("")); + Plugin p = util.dynamicLoadPlugin(); p.init(null, null); p.close(); diff --git a/fe/src/test/java/org/apache/doris/plugin/PluginMgrTest.java b/fe/src/test/java/org/apache/doris/plugin/PluginMgrTest.java index 547e4206da..6f046e754c 100644 --- a/fe/src/test/java/org/apache/doris/plugin/PluginMgrTest.java +++ b/fe/src/test/java/org/apache/doris/plugin/PluginMgrTest.java @@ -20,71 +20,51 @@ package org.apache.doris.plugin; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import org.apache.doris.analysis.InstallPluginStmt; +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.Config; +import org.apache.doris.common.UserException; +import org.apache.doris.common.io.DataOutputBuffer; +import org.apache.doris.utframe.UtFrameUtils; + +import org.apache.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; import java.nio.file.Files; - -import org.apache.commons.io.FileUtils; -import org.apache.doris.analysis.InstallPluginStmt; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.FakeCatalog; -import org.apache.doris.common.Config; -import org.apache.doris.common.FeConstants; -import org.apache.doris.common.UserException; -import org.apache.doris.common.io.DataOutputBuffer; -import org.apache.doris.common.jmockit.Deencapsulation; -import org.junit.Before; -import org.junit.Test; +import java.util.UUID; public class PluginMgrTest { - class TestPlugin extends Plugin implements AuditPlugin { + private static String runningDir = "fe/mocked/PluginMgrTest/" + UUID.randomUUID().toString() + "/"; - @Override - public boolean eventFilter(short type, short masks) { - return type == AuditEvent.AUDIT_CONNECTION; - } - - @Override - public void exec(AuditEvent event) { - } + @BeforeClass + public static void beforeClass() throws Exception { + UtFrameUtils.createMinDorisCluster(runningDir); } - - private Catalog catalog; - - private FakeCatalog fakeCatalog; - - private PluginMgr mgr; + @AfterClass + public static void tearDown() { + File file = new File(runningDir); + file.delete(); + } @Before - public void setUp() { - try { - fakeCatalog = new FakeCatalog(); - catalog = Deencapsulation.newInstance(Catalog.class); - - FakeCatalog.setCatalog(catalog); - FakeCatalog.setMetaVersion(FeConstants.meta_version); - - FileUtils.deleteQuietly(PluginTestUtil.getTestFile("target")); - assertFalse(Files.exists(PluginTestUtil.getTestPath("target"))); - Files.createDirectory(PluginTestUtil.getTestPath("target")); - assertTrue(Files.exists(PluginTestUtil.getTestPath("target"))); - - Config.plugin_dir = PluginTestUtil.getTestPathString("target"); - - mgr = new PluginMgr(); - PluginInfo info = new PluginInfo("TestPlugin", PluginInfo.PluginType.AUDIT, "use for test"); - assertTrue(mgr.registerPlugin(info, new TestPlugin())); - - } catch (IOException e) { - e.printStackTrace(); - } + public void setUp() throws IOException { + FileUtils.deleteQuietly(PluginTestUtil.getTestFile("target")); + assertFalse(Files.exists(PluginTestUtil.getTestPath("target"))); + Files.createDirectory(PluginTestUtil.getTestPath("target")); + assertTrue(Files.exists(PluginTestUtil.getTestPath("target"))); + Config.plugin_dir = PluginTestUtil.getTestPathString("target"); } @Test @@ -94,30 +74,31 @@ public class PluginMgrTest { assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar"))); InstallPluginStmt stmt = new InstallPluginStmt(PluginTestUtil.getTestPathString("auditdemo.zip")); - mgr.installPlugin(stmt); + Catalog.getCurrentCatalog().installPlugin(stmt); - assertEquals(2, mgr.getActivePluginList(PluginInfo.PluginType.AUDIT).size()); + PluginMgr pluginMgr = Catalog.getCurrentPluginMgr(); - Plugin p = mgr.getActivePlugin("audit_plugin_demo", PluginInfo.PluginType.AUDIT); + assertEquals(2, pluginMgr.getActivePluginList(PluginInfo.PluginType.AUDIT).size()); + + Plugin p = pluginMgr.getActivePlugin("audit_plugin_demo", PluginInfo.PluginType.AUDIT); assertNotNull(p); assertTrue(p instanceof AuditPlugin); - assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_QUERY, AuditEvent.AUDIT_QUERY_START)); - assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_QUERY, AuditEvent.AUDIT_QUERY_END)); - assertFalse(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_CONNECTION, AuditEvent.AUDIT_QUERY_END)); + assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.EventType.AFTER_QUERY)); + assertFalse(((AuditPlugin) p).eventFilter(AuditEvent.EventType.BEFORE_QUERY)); assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo"))); assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar"))); - assertEquals(1, mgr.getAllDynamicPluginInfo().size()); - PluginInfo info = mgr.getAllDynamicPluginInfo().get(0); + assertEquals(1, pluginMgr.getAllDynamicPluginInfo().size()); + PluginInfo info = pluginMgr.getAllDynamicPluginInfo().get(0); assertEquals("audit_plugin_demo", info.getName()); assertEquals(PluginInfo.PluginType.AUDIT, info.getType()); assertEquals("just for test", info.getDescription()); assertEquals("plugin.AuditPluginDemo", info.getClassName()); - mgr.uninstallPlugin("audit_plugin_demo"); + pluginMgr.uninstallPlugin("audit_plugin_demo"); assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo"))); assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar"))); @@ -139,25 +120,27 @@ public class PluginMgrTest { PluginTestUtil.getTestPath("test_plugin").toFile()); InstallPluginStmt stmt = new InstallPluginStmt(PluginTestUtil.getTestPathString("test_plugin")); - mgr.installPlugin(stmt); + Catalog.getCurrentCatalog().installPlugin(stmt); + + PluginMgr pluginMgr = Catalog.getCurrentPluginMgr(); assertFalse(Files.exists(PluginTestUtil.getTestPath("test_plugin"))); assertFalse(Files.exists(PluginTestUtil.getTestPath("test_plugin/auditdemo.jar"))); - Plugin p = mgr.getActivePlugin("audit_plugin_demo", PluginInfo.PluginType.AUDIT); + Plugin p = pluginMgr.getActivePlugin("audit_plugin_demo", PluginInfo.PluginType.AUDIT); - assertEquals(2, mgr.getActivePluginList(PluginInfo.PluginType.AUDIT).size()); + assertEquals(2, pluginMgr.getActivePluginList(PluginInfo.PluginType.AUDIT).size()); assertNotNull(p); assertTrue(p instanceof AuditPlugin); - assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_QUERY, AuditEvent.AUDIT_QUERY_START)); - assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_QUERY, AuditEvent.AUDIT_QUERY_END)); - assertFalse(((AuditPlugin) p).eventFilter(AuditEvent.AUDIT_CONNECTION, AuditEvent.AUDIT_QUERY_END)); + assertTrue(((AuditPlugin) p).eventFilter(AuditEvent.EventType.AFTER_QUERY)); + assertFalse(((AuditPlugin) p).eventFilter(AuditEvent.EventType.BEFORE_QUERY)); assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo"))); assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar"))); - mgr.uninstallPlugin("audit_plugin_demo"); + testSerializeBuiltinPlugin(pluginMgr); + pluginMgr.uninstallPlugin("audit_plugin_demo"); assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo"))); assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar"))); @@ -168,10 +151,7 @@ public class PluginMgrTest { } } - - @Test - public void testSerializeBuiltinPlugin() { - + private void testSerializeBuiltinPlugin(PluginMgr mgr) { try { DataOutputBuffer dob = new DataOutputBuffer(); DataOutputStream dos = new DataOutputStream(dob); @@ -180,47 +160,10 @@ public class PluginMgrTest { PluginMgr test = new PluginMgr(); test.readFields(new DataInputStream(new ByteArrayInputStream(dob.getData()))); - assertEquals(0, test.getAllDynamicPluginInfo().size()); + assertEquals(1, test.getAllDynamicPluginInfo().size()); } catch (IOException e) { e.printStackTrace(); } } - - @Test - public void testSerializeDynamicPlugin() { - try { - assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo"))); - assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar"))); - - InstallPluginStmt stmt = new InstallPluginStmt(PluginTestUtil.getTestPathString("auditdemo.zip")); - mgr.installPlugin(stmt); - - assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo"))); - assertTrue(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar"))); - - assertEquals(1, mgr.getAllDynamicPluginInfo().size()); - - DataOutputBuffer dob = new DataOutputBuffer(); - DataOutputStream dos = new DataOutputStream(dob); - mgr.write(dos); - - PluginMgr test = new PluginMgr(); - - assertNotNull(dob); - assertNotNull(test); - DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dob.getData())); - test.readFields(dis); - assertEquals(1, test.getAllDynamicPluginInfo().size()); - - mgr.uninstallPlugin("audit_plugin_demo"); - - assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo"))); - assertFalse(Files.exists(PluginTestUtil.getTestPath("target/audit_plugin_demo/auditdemo.jar"))); - } catch (IOException | UserException e) { - e.printStackTrace(); - } - } - - } diff --git a/fe/src/test/java/org/apache/doris/qe/AuditEventProcessorTest.java b/fe/src/test/java/org/apache/doris/qe/AuditEventProcessorTest.java new file mode 100644 index 0000000000..3d2a45ad29 --- /dev/null +++ b/fe/src/test/java/org/apache/doris/qe/AuditEventProcessorTest.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.util.DigitalVersion; +import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.plugin.PluginInfo; +import org.apache.doris.plugin.AuditEvent.AuditEventBuilder; +import org.apache.doris.plugin.AuditEvent.EventType; +import org.apache.doris.qe.AuditEventProcessor; +import org.apache.doris.qe.AuditLogBuilder; +import org.apache.doris.utframe.UtFrameUtils; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +public class AuditEventProcessorTest { + + private static String runningDir = "fe/mocked/AuditProcessorTest/" + UUID.randomUUID().toString() + "/"; + + @BeforeClass + public static void beforeClass() throws Exception { + UtFrameUtils.createMinDorisCluster(runningDir); + } + + @AfterClass + public static void tearDown() { + File file = new File(runningDir); + file.delete(); + } + + @Test + public void testAuditEvent() { + AuditEvent event = new AuditEvent.AuditEventBuilder().setEventType(EventType.AFTER_QUERY) + .setTimestamp(System.currentTimeMillis()) + .setClientIp("127.0.0.1") + .setUser("user1") + .setDb("db1") + .setState("EOF") + .setQueryTime(2000) + .setScanBytes(100000) + .setScanRows(200000) + .setReturnRows(1) + .setStmtId(1234) + .setStmt("select * from tbl1").build(); + + Assert.assertEquals("127.0.0.1", event.clientIp); + Assert.assertEquals(200000, event.scanRows); + } + + @Test + public void testAuditLogBuilder() throws IOException { + try (AuditLogBuilder auditLogBuilder = new AuditLogBuilder()) { + PluginInfo pluginInfo = auditLogBuilder.getPluginInfo(); + Assert.assertEquals(DigitalVersion.fromString("0.12.0"), pluginInfo.getVersion()); + Assert.assertEquals(DigitalVersion.fromString("1.8.31"), pluginInfo.getJavaVersion()); + long start = System.currentTimeMillis(); + for (int i = 0; i < 10000; i++) { + AuditEvent event = new AuditEvent.AuditEventBuilder().setEventType(EventType.AFTER_QUERY) + .setTimestamp(System.currentTimeMillis()) + .setClientIp("127.0.0.1") + .setUser("user1") + .setDb("db1") + .setState("EOF") + .setQueryTime(2000) + .setScanBytes(100000) + .setScanRows(200000) + .setReturnRows(i) + .setStmtId(1234) + .setStmt("select * from tbl1").build(); + if (auditLogBuilder.eventFilter(event.type)) { + auditLogBuilder.exec(event); + } + } + long total = System.currentTimeMillis() - start; + System.out.println("total(ms): " + total + ", avg: " + total / 10000.0); + } + } + + @Test + public void testAuditEventProcessor() throws IOException { + AuditEventProcessor processor = Catalog.getCurrentAuditEventProcessor(); + long start = System.currentTimeMillis(); + for (int i = 0; i < 10000; i++) { + AuditEvent event = new AuditEvent.AuditEventBuilder().setEventType(EventType.AFTER_QUERY) + .setTimestamp(System.currentTimeMillis()) + .setClientIp("127.0.0.1") + .setUser("user1") + .setDb("db1") + .setState("EOF") + .setQueryTime(2000) + .setScanBytes(100000) + .setScanRows(200000) + .setReturnRows(i) + .setStmtId(1234) + .setStmt("select * from tbl1").build(); + processor.handleAuditEvent(event); + } + long total = System.currentTimeMillis() - start; + System.out.println("total(ms): " + total + ", avg: " + total / 10000.0); + } +} \ No newline at end of file diff --git a/fe/src/test/java/org/apache/doris/qe/ConnectProcessorTest.java b/fe/src/test/java/org/apache/doris/qe/ConnectProcessorTest.java index 641c3f6b4d..678751b608 100644 --- a/fe/src/test/java/org/apache/doris/qe/ConnectProcessorTest.java +++ b/fe/src/test/java/org/apache/doris/qe/ConnectProcessorTest.java @@ -17,8 +17,6 @@ package org.apache.doris.qe; -import mockit.Expectations; -import mockit.Mocked; import org.apache.doris.analysis.AccessTestUtil; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.jmockit.Deencapsulation; @@ -29,6 +27,7 @@ import org.apache.doris.mysql.MysqlEofPacket; import org.apache.doris.mysql.MysqlErrPacket; import org.apache.doris.mysql.MysqlOkPacket; import org.apache.doris.mysql.MysqlSerializer; +import org.apache.doris.plugin.AuditEvent.AuditEventBuilder; import org.apache.doris.proto.PQueryStatistics; import org.apache.doris.thrift.TUniqueId; @@ -41,13 +40,16 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.SocketChannel; +import mockit.Expectations; +import mockit.Mocked; + public class ConnectProcessorTest { private static ByteBuffer initDbPacket; private static ByteBuffer pingPacket; private static ByteBuffer quitPacket; private static ByteBuffer queryPacket; private static ByteBuffer fieldListPacket; - private static AuditBuilder auditBuilder = new AuditBuilder(); + private static AuditEventBuilder auditBuilder = new AuditEventBuilder(); private static ConnectContext myContext; @Mocked @@ -202,7 +204,7 @@ public class ConnectProcessorTest { minTimes = 0; result = catalog; - context.getAuditBuilder(); + context.getAuditEventBuilder(); minTimes = 0; result = auditBuilder; diff --git a/fe/src/test/java/org/apache/doris/utframe/UtFrameUtils.java b/fe/src/test/java/org/apache/doris/utframe/UtFrameUtils.java index d7d9b637e1..ac538e7324 100644 --- a/fe/src/test/java/org/apache/doris/utframe/UtFrameUtils.java +++ b/fe/src/test/java/org/apache/doris/utframe/UtFrameUtils.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; import org.apache.doris.common.util.SqlParserUtils; @@ -126,6 +127,7 @@ public class UtFrameUtils { if (Strings.isNullOrEmpty(dorisHome)) { dorisHome = Files.createTempDirectory("DORIS_HOME").toAbsolutePath().toString(); } + Config.plugin_dir = dorisHome + "/plugins"; int fe_http_port = findValidPort(); int fe_rpc_port = findValidPort(); diff --git a/fe/src/test/resources/plugin_test/auditdemo.zip b/fe/src/test/resources/plugin_test/auditdemo.zip index a2787e576a..6c1c2968de 100644 Binary files a/fe/src/test/resources/plugin_test/auditdemo.zip and b/fe/src/test/resources/plugin_test/auditdemo.zip differ diff --git a/fe/src/test/resources/plugin_test/test_local_plugin/auditdemo.jar b/fe/src/test/resources/plugin_test/test_local_plugin/auditdemo.jar index 5fa61f43cf..c4b3ffad7a 100644 Binary files a/fe/src/test/resources/plugin_test/test_local_plugin/auditdemo.jar and b/fe/src/test/resources/plugin_test/test_local_plugin/auditdemo.jar differ diff --git a/fe_plugins/README.md b/fe_plugins/README.md deleted file mode 100644 index 574e0d3956..0000000000 --- a/fe_plugins/README.md +++ /dev/null @@ -1,245 +0,0 @@ -# README -[TOC] -## Introduction -fe_plugins is the parent module of the fe plugins. It can uniformly manage the third-party library information that - the plugin depends on. Adding a plugin can add a submodule implementation under fe_plugins - -## Plugin - -A FE Plugin can be a .zip package or a directory, which contains at least two parts: the plugin.properties and .jar files. The plugin.properties file is used to describe the plugin information. - -The file structure of a Plugin looks like this: -``` -# plugin .zip -auditodemo.zip: --plugin.properties --auditdemo.jar --xxx.config --data \ --test_data - -# plugin local directory -auditodemo /: --plugin.properties --auditdemo.jar --xxx.config --data \ --test_data -``` - -plugin.properties example: - -``` -### required: -# -# the plugin name -name = audit_plugin_demo -# -# the plugin type -type = AUDIT -# -# simple summary of the plugin -description = just for test -# -# Doris's version, like: 0.11.0 -version = 0.11.0 - -### FE-Plugin optional: -# -# version of java the code is built against -# use the command "java -version" value, like 1.8.0, 9.0.1, 13.0.4 -java.version = 1.8.31 -# -# the name of the class to load, fully-qualified. -classname = plugin.AuditPluginDemo - -### BE-Plugin optional: -# the name of the so to load -soName = example.so -``` - -## Write A Plugin -### create module -We can add a submodule in the fe_plugins directory to implement Plugin and create a project: -``` -mvn archetype: generate -DarchetypeCatalog = internal -DgroupId = org.apache -DartifactId = doris-fe-test -DinteractiveMode = false -``` - -The command produces a new mvn project, and a new submodule is automatically added to fe_plugins / pom.xml: -``` -    ..... -     org.apache -     doris-fe-plugins -     pom -     1.0-SNAPSHOT -     -         auditdemo -        # new plugin module -         doris-fe-test -     - -     -         $ {basedir} /../../ -     -    ..... -``` - -The new plugin project file structure is as follows: -``` --doris-fe-test / --pom.xml --src / ----- main / java / org / apache / -------App.java # mvn auto generate, ignore ----- test / java / org / apache -``` - -We will add an assembly folder under main to store plugin.properties and zip.xml. After completion, the file structure is as follows: -``` --doris-fe-test / --pom.xml --src / ----- main / ------- assembly / --------- plugin.properties --------- zip.xml ------- java / org / apache / ---------App.java # mvn auto generate, ignore ----- test / java / org / apache -``` - -### add zip.xml -zip.xml, used to describe the content of the final package of the plugin (.jar file, plugin.properties): - -``` - -     plugin -     -         zip -     -     -     false -     -         -             target -             -                 *. jar -             -             / -         - -         -             src / main / assembly -             -                 plugin.properties -             -             / -         -     - -``` - - -### update pom.xml -Then we need to update pom.xml, add doris-fe dependency, and modify maven packaging way: -``` - - -     -         org.apache -         doris-fe-test -         1.0-SNAPSHOT -     -     4.0.0 - - -     doris-fe-test -     jar -     -         -             org.apache -             doris-fe -         -     -     -         doris-fe-test -         -             -                 maven-assembly-plugin -                 2.4.1 -                 -                     false -                     -                         src / main / assembly / zip.xml -                     -                 -                 -                     -                         make-assembly -                         package -                         -                             single -                         -                     -                 -             -         -     - - -``` - -### implement plugin -Then we can happily implement Plugin according to the needs - -### compile -Before compiling the plugin, you must first execute `sh build.sh --fe` of Doris to complete the compilation of Doris FE. - -Finally, execute `sh build_plugin.sh` in the ${DORIS_HOME} path and you will find the plugin .zip file in fe_plugins/output - -### other way -The easiest way, you can implement your plugin by modifying an example - -## Deploy -Doris's plugin can be deployed in three ways: - -* Http or Https .zip, like http://xxx.xxxxxx.com/data/plugin.zip, Doris will download this .zip file -* Local .zip, like /home/work/data/plugin.zip, need to be deployed on all FE and BE nodes -* Local directory, like /home/work/data/plugin, .zip decompressed folder, need to be deployed on all FE, BE nodes - -Note: Need to ensure that the plugin .zip file is available in the life cycle of doris! - -## Use - -Install and uninstall the plugin through the install/uninstall statements: - -``` -mysql> -mysql> -mysql> install plugin from "/home/users/seaven/auditdemo.zip"; -Query OK, 0 rows affected (0.09 sec) - -mysql> -mysql> -mysql> -mysql> show plugins; -+ ------------------- + ------- + --------------- + ----- ---- + ------------- + ------------------------ + ------ -+ --------------------------------- + ----------- + -Name | Type | Description | Version | JavaVersion | ClassName | SoName | Sources | -+ ------------------- + ------- + --------------- + ----- ---- + ------------- + ------------------------ + ------ -+ --------------------------------- + ----------- + -audit_plugin_demo | AUDIT | just for test | 0.11.0 | 1.8.31 | plugin.AuditPluginDemo | NULL | /home/users/hekai/auditdemo.zip | INSTALLED | -+ ------------------- + ------- + --------------- + ----- ---- + ------------- + ------------------------ + ------ -+ --------------------------------- + ----------- + -1 row in set (0.02 sec) -mysql> -mysql> -mysql> -mysql> uninstall plugin audit_plugin_demo; -Query OK, 0 rows affected (0.05 sec) - -mysql> show plugins; -Empty set (0.00 sec) - -mysql> - -``` \ No newline at end of file diff --git a/fe_plugins/auditdemo/pom.xml b/fe_plugins/auditdemo/pom.xml index 0b4df42392..37403eb371 100644 --- a/fe_plugins/auditdemo/pom.xml +++ b/fe_plugins/auditdemo/pom.xml @@ -1,6 +1,6 @@ - org.apache @@ -18,7 +18,6 @@ doris-fe - org.apache.logging.log4j @@ -71,4 +70,4 @@ - \ No newline at end of file + diff --git a/fe_plugins/auditdemo/src/main/assembly/plugin.properties b/fe_plugins/auditdemo/src/main/assembly/plugin.properties index 70e19b4511..c51d54fda4 100755 --- a/fe_plugins/auditdemo/src/main/assembly/plugin.properties +++ b/fe_plugins/auditdemo/src/main/assembly/plugin.properties @@ -11,7 +11,6 @@ description=just for test # # Doris's version, like: 0.11.0 version=0.11.0 - ### FE-Plugin optional: # # version of java the code is built against @@ -19,8 +18,7 @@ version=0.11.0 java.version=1.8.31 # # the name of the class to load, fully-qualified. -classname=plugin.AuditPluginDemo - +classname=org.apache.doris.plugin.audit.AuditPluginDemo ### BE-Plugin optional: # # \ No newline at end of file diff --git a/fe_plugins/auditdemo/src/main/java/plugin/AuditPluginDemo.java b/fe_plugins/auditdemo/src/main/java/org/apache/doris/plugin/audit/AuditPluginDemo.java similarity index 72% rename from fe_plugins/auditdemo/src/main/java/plugin/AuditPluginDemo.java rename to fe_plugins/auditdemo/src/main/java/org/apache/doris/plugin/audit/AuditPluginDemo.java index 886c834031..5cbb4f028b 100755 --- a/fe_plugins/auditdemo/src/main/java/plugin/AuditPluginDemo.java +++ b/fe_plugins/auditdemo/src/main/java/org/apache/doris/plugin/audit/AuditPluginDemo.java @@ -15,25 +15,25 @@ // specific language governing permissions and limitations // under the License. -package plugin; - -import java.io.IOException; +package org.apache.doris.plugin.audit; import org.apache.doris.plugin.AuditEvent; import org.apache.doris.plugin.AuditPlugin; import org.apache.doris.plugin.Plugin; import org.apache.doris.plugin.PluginContext; +import org.apache.doris.plugin.PluginException; import org.apache.doris.plugin.PluginInfo; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.IOException; + public class AuditPluginDemo extends Plugin implements AuditPlugin { private final static Logger LOG = LogManager.getLogger(AuditPluginDemo.class); - private final static short EVENT_MASKS = AuditEvent.AUDIT_QUERY_START | AuditEvent.AUDIT_QUERY_END; - @Override - public void init(PluginInfo info, PluginContext ctx) { + public void init(PluginInfo info, PluginContext ctx) throws PluginException { super.init(info, ctx); LOG.info("this is audit plugin demo init"); } @@ -44,17 +44,11 @@ public class AuditPluginDemo extends Plugin implements AuditPlugin { LOG.info("this is audit plugin demo close"); } - @Override - public boolean eventFilter(short type, short masks) { - return type == AuditEvent.AUDIT_QUERY && (masks & EVENT_MASKS) != 0; + public boolean eventFilter(AuditEvent.EventType type) { + return type == AuditEvent.EventType.AFTER_QUERY; } - @Override public void exec(AuditEvent event) { - LOG.info("audit demo plugin log: event={} masks={} user={}, ip={}, query={}", event.getEventType(), - event.getEventMasks(), - event.getUser(), - event.getIp(), - event.getQuery()); + LOG.info("audit demo plugin log exec"); } } diff --git a/fe_plugins/auditloader/pom.xml b/fe_plugins/auditloader/pom.xml new file mode 100644 index 0000000000..655ac69507 --- /dev/null +++ b/fe_plugins/auditloader/pom.xml @@ -0,0 +1,73 @@ + + + + org.apache + doris-fe-plugins + 1.0-SNAPSHOT + + 4.0.0 + + auditloader + jar + + + + org.apache + doris-fe + + + + + org.apache.logging.log4j + log4j-api + + + + + org.apache.logging.log4j + log4j-core + + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + log4j + log4j + + + + + + auditloader + + + + maven-assembly-plugin + 2.4.1 + + false + + src/main/assembly/zip.xml + + + + + make-assembly + package + + single + + + + + + + + diff --git a/fe_plugins/auditloader/src/main/assembly/plugin.conf b/fe_plugins/auditloader/src/main/assembly/plugin.conf new file mode 100755 index 0000000000..3f2e90cad1 --- /dev/null +++ b/fe_plugins/auditloader/src/main/assembly/plugin.conf @@ -0,0 +1,24 @@ +### plugin configuration + +# The max size of a batch, default is 50MB +max_batch_size=52428800 + +# The max interval of batch loaded, default is 60 seconds +max_batch_interval_sec=60 + +# Doris FE host for loading the audit, default is 127.0.0.1:8030. +# this should be the host port for stream load +frontend_host_port=127.0.0.1:8030 + +# Database of the audit table +database=doris_audit_db__ + +# Audit table name, to save the audit data. +table=doris_audit_tbl__ + +# Doris user. This user must have LOAD_PRIV to the audit table. +user=root + +# Doris user's password +password= + diff --git a/fe_plugins/auditloader/src/main/assembly/plugin.properties b/fe_plugins/auditloader/src/main/assembly/plugin.properties new file mode 100755 index 0000000000..05cb6c7faf --- /dev/null +++ b/fe_plugins/auditloader/src/main/assembly/plugin.properties @@ -0,0 +1,6 @@ +name=AuditLoader +type=AUDIT +description=load audit log to olap load, and user can view the statistic of queries +version=0.12.0 +java.version=1.8.0 +classname=org.apache.doris.plugin.audit.AuditLoaderPlugin \ No newline at end of file diff --git a/fe_plugins/auditloader/src/main/assembly/zip.xml b/fe_plugins/auditloader/src/main/assembly/zip.xml new file mode 100644 index 0000000000..9fb92265cc --- /dev/null +++ b/fe_plugins/auditloader/src/main/assembly/zip.xml @@ -0,0 +1,25 @@ + + plugin + + zip + + false + + + target + + *.jar + + / + + + + src/main/assembly + + plugin.properties + plugin.conf + + / + + + diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java new file mode 100755 index 0000000000..c2d8c78dfc --- /dev/null +++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.plugin.audit; + +import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.plugin.AuditPlugin; +import org.apache.doris.plugin.Plugin; +import org.apache.doris.plugin.PluginContext; +import org.apache.doris.plugin.PluginException; +import org.apache.doris.plugin.PluginInfo; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; +import java.util.Properties; +import java.util.TimeZone; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +/* + * This plugin will load audit log to specified doris table at specified interval + */ +public class AuditLoaderPlugin extends Plugin implements AuditPlugin { + private final static Logger LOG = LogManager.getLogger(AuditLoaderPlugin.class); + + private static SimpleDateFormat DATETIME_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + private StringBuilder auditBuffer = new StringBuilder(); + private long lastLoadTime = 0; + + private BlockingQueue batchQueue = new LinkedBlockingDeque(1); + private DorisStreamLoader streamLoader; + private Thread loadThread; + + private AuditLoaderConf conf; + private volatile boolean isClosed = false; + private volatile boolean isInit = false; + + @Override + public void init(PluginInfo info, PluginContext ctx) throws PluginException { + super.init(info, ctx); + + synchronized (this) { + if (isInit) { + return; + } + this.lastLoadTime = System.currentTimeMillis(); + + loadConfig(ctx); + + this.streamLoader = new DorisStreamLoader(conf); + this.loadThread = new Thread(new LoadWorker(this.streamLoader), "audit loader thread"); + this.loadThread.start(); + } + } + + private void loadConfig(PluginContext ctx) throws PluginException { + Path pluginPath = FileSystems.getDefault().getPath(ctx.getPluginPath()); + if (!Files.exists(pluginPath)) { + throw new PluginException("plugin path does not exist: " + pluginPath); + } + + Path confFile = pluginPath.resolve("plugin.conf"); + if (!Files.exists(confFile)) { + throw new PluginException("plugin conf file does not exist: " + confFile); + } + + final Properties props = new Properties(); + try (InputStream stream = Files.newInputStream(confFile)) { + props.load(stream); + } catch (IOException e) { + throw new PluginException(e.getMessage()); + } + final Map properties = props.stringPropertyNames().stream() + .collect(Collectors.toMap(Function.identity(), props::getProperty)); + + conf = new AuditLoaderConf(); + conf.init(properties); + } + + @Override + public void close() throws IOException { + super.close(); + isClosed = true; + if (loadThread != null) { + try { + loadThread.join(); + } catch (InterruptedException e) { + LOG.debug("encounter exception when closing the audit loader", e); + } + } + } + + public boolean eventFilter(AuditEvent.EventType type) { + return type == AuditEvent.EventType.AFTER_QUERY; + } + + public void exec(AuditEvent event) { + assembleAudit(event); + loadIfNecessary(); + } + + private void assembleAudit(AuditEvent event) { + auditBuffer.append(event.queryId).append("\t"); + auditBuffer.append(longToTimeString(event.timestamp)).append("\t"); + auditBuffer.append(event.clientIp).append("\t"); + auditBuffer.append(event.user).append("\t"); + auditBuffer.append(event.db).append("\t"); + auditBuffer.append(event.state).append("\t"); + auditBuffer.append(event.queryTime).append("\t"); + auditBuffer.append(event.scanBytes).append("\t"); + auditBuffer.append(event.scanRows).append("\t"); + auditBuffer.append(event.returnRows).append("\t"); + auditBuffer.append(event.stmtId).append("\t"); + auditBuffer.append(event.isQuery ? 1 : 0).append("\t"); + auditBuffer.append(event.feIp).append("\t"); + // trim the query to avoid too long + int maxLen = Math.min(2048, event.stmt.length()); + String stmt = event.stmt.substring(0, maxLen).replace("\t", " "); + LOG.debug("receive audit event with stmt: {}", stmt); + auditBuffer.append(stmt).append("\n"); + } + + private void loadIfNecessary() { + if (auditBuffer.length() < conf.maxBatchSize && System.currentTimeMillis() - lastLoadTime < conf.maxBatchIntervalSec * 1000) { + return; + } + + lastLoadTime = System.currentTimeMillis(); + // begin to load + try { + if (!batchQueue.isEmpty()) { + // TODO(cmy): if queue is not empty, which means the last batch is not processed. + // In order to ensure that the system can run normally, here we directly + // discard the current batch. If this problem occurs frequently, + // improvement can be considered. + throw new PluginException("The previous batch is not processed, and the current batch is discarded."); + } + + batchQueue.put(this.auditBuffer); + } catch (Exception e) { + LOG.debug("encounter exception when putting current audit batch, discard current batch", e); + } finally { + // make a new string builder to receive following events. + this.auditBuffer = new StringBuilder(); + } + + return; + } + + public static class AuditLoaderConf { + public static final String PROP_MAX_BATCH_SIZE = "max_batch_size"; + public static final String PROP_MAX_BATCH_INTERVAL_SEC = "max_batch_interval_sec"; + public static final String PROP_FRONTEND_HOST_PORT = "frontend_host_port"; + public static final String PROP_USER = "user"; + public static final String PROP_PASSWORD = "password"; + public static final String PROP_DATABASE = "database"; + public static final String PROP_TABLE = "table"; + + public long maxBatchSize = 50 * 1024 * 1024; + public long maxBatchIntervalSec = 60; + public String frontendHostPort = "127.0.0.1:9030"; + public String user = "root"; + public String password = ""; + public String database = "__doris_audit_db"; + public String table = "__doris_audit_tbl"; + + public void init(Map properties) throws PluginException { + try { + if (properties.containsKey(PROP_MAX_BATCH_SIZE)) { + maxBatchSize = Long.valueOf(properties.get(PROP_MAX_BATCH_SIZE)); + } + if (properties.containsKey(PROP_MAX_BATCH_INTERVAL_SEC)) { + maxBatchIntervalSec = Long.valueOf(properties.get(PROP_MAX_BATCH_INTERVAL_SEC)); + } + if (properties.containsKey(PROP_FRONTEND_HOST_PORT)) { + frontendHostPort = properties.get(PROP_FRONTEND_HOST_PORT); + } + if (properties.containsKey(PROP_USER)) { + user = properties.get(PROP_USER); + } + if (properties.containsKey(PROP_PASSWORD)) { + password = properties.get(PROP_PASSWORD); + } + if (properties.containsKey(PROP_DATABASE)) { + database = properties.get(PROP_DATABASE); + } + if (properties.containsKey(PROP_TABLE)) { + table = properties.get(PROP_TABLE); + } + } catch (Exception e) { + throw new PluginException(e.getMessage()); + } + } + } + + private class LoadWorker implements Runnable { + private DorisStreamLoader loader; + + public LoadWorker(DorisStreamLoader loader) { + this.loader = loader; + } + + public void run() { + while (!isClosed) { + try { + StringBuilder batch = batchQueue.poll(5, TimeUnit.SECONDS); + if (batch == null) { + continue; + } + + DorisStreamLoader.LoadResponse response = loader.loadBatch(batch); + LOG.debug("audit loader response: {}", response); + } catch (InterruptedException e) { + LOG.debug("encounter exception when loading current audit batch", e); + continue; + } + } + } + } + + public static synchronized String longToTimeString(long timeStamp) { + if (timeStamp <= 0L) { + return "1900-01-01 00:00:00"; + } + return DATETIME_FORMAT.format(new Date(timeStamp)); + } +} diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java new file mode 100644 index 0000000000..8dbac8f523 --- /dev/null +++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.plugin.audit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Calendar; + +public class DorisStreamLoader { + private final static Logger LOG = LogManager.getLogger(DorisStreamLoader.class); + private static String loadUrlPattern = "http://%s/api/%s/%s/_stream_load?"; + private String hostPort; + private String db; + private String tbl; + private String user; + private String passwd; + private String loadUrlStr; + private String authEncoding; + + public DorisStreamLoader(AuditLoaderPlugin.AuditLoaderConf conf) { + this.hostPort = conf.frontendHostPort; + this.db = conf.database; + this.tbl = conf.table; + this.user = conf.user; + this.passwd = conf.password; + + this.loadUrlStr = String.format(loadUrlPattern, hostPort, db, tbl); + this.authEncoding = Base64.getEncoder().encodeToString(String.format("%s:%s", user, passwd).getBytes(StandardCharsets.UTF_8)); + } + + public static void main(String[] args) { + try { + AuditLoaderPlugin.AuditLoaderConf conf = new AuditLoaderPlugin.AuditLoaderConf(); + conf.frontendHostPort = "fe_host"; + conf.database = "db1"; + conf.table = "tbl1"; + conf.user = "root"; + conf.password = ""; + + DorisStreamLoader loader = new DorisStreamLoader(conf); + + StringBuilder sb = new StringBuilder(); + sb.append("1\t2\n3\t4\n"); + + System.out.println("before load"); + LoadResponse loadResponse = loader.loadBatch(sb); + + System.out.println(loadResponse); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + public LoadResponse loadBatch(StringBuilder sb) { + Calendar calendar = Calendar.getInstance(); + String label = String.format("audit_%s%02d%02d_%02d%02d%02d", + calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH), calendar.get(Calendar.DAY_OF_MONTH), + calendar.get(Calendar.HOUR), calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND)); + + + try { + // build requst + URL loadUrl = new URL(loadUrlStr); + HttpURLConnection conn = (HttpURLConnection) loadUrl.openConnection(); + conn.setRequestMethod("PUT"); + conn.setRequestProperty("Authorization", "Basic " + authEncoding); + conn.addRequestProperty("Expect", "100-continue"); + conn.addRequestProperty("Content-Type", "text/plain; charset=UTF-8"); + + conn.addRequestProperty("label", label); + conn.addRequestProperty("max_fiter_ratio", "1.0"); + + conn.setDoOutput(true); + conn.setDoInput(true); + + // send data + BufferedOutputStream bos = new BufferedOutputStream(conn.getOutputStream()); + bos.write(sb.toString().getBytes()); + bos.close(); + + // get respond + int status = conn.getResponseCode(); + String respMsg = conn.getResponseMessage(); + + InputStream stream = (InputStream) conn.getContent(); + BufferedReader br = new BufferedReader(new InputStreamReader(stream)); + StringBuilder response = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + response.append(line); + } + + LOG.info("AuditLoader plugin load with label: {}, response code: {}, msg: {}, content: {}", + label, status, respMsg, response.toString()); + + return new LoadResponse(status, respMsg, response.toString()); + + } catch (Exception e) { + e.printStackTrace(); + String err = "failed to load audit via AuditLoader plugin with label: " + label; + LOG.warn(err, e); + return new LoadResponse(-1, e.getMessage(), err); + } + } + + public static class LoadResponse { + public int status; + public String respMsg; + public String respContent; + + public LoadResponse(int status, String respMsg, String respContent) { + this.status = status; + this.respMsg = respMsg; + this.respContent = respContent; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("status: ").append(status); + sb.append(", resp msg: ").append(respMsg); + sb.append(", resp content: ").append(respContent); + return sb.toString(); + } + } +} \ No newline at end of file diff --git a/fe_plugins/pom.xml b/fe_plugins/pom.xml index 81dc39584c..f2fcaf02da 100644 --- a/fe_plugins/pom.xml +++ b/fe_plugins/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 org.apache @@ -8,10 +9,11 @@ 1.0-SNAPSHOT auditdemo + auditloader - ${basedir}/../../ + ${env.DORIS_HOME} @@ -79,4 +81,18 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.8 + 1.8 + + + + +