From 70236adc1f0a6289bf1bff874fa156852026200b Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Fri, 5 May 2023 14:42:43 +0800 Subject: [PATCH] [Refactor](doc)(config)(variable) use script to generate doc for FE config and session variables (#19246) The document of configs(FE and BE) and session variables is hard to maintain. Because developer need to modify both code and document. And you can see that some of config's document is missing. So I plan to write the document of config or variables directly in code, and using script to generate document automatically. How To This CL mainly changes: Add field in Config and Session Variables' annaotion description: The description of the config or variable item. It is a String array. And first element is in Chinese, second is in English options: the valid options if the config or variable is enum. Add a scripts docs/generate-config-and-variable-doc.sh Simple run sh docs/generate-config-and-variable-doc.sh and it will generate docs of FE config and variables, And save it under docs/admin-manual/config/fe-config.md and docs/advanced/variables.md, both in Chinese and in English. And there are template markdowns for this script to read and replace with real doc content. TODO Too many description need to be filled. I will finish them in next PR. And now the origin doc remain unchanged. Find a way to check the description field of config and variables, to make sure we won't missing it. Generate doc for BE config. --- .../admin-manual/config/fe-config-template.md | 133 ++ docs/en/docs/advanced/variables-template.md | 116 ++ docs/generate-config-and-variable-doc.sh | 36 + .../admin-manual/config/fe-config-template.md | 132 ++ .../zh-CN/docs/advanced/variables-template.md | 115 ++ .../java/org/apache/doris/common/Config.java | 1159 +++++++---------- .../org/apache/doris/common/ConfigBase.java | 14 +- .../org/apache/doris/common/EnvUtils.java | 32 + fe/fe-core/pom.xml | 26 +- .../doris/common/util/DocGenerator.java | 303 +++++ .../IcebergTableCreationRecordMgr.java | 2 +- .../java/org/apache/doris/load/DeleteJob.java | 2 +- .../org/apache/doris/load/DppScheduler.java | 2 +- .../java/org/apache/doris/load/LoadJob.java | 6 - .../doris/load/loadv2/SparkLoadJob.java | 3 +- .../org/apache/doris/qe/GlobalVariable.java | 2 +- .../org/apache/doris/qe/SessionVariable.java | 2 - .../java/org/apache/doris/qe/VariableMgr.java | 11 +- .../apache/doris/service/FrontendOptions.java | 10 - .../apache/doris/load/DppSchedulerTest.java | 224 ---- 20 files changed, 1361 insertions(+), 969 deletions(-) create mode 100644 docs/en/docs/admin-manual/config/fe-config-template.md create mode 100644 docs/en/docs/advanced/variables-template.md create mode 100755 docs/generate-config-and-variable-doc.sh create mode 100644 docs/zh-CN/docs/admin-manual/config/fe-config-template.md create mode 100644 docs/zh-CN/docs/advanced/variables-template.md create mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/EnvUtils.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/load/DppSchedulerTest.java diff --git a/docs/en/docs/admin-manual/config/fe-config-template.md b/docs/en/docs/admin-manual/config/fe-config-template.md new file mode 100644 index 0000000000..9e4650d0b5 --- /dev/null +++ b/docs/en/docs/admin-manual/config/fe-config-template.md @@ -0,0 +1,133 @@ +--- +{ + "title": "FE Configuration", + "language": "en", + "toc_min_heading_level": 2, + "toc_max_heading_level": 4 +} +--- + + + + + +# FE Configuration + +This document mainly introduces the relevant configuration items of FE. + +The FE configuration file `fe.conf` is usually stored in the `conf/` directory of the FE deployment path. In version 0.14, another configuration file `fe_custom.conf` will be introduced. The configuration file is used to record the configuration items that are dynamically configured and persisted by the user during operation. + +After the FE process is started, it will read the configuration items in `fe.conf` first, and then read the configuration items in `fe_custom.conf`. The configuration items in `fe_custom.conf` will overwrite the same configuration items in `fe.conf`. + +The location of the `fe_custom.conf` file can be configured in `fe.conf` through the `custom_config_dir` configuration item. + +## View configuration items + +There are two ways to view the configuration items of FE: + +1. FE web page + + Open the FE web page `http://fe_host:fe_http_port/variable` in the browser. You can see the currently effective FE configuration items in `Configure Info`. + +2. View by command + + After the FE is started, you can view the configuration items of the FE in the MySQL client with the following command: + + `ADMIN SHOW FRONTEND CONFIG;` + + The meanings of the columns in the results are as follows: + + * Key: the name of the configuration item. + * Value: The value of the current configuration item. + * Type: The configuration item value type, such as integer or string. + * IsMutable: whether it can be dynamically configured. If true, the configuration item can be dynamically configured at runtime. If false, it means that the configuration item can only be configured in `fe.conf` and takes effect after restarting FE. + * MasterOnly: Whether it is a unique configuration item of Master FE node. If it is true, it means that the configuration item is meaningful only at the Master FE node, and is meaningless to other types of FE nodes. If false, it means that the configuration item is meaningful in all types of FE nodes. + * Comment: The description of the configuration item. + +## Set configuration items + +There are two ways to configure FE configuration items: + +1. Static configuration + + Add and set configuration items in the `conf/fe.conf` file. The configuration items in `fe.conf` will be read when the FE process starts. Configuration items not in `fe.conf` will use default values. + +2. Dynamic configuration via MySQL protocol + + After the FE starts, you can set the configuration items dynamically through the following commands. This command requires administrator privilege. + + `ADMIN SET FRONTEND CONFIG (" fe_config_name "=" fe_config_value ");` + + Not all configuration items support dynamic configuration. You can check whether the dynamic configuration is supported by the `IsMutable` column in the` ADMIN SHOW FRONTEND CONFIG; `command result. + + If the configuration item of `MasterOnly` is modified, the command will be directly forwarded to the Master FE and only the corresponding configuration item in the Master FE will be modified. + + **Configuration items modified in this way will become invalid after the FE process restarts.** + + For more help on this command, you can view it through the `HELP ADMIN SET CONFIG;` command. + +3. Dynamic configuration via HTTP protocol + + For details, please refer to [Set Config Action](../http-actions/fe/set-config-action.md) + + This method can also persist the modified configuration items. The configuration items will be persisted in the `fe_custom.conf` file and will still take effect after FE is restarted. + +## Examples + +1. Modify `async_pending_load_task_pool_size` + + Through `ADMIN SHOW FRONTEND CONFIG;` you can see that this configuration item cannot be dynamically configured (`IsMutable` is false). You need to add in `fe.conf`: + + `async_pending_load_task_pool_size = 20` + + Then restart the FE process to take effect the configuration. + +2. Modify `dynamic_partition_enable` + + Through `ADMIN SHOW FRONTEND CONFIG;` you can see that the configuration item can be dynamically configured (`IsMutable` is true). And it is the unique configuration of Master FE. Then first we can connect to any FE and execute the following command to modify the configuration: + + ``` + ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true"); ` + ``` + + Afterwards, you can view the modified value with the following command: + + ``` + set forward_to_master = true; + ADMIN SHOW FRONTEND CONFIG; + ``` + + After modification in the above manner, if the Master FE restarts or a Master election is performed, the configuration will be invalid. You can add the configuration item directly in `fe.conf` and restart the FE to make the configuration item permanent. + +3. Modify `max_distribution_pruner_recursion_depth` + + Through `ADMIN SHOW FRONTEND CONFIG;` you can see that the configuration item can be dynamically configured (`IsMutable` is true). It is not unique to Master FE. + + Similarly, we can modify the configuration by dynamically modifying the configuration command. Because this configuration is not unique to the Master FE, user need to connect to different FEs separately to modify the configuration dynamically, so that all FEs use the modified configuration values. + +## Configurations + +> Note: +> +> The following content is automatically generated by `docs/generate-config-and-variable-doc.sh`. +> +> If you need to modify, please modify the description information in `fe/fe-common/src/main/java/org/apache/doris/common/Config.java`. + +<--DOC_PLACEHOLDER--> diff --git a/docs/en/docs/advanced/variables-template.md b/docs/en/docs/advanced/variables-template.md new file mode 100644 index 0000000000..b6be0253b4 --- /dev/null +++ b/docs/en/docs/advanced/variables-template.md @@ -0,0 +1,116 @@ +--- +{ + "title": "Variable", + "language": "en" +} +--- + + + +# Variable + +This document focuses on currently supported variables. + +Variables in Doris refer to variable settings in MySQL. However, some of the variables are only used to be compatible with some MySQL client protocols, and do not produce their actual meaning in the MySQL database. + +## Variable setting and viewing + +### View + +All or specified variables can be viewed via `SHOW VARIABLES [LIKE 'xxx'];`. Such as: + +``` +SHOW VARIABLES; +SHOW VARIABLES LIKE '%time_zone%'; +``` + +### Settings + +Note that before version 1.1, after the setting takes effect globally, the setting value will be inherited in subsequent new session connections, but the value in the current session will remain unchanged. +After version 1.1 (inclusive), after the setting takes effect globally, the setting value will be used in subsequent new session connections, and the value in the current session will also change. + +For session-only, set by the `SET var_name=xxx;` statement. Such as: + +``` +SET exec_mem_limit = 137438953472; +SET forward_to_master = true; +SET time_zone = "Asia/Shanghai"; +``` + +For global-level, set by `SET GLOBAL var_name=xxx;`. Such as: + +``` +SET GLOBAL exec_mem_limit = 137438953472 +``` + +> Note 1: Only ADMIN users can set variable at global-level. + +Variables that support both session-level and global-level setting include: + +* `time_zone` +* `wait_timeout` +* `sql_mode` +* `enable_profile` +* `query_timeout` +* `insert_timeout` +* `exec_mem_limit` +* `batch_size` +* `parallel_fragment_exec_instance_num` +* `parallel_exchange_instance_num` +* `allow_partition_column_nullable` +* `insert_visible_timeout_ms` +* `enable_fold_constant_by_be` + +Variables that support only global-level setting include: + +* `default_rowset_type` +* `default_password_lifetime` +* `password_history` +* `validate_password_policy` + +At the same time, variable settings also support constant expressions. Such as: + +``` +SET exec_mem_limit = 10 * 1024 * 1024 * 1024; +SET forward_to_master = concat('tr', 'u', 'e'); +``` + +### Set variables in the query statement + +In some scenarios, we may need to set variables specifically for certain queries. +The SET_VAR hint sets the session value of a system variable temporarily (for the duration of a single statement). Examples: + +``` +SELECT /*+ SET_VAR(exec_mem_limit = 8589934592) */ name FROM people ORDER BY name; +SELECT /*+ SET_VAR(query_timeout = 1, enable_partition_cache=true) */ sleep(3); +``` + +Note that the comment must start with /*+ and can only follow the SELECT. + +## Supported variables + +> Note: +> +> The following content is automatically generated by `docs/generate-config-and-variable-doc.sh`. +> +> To modify, please modify `fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java` and `fe/fe-core/src/main/java/org/apache/ Description information in doris/qe/GlobalVariable.java`. + +<--DOC_PLACEHOLDER--> + diff --git a/docs/generate-config-and-variable-doc.sh b/docs/generate-config-and-variable-doc.sh new file mode 100755 index 0000000000..55a9f19328 --- /dev/null +++ b/docs/generate-config-and-variable-doc.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +######################################################################### +# This script is used to generate doc for FE config and session variables +######################################################################### + +set -eo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +export DORIS_HOME="${ROOT}/../" + +"${DORIS_HOME}/generated-source.sh" + +cd "${DORIS_HOME}/fe" +mvn compile -Dskip.clean=true -DskipTests -Dcheckstyle.skip=true exec:java@doc -pl fe-core +cd "${DORIS_HOME}/" + +echo "Done" +exit 0 diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config-template.md b/docs/zh-CN/docs/admin-manual/config/fe-config-template.md new file mode 100644 index 0000000000..025475e3ab --- /dev/null +++ b/docs/zh-CN/docs/admin-manual/config/fe-config-template.md @@ -0,0 +1,132 @@ +--- +{ + "title": "FE 配置项", + "language": "zh-CN", + "toc_min_heading_level": 2, + "toc_max_heading_level": 4 +} + +--- + + + +# Doris FE配置参数 + +该文档主要介绍 FE 的相关配置项。 + +FE 的配置文件 `fe.conf` 通常存放在 FE 部署路径的 `conf/` 目录下。 而在 0.14 版本中会引入另一个配置文件 `fe_custom.conf`。该配置文件用于记录用户在运行时动态配置并持久化的配置项。 + +FE 进程启动后，会先读取 `fe.conf` 中的配置项，之后再读取 `fe_custom.conf` 中的配置项。`fe_custom.conf` 中的配置项会覆盖 `fe.conf` 中相同的配置项。 + +`fe_custom.conf` 文件的位置可以在 `fe.conf` 通过 `custom_config_dir` 配置项配置。 + +## 查看配置项 + +FE 的配置项有两种方式进行查看： + +1. FE 前端页面查看 + + 在浏览器中打开 FE 前端页面 `http://fe_host:fe_http_port/variable`。在 `Configure Info` 中可以看到当前生效的 FE 配置项。 + +2. 通过命令查看 + + FE 启动后，可以在 MySQL 客户端中，通过以下命令查看 FE 的配置项： + + `ADMIN SHOW FRONTEND CONFIG;` + + 结果中各列含义如下： + + - Key：配置项名称。 + - Value：当前配置项的值。 + - Type：配置项值类型，如果整型、字符串。 + - IsMutable：是否可以动态配置。如果为 true，表示该配置项可以在运行时进行动态配置。如果false，则表示该配置项只能在 `fe.conf` 中配置并且重启 FE 后生效。 + - MasterOnly：是否为 Master FE 节点独有的配置项。如果为 true，则表示该配置项仅在 Master FE 节点有意义，对其他类型的 FE 节点无意义。如果为 false，则表示该配置项在所有 FE 节点中均有意义。 + - Comment：配置项的描述。 + +## 设置配置项 + +FE 的配置项有两种方式进行配置： + +1. 静态配置 + + 在 `conf/fe.conf` 文件中添加和设置配置项。`fe.conf` 中的配置项会在 FE 进程启动时被读取。没有在 `fe.conf` 中的配置项将使用默认值。 + +2. 通过 MySQL 协议动态配置 + + FE 启动后，可以通过以下命令动态设置配置项。该命令需要管理员权限。 + + `ADMIN SET FRONTEND CONFIG ("fe_config_name" = "fe_config_value");` + + 不是所有配置项都支持动态配置。可以通过 `ADMIN SHOW FRONTEND CONFIG;` 命令结果中的 `IsMutable` 列查看是否支持动态配置。 + + 如果是修改 `MasterOnly` 的配置项，则该命令会直接转发给 Master FE 并且仅修改 Master FE 中对应的配置项。 + + **通过该方式修改的配置项将在 FE 进程重启后失效。** + + 更多该命令的帮助，可以通过 `HELP ADMIN SET CONFIG;` 命令查看。 + +3. 通过 HTTP 协议动态配置 + + 具体请参阅 [Set Config Action](../http-actions/fe/set-config-action.md) + + 该方式也可以持久化修改后的配置项。配置项将持久化在 `fe_custom.conf` 文件中，在 FE 重启后仍会生效。 + +## 应用举例 + +1. 修改 `async_pending_load_task_pool_size` + + 通过 `ADMIN SHOW FRONTEND CONFIG;` 可以查看到该配置项不能动态配置（`IsMutable` 为 false）。则需要在 `fe.conf` 中添加： + + `async_pending_load_task_pool_size=20` + + 之后重启 FE 进程以生效该配置。 + +2. 修改 `dynamic_partition_enable` + + 通过 `ADMIN SHOW FRONTEND CONFIG;` 可以查看到该配置项可以动态配置（`IsMutable` 为 true）。并且是 Master FE 独有配置。则首先我们可以连接到任意 FE，执行如下命令修改配置： + + ```text + ADMIN SET FRONTEND CONFIG ("dynamic_partition_enable" = "true");` + ``` + + 之后可以通过如下命令查看修改后的值： + + ```text + set forward_to_master=true; + ADMIN SHOW FRONTEND CONFIG; + ``` + + 通过以上方式修改后，如果 Master FE 重启或进行了 Master 切换，则配置将失效。可以通过在 `fe.conf` 中直接添加配置项，并重启 FE 后，永久生效该配置项。 + +3. 修改 `max_distribution_pruner_recursion_depth` + + 通过 `ADMIN SHOW FRONTEND CONFIG;` 可以查看到该配置项可以动态配置（`IsMutable` 为 true）。并且不是 Master FE 独有配置。 + + 同样，我们可以通过动态修改配置的命令修改该配置。因为该配置不是 Master FE 独有配置，所以需要单独连接到不同的 FE，进行动态修改配置的操作，这样才能保证所有 FE 都使用了修改后的配置值 + +## 配置项列表 + +> 注： +> +> 以下内容由 `docs/generate-config-and-variable-doc.sh` 自动生成。 +> +> 如需修改，请修改 `fe/fe-common/src/main/java/org/apache/doris/common/Config.java` 中的描述信息。 + +<--DOC_PLACEHOLDER--> diff --git a/docs/zh-CN/docs/advanced/variables-template.md b/docs/zh-CN/docs/advanced/variables-template.md new file mode 100644 index 0000000000..6cdade26a0 --- /dev/null +++ b/docs/zh-CN/docs/advanced/variables-template.md @@ -0,0 +1,115 @@ +--- +{ + "title": "变量", + "language": "zh-CN" +} +--- + + + +# 变量 + +本文档主要介绍当前支持的变量（variables）。 + +Doris 中的变量参考 MySQL 中的变量设置。但部分变量仅用于兼容一些 MySQL 客户端协议，并不产生其在 MySQL 数据库中的实际意义。 + +## 变量设置与查看 + +### 查看 + +可以通过 `SHOW VARIABLES [LIKE 'xxx'];` 查看所有或指定的变量。如： + +```sql +SHOW VARIABLES; +SHOW VARIABLES LIKE '%time_zone%'; +``` + +### 设置 + +部分变量可以设置全局生效或仅当前会话生效。 + +注意，在 1.1 版本之前，设置全局生效后，后续新的会话连接中会沿用设置值，但当前会话中的值不变。 +而在 1.1 版本（含）之后，设置全局生效后，后续新的会话连接中会沿用设置值，当前会话中的值也会改变。 + +仅当前会话生效，通过 `SET var_name=xxx;` 语句来设置。如： + +```sql +SET exec_mem_limit = 137438953472; +SET forward_to_master = true; +SET time_zone = "Asia/Shanghai"; +``` + +全局生效，通过 `SET GLOBAL var_name=xxx;` 设置。如： + +```sql +SET GLOBAL exec_mem_limit = 137438953472 +``` + +> 注1：只有 ADMIN 用户可以设置变量的全局生效。 + +既支持当前会话生效又支持全局生效的变量包括： + +- `time_zone` +- `wait_timeout` +- `sql_mode` +- `enable_profile` +- `query_timeout` +- `insert_timeout` +- `exec_mem_limit` +- `batch_size` +- `allow_partition_column_nullable` +- `insert_visible_timeout_ms` +- `enable_fold_constant_by_be` + +只支持全局生效的变量包括： + +- `default_rowset_type` +- `default_password_lifetime` +- `password_history` +- `validate_password_policy` + +同时，变量设置也支持常量表达式。如： + +```sql +SET exec_mem_limit = 10 * 1024 * 1024 * 1024; +SET forward_to_master = concat('tr', 'u', 'e'); +``` + +### 在查询语句中设置变量 + +在一些场景中，我们可能需要对某些查询有针对性的设置变量。 通过使用SET_VAR提示可以在查询中设置会话变量（在单个语句内生效）。例子： + +```sql +SELECT /*+ SET_VAR(exec_mem_limit = 8589934592) */ name FROM people ORDER BY name; +SELECT /*+ SET_VAR(query_timeout = 1, enable_partition_cache=true) */ sleep(3); +``` + +注意注释必须以/*+ 开头，并且只能跟随在SELECT之后。 + +## 支持的变量 + +> 注： +> +> 以下内容由 `docs/generate-config-and-variable-doc.sh` 自动生成。 +> +> 如需修改，请修改 `fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java` 和 `fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java` 中的描述信息。 + +<--DOC_PLACEHOLDER--> + diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 0322385c9b..0cb93bef9b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -21,16 +21,14 @@ import org.apache.doris.common.ExperimentalUtil.ExperimentalType; public class Config extends ConfigBase { - /** - * Dir of custom config file - */ - @ConfField - public static String custom_config_dir = System.getenv("DORIS_HOME") + "/conf"; + @ConfField(description = {"用户自定义配置文件的路径，用于存放 fe_custom.conf。该文件中的配置会覆盖 fe.conf 中的配置", + "The path of the user-defined configuration file, used to store fe_custom.conf. " + + "The configuration in this file will override the configuration in fe.conf"}) + public static String custom_config_dir = EnvUtils.getDorisHome() + "/conf"; - /** - * The max size of one sys log and audit log - */ - @ConfField public static int log_roll_size_mb = 1024; // 1 GB + @ConfField(description = {"fe.log 和 fe.audit.log 的最大文件大小。超过这个大小后，日志文件会被切分", + "The maximum file size of fe.log and fe.audit.log. After exceeding this size, the log file will be split"}) + public static int log_roll_size_mb = 1024; // 1 GB /** * sys_log_dir: @@ -63,881 +61,614 @@ public class Config extends ConfigBase { * 60m 60 mins * 120s 120 seconds */ - @ConfField + @ConfField(description = {"FE 日志文件的存放路径，用于存放 fe.log。", + "The path of the FE log file, used to store fe.log"}) public static String sys_log_dir = System.getenv("DORIS_HOME") + "/log"; - @ConfField + + @ConfField(description = {"FE 日志的级别", "The level of FE log"}, options = {"INFO", "WARNING", "ERROR", "FATAL"}) public static String sys_log_level = "INFO"; - @ConfField public static int sys_log_roll_num = 10; - @ConfField + + @ConfField(description = {"FE 日志文件的最大数量。超过这个数量后，最老的日志文件会被删除", + "The maximum number of FE log files. After exceeding this number, the oldest log file will be deleted"}) + public static int sys_log_roll_num = 10; + + @ConfField(description = { + "Verbose 模块。VERBOSE 级别的日志是通过 log4j 的 DEBUG 级别实现的。" + + "如设置为 `org.apache.doris.catalog`，则会打印这个 package 下的类的 DEBUG 日志。", + "Verbose module. The VERBOSE level log is implemented by the DEBUG level of log4j. " + + "If set to `org.apache.doris.catalog`, " + + "the DEBUG log of the class under this package will be printed."}) public static String[] sys_log_verbose_modules = {}; - @ConfField public static String sys_log_roll_interval = "DAY"; - @ConfField public static String sys_log_delete_age = "7d"; - @Deprecated - @ConfField public static String sys_log_roll_mode = "SIZE-MB-1024"; + @ConfField(description = {"FE 日志文件的切分周期", "The split cycle of the FE log file"}, options = {"DAY", "HOUR"}) + public static String sys_log_roll_interval = "DAY"; + @ConfField(description = { + "FE 日志文件的最大存活时间。超过这个时间后，日志文件会被删除。支持的格式包括：7d, 10h, 60m, 120s", + "The maximum survival time of the FE log file. After exceeding this time, the log file will be deleted. " + + "Supported formats include: 7d, 10h, 60m, 120s"}) + public static String sys_log_delete_age = "7d"; - /** - * audit_log_dir: - * This specifies FE audit log dir. - * Audit log fe.audit.log contains all requests with related infos such as user, host, cost, status, etc. - * - * audit_log_roll_num: - * Maximal FE audit log files to be kept within an audit_log_roll_interval. - * - * audit_log_modules: - * Slow query contains all queries which cost exceed *qe_slow_log_ms* - * - * qe_slow_log_ms: - * If the response time of a query exceed this threshold, it will be recorded in audit log as slow_query. - * - * audit_log_roll_interval: - * DAY: log suffix is yyyyMMdd - * HOUR: log suffix is yyyyMMddHH - * - * audit_log_delete_age: - * default is 30 days, if log's last modify time is 30 days ago, it will be deleted. - * support format: - * 7d 7 days - * 10h 10 hours - * 60m 60 mins - * 120s 120 seconds - */ - @ConfField + @ConfField(description = {"FE 审计日志文件的存放路径，用于存放 fe.audit.log。", + "The path of the FE audit log file, used to store fe.audit.log"}) public static String audit_log_dir = System.getenv("DORIS_HOME") + "/log"; - @ConfField + @ConfField(description = {"FE 审计日志文件的最大数量。超过这个数量后，最老的日志文件会被删除", + "The maximum number of FE audit log files. " + + "After exceeding this number, the oldest log file will be deleted"}) public static int audit_log_roll_num = 90; - @ConfField + @ConfField(description = {"FE 审计日志文件的种类", "The type of FE audit log file"}, + options = {"slow_query", "query", "load", "stream_load"}) public static String[] audit_log_modules = {"slow_query", "query", "load", "stream_load"}; - @ConfField(mutable = true) + @ConfField(mutable = true, description = {"慢查询的阈值，单位为毫秒。如果一个查询的响应时间超过这个阈值，" + + "则会被记录在 audit log 中。", + "The threshold of slow query, in milliseconds. " + + "If the response time of a query exceeds this threshold, it will be recorded in audit log."}) public static long qe_slow_log_ms = 5000; - @ConfField + @ConfField(description = {"FE 审计日志文件的切分周期", "The split cycle of the FE audit log file"}, + options = {"DAY", "HOUR"}) public static String audit_log_roll_interval = "DAY"; - @ConfField + @ConfField(description = { + "FE 审计日志文件的最大存活时间。超过这个时间后，日志文件会被删除。支持的格式包括：7d, 10h, 60m, 120s", + "The maximum survival time of the FE audit log file. " + + "After exceeding this time, the log file will be deleted. " + + "Supported formats include: 7d, 10h, 60m, 120s"}) public static String audit_log_delete_age = "30d"; - @Deprecated - @ConfField - public static String audit_log_roll_mode = "TIME-DAY"; - /** - * plugin_dir: - * plugin install directory - */ - @ConfField + @ConfField(description = {"插件的安装目录", "The installation directory of the plugin"}) public static String plugin_dir = System.getenv("DORIS_HOME") + "/plugins"; - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"是否启用插件", "Whether to enable the plugin"}) public static boolean plugin_enable = true; - /** - * The default path to save jdbc drivers. - * You can put all jdbc drivers in this path, and when creating jdbc resource with only jdbc driver file name, - * Doris will find jars from this path. - */ - @ConfField + @ConfField(description = { + "JDBC 驱动的存放路径。在创建 JDBC Catalog 时，如果指定的驱动文件路径不是绝对路径，则会在这个目录下寻找", + "The path to save jdbc drivers. When creating JDBC Catalog," + + "if the specified driver file path is not an absolute path, Doris will find jars from this path"}) public static String jdbc_drivers_dir = System.getenv("DORIS_HOME") + "/jdbc_drivers"; - /** - * The default parallelism of the load execution plan - * on a single node when the broker load is submitted - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"broker load 时，单个节点上 load 执行计划的默认并行度", + "The default parallelism of the load execution plan on a single node when the broker load is submitted"}) public static int default_load_parallelism = 1; - /** - * Labels of finished or cancelled load jobs will be removed after *label_keep_max_second* - * The removed labels can be reused. - * Set a short time will lower the FE memory usage. - * (Because all load jobs' info is kept in memory before being removed) - */ - @ConfField(mutable = true) + @ConfField(mutable = true, masterOnly = true, description = { + "已完成或取消的导入作业信息的 label 会在这个时间后被删除。被删除的 label 可以被重用。", + "Labels of finished or cancelled load jobs will be removed after this time" + + "The removed labels can be reused."}) public static int label_keep_max_second = 3 * 24 * 3600; // 3 days - // For some high frequency load job such as - // INSERT, STREAMING LOAD, ROUTINE_LOAD_TASK, DELETE - // Remove the finished job or task if expired. - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "针对一些高频的导入作业，比如 INSERT, STREAMING LOAD, ROUTINE_LOAD_TASK, DELETE" + + "如果导入作业或者任务已经完成，且超过这个时间后，会被删除。被删除的作业或者任务可以被重用。", + "For some high frequency load jobs such as INSERT, STREAMING LOAD, ROUTINE_LOAD_TASK, DELETE" + + "Remove the finished job or task if expired. The removed job or task can be reused."}) public static int streaming_label_keep_max_second = 43200; // 12 hour - /** - * The max keep time of some kind of jobs. - * like alter job or export job. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "针对 ALTER, EXPORT 作业，如果作业已经完成，且超过这个时间后，会被删除。", + "For ALTER, EXPORT jobs, remove the finished job if expired."}) public static int history_job_keep_max_second = 7 * 24 * 3600; // 7 days - /** - * the transaction will be cleaned after transaction_clean_interval_second seconds - * if the transaction is visible or aborted - * we should make this interval as short as possible and each clean cycle as soon as possible - */ - @ConfField + @ConfField(description = {"事务的清理周期，单位为秒。每个周期内，将会清理已经结束的并且过期的历史事务信息", + "The clean interval of transaction, in seconds. " + + "In each cycle, the expired history transaction will be cleaned"}) public static int transaction_clean_interval_second = 30; - /** - * Load label cleaner will run every *label_clean_interval_second* to clean the outdated jobs. - */ - @ConfField + @ConfField(description = {"导入作业的清理周期，单位为秒。每个周期内，将会清理已经结束的并且过期的导入作业", + "The clean interval of load job, in seconds. " + + "In each cycle, the expired history load job will be cleaned"}) public static int label_clean_interval_second = 1 * 3600; // 1 hours - // Configurations for meta data durability - /** - * Doris meta data will be saved here. - * The storage of this dir is highly recommended as to be: - * 1. High write performance (SSD) - * 2. Safe (RAID) - */ - @ConfField + @ConfField(description = {"元数据的存储目录", "The directory to save Doris meta data"}) public static String meta_dir = System.getenv("DORIS_HOME") + "/doris-meta"; - /** - * temp dir is used to save intermediate results of some process, such as backup and restore process. - * file in this dir will be cleaned after these process is finished. - */ - @ConfField + @ConfField(description = {"临时文件的存储目录", "The directory to save Doris temp data"}) public static String tmp_dir = System.getenv("DORIS_HOME") + "/temp_dir"; - /** - * Edit log type. - * BDB: write log to bdbje - * LOCAL: use local file to save edit log, only used for unit test - */ - @ConfField + @ConfField(description = {"元数据日志的存储类型。BDB: 日志存储在 BDBJE 中。LOCAL：日志存储在本地文件中（仅用于测试）", + "The storage type of the metadata log. BDB: Logs are stored in BDBJE. " + + "LOCAL: logs are stored in a local file (for testing only)"}, options = {"BDB", "LOCAL"}) public static String edit_log_type = "bdb"; - /** - * bdbje port - */ - @ConfField + @ConfField(description = {"BDBJE 的端口号", "The port of BDBJE"}) public static int edit_log_port = 9010; - /** - * Master FE will save image every *edit_log_roll_num* meta journals. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "BDBJE 的日志滚动大小。当日志条目数超过这个值后，会触发日志滚动", + "The log roll size of BDBJE. When the number of log entries exceeds this value, the log will be rolled"}) public static int edit_log_roll_num = 50000; - /** - * Non-master FE will stop offering service - * if meta data delay gap exceeds *meta_delay_toleration_second* - */ - @ConfField public static int meta_delay_toleration_second = 300; // 5 min + @ConfField(description = {"元数据同步的容忍延迟时间，单位为秒。如果元数据的延迟超过这个值，非主 FE 会停止提供服务", + "The toleration delay time of meta data synchronization, in seconds. " + + "If the delay of meta data exceeds this value, non-master FE will stop offering service"}) + public static int meta_delay_toleration_second = 300; // 5 min - /** - * Master FE sync policy of bdbje. - * If you only deploy one Follower FE, set this to 'SYNC'. If you deploy more than 3 Follower FE, - * you can set this and the following 'replica_sync_policy' to WRITE_NO_SYNC. - * more info, see: http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.SyncPolicy.html - */ - @ConfField public static String master_sync_policy = "SYNC"; // SYNC, NO_SYNC, WRITE_NO_SYNC + @ConfField(description = {"元数据日志的写同步策略。如果仅部署一个 Follower FE，" + + "则推荐设置为 `SYNC`，如果有多个 Follower FE，则可以设置为 `WRITE_NO_SYNC`。" + + "可参阅：http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.SyncPolicy.html", + "The sync policy of meta data log. If you only deploy one Follower FE, " + + "set this to `SYNC`. If you deploy more than 3 Follower FE, " + + "you can set this and the following `replica_sync_policy` to `WRITE_NO_SYNC`. " + + "See: http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.SyncPolicy.html"}, + options = {"SYNC", "NO_SYNC", "WRITE_NO_SYNC"}) + public static String master_sync_policy = "SYNC"; // SYNC, NO_SYNC, WRITE_NO_SYNC - /** - * Follower FE sync policy of bdbje. - */ - @ConfField public static String replica_sync_policy = "SYNC"; // SYNC, NO_SYNC, WRITE_NO_SYNC + @ConfField(description = {"同 `master_sync_policy`", "Same as `master_sync_policy`"}, + options = {"SYNC", "NO_SYNC", "WRITE_NO_SYNC"}) + public static String replica_sync_policy = "SYNC"; // SYNC, NO_SYNC, WRITE_NO_SYNC - /** - * Replica ack policy of bdbje. - * more info, see: http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.ReplicaAckPolicy.html - */ - @ConfField public static String replica_ack_policy = "SIMPLE_MAJORITY"; // ALL, NONE, SIMPLE_MAJORITY + @ConfField(description = {"BDBJE 节点间同步策略，" + + "可参阅：http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.ReplicaAckPolicy.html", + "The replica ack policy of bdbje. " + + "See: http://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/Durability.ReplicaAckPolicy.html"}, + options = {"ALL", "NONE", "SIMPLE_MAJORITY"}) + public static String replica_ack_policy = "SIMPLE_MAJORITY"; // ALL, NONE, SIMPLE_MAJORITY - /** - * The heartbeat timeout of bdbje between master and follower. - * the default is 30 seconds, which is same as default value in bdbje. - * If the network is experiencing transient problems, of some unexpected long java GC annoying you, - * you can try to increase this value to decrease the chances of false timeouts - */ - @ConfField + @ConfField(description = {"BDBJE 主从节点间心跳超时时间，单位为秒。默认值为 30 秒，与 BDBJE 的默认值相同。" + + "如果网络不稳定，或者 Java GC 经常导致长时间的暂停，可以适当增大这个值，减少误报超时的概率", + "The heartbeat timeout of bdbje between master and follower, in seconds. " + + "The default is 30 seconds, which is same as default value in bdbje. " + + "If the network is experiencing transient problems, " + + "of some unexpected long java GC annoying you, " + + "you can try to increase this value to decrease the chances of false timeouts"}) public static int bdbje_heartbeat_timeout_second = 30; - /** - * The lock timeout of bdbje operation - * If there are many LockTimeoutException in FE WARN log, you can try to increase this value - */ - @ConfField + @ConfField(description = {"BDBJE 操作的锁超时时间，单位为秒。如果 FE 的 WARN 日志中出现大量的 LockTimeoutException，" + + "可以适当增大这个值", + "The lock timeout of bdbje operation, in seconds. " + + "If there are many LockTimeoutException in FE WARN log, you can try to increase this value"}) public static int bdbje_lock_timeout_second = 1; - /** - * The replica ack timeout when writing to bdbje - * When writing some relatively large logs, the ack time may time out, resulting in log writing failure. - * At this time, you can increase this value appropriately. - */ - @ConfField + @ConfField(description = {"BDBJE 主从节点间同步的超时时间，单位为秒。如果出现大量的 ReplicaWriteException，" + + "可以适当增大这个值", + "The replica ack timeout of bdbje between master and follower, in seconds. " + + "If there are many ReplicaWriteException in FE WARN log, you can try to increase this value"}) public static int bdbje_replica_ack_timeout_second = 10; - /** - * The desired upper limit on the number of bytes of reserved space to - * retain in a replicated JE Environment. - * You only need to decrease this value if your FE meta disk is really small. - * And don't need to increase this value. - */ - @ConfField + @ConfField(description = {"BDBJE 所需的空闲磁盘空间大小。如果空闲磁盘空间小于这个值，则BDBJE将无法写入。", + "Amount of free disk space required by BDBJE. " + + "If the free disk space is less than this value, BDBJE will not be able to write."}) public static int bdbje_reserved_disk_bytes = 1 * 1024 * 1024 * 1024; // 1G - /** - * num of thread to handle heartbeat events in heartbeat_mgr. - */ - @ConfField(masterOnly = true) + @ConfField(masterOnly = true, description = {"心跳线程池的线程数", + "Num of thread to handle heartbeat events"}) public static int heartbeat_mgr_threads_num = 8; - /** - * blocking queue size to store heartbeat task in heartbeat_mgr. - */ - @ConfField(masterOnly = true) + @ConfField(masterOnly = true, description = {"心跳线程池的队列大小", + "Queue size to store heartbeat task in heartbeat_mgr"}) public static int heartbeat_mgr_blocking_queue_size = 1024; - /** - * max num of thread to handle agent task in agent task thread-pool. - */ - @ConfField(masterOnly = true) + @ConfField(masterOnly = true, description = {"Agent任务线程池的线程数", + "Num of thread to handle agent task in agent task thread-pool"}) public static int max_agent_task_threads_num = 4096; - /** - * the max txn number which bdbje can rollback when trying to rejoin the group - */ - @ConfField public static int txn_rollback_limit = 100; + @ConfField(description = {"BDBJE 重加入集群时，最多回滚的事务数。如果回滚的事务数超过这个值，" + + "则 BDBJE 将无法重加入集群，需要手动清理 BDBJE 的数据。", + "The max txn number which bdbje can rollback when trying to rejoin the group. " + + "If the number of rollback txn is larger than this value, " + + "bdbje will not be able to rejoin the group, and you need to clean up bdbje data manually."}) + public static int txn_rollback_limit = 100; - /** - * Specified an IP for frontend, instead of the ip get by *InetAddress.getByName*. - * This can be used when *InetAddress.getByName* get an unexpected IP address. - * Default is "0.0.0.0", which means not set. - * CAN NOT set this as a hostname, only IP. - */ - @Deprecated - @ConfField - public static String frontend_address = "0.0.0.0"; + @ConfField(description = {"优先使用的网络地址，如果 FE 有多个网络地址，" + + "可以通过这个配置来指定优先使用的网络地址。" + + "这是一个分号分隔的列表，每个元素是一个 CIDR 表示的网络地址", + "The preferred network address. If FE has multiple network addresses, " + + "this configuration can be used to specify the preferred network address. " + + "This is a semicolon-separated list, " + + "each element is a CIDR representation of the network address"}) + public static String priority_networks = ""; - /** - * Declare a selection strategy for those servers have many ips. - * Note that there should at most one ip match this list. - * this is a list in semicolon-delimited format, in CIDR notation, e.g. 10.10.10.0/24 - * If no ip match this rule, will choose one randomly. - */ - @ConfField public static String priority_networks = ""; + @ConfField(description = {"是否重置 BDBJE 的复制组，如果所有的可选节点都无法启动，" + + "可以将元数据拷贝到另一个节点，并将这个配置设置为 true，尝试重启 FE。更多信息请参阅官网的元数据故障恢复文档。", + "If true, FE will reset bdbje replication group(that is, to remove all electable nodes info) " + + "and is supposed to start as Master. " + + "If all the electable nodes can not start, we can copy the meta data " + + "to another node and set this config to true to try to restart the FE. " + + "For more information, please refer to the metadata failure recovery document " + + "on the official website."}) + public static String metadata_failure_recovery = "false"; - /** - * If true, FE will reset bdbje replication group(that is, to remove all electable nodes info) - * and is supposed to start as Master. - * If all the electable nodes can not start, we can copy the meta data - * to another node and set this config to true to try to restart the FE. - */ - @ConfField public static String metadata_failure_recovery = "false"; - - /** - * If true, non-master FE will ignore the meta data delay gap between Master FE and its self, - * even if the metadata delay gap exceeds *meta_delay_toleration_second*. - * Non-master FE will still offer read service. - * - * This is helpful when you try to stop the Master FE for a relatively long time for some reason, - * but still wish the non-master FE can offer read service. - */ - @ConfField(mutable = true) + @ConfField(mutable = true, description = {"是否忽略元数据延迟，如果 FE 的元数据延迟超过这个阈值，" + + "则非 Master FE 仍然提供读服务。这个配置可以用于当 Master FE 因为某些原因停止了较长时间，" + + "但是仍然希望非 Master FE 可以提供读服务。", + "If true, non-master FE will ignore the meta data delay gap between Master FE and its self, " + + "even if the metadata delay gap exceeds this threshold. " + + "Non-master FE will still offer read service. " + + "This is helpful when you try to stop the Master FE for a relatively long time for some reason, " + + "but still wish the non-master FE can offer read service."}) public static boolean ignore_meta_check = false; - /** - * Set the maximum acceptable clock skew between non-master FE to Master FE host. - * This value is checked whenever a non-master FE establishes a connection to master FE via BDBJE. - * The connection is abandoned if the clock skew is larger than this value. - */ - @ConfField public static long max_bdbje_clock_delta_ms = 5000; // 5s + @ConfField(description = {"非 Master FE 与 Master FE 的最大时钟偏差，单位为毫秒。" + + "这个配置用于在非 Master FE 与 Master FE 之间建立 BDBJE 连接时检查时钟偏差，" + + "如果时钟偏差超过这个阈值，则 BDBJE 连接会被放弃。", + "The maximum clock skew between non-master FE to Master FE host, in milliseconds. " + + "This value is checked whenever a non-master FE establishes a connection to master FE via BDBJE. " + + "The connection is abandoned if the clock skew is larger than this value."}) + public static long max_bdbje_clock_delta_ms = 5000; // 5s - /** - * Whether to enable all http interface authentication - */ - @ConfField public static boolean enable_all_http_auth = false; + @ConfField(description = {"是否启用所有 http 接口的认证", + "Whether to enable all http interface authentication"}, expType = ExperimentalType.EXPERIMENTAL) + public static boolean enable_all_http_auth = false; - /** - * Fe http port - * Currently, all FEs' http port must be same. - */ - @ConfField public static int http_port = 8030; + @ConfField(description = {"FE http 端口，目前所有 FE 的 http 端口必须相同", + "Fe http port, currently all FE's http port must be same"}) + public static int http_port = 8030; - /** - * Fe https port - * Currently, all FEs' https port must be same. - */ - @ConfField public static int https_port = 8050; + @ConfField(description = {"FE https 端口，目前所有 FE 的 https 端口必须相同", + "Fe https port, currently all FE's https port must be same"}) + public static int https_port = 8050; - /** - * ssl key store path. - * If you want to change the path, you need to create corresponding directory. - */ - @ConfField public static String key_store_path = System.getenv("DORIS_HOME") - + "/conf/ssl/doris_ssl_certificate.keystore"; + @ConfField(description = {"FE https 服务的 key store 路径", + "The key store path of FE https service"}) + public static String key_store_path = System.getenv("DORIS_HOME") + + "/conf/ssl/doris_ssl_certificate.keystore"; - /** - * ssl key store password. Null by default. - */ - @ConfField public static String key_store_password = ""; + @ConfField(description = {"FE https 服务的 key store 密码", + "The key store password of FE https service"}) + public static String key_store_password = ""; - /** - * ssl key store type. "JKS" by default. - */ - @ConfField public static String key_store_type = "JKS"; + @ConfField(description = {"FE https 服务的 key store 类型", + "The key store type of FE https service"}) + public static String key_store_type = "JKS"; - /** - * ssl key store alias. "doris_ssl_certificate" by default. - */ - @ConfField public static String key_store_alias = "doris_ssl_certificate"; + @ConfField(description = {"FE https 服务的 key store 别名", + "The key store alias of FE https service"}) + public static String key_store_alias = "doris_ssl_certificate"; - /** - * https enable flag. false by default. - * If the value is false, http is supported. Otherwise, https is supported. - * Currently doris uses many ports, so http and https are not supported at the same time. - */ - @ConfField public static boolean enable_https = false; + @ConfField(description = {"是否启用 https，如果启用，http 端口将不可用", + "Whether to enable https, if enabled, http port will not be available"}, + expType = ExperimentalType.EXPERIMENTAL) + public static boolean enable_https = false; - /** - * Jetty container default configuration - * Jetty's thread architecture model is very simple, divided into three thread pools: - * acceptors,selectors and workers. Acceptors are responsible for accepting new connections, - * and then hand over to selectors to process the unpacking of the HTTP message protocol, - * and finally workers process the request. The first two thread pools adopt a non-blocking model, - * and one thread can handle the read and write of many sockets, so the number of thread pools is small. - * - * For most projects, only 1-2 acceptors threads are needed, and 2 to 4 selectors threads are sufficient. - * Workers are obstructive business logic, often have more database operations, and require a large number of - * threads. The specific number depends on the proportion of QPS and IO events of the application. The higher the - * QPS, the more threads are required, the higher the proportion of IO, the more threads waiting, and the more - * total threads required. - */ - @ConfField public static int jetty_server_acceptors = 2; - @ConfField public static int jetty_server_selectors = 4; - @ConfField public static int jetty_server_workers = 0; + @ConfField(description = {"Jetty 的 acceptor 线程数。Jetty的线程架构模型很简单，分为三个线程池：acceptor、selector 和 worker。" + + "acceptor 负责接受新的连接，然后交给 selector 处理HTTP报文协议的解包，最后由 worker 处理请求。" + + "前两个线程池采用非阻塞模型，并且一个线程可以处理很多socket的读写，所以线程池的数量少。" + + "对于大多数项目，只需要 1-2 个 acceptor 线程，2 到 4 个就足够了。Worker 的数量取决于应用的QPS和IO事件的比例。" + + "越高QPS，或者IO占比越高，等待的线程越多，需要的线程总数越多。", + "The number of acceptor threads for Jetty. Jetty's thread architecture model is very simple, " + + "divided into three thread pools: acceptor, selector and worker. " + + "The acceptor is responsible for accepting new connections, " + + "and then handing it over to the selector to process the unpacking of the HTTP message protocol, " + + "and finally the worker processes the request. " + + "The first two thread pools adopt a non-blocking model, " + + "and one thread can handle many socket reads and writes, " + + "so the number of thread pools is small. For most projects, " + + "only 1-2 acceptor threads are needed, 2 to 4 should be enough. " + + "The number of workers depends on the ratio of QPS and IO events of the application. " + + "The higher the QPS, or the higher the IO ratio, the more threads are waiting, " + + "and the more threads are required."}) + public static int jetty_server_acceptors = 2; + @ConfField(description = {"Jetty 的 selector 线程数。", "The number of selector threads for Jetty."}) + public static int jetty_server_selectors = 4; + @ConfField(description = {"Jetty 的 worker 线程数。0 表示使用默认线程池。", + "The number of worker threads for Jetty. 0 means using the default thread pool."}) + public static int jetty_server_workers = 0; - /** - * Configure the default minimum and maximum number of threads for jetty. - * The default minimum and maximum number of threads for jetty is 10 and the maximum is 200. - * If this is relatively small in a high-concurrency import scenario, - * users can adjust it according to their own conditions. - */ - @ConfField public static int jetty_threadPool_minThreads = 20; - @ConfField public static int jetty_threadPool_maxThreads = 400; + @ConfField(description = {"Jetty 的线程池的默认最小线程数。", + "The default minimum number of threads for jetty."}) + public static int jetty_threadPool_minThreads = 20; + @ConfField(description = {"Jetty 的线程池的默认最大线程数。", + "The default maximum number of threads for jetty."}) + public static int jetty_threadPool_maxThreads = 400; - /** - * Jetty maximum number of bytes in put or post method,default:100MB - */ - @ConfField public static int jetty_server_max_http_post_size = 100 * 1024 * 1024; + @ConfField(description = {"Jetty 的最大 HTTP POST 大小，单位是字节，默认值是 100MB。", + "The maximum HTTP POST size of Jetty, in bytes, the default value is 100MB."}) + public static int jetty_server_max_http_post_size = 100 * 1024 * 1024; - /** - * http header size configuration parameter, the default value is 10K - */ - @ConfField public static int jetty_server_max_http_header_size = 10240; + @ConfField(description = {"Jetty 的最大 HTTP header 大小，单位是字节，默认值是 10KB。", + "The maximum HTTP header size of Jetty, in bytes, the default value is 10KB."}) + public static int jetty_server_max_http_header_size = 10240; - /** - * Mini load disabled by default - */ - @ConfField public static boolean disable_mini_load = true; + @ConfField(description = {"是否禁用 mini load，默认禁用", + "Whether to disable mini load, disabled by default"}) + public static boolean disable_mini_load = true; - /** - * The backlog_num for mysql nio server - * When you enlarge this backlog_num, you should enlarge the value in - * the linux /proc/sys/net/core/somaxconn file at the same time - */ - @ConfField public static int mysql_nio_backlog_num = 1024; + @ConfField(description = {"mysql nio server 的 backlog 数量。" + + "如果调大这个值，则需同时调整 /proc/sys/net/core/somaxconn 的值", + "The backlog number of mysql nio server. " + + "If you enlarge this value, you should enlarge the value in " + + "`/proc/sys/net/core/somaxconn` at the same time"}) + public static int mysql_nio_backlog_num = 1024; - /** - * The connection timeout and socket timeout config for thrift server - * The default value for thrift_client_timeout_ms is set to be zero to prevent readtimeout - * - */ - @ConfField public static int thrift_client_timeout_ms = 0; + @ConfField(description = {"thrift client 的连接超时时间，单位是毫秒。0 表示不设置超时时间。", + "The connection timeout of thrift client, in milliseconds. 0 means no timeout."}) + public static int thrift_client_timeout_ms = 0; - /** - * The backlog_num for thrift server - * When you enlarge this backlog_num, you should ensure it's value larger than - * the linux /proc/sys/net/core/somaxconn config - */ - @ConfField public static int thrift_backlog_num = 1024; + @ConfField(description = {"thrift server 的 backlog 数量。" + + "如果调大这个值，则需同时调整 /proc/sys/net/core/somaxconn 的值", + "The backlog number of thrift server. " + + "If you enlarge this value, you should enlarge the value in " + + "`/proc/sys/net/core/somaxconn` at the same time"}) + public static int thrift_backlog_num = 1024; - /** - * FE thrift server port - */ - @ConfField public static int rpc_port = 9020; + @ConfField(description = {"FE thrift server 的端口号", "The port of FE thrift server"}) + public static int rpc_port = 9020; - /** - * FE mysql server port - */ - @ConfField public static int query_port = 9030; + @ConfField(description = {"FE MySQL server 的端口号", "The port of FE MySQL server"}) + public static int query_port = 9030; - /** - * num of thread to handle io events in mysql. - */ - @ConfField public static int mysql_service_io_threads_num = 4; + @ConfField(description = {"MySQL 服务的 IO 线程数", "The number of IO threads in MySQL service"}) + public static int mysql_service_io_threads_num = 4; - /** - * max num of thread to handle task in mysql. - */ - @ConfField public static int max_mysql_service_task_threads_num = 4096; + @ConfField(description = {"MySQL 服务的最大任务线程数", "The max number of task threads in MySQL service"}) + public static int max_mysql_service_task_threads_num = 4096; - /** - * node(FE or BE) will be considered belonging to the same Palo cluster if they have same cluster id. - * Cluster id is usually a random integer generated when master FE start at first time. - * You can also specify one. - */ - @ConfField public static int cluster_id = -1; + @ConfField(description = { + "集群 ID，用于内部认证。通常在集群第一次启动时，会随机生成一个 cluster id. 用户也可以手动指定。", + "Cluster id used for internal authentication. Usually a random integer generated when master FE " + + "start at first time. You can also specify one."}) + public static int cluster_id = -1; - /** - * Cluster token used for internal authentication. - */ - @ConfField public static String auth_token = ""; + @ConfField(description = {"集群 token，用于内部认证。", + "Cluster token used for internal authentication."}) + public static String auth_token = ""; - // Configurations for load, clone, create table, alter table etc. We will rarely change them - /** - * Maximal waiting time for creating a single replica. - * eg. - * if you create a table with #m tablets and #n replicas for each tablet, - * the create table request will run at most (m * n * tablet_create_timeout_second) before timeout. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, + description = {"创建单个 Replica 的最大超时时间，单位是秒。如果你要创建 m 个 tablet，每个 tablet 有 n 个 replica。" + + "则总的超时时间为 `m * n * tablet_create_timeout_second`", + "Maximal waiting time for creating a single replica, in seconds. " + + "eg. if you create a table with #m tablets and #n replicas for each tablet, " + + "the create table request will run at most " + + "(m * n * tablet_create_timeout_second) before timeout"}) public static int tablet_create_timeout_second = 1; - /** - * In order not to wait too long for create table(index), set a max timeout. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"创建表的最大超时时间，单位是秒。", + "Maximal waiting time for creating a table, in seconds."}) public static int max_create_table_timeout_second = 3600; - /** - * Maximal waiting time for all publish version tasks of one transaction to be finished - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"导入 Publish 阶段的最大超时时间，单位是秒。", + "Maximal waiting time for all publish version tasks of one transaction to be finished, in seconds."}) public static int publish_version_timeout_second = 30; // 30 seconds - /** - * Maximal waiting time for all data inserted before one transaction to be committed - * This is the timeout second for the command "commit" - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"提交事务的最大超时时间，单位是秒。" + + "该参数仅用于事务型 insert 操作中。", + "Maximal waiting time for all data inserted before one transaction to be committed, in seconds. " + + "This parameter is only used for transactional insert operation"}) public static int commit_timeout_second = 30; // 30 seconds - /** - * minimal intervals between two publish version action - */ - @ConfField public static int publish_version_interval_ms = 10; + @ConfField(masterOnly = true, description = {"Publish 任务触发线程的执行间隔，单位是毫秒。", + "The interval of publish task trigger thread, in milliseconds"}) + public static int publish_version_interval_ms = 10; - /** - * The thrift server max worker threads - */ - @ConfField public static int thrift_server_max_worker_threads = 4096; + @ConfField(description = {"thrift server 的最大 worker 线程数", "The max worker threads of thrift server"}) + public static int thrift_server_max_worker_threads = 4096; - /** - * Maximal wait seconds for straggler node in load - * eg. - * there are 3 replicas A, B, C - * load is already quorum finished(A,B) at t1 and C is not finished - * if (current_time - t1) > 300s, then palo will treat C as a failure node - * will call transaction manager to commit the transaction and tell transaction manager - * that C is failed - * - * This is also used when waiting for publish tasks - * - * TODO this parameter is the default value for all job and the DBA could specify it for separate job - */ - @ConfField(mutable = true, masterOnly = true) - public static int load_straggler_wait_second = 300; + @ConfField(mutable = true, masterOnly = true, description = {"Delete 操作的最大超时时间，单位是秒。", + "Maximal timeout for delete job, in seconds."}) + public static int delete_job_max_timeout_second = 300; - /** - * The load scheduler running interval. - * A load job will transfer its state from PENDING to LOADING to FINISHED. - * The load scheduler will transfer load job from PENDING to LOADING - * while the txn callback will transfer load job from LOADING to FINISHED. - * So a load job will cost at most one interval to finish when the concurrency has not reached the upper limit. - */ - @ConfField public static int load_checker_interval_second = 5; + @ConfField(description = {"load job 调度器的执行间隔，单位是秒。", + "The interval of load job scheduler, in seconds."}) + public static int load_checker_interval_second = 5; - /** - * The spark load scheduler running interval. - * Default 60 seconds, because spark load job is heavy and yarn client returns slowly. - */ - @ConfField public static int spark_load_checker_interval_second = 60; + @ConfField(description = {"spark load job 调度器的执行间隔，单位是秒。", + "The interval of spark load job scheduler, in seconds."}) + public static int spark_load_checker_interval_second = 60; - /** - * Concurrency of HIGH priority pending load jobs. - * Load job priority is defined as HIGH or NORMAL. - * All mini batch load jobs are HIGH priority, other types of load jobs are NORMAL priority. - * Priority is set to avoid that a slow load job occupies a thread for a long time. - * This is just a internal optimized scheduling policy. - * Currently, you can not specified the job priority manually, - * and do not change this if you know what you are doing. - */ - @ConfField public static int load_pending_thread_num_high_priority = 3; - /** - * Concurrency of NORMAL priority pending load jobs. - * Do not change this if you know what you are doing. - */ - @ConfField public static int load_pending_thread_num_normal_priority = 10; - /** - * Concurrency of HIGH priority etl load jobs. - * Do not change this if you know what you are doing. - */ - @ConfField public static int load_etl_thread_num_high_priority = 3; - /** - * Concurrency of NORMAL priority etl load jobs. - * Do not change this if you know what you are doing. - */ - @ConfField public static int load_etl_thread_num_normal_priority = 10; - /** - * Not available. - */ - @ConfField(mutable = true, masterOnly = true) - public static int load_input_size_limit_gb = 0; // GB, 0 is no limit - /** - * Not available. - */ - @ConfField(mutable = true, masterOnly = true) - public static int load_running_job_num_limit = 0; // 0 is no limit - /** - * Default broker load timeout - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Broker load 的默认超时时间，单位是秒。", + "Default timeout for broker load job, in seconds."}) public static int broker_load_default_timeout_second = 14400; // 4 hour - /** - * Broker rpc timeout - */ - @ConfField public static int broker_timeout_ms = 10000; // 10s - /** - * Default non-streaming mini load timeout - */ - @Deprecated - @ConfField(mutable = true, masterOnly = true) - public static int mini_load_default_timeout_second = 3600; // 1 hour + @ConfField(description = {"和 Broker 进程交互的 RPC 的超时时间，单位是毫秒。", + "The timeout of RPC between FE and Broker, in milliseconds"}) + public static int broker_timeout_ms = 10000; // 10s - /** - * Default insert load timeout - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Insert load 的默认超时时间，单位是秒。", + "Default timeout for insert load job, in seconds."}) public static int insert_load_default_timeout_second = 3600; // 1 hour - /** - * Default stream load and streaming mini load timeout - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Stream load 的默认超时时间，单位是秒。", + "Default timeout for stream load job, in seconds."}) public static int stream_load_default_timeout_second = 86400 * 3; // 3days - /** - * Default stream load pre-commit status timeout - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Stream load 的默认预提交超时时间，单位是秒。", + "Default pre-commit timeout for stream load job, in seconds."}) public static int stream_load_default_precommit_timeout_second = 3600; // 3600s - /** - * Max load timeout applicable to all type of load except for stream load - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Load 的最大超时时间，单位是秒。", + "Maximal timeout for load job, in seconds."}) public static int max_load_timeout_second = 259200; // 3days - /** - * Max stream load and streaming mini load timeout - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Stream load 的最大超时时间，单位是秒。", + "Maximal timeout for stream load job, in seconds."}) public static int max_stream_load_timeout_second = 259200; // 3days - /** - * Min stream load timeout applicable to all type of load - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Load 的最小超时时间，单位是秒。", + "Minimal timeout for load job, in seconds."}) public static int min_load_timeout_second = 1; // 1s - /** - * Default hadoop load timeout - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Hadoop load 的默认超时时间，单位是秒。", + "Default timeout for hadoop load job, in seconds."}) public static int hadoop_load_default_timeout_second = 86400 * 3; // 3 day - // Configurations for spark load - /** - * Default spark dpp version - */ - @ConfField + @ConfField(description = {"Spark DPP 程序的版本", "Default spark dpp version"}) public static String spark_dpp_version = "1.0.0"; - /** - * Default spark load timeout - */ - @ConfField(mutable = true, masterOnly = true) + + @ConfField(mutable = true, masterOnly = true, description = {"Spark load 的默认超时时间，单位是秒。", + "Default timeout for spark load job, in seconds."}) public static int spark_load_default_timeout_second = 86400; // 1 day - /** - * Default spark home dir - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Spark Load 所使用的 Spark 程序目录", + "Spark dir for Spark Load"}) public static String spark_home_default_dir = System.getenv("DORIS_HOME") + "/lib/spark2x"; - /** - * Default spark dependencies path - */ - @ConfField + @ConfField(description = {"Spark load 所使用的依赖项目录", "Spark dependencies dir for Spark Load"}) public static String spark_resource_path = ""; - /** - * The specified spark launcher log dir - */ - @ConfField + @ConfField(description = {"Spark launcher 日志路径", "Spark launcher log dir"}) public static String spark_launcher_log_dir = sys_log_dir + "/spark_launcher_log"; - /** - * Default yarn client path - */ - @ConfField + @ConfField(description = {"Yarn client 的路径", "Yarn client path"}) public static String yarn_client_path = System.getenv("DORIS_HOME") + "/lib/yarn-client/hadoop/bin/yarn"; - /** - * Default yarn config file directory - * Each time before running the yarn command, we need to check that the - * config file exists under this path, and if not, create them. - */ - @ConfField + @ConfField(description = {"Yarn 配置文件的路径", "Yarn config path"}) public static String yarn_config_dir = System.getenv("DORIS_HOME") + "/lib/yarn-config"; - /** - * Maximal intervals between two syncJob's commits. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Sync job 的最大提交间隔，单位是秒。", + "Maximal intervals between two sync job's commits."}) public static long sync_commit_interval_second = 10; - /** - * Sync checker's running interval. - */ - @ConfField public static int sync_checker_interval_second = 5; + @ConfField(description = {"Sync job 调度器的执行间隔，单位是秒。", + "The interval of sync job scheduler, in seconds."}) + public static int sync_checker_interval_second = 5; - /** - * max num of thread to handle sync task in sync task thread-pool. - */ - @ConfField public static int max_sync_task_threads_num = 10; + @ConfField(description = {"Sync job 的最大并发数。", + "Maximal concurrent num of sync job."}) + public static int max_sync_task_threads_num = 10; - - /** - * Min event size that a sync job will commit. - * When receiving events less than it, SyncJob will continue - * to wait for the next batch of data until the time exceeds - * `sync_commit_interval_second`. - * The default value is 10000 (canal default event buffer size is 16384). - * You should set it smaller than canal buffer size. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Sync job 的最小提交事件数。如果收到的事件数小于该值，" + + "Sync Job 会继续等待下一批数据，直到时间超过 `sync_commit_interval_second`。这个值应小于 canal 的缓冲区大小。", + "Min events that a sync job will commit. When receiving events less than it, SyncJob will continue " + + "to wait for the next batch of data until the time exceeds `sync_commit_interval_second`."}) public static long min_sync_commit_size = 10000; - /** - * Min bytes that a sync job will commit. - * When receiving bytes less than it, SyncJob will continue - * to wait for the next batch of data until the time exceeds - * `sync_commit_interval_second`. - * The default value is 15 MB (canal default memory is 16 MB). - * You should set it slightly smaller than canal memory. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Sync job 的最小提交字节数。如果收到的字节数小于该值，" + + "Sync Job 会继续等待下一批数据，直到时间超过 `sync_commit_interval_second`。这个值应小于 canal 的缓冲区大小。", + "Min bytes that a sync job will commit. When receiving bytes less than it, SyncJob will continue " + + "to wait for the next batch of data until the time exceeds `sync_commit_interval_second`."}) public static long min_bytes_sync_commit = 15 * 1024 * 1024; // 15 MB - /** - * Max bytes that a sync job will commit. - * When receiving bytes less than it, SyncJob will commit - * all data immediately. - * The default value is 64 MB (canal default memory is 16 MB). - * You should set it larger than canal memory and - * `min_bytes_sync_commit`. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Sync job 的最大提交字节数。如果收到的字节数大于该值，" + + "Sync Job 会立即提交所有数据。这个值应大于 canal 的缓冲区大小和 `min_bytes_sync_commit`。", + "Max bytes that a sync job will commit. When receiving bytes larger than it, SyncJob will commit " + + "all data immediately. You should set it larger than canal memory and " + + "`min_bytes_sync_commit`."}) public static long max_bytes_sync_commit = 64 * 1024 * 1024; // 64 MB - /** - * Default number of waiting jobs for routine load and version 2 of load - * This is a desired number. - * In some situation, such as switch the master, the current number is maybe more than desired_max_waiting_jobs - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Broker Load 的最大等待 job 数量。" + + "这个值是一个期望值。在某些情况下，比如切换 master，当前等待的 job 数量可能会超过这个值。", + "Maximal number of waiting jobs for Broker Load. This is a desired number. " + + "In some situation, such as switch the master, " + + "the current number is maybe more than this value."}) public static int desired_max_waiting_jobs = 100; - /** - * fetch stream load record interval. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"FE 从 BE 获取 Stream Load 作业信息的间隔。", + "The interval of FE fetch stream load record from BE."}) public static int fetch_stream_load_record_interval_second = 120; - /** - * Default max number of recent stream load record that can be stored in memory. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = {"Stream load 的默认最大记录数。", + "Default max number of recent stream load record that can be stored in memory."}) public static int max_stream_load_record_size = 5000; - /** - * Default max number of recent iceberg database table creation record that can be stored in memory. - */ - @ConfField(mutable = true, masterOnly = true) - public static int max_iceberg_table_creation_record_size = 2000; - - /** - * Whether to disable show stream load and clear stream load records in memory. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "是否禁用 show stream load 和 clear stream load 命令，以及是否清理内存中的 stream load 记录。", + "Whether to disable show stream load and clear stream load records in memory."}) public static boolean disable_show_stream_load = false; - /** - * Whether to enable to write single replica for stream load and broker load. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "是否启用 stream load 和 broker load 的单副本写入。", + "Whether to enable to write single replica for stream load and broker load."}, + expType = ExperimentalType.EXPERIMENTAL) public static boolean enable_single_replica_load = false; - /** - * maximum concurrent running txn num including prepare, commit txns under a single db - * txn manager will reject coming txns - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "单个数据库最大并发运行的事务数，包括 prepare 和 commit 事务。", + "Maximum concurrent running txn num including prepare, commit txns under a single db.", + "Txn manager will reject coming txns."}) public static int max_running_txn_num_per_db = 100; - /** - * This configuration is just for compatible with old version, - * this config has been replaced by async_loading_load_task_pool_size, - * it will be removed in the future. - */ - @Deprecated - @ConfField(mutable = false, masterOnly = true) - public static int async_load_task_pool_size = 10; - - /** - * The pending_load task executor pool size. This pool size limits the max running pending_load tasks. - * Currently, it only limits the pending_load task of broker load and spark load. - * It should be less than 'max_running_txn_num_per_db' - */ - @ConfField(mutable = false, masterOnly = true) + @ConfField(masterOnly = true, description = {"pending load task 执行线程数。这个配置可以限制当前等待的导入作业数。" + + "并且应小于 `max_running_txn_num_per_db`。", + "The pending load task executor pool size. " + + "This pool size limits the max running pending load tasks.", + "Currently, it only limits the pending load task of broker load and spark load.", + "It should be less than `max_running_txn_num_per_db`"}) public static int async_pending_load_task_pool_size = 10; - /** - * The loading_load task executor pool size. This pool size limits the max running loading_load tasks. - * Currently, it only limits the loading_load task of broker load. - */ - @ConfField(mutable = false, masterOnly = true) - public static int async_loading_load_task_pool_size = async_load_task_pool_size; + @ConfField(masterOnly = true, description = {"loading load task 执行线程数。这个配置可以限制当前正在导入的作业数。", + "The loading load task executor pool size. " + + "This pool size limits the max running loading load tasks.", + "Currently, it only limits the loading load task of broker load."}) + public static int async_loading_load_task_pool_size = 10; - /** - * Same meaning as *tablet_create_timeout_second*, but used when delete a tablet. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "和 `tablet_create_timeout_second` 含义相同，但是是用于 Delete 操作的。", + "The same meaning as `tablet_create_timeout_second`, but used when delete a tablet."}) public static int tablet_delete_timeout_second = 2; - /** - * the minimal delay seconds between a replica is failed and fe try to recovery it using clone. - */ - @ConfField(mutable = true, masterOnly = true) - public static int replica_delay_recovery_second = 0; - /** - * Balance threshold of data size in BE. - * The balance algorithm is: - * 1. Calculate the average used capacity(AUC) of the entire cluster. (total data size / total backends num) - * 2. The high water level is (AUC * (1 + clone_capacity_balance_threshold)) - * 3. The low water level is (AUC * (1 - clone_capacity_balance_threshold)) - * The Clone checker will try to move replica from high water level BE to low water level BE. - */ - @ConfField(mutable = true, masterOnly = true) - public static double clone_capacity_balance_threshold = 0.2; - /** - * Balance threshold of num of replicas in Backends. - */ - @ConfField(mutable = true, masterOnly = true) - public static double clone_distribution_balance_threshold = 0.2; - /** - * The high water of disk capacity used percent. - * This is used for calculating load score of a backend. - */ - @ConfField(mutable = true, masterOnly = true) - public static double capacity_used_percent_high_water = 0.75; - /** - * Maximal timeout of ALTER TABLE request. Set long enough to fit your table data size. - */ - @ConfField(mutable = true, masterOnly = true) - public static int alter_table_timeout_second = 86400 * 30; // 1month - /** - * If a backend is down for *max_backend_down_time_second*, a BACKEND_DOWN event will be triggered. - * Do not set this if you know what you are doing. - */ - @ConfField(mutable = true, masterOnly = true) - public static int max_backend_down_time_second = 3600; // 1h - /** - * If disable_storage_medium_check is true, ReportHandler would not check tablet's storage medium - * and disable storage cool down function, the default value is false. - * You can set the value true when you don't care what the storage medium of the tablet is. - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "磁盘使用率的高水位线。用于计算 BE 的负载分数。", + "The high water of disk capacity used percent. This is used for calculating load score of a backend."}) + public static double capacity_used_percent_high_water = 0.75; + + @ConfField(mutable = true, masterOnly = true, description = { + "ALTER TABLE 请求的最大超时时间。设置的足够长以适应表的数据量。", + "Maximal timeout of ALTER TABLE request. Set long enough to fit your table data size."}) + public static int alter_table_timeout_second = 86400 * 30; // 1month + + @ConfField(mutable = true, masterOnly = true, description = { + "是否禁用存储介质检查。如果禁用，ReportHandler 将不会检查 tablet 的存储介质，" + + "并且禁用存储介质冷却功能。默认值为 false。", + "When disable_storage_medium_check is true, ReportHandler would not check tablet's storage medium " + + "and disable storage cool down function."}) public static boolean disable_storage_medium_check = false; - /** - * When create a table(or partition), you can specify its storage medium(HDD or SSD). - * If not set, this specifies the default medium when created. - */ - @ConfField public static String default_storage_medium = "HDD"; - /** - * After dropping database(table/partition), you can recover it by using RECOVER stmt. - * And this specifies the maximal data retention time. After time, the data will be deleted permanently. - */ - @ConfField(mutable = true, masterOnly = true) + + @ConfField(description = {"创建表或分区时，可以指定存储介质(HDD 或 SSD)。如果未指定，" + + "则使用此配置指定的默认介质。", + "When create a table(or partition), you can specify its storage medium(HDD or SSD)."}) + public static String default_storage_medium = "HDD"; + + @ConfField(mutable = true, masterOnly = true, description = { + "删除数据库(表/分区)后，可以使用 RECOVER 语句恢复。此配置指定了数据的最大保留时间。" + + "超过此时间，数据将被永久删除。", + "After dropping database(table/partition), you can recover it by using RECOVER stmt.", + "And this specifies the maximal data retention time. After time, the data will be deleted permanently."}) public static long catalog_trash_expire_second = 86400L; // 1day - /** - * Maximal bytes that a single broker scanner will read. - * Do not set this if you know what you are doing. - */ - @ConfField(mutable = true, masterOnly = true) + + @ConfField(mutable = true, masterOnly = true, description = { + "单个 broker scanner 读取的最小字节数。Broker Load 切分文件时，" + + "如果切分后的文件大小小于此值，将不会切分。", + "Minimal bytes that a single broker scanner will read. When splitting file in broker load, " + + "if the size of split file is less than this value, it will not be split."}) public static long min_bytes_per_broker_scanner = 67108864L; // 64MB - /** - * Maximal concurrency of broker scanners. - * Do not set this if you know what you are doing. - */ - @ConfField(mutable = true, masterOnly = true) + + @ConfField(mutable = true, masterOnly = true, description = { + "单个 broker scanner 的最大并发数。", "Maximal concurrency of broker scanners."}) public static int max_broker_concurrency = 10; - /** - * Export checker's running interval. - */ - @ConfField public static int export_checker_interval_second = 5; - /** - * Limitation of the concurrency of running export jobs. - * Default is 5. - * 0 is unlimited - */ - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "导出作业的最大并发数。", "Limitation of the concurrency of running export jobs."}) public static int export_running_job_num_limit = 5; - /** - * Default timeout of export jobs. - */ - @ConfField(mutable = true, masterOnly = true) + + @ConfField(mutable = true, masterOnly = true, description = { + "导出作业的默认超时时间。", "Default timeout of export jobs."}) public static int export_task_default_timeout_second = 2 * 3600; // 2h - /** - * Number of tablets per export query plan - */ - @ConfField(mutable = true, masterOnly = true) + + @ConfField(mutable = true, masterOnly = true, description = { + "每个导出作业的需要处理的 tablet 数量。", "Number of tablets need to be handled per export job."}) public static int export_tablet_num_per_task = 5; - // Configurations for consistency check - /** - * Consistency checker will run from *consistency_check_start_time* to *consistency_check_end_time*. - * If start time == end time, the checker will stop scheduling. - * And default is disabled. - * TODO(cmy): Disable by default because current checksum logic has some bugs. - * And it will also bring some overhead. - */ - @ConfField(mutable = true, masterOnly = true) + // TODO(cmy): Disable by default because current checksum logic has some bugs. + @ConfField(mutable = true, masterOnly = true, description = { + "一致性检查的开始时间。与 `consistency_check_end_time` 配合使用，决定一致性检查的起止时间。" + + "如果将两个参数设置为相同的值，则一致性检查将不会被调度。", + "Start time of consistency check. Used with `consistency_check_end_time` " + + "to decide the start and end time of consistency check. " + + "If set to the same value, consistency check will not be scheduled."}) public static String consistency_check_start_time = "23"; - @ConfField(mutable = true, masterOnly = true) + @ConfField(mutable = true, masterOnly = true, description = { + "一致性检查的结束时间。与 `consistency_check_start_time` 配合使用，决定一致性检查的起止时间。" + + "如果将两个参数设置为相同的值，则一致性检查将不会被调度。", + "End time of consistency check. Used with `consistency_check_start_time` " + + "to decide the start and end time of consistency check. " + + "If set to the same value, consistency check will not be scheduled."}) public static String consistency_check_end_time = "23"; - /** - * Default timeout of a single consistency check task. Set long enough to fit your tablet size. - */ - @ConfField(mutable = true, masterOnly = true) + + @ConfField(mutable = true, masterOnly = true, description = { + "单个一致性检查任务的默认超时时间。设置的足够长以适应表的数据量。", + "Default timeout of a single consistency check task. Set long enough to fit your tablet size."}) public static long check_consistency_default_timeout_second = 600; // 10 min - // Configurations for query engine - /** - * Maximal number of connections per FE. - */ - @ConfField public static int qe_max_connection = 1024; + @ConfField(description = {"单个 FE 的 MySQL Server 的最大连接数。", + "Maximal number of connections of MySQL server per FE."}) + public static int qe_max_connection = 1024; - /** - * Maximal number of thread in connection-scheduler-pool. - */ - @ConfField public static int max_connection_scheduler_threads_num = 4096; + @ConfField(description = {"MySQL 连接调度线程池的最大线程数。", + "Maximal number of thread in MySQL connection-scheduler-pool."}) + public static int max_connection_scheduler_threads_num = 4096; - /** - * The memory_limit for colocote join PlanFragment instance = - * exec_mem_limit / min (query_colocate_join_memory_limit_penalty_factor, instance_num) - */ - @ConfField(mutable = true) + @ConfField(mutable = true, description = {"Colocate join 每个 instance 的内存 penalty 系数。" + + "计算方式：`exec_mem_limit / min (query_colocate_join_memory_limit_penalty_factor, instance_num)`", + "Colocate join PlanFragment instance memory limit penalty factor.", + "The memory_limit for colocote join PlanFragment instance = " + + "`exec_mem_limit / min (query_colocate_join_memory_limit_penalty_factor, instance_num)`"}) public static int query_colocate_join_memory_limit_penalty_factor = 1; /** diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java index eac6973d52..8774e65206 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java @@ -57,6 +57,15 @@ public class ConfigBase { ExperimentalType expType() default ExperimentalType.NONE; Class callback() default DefaultConfHandler.class; + + // description for this config item. + // There should be 2 elements in the array. + // The first element is the description in Chinese. + // The second element is the description in English. + String[] description() default {"待补充", "TODO"}; + + // Enum options for this config item, if it has. + String[] options() default {}; } public interface ConfHandler { @@ -221,11 +230,6 @@ public class ConfigBase { } setConfigField(f, confVal); - - // to be compatible with old version - if (confKey.equalsIgnoreCase("async_load_task_pool_size")) { - Config.async_loading_load_task_pool_size = Config.async_load_task_pool_size; - } } } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/EnvUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/EnvUtils.java new file mode 100644 index 0000000000..94a90e6ac4 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/common/EnvUtils.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +import com.google.common.base.Strings; + +public class EnvUtils { + // get doris home from env, + // if not set, return ${Env.DORIS_HOME} as a placeholder + public static String getDorisHome() { + String dorisHome = System.getenv("DORIS_HOME"); + if (Strings.isNullOrEmpty(dorisHome)) { + dorisHome = "${Env.DORIS_HOME}"; + } + return dorisHome; + } +} diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 23ab52cba4..7819c44ef6 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -797,7 +797,7 @@ under the License. org.codehaus.mojo exec-maven-plugin - 3.0.0 + 3.1.0 gensrc @@ -818,6 +818,26 @@ under the License. ${skip.plugin} + + doc + process-classes + + java + + + org.apache.doris.common.util.DocGenerator + + ${doris.home}/docs/en/docs/admin-manual/config/fe-config-template.md + ${doris.home}/docs/zh-CN/docs/admin-manual/config/fe-config-template.md + ${doris.home}/docs/en/docs/admin-manual/config/fe-config.md + ${doris.home}/docs/zh-CN/docs/admin-manual/config/fe-config.md + ${doris.home}/docs/en/docs/advanced/variables-template.md + ${doris.home}/docs/zh-CN/docs/advanced/variables-template.md + ${doris.home}/docs/en/docs/advanced/variables.md + ${doris.home}/docs/zh-CN/docs/advanced/variables.md + + + @@ -936,6 +956,10 @@ under the License. org.apache.maven.plugins maven-clean-plugin + 3.1.0 + + ${skip.clean} + auto-clean diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java new file mode 100644 index 0000000000..c54486d995 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java @@ -0,0 +1,303 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.common.Config; +import org.apache.doris.common.ConfigBase.ConfField; +import org.apache.doris.common.ExperimentalUtil; +import org.apache.doris.common.ExperimentalUtil.ExperimentalType; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.VariableMgr; + +import com.google.common.collect.Maps; +import org.apache.commons.io.output.FileWriterWithEncoding; +import org.apache.parquet.Strings; +import org.jetbrains.annotations.NotNull; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; + +/** + * This class is used to generate doc for FE config and session variable. + * The doc is generated from Config.java and SessionVariable.java + */ +public class DocGenerator { + + private static final String PLACEHOLDER = "<--DOC_PLACEHOLDER-->"; + private static final String[] TYPE = new String[] {"类型：", "Type: "}; + private static final String[] DEFAULT_VALYUE = new String[] {"默认值：", "Default: "}; + private static final String[] OPTIONS = new String[] {"可选值：", "Options: "}; + private static final String[] CONF_MUTABLE = new String[] {"是否可动态修改：", "Mutable: "}; + private static final String[] CONF_MASTER_ONLY = new String[] {"是否为 Master FE 节点独有的配置项：", + "Master only: "}; + private static final String[] VAR_READ_ONLY = new String[] {"只读变量：", "Read Only: "}; + private static final String[] VAR_GLOBAL_ONLY = new String[] {"仅全局变量：", "Global only: "}; + + private String configDocTemplatePath; + private String configDocTemplatePathCN; + private String configDocOutputPath; + private String configDocOutputPathCN; + private String sessionVariableDocTemplatePath; + private String sessionVariableDocTemplatePathCN; + private String sessionVariableDocOutputPath; + private String sessionVariableDocOutputPathCN; + + private enum Lang { + CN(0), + EN(1); + + private int idx; + + Lang(int idx) { + this.idx = idx; + } + } + + public DocGenerator(String configDocTemplatePath, String configDocTemplatePathCN, + String configDocOutputPath, String configDocOutputPathCN, + String sessionVariableDocTemplatePath, String sessionVariableDocTemplatePathCN, + String sessionVariableDocOutputPath, String sessionVariableDocOutputPathCN) { + this.configDocTemplatePath = configDocTemplatePath; + this.configDocTemplatePathCN = configDocTemplatePathCN; + this.configDocOutputPath = configDocOutputPath; + this.configDocOutputPathCN = configDocOutputPathCN; + this.sessionVariableDocOutputPath = sessionVariableDocOutputPath; + this.sessionVariableDocTemplatePathCN = sessionVariableDocTemplatePathCN; + this.sessionVariableDocTemplatePath = sessionVariableDocTemplatePath; + this.sessionVariableDocOutputPathCN = sessionVariableDocOutputPathCN; + } + + public void generate() throws Exception { + generateConfigDoc(); + generateSessionVariableDoc(); + } + + private void generateConfigDoc() throws Exception { + // 1. CN + String contentCN = readDocTemplate(this.configDocTemplatePathCN); + contentCN = contentCN.replace(PLACEHOLDER, genFEConfigDoc(Lang.CN)); + // 2. EN + String content = readDocTemplate(this.configDocTemplatePath); + content = content.replace(PLACEHOLDER, genFEConfigDoc(Lang.EN)); + // 3. write CN + writeDoc(contentCN, this.configDocOutputPathCN); + // 4. write EN + writeDoc(content, this.configDocOutputPath); + } + + private void generateSessionVariableDoc() throws Exception { + // 1. CN + String contentCN = readDocTemplate(this.sessionVariableDocTemplatePathCN); + contentCN = contentCN.replace(PLACEHOLDER, genSessionVariableDoc(Lang.CN)); + // 2. EN + String content = readDocTemplate(this.sessionVariableDocTemplatePath); + content = content.replace(PLACEHOLDER, genSessionVariableDoc(Lang.EN)); + // 3. write CN + writeDoc(contentCN, this.sessionVariableDocOutputPathCN); + // 4. write EN + writeDoc(content, this.sessionVariableDocOutputPath); + } + + private String readDocTemplate(String templatePath) throws Exception { + StringBuilder sb = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(templatePath))) { + String line; + while ((line = br.readLine()) != null) { + sb.append(line).append("\n"); + } + } + return sb.toString(); + } + + private void writeDoc(String content, String outputPath) throws Exception { + try (BufferedWriter bw = new BufferedWriter(new FileWriterWithEncoding(outputPath, StandardCharsets.UTF_8))) { + bw.write(content); + } + } + + // generate doc for FE configs. + // Content will be sorted by config name. + private String genFEConfigDoc(Lang lang) throws IllegalAccessException { + Map sortedDoc = Maps.newTreeMap(); + Class confClass = Config.class; + for (Field field : confClass.getFields()) { + try { + String res = genSingleConfFieldDoc(field, lang); + if (!Strings.isNullOrEmpty(res)) { + sortedDoc.put(field.getName(), res); + } + } catch (Exception e) { + System.out.println("Failed to generate doc for field: " + field.getName()); + throw e; + } + } + return printSortedMap(sortedDoc); + } + + // Generate doc for a single config field. + private String genSingleConfFieldDoc(Field field, Lang lang) throws IllegalAccessException { + StringBuilder sb = new StringBuilder(); + ConfField confField = field.getAnnotation(ConfField.class); + if (confField == null) { + return null; + } + String configName = field.getName(); + if (confField.expType() == ExperimentalType.EXPERIMENTAL) { + configName = ExperimentalUtil.EXPERIMENTAL_PREFIX + configName; + } + sb.append("### `").append(configName).append("`\n\n"); + sb.append(confField.description()[lang.idx]).append("\n\n"); + sb.append(TYPE[lang.idx]).append("`").append(field.getType().getSimpleName()).append("`\n\n"); + sb.append(DEFAULT_VALYUE[lang.idx]).append("`").append(getStringValue(field, null)).append("`\n\n"); + if (confField.options().length > 0) { + sb.append(OPTIONS[lang.idx]); + for (int i = 0; i < confField.options().length; i++) { + sb.append("`").append(confField.options()[i]).append("`"); + if (i != confField.options().length - 1) { + sb.append(", "); + } + } + sb.append("\n\n"); + } + sb.append(CONF_MUTABLE[lang.idx]).append("`").append(confField.mutable()).append("`\n\n"); + sb.append(CONF_MASTER_ONLY[lang.idx]).append("`").append(confField.masterOnly()).append("`\n\n"); + return sb.toString(); + } + + private static String getStringValue(Field field, Object instance) throws IllegalAccessException { + if (field.getType().isArray()) { + return Arrays.toString((Object[]) field.get(instance)); + } else { + return String.valueOf(field.get(instance)); + } + } + + // generate doc for Session Variables + // Content will be sorted by variables' name. + private String genSessionVariableDoc(Lang lang) throws IllegalAccessException { + Map sortedDoc = Maps.newTreeMap(); + // 1. session variables + SessionVariable sv = new SessionVariable(); + Class svClass = SessionVariable.class; + for (Field field : svClass.getFields()) { + try { + String res = genSingleSessionVariableDoc(sv, field, lang); + if (!Strings.isNullOrEmpty(res)) { + sortedDoc.put(field.getAnnotation(VariableMgr.VarAttr.class).name(), res); + } + } catch (Exception e) { + System.out.println("Failed to generate doc for " + field.getName()); + throw e; + } + } + // 2. global variables + Class gvClass = GlobalVariable.class; + for (Field field : gvClass.getFields()) { + try { + String res = genSingleSessionVariableDoc(null, field, lang); + if (!Strings.isNullOrEmpty(res)) { + sortedDoc.put(field.getAnnotation(VariableMgr.VarAttr.class).name(), res); + } + } catch (Exception e) { + System.out.println("Failed to generate doc for field: " + field.getName()); + throw e; + } + } + return printSortedMap(sortedDoc); + } + + @NotNull + private static String printSortedMap(Map sortedDoc) { + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : sortedDoc.entrySet()) { + sb.append(entry.getValue()); + } + return sb.toString(); + } + + private String genSingleSessionVariableDoc(SessionVariable sv, Field field, Lang lang) + throws IllegalAccessException { + VariableMgr.VarAttr varAttr = field.getAnnotation(VariableMgr.VarAttr.class); + if (varAttr == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + String varName = varAttr.name(); + if (varAttr.expType() == ExperimentalType.EXPERIMENTAL) { + varName = ExperimentalUtil.EXPERIMENTAL_PREFIX + varName; + } + sb.append("### `").append(varName).append("`\n\n"); + sb.append(varAttr.description()[lang.idx]).append("\n\n"); + sb.append(TYPE[lang.idx]).append("`").append(field.getType().getSimpleName()).append("`\n\n"); + sb.append(DEFAULT_VALYUE[lang.idx]).append("`").append(getStringValue(field, sv)).append("`\n\n"); + if (varAttr.options().length > 0) { + sb.append(OPTIONS[lang.idx]); + for (int i = 0; i < varAttr.options().length; i++) { + sb.append("`").append(varAttr.options()[i]).append("`"); + if (i != varAttr.options().length - 1) { + sb.append(", "); + } + } + sb.append("\n\n"); + } + sb.append(VAR_READ_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.READ_ONLY).append("`\n\n"); + sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.GLOBAL).append("`\n\n"); + return sb.toString(); + } + + /** + * generate config and session variable doc from given templates + * + * @param args args[0]: config doc template path + * args[1]: config doc template path CN + * args[2]: config doc output path + * args[3]: config doc output path CN + * args[4]: session variable doc template path + * args[5]: session variable doc template path CN + * args[6]: session variable doc output path + * args[7]: session variable doc output path CN + */ + public static void main(String[] args) { + String configDocTemplatePath = args[0]; + String configDocTemplatePathCN = args[1]; + String configDocOutputPath = args[2]; + String configDocOutputPathCN = args[3]; + String sessionVariableDocTemplatePath = args[4]; + String sessionVariableDocTemplatePathCN = args[5]; + String sessionVariableDocOutputPath = args[6]; + String sessionVariableDocOutputPathCN = args[7]; + DocGenerator docGenerator = new DocGenerator( + configDocTemplatePath, configDocTemplatePathCN, + configDocOutputPath, configDocOutputPathCN, + sessionVariableDocTemplatePath, sessionVariableDocTemplatePathCN, + sessionVariableDocOutputPath, sessionVariableDocOutputPathCN); + try { + docGenerator.generate(); + System.out.println("Done!"); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/IcebergTableCreationRecordMgr.java b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/IcebergTableCreationRecordMgr.java index a98dd7f2ab..a7adc1bbd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/IcebergTableCreationRecordMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/IcebergTableCreationRecordMgr.java @@ -257,7 +257,7 @@ public class IcebergTableCreationRecordMgr extends MasterDaemon { } public boolean isQueueFull() { - return tableCreationRecordQueue.size() >= Config.max_iceberg_table_creation_record_size; + return tableCreationRecordQueue.size() >= 2000; } private void readLock() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteJob.java index 4a330f878a..3723c226c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteJob.java @@ -204,6 +204,6 @@ public class DeleteJob extends AbstractTxnStateChangeCallback { } // timeout is between 30 seconds to 5 min long timeout = Math.max(totalTablets.size() * Config.tablet_delete_timeout_second * 1000L, 30000L); - return Math.min(timeout, Config.load_straggler_wait_second * 1000L); + return Math.min(timeout, Config.delete_job_max_timeout_second * 1000L); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DppScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DppScheduler.java index 6352e766ac..c037f25b60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DppScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DppScheduler.java @@ -385,7 +385,7 @@ public class DppScheduler { } // check input size limit - int inputSizeLimitGB = Config.load_input_size_limit_gb; + int inputSizeLimitGB = 0; if (inputSizeLimitGB != 0) { if (totalSizeB > inputSizeLimitGB * GB) { String failMsg = "Input file size[" + (float) totalSizeB / GB + "GB]" diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/LoadJob.java index 684db1b997..cf5220edf3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/LoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/LoadJob.java @@ -586,12 +586,6 @@ public class LoadJob implements Writable { return deleteInfo; } - public long getDeleteJobTimeout() { - // timeout is between 30 seconds to 5 min - long timeout = Math.max(idToTabletLoadInfo.size() * Config.tablet_delete_timeout_second * 1000L, 30000L); - return Math.min(timeout, Config.load_straggler_wait_second * 1000L); - } - @Override public String toString() { return "LoadJob [id=" + id + ", dbId=" + dbId + ", label=" + label + ", timeoutSecond=" + timeoutSecond diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java index bc1465f476..0f4aebbc30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java @@ -44,7 +44,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.common.DataQualityException; import org.apache.doris.common.DdlException; import org.apache.doris.common.DuplicatedRequestException; @@ -617,7 +616,7 @@ public class SparkLoadJob extends BulkLoadJob { } // if all replicas are finished or stay in quorum finished for long time, try to commit it. - long stragglerTimeout = Config.load_straggler_wait_second * 1000; + long stragglerTimeout = 300 * 1000; if ((quorumFinishTimestamp > 0 && System.currentTimeMillis() - quorumFinishTimestamp > stragglerTimeout) || fullTablets.containsAll(totalTablets)) { canCommitJob = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java index ff2840ef60..1e6fff1382 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -97,7 +97,7 @@ public final class GlobalVariable { @VariableMgr.VarAttr(name = VALIDATE_PASSWORD_POLICY, flag = VariableMgr.GLOBAL) public static long validatePasswordPolicy = 0; - // Don't allow create instance. + // Don't allow to create instance. private GlobalVariable() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 298710ee7a..00fb85e17c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -321,8 +321,6 @@ public class SessionVariable implements Serializable, Writable { // if it is setStmt, we needn't collect session origin value public boolean isSingleSetVar = false; - - @VariableMgr.VarAttr(name = INSERT_VISIBLE_TIMEOUT_MS, needForward = true) public long insertVisibleTimeoutMs = DEFAULT_INSERT_VISIBLE_TIMEOUT_MS; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 9d4bbb2a3a..f714bb2d8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -636,7 +636,7 @@ public class VariableMgr { } @Retention(RetentionPolicy.RUNTIME) - public static @interface VarAttr { + public @interface VarAttr { // Name in show variables and set statement; String name(); @@ -659,6 +659,15 @@ public class VariableMgr { boolean fuzzy() default false; ExperimentalType expType() default ExperimentalType.NONE; + + // description for this config item. + // There should be 2 elements in the array. + // The first element is the description in Chinese. + // The second element is the description in English. + String[] description() default {"待补充", "TODO"}; + + // Enum options for this config item, if it has. + String[] options() default {}; } private static class VarContext { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendOptions.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendOptions.java index 8a6f83cd83..ba7cdab462 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendOptions.java @@ -24,7 +24,6 @@ import org.apache.doris.common.util.NetUtils; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.net.InetAddresses; -import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -44,15 +43,6 @@ public class FrontendOptions { public static void init() throws UnknownHostException { localAddr = null; - if (!"0.0.0.0".equals(Config.frontend_address)) { - if (!InetAddressValidator.getInstance().isValidInet4Address(Config.frontend_address)) { - throw new UnknownHostException("invalid frontend_address: " + Config.frontend_address); - } - localAddr = InetAddress.getByName(Config.frontend_address); - LOG.info("use configured address. {}", localAddr); - return; - } - analyzePriorityCidrs(); // if not set frontend_address, get a non-loopback ip diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/DppSchedulerTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/DppSchedulerTest.java deleted file mode 100644 index 9e0fdbc9b6..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/load/DppSchedulerTest.java +++ /dev/null @@ -1,224 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.load; - -import org.apache.doris.common.Config; -import org.apache.doris.common.util.CommandResult; -import org.apache.doris.common.util.UnitTestUtil; -import org.apache.doris.common.util.Util; -import org.apache.doris.thrift.TEtlState; - -import mockit.Expectations; -import mockit.Mocked; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class DppSchedulerTest { - private DppScheduler dppScheduler; - - @Mocked - private Util util; - - @Before - public void setUp() { - UnitTestUtil.initDppConfig(); - dppScheduler = new DppScheduler(Load.dppDefaultConfig); - } - - @Ignore - @Test - public void testCalcReduceNumByInputSize() throws Exception { - // mock hadoop count - String fileInfos = " 0 1 1000000000 /label2/export/label2.10007.10007.10005\n" - + " 0 1 1000000001 /label2/export/label2.10007.10007.10006"; - CommandResult commandResult = new CommandResult(); - commandResult.setReturnCode(0); - commandResult.setStdout(fileInfos); - new Expectations(util) { - { - Util.executeCommand(anyString, (String[]) any); - times = 3; - result = commandResult; - } - }; - - // get method - Method calcReduceNumByInputSize = UnitTestUtil.getPrivateMethod( - DppScheduler.class, "calcReduceNumByInputSize", new Class[] {Set.class}); - - // normal test - Set inputPaths = new HashSet(); - Config.load_input_size_limit_gb = 0; - Config.dpp_bytes_per_reduce = 2000000000; - Assert.assertEquals(2, calcReduceNumByInputSize.invoke(dppScheduler, new Object[] {inputPaths})); - Config.dpp_bytes_per_reduce = 2000000002; - Assert.assertEquals(1, calcReduceNumByInputSize.invoke(dppScheduler, new Object[] {inputPaths})); - - // input file size exceeds limit - Config.load_input_size_limit_gb = 1; - try { - calcReduceNumByInputSize.invoke(dppScheduler, new Object[]{inputPaths}); - Assert.assertTrue(false); - } catch (Exception e) { - Assert.assertTrue(true); - } - Config.load_input_size_limit_gb = 0; - } - - @Test - public void testCalcReduceNumByTablet() throws Exception { - Map jobConf = new HashMap(); - Map tables = new HashMap(); - jobConf.put("tables", tables); - Map table = new HashMap(); - tables.put("0", table); - Map views = new HashMap(); - table.put("views", views); - Map view1 = new HashMap(); - view1.put("hash_mod", 10); - views.put("view1", view1); - Map view2 = new HashMap(); - List rangeList = new ArrayList(); - for (int i = 0; i < 5; i++) { - rangeList.add(i); - } - view2.put("key_ranges", rangeList); - views.put("view2", view2); - - Method calcReduceNumByTablet = UnitTestUtil.getPrivateMethod( - DppScheduler.class, "calcReduceNumByTablet", new Class[] {Map.class}); - Assert.assertEquals(15, calcReduceNumByTablet.invoke(dppScheduler, new Object[] {jobConf})); - } - - @Test - public void testGetEtlJobStatus() { - String jobStatus = "Job: job_201501261830_12231\n" - + "file: hdfs://host:54310/system/mapred/job_201501261830_12231/job.xml\n" - + "tracking URL: http://host:8030/jobdetails.jsp?jobid=job_201501261830_12231\n" - + "job state: 1\n" - + "map() completion: 0.9036233\n" - + "reduce() completion: 0.0\n" - + "Counters: 14\n" - + " File Systems\n" - + " HDFS bytes read=398065481\n" - + " DPP\n" - + " dpp.abnorm.ALL=0\n" - + " dpp.norm.ALL=0\n" - + " Map-Reduce Framework\n" - + " Map input records=4085933\n" - + " Map output bytes=503053858"; - CommandResult commandResult = new CommandResult(); - commandResult.setReturnCode(0); - commandResult.setStdout(jobStatus); - new Expectations(util) { - { - Util.executeCommand(anyString, (String[]) any); - times = 1; - result = commandResult; - } - }; - - EtlStatus status = dppScheduler.getEtlJobStatus("etlJobId"); - Assert.assertEquals(TEtlState.RUNNING, status.getState()); - Assert.assertEquals("0", status.getCounters().get("dpp.abnorm.ALL")); - Assert.assertEquals("0.9036233", status.getStats().get("map() completion")); - } - - @Test - public void testGetEtlFileList() { - String outputPath = "/label_0"; - CommandResult successLsResult = new CommandResult(); - successLsResult.setReturnCode(0); - CommandResult failLsResult = new CommandResult(); - failLsResult.setReturnCode(-1); - CommandResult successTestDirResult = new CommandResult(); - successTestDirResult.setReturnCode(0); - CommandResult failTestDirResult = new CommandResult(); - failTestDirResult.setReturnCode(-1); - - String files = "-rw-r--r-- 3 palo palo 29896160 2015-02-03 13:10 /label_0/export/label_0.32241.32241.0\n" - + "-rw-r--r-- 3 palo palo 29896161 2015-02-03 13:10 /label_0/export/label_0.32241.32241.1"; - successLsResult.setStdout(files); - new Expectations(util) { - { - Util.executeCommand(anyString, (String[]) any); - times = 6; - returns( - // success - successLsResult, - // ls fail - failLsResult, - // outputPath not exist - failTestDirResult, - // ls fail - failLsResult, - // success - successTestDirResult, - // fileDir not exist - failTestDirResult - ); - } - }; - Map fileMap = dppScheduler.getEtlFiles(outputPath); - Assert.assertEquals(2, fileMap.size()); - Assert.assertNull(dppScheduler.getEtlFiles(outputPath)); - - fileMap = dppScheduler.getEtlFiles(outputPath); - Assert.assertNotNull(fileMap); - Assert.assertTrue(fileMap.isEmpty()); - } - - @Test - public void testKillEtlJob() { - CommandResult commandResult = new CommandResult(); - new Expectations(util) { - { - Util.executeCommand(anyString, (String[]) any); - times = 1; - result = commandResult; - } - }; - - dppScheduler.killEtlJob("etlJobId"); - } - - @Test - public void testGetEtlOutputPath() { - DppConfig dppConfig = Load.dppDefaultConfig.getCopiedDppConfig(); - long dbId = 0; - String loadLabel = "test_label"; - String etlOutputDir = "10000"; - - String actualPath = DppScheduler.getEtlOutputPath(dppConfig.getFsDefaultName(), dppConfig.getOutputPath(), - dbId, loadLabel, etlOutputDir); - String expectedPath = dppConfig.getFsDefaultName() + dppConfig.getOutputPath() + "/" + String.valueOf(dbId) - + "/" + loadLabel + "/" + etlOutputDir; - Assert.assertEquals(expectedPath, actualPath); - } - -}