From 8df63bc191fd123df5986084c1e453db732fda29 Mon Sep 17 00:00:00 2001 From: WingC <1018957763@qq.com> Date: Thu, 16 Jan 2020 21:54:26 +0800 Subject: [PATCH] [Doc] Add en doc for dynamic partition feature (#2764) --- .../administrator-guide/dynamic-partition.md | 2 +- .../dynamic-partition_EN.md | 185 ++++++++++++++++++ .../Data Definition/ALTER TABLE_EN.md | 10 +- .../Data Definition/CREATE TABLE_EN.md | 60 +++++- .../SHOW DYNAMIC PARTITION TABLES_EN.md | 29 +++ 5 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 docs/documentation/en/administrator-guide/dynamic-partition_EN.md create mode 100644 docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES_EN.md diff --git a/docs/documentation/cn/administrator-guide/dynamic-partition.md b/docs/documentation/cn/administrator-guide/dynamic-partition.md index 4ac8c20373..9c60118f6b 100644 --- a/docs/documentation/cn/administrator-guide/dynamic-partition.md +++ b/docs/documentation/cn/administrator-guide/dynamic-partition.md @@ -91,7 +91,7 @@ ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "true", "dynamic ### 停止动态分区功能 -如果需要对集群中所有动态分区表停止动态分区功能，则需要在fe.conf中设置`dynamic_partition_enable=true` +如果需要对集群中所有动态分区表停止动态分区功能，则需要在fe.conf中设置`dynamic_partition_enable=false` 如果需要对指定表停止动态分区功能，则可以通过以下命令修改表的属性 ``` diff --git a/docs/documentation/en/administrator-guide/dynamic-partition_EN.md b/docs/documentation/en/administrator-guide/dynamic-partition_EN.md new file mode 100644 index 0000000000..3dc9ef12bc --- /dev/null +++ b/docs/documentation/en/administrator-guide/dynamic-partition_EN.md @@ -0,0 +1,185 @@ + + +# Dynamic Partition + +Dynamic partition is a new feature introduced in Doris verion 0.12. It's designed to manage partition's Time-to-Life (TTL), reducing the burden on users. + +The original design, implementation and effect can be referred to [ISSUE 2262](https://github.com/apache/incubator-doris/issues/2262)。 + +Currently, the function of adding partitions dynamically is implemented, and the next version will support removing partitions dynamically. + +## Noun Interpretation + +* FE: Frontend, the front-end node of Doris. Responsible for metadata management and request access. +* BE: Backend, Doris's back-end node. Responsible for query execution and data storage. + +## Principle + +In some scenarios, the user will create partitions for the table according to the day and perform routine tasks regularly every day. In this case, the user needs to manually manage the partition, otherwise the data import may fail because the partition is forgot to create, which brings additional maintenance costs to the user. + +The design of implementation is that FE will starts a background thread that determines whether or not to start the thread and the scheduling frequency of the thread based on the parameters `dynamic_partition_enable` and `dynamic_partition_check_interval_seconds` in `fe.conf`. + +When create a olap table, the `dynamic_partition` properties will be assigned. FE will parse `dynamic_partition` properties and check the legitimacy of the input parameters firstly, and then persist the properties to FE metadata, register the table to the list of dynamic partition at the same time. Daemon thread will scan the dynamic partition list periodically according to the configuration parameters, +read dynamic partition properties of the table, and doing the task of adding partitions. The scheduling information of each time will be kept in the memory of FE. You can check whether the scheduling task is successful through `SHOW DYNAMIC PARTITION TABLES`. + +## Usage + +### Establishment of tables + +When creating a table, you can specify the attribute `dynamic_partition` in `PROPERTIES`, which means that the table is a dynamic partition table. + +Examples: + +``` +CREATE TABLE example_db.dynamic_partition +( +k1 DATE, +k2 INT, +k3 SMALLINT, +v1 VARCHAR(2048), +v2 DATETIME DEFAULT "2014-02-04 15:36:00" +) +ENGINE=olap +DUPLICATE KEY(k1, k2, k3) +PARTITION BY RANGE (k1) +( +PARTITION p1 VALUES LESS THAN ("2014-01-01"), +PARTITION p2 VALUES LESS THAN ("2014-06-01"), +PARTITION p3 VALUES LESS THAN ("2014-12-01") +) +DISTRIBUTED BY HASH(k2) BUCKETS 32 +PROPERTIES( +"storage_medium" = "SSD", +"dynamic_partition.enable" = "true" +"dynamic_partition.time_unit" = "DAY", +"dynamic_partition.end" = "3", +"dynamic_partition.prefix" = "p", +"dynamic_partition.buckets" = "32" + ); +``` +Create a dynamic partition table, specify enable dynamic partition features, take today is 2020-01-08 for example, at every time of scheduling, will create today and after 3 days in advance of four partitions +(if the partition is existed, the task will be ignored), partition name respectively according to the specified prefix `p20200108` `p20200109` `p20200110` `p20200111`, each partition to 32 the number of points barrels, each partition scope is as follows: +``` +[types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) +[types: [DATE]; keys: [2020-01-09]; ‥types: [DATE]; keys: [2020-01-10]; ) +[types: [DATE]; keys: [2020-01-10]; ‥types: [DATE]; keys: [2020-01-11]; ) +[types: [DATE]; keys: [2020-01-11]; ‥types: [DATE]; keys: [2020-01-12]; ) +``` + +### Enable Dynamic Partition Feature + +1. First of all, `dynamic_partition_enable=true` needs to be set in fe.conf, which can be specified by modifying the configuration file when the cluster starts up, or dynamically modified by HTTP interface at run time + +2. If you need to add dynamic partitioning properties to a table prior to version 0.12, you need to modify the properties of the table with the following command + +``` +ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.end" = "3", "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32"); +``` + +### Disable Dynamic Partition Feature + +If you need to stop dynamic partitioning for all dynamic partitioning tables in the cluster, you need to set 'dynamic_partition_enable=true' in fe.conf + +If you need to stop dynamic partitioning for a specified table, you can modify the properties of the table with the following command + +``` +ALTER TABLE dynamic_partition set ("dynamic_partition.enable" = "false") +``` + +### Modify Dynamic Partition Properties + +You can modify the properties of the dynamic partition with the following command + +``` +ALTER TABLE dynamic_partition set("key" = "value") +``` + +### Check Dynamic Partition Table Scheduling Status + +You can further view the scheduling of dynamic partitioned tables by using the following command: + +``` +SHOW DYNAMIC PARTITION TABLES; + ++-------------------+--------+----------+------+--------+---------+---------------------+---------------------+--------+------+ +| TableName | Enable | TimeUnit | End | Prefix | Buckets | LastUpdateTime | LastSchedulerTime | State | Msg | ++-------------------+--------+----------+------+--------+---------+---------------------+---------------------+--------+------+ +| dynamic_partition | true | DAY | 3 | p | 32 | 2020-01-08 20:19:09 | 2020-01-08 20:19:34 | NORMAL | N/A | ++-------------------+--------+----------+------+--------+---------+---------------------+---------------------+--------+------+ +1 row in set (0.00 sec) + +``` + +* LastUpdateTime: The last time of modifying dynamic partition properties +* LastSchedulerTime: The last time of performing dynamic partition scheduling +* State: The state of the last execution of dynamic partition scheduling +* Msg: Error message for the last time dynamic partition scheduling was performed + +## Advanced Operation + +### FE Configuration Item + +* dynamic\_partition\_enable + + Whether to enable Doris's dynamic partition feature. The default value is false, which is off. This parameter only affects the partitioning operation of dynamic partition tables, not normal tables. + +* dynamic\_partition\_check\_interval\_seconds + + The execution frequency of dynamically partitioned threads, by default 3600(1 hour), which means scheduled every 1 hour. + +### HTTP Restful API + +Doris provides an HTTP Restful API for modifying dynamic partition configuration parameters at run time. + +The API is implemented in FE, user can access it by `fe_host:fe_http_port`.The operation needs admin privilege. + +1. Set dynamic_partition_enable to true or false + + * Set to true + + ``` + GET /api/_set_config?dynamic_partition_enable=true + + For example: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=true + + Return Code：200 + ``` + + * Set to false + + ``` + GET /api/_set_config?dynamic_partition_enable=false + + For example: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_enable=false + + Return Code：200 + ``` + +2. Set the scheduling frequency for dynamic partition + + * Set schedule frequency to 12 hours. + + ``` + GET /api/_set_config?dynamic_partition_check_interval_seconds=432000 + + For example: curl --location-trusted -u username:password -XGET http://fe_host:fe_http_port/api/_set_config?dynamic_partition_check_interval_seconds=432000 + + Return Code：200 + ``` diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md index 2a31cb70ef..caf152f2fe 100644 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md +++ b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md @@ -146,7 +146,7 @@ under the License. 1) All columns in index must be written 2) value is listed after the key column - 6. Modify the properties of the table, currently supports modifying the bloom filter column and the colocate_with attribute. + 6. Modify the properties of the table, currently supports modifying the bloom filter column, the colocate_with attribute and the dynamic_partition attribute. grammar: PROPERTIES ("key"="value") note: @@ -286,6 +286,14 @@ under the License. 13. Change the bucketing mode of the table from Random Distribution to Hash Distribution ALTER TABLE example_db.my_table set ("distribution_type" = "hash"); + + 14. Modify the dynamic partition properties of the table (support adding dynamic partition properties to tables without dynamic partition properties) + + ALTER TABLE example_db.my_table set ("dynamic_partition_enable" = "false"); + + If you need to add dynamic partition attributes to a table without dynamic partition attributes, you need to specify all dynamic partition attributes + + ALTER TABLE example_db.my_table set ("dynamic_partition. Enable "= "true", dynamic_partition. Time_unit" = "DAY", "dynamic_partition. End "= "3", "dynamic_partition. Prefix" = "p", "Dynamic_partition. Buckets" = "32"); [rename] 1. Modify the table named table1 to table2 diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md index 6dc27119e1..843eb21b74 100644 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md +++ b/docs/documentation/en/sql-reference/sql-statements/Data Definition/CREATE TABLE_EN.md @@ -227,6 +227,28 @@ Syntax: "colocate_with"="table1" ) ``` + + 4) if you want to use the dynamic partitioning feature, specify it in properties + + ``` + PROPERTIES ( + "dynamic_partition.enable" = "true|false", + "dynamic_partition.time_unit" = "DAY|WEEK|MONTH", + "dynamic_partitoin.end" = "${integer_value}", + "dynamic_partition.prefix" = "${string_value}", + "dynamic_partition.buckets" = "${integer_value} + ) + ``` + + Dynamic_partition. Enable: specifies whether dynamic partitioning at the table level is enabled + + Dynamic_partition. Time_unit: used to specify the time unit for dynamically adding partitions, which can be selected as DAY, WEEK, and MONTH. + + Dynamic_partition. End: used to specify the number of partitions created in advance + + Dynamic_partition. Prefix: used to specify the partition name prefix to be created, such as the partition name prefix p, automatically creates the partition name p20200108 + + Dynamic_partition. Buckets: specifies the number of partition buckets that are automatically created ## example @@ -442,7 +464,7 @@ Syntax: ); ``` -10. 创建一个带有bitmap 索引的表 +10. Create a table with a bitmap index ``` CREATE TABLE example_db.table_hash @@ -459,6 +481,42 @@ Syntax: DISTRIBUTED BY HASH(k1) BUCKETS 32 PROPERTIES ("storage_type"="column"); ``` + +11. Create a dynamic partitioning table (dynamic partitioning needs to be enabled in FE configuration), which creates partitions 3 days in advance every day. For example, if today is' 2020-01-08 ', partitions named 'p20200108', 'p20200109', 'p20200110', 'p20200111' will be created. + + ``` + [types: [DATE]; keys: [2020-01-08]; ‥types: [DATE]; keys: [2020-01-09]; ) + [types: [DATE]; keys: [2020-01-09]; ‥types: [DATE]; keys: [2020-01-10]; ) + [types: [DATE]; keys: [2020-01-10]; ‥types: [DATE]; keys: [2020-01-11]; ) + [types: [DATE]; keys: [2020-01-11]; ‥types: [DATE]; keys: [2020-01-12]; ) + ``` + + ``` + CREATE TABLE example_db.dynamic_partition + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + v1 VARCHAR(2048), + v2 DATETIME DEFAULT "2014-02-04 15:36:00" + ) + ENGINE=olap + DUPLICATE KEY(k1, k2, k3) + PARTITION BY RANGE (k1) + ( + PARTITION p1 VALUES LESS THAN ("2014-01-01"), + PARTITION p2 VALUES LESS THAN ("2014-06-01"), + PARTITION p3 VALUES LESS THAN ("2014-12-01") + ) + DISTRIBUTED BY HASH(k2) BUCKETS 32 + PROPERTIES( + "storage_medium" = "SSD", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "32" + ); + ``` ## keyword diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES_EN.md new file mode 100644 index 0000000000..b0edcbf509 --- /dev/null +++ b/docs/documentation/en/sql-reference/sql-statements/Data Manipulation/SHOW DYNAMIC PARTITION TABLES_EN.md @@ -0,0 +1,29 @@ + + + # SHOW DYNAMIC PARTITION TABLES +## description + This statement is used to display all dynamically partitioned table states under the current db + Grammar： + SHOW DYNAMIC PARTITION TABLES [FROM db_name]; + + ## example + 1. Displays all dynamically partitioned table states for the database + SHOW DYNAMIC PARTITION TABLES FROM database; + + ## keyword + SHOW,DYNAMIC,PARTITION,TABLES