diff --git a/be/test/runtime/load_channel_mgr_test.cpp b/be/test/runtime/load_channel_mgr_test.cpp index 8997d77dac..bbb590c8b0 100644 --- a/be/test/runtime/load_channel_mgr_test.cpp +++ b/be/test/runtime/load_channel_mgr_test.cpp @@ -98,6 +98,7 @@ public: open_status = OLAP_SUCCESS; add_status = OLAP_SUCCESS; close_status = OLAP_SUCCESS; + config::streaming_load_rpc_max_alive_time_sec = 120; } private: }; diff --git a/docs/documentation/cn/getting-started/data-partition.md b/docs/documentation/cn/getting-started/data-partition.md index 992ea27718..293e2b9377 100644 --- a/docs/documentation/cn/getting-started/data-partition.md +++ b/docs/documentation/cn/getting-started/data-partition.md @@ -87,13 +87,14 @@ Doris 支持两层的数据划分。第一层是 Partition，仅支持 Range 的 1. Partition * Partition 列可以指定一列或多列。分区类必须为 KEY 列。多列分区的使用方式在后面 **多列分区** 小结介绍。 - * Partition 的区间界限是左闭右开。比如如上示例，如果想在 p201702 存储所有2月份的数据，则分区值需要输入 "2017-03-01"，即范围为：[2017-02-01, 2017-03-01)。 * 不论分区列是什么类型，在写分区值时，都需要加双引号。 * 分区列通常为时间列，以方便的管理新旧数据。 * 分区数量理论上没有上限。 * 当不使用 Partition 建表时，系统会自动生成一个和表名同名的，全值范围的 Partition。该 Partition 对用户不可见，并且不可删改。 + * Partition 支持通过 `VALUES LESS THAN (...)` 仅指定上界，系统会将前一个分区的上界作为该分区的下界，生成一个左闭右开的区间。通过，也支持通过 `VALUES [...)` 指定同时指定上下界，生成一个左闭右开的区间。 + + * 通过 `VALUES [...)` 同是指定上下界比较容易理解。这里举例说明，当使用 `VALUES LESS THAN (...)` 语句进行分区的增删操作时，分区范围的变化情况： - 这里举例说明，当分区在进行增删操作时，分区范围的变化情况。 * 如上示例，当建表完成后，会自动生成如下3个分区： ``` @@ -114,9 +115,9 @@ Doris 支持两层的数据划分。第一层是 Partition，仅支持 Range 的 * 此时我们删除分区 p201703，则分区结果如下： ``` - * p201701: [MIN_VALUE, 2017-02-01) - * p201702: [2017-02-01, 2017-03-01) - * p201705: [2017-04-01, 2017-06-01) + p201701: [MIN_VALUE, 2017-02-01) + p201702: [2017-02-01, 2017-03-01) + p201705: [2017-04-01, 2017-06-01) ``` > 注意到 p201702 和 p201705 的分区范围并没有发生变化，而这两个分区之间，出现了一个空洞：[2017-03-01, 2017-04-01)。即如果导入的数据范围在这个空洞范围内，是无法导入的。 @@ -124,17 +125,17 @@ Doris 支持两层的数据划分。第一层是 Partition，仅支持 Range 的 * 继续删除分区 p201702，分区结果如下： ``` - * p201701: [MIN_VALUE, 2017-02-01) - * p201705: [2017-04-01, 2017-06-01) - * 空洞范围变为：[2017-02-01, 2017-04-01) + p201701: [MIN_VALUE, 2017-02-01) + p201705: [2017-04-01, 2017-06-01) + 空洞范围变为：[2017-02-01, 2017-04-01) ``` * 现在增加一个分区 p201702new VALUES LESS THAN ("2017-03-01")，分区结果如下： ``` - * p201701: [MIN_VALUE, 2017-02-01) - * p201702new: [2017-02-01, 2017-03-01) - * p201705: [2017-04-01, 2017-06-01) + p201701: [MIN_VALUE, 2017-02-01) + p201702new: [2017-02-01, 2017-03-01) + p201705: [2017-04-01, 2017-06-01) ``` > 可以看到空洞范围缩小为：[2017-03-01, 2017-04-01) @@ -142,14 +143,15 @@ Doris 支持两层的数据划分。第一层是 Partition，仅支持 Range 的 * 现在删除分区 p201701，并添加分区 p201612 VALUES LESS THAN ("2017-01-01")，分区结果如下： ``` - * p201612: [MIN_VALUE, 2017-01-01) - * p201702new: [2017-02-01, 2017-03-01) - * p201705: [2017-04-01, 2017-06-01) + p201612: [MIN_VALUE, 2017-01-01) + p201702new: [2017-02-01, 2017-03-01) + p201705: [2017-04-01, 2017-06-01) ``` > 即出现了一个新的空洞：[2017-01-01, 2017-02-01) - - 综上，分区的删除不会改变已存在分区的范围。删除分区可能出现空洞。增加分区时，分区的下界紧接上一个分区的上界。 + + 综上，分区的删除不会改变已存在分区的范围。删除分区可能出现空洞。通过 `VALUES LESS THAN` 语句增加分区时，分区的下界紧接上一个分区的上界。 + 不可添加范围重叠的分区。 2. Bucket diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md index cf6b993c8a..722351d52d 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md +++ b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md @@ -15,10 +15,14 @@ partition 支持如下几种修改方式 1. 增加分区 语法： - ADD PARTITION [IF NOT EXISTS] partition_name VALUES LESS THAN [MAXVALUE|("value1")] ["key"="value"] + ADD PARTITION [IF NOT EXISTS] partition_name + partition_desc ["key"="value"] [DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]] 注意： - 1) 分区为左闭右开区间，用户指定右边界，系统自动确定左边界 + 1) partition_desc 支持一下两种写法： + * VALUES LESS THAN [MAXVALUE|("value1", ...)] + * VALUES [("value1", ...), ("value1", ...)) + 1) 分区为左闭右开区间，如果用户仅指定右边界，系统会自动确定左边界 2) 如果没有指定分桶方式，则自动使用建表使用的分桶方式 3) 如指定分桶方式，只能修改分桶数，不可修改分桶方式或分桶列 4) ["key"="value"] 部分可以设置分区的一些属性，具体说明见 CREATE TABLE @@ -158,6 +162,11 @@ 5. 删除分区 ALTER TABLE example_db.my_table DROP PARTITION p1; + + 6. 增加一个指定上下界的分区 + + ALTER TABLE example_db.my_table + ADD PARTITION p1 VALUES [("2014-01-01"), ("2014-02-01")); [rollup] 1. 创建 index: example_rollup_index，基于 base index（k1,k2,k3,v1,v2）。列式存储。 diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 9a3752d08d..63eeb65d31 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -133,6 +133,7 @@ 注意： 1) 分区一般用于时间维度的数据管理 2) 有数据回溯需求的，可以考虑首个分区为空分区，以便后续增加分区 + 2）Fixed Range 语法： PARTITION BY RANGE (k1, k2, k3, ...) @@ -226,8 +227,9 @@ "storage_cooldown_time" = "2015-06-04 00:00:00" ); - 3. 创建一个 olap 表，使用 Key Range 分区，使用Hash分桶，默认使用列存， + 3. 创建一个 olap 表，使用 Range 分区，使用Hash分桶，默认使用列存， 相同key的记录同时存在，设置初始存储介质和冷却时间 + 1）LESS THAN CREATE TABLE example_db.table_range ( @@ -257,6 +259,7 @@ [ {"2014-06-01"}, {"2014-12-01"} ) 不在这些分区范围内的数据将视为非法数据被过滤 + 2) Fixed Range CREATE TABLE table_range ( diff --git a/docs/documentation/en/getting-started/data-partition_EN.md b/docs/documentation/en/getting-started/data-partition_EN.md index 8a67fcabd2..7d0b6561db 100644 --- a/docs/documentation/en/getting-started/data-partition_EN.md +++ b/docs/documentation/en/getting-started/data-partition_EN.md @@ -1,269 +1,272 @@ -# Data partition +# Data Partition -This document mainly introduces Doris's table building and data partitioning, as well as possible problems and solutions in table building operation. +This document mainly introduces Doris's table construction and data partitioning, as well as problems and solutions that may be encountered in the construction of the table. -## Basic concepts +## Basic Concepts -In Doris, data is logically described in the form of tables. +In Doris, data is logically described in the form of a table. ### Row & Column -A table consists of rows and columns. Row is a row of user data. Column is used to describe different fields in a row of data. +A table includes rows (rows) and columns (columns). Row is a row of data for the user. Column is used to describe different fields in a row of data. -Columns can be divided into two categories: Key and Value. From a business perspective, Key and Value can correspond to dimension columns and indicator columns, respectively. From the point of view of the aggregation model, the same row of the Key column is aggregated into a row. The aggregation of the Value column is specified by the user when the table is built. For an introduction to more aggregation models, see [Doris data model] (. / data-model-rollup. md). +Column can be divided into two broad categories: Key and Value. From a business perspective, Key and Value can correspond to dimension columns and metric columns, respectively. From the perspective of the aggregation model, the same row of Key columns will be aggregated into one row. The way the Value column is aggregated is specified by the user when the table is built. For an introduction to more aggregation models, see the [Doris Data Model] (./data-model-rollup.md). -'35;\ 35;\ 35; Tablet & Partition +### Tablet & Partition -In Doris's storage engine, user data is divided horizontally into several data fragments (Tablets, also known as data buckets). Each table contains several rows of data. Data between Tablets does not intersect and is physically stored independently. +In Doris's storage engine, user data is horizontally divided into several data slices (also known as data buckets). Each tablet contains several rows of data. The data between the individual tablets does not intersect and is physically stored independently. -Multiple Tablets logically belong to different Partitions. A Tablet belongs to only one Partition. A Partition contains several Tablets. Because Tablet is physically stored independently, it can be considered that Partition is physically independent. Tablet is the smallest physical storage unit for data movement, replication and other operations. +Multiple tablets are logically attributed to different partitions. A tablet belongs to only one Partition. And a Partition contains several Tablets. Because the tablet is physically stored independently, it can be considered that the Partition is physically independent. Tablet is the smallest physical storage unit for data movement, replication, and so on. -Several Partitions form a Table. Partition can be regarded as the smallest management unit logically. Data import and deletion can be done for a single partition. +Several Partitions form a Table. Partition can be thought of as the smallest logical unit of management. Importing and deleting data can be done for one Partition or only for one Partition. -## Data partition +## Data division -We illustrate Doris's data partition with a table-building operation. +We use a table-building operation to illustrate Doris' data partitioning. -Doris's table building is a synchronization command. The command returns success, which means that the table building is successful. +Doris's built-in table is a synchronous command. If the command returns successfully, it means that the table is built successfully. -可以通过 `HELP CREATE TABLE;` 查看更多帮助。 +See more help with `HELP CREATE TABLE;`. -This section presents an example of how Doris is constructed. +This section introduces Doris's approach to building tables with an example. ``` CREATE TABLE IF NOT EXISTS example_db.expamle_tbl ( -`user_id` LARGEINT NOT NULL COMMENT "用户id", -"Date `date not null how `index `Fufu 8;'Back -`timestamp` DATETIME NOT NULL COMMENT "数据灌入的时间戳", -` City `VARCHAR (20) COMMENT `User City', -"Age" SMALLINT COMMENT "29992;" 25143;"24180;" 40836 ", -`sex` TINYINT COMMENT "用户性别", -"last visit date" DATETIME REPLACE DEFAULT "1970 -01 -01 00:00" COMMENT "25143;" 27425;"35775;" 3838382", -`cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费", -Best Answer: Best Answer -How about "99999" as time goes by??????????????????????????????????????????????????????????????????????????????????????????? +    `user_id` LARGEINT NOT NULL COMMENT "user id", +    `date` DATE NOT NULL COMMENT "Data fill in date time", +    `timestamp` DATETIME NOT NULL COMMENT "Timestamp of data being poured", +    `city` VARCHAR(20) COMMENT "The city where the user is located", +    `age` SMALLINT COMMENT "user age", +    `sex` TINYINT COMMENT "User Gender", +    `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "User last visit time", +    `cost` BIGINT SUM DEFAULT "0" COMMENT "Total user consumption", +    `max_dwell_time` INT MAX DEFAULT "0" COMMENT "User maximum dwell time", +    `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "User minimum dwell time", ) ENGINE=olap AGGREGATE KEY(`user_id`, `date`, `timestamp`, `city`, `age`, `sex`) PARTITION BY RANGE(`date`) ( -Segmentation `P201701 `value'equals `2017-02-01', -Segmentation `P201702 `Value equals'(2017-03-01'), -Segmentation `P201703'`Value less than' (2017-04-01') +    PARTITION `p201701` VALUES LESS THAN ("2017-02-01"), +    PARTITION `p201702` VALUES LESS THAN ("2017-03-01"), +    PARTITION `p201703` VALUES LESS THAN ("2017-04-01") ) DISTRIBUTED BY HASH(`user_id`) BUCKETS 16 PROPERTIES ( -"Replication\ num" = "3", -"storage_medium" = "SSD", -"Storage = U Cooldown = U Time" = "2018-01-12:00: +    "replication_num" = "3", +    "storage_medium" = "SSD", +    "storage_cooldown_time" = "2018-01-01 12:00:00" ); ``` ### Column Definition -Here we only take the AGGREGATE KEY data model as an example to illustrate. Refer to [Doris data model] (. / data-model-rollup. md) for more data models. +Here we only use the AGGREGATE KEY data model as an example. See the [Doris Data Model] (./data-model-rollup.md) for more data models. -The basic types of columns can be viewed by executing `HELP CREATE TABLE'in mysql-client. +The basic type of column can be viewed by executing `HELP CREATE TABLE;` in mysql-client. -In the AGGREGATE KEY data model, all columns without specified aggregation methods (SUM, REPLACE, MAX, MIN) are treated as Key columns. The rest are Value columns. +In the AGGREGATE KEY data model, all columns that do not specify an aggregation mode (SUM, REPLACE, MAX, MIN) are treated as Key columns. The rest is the Value column. When defining columns, you can refer to the following suggestions: 1. The Key column must precede all Value columns. -2. Choose integer type as far as possible. Because integer types are much more efficient than strings in computation and lookup. -3. For the selection principle of integer types of different lengths, follow the ** sufficient can **. -4. For the length of VARCHAR and STING types, follow ** suffice **. +2. Try to choose the type of integer. Because integer type calculations and lookups are much more efficient than strings. +3. For the selection principle of integer types of different lengths, follow ** enough to **. +4. For lengths of type VARCHAR and STRING, follow ** is sufficient. 5. The total byte length of all columns (including Key and Value) cannot exceed 100KB. -### Zoning and Bucketing +### Partitioning and binning -Doris supports two-tier data partitioning. The first layer is Partition, which only supports Range partitioning. The second layer is Bucket (Tablet), which only supports Hash partitioning. +Doris supports two levels of data partitioning. The first layer is Partition, which only supports the division of Range. The second layer is Bucket (Tablet), which only supports the way Hash is divided. -You can also use only one layer of partition. Bucket partitioning is only supported when using one-tier partitioning. +It is also possible to use only one layer of partitioning. When using a layer partition, only Bucket partitioning is supported. 1. Partition -* Partition columns can specify one or more columns. Partition classes must be KEY columns. The usage of multi-column partitions is introduced in the following ** summary of multi-column partitions. -* Partition's boundaries are left-closed and right-open. For example, if you want to store all February data in p201702, you need to enter the partition value "2017-03-01", that is, the range: [2017-02-01, 2017-03-01]. -* Regardless of the partition column type, double quotation marks are required when writing partition values. -* Partition columns are usually time columns to facilitate the management of old and new data. -* There is no theoretical upper limit on the number of zones. -* When Partition is not used to build tables, the system automatically generates a full-range Partition with the same name as the table name. The Partition is invisible to users and cannot be deleted. + * The Partition column can specify one or more columns. The partition class must be a KEY column. The use of multi-column partitions is described later in the **Multi-column partitioning** summary. +     + * Regardless of the type of partition column, double quotes are required when writing partition values. + * Partition columns are usually time columns for easy management of old and new data. + * There is no theoretical limit on the number of partitions. + * When you do not use Partition to build a table, the system will automatically generate a Partition with the same name as the table name. This Partition is not visible to the user and cannot be modified. + * Partition supports only the upper bound by `VALUES LESS THAN (...)`, the system will use the upper bound of the previous partition as the lower bound of the partition, and generate a left closed right open interval. Passing, also supports specifying the upper and lower bounds by `VALUES [...)`, and generating a left closed right open interval. -An example is given to illustrate the change of partition scope when adding or deleting partitions. -* As shown in the example above, when the table is completed, the following three partitions are automatically generated: + * It is easier to understand by specifying `VALUES [...)`. Here is an example of the change in partition range when adding or deleting partitions using the `VALUES LESS THAN (...)` statement: +     + * As the example above, when the table is built, the following 3 partitions are automatically generated: -``` -p201701: [MIN VALUE, 2017 -02 -01] -p201702: [2017-02-01, 2017-03-01) -p201703: [2017-03-01, 2017-04-01) -``` +            ``` +            P201701: [MIN_VALUE, 2017-02-01) +            P201702: [2017-02-01, 2017-03-01) +            P201703: [2017-03-01, 2017-04-01) +            ``` +         +        * When we add a partition p201705 VALUES LESS THAN ("2017-06-01"), the partition results are as follows: -* When we add a partition p201705 VALUES LESS THAN ("2017-06-01"), the partition results are as follows: +            ``` +            P201701: [MIN_VALUE, 2017-02-01) +            P201702: [2017-02-01, 2017-03-01) +            P201703: [2017-03-01, 2017-04-01) +            P201705: [2017-04-01, 2017-06-01) +            ``` +             +        * At this point we delete the partition p201703, the partition results are as follows: +         +            ``` +            p201701: [MIN_VALUE, 2017-02-01) +            p201702: [2017-02-01, 2017-03-01) +            p201705: [2017-04-01, 2017-06-01) +            ``` +             +            > Note that the partition range of p201702 and p201705 has not changed, and there is a hole between the two partitions: [2017-03-01, 2017-04-01). That is, if the imported data range is within this hole, it cannot be imported. +             +        * Continue to delete partition p201702, the partition results are as follows: +         +            ``` +            p201701: [MIN_VALUE, 2017-02-01) +            p201705: [2017-04-01, 2017-06-01) +            The void range becomes: [2017-02-01, 2017-04-01) +            ``` +             +        * Now add a partition p201702new VALUES LESS THAN ("2017-03-01"), the partition results are as follows: +             +            ``` +            p201701: [MIN_VALUE, 2017-02-01) +            p201702new: [2017-02-01, 2017-03-01) +            p201705: [2017-04-01, 2017-06-01) +            ``` +             +            > You can see that the hole size is reduced to: [2017-03-01, 2017-04-01) +             +        * Now delete partition p201701 and add partition p201612 VALUES LESS THAN ("2017-01-01"), the partition result is as follows: -``` -p201701: [MIN VALUE, 2017 -02 -01] -p201702: [2017-02-01, 2017-03-01) -p201703: [2017-03-01, 2017-04-01) -p201705: [2017-04-01, 2017-06-01) -``` - -* When we delete partition p201703, the partition results are as follows: - -``` -*p201701: [MIN VALUE, 2017 -02 -01] -* p201702: [2017-02-01, 2017-03-01) -* p201705: [2017-04-01, 2017-06-01) -``` - -> Notice that the partition ranges of p201702 and p201705 have not changed, and there is a gap between the two partitions: [2017-03-01, 2017-04-01]. That is, if the imported data range is within this empty range, it is imported as before. - -* Continue to delete partition p201702, partition results are as follows: - -``` -*p201701: [MIN VALUE, 2017 -02 -01] -* p201705: [2017-04-01, 2017-06-01) -* The void range becomes: [2017-02-01, 2017-04-01] -``` - -* Now add a partition p201702 new VALUES LESS THAN ("2017-03-01"). The partition results are as follows: - -``` -*p201701: [MIN VALUE, 2017 -02 -01] -*p201702new: [2017 -02 -01, 2017 -03 -01] -*p201705: [2017 -04 -01, 2017 -06 -01] -``` - -> It can be seen that the void range is reduced to: [2017-03-01, 2017-04-01] - -* Now delete partition p201701 and add partition p201612 VALUES LESS THAN ("2017-01-01"). The partition results are as follows: - -``` -*p201612: [MIN VALUE, 2017 -01 -01] -*p201702new: [2017 -02 -01, 2017 -03 -01] -*p201705: [2017 -04 -01, 2017 -06 -01] -``` - -> A new void appears: [2017-01-01, 2017-02-01] - -In summary, deletion of partitions does not change the scope of existing partitions. Deleting partitions may cause holes. When partitions are added, the lower bound of a partition is immediately followed by the upper bound of a partition. -Partitions with overlapping ranges cannot be added. +            ``` +            p201612: [MIN_VALUE, 2017-01-01) +            p201702new: [2017-02-01, 2017-03-01) +            p201705: [2017-04-01, 2017-06-01) + ``` +             + > A new void appeared: [2017-01-01, 2017-02-01) +         +    In summary, the deletion of a partition does not change the scope of an existing partition. There may be holes in deleting partitions. When a partition is added by the `VALUES LESS THAN` statement, the lower bound of the partition immediately follows the upper bound of the previous partition. +     +    You cannot add partitions with overlapping ranges. 2. Bucket -* If Partition is used, the `DISTRIBUTED...'statement describes the partitioning rules of data within ** partitions. If Partition is not used, the partitioning rules for the data of the entire table are described. -* Bucket columns can be multiple columns, but must be Key columns. Bucket columns can be the same or different as ARTITION columns. -* The choice of bucket columns is a trade-off between ** query throughput ** and ** query concurrency **: +    * If a Partition is used, the `DISTRIBUTED ...` statement describes the division rules for the data in each partition. If you do not use Partition, it describes the rules for dividing the data of the entire table. +    * The bucket column can be multiple columns, but it must be a Key column. The bucket column can be the same or different from the Partition column. +    * The choice of bucket column is a trade-off between **query throughput** and **query concurrency**: -1. If multiple bucket columns are selected, the data will be more evenly distributed. But if the query condition does not contain the equivalent condition of all bucket columns, a query scans all buckets. This increases the throughput of queries, but increases the latency of individual queries. This approach is suitable for query scenarios with high throughput and low concurrency. -2. If only one or a few bucket columns are selected, point query can query only one bucket. This method is suitable for high concurrent point query scenarios. +        1. If you select multiple bucket columns, the data is more evenly distributed. However, if the query condition does not include the equivalent condition for all bucket columns, a query will scan all buckets. The throughput of such queries will increase, but the latency of a single query will increase. This method is suitable for large throughput and low concurrent query scenarios. +        2. If you select only one or a few bucket columns, the point query can query only one bucket. This approach is suitable for high-concurrency point query scenarios. +         +    * There is no theoretical limit on the number of buckets. -* There is theoretically no upper limit on the number of buckets. - -3. Suggestions on the quantity and data quantity of Partition and Bucket. - -* The total number of tables in a table is equal to (Partition num * Bucket num). -* The number of tables in a table is recommended to be slightly more than the number of disks in the entire cluster, regardless of capacity expansion. -* There is no upper and lower bound theoretically for the data volume of a single Tablet, but it is recommended to be within the range of 1G - 10G. If the amount of single Tablet data is too small, the aggregation effect of data is not good, and the pressure of metadata management is high. If the amount of data is too large, it is not conducive to the migration and completion of replicas, and will increase the cost of failed retries of Schema Change or Rollup operations (the granularity of these failed retries is Tablet). -* When Tablet's principle of data quantity conflicts with that of quantity, it is suggested that priority be given to the principle of data quantity. -* When tabulating, the number of Buckets per partition is specified uniformly. However, when adding partitions dynamically (`ADD PARTITION'), you can specify the number of Buckets for new partitions separately. This function can be used to deal with data shrinkage or expansion conveniently. -* Once specified, the number of Buckets for a Partition cannot be changed. Therefore, in determining the number of Buckets, it is necessary to consider the situation of cluster expansion in advance. For example, currently there are only three hosts, and each host has one disk. If the number of Buckets is set to 3 or less, concurrency cannot be improved even if machines are added later. -* For example, suppose there are 10 BEs, one disk per BE. If the total size of a table is 500 MB, 4-8 fragments can be considered. 5GB: 8-16. 50GB: 32. 500GB: Recommended partition, each partition size is about 50GB, each partition 16-32 partitions. 5TB: Recommended partitions, each partition size is about 50GB, each partition 16-32 partitions. - -> Note: The amount of data in the table can be viewed by the `show data'command, and the result is divided by the number of copies, that is, the amount of data in the table. +3. Recommendations on the number and amount of data for Partitions and Buckets. +    * The total number of tablets in a table is equal to (Partition num * Bucket num). +    * The number of tablets in a table, which is slightly more than the number of disks in the entire cluster, regardless of capacity expansion. +    * The data volume of a single tablet does not theoretically have an upper and lower bound, but is recommended to be in the range of 1G - 10G. If the amount of data for a single tablet is too small, the aggregation of the data is not good and the metadata management pressure is high. If the amount of data is too large, it is not conducive to the migration, completion, and increase the cost of Schema Change or Rollup operation failure retry (the granularity of these operations failure retry is Tablet). +    * When the tablet's data volume principle and quantity principle conflict, it is recommended to prioritize the data volume principle. +    * When building a table, the number of Buckets for each partition is uniformly specified. However, when dynamically increasing partitions (`ADD PARTITION`), you can specify the number of Buckets for the new partition separately. This feature can be used to easily reduce or expand data. +    * Once the number of Buckets for a Partition is specified, it cannot be changed. Therefore, when determining the number of Buckets, you need to consider the expansion of the cluster in advance. For example, there are currently only 3 hosts, and each host has 1 disk. If the number of Buckets is only set to 3 or less, then even if you add more machines later, you can't increase the concurrency. +    * Give some examples: Suppose there are 10 BEs, one for each BE disk. If the total size of a table is 500MB, you can consider 4-8 shards. 5GB: 8-16. 50GB: 32. 500GB: Recommended partitions, each partition is about 50GB in size, with 16-32 shards per partition. 5TB: Recommended partitions, each with a size of around 50GB and 16-32 shards per partition. +     +    > Note: The amount of data in the table can be viewed by the `show data` command. The result is divided by the number of copies, which is the amount of data in the table. +     #### Multi-column partition -Doris supports specifying multiple columns as partitioned columns, as shown below: +Doris supports specifying multiple columns as partition columns, examples are as follows: ``` PARTITION BY RANGE(`date`, `id`) ( -Separating `P201701 `U1000 `values less than'(2017-02-01', `1000'), -Split `P201702 `U2000 `values less than'(2017-03-01', `2000'), -Segmentation `P201703 `U'all `values less than (`2017-04-01') +    PARTITION `p201701_1000` VALUES LESS THAN ("2017-02-01", "1000"), +    PARTITION `p201702_2000` VALUES LESS THAN ("2017-03-01", "2000"), +    PARTITION `p201703_all` VALUES LESS THAN ("2017-04-01") ) ``` -In the above example, we specify `date'(DATE type) and `id' (INT type) as partition columns. The final partition of the above example is as follows: +In the above example, we specify `date` (DATE type) and `id` (INT type) as partition columns. The resulting partitions in the above example are as follows: ``` -*p201701.1000: [(MIN VALUE, MIN VALUE), ("2017 -02 -01", "1000") -(2017 -02 -01, 1000), ("2017 -03 -01", "2000") -*p201703 all: [("2017 -03 -01", "2000"), ("2017 -04 -01", MIN VALUE)) +*p201701_1000: [(MIN_VALUE, MIN_VALUE), ("2017-02-01", "1000") ) +*p201702_2000: [("2017-02-01", "1000"), ("2017-03-01", "2000") ) +*p201703_all: [("2017-03-01", "2000"), ("2017-04-01", MIN_VALUE)) ``` -Note that the last partition user specifies only the partition value of the `date'column by default, so the partition value of the `id' column is filled in by default `MIN_VALUE'. When the user inserts data, the partition column values are compared sequentially, and the corresponding partitions are finally obtained. Examples are as follows: +Note that the last partition user defaults only the partition value of the `date` column, so the partition value of the `id` column will be filled with `MIN_VALUE` by default. When the user inserts data, the partition column values ​​are compared in order, and the corresponding partition is finally obtained. Examples are as follows: ``` -* Data - > Partition -*2017 -01, 200 --> p201701 -u 1000 -* 2017-01-01, 2000 --> p201701_1000 -*2017 -02 -01, 100 --> p201701 -u 1000 -* 2017-02-01, 2000 --> p201702_2000 -* 2017-02-15, 5000 --> p201702_2000 -* 2017-03-01, 2000 --> p201703_all -* 2017-03-10, 1-> P201703 all -* 2017-04-01, 1000 - > Unable to import -* 2017-05-01, 1000 - > Unable to import +* Data --> Partition +* 2017-01-01, 200 --> p201701_1000 +* 2017-01-01, 2000 --> p201701_1000 +* 2017-02-01, 100 --> p201701_1000 +* 2017-02-01, 2000 --> p201702_2000 +* 2017-02-15, 5000 --> p201702_2000 +* 2017-03-01, 2000 --> p201703_all +* 2017-03-10, 1 --> p201703_all +* 2017-04-01, 1000 --> Unable to import +* 2017-05-01, 1000 --> Unable to import ``` ### PROPERTIES -In the final PROPERTIES of the table statement, you can specify the following two parameters: +In the last PROPERTIES of the table statement, you can specify the following two parameters: -One copy +Replication_num -* Number of copies per Tablet. The default is 3. It is recommended that the default be maintained. In the table statement, the number of Tablet replicas in all Partitions is specified uniformly. When adding a new partition, you can specify the number of Tablet copies in the new partition separately. -* The number of copies can be modified at run time. It is strongly recommended that odd numbers be maintained. -* The maximum number of copies depends on the number of independent IP in the cluster (note that it is not the number of BEs). The principle of duplicate distribution in Doris is that duplicates of the same Tablet are not allowed to be distributed on the same physical machine, while identifying the physical machine is through IP. Therefore, even if three or more BE instances are deployed on the same physical machine, if the IP of these BEs is the same, only 1 copy number can be set. -* For some small and infrequently updated dimension tables, you can consider setting more copies. In this way, when Join queries, there is a greater probability of local data Join. +    * The number of copies per tablet. The default is 3, it is recommended to keep the default. In the build statement, the number of Tablet copies in all Partitions is uniformly specified. When you add a new partition, you can individually specify the number of copies of the tablet in the new partition. +    * The number of copies can be modified at runtime. It is strongly recommended to keep odd numbers. +    * The maximum number of copies depends on the number of independent IPs in the cluster (note that it is not the number of BEs). The principle of replica distribution in Doris is that the copies of the same Tablet are not allowed to be distributed on the same physical machine, and the physical machine is identified as IP. Therefore, even if 3 or more BE instances are deployed on the same physical machine, if the BEs have the same IP, you can only set the number of copies to 1. +    * For some small, and infrequently updated dimension tables, consider setting more copies. In this way, when joining queries, there is a greater probability of local data join. 2. storage_medium & storage\_cooldown\_time -* The data storage directory of BE can be explicitly specified as SSD or HDD (distinguished by. SSD or. HDD suffixes). When creating a table, you can specify all the media that Partition initially stores. Note that the suffix function is to explicitly specify the disk media without checking whether it matches the actual media type. -* The default initial storage medium is HDD. If SSD is specified, the data is initially stored on SSD. -* If storage cooldown time is not specified, data will be automatically migrated from SSD to HDD 7 days later by default. If storage cooldown time is specified, the data migrates only after the storage_cooldown_time time time is reached. -* Note that when storage_media is specified, this parameter is just a "best effort" setting. Even if SSD storage medium is not set up in the cluster, it will not report errors, but will be automatically stored in the available data directory. Similarly, if SSD media is inaccessible and space is insufficient, it may cause data to be stored directly on other available media initially. When data is migrated to HDD at maturity, if HDD media is inaccessible and space is insufficient, the migration may fail (but it will keep trying). +    * The BE data storage directory can be explicitly specified as SSD or HDD (differentiated by .SSD or .HDD suffix). When you build a table, you can uniformly specify the media for all Partition initial storage. Note that the suffix is ​​to explicitly specify the disk media without checking to see if it matches the actual media type. +    * The default initial storage medium is HDD. If specified as an SSD, the data is initially stored on the SSD. +    * If storage\_cooldown\_time is not specified, the data is automatically migrated from the SSD to the HDD after 7 days by default. If storage\_cooldown\_time is specified, the data will not migrate until the storage_cooldown_time time is reached. +    * Note that this parameter is just a "best effort" setting when storage_medium is specified. Even if no SSD storage media is set in the cluster, no error is reported and it is automatically stored in the available data directory. Similarly, if the SSD media is inaccessible and out of space, the data may initially be stored directly on other available media. When the data expires and is migrated to the HDD, if the HDD media is inaccessible and there is not enough space, the migration may fail (but will continue to try). ### ENGINE -In this example, the ENGINE type is olap, which is the default ENGINE type. In Doris, only this ENGINE type is responsible for data management and storage by Doris. Other ENGINE types, such as mysql, broker, es, etc., are essentially mappings to tables in other external databases or systems to ensure that Doris can read these data. Doris itself does not create, manage and store any tables and data of non-olap ENGINE type. +In this example, the type of ENGINE is olap, the default ENGINE type. In Doris, only this ENGINE type is managed and stored by Doris. Other ENGINE types, such as mysql, broker, es, etc., are essentially mappings to tables in other external databases or systems to ensure that Doris can read the data. And Doris itself does not create, manage, and store any tables and data of a non-olap ENGINE type. -### Others +### Other -` IF NOT EXISTS ` indicates that if the table has not been created, it will be created. Note that only the existence of table names is judged here, not whether the new table structure is the same as the existing table structure. So if there is a table with the same name but different structure, the command returns success, but it does not mean that a new table and new structure have been created. +    `IF NOT EXISTS` indicates that if the table has not been created, it is created. Note that only the table name is judged here, and it is not determined whether the new table structure is the same as the existing table structure. So if there is a table with the same name but different structure, the command will also return success, but it does not mean that a new table and a new structure have been created. -## Common Questions +## common problem -### Common problems in table building operation +### Build Table Operations FAQ -1. If a grammatical error occurs in a long table-building statement, the phenomenon of incomplete grammatical error hints may occur. Here is a list of possible grammatical errors for manual error correction: +1. If a syntax error occurs in a long build statement, a syntax error may be incomplete. Here is a list of possible syntax errors for manual error correction: -* Error in grammatical structure. Please read `HELP CREATE TABLE'carefully; `Check the relevant grammatical structure. -* Keep words. When a user-defined name encounters a reserved word, it needs to be caused by a back quotation mark `. It is recommended that all custom names be generated using this symbol. -* Chinese characters or full-angle characters. Non-utf8 coded Chinese characters, or hidden full-angle characters (spaces, punctuation, etc.), can lead to grammatical errors. It is recommended to use a text editor with invisible characters to check. +    * The syntax is incorrect. Please read `HELP CREATE TABLE;` carefully to check the relevant syntax structure. +    * Reserved words. When the user-defined name encounters a reserved word, it needs to be enclosed in the backquote ``. It is recommended that all custom names be generated using this symbol. +    * Chinese characters or full-width characters. Non-utf8 encoded Chinese characters, or hidden full-width characters (spaces, punctuation, etc.) can cause syntax errors. It is recommended to check with a text editor with invisible characters. 2. `Failed to create partition [xxx] . Timeout` -Doris tables are created in order of partition granularity. This error may be reported when a Partition creation fails. Even if you don't use Partition, you will report `Failed to create Partition'when a table is built incorrectly, because Doris will create an immutable default Artition for a table that does not specify Partition, as described earlier. +    Doris builds are created in order of Partition granularity. This error may be reported when a Partition creation fails. Even if you don't use Partition, you will report `Failed to create partition` when there is a problem with the built table, because as mentioned earlier, Doris will create an unchangeable default Partition for tables that do not have a Partition specified. +     +    When this error is encountered, it is usually the BE that has encountered problems creating data fragments. You can follow the steps below to troubleshoot: +     +    1. In fe.log, find the `Failed to create partition` log for the corresponding point in time. In this log, a series of numbers like `{10001-10010}` will appear. The first number of the pair is the Backend ID and the second number is the Tablet ID. As for the pair of numbers above, on the Backend with ID 10001, creating a tablet with ID 10010 failed. +    2. Go to the be.INFO log corresponding to Backend and find the log related to the tablet id in the corresponding time period. You can find the error message. +    3. Listed below are some common tablet creation failure errors, including but not limited to: +        * BE did not receive the relevant task, and the tablet id related log could not be found in be.INFO. Or the BE is created successfully, but the report fails. For the above questions, see [Deployment and Upgrade Documentation] to check the connectivity of FE and BE. +        * Pre-allocated memory failed. It may be that the length of a line in a row in the table exceeds 100KB. +        * `Too many open files`. The number of open file handles exceeds the Linux system limit. The handle limit of the Linux system needs to be modified. -When encountering this error, BE usually encounters problems in creating data fragments. Reference can be made to the following steps: +    You can also extend the timeout by setting `tablet_create_timeout_second=xxx` in fe.conf. The default is 2 seconds. -1. In fe.log, find the `Failed to create Partition'log for the corresponding time point. In this log, there will be a series of number pairs similar to {10001-10010}. The first number of the number pair represents the Backend ID, and the second number represents the Tablet ID. As the above number pair indicates, on the Backend with ID 10001, the creation of a Tablet with ID 10010 failed. -2. Go to the be.INFO log corresponding to Backend and find the tablet id-related log within the corresponding time period to find the error information. -3. Following are some common tablet creation failures, including but not limited to: -* BE did not receive the related task, and the tablet ID related log could not be found in be.INFO at this time. Or BE was created successfully, but failed to report. For the above questions, see the Deployment and Upgrade Document to check the connectivity between FE and BE. -* Pre-allocated memory failed. Perhaps the byte length of a row in the table exceeds 100KB. -*` Too many open files `. The number of open file handles exceeds the Linux system limit. Handle limit of Linux system needs to be modified. +3. The build table command does not return results for a long time. -You can also extend the timeout time by setting `tablet_create_timeout_second= xxx'in fe.conf. The default is 2 seconds. - -3. Tabulation commands do not return results for a long time. - -Doris's build command is a synchronization command. The command's timeout time is currently set in a relatively simple (tablet num * replication num) second. If you create more data fragments, and some of them fail to create fragments, you may be waiting for a longer timeout before returning an error. - -Normally, the build statement will return in a few seconds or a dozen seconds. If it takes more than one minute, it is recommended to cancel this operation directly and go to the FE or BE log to check for related errors. +    Doris's table creation command is a synchronous command. The timeout of this command is currently set to be relatively simple, ie (tablet num * replication num) seconds. If you create more data fragments and have fragment creation failed, it may cause an error to be returned after waiting for a long timeout. +     +    Under normal circumstances, the statement will return in a few seconds or ten seconds. If it is more than one minute, it is recommended to cancel this operation directly and go to the FE or BE log to view the related errors. \ No newline at end of file diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md index 35b38c0a4b..63d27eb00e 100644 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md +++ b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md @@ -1,244 +1,262 @@ # ALTER TABLE -## Description -This statement is used to modify an existing table. If no rollup index is specified, the default operation is base index. -该语句分为三种操作类型： schema change 、rollup 、partition -These three types of operations cannot appear in an ALTER TABLE statement at the same time. -Where schema change and rollup are asynchronous operations, task submission returns if it succeeds. You can then use the SHOW ALTER command to view progress. -Partition is a synchronous operation, and the return of the command indicates that the execution is complete. +## description +    This statement is used to modify an existing table. If no rollup index is specified, the base operation is the default. +    The statement is divided into three types of operations: schema change, rollup, partition +    These three types of operations cannot appear in an ALTER TABLE statement at the same time. +    Where schema change and rollup are asynchronous operations and are returned if the task commits successfully. You can then use the SHOW ALTER command to view the progress. +    Partition is a synchronous operation, and a command return indicates that execution is complete. -Grammar: -ALTER TABLE [database.]table -alter_clause1[, alter_clause2, ...]; +    grammar: +        ALTER TABLE [database.]table +        Alter_clause1[, alter_clause2, ...]; -alter_clause 分为 partition 、rollup、schema change 和 rename 四种。 +    The alter_clause is divided into partition, rollup, schema change, and rename. -partition 支持如下几种修改方式 -1. Adding partitions -Grammar: -ADD PARTITION [IF NOT EXISTS] partition_name VALUES LESS THAN [MAXVALUE|("value1")] ["key"="value"] -[DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]] -Be careful: -1) The partition is left-closed and right-open, the user specifies the right boundary, and the system automatically determines the left boundary. -2) If no bucket-dividing method is specified, the bucket-dividing method used in table-building will be used automatically. -3) If the barrel-dividing method is specified, only the number of barrels can be modified, but not the barrel-dividing method or the barrel-dividing column. -4) The ["key"= "value"] section can set some properties of the partition, as specified in CREATE TABLE +    Partition supports the following modifications +    Increase the partition +        grammar: +            ADD PARTITION [IF NOT EXISTS] partition_name +            Partition_desc ["key"="value"] +            [DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num]] +        note: +            1) partition_desc supports two ways of writing: +                * VALUES LESS THAN [MAXVALUE|("value1", ...)] +                * VALUES [("value1", ...), ("value1", ...)) +            1) The partition is the left closed right open interval. If the user only specifies the right boundary, the system will automatically determine the left boundary. +            2) If the bucket mode is not specified, the bucket method used by the built-in table is automatically used. +            3) If the bucket mode is specified, only the bucket number can be modified, and the bucket mode or bucket column cannot be modified. +            4) ["key"="value"] section can set some properties of the partition, see CREATE TABLE for details. -2. Delete partitions -Grammar: -DROP PARTITION [IF EXISTS] partition_name -Be careful: -1) A partitioned table should have at least one partition. -2) During the execution of DROP PARTITION, deleted partitions can be restored through RECOVER statements. See RECOVER statement for details - -3. Modify partition attributes -Grammar: -MODIFY PARTITION partition u name SET ("key" ="value",...) -Explain: -1) Currently, three attributes, storage_medium, storage_cooldown_time and replication_num, are supported to modify partitions. -2) For a single partition table, partition_name is the same table name. - -Rollup supports the following ways of creation: -One. 1.2.1.1.1.1.1.1.1. -Grammar: -ADD ROLLUP rollup_name (column_name1, column_name2, ...) -[FROM from index name] -[PROPERTIES ("key"="value", ...)] -Be careful: -1) If no from_index_name is specified, it is created by default from base index -2) The column in the rollup table must be an existing column from_index -3) In properties, you can specify the storage format. See CREATE TABLE for details. - -2. 1.2.2.2.2.2.2.2.2. -Grammar: -DROP ROLLUP rollup_name -[PROPERTIES ("key"="value", ...)] -Be careful: -1) Base index cannot be deleted -2) During the execution of DROP ROLLUP, the deleted rollup index can be restored by RECOVER statement. See RECOVER statement for details - - -schema change 支持如下几种修改方式： -1. Add a column to the specified index location -Grammar: -ADD COLUMN column_name column_type [KEY | agg_type] [DEFAULT "default_value"] -[AFTER column_name|FIRST] -[TO rollup_index_name] -[PROPERTIES ("key"="value", ...)] -Be careful: -1) If the value column is added to the aggregation model, agg_type needs to be specified -2) If the key column is added to the non-aggregate model, KEY keywords need to be specified. -3) Cannot add columns already existing in base index in rollup index -If necessary, you can re-create a rollup index. - -2. Add multiple columns to the specified index -Grammar: -ADD COLUMN (column_name1 column_type [KEY | agg_type] DEFAULT "default_value", ...) -[TO rollup_index_name] -[PROPERTIES ("key"="value", ...)] -Be careful: -1) If the value column is added to the aggregation model, agg_type needs to be specified -2) If the key column is added to the non-aggregate model, KEY keywords need to be specified. -3) Cannot add columns already existing in base index in rollup index -(You can re-create a rollup index if you need to) - -3. Delete a column from the specified index -Grammar: -DROP COLUMN column_name -[FROM rollup_index_name] -Be careful: -1) Partition columns cannot be deleted -2) If a column is deleted from base index, it will also be deleted if it is included in rollup index - -4. Modify the column type and column location of the specified index -Grammar: -MODIFY COLUMN column_name column_type [KEY | agg_type] [NULL | NOT NULL] [DEFAULT "default_value"] -[AFTER column_name|FIRST] -[FROM rollup_index_name] -[PROPERTIES ("key"="value", ...)] -Be careful: -1) The aggregation model needs to specify agg_type if it modifies the value column -2) If you modify the key column for a non-aggregated type, you need to specify the KEY keyword -3) The type of column can only be modified, and the other attributes of the column remain the same (that is, other attributes should be explicitly written in the statement according to the original attributes, see example 8). -4) Partition column cannot be modified -5) The following types of conversion are currently supported (accuracy loss is guaranteed by users) -TINYINT/SMALLINT/INT/BIGINT is converted to TINYINT/SMALLINT/INT/BIGINT/DOUBLE. -LARGEINT 转换成 DOUBLE -VARCHAR 25345;'20462;' 25913;'38271;' 24230s; -6) Conversion from NULL to NOT NULL is not supported - -5. Reordering columns with specified index -Grammar: -ORDER BY (column_name1, column_name2, ...) -[FROM rollup_index_name] -[PROPERTIES ("key"="value", ...)] -Be careful: -1) All columns in index should be written out -2) Value is listed after the key column - -6. Modify table attributes, which currently support modifying bloom filter columns and colocate_with attributes -Grammar: -PROPERTIES ("key"="value") -Be careful: -You can also incorporate it into the schema change operation above to modify it, as shown in the following example - - -Rename supports the modification of the following names: -1. Modify the table name -Grammar: -RENAME new_table_name; - -2. 1.2.2.5.5.5.5.;5.5.5.5.5.5. -Grammar: -RENAME ROLLUP old_rollup_name new_rollup_name; - -3. 修改 partition 名称 -Grammar: -Rename old partition name and new partition name +    2. Delete the partition +        grammar: +            DROP PARTITION [IF EXISTS] partition_name +        note: +            1) Use a partitioned table to keep at least one partition. +            2) Execute DROP PARTITION For a period of time, the deleted partition can be recovered by the RECOVER statement. See the RECOVER statement for details. +             +    3. Modify the partition properties +        grammar: +            MODIFY PARTITION partition_name SET ("key" = "value", ...) +        Description: +            1) The storage_medium, storage_cooldown_time, and replication_num attributes of the modified partition are currently supported. +            2) For single-partition tables, partition_name is the same as the table name. +         +    Rollup supports the following ways to create: +    1. Create a rollup index +        grammar: +            ADD ROLLUP rollup_name (column_name1, column_name2, ...) +            [FROM from_index_name] +            [PROPERTIES ("key"="value", ...)] +        note: +            1) If from_index_name is not specified, it is created by default from base index +            2) The columns in the rollup table must be existing columns in from_index +            3) In properties, you can specify the storage format. See CREATE TABLE for details. +             +    2. Delete the rollup index +        grammar: +            DROP ROLLUP rollup_name +            [PROPERTIES ("key"="value", ...)] +        note: +            1) Cannot delete base index +            2) Execute DROP ROLLUP For a period of time, the deleted rollup index can be restored by the RECOVER statement. See the RECOVER statement for details. +     +             +    Schema change supports the following modifications: +    1. Add a column to the specified location of the specified index +        grammar: +            ADD COLUMN column_name column_type [KEY | agg_type] [DEFAULT "default_value"] +            [AFTER column_name|FIRST] +            [TO rollup_index_name] +            [PROPERTIES ("key"="value", ...)] +        note: +            1) Aggregate model If you add a value column, you need to specify agg_type +            2) Non-aggregate models (such as DUPLICATE KEY) If you add a key column, you need to specify the KEY keyword. +            3) You cannot add a column that already exists in the base index to the rollup index +                Recreate a rollup index if needed +             +    2. Add multiple columns to the specified index +        grammar: +            ADD COLUMN (column_name1 column_type [KEY | agg_type] DEFAULT "default_value", ...) +            [TO rollup_index_name] +            [PROPERTIES ("key"="value", ...)] +        note: +            1) Aggregate model If you add a value column, you need to specify agg_type +            2) Non-aggregate model If you add a key column, you need to specify the KEY keyword. +            3) You cannot add a column that already exists in the base index to the rollup index +            (You can recreate a rollup index if needed) +     +    3. Remove a column from the specified index +        grammar: +            DROP COLUMN column_name +            [FROM rollup_index_name] +        note: +            1) Cannot delete partition column +            2) If the column is removed from the base index, it will also be deleted if the column is included in the rollup index +         +    4. Modify the column type and column position of the specified index +        grammar: +            MODIFY COLUMN column_name column_type [KEY | agg_type] [NULL | NOT NULL] [DEFAULT "default_value"] +            [AFTER column_name|FIRST] +            [FROM rollup_index_name] +            [PROPERTIES ("key"="value", ...)] +        note: +            1) Aggregate model If you modify the value column, you need to specify agg_type +            2) Non-aggregate type If you modify the key column, you need to specify the KEY keyword. +            3) Only the type of the column can be modified. The other attributes of the column remain as they are (ie other attributes need to be explicitly written in the statement according to the original attribute, see example 8) +            4) The partition column cannot be modified +            5) The following types of conversions are currently supported (accuracy loss is guaranteed by the user) +                TINYINT/SMALLINT/INT/BIGINT is converted to TINYINT/SMALLINT/INT/BIGINT/DOUBLE. +                Convert LARGEINT to DOUBLE +                VARCHAR supports modification of maximum length +            6) Does not support changing from NULL to NOT NULL +                 +    5. Reorder the columns of the specified index +        grammar: +            ORDER BY (column_name1, column_name2, ...) +            [FROM rollup_index_name] +            [PROPERTIES ("key"="value", ...)] +        note: +            1) All columns in index must be written +            2) value is listed after the key column +             +    6. Modify the properties of the table, currently supports modifying the bloom filter column and the colocate_with attribute. +        grammar: +            PROPERTIES ("key"="value") +        note: +            Can also be merged into the above schema change operation to modify, see the example below +      +    Rename supports modification of the following names: +    1. Modify the table name +        grammar: +            RENAME new_table_name; +             +    2. Modify the rollup index name +        grammar: +            RENAME ROLLUP old_rollup_name new_rollup_name; +             +    3. Modify the partition name +        grammar: +            RENAME PARTITION old_partition_name new_partition_name; +       ## example -[partition] -1. Increase partitions, existing partitions [MIN, 2013-01-01], increase partitions [2013-01-01, 2014-01-01], using default bucket partitioning -ALTER TABLE example_db.my_table -ADD PARTITION p1 VALUES LESS THAN ("2014-01-01"); +    [partition] +    1. Add partition, existing partition [MIN, 2013-01-01), add partition [2013-01-01, 2014-01-01), use default bucket mode +        ALTER TABLE example_db.my_table +        ADD PARTITION p1 VALUES LESS THAN ("2014-01-01"); -2. Increase partitions and use new buckets -ALTER TABLE example_db.my_table -ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") -DISTRIBUTED BY HASH(k1) BUCKETS 20; +    2. Increase the partition and use the new number of buckets +        ALTER TABLE example_db.my_table +        ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") +        DISTRIBUTED BY HASH(k1) BUCKETS 20; -3. Delete partitions -ALTER TABLE example_db.my_table -DROP PARTITION p1; +    3. Increase the partition and use the new number of copies +        ALTER TABLE example_db.my_table +        ADD PARTITION p1 VALUES LESS THAN ("2015-01-01") +        ("replication_num"="1"); -[rollup] -1. Create index: example_rollup_index, based on base index (k1, k2, k3, v1, v2). Formula storage. -ALTER TABLE example_db.my_table -ADD ROLLUP example_rollup_index(k1, k3, v1, v2) -PROPERTIES("storage_type"="column"); +    4. Modify the number of partition copies +        ALTER TABLE example_db.my_table +        MODIFY PARTITION p1 SET("replication_num"="1"); -2. Create index: example_rollup_index2, based on example_rollup_index (k1, k3, v1, v2) -ALTER TABLE example_db.my_table -ADD ROLLUP example_rollup_index2 (k1, v1) -FROM example_rollup_index; +    5. Delete the partition +        ALTER TABLE example_db.my_table +        DROP PARTITION p1; +         +    6. Add a partition that specifies the upper and lower bounds -3. Delete index: example_rollup_index2 -ALTER TABLE example_db.my_table -DROP ROLLUP example_rollup_index2; +        ALTER TABLE example_db.my_table +        ADD PARTITION p1 VALUES [("2014-01-01"), ("2014-02-01")); -[schema change] -1. Add a key column new_col (non-aggregate model) to col1 of example_rollup_index -ALTER TABLE example_db.my_table -ADD COLUMN new_col INT KEY DEFAULT "0" AFTER col1 -TO example_rollup_index; +    [rollup] +    1. Create index: example_rollup_index, based on base index(k1,k2,k3,v1,v2). Columnar storage. +        ALTER TABLE example_db.my_table +        ADD ROLLUP example_rollup_index(k1, k3, v1, v2) +        PROPERTIES("storage_type"="column"); +         +    2. Create index: example_rollup_index2, based on example_rollup_index(k1,k3,v1,v2) +        ALTER TABLE example_db.my_table +        ADD ROLLUP example_rollup_index2 (k1, v1) +        FROM example_rollup_index; +     +    3. Delete index: example_rollup_index2 +        ALTER TABLE example_db.my_table +        DROP ROLLUP example_rollup_index2; -2. Add a value column new_col (non-aggregate model) to col1 of example_rollup_index -ALTER TABLE example_db.my_table -ADD COLUMN new_col INT DEFAULT "0" AFTER col1 -TO example_rollup_index; +    [schema change] +    1. Add a key column new_col to the col1 of example_rollup_index (non-aggregate model) +        ALTER TABLE example_db.my_table +        ADD COLUMN new_col INT KEY DEFAULT "0" AFTER col1 +        TO example_rollup_index; -3. Add a key column new_col (aggregation model) to col1 of example_rollup_index -ALTER TABLE example_db.my_table -ADD COLUMN new_col INT DEFAULT "0" AFTER col1 -TO example_rollup_index; +    2. Add a value column new_col to the col1 of example_rollup_index (non-aggregate model) +          ALTER TABLE example_db.my_table +          ADD COLUMN new_col INT DEFAULT "0" AFTER col1 +          TO example_rollup_index; -4. Add a value column new_col SUM aggregation type (aggregation model) to col1 of example_rollup_index -ALTER TABLE example_db.my_table -ADD COLUMN new_col INT SUM DEFAULT "0" AFTER col1 -TO example_rollup_index; +    3. Add a key column new_col (aggregation model) to col1 of example_rollup_index +          ALTER TABLE example_db.my_table +          ADD COLUMN new_col INT DEFAULT "0" AFTER col1 +          TO example_rollup_index; -5. Add multiple columns to example_rollup_index (aggregation model) -ALTER TABLE example_db.my_table -ADD COLUMN (col1 INT DEFAULT "1", col2 FLOAT SUM DEFAULT "2.3") -TO example_rollup_index; +    4. Add a value column to the col1 of example_rollup_index. new_col SUM aggregation type (aggregation model) +          ALTER TABLE example_db.my_table +          ADD COLUMN new_col INT SUM DEFAULT "0" AFTER col1 +          TO example_rollup_index; +     +    5. Add multiple columns to the example_rollup_index (aggregate model) +        ALTER TABLE example_db.my_table +        ADD COLUMN (col1 INT DEFAULT "1", col2 FLOAT SUM DEFAULT "2.3") +        TO example_rollup_index; +     +    6. Remove a column from example_rollup_index +        ALTER TABLE example_db.my_table +        DROP COLUMN col2 +        FROM example_rollup_index; +         +    7. Modify the base index's col1 column to be of type BIGINT and move to the col2 column +        ALTER TABLE example_db.my_table +        MODIFY COLUMN col1 BIGINT DEFAULT "1" AFTER col2; -6. Delete a column from example_rollup_index -ALTER TABLE example_db.my_table -DROP COLUMN col2 -FROM example_rollup_index; +    8. Modify the maximum length of the val1 column of the base index. The original val1 is (val1 VARCHAR(32) REPLACE DEFAULT "abc") +        ALTER TABLE example_db.my_table +        MODIFY COLUMN val1 VARCHAR(64) REPLACE DEFAULT "abc"; +     +    9. Reorder the columns in example_rollup_index (set the original column order: k1, k2, k3, v1, v2) +        ALTER TABLE example_db.my_table +        ORDER BY (k3, k1, k2, v2, v1) +        FROM example_rollup_index; +         +    10. Perform both operations simultaneously +        ALTER TABLE example_db.my_table +        ADD COLUMN v2 INT MAX DEFAULT "0" AFTER k2 TO example_rollup_index, +        ORDER BY (k3,k1,k2,v2,v1) FROM example_rollup_index; -7. Modify the col1 column type of base index to BIGINT and move to the back of col2 column -ALTER TABLE example_db.my_table -MODIFY COLUMN col1 BIGINT DEFAULT "1" AFTER col2; +    11. Modify the bloom filter column of the table +        ALTER TABLE example_db.my_table SET ("bloom_filter_columns"="k1,k2,k3"); -8. 修改 base index 的 val1 列最大长度。原 val1 为 (val1 VARCHAR(32) REPLACE DEFAULT "abc") -ALTER TABLE example_db.my_table -MODIFY COLUMN val1 VARCHAR(64) REPLACE DEFAULT "abc"; +        Can also be merged into the above schema change operation (note that the syntax of multiple clauses is slightly different) +        ALTER TABLE example_db.my_table +        DROP COLUMN col2 +        PROPERTIES ("bloom_filter_columns"="k1,k2,k3"); -9. Rearrange the columns in example_rollup_index (set the original column order to k1, k2, k3, v1, v2) -ALTER TABLE example_db.my_table -ORDER BY (k3,k1,k2,v2,v1) -FROM example_rollup_index; +    12. Modify the Colocate property of the table -10. Perform two operations simultaneously -ALTER TABLE example_db.my_table -ADD COLUMN v2 INT MAX DEFAULT "0" AFTER k2 TO example_rollup_index, -ORDER BY (k3,k1,k2,v2,v1) FROM example_rollup_index; +        ALTER TABLE example_db.my_table set ("colocate_with" = "t1"); -11. 20462;- 259130;-bloom filter -210151; -ALTER TABLE example_db.my_table SET ("bloom_filter_columns"="k1,k2,k3"); - -You can also incorporate it into the schema change operation above (note that the grammar of multiple clauses is slightly different) -ALTER TABLE example_db.my_table -DROP COLUMN col2 -PROPERTIES ("bloom_filter_columns"="k1,k2,k3"); - -12. Modify the Colocate property of the table -ALTER TABLE example_db.my_table set ("colocate_with"="t1"); - -13. Change the Distribution type from Random to Hash - -ALTER TABLE example_db.my_table set ("distribution_type" = "hash"); - -[Rename] -1. Modify the table named Table 1 to table2 -ALTER TABLE table1 RENAME table2; - -2. 将表 example_table 中名为 rollup1 的 rollup index 修改为 rollup2 -ALTER TABLE example_table RENAME ROLLUP rollup1 rollup2; - -3. 将表 example_table 中名为 p1 的 partition 修改为 p2 -ALTER TABLE example_table RENAME PARTITION p1 p2; +    13. Change the bucketing mode of the table from Random Distribution to Hash Distribution +        ALTER TABLE example_db.my_table set ("distribution_type" = "hash"); +         +    [rename] +    1. Modify the table named table1 to table2 +        ALTER TABLE table1 RENAME table2; +         +    2. Modify the rollup index named rollup1 in the table example_table to rollup2 +        ALTER TABLE example_table RENAME ROLLUP rollup1 rollup2; +         +    3. Modify the partition named p1 in the table example_table to p2 +        ALTER TABLE example_table RENAME PARTITION p1 p2; +         ## keyword -ALTER,TABLE,ROLLUP,COLUMN,PARTITION,RENAME - +    ALTER, TABLE, ROLLUP, COLUMN, PARTITION, RENAME \ No newline at end of file diff --git a/fe/src/main/cup/sql_parser.cup b/fe/src/main/cup/sql_parser.cup index 156ff902b9..4ac0e37a20 100644 --- a/fe/src/main/cup/sql_parser.cup +++ b/fe/src/main/cup/sql_parser.cup @@ -1578,7 +1578,7 @@ partition_key_list ::= :} | partition_key_list:l COMMA KW_MAX_VALUE {: - l.add(PartitionValue.createMaxValue()); + l.add(PartitionValue.MAX_VALUE); RESULT = l; :} | STRING_LITERAL:item @@ -1587,7 +1587,7 @@ partition_key_list ::= :} | KW_MAX_VALUE {: - RESULT = Lists.newArrayList(PartitionValue.createMaxValue()); + RESULT = Lists.newArrayList(PartitionValue.MAX_VALUE); :} ; diff --git a/fe/src/main/java/org/apache/doris/analysis/AddPartitionClause.java b/fe/src/main/java/org/apache/doris/analysis/AddPartitionClause.java index 4ccb85aabd..9cc1b7028e 100644 --- a/fe/src/main/java/org/apache/doris/analysis/AddPartitionClause.java +++ b/fe/src/main/java/org/apache/doris/analysis/AddPartitionClause.java @@ -18,13 +18,7 @@ package org.apache.doris.analysis; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.io.Text; -import com.google.common.collect.Maps; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.util.Map; // clause which is used to add partition @@ -34,12 +28,6 @@ public class AddPartitionClause extends AlterClause { private DistributionDesc distributionDesc; private Map properties; - public AddPartitionClause() { - // for persist - partitionDesc = new SingleRangePartitionDesc(); - properties = Maps.newHashMap(); - } - public SingleRangePartitionDesc getSingeRangePartitionDesc() { return partitionDesc; } @@ -81,48 +69,4 @@ public class AddPartitionClause extends AlterClause { public String toString() { return toSql(); } - - @Override - public void write(DataOutput out) throws IOException { - String className = AddPartitionClause.class.getCanonicalName(); - Text.writeString(out, className); - - partitionDesc.write(out); - if (distributionDesc == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - distributionDesc.write(out); - } - - if (properties == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int count = properties.size(); - out.writeInt(count); - for (Map.Entry prop : properties.entrySet()) { - Text.writeString(out, prop.getKey()); - Text.writeString(out, prop.getValue()); - } - } - } - - @Override - public void readFields(DataInput in) throws IOException { - partitionDesc.readFields(in); - boolean has = in.readBoolean(); - if (has) { - distributionDesc = DistributionDesc.read(in); - } - - if (in.readBoolean()) { - int count = in.readInt(); - for (int i = 0; i < count; i++) { - String key = Text.readString(in); - String value = Text.readString(in); - properties.put(key, value); - } - } - } } diff --git a/fe/src/main/java/org/apache/doris/analysis/AddRollupClause.java b/fe/src/main/java/org/apache/doris/analysis/AddRollupClause.java index 454cf31cb7..0bbf4dc1d2 100644 --- a/fe/src/main/java/org/apache/doris/analysis/AddRollupClause.java +++ b/fe/src/main/java/org/apache/doris/analysis/AddRollupClause.java @@ -22,7 +22,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; -import org.apache.doris.common.io.Text; import org.apache.doris.common.util.PropertyAnalyzer; import com.google.common.base.Strings; @@ -30,8 +29,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import java.io.DataOutput; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; @@ -133,59 +130,4 @@ public class AddRollupClause extends AlterClause { public String toString() { return toSql(); } - - @Override - public void write(DataOutput out) throws IOException { - String className = AddRollupClause.class.getCanonicalName(); - Text.writeString(out, className); - - Text.writeString(out, rollupName); - - int count = columnNames.size(); - out.writeInt(count); - for (String colName : columnNames) { - Text.writeString(out, colName); - } - - if (baseRollupName == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - Text.writeString(out, baseRollupName); - } - - if (properties == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - count = properties.size(); - out.writeInt(count); - for (Map.Entry prop : properties.entrySet()) { - Text.writeString(out, prop.getKey()); - Text.writeString(out, prop.getValue()); - } - } - } - - public void readFields(java.io.DataInput in) throws IOException { - rollupName = Text.readString(in); - int count = in.readInt(); - for (int i = 0; i < count; i++) { - String colName = Text.readString(in); - columnNames.add(colName); - } - - if (in.readBoolean()) { - baseRollupName = Text.readString(in); - } - - if (in.readBoolean()) { - count = in.readInt(); - for (int i = 0; i < count; i++) { - String key = Text.readString(in); - String value = Text.readString(in); - properties.put(key, value); - } - } - } } diff --git a/fe/src/main/java/org/apache/doris/analysis/AlterClause.java b/fe/src/main/java/org/apache/doris/analysis/AlterClause.java index 4eb2275842..6dd222a863 100644 --- a/fe/src/main/java/org/apache/doris/analysis/AlterClause.java +++ b/fe/src/main/java/org/apache/doris/analysis/AlterClause.java @@ -18,20 +18,13 @@ package org.apache.doris.analysis; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; import org.apache.commons.lang.NotImplementedException; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.Map; // Alter table clause. -public class AlterClause implements Writable { +public class AlterClause { public void analyze(Analyzer analyzer) throws AnalysisException { throw new NotImplementedException(); } @@ -43,37 +36,4 @@ public class AlterClause implements Writable { public String toSql() { throw new NotImplementedException(); } - - public static AlterClause read(DataInput in) throws IOException { - String className = Text.readString(in); - if (className.startsWith("com.baidu.palo")) { - // we need to be compatible with former class name - className = className.replaceFirst("com.baidu.palo", "org.apache.doris"); - } - AlterClause alterClause = null; - try { - Class derivedClass = (Class) Class.forName(className); - alterClause = derivedClass.newInstance(); - Class[] paramTypes = { DataInput.class }; - Method readMethod = derivedClass.getMethod("readFields", paramTypes); - Object[] params = { in }; - readMethod.invoke(alterClause, params); - - return alterClause; - } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException - | SecurityException | IllegalArgumentException | InvocationTargetException e) { - throw new IOException("failed read AlterClause", e); - } - } - - @Override - public void write(DataOutput out) throws IOException { - throw new NotImplementedException(); - } - - @Override - public void readFields(DataInput in) throws IOException { - throw new NotImplementedException(); - } - } diff --git a/fe/src/main/java/org/apache/doris/analysis/AlterClusterClause.java b/fe/src/main/java/org/apache/doris/analysis/AlterClusterClause.java index 2f91b9d111..7e173b6a07 100644 --- a/fe/src/main/java/org/apache/doris/analysis/AlterClusterClause.java +++ b/fe/src/main/java/org/apache/doris/analysis/AlterClusterClause.java @@ -17,15 +17,12 @@ package org.apache.doris.analysis; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Map; - import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import java.util.Map; + public class AlterClusterClause extends AlterClause { private AlterClusterType type; private Map properties; @@ -60,32 +57,11 @@ public class AlterClusterClause extends AlterClause { return super.toSql(); } - @Override - public void write(DataOutput out) throws IOException { - // TODO Auto-generated method stub - super.write(out); - } - - @Override - public void readFields(DataInput in) throws IOException { - // TODO Auto-generated method stub - super.readFields(in); - } - public int getInstanceNum() { return instanceNum; } - public void setInstanceNum(int instanceNum) { - this.instanceNum = instanceNum; - } - public String getPassword() { return password; } - - public void setPassword(String password) { - this.password = password; - } - } diff --git a/fe/src/main/java/org/apache/doris/analysis/AlterTableStmt.java b/fe/src/main/java/org/apache/doris/analysis/AlterTableStmt.java index 7ca17d36f6..f58dd8c5a5 100644 --- a/fe/src/main/java/org/apache/doris/analysis/AlterTableStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/AlterTableStmt.java @@ -22,26 +22,23 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; -import org.apache.doris.common.io.Writable; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; -import com.google.common.collect.Lists; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.util.List; // Alter table statement. -public class AlterTableStmt extends DdlStmt implements Writable { +public class AlterTableStmt extends DdlStmt { private TableName tbl; private List ops; - public AlterTableStmt() { - // for persist - tbl = new TableName(); - ops = Lists.newArrayList(); + public AlterTableStmt(TableName tbl, List ops) { + this.tbl = tbl; + this.ops = ops; + } + + public void setTableName(String newTableName) { + tbl = new TableName(tbl.getDb(), newTableName); } public TableName getTbl() { @@ -52,14 +49,6 @@ public class AlterTableStmt extends DdlStmt implements Writable { return ops; } - public AlterTableStmt(TableName tbl, List ops) { - this.tbl = tbl; - this.ops = ops; - } - - public void setTableName(String newTableName) { - tbl = new TableName(tbl.getDb(), newTableName); - } @Override public void analyze(Analyzer analyzer) throws AnalysisException, UserException { @@ -103,24 +92,4 @@ public class AlterTableStmt extends DdlStmt implements Writable { public String toString() { return toSql(); } - - @Override - public void write(DataOutput out) throws IOException { - tbl.write(out); - int count = ops.size(); - out.writeInt(count); - for (AlterClause alterClause : ops) { - alterClause.write(out); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - tbl.readFields(in); - int count = in.readInt(); - for (int i = 0; i < count; i++) { - AlterClause alterClause = AlterClause.read(in); - ops.add(alterClause); - } - } } diff --git a/fe/src/main/java/org/apache/doris/analysis/PartitionDesc.java b/fe/src/main/java/org/apache/doris/analysis/PartitionDesc.java index 3dda238854..4f5e89148e 100644 --- a/fe/src/main/java/org/apache/doris/analysis/PartitionDesc.java +++ b/fe/src/main/java/org/apache/doris/analysis/PartitionDesc.java @@ -22,24 +22,16 @@ import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; import org.apache.commons.lang.NotImplementedException; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.util.List; import java.util.Map; -public class PartitionDesc implements Writable { +public class PartitionDesc { protected PartitionType type; - public PartitionDesc() { - } - public void analyze(List columnDefs, Map otherProperties) throws AnalysisException { throw new NotImplementedException(); } @@ -56,25 +48,4 @@ public class PartitionDesc implements Writable { throws DdlException { throw new NotImplementedException(); } - - public static PartitionDesc read(DataInput in) throws IOException { - PartitionType type = PartitionType.valueOf(Text.readString(in)); - if (type == PartitionType.RANGE) { - PartitionDesc desc = new RangePartitionDesc(); - desc.readFields(in); - return desc; - } else { - throw new IOException("Unknow partition type"); - } - } - - @Override - public void write(DataOutput out) throws IOException { - Text.writeString(out, type.name()); - } - - @Override - public void readFields(DataInput in) throws IOException { - throw new NotImplementedException(); - } } diff --git a/fe/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java b/fe/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java index 371372941d..88380f14b8 100644 --- a/fe/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java +++ b/fe/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java @@ -17,21 +17,18 @@ package org.apache.doris.analysis; -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; +import org.apache.doris.common.AnalysisException; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.collect.Lists; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.util.List; -// 用于表达在创建表、创建rollup中key range partition中所使用的key信息 -// 在知道具体列信息后，可以转成PartitionKey，仅仅在语法解析中短暂的有意义 -public class PartitionKeyDesc implements Writable { +// Describe the partition key values in create table or add partition clause +public class PartitionKeyDesc { + public static final PartitionKeyDesc MAX_VALUE = new PartitionKeyDesc(); + public enum PartitionRangeType { INVALID, LESS_THAN, @@ -41,30 +38,28 @@ public class PartitionKeyDesc implements Writable { private List lowerValues; private List upperValues; private PartitionRangeType partitionType; + public static PartitionKeyDesc createMaxKeyDesc() { - return new PartitionKeyDesc(); + return MAX_VALUE; } - public PartitionKeyDesc() { - partitionType = PartitionRangeType.LESS_THAN; //LESS_THAN is default type. + private PartitionKeyDesc() { + partitionType = PartitionRangeType.LESS_THAN; // LESS_THAN is default type. } - // used by SQL parser + // values less than public PartitionKeyDesc(List upperValues) { this.upperValues = upperValues; partitionType = PartitionRangeType.LESS_THAN; } + // fixed range public PartitionKeyDesc(List lowerValues, List upperValues) { this.lowerValues = lowerValues; this.upperValues = upperValues; partitionType = PartitionRangeType.FIXED; } - public void setLowerValues(List lowerValues) { - this.lowerValues = lowerValues; - } - public List getLowerValues() { return lowerValues; } @@ -74,7 +69,7 @@ public class PartitionKeyDesc implements Writable { } public boolean isMax() { - return lowerValues == null && upperValues == null; + return this == MAX_VALUE; } public boolean hasLowerValues() { @@ -89,65 +84,65 @@ public class PartitionKeyDesc implements Writable { return partitionType; } - public String toSql() { - if (this.isMax()) { - return "MAXVALUE"; + public void analyze(int partColNum) throws AnalysisException { + if (!isMax()) { + if (upperValues.isEmpty() || upperValues.size() > partColNum) { + throw new AnalysisException("Partiton values number is more than partition column number: " + toSql()); + } + } + + // currently, we do not support MAXVALUE in partition range values. eg: ("100", "200", MAXVALUE); + // maybe support later. + if (lowerValues != null) { + for (PartitionValue lowerVal : lowerValues) { + if (lowerVal.isMax()) { + throw new AnalysisException("Not support MAXVALUE in partition range values."); + } + } } if (upperValues != null) { - StringBuilder sb = new StringBuilder("("); - Joiner.on(", ").appendTo(sb, Lists.transform(upperValues, new Function() { - @Override - public String apply(PartitionValue v) { - return "'" + v.getStringValue() + "'"; + for (PartitionValue upperVal : upperValues) { + if (upperVal.isMax()) { + throw new AnalysisException("Not support MAXVALUE in partition range values."); } - })).append(")"); + } + } + } + + // returns: + // 1: MAXVALUE + // 2: ("100", "200", MAXVALUE) + // 3: [("100", "200"), ("300", "200")) + public String toSql() { + if (isMax()) { + return "MAXVALUE"; + } + + if (partitionType == PartitionRangeType.LESS_THAN) { + return getPartitionValuesStr(upperValues); + } else if (partitionType == PartitionRangeType.FIXED) { + StringBuilder sb = new StringBuilder("["); + sb.append(getPartitionValuesStr(lowerValues)).append(", ").append(getPartitionValuesStr(upperValues)); + sb.append(")"); return sb.toString(); } else { - return "()"; + return "INVALID"; } } - @Override - public void write(DataOutput out) throws IOException { - int count = lowerValues == null ? 0 : lowerValues.size(); - out.writeInt(count); - if (count > 0) { - for (PartitionValue value : lowerValues) { - Text.writeString(out, value.getStringValue()); + private String getPartitionValuesStr(List values) { + StringBuilder sb = new StringBuilder("("); + Joiner.on(", ").appendTo(sb, Lists.transform(values, new Function() { + @Override + public String apply(PartitionValue v) { + if (v.isMax()) { + return v.getStringValue(); + } else { + return "'" + v.getStringValue() + "'"; + } } - } - - count = upperValues == null ? 0: upperValues.size(); - out.writeInt(count); - if (count > 0) { - for (PartitionValue value : upperValues) { - Text.writeString(out, value.getStringValue()); - } - } - - } - - @Override - public void readFields(DataInput in) throws IOException { - int count = in.readInt(); - for (int i = 0; i < count; i++) { - String v = Text.readString(in); - if (v.equals("MAXVALUE")) { - lowerValues.add(new PartitionValue()); - } else { - lowerValues.add(new PartitionValue(v)); - } - } - - count = in.readInt(); - for (int i = 0; i < count; i++) { - String v = Text.readString(in); - if (v.equals("MAXVALUE")) { - upperValues.add(new PartitionValue()); - } else { - upperValues.add(new PartitionValue(v)); - } - } + })).append(")"); + return sb.toString(); } } diff --git a/fe/src/main/java/org/apache/doris/analysis/PartitionValue.java b/fe/src/main/java/org/apache/doris/analysis/PartitionValue.java index 6a2a414624..f61c4be669 100644 --- a/fe/src/main/java/org/apache/doris/analysis/PartitionValue.java +++ b/fe/src/main/java/org/apache/doris/analysis/PartitionValue.java @@ -21,27 +21,20 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; public class PartitionValue { + public static final PartitionValue MAX_VALUE = new PartitionValue(); + private String value; - private boolean isMaxValue; - public static PartitionValue createMaxValue() { - PartitionValue value = new PartitionValue(); - value.isMaxValue = true; - value.value = null; - return value; - } - - public PartitionValue() { + private PartitionValue() { } public PartitionValue(String value) { - this.isMaxValue = false; this.value = value; } public LiteralExpr getValue(Type type) throws AnalysisException { - if (isMaxValue) { + if (isMax()) { return LiteralExpr.createInfinity(type, true); } else { return LiteralExpr.create(value, type); @@ -49,11 +42,11 @@ public class PartitionValue { } public boolean isMax() { - return isMaxValue; + return this == MAX_VALUE; } public String getStringValue() { - if (isMaxValue) { + if (isMax()) { return "MAXVALUE"; } else { return value; diff --git a/fe/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java b/fe/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java index b7449e2bfd..005676a77b 100644 --- a/fe/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java +++ b/fe/src/main/java/org/apache/doris/analysis/RangePartitionDesc.java @@ -17,22 +17,18 @@ package org.apache.doris.analysis; +import org.apache.doris.analysis.PartitionKeyDesc.PartitionRangeType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.RangePartitionInfo; -import org.apache.doris.analysis.PartitionKeyDesc.PartitionRangeType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import org.apache.doris.common.io.Text; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; @@ -42,12 +38,6 @@ public class RangePartitionDesc extends PartitionDesc { private List partitionColNames; private List singleRangePartitionDescs; - public RangePartitionDesc() { - type = PartitionType.RANGE; - partitionColNames = Lists.newArrayList(); - singleRangePartitionDescs = Lists.newArrayList(); - } - public RangePartitionDesc(List partitionColNames, List singlePartitionDescs) { type = PartitionType.RANGE; @@ -192,36 +182,4 @@ public class RangePartitionDesc extends PartitionDesc { } return rangePartitionInfo; } - - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - - int count = partitionColNames.size(); - out.writeInt(count); - for (String colName : partitionColNames) { - Text.writeString(out, colName); - } - - count = singleRangePartitionDescs.size(); - out.writeInt(count); - for (SingleRangePartitionDesc singleRangePartitionDesc : singleRangePartitionDescs) { - singleRangePartitionDesc.write(out); - } - } - - @Override - public void readFields(DataInput in) throws IOException { - int count = in.readInt(); - for (int i = 0; i < count; i++) { - partitionColNames.add(Text.readString(in)); - } - - count = in.readInt(); - for (int i = 0; i < count; i++) { - SingleRangePartitionDesc desc = new SingleRangePartitionDesc(); - desc.readFields(in); - singleRangePartitionDescs.add(desc); - } - } } diff --git a/fe/src/main/java/org/apache/doris/analysis/SingleRangePartitionDesc.java b/fe/src/main/java/org/apache/doris/analysis/SingleRangePartitionDesc.java index e807458f84..ebf98215c5 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SingleRangePartitionDesc.java +++ b/fe/src/main/java/org/apache/doris/analysis/SingleRangePartitionDesc.java @@ -17,27 +17,22 @@ package org.apache.doris.analysis; +import org.apache.doris.analysis.PartitionKeyDesc.PartitionRangeType; import org.apache.doris.catalog.DataProperty; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.Pair; -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.PropertyAnalyzer; import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; import com.google.common.base.Joiner.MapJoiner; -import com.google.common.collect.Maps; +import com.google.common.base.Preconditions; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.util.Map; -public class SingleRangePartitionDesc implements Writable { +public class SingleRangePartitionDesc { private boolean isAnalyzed; private boolean ifNotExists; @@ -50,10 +45,6 @@ public class SingleRangePartitionDesc implements Writable { private Short replicationNum; private Pair versionInfo; - public SingleRangePartitionDesc() { - partitionKeyDesc = new PartitionKeyDesc(); - } - public SingleRangePartitionDesc(boolean ifNotExists, String partName, PartitionKeyDesc partitionKeyDesc, Map properties) { this.ifNotExists = ifNotExists; @@ -103,11 +94,7 @@ public class SingleRangePartitionDesc implements Writable { FeNameFormat.checkPartitionName(partName); - if (!partitionKeyDesc.isMax()) { - if (partitionKeyDesc.getUpperValues().isEmpty() || partitionKeyDesc.getUpperValues().size() > partColNum) { - throw new AnalysisException("Invalid partition value number: " + partitionKeyDesc.toSql()); - } - } + partitionKeyDesc.analyze(partColNum); if (otherProperties != null) { // use given properties @@ -150,8 +137,12 @@ public class SingleRangePartitionDesc implements Writable { public String toSql() { StringBuilder sb = new StringBuilder(); - sb.append("PARTITION "); - sb.append(partName + " VALUES LESS THEN "); + sb.append("PARTITION ").append(partName); + if (partitionKeyDesc.getPartitionType() == PartitionRangeType.LESS_THAN) { + sb.append(" VALUES LESS THEN "); + } else { + sb.append(" VALUES "); + } sb.append(partitionKeyDesc.toSql()); if (properties != null && !properties.isEmpty()) { @@ -167,51 +158,4 @@ public class SingleRangePartitionDesc implements Writable { public String toString() { return toSql(); } - - @Override - public void write(DataOutput out) throws IOException { - out.writeBoolean(isAnalyzed); - out.writeBoolean(ifNotExists); - Text.writeString(out, partName); - - partitionKeyDesc.write(out); - - if (properties == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - int count = properties.size(); - out.writeInt(count); - for (Map.Entry enytry : properties.entrySet()) { - Text.writeString(out, enytry.getKey()); - Text.writeString(out, enytry.getValue()); - } - } - - partitionDataProperty.write(out); - out.writeShort(replicationNum); - } - - @Override - public void readFields(DataInput in) throws IOException { - isAnalyzed = in.readBoolean(); - ifNotExists = in.readBoolean(); - partName = Text.readString(in); - - partitionKeyDesc.readFields(in); - - boolean hasProp = in.readBoolean(); - if (hasProp) { - properties = Maps.newHashMap(); - int count = in.readInt(); - for (int i = 0; i < count; i++) { - String key = Text.readString(in); - String value = Text.readString(in); - properties.put(key, value); - } - } - - partitionDataProperty = DataProperty.read(in); - replicationNum = in.readShort(); - } } diff --git a/fe/src/main/java/org/apache/doris/backup/MetaDownloadTask.java b/fe/src/main/java/org/apache/doris/backup/MetaDownloadTask.java deleted file mode 100644 index c76d21dc72..0000000000 --- a/fe/src/main/java/org/apache/doris/backup/MetaDownloadTask.java +++ /dev/null @@ -1,366 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.backup; - -import org.apache.doris.analysis.AddPartitionClause; -import org.apache.doris.analysis.AlterTableStmt; -import org.apache.doris.analysis.CreateTableStmt; -import org.apache.doris.analysis.SingleRangePartitionDesc; -import org.apache.doris.catalog.Catalog; -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.RangePartitionInfo; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.Table.TableType; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.UserException; -import org.apache.doris.common.util.CommandResult; -import org.apache.doris.common.util.Util; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class MetaDownloadTask extends ResultfulTask { - private static final Logger LOG = LogManager.getLogger(MetaDownloadTask.class); - - private final long jobId; - private final String dbName; - private final String label; - - private final String localDirName; - private final String remotePath; - - private final CommandBuilder commandBuilder; - private final PathBuilder pathBuilder; - - private final Map> tableToPartitionNames; - private final Map tableToCreateTableStmt; - private final Map tableToRollupStmt; - private final com.google.common.collect.Table tableToPartitionStmts; - private final Map tableToReplace; - - private final Map tableRenameMap; - - public MetaDownloadTask(long jobId, String dbName, String label, String localDirName, String remotePath, - PathBuilder pathBuilder, CommandBuilder commandBuilder, - Map> tableToPartitionNames, - Map tableToCreateTableStmt, - Map tableToRollupStmt, - com.google.common.collect.Table tableToPartitionStmts, - Map tableToReplace, Map tableRenameMap) { - this.jobId = jobId; - this.dbName = dbName; - this.label = label; - this.localDirName = localDirName; - this.remotePath = remotePath; - - this.pathBuilder = pathBuilder; - this.commandBuilder = commandBuilder; - this.tableToPartitionNames = tableToPartitionNames; - this.tableToCreateTableStmt = tableToCreateTableStmt; - this.tableToRollupStmt = tableToRollupStmt; - this.tableToPartitionStmts = tableToPartitionStmts; - this.tableToReplace = tableToReplace; - - this.tableRenameMap = tableRenameMap; - } - - @Override - public String call() { - try { - String local = pathBuilder.getRoot().getFullPath() + PathBuilder.MANIFEST_NAME; - String remote = PathBuilder.createPath(remotePath, PathBuilder.MANIFEST_NAME); - downloadFile(local, remote); - - // read manifest - DirSaver rootDir; - - rootDir = ObjectWriter.readManifest(local); - - LOG.info("get backup label: {}", rootDir.getName()); - - // rename rootDir to restore label so that it map to the current dir - rootDir.setName(localDirName); - pathBuilder.setRoot(rootDir); - - // check if all restore obj include - checkRestoreObjs(); - - } catch (Exception e) { - setErrMsg(e); - LOG.warn("get exception", e); - return errMsg; - } - - return null; - } - - private void checkRestoreObjs() throws UserException { - try { - Database db = Catalog.getInstance().getDb(dbName); - if (db == null) { - throw new UserException("Database[" + dbName + "] does not exist"); - } - - // 1.1 check restored objs exist - for (Map.Entry> entry : tableToPartitionNames.entrySet()) { - String newTableName = entry.getKey(); - String tableName = tableRenameMap.get(newTableName); - checkContains(PathBuilder.createPath(dbName, tableName)); - - Set partitionNames = entry.getValue(); - if (!partitionNames.isEmpty()) { - for (String partitionName : partitionNames) { - checkContains(PathBuilder.createPath(dbName, tableName, partitionName)); - } - } - } - - // 1.2 add all metas - if (tableToPartitionNames.isEmpty()) { - Preconditions.checkState(tableRenameMap.isEmpty()); - DirSaver dbDir = (DirSaver) pathBuilder.getRoot().getChild(dbName); - if (dbDir == null) { - throw new UserException("Backup path does not contains database[" + dbName + "]"); - } - Collection tableDirs = dbDir.getChildren(); - for (FileSaverI child : tableDirs) { - if (child instanceof DirSaver) { - Set partitionNames = Sets.newHashSet(); - tableToPartitionNames.put(child.getName(), partitionNames); - } - tableRenameMap.put(child.getName(), child.getName()); - } - } - - // 2. download all ddl file - String local = null; - String remote = null; - for (Map.Entry> entry : tableToPartitionNames.entrySet()) { - // create table stmt - String newTableName = entry.getKey(); - String tableName = tableRenameMap.get(newTableName); - local = PathBuilder.createPath(pathBuilder.getRoot().getFullPath(), dbName, newTableName, - PathBuilder.CREATE_TABLE_STMT_FILE); - remote = PathBuilder.createPath(remotePath, dbName, tableName, PathBuilder.CREATE_TABLE_STMT_FILE); - downloadFile(local, remote); - CreateTableStmt createTableStmt = ObjectWriter.readCreateTableStmt(local); - createTableStmt.setTableName(newTableName); - tableToCreateTableStmt.put(newTableName, createTableStmt); - - // rollup stmt if exists - String rollupStmtPath = PathBuilder.createPath(dbName, tableName, PathBuilder.ADD_ROLLUP_STMT_FILE); - if (pathBuilder.getRoot().contains(rollupStmtPath)) { - local = PathBuilder.createPath(pathBuilder.getRoot().getFullPath(), dbName, newTableName, - PathBuilder.ADD_ROLLUP_STMT_FILE); - remote = PathBuilder.createPath(remotePath, dbName, tableName, PathBuilder.ADD_ROLLUP_STMT_FILE); - downloadFile(local, remote); - List rollupStmts = ObjectWriter.readAlterTableStmt(local); - Preconditions.checkState(rollupStmts.size() == 1); - AlterTableStmt rollupStmt = rollupStmts.get(0); - rollupStmt.setTableName(newTableName); - tableToRollupStmt.put(newTableName, rollupStmt); - } - - // add partition stmt if exists - Set partitionNames = entry.getValue(); - if (!partitionNames.isEmpty()) { - List partitionStmts = Lists.newArrayList(); - for (String partitionName : partitionNames) { - local = PathBuilder.createPath(pathBuilder.getRoot().getFullPath(), dbName, newTableName, - partitionName, PathBuilder.ADD_PARTITION_STMT_FILE); - remote = PathBuilder.createPath(remotePath, dbName, tableName, partitionName, - PathBuilder.ADD_PARTITION_STMT_FILE); - downloadFile(local, remote); - partitionStmts.addAll(ObjectWriter.readAlterTableStmt(local)); - } - - for (AlterTableStmt partitionStmt : partitionStmts) { - Preconditions.checkState(partitionStmt.getOps().size() == 1); - AddPartitionClause clause = (AddPartitionClause) partitionStmt.getOps().get(0); - String partitionName = clause.getSingeRangePartitionDesc().getPartitionName(); - partitionStmt.setTableName(newTableName); - tableToPartitionStmts.put(newTableName, partitionName, partitionStmt); - } - } else { - // get all existed partitions - FileSaverI tableSaver = pathBuilder.getRoot().getChild(dbName).getChild(tableName); - if (!(tableSaver instanceof DirSaver)) { - throw new UserException("Table[" + tableName + "] dir does not exist"); - } - - List partitionStmts = Lists.newArrayList(); - DirSaver tableDir = (DirSaver) tableSaver; - Collection children = tableDir.getChildren(); - for (FileSaverI child : children) { - if (!(child instanceof DirSaver)) { - continue; - } - - DirSaver partitionDir = (DirSaver) child; - if (partitionDir.hasChild(PathBuilder.ADD_PARTITION_STMT_FILE)) { - local = PathBuilder.createPath(pathBuilder.getRoot().getFullPath(), dbName, newTableName, - partitionDir.getName(), PathBuilder.ADD_PARTITION_STMT_FILE); - remote = PathBuilder.createPath(remotePath, dbName, tableName, partitionDir.getName(), - PathBuilder.ADD_PARTITION_STMT_FILE); - downloadFile(local, remote); - partitionStmts.addAll(ObjectWriter.readAlterTableStmt(local)); - } - } - - for (AlterTableStmt partitionStmt : partitionStmts) { - Preconditions.checkState(partitionStmt.getOps().size() == 1); - AddPartitionClause clause = (AddPartitionClause) partitionStmt.getOps().get(0); - String partitionName = clause.getSingeRangePartitionDesc().getPartitionName(); - partitionStmt.setTableName(newTableName); - tableToPartitionStmts.put(newTableName, partitionName, partitionStmt); - } - } - } // end for tableToPartitions - - // 3. check validation - db.readLock(); - try { - for (Map.Entry> entry : tableToPartitionNames.entrySet()) { - String newTableName = entry.getKey(); - Set partitionNames = entry.getValue(); - Table table = db.getTable(newTableName); - if (table == null) { - tableToReplace.put(newTableName, true); - continue; - } else { - if (partitionNames.isEmpty()) { - throw new UserException("Table[" + newTableName + "]' already exist. " - + "Drop table first or restore to another table"); - } - } - - // table - CreateTableStmt stmt = tableToCreateTableStmt.get(newTableName); - if (table.getSignature(BackupVersion.VERSION_1) != stmt.getTableSignature()) { - throw new UserException("Table[" + newTableName + "]'s struct is not same"); - } - - // partition - Preconditions.checkState(!partitionNames.isEmpty()); - Preconditions.checkState(table.getType() == TableType.OLAP); - OlapTable olapTable = (OlapTable) table; - for (String partitionName : partitionNames) { - Partition partition = olapTable.getPartition(partitionName); - if (partition == null) { - // check range valid - checkRangeValid(olapTable, partitionName); - } else { - // do not allow overwrite a partition - throw new UserException("Partition[" + partitionName + "]' already exist in table[" - + newTableName + "]. Drop partition first or restore to another table"); - } - } - tableToReplace.put(newTableName, false); - } - } finally { - db.readUnlock(); - } - } catch (Exception e) { - throw new UserException(e.getMessage(), e); - } - } - - private void checkRangeValid(OlapTable olapTable, String partitionName) throws AnalysisException, DdlException { - AlterTableStmt stmt = tableToPartitionStmts.get(olapTable.getName(), partitionName); - RangePartitionInfo partitionInfo = (RangePartitionInfo) olapTable.getPartitionInfo(); - AddPartitionClause clause = (AddPartitionClause) stmt.getOps().get(0); - SingleRangePartitionDesc desc = clause.getSingeRangePartitionDesc(); - desc.analyze(partitionInfo.getPartitionColumns().size(), null); - partitionInfo.checkAndCreateRange(desc); - } - - private void checkContains(String path) throws DdlException { - if (!pathBuilder.getRoot().contains(path)) { - throw new DdlException("path[" + path + "] is not backuped"); - } - } - - private void downloadFile(String local, String remote) throws IOException { - String downloadCmd = null; - boolean succeed = false; - String msg = null; - for (int i = 0; i < MAX_RETRY_TIME; i++) { - try { - downloadCmd = commandBuilder.downloadCmd(label, local, remote); - } catch (IOException e) { - msg = e.getMessage(); - LOG.warn(msg + ". job[{}]. retry: {}", jobId, i); - try { - Thread.sleep(MAX_RETRY_INTERVAL_MS); - continue; - } catch (InterruptedException e1) { - LOG.warn(e.getMessage()); - break; - } - } - - String[] envp = { "LC_ALL=" + Config.locale }; - CommandResult result = Util.executeCommand(downloadCmd, envp); - if (result.getReturnCode() != 0) { - msg = "failed to download file[" + result + "]. job[" + jobId + "]"; - LOG.warn("{}. job[{}]. retry: {}", msg, jobId, i); - try { - Thread.sleep(MAX_RETRY_INTERVAL_MS); - continue; - } catch (InterruptedException e) { - LOG.warn(e.getMessage()); - break; - } - } - - // check file exist - File file = new File(local); - if (!file.exists()) { - msg = "can not find downloaded file: " + local; - LOG.warn(msg); - succeed = false; - break; - } - - succeed = true; - } - - if (!succeed) { - throw new IOException(msg); - } - - LOG.info("finished download file: {}. job[{}]", local, jobId); - } - - -} diff --git a/fe/src/main/java/org/apache/doris/backup/MetaUploadTask.java b/fe/src/main/java/org/apache/doris/backup/MetaUploadTask.java deleted file mode 100644 index 23b7168015..0000000000 --- a/fe/src/main/java/org/apache/doris/backup/MetaUploadTask.java +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.backup; - -import org.apache.doris.common.Config; -import org.apache.doris.common.util.CommandResult; -import org.apache.doris.common.util.Util; - -import java.util.concurrent.Callable; - -public class MetaUploadTask implements Callable { - - private String cmd; - - public MetaUploadTask(String cmd) { - this.cmd = cmd; - } - - @Override - public String call() throws Exception { - String[] envp = { "LC_ALL=" + Config.locale }; - CommandResult result = Util.executeCommand(cmd, envp); - if (result.getReturnCode() != 0) { - return result.toString(); - } - return null; - } -} diff --git a/fe/src/main/java/org/apache/doris/backup/ObjectWriter.java b/fe/src/main/java/org/apache/doris/backup/ObjectWriter.java index 4b59e1d2fb..0028e2486f 100644 --- a/fe/src/main/java/org/apache/doris/backup/ObjectWriter.java +++ b/fe/src/main/java/org/apache/doris/backup/ObjectWriter.java @@ -17,7 +17,6 @@ package org.apache.doris.backup; -import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.Util; @@ -116,17 +115,6 @@ public class ObjectWriter { throw new RuntimeException("Don't support CreateTableStmt serialization anymore."); } - public static List readAlterTableStmt(String filePath) throws IOException { - List stmts = null; - try { - stmts = read(filePath, AlterTableStmt.class); - } catch (IOException e) { - LOG.warn("failed to read AlterTableStmt: " + filePath, e); - throw e; - } - return stmts; - } - public static DirSaver readManifest(String filePath) throws IOException { List stmts = null; try { diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index 95cddb5158..32c5d176bb 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -3800,6 +3800,10 @@ public class Catalog { } sb.append(Joiner.on(", ").join(keysColumnNames)).append(")"); + if (!Strings.isNullOrEmpty(table.getComment())) { + sb.append("\nCOMMENT \"").append(table.getComment()).append("\""); + } + // partition PartitionInfo partitionInfo = olapTable.getPartitionInfo(); List partitionId = null; @@ -3859,6 +3863,9 @@ public class Catalog { sb.append("\n)"); } else if (table.getType() == TableType.MYSQL) { MysqlTable mysqlTable = (MysqlTable) table; + if (!Strings.isNullOrEmpty(table.getComment())) { + sb.append("\nCOMMENT \"").append(table.getComment()).append("\""); + } // properties sb.append("\nPROPERTIES (\n"); sb.append("\"host\" = \"").append(mysqlTable.getHost()).append("\",\n"); @@ -3870,6 +3877,9 @@ public class Catalog { sb.append(")"); } else if (table.getType() == TableType.BROKER) { BrokerTable brokerTable = (BrokerTable) table; + if (!Strings.isNullOrEmpty(table.getComment())) { + sb.append("\nCOMMENT \"").append(table.getComment()).append("\""); + } // properties sb.append("\nPROPERTIES (\n"); sb.append("\"broker_name\" = \"").append(brokerTable.getBrokerName()).append("\",\n"); @@ -3885,6 +3895,9 @@ public class Catalog { } } else if (table.getType() == TableType.ELASTICSEARCH) { EsTable esTable = (EsTable) table; + if (!Strings.isNullOrEmpty(table.getComment())) { + sb.append("\nCOMMENT \"").append(table.getComment()).append("\""); + } // partition PartitionInfo partitionInfo = esTable.getPartitionInfo(); @@ -3912,10 +3925,6 @@ public class Catalog { sb.append("\"transport\" = \"").append(esTable.getTransport()).append("\"\n"); sb.append(")"); } - - if (!Strings.isNullOrEmpty(table.getComment())) { - sb.append("\nCOMMENT \"").append(table.getComment()).append("\""); - } sb.append(";"); createTableStmt.add(sb.toString()); @@ -3935,8 +3944,9 @@ public class Catalog { sb = new StringBuilder(); Partition partition = olapTable.getPartition(entry.getKey()); sb.append("ALTER TABLE ").append(table.getName()); - sb.append(" ADD PARTITION ").append(partition.getName()).append(" VALUES LESS THAN "); - sb.append(entry.getValue().upperEndpoint().toSql()); + sb.append(" ADD PARTITION ").append(partition.getName()).append(" VALUES ["); + sb.append(entry.getValue().lowerEndpoint().toSql()); + sb.append(", ").append(entry.getValue().upperEndpoint().toSql()).append(")"); sb.append("(\"version_info\" = \""); sb.append(Joiner.on(",").join(partition.getVisibleVersion(), partition.getVisibleVersionHash())) diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index d02f957302..e8e4ca4483 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -17,13 +17,10 @@ package org.apache.doris.catalog; -import org.apache.doris.analysis.AddPartitionClause; import org.apache.doris.analysis.AddRollupClause; import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.CreateTableStmt; -import org.apache.doris.analysis.DistributionDesc; -import org.apache.doris.analysis.SingleRangePartitionDesc; import org.apache.doris.analysis.TableName; import org.apache.doris.backup.Status; import org.apache.doris.backup.Status.ErrCode; @@ -646,29 +643,6 @@ public class OlapTable extends Table { return alterTableStmt; } - public AlterTableStmt toAddPartitionStmt(String dbName, String partitionName) { - Preconditions.checkState(partitionInfo.getType() == PartitionType.RANGE); - RangePartitionInfo rangePartitionInfo = (RangePartitionInfo) partitionInfo; - List alterClauses = Lists.newArrayList(); - - Partition partition = nameToPartition.get(partitionName); - Map properties = Maps.newHashMap(); - long version = partition.getVisibleVersion(); - long versionHash = partition.getVisibleVersionHash(); - properties.put(PropertyAnalyzer.PROPERTIES_VERSION_INFO, version + "," + versionHash); - properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, - String.valueOf(partitionInfo.getReplicationNum(partition.getId()))); - - SingleRangePartitionDesc singleDesc = - rangePartitionInfo.toSingleRangePartitionDesc(partition.getId(), partitionName, properties); - DistributionDesc distributionDesc = partition.getDistributionInfo().toDistributionDesc(); - - AddPartitionClause addPartitionClause = new AddPartitionClause(singleDesc, distributionDesc, null); - alterClauses.add(addPartitionClause); - AlterTableStmt stmt = new AlterTableStmt(new TableName(dbName, name), alterClauses); - return stmt; - } - @Override public CreateTableStmt toCreateTableStmt(String dbName) { throw new RuntimeException("Don't support anymore"); diff --git a/fe/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/src/main/java/org/apache/doris/catalog/PartitionKey.java index a80a180229..b1681d4440 100644 --- a/fe/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -172,14 +172,15 @@ public class PartitionKey implements Comparable, Writable { } } + // return: ("100", "200", "300") public String toSql() { - StringBuilder strBuilder = new StringBuilder(); + StringBuilder sb = new StringBuilder("("); int i = 0; for (LiteralExpr expr : keys) { Object value = null; if (expr == MaxLiteral.MAX_VALUE) { value = expr.toSql(); - strBuilder.append(value); + sb.append(value); continue; } else { value = "\"" + expr.getRealValue() + "\""; @@ -188,14 +189,15 @@ public class PartitionKey implements Comparable, Writable { value = dateLiteral.toSql(); } } - strBuilder.append(value); + sb.append(value); if (keys.size() - 1 != i) { - strBuilder.append(", "); + sb.append(", "); } i++; } - return strBuilder.toString(); + sb.append(")"); + return sb.toString(); } @Override diff --git a/fe/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java b/fe/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java index 32ca86372a..9cfb45c50c 100644 --- a/fe/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java +++ b/fe/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java @@ -18,12 +18,9 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.PartitionKeyDesc; -import org.apache.doris.analysis.PartitionValue; import org.apache.doris.analysis.SingleRangePartitionDesc; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import org.apache.doris.common.FeNameFormat; -import org.apache.doris.common.util.PropertyAnalyzer; import com.google.common.base.Preconditions; import com.google.common.collect.BoundType; @@ -95,79 +92,10 @@ public class RangePartitionInfo extends PartitionInfo { public Range checkAndCreateRange(SingleRangePartitionDesc desc) throws DdlException { Range newRange = null; + PartitionKeyDesc partitionKeyDesc = desc.getPartitionKeyDesc(); // check range try { - // create single value partition key - PartitionKeyDesc partKeyDesc = desc.getPartitionKeyDesc(); - PartitionKey singlePartitionKey = null; - if (partKeyDesc.isMax()) { - singlePartitionKey = PartitionKey.createInfinityPartitionKey(partitionColumns, true); - } else { - singlePartitionKey = PartitionKey.createPartitionKey(partKeyDesc.getUpperValues(), partitionColumns); - } - - if (singlePartitionKey.isMinValue()) { - throw new DdlException("Partition value should not be MIN VALUE: " + singlePartitionKey.toSql()); - } - - List>> entries = - new ArrayList>>(this.idToRange.entrySet()); - Collections.sort(entries, RANGE_MAP_ENTRY_COMPARATOR); - - Range lastRange = null; - Range nextRange = null; - for (Map.Entry> entry : entries) { - nextRange = entry.getValue(); - - // check if equals to upper bound - PartitionKey upperKey = nextRange.upperEndpoint(); - if (upperKey.compareTo(singlePartitionKey) >= 0) { - PartitionKey lowKey = null; - if (partKeyDesc.hasLowerValues()) { - lowKey = PartitionKey.createPartitionKey(partKeyDesc.getLowerValues(), partitionColumns); - } else { - if (lastRange == null) { - lowKey = PartitionKey.createInfinityPartitionKey(partitionColumns, false); - } else { - lowKey = lastRange.upperEndpoint(); - } - } - // check: [left, right), error if left equal right - if (lowKey.compareTo(singlePartitionKey) >= 0) { - throw new IllegalArgumentException("The right value must be more than the left value"); - } - newRange = Range.closedOpen(lowKey, singlePartitionKey); - - // check if range intersected - checkRangeIntersect(newRange, nextRange); - break; - } - lastRange = nextRange; - } // end for ranges - - if (newRange == null) { - PartitionKey lowKey = null; - if (partKeyDesc.hasLowerValues()) { - lowKey = PartitionKey.createPartitionKey(partKeyDesc.getLowerValues(), partitionColumns); - } else { - if (lastRange == null) { - // add first partition to this table. so the lower key is MIN - lowKey = PartitionKey.createInfinityPartitionKey(partitionColumns, false); - } else { - lowKey = lastRange.upperEndpoint(); - } - } - // check: [left, right), error if left equal right - if (lowKey.compareTo(singlePartitionKey) >= 0) { - throw new IllegalArgumentException("The right value must be more than the left value"); - } - newRange = Range.closedOpen(lowKey, singlePartitionKey); - - // check if range intersected. The first Partition if lastRange == null - if (lastRange != null) { - checkRangeIntersect(newRange, lastRange); - } - } + newRange = createAndCheckNewRange(partitionKeyDesc); } catch (AnalysisException e) { throw new DdlException("Invalid range value format： " + e.getMessage()); } @@ -176,6 +104,72 @@ public class RangePartitionInfo extends PartitionInfo { return newRange; } + // create a new range and check it. + private Range createAndCheckNewRange(PartitionKeyDesc partKeyDesc) + throws AnalysisException, DdlException { + Range newRange = null; + // generate and sort the existing ranges + List>> sortedRanges = new ArrayList>>( + this.idToRange.entrySet()); + Collections.sort(sortedRanges, RANGE_MAP_ENTRY_COMPARATOR); + + // create upper values for new range + PartitionKey newRangeUpper = null; + if (partKeyDesc.isMax()) { + newRangeUpper = PartitionKey.createInfinityPartitionKey(partitionColumns, true); + } else { + newRangeUpper = PartitionKey.createPartitionKey(partKeyDesc.getUpperValues(), partitionColumns); + } + if (newRangeUpper.isMinValue()) { + throw new DdlException("Partition's upper value should not be MIN VALUE: " + partKeyDesc.toSql()); + } + + Range lastRange = null; + Range currentRange = null; + for (Map.Entry> entry : sortedRanges) { + currentRange = entry.getValue(); + // check if equals to upper bound + PartitionKey upperKey = currentRange.upperEndpoint(); + if (upperKey.compareTo(newRangeUpper) >= 0) { + newRange = checkNewRange(partKeyDesc, newRangeUpper, lastRange, currentRange); + break; + } else { + lastRange = currentRange; + } + } // end for ranges + + if (newRange == null) /* the new range's upper value is larger than any existing ranges */ { + newRange = checkNewRange(partKeyDesc, newRangeUpper, lastRange, currentRange); + } + return newRange; + } + + private Range checkNewRange(PartitionKeyDesc partKeyDesc, PartitionKey newRangeUpper, + Range lastRange, Range currentRange) throws AnalysisException, DdlException { + Range newRange; + PartitionKey lowKey = null; + if (partKeyDesc.hasLowerValues()) { + lowKey = PartitionKey.createPartitionKey(partKeyDesc.getLowerValues(), partitionColumns); + } else { + if (lastRange == null) { + lowKey = PartitionKey.createInfinityPartitionKey(partitionColumns, false); + } else { + lowKey = lastRange.upperEndpoint(); + } + } + // check: [left, right), error if left equal right + if (lowKey.compareTo(newRangeUpper) >= 0) { + throw new AnalysisException("The lower values must smaller than upper values"); + } + newRange = Range.closedOpen(lowKey, newRangeUpper); + + if (currentRange != null) { + // check if range intersected + checkRangeIntersect(newRange, currentRange); + } + return newRange; + } + public static void checkRangeIntersect(Range range1, Range range2) throws DdlException { if (range2.isConnected(range1)) { if (!range2.intersection(range1).isEmpty()) { @@ -307,34 +301,6 @@ public class RangePartitionInfo extends PartitionInfo { return null; } - public SingleRangePartitionDesc toSingleRangePartitionDesc(long partitionId, String partitionName, - Map properties) { - Range range = idToRange.get(partitionId); - List upperValues = Lists.newArrayList(); - List lowerValues = Lists.newArrayList(); - // FIXME(cmy): check here(getStringValue) - lowerValues.add(new PartitionValue(range.lowerEndpoint().getKeys().get(0).getStringValue())); - - PartitionKey upperKey = range.upperEndpoint(); - PartitionKeyDesc keyDesc = null; - if (upperKey.isMaxValue()) { - keyDesc = PartitionKeyDesc.createMaxKeyDesc(); - keyDesc.setLowerValues(lowerValues); - } else { - upperValues.add(new PartitionValue(range.upperEndpoint().getKeys().get(0).getStringValue())); - keyDesc = new PartitionKeyDesc(lowerValues, upperValues); - } - - SingleRangePartitionDesc singleDesc = new SingleRangePartitionDesc(false, partitionName, keyDesc, properties); - - if (properties != null) { - // properties - Short replicationNum = getReplicationNum(partitionId); - properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, replicationNum.toString()); - } - return singleDesc; - } - public boolean checkRange(Range newRange) { for (Range range : idToRange.values()) { if (range.isConnected(newRange)) { @@ -409,31 +375,16 @@ public class RangePartitionInfo extends PartitionInfo { new ArrayList>>(this.idToRange.entrySet()); Collections.sort(entries, RANGE_MAP_ENTRY_COMPARATOR); - Range lastRange = null; idx = 0; for (Map.Entry> entry : entries) { Partition partition = table.getPartition(entry.getKey()); String partitionName = partition.getName(); Range range = entry.getValue(); - if (idx == 0) { - // first partition - if (!range.lowerEndpoint().isMinValue()) { - sb.append("PARTITION ").append(FeNameFormat.FORBIDDEN_PARTITION_NAME).append(idx) - .append(" VALUES LESS THAN (").append(range.lowerEndpoint().toSql()).append(")"); - sb.append(",\n"); - } - } else { - Preconditions.checkNotNull(lastRange); - if (!lastRange.upperEndpoint().equals(range.lowerEndpoint())) { - sb.append("PARTITION ").append(FeNameFormat.FORBIDDEN_PARTITION_NAME).append(idx) - .append(" VALUES LESS THAN (").append(range.lowerEndpoint().toSql()).append(")"); - sb.append(",\n"); - } - } - - sb.append("PARTITION ").append(partitionName).append(" VALUES LESS THAN ("); - sb.append(range.upperEndpoint().toSql()).append(")"); + // print all partitions' range is fixed range, even if some of them is created by less than range + sb.append("PARTITION ").append(partitionName).append(" VALUES ["); + sb.append(range.lowerEndpoint().toSql()); + sb.append(", ").append(range.upperEndpoint().toSql()).append(")"); if (partitionId != null) { partitionId.add(entry.getKey()); @@ -444,8 +395,6 @@ public class RangePartitionInfo extends PartitionInfo { sb.append(",\n"); } idx++; - - lastRange = range; } sb.append(")"); return sb.toString(); diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index 2ac879cda8..20c52bf440 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -628,13 +628,13 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements // progress switch (state) { case PENDING: - jobInfo.add("ETL:N/A; LOAD:0%"); + jobInfo.add("ETL:0%; LOAD:0%"); break; case CANCELLED: jobInfo.add("ETL:N/A; LOAD:N/A"); break; default: - jobInfo.add("ETL:N/A; LOAD:" + progress + "%"); + jobInfo.add("ETL:100%; LOAD:" + progress + "%"); break; } diff --git a/fe/src/main/java/org/apache/doris/system/Frontend.java b/fe/src/main/java/org/apache/doris/system/Frontend.java index 0a315c8ce4..31b46085c1 100644 --- a/fe/src/main/java/org/apache/doris/system/Frontend.java +++ b/fe/src/main/java/org/apache/doris/system/Frontend.java @@ -29,7 +29,6 @@ import java.io.DataOutput; import java.io.IOException; public class Frontend implements Writable { - private FrontendNodeType role; private String nodeName; private String host; @@ -45,9 +44,6 @@ public class Frontend implements Writable { private boolean isAlive = false; public Frontend() { - role = FrontendNodeType.UNKNOWN; - host = ""; - editLogPort = 0; } public Frontend(FrontendNodeType role, String nodeName, String host, int editLogPort) { @@ -56,19 +52,11 @@ public class Frontend implements Writable { this.host = host; this.editLogPort = editLogPort; } - - public void setRole(FrontendNodeType role) { - this.role = role; - } - + public FrontendNodeType getRole() { return this.role; } - public void setHost(String host) { - this.host = host; - } - public String getHost() { return this.host; } @@ -88,10 +76,6 @@ public class Frontend implements Writable { public boolean isAlive() { return isAlive; } - - public void setEditLogPort(int editLogPort) { - this.editLogPort = editLogPort; - } public int getEditLogPort() { return this.editLogPort; diff --git a/fe/src/test/java/org/apache/doris/backup/AlterClauseRWTest.java b/fe/src/test/java/org/apache/doris/backup/AlterClauseRWTest.java deleted file mode 100644 index 8030b07407..0000000000 --- a/fe/src/test/java/org/apache/doris/backup/AlterClauseRWTest.java +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.backup; - -import org.apache.doris.analysis.AddPartitionClause; -import org.apache.doris.analysis.AddRollupClause; -import org.apache.doris.analysis.AlterClause; -import org.apache.doris.analysis.AlterTableStmt; -import org.apache.doris.analysis.DistributionDesc; -import org.apache.doris.analysis.PartitionKeyDesc; -import org.apache.doris.analysis.PartitionValue; -import org.apache.doris.analysis.RandomDistributionDesc; -import org.apache.doris.analysis.SingleRangePartitionDesc; -import org.apache.doris.analysis.TableName; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import org.junit.Before; -import org.junit.Test; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -public class AlterClauseRWTest { - - @Before - public void setUp() { - System.out.println(AddPartitionClause.class.getName()); - System.out.println(AddPartitionClause.class.getCanonicalName()); - } - - @Test - public void test() throws FileNotFoundException { - List clauses = Lists.newArrayList(); - AlterTableStmt stmt = new AlterTableStmt(new TableName("db", "tbl"), clauses); - - - File file = new File("./addPartitionClause"); - try { - file.createNewFile(); - DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); - - // add partition clause - String partititionName = "p1"; - List values = Lists.newArrayList(new PartitionValue("100")); - PartitionKeyDesc keyDesc = new PartitionKeyDesc(values); - Map properties = Maps.newHashMap(); - SingleRangePartitionDesc partitionDesc = new SingleRangePartitionDesc(false, partititionName, keyDesc, - properties); - DistributionDesc distributionDesc = new RandomDistributionDesc(32); - AddPartitionClause addPartitionClause = new AddPartitionClause(partitionDesc, distributionDesc, properties); - clauses.add(addPartitionClause); - - // add rollup clause - AddRollupClause rollupClause = new AddRollupClause("rollup", Lists.newArrayList("k1", "v1"), - null, null, properties); - clauses.add(rollupClause); - - // write - stmt.write(out); - out.flush(); - out.close(); - - System.out.println(stmt.toSql()); - - // read - DataInputStream in = new DataInputStream(new FileInputStream(file)); - stmt = new AlterTableStmt(); - stmt.readFields(in); - System.out.println(stmt.toSql()); - - in.close(); - file.delete(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - -} diff --git a/fe/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java b/fe/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java index 0aa9632b3d..4168fdb31f 100644 --- a/fe/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java +++ b/fe/src/test/java/org/apache/doris/catalog/RangePartitionInfoTest.java @@ -168,7 +168,7 @@ public class RangePartitionInfoTest { Lists.newArrayList(new PartitionValue("20190101"), new PartitionValue("100"))); PartitionKeyDesc p4 = new PartitionKeyDesc( Lists.newArrayList(new PartitionValue("20190105"), new PartitionValue("100")), - Lists.newArrayList(new PartitionValue("20190120"), PartitionValue.createMaxValue())); + Lists.newArrayList(new PartitionValue("20190120"), new PartitionValue("10000000000"))); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p1", p1, null)); singleRangePartitionDescs.add(new SingleRangePartitionDesc(false, "p2", p2, null)); @@ -206,7 +206,7 @@ public class RangePartitionInfoTest { Lists.newArrayList(new PartitionValue("2019-02-01"), new PartitionValue("100"), new PartitionValue("200"))); PartitionKeyDesc p2 = new PartitionKeyDesc( Lists.newArrayList(new PartitionValue("2020-02-01"), new PartitionValue("100"), new PartitionValue("200")), - Lists.newArrayList(PartitionValue.createMaxValue())); + Lists.newArrayList(new PartitionValue("10000000000"))); PartitionKeyDesc p3 = new PartitionKeyDesc( Lists.newArrayList(new PartitionValue("2021-02-01")));