database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[branch-2.1](partition) Support use Auto and Dynamic partition at the same time (#39580) (#40649)

Browse Source 
pick https://github.com/apache/doris/pull/39580
This commit is contained in:
zclllhhjj
2024-09-11 15:35:20 +08:00
committed by GitHub
parent 279c58fbc7
commit d554f600bc
5 changed files with 183 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View File

@ -17,6 +17,9 @@
package org.apache.doris.common.util;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
@ -127,13 +130,14 @@ public class DynamicPartitionUtil {
return DynamicPartitionProperty.MIN_START_OFFSET;
}
private static int checkEnd(String end) throws DdlException {
private static int checkEnd(String end, boolean enableAutoPartition) throws DdlException {
if (Strings.isNullOrEmpty(end)) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_EMPTY);
}
try {
int endInt = Integer.parseInt(end);
if (endInt <= 0) {
// with auto partition sometime we dont like to create future partition by dynamic partition.
if (endInt < 0 || endInt == 0 && !enableAutoPartition) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_ZERO, end);
}
return endInt;
@ -503,6 +507,25 @@ public class DynamicPartitionUtil {
}
}
public static void partitionIntervalCompatible(String dynamicUnit, ArrayList<Expr> autoExprs)
throws AnalysisException {
if (autoExprs == null) {
return;
}
for (Expr autoExpr : autoExprs) {
Expr func = (FunctionCallExpr) autoExpr;
for (Expr child : func.getChildren()) {
if (child instanceof LiteralExpr) {
String autoUnit = ((LiteralExpr) child).getStringValue();
if (!dynamicUnit.equalsIgnoreCase(autoUnit)) {
throw new AnalysisException("If support auto partition and dynamic partition at same time, "
+ "they must have the same interval unit.");
}
}
}
}
}
// Analyze all properties to check their validation
public static Map<String, String> analyzeDynamicPartition(Map<String, String> properties,
OlapTable olapTable, Database db) throws UserException {
@ -511,6 +534,12 @@ public class DynamicPartitionUtil {
if (properties.containsKey(DynamicPartitionProperty.TIME_UNIT)) {
String timeUnitValue = properties.get(DynamicPartitionProperty.TIME_UNIT);
checkTimeUnit(timeUnitValue, olapTable.getPartitionInfo());
// if both enabled, must use same interval.
if (olapTable.getPartitionInfo().enableAutomaticPartition()) {
partitionIntervalCompatible(timeUnitValue, olapTable.getPartitionInfo().getPartitionExprs());
}
properties.remove(DynamicPartitionProperty.TIME_UNIT);
analyzedProperties.put(DynamicPartitionProperty.TIME_UNIT, timeUnitValue);
}
@ -535,11 +564,7 @@ public class DynamicPartitionUtil {
analyzedProperties.put(DynamicPartitionProperty.ENABLE, enableValue);
}
if (Boolean.parseBoolean(analyzedProperties.getOrDefault(DynamicPartitionProperty.ENABLE, "true"))
&& olapTable.getPartitionInfo().enableAutomaticPartition()) {
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
}
boolean enableAutoPartition = olapTable.getPartitionInfo().enableAutomaticPartition();
// If dynamic property "start" is not specified, use Integer.MIN_VALUE as default
int start = DynamicPartitionProperty.MIN_START_OFFSET;
@ -554,7 +579,7 @@ public class DynamicPartitionUtil {
boolean hasEnd = false;
if (properties.containsKey(DynamicPartitionProperty.END)) {
String endValue = properties.get(DynamicPartitionProperty.END);
end = checkEnd(endValue);
end = checkEnd(endValue, enableAutoPartition);
properties.remove(DynamicPartitionProperty.END);
analyzedProperties.put(DynamicPartitionProperty.END, endValue);
hasEnd = true;

6
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -2859,8 +2859,10 @@ public class InternalCatalog implements CatalogIf<Database> {
.getDynamicPartitionProperty();
if (dynamicProperty.isExist() && dynamicProperty.getEnable()
&& partitionDesc.isAutoCreatePartitions()) {
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
String dynamicUnit = dynamicProperty.getTimeUnit();
ArrayList<Expr> autoExprs = partitionDesc.getPartitionExprs();
// check same interval. fail will leading to AnalysisException
DynamicPartitionUtil.partitionIntervalCompatible(dynamicUnit, autoExprs);
}
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());

10
regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out
View File

@ -102,10 +102,7 @@ xxX 3
2013-12-12T00:00 2013-12-12T00:00
2020-12-12T00:00 2020-12-12T12:12:12.123456
-- !sql_overwrite1 --
Yyy
-- !sql_overwrite2 --
-- !sql_overwrite --
Xxx
-- !sql_non_order1 --
@ -117,3 +114,8 @@ Xxx
-- !sql_non_order3 --
3 2013-12-12T00:00
-- !sql_dynamic_auto --
2024-01-01T00:00
2900-01-01T00:00
3000-01-01T00:00

251
regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
View File

@ -16,6 +16,9 @@
// under the License.
suite("test_auto_partition_behavior") {
sql "set experimental_enable_nereids_planner=true;"
sql "set enable_fallback_to_original_planner=false;"
/// unique key table
sql "drop table if exists unique_table"
sql """
@ -165,18 +168,6 @@ suite("test_auto_partition_behavior") {
);
"""
sql """ insert into rewrite values ("Xxx"); """
// legacy planner
sql " set experimental_enable_nereids_planner=false "
try {
sql """ insert overwrite table rewrite partition(p1) values ("XXX") """
fail()
} catch (Exception e) {
assertTrue(e.getMessage().contains("Insert has filtered data in strict mode"))
}
sql """ insert overwrite table rewrite partition(p1) values ("Yyy") """
qt_sql_overwrite1 """ select * from rewrite """ // Yyy
// nereids planner
sql " set experimental_enable_nereids_planner=true "
try {
sql """ insert overwrite table rewrite partition(p1) values ("") """
fail()
@ -184,7 +175,7 @@ suite("test_auto_partition_behavior") {
assertTrue(e.getMessage().contains("Insert has filtered data in strict mode"))
}
sql """ insert overwrite table rewrite partition(p1) values ("Xxx") """
qt_sql_overwrite2 """ select * from rewrite """ // Xxx
qt_sql_overwrite """ select * from rewrite """ // Xxx
sql " drop table if exists non_order; "
sql """
@ -209,22 +200,6 @@ suite("test_auto_partition_behavior") {
qt_sql_non_order3 """ select * from non_order where k1 = '2013-12-12'; """
// range partition can't auto create null partition
sql " set experimental_enable_nereids_planner=true "
sql "drop table if exists invalid_null_range"
test {
sql """
create table invalid_null_range(
k0 datetime(6) null
)
auto partition by range (date_trunc(k0, 'hour'))
(
)
DISTRIBUTED BY HASH(`k0`) BUCKETS 2
properties("replication_num" = "1");
"""
exception "AUTO RANGE PARTITION doesn't support NULL column"
}
sql " set experimental_enable_nereids_planner=false "
sql "drop table if exists invalid_null_range"
test {
sql """
@ -240,7 +215,36 @@ suite("test_auto_partition_behavior") {
exception "AUTO RANGE PARTITION doesn't support NULL column"
}
sql "drop table if exists test_dynamic"
// dynamic + auto partition
sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '1') """
// PROHIBIT different timeunit of interval when use both auto & dynamic partition
test{
sql """
CREATE TABLE tbl3
(
k1 DATETIME NOT NULL,
col1 int
)
auto partition by range (date_trunc(`k1`, 'year')) ()
DISTRIBUTED BY HASH(k1)
PROPERTIES
(
"replication_num" = "1",
"dynamic_partition.create_history_partition"="true",
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "HOUR",
"dynamic_partition.start" = "-2",
"dynamic_partition.end" = "2",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "8"
);
"""
exception "If support auto partition and dynamic partition at same time, they must have the same interval unit."
}
sql " drop table if exists test_dynamic "
sql """
create table test_dynamic(
k0 DATE not null
@ -249,32 +253,6 @@ suite("test_auto_partition_behavior") {
DISTRIBUTED BY HASH(`k0`) BUCKETS auto
properties("replication_num" = "1");
"""
// PROHIBIT different timeunit of interval when use both auto & dynamic partition
sql "set experimental_enable_nereids_planner=true;"
test{
sql """
CREATE TABLE tbl3
(
k1 DATETIME NOT NULL,
col1 int
)
auto partition by range (date_trunc(`k1`, 'year')) ()
DISTRIBUTED BY HASH(k1)
PROPERTIES
(
"replication_num" = "1",
"dynamic_partition.create_history_partition"="true",
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "HOUR",
"dynamic_partition.start" = "-2",
"dynamic_partition.end" = "2",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "8"
);
"""
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}
test {
sql """
ALTER TABLE test_dynamic set (
@ -285,45 +263,70 @@ suite("test_auto_partition_behavior") {
"dynamic_partition.buckets" = "32"
);
"""
exception "Can't use Dynamic Partition and Auto Partition at the same time"
exception "If support auto partition and dynamic partition at same time, they must have the same interval unit."
}
sql """
ALTER TABLE test_dynamic set (
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "YeAr",
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32"
);
"""
sql " drop table if exists auto_dynamic "
sql """
create table auto_dynamic(
k0 datetime(6) NOT NULL
)
auto partition by range (date_trunc(k0, 'hour'))
(
)
DISTRIBUTED BY HASH(`k0`) BUCKETS 2
properties(
"dynamic_partition.enable" = "true",
"dynamic_partition.prefix" = "p",
"dynamic_partition.create_history_partition" = "true",
"dynamic_partition.start" = "-5",
"dynamic_partition.end" = "0",
"dynamic_partition.time_unit" = "hour",
"replication_num" = "1"
);
"""
def part_result = sql " show partitions from auto_dynamic "
assertEquals(part_result.size, 6)
sql " drop table if exists auto_dynamic "
sql """
create table auto_dynamic(
k0 datetime(6) NOT NULL
)
auto partition by range (date_trunc(k0, 'year'))
(
)
DISTRIBUTED BY HASH(`k0`) BUCKETS 2
properties(
"dynamic_partition.enable" = "true",
"dynamic_partition.prefix" = "p",
"dynamic_partition.start" = "-50",
"dynamic_partition.end" = "0",
"dynamic_partition.time_unit" = "year",
"replication_num" = "1"
);
"""
part_result = sql " show partitions from auto_dynamic "
assertEquals(part_result.size, 1)
sql " insert into auto_dynamic values ('2024-01-01'), ('2900-01-01'), ('1900-01-01'), ('3000-01-01'); "
sleep(3000)
part_result = sql " show partitions from auto_dynamic "
log.info("${part_result}".toString())
assertEquals(part_result.size, 3)
qt_sql_dynamic_auto "select * from auto_dynamic order by k0;"
sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '600') """
sql "set experimental_enable_nereids_planner=false;"
test{
sql """
CREATE TABLE tbl3
(
k1 DATETIME NOT NULL,
col1 int
)
auto partition by range (date_trunc(`k1`, 'year')) ()
DISTRIBUTED BY HASH(k1)
PROPERTIES
(
"replication_num" = "1",
"dynamic_partition.create_history_partition"="true",
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "HOUR",
"dynamic_partition.start" = "-2",
"dynamic_partition.end" = "2",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "8"
);
"""
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}
test {
sql """
ALTER TABLE test_dynamic set (
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "DAY",
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32"
);
"""
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}
// prohibit too long value for partition column
sql "drop table if exists `long_value`"
@ -345,25 +348,10 @@ suite("test_auto_partition_behavior") {
exception "Partition name's length is over limit of 50."
}
// illegal partiton definetion
sql "set experimental_enable_nereids_planner=false;"
test{
sql """
create table illegal(
k0 datetime(6) NOT null,
k1 datetime(6) NOT null
)
auto partition by range (date_trunc(k0, k1, 'hour'))
(
)
DISTRIBUTED BY HASH(`k0`) BUCKETS 2
properties("replication_num" = "1");
"""
exception "auto create partition only support one slotRef in function expr"
}
sql "set experimental_enable_nereids_planner=true;"
sql "set enable_fallback_to_original_planner=false;"
/// illegal partition exprs
test{
sql """
create table illegal(
@ -393,19 +381,30 @@ suite("test_auto_partition_behavior") {
"""
exception "partition expr date_trunc is illegal!"
}
sql "set experimental_enable_nereids_planner=false;"
test{
sql """
create table illegal(
k0 datetime(6) NOT null,
k1 int NOT null
)
auto partition by range (date_trunc(k1, 'hour'))
(
)
DISTRIBUTED BY HASH(`k0`) BUCKETS 2
properties("replication_num" = "1");
"""
exception "Auto range partition needs Date/DateV2/Datetime/DatetimeV2 column as partition column"
}
// altering table property effects new partitions.
sql " drop table if exists test_change "
sql """
create table test_change(
k0 datetime not null
)
auto partition by range (date_trunc(k0, 'year'))
(
)
DISTRIBUTED BY HASH(`k0`) BUCKETS 2
properties("replication_num" = "1");
"""
def replicaNum = get_table_replica_num("test_change")
logger.info("get table replica num: " + replicaNum)
sql """ insert into test_change values ("20201212"); """
part_result = sql " show tablets from test_change "
assertEquals(part_result.size, 2 * replicaNum)
sql """ ALTER TABLE test_change MODIFY DISTRIBUTION DISTRIBUTED BY HASH(k0) BUCKETS 50; """
sql """ insert into test_change values ("20001212"); """
part_result = sql " show tablets from test_change "
assertEquals(part_result.size, 52 * replicaNum)
}

31
regression-test/suites/partition_p1/dynamic_partition/test_dynamic_partition.groovy
View File

@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
suite("test_dynamic_partition_with_update","nonConcurrent") {
def tbl = "test_dynamic_partition_with_update"
sql "drop table if exists ${tbl}"
suite("test_dynamic_partition_with_update", "nonConcurrent") {
sql "drop table if exists test_dynamic_partition_with_update"
sql """
CREATE TABLE IF NOT EXISTS ${tbl}
CREATE TABLE IF NOT EXISTS test_dynamic_partition_with_update
( k1 date NOT NULL )
PARTITION BY RANGE(k1) ( )
DISTRIBUTED BY HASH(k1) BUCKETS 1
@ -36,38 +35,38 @@ suite("test_dynamic_partition_with_update","nonConcurrent") {
"""
// set check interval time
sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '2') """
sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '1') """
// check table init
def result = sql "show partitions from ${tbl}"
def result = sql "show partitions from test_dynamic_partition_with_update"
assertEquals(7, result.size())
result = sql "show dynamic partition tables"
assertEquals("true",result.get(0).get(1))
// disable dynamic partition to insert partition
sql """ alter table ${tbl} set ('dynamic_partition.enable' = 'false') """
sql """ alter table test_dynamic_partition_with_update set ('dynamic_partition.enable' = 'false') """
result = sql "show dynamic partition tables"
assertEquals("false",result.get(0).get(1))
// manually insert partition
sql """ alter table ${tbl} add partition p1 values [("2020-01-02"), ("2020-01-05")) """
sql """ alter table ${tbl} add partition p2 values [("2020-05-02"), ("2020-06-06")) """
sql """ alter table ${tbl} add partition p3 values [("2020-07-04"), ("2020-07-28")) """
sql """ alter table ${tbl} add partition p4 values [("2999-04-25"), ("2999-04-28")) """
sql """ alter table test_dynamic_partition_with_update add partition p1 values [("2020-01-02"), ("2020-01-05")) """
sql """ alter table test_dynamic_partition_with_update add partition p2 values [("2020-05-02"), ("2020-06-06")) """
sql """ alter table test_dynamic_partition_with_update add partition p3 values [("2020-07-04"), ("2020-07-28")) """
sql """ alter table test_dynamic_partition_with_update add partition p4 values [("2999-04-25"), ("2999-04-28")) """
// check size
result = sql "show partitions from ${tbl}"
result = sql "show partitions from test_dynamic_partition_with_update"
assertEquals(11, result.size())
sql """ alter table ${tbl} set ('dynamic_partition.enable' = 'true') """
sql """ alter table test_dynamic_partition_with_update set ('dynamic_partition.enable' = 'true') """
result = sql "show dynamic partition tables"
assertEquals("true",result.get(0).get(1))
// check and update
sleep(5000);
sleep(3000)
// check size
result = sql "show partitions from ${tbl}"
result = sql "show partitions from test_dynamic_partition_with_update"
assertEquals(8, result.size())
sql "drop table ${tbl}"
sql """ admin set frontend config ('dynamic_partition_check_interval_seconds' = '600') """
}