database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](auto-partition) Forbid use Auto and Dynamic partition at the same time (#33736)

Browse Source
This commit is contained in:
zclllyybb
2024-04-19 20:15:01 +08:00
committed by yiguolei
parent bec7c36c46
commit 42e91149e4
5 changed files with 84 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
View File

@ -102,7 +102,6 @@ public class PartitionDesc {
// 1. partition by list (column) : now support one slotRef
// 2. partition by range(column/function(column)) : support slotRef and some
// special function eg: date_trunc, date_floor/ceil
// not only for auto partition. maybe we should check for project partitiion also
public static List<String> getColNamesFromExpr(ArrayList<Expr> exprs, boolean isListPartition,
boolean isAutoPartition)
throws AnalysisException {
@ -206,16 +205,27 @@ public class PartitionDesc {
throw new AnalysisException(
"The partition column must be NOT NULL with allow_partition_column_nullable OFF");
}
if (this instanceof RangePartitionDesc && isAutoCreatePartitions && columnDef.isAllowNull()) {
throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column");
}
if (this instanceof RangePartitionDesc && partitionExprs != null) {
if (partitionExprs.get(0) instanceof FunctionCallExpr) {
if (!columnDef.getType().isDateType()) {
throw new AnalysisException(
"Auto range partition needs Date/DateV2/"
+ "Datetime/DatetimeV2 column as partition column"
+ partitionExprs.get(0).toSql());
if (this instanceof RangePartitionDesc && isAutoCreatePartitions) {
if (columnDef.isAllowNull()) {
throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column");
}
if (partitionExprs != null) {
for (Expr expr : partitionExprs) {
if (!(expr instanceof FunctionCallExpr) || !RANGE_PARTITION_FUNCTIONS
.contains(((FunctionCallExpr) expr).getFnName().getFunction())) {
throw new AnalysisException(
"auto create partition only support slotRef and "
+ "date_trunc/date_floor/date_ceil function in range partitions. "
+ expr.toSql());
}
}
if (partitionExprs.get(0) instanceof FunctionCallExpr) {
if (!columnDef.getType().isDateType()) {
throw new AnalysisException(
"Auto range partition needs Date/DateV2/"
+ "Datetime/DatetimeV2 column as partition column but got"
+ partitionExprs.get(0).toSql());
}
}
}
}

6
fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View File

@ -535,6 +535,12 @@ public class DynamicPartitionUtil {
analyzedProperties.put(DynamicPartitionProperty.ENABLE, enableValue);
}
if (Boolean.parseBoolean(analyzedProperties.getOrDefault(DynamicPartitionProperty.ENABLE, "true"))
&& olapTable.getPartitionInfo().enableAutomaticPartition()) {
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
}
// If dynamic property "start" is not specified, use Integer.MIN_VALUE as default
int start = DynamicPartitionProperty.MIN_START_OFFSET;
if (properties.containsKey(DynamicPartitionProperty.START)) {

17
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -2645,21 +2645,8 @@ public class InternalCatalog implements CatalogIf<Database> {
.getDynamicPartitionProperty();
if (dynamicProperty.isExist() && dynamicProperty.getEnable()
&& partitionDesc.isAutoCreatePartitions()) {
String dynamicUnit = dynamicProperty.getTimeUnit();
ArrayList<Expr> autoExprs = partitionDesc.getPartitionExprs();
for (Expr autoExpr : autoExprs) {
Expr func = (FunctionCallExpr) autoExpr;
for (Expr child : func.getChildren()) {
if (child instanceof LiteralExpr) {
String autoUnit = ((LiteralExpr) child).getStringValue();
if (!dynamicUnit.equalsIgnoreCase(autoUnit)) {
throw new AnalysisException(
"If support auto partition and dynamic partition at same time, "
+ "they must have the same interval unit.");
}
}
}
}
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
}
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());

6
fe/fe-core/src/main/java/org/apache/doris/nereids/parser/PartitionTableInfo.java
View File

@ -140,8 +140,10 @@ public class PartitionTableInfo {
throw new AnalysisException(
"The partition column must be NOT NULL with allow_partition_column_nullable OFF");
}
if (isAutoPartition && partitionType.equalsIgnoreCase(PartitionType.RANGE.name()) && column.isNullable()) {
throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column");
if (partitionType.equalsIgnoreCase(PartitionType.RANGE.name()) && isAutoPartition) {
if (column.isNullable()) {
throw new AnalysisException("AUTO RANGE PARTITION doesn't support NULL column");
}
}
}

53
regression-test/suites/partition_p0/auto_partition/test_auto_partition_behavior.groovy
View File

@ -240,6 +240,16 @@ suite("test_auto_partition_behavior") {
exception "AUTO RANGE PARTITION doesn't support NULL column"
}
sql "drop table if exists test_dynamic"
sql """
create table test_dynamic(
k0 DATE not null
)
auto partition by range (date_trunc(k0, 'year')) ()
DISTRIBUTED BY HASH(`k0`) BUCKETS auto
properties("replication_num" = "1");
"""
// PROHIBIT different timeunit of interval when use both auto & dynamic partition
sql "set experimental_enable_nereids_planner=true;"
test{
@ -263,7 +273,19 @@ suite("test_auto_partition_behavior") {
"dynamic_partition.buckets" = "8"
);
"""
exception "If support auto partition and dynamic partition at same time, they must have the same interval unit."
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}
test {
sql """
ALTER TABLE test_dynamic set (
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "DAY",
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32"
);
"""
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}
sql "set experimental_enable_nereids_planner=false;"
@ -288,7 +310,19 @@ suite("test_auto_partition_behavior") {
"dynamic_partition.buckets" = "8"
);
"""
exception "If support auto partition and dynamic partition at same time, they must have the same interval unit."
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}
test {
sql """
ALTER TABLE test_dynamic set (
"dynamic_partition.enable" = "true",
"dynamic_partition.time_unit" = "DAY",
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32"
);
"""
exception "Can't use Dynamic Partition and Auto Partition at the same time"
}
// prohibit too long value for partition column
@ -359,4 +393,19 @@ suite("test_auto_partition_behavior") {
"""
exception "partition expr date_trunc is illegal!"
}
sql "set experimental_enable_nereids_planner=false;"
test{
sql """
create table illegal(
k0 datetime(6) NOT null,
k1 int NOT null
)
auto partition by range (date_trunc(k1, 'hour'))
(
)
DISTRIBUTED BY HASH(`k0`) BUCKETS 2
properties("replication_num" = "1");
"""
exception "Auto range partition needs Date/DateV2/Datetime/DatetimeV2 column as partition column"
}
}