database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[branch-2.1](partition) Support use Auto and Dynamic partition at the same time (#39580) (#40649)

Browse Source 
pick https://github.com/apache/doris/pull/39580
This commit is contained in:
zclllhhjj
2024-09-11 15:35:20 +08:00
committed by GitHub
parent 279c58fbc7
commit d554f600bc
5 changed files with 183 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View File

@ -17,6 +17,9 @@
package org.apache.doris.common.util;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
@ -127,13 +130,14 @@ public class DynamicPartitionUtil {
return DynamicPartitionProperty.MIN_START_OFFSET;
}
private static int checkEnd(String end) throws DdlException {
private static int checkEnd(String end, boolean enableAutoPartition) throws DdlException {
if (Strings.isNullOrEmpty(end)) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_EMPTY);
}
try {
int endInt = Integer.parseInt(end);
if (endInt <= 0) {
// with auto partition sometime we dont like to create future partition by dynamic partition.
if (endInt < 0 || endInt == 0 && !enableAutoPartition) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_ZERO, end);
}
return endInt;
@ -503,6 +507,25 @@ public class DynamicPartitionUtil {
}
}
public static void partitionIntervalCompatible(String dynamicUnit, ArrayList<Expr> autoExprs)
throws AnalysisException {
if (autoExprs == null) {
return;
}
for (Expr autoExpr : autoExprs) {
Expr func = (FunctionCallExpr) autoExpr;
for (Expr child : func.getChildren()) {
if (child instanceof LiteralExpr) {
String autoUnit = ((LiteralExpr) child).getStringValue();
if (!dynamicUnit.equalsIgnoreCase(autoUnit)) {
throw new AnalysisException("If support auto partition and dynamic partition at same time, "
+ "they must have the same interval unit.");
}
}
}
}
}
// Analyze all properties to check their validation
public static Map<String, String> analyzeDynamicPartition(Map<String, String> properties,
OlapTable olapTable, Database db) throws UserException {
@ -511,6 +534,12 @@ public class DynamicPartitionUtil {
if (properties.containsKey(DynamicPartitionProperty.TIME_UNIT)) {
String timeUnitValue = properties.get(DynamicPartitionProperty.TIME_UNIT);
checkTimeUnit(timeUnitValue, olapTable.getPartitionInfo());
// if both enabled, must use same interval.
if (olapTable.getPartitionInfo().enableAutomaticPartition()) {
partitionIntervalCompatible(timeUnitValue, olapTable.getPartitionInfo().getPartitionExprs());
}
properties.remove(DynamicPartitionProperty.TIME_UNIT);
analyzedProperties.put(DynamicPartitionProperty.TIME_UNIT, timeUnitValue);
}
@ -535,11 +564,7 @@ public class DynamicPartitionUtil {
analyzedProperties.put(DynamicPartitionProperty.ENABLE, enableValue);
}
if (Boolean.parseBoolean(analyzedProperties.getOrDefault(DynamicPartitionProperty.ENABLE, "true"))
&& olapTable.getPartitionInfo().enableAutomaticPartition()) {
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
}
boolean enableAutoPartition = olapTable.getPartitionInfo().enableAutomaticPartition();
// If dynamic property "start" is not specified, use Integer.MIN_VALUE as default
int start = DynamicPartitionProperty.MIN_START_OFFSET;
@ -554,7 +579,7 @@ public class DynamicPartitionUtil {
boolean hasEnd = false;
if (properties.containsKey(DynamicPartitionProperty.END)) {
String endValue = properties.get(DynamicPartitionProperty.END);
end = checkEnd(endValue);
end = checkEnd(endValue, enableAutoPartition);
properties.remove(DynamicPartitionProperty.END);
analyzedProperties.put(DynamicPartitionProperty.END, endValue);
hasEnd = true;

6
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -2859,8 +2859,10 @@ public class InternalCatalog implements CatalogIf<Database> {
.getDynamicPartitionProperty();
if (dynamicProperty.isExist() && dynamicProperty.getEnable()
&& partitionDesc.isAutoCreatePartitions()) {
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
String dynamicUnit = dynamicProperty.getTimeUnit();
ArrayList<Expr> autoExprs = partitionDesc.getPartitionExprs();
// check same interval. fail will leading to AnalysisException
DynamicPartitionUtil.partitionIntervalCompatible(dynamicUnit, autoExprs);
}
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());