database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature] support hour time unit with dynamic parition (#4514)

Browse Source 
Many tables are so large that need seperate partitions with "HOUR" time unit.
But now dynamic partition doesn't support "HOUR" time unit and it was marked as "TODO".
So I support the feature and it works.
This commit is contained in:
caoyang10
2020-09-06 20:25:27 +08:00
committed by GitHub
parent 13e2cf172f
commit 03d9f6d8b4
4 changed files with 35 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
View File

@ -210,7 +210,7 @@ public enum ErrorCode {
"Table %s is not a colocated table"),
ERR_INVALID_OPERATION(5065, new byte[] { '4', '2', '0', '0', '0' }, "Operation %s is invalid"),
ERROR_DYNAMIC_PARTITION_TIME_UNIT(5065, new byte[] {'4', '2', '0', '0', '0'},
"Unsupported time unit %s. Expect DAY/WEEK/MONTH."),
"Unsupported time unit %s. Expect HOUR/DAY/WEEK/MONTH."),
ERROR_DYNAMIC_PARTITION_START_ZERO(5066, new byte[] {'4', '2', '0', '0', '0'},
"Dynamic partition start must less than 0"),
ERROR_DYNAMIC_PARTITION_START_FORMAT(5066, new byte[] {'4', '2', '0', '0', '0'},

30
fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View File

@ -63,6 +63,7 @@ public class DynamicPartitionUtil {
public static void checkTimeUnit(String timeUnit) throws DdlException {
if (Strings.isNullOrEmpty(timeUnit)
|| !(timeUnit.equalsIgnoreCase(TimeUnit.DAY.toString())
|| timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())
|| timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())
|| timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString()))) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_TIME_UNIT, timeUnit);
@ -365,6 +366,8 @@ public class DynamicPartitionUtil {
return formattedDateStr.substring(0, 8);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.MONTH.toString())) {
return formattedDateStr.substring(0, 6);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) {
return formattedDateStr.substring(0, 10);
} else {
formattedDateStr = formattedDateStr.substring(0, 8);
Calendar calendar = Calendar.getInstance(tz);
@ -385,7 +388,8 @@ public class DynamicPartitionUtil {
}
// return the partition range date string formatted as yyyy-MM-dd[ HH:mm::ss]
// TODO: support HOUR and YEAR
// add support: HOUR by caoyang10
// TODO: support YEAR
public static String getPartitionRangeString(DynamicPartitionProperty property, ZonedDateTime current,
int offset, String format) {
String timeUnit = property.getTimeUnit();
@ -393,10 +397,27 @@ public class DynamicPartitionUtil {
return getPartitionRangeOfDay(current, offset, format);
} else if (timeUnit.equalsIgnoreCase(TimeUnit.WEEK.toString())) {
return getPartitionRangeOfWeek(current, offset, property.getStartOfWeek(), format);
} else { // MONTH
} else if (timeUnit.equalsIgnoreCase(TimeUnit.HOUR.toString())) { // MONTH
return getPartitionRangeOfHour(current, offset, format);
} else {
return getPartitionRangeOfMonth(current, offset, property.getStartOfMonth(), format);
}
}
/**
* return formatted string of partition range in HOUR granularity.
* offset: The offset from the current hour. 0 means current hour, 1 means next hour, -1 last hour.
* format: the format of the return date string.
*
* Eg:
* Today is 2020-05-24 00:12:34, offset = -1
* It will return 2020-05-23 23:00:00
* Today is 2020-05-24 00, offset = 1
* It will return 2020-05-24 01:00:00
*/
private static String getPartitionRangeOfHour(ZonedDateTime current, int offset, String format) {
return getFormattedTimeWithoutMinuteSecond(current.plusHours(offset), format);
}
/**
* return formatted string of partition range in DAY granularity.
@ -460,6 +481,11 @@ public class DynamicPartitionUtil {
return DateTimeFormatter.ofPattern(format).format(timeWithoutHourMinuteSecond);
}
private static String getFormattedTimeWithoutMinuteSecond(ZonedDateTime zonedDateTime, String format) {
ZonedDateTime timeWithoutMinuteSecond = zonedDateTime.withMinute(0).withSecond(0);
return DateTimeFormatter.ofPattern(format).format(timeWithoutMinuteSecond);
}
/**
* Used to indicate the start date.
* Taking the year as the granularity, it can indicate the month and day as the start date.