database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](dynamic partition) fix dynamic partition storage medium not working (#29490)

Browse Source
This commit is contained in:
yujun
2024-01-11 20:48:39 +08:00
committed by yiguolei
parent ad2c13e009
commit 2a51750abd
3 changed files with 193 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
View File

@ -19,7 +19,6 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.DynamicPartitionUtil;
@ -27,6 +26,8 @@ import org.apache.doris.common.util.DynamicPartitionUtil.StartOfDate;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.TimeUtils;
import com.google.common.base.Strings;
import java.util.Map;
import java.util.TimeZone;
@ -97,7 +98,7 @@ public class DynamicPartitionProperty {
this.reservedHistoryPeriods = properties.getOrDefault(
RESERVED_HISTORY_PERIODS, NOT_SET_RESERVED_HISTORY_PERIODS);
this.storagePolicy = properties.getOrDefault(STORAGE_POLICY, "");
this.storageMedium = properties.getOrDefault(STORAGE_MEDIUM, Config.default_storage_medium);
this.storageMedium = properties.getOrDefault(STORAGE_MEDIUM, "");
createStartOfs(properties);
} else {
this.exist = false;
@ -228,8 +229,10 @@ public class DynamicPartitionProperty {
+ ",\n\"" + HISTORY_PARTITION_NUM + "\" = \"" + historyPartitionNum + "\""
+ ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum + "\""
+ ",\n\"" + RESERVED_HISTORY_PERIODS + "\" = \"" + reservedHistoryPeriods + "\""
+ ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\""
+ ",\n\"" + STORAGE_MEDIUM + "\" = \"" + storageMedium + "\"";
+ ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\"";
if (!Strings.isNullOrEmpty(storageMedium)) {
res += ",\n\"" + STORAGE_MEDIUM + "\" = \"" + storageMedium + "\"";
}
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\"";
} else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) {

30
fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
View File

@ -52,6 +52,7 @@ import org.apache.doris.common.util.RangeUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.thrift.TStorageMedium;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
@ -351,19 +352,28 @@ public class DynamicPartitionScheduler extends MasterDaemon {
*/
private void setStorageMediumProperty(HashMap<String, String> partitionProperties,
DynamicPartitionProperty property, ZonedDateTime now, int hotPartitionNum, int offset) {
if ((hotPartitionNum <= 0 || offset + hotPartitionNum <= 0) && !property.getStorageMedium()
.equalsIgnoreCase("ssd")) {
return;
}
String cooldownTime;
if (property.getStorageMedium().equalsIgnoreCase("ssd")) {
cooldownTime = TimeUtils.longToTimeString(DataProperty.MAX_COOLDOWN_TIME_MS);
// 1. no hot partition, then use dynamic_partition.storage_medium
// 2. has hot partition
// 1) dynamic_partition.storage_medium = 'ssd', then use ssd;
// 2) otherwise
// a. cooldown partition, then use hdd
// b. hot partition. then use ssd
if (hotPartitionNum <= 0 || property.getStorageMedium().equalsIgnoreCase("ssd")) {
if (!Strings.isNullOrEmpty(property.getStorageMedium())) {
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, property.getStorageMedium());
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
TimeUtils.longToTimeString(DataProperty.MAX_COOLDOWN_TIME_MS));
}
} else if (offset + hotPartitionNum <= 0) {
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, TStorageMedium.HDD.name());
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME,
TimeUtils.longToTimeString(DataProperty.MAX_COOLDOWN_TIME_MS));
} else {
cooldownTime = DynamicPartitionUtil.getPartitionRangeString(
String cooldownTime = DynamicPartitionUtil.getPartitionRangeString(
property, now, offset + hotPartitionNum, DynamicPartitionUtil.DATETIME_FORMAT);
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, TStorageMedium.SSD.name());
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME, cooldownTime);
}
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM, TStorageMedium.SSD.name());
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_COOLDOWN_TIME, cooldownTime);
}
private void setStoragePolicyProperty(HashMap<String, String> partitionProperties,