database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[config](storage-policy) add a FE config to disable storage policy by default (#14655)

Browse Source 
the cold-hot separation feature is still
under development. And seems there are some unsolved feature remains.
So I add a fe config enable_storage_policy, and default is false, to disable the creation and usage of storage policy by default.

So that user can aware that he is using an experimental feature on his own, and it will not be released formally in v1.2.0.

Disable storage policy by default, user can not use or create storage policy. Configured by enable_storage_policy.

Remove property remote_storage_policy, it is duplicate with storage_policy

Change the persist field in DataProperty.java.
And remove remoteCooldownTime from DataProperty, because it can be got from StoragePolicy.
This commit is contained in:
Mingyu Chen
2022-11-30 10:04:33 +08:00
committed by GitHub
parent dd7ec8f4ca
commit ca90253b09
29 changed files with 219 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
docs/en/docs/admin-manual/config/fe-config.md
View File

@ -2347,3 +2347,14 @@ Default: 3
Is it possible to dynamically configure: true
Is it a configuration item unique to the Master FE node: true
### `enable_storage_policy`
Whether to enable the Storage Policy feature. This feature allows users to separate hot and cold data. This feature is still under development. Recommended for test environments only.
Default: false
Is it possible to dynamically configure: true
Is it a configuration item unique to the Master FE node: true

11
docs/zh-CN/docs/admin-manual/config/fe-config.md
View File

@ -2403,3 +2403,14 @@ hive partition 的最大缓存数量。
是否可以动态配置：true
是否为 Master FE 节点独有的配置项：true
### `enable_storage_policy`
是否开启 Storage Policy 功能。该功能用户冷热数据分离功能。该功能仍在开发中，不排除后续后功能修改或重构。仅建议测试环境使用。
默认值：false。即不开启
是否可以动态配置：true
是否为 Master FE 节点独有的配置项：true

5
fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
View File

@ -1712,9 +1712,8 @@ public class SchemaChangeHandler extends AlterHandler {
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)) {
Env.getCurrentEnv().modifyTableReplicaAllocation(db, olapTable, properties);
return;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY)) {
olapTable.setRemoteStoragePolicy(
properties.get(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY));
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)) {
olapTable.setStoragePolicy(properties.get(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY));
return;
}
}

5
fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
@ -92,6 +93,10 @@ public class CreatePolicyStmt extends DdlStmt {
super.analyze(analyzer);
switch (type) {
case STORAGE:
if (!Config.enable_storage_policy) {
throw new UserException("storage policy feature is disabled by default. "
+ "Enable it by setting 'enable_storage_policy=true' in fe.conf");
}
break;
case ROW:
default:

2
fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
View File

@ -97,8 +97,6 @@ public class ModifyTablePropertiesClause extends AlterTableClause {
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY)) {
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY)) {
// do nothing, just check valid.
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_TABLET_TYPE)) {
throw new AnalysisException("Alter tablet type not supported");
} else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)) {

44
fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java
View File

@ -38,15 +38,12 @@ public class DataProperty implements Writable {
);
public static final long MAX_COOLDOWN_TIME_MS = 253402271999000L; // 9999-12-31 23:59:59
@SerializedName(value = "storageMedium")
@SerializedName(value = "storageMedium")
private TStorageMedium storageMedium;
@SerializedName(value = "cooldownTimeMs")
@SerializedName(value = "cooldownTimeMs")
private long cooldownTimeMs;
@SerializedName(value = "remoteStoragePolicy")
private String remoteStoragePolicy;
// cooldown time for remote storage
@SerializedName(value = "remoteCooldownTimeMs")
private long remoteCooldownTimeMs;
@SerializedName(value = "storagePolicy")
private String storagePolicy;
private DataProperty() {
// for persist
@ -60,8 +57,7 @@ public class DataProperty implements Writable {
} else {
this.cooldownTimeMs = MAX_COOLDOWN_TIME_MS;
}
this.remoteStoragePolicy = "";
this.remoteCooldownTimeMs = MAX_COOLDOWN_TIME_MS;
this.storagePolicy = "";
}
/**
@ -69,18 +65,12 @@ public class DataProperty implements Writable {
*
* @param medium storage medium for the init storage of the table
* @param cooldown cool down time for SSD->HDD
* @param remoteStoragePolicy remote storage policy for remote storage
* @param remoteCooldownTimeMs remote storage cooldown time
* @param storagePolicy remote storage policy for remote storage
*/
public DataProperty(TStorageMedium medium, long cooldown, String remoteStoragePolicy, long remoteCooldownTimeMs) {
public DataProperty(TStorageMedium medium, long cooldown, String storagePolicy) {
this.storageMedium = medium;
this.cooldownTimeMs = cooldown;
this.remoteStoragePolicy = remoteStoragePolicy;
if (remoteCooldownTimeMs > 0) {
this.remoteCooldownTimeMs = remoteCooldownTimeMs;
} else {
this.remoteCooldownTimeMs = MAX_COOLDOWN_TIME_MS;
}
this.storagePolicy = storagePolicy;
}
public TStorageMedium getStorageMedium() {
@ -91,12 +81,8 @@ public class DataProperty implements Writable {
return cooldownTimeMs;
}
public String getRemoteStoragePolicy() {
return remoteStoragePolicy;
}
public long getRemoteCooldownTimeMs() {
return remoteCooldownTimeMs;
public String getStoragePolicy() {
return storagePolicy;
}
public static DataProperty read(DataInput in) throws IOException {
@ -118,12 +104,12 @@ public class DataProperty implements Writable {
public void readFields(DataInput in) throws IOException {
storageMedium = TStorageMedium.valueOf(Text.readString(in));
cooldownTimeMs = in.readLong();
remoteStoragePolicy = "";
storagePolicy = "";
}
@Override
public int hashCode() {
return Objects.hash(storageMedium, cooldownTimeMs, remoteStoragePolicy, remoteCooldownTimeMs);
return Objects.hash(storageMedium, cooldownTimeMs, storagePolicy);
}
@Override
@ -140,8 +126,7 @@ public class DataProperty implements Writable {
return this.storageMedium == other.storageMedium
&& this.cooldownTimeMs == other.cooldownTimeMs
&& this.remoteStoragePolicy.equals(other.remoteStoragePolicy)
&& this.remoteCooldownTimeMs == other.remoteCooldownTimeMs;
&& this.storagePolicy.equals(other.storagePolicy);
}
@Override
@ -149,8 +134,7 @@ public class DataProperty implements Writable {
StringBuilder sb = new StringBuilder();
sb.append("Storage medium[").append(this.storageMedium).append("]. ");
sb.append("cool down[").append(TimeUtils.longToTimeString(cooldownTimeMs)).append("]. ");
sb.append("remote storage policy[").append(this.remoteStoragePolicy).append("]. ");
sb.append("remote cooldown time[").append(TimeUtils.longToTimeString(remoteCooldownTimeMs)).append("]. ");
sb.append("remote storage policy[").append(this.storagePolicy).append("]. ");
return sb.toString();
}
}

12
fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java
View File

@ -46,7 +46,7 @@ public class DynamicPartitionProperty {
public static final String HISTORY_PARTITION_NUM = "dynamic_partition.history_partition_num";
public static final String HOT_PARTITION_NUM = "dynamic_partition.hot_partition_num";
public static final String RESERVED_HISTORY_PERIODS = "dynamic_partition.reserved_history_periods";
public static final String REMOTE_STORAGE_POLICY = "dynamic_partition.remote_storage_policy";
public static final String STORAGE_POLICY = "dynamic_partition.storage_policy";
public static final int MIN_START_OFFSET = Integer.MIN_VALUE;
public static final int MAX_END_OFFSET = Integer.MAX_VALUE;
@ -73,7 +73,7 @@ public class DynamicPartitionProperty {
// If not set, default is 0
private int hotPartitionNum;
private String reservedHistoryPeriods;
private String remoteStoragePolicy;
private String storagePolicy;
public DynamicPartitionProperty(Map<String, String> properties) {
if (properties != null && !properties.isEmpty()) {
@ -93,7 +93,7 @@ public class DynamicPartitionProperty {
this.hotPartitionNum = Integer.parseInt(properties.getOrDefault(HOT_PARTITION_NUM, "0"));
this.reservedHistoryPeriods = properties.getOrDefault(
RESERVED_HISTORY_PERIODS, NOT_SET_RESERVED_HISTORY_PERIODS);
this.remoteStoragePolicy = properties.getOrDefault(REMOTE_STORAGE_POLICY, "");
this.storagePolicy = properties.getOrDefault(STORAGE_POLICY, "");
createStartOfs(properties);
} else {
this.exist = false;
@ -173,8 +173,8 @@ public class DynamicPartitionProperty {
return hotPartitionNum;
}
public String getRemoteStoragePolicy() {
return remoteStoragePolicy;
public String getStoragePolicy() {
return storagePolicy;
}
public String getStartOfInfo() {
@ -220,7 +220,7 @@ public class DynamicPartitionProperty {
+ ",\n\"" + HISTORY_PARTITION_NUM + "\" = \"" + historyPartitionNum + "\""
+ ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum + "\""
+ ",\n\"" + RESERVED_HISTORY_PERIODS + "\" = \"" + reservedHistoryPeriods + "\""
+ ",\n\"" + REMOTE_STORAGE_POLICY + "\" = \"" + remoteStoragePolicy + "\"";
+ ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\"";
if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) {
res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\"";
} else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) {

6
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View File

@ -2971,12 +2971,6 @@ public class Env {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT).append("\" = \"");
sb.append(olapTable.getStorageFormat()).append("\"");
// remote storage
String remoteStoragePolicy = olapTable.getRemoteStoragePolicy();
if (!Strings.isNullOrEmpty(remoteStoragePolicy)) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY).append("\" = \"");
sb.append(remoteStoragePolicy).append("\"");
}
// compression type
if (olapTable.getCompressionType() != TCompressionType.LZ4F) {
sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_COMPRESSION).append("\" = \"");

33
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -36,6 +36,7 @@ import org.apache.doris.catalog.Tablet.TabletStatus;
import org.apache.doris.clone.TabletSchedCtx;
import org.apache.doris.clone.TabletScheduler;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
@ -68,6 +69,7 @@ import com.google.common.collect.Sets;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.parquet.Strings;
import java.io.DataInput;
import java.io.DataOutput;
@ -1627,7 +1629,11 @@ public class OlapTable extends Table {
tableProperty.buildEnableLightSchemaChange();
}
public void setStoragePolicy(String storagePolicy) {
public void setStoragePolicy(String storagePolicy) throws UserException {
if (!Config.enable_storage_policy && !Strings.isNullOrEmpty(storagePolicy)) {
throw new UserException("storage policy feature is disabled by default. "
+ "Enable it by setting 'enable_storage_policy=true' in fe.conf");
}
if (tableProperty == null) {
tableProperty = new TableProperty(new HashMap<>());
}
@ -1666,19 +1672,6 @@ public class OlapTable extends Table {
tableProperty.buildDataSortInfo();
}
/**
* set remote storage policy for table.
*
* @param remoteStoragePolicy remote storage policy name
*/
public void setRemoteStoragePolicy(String remoteStoragePolicy) {
if (tableProperty == null) {
tableProperty = new TableProperty(new HashMap<>());
}
tableProperty.setRemoteStoragePolicy(remoteStoragePolicy);
tableProperty.buildRemoteStoragePolicy();
}
// return true if partition with given name already exist, both in partitions and temp partitions.
// return false otherwise
public boolean checkPartitionNameExist(String partitionName) {
@ -1846,18 +1839,6 @@ public class OlapTable extends Table {
return tableProperty.getDataSortInfo();
}
/**
* get remote storage policy name.
*
* @return remote storage policy name for this table.
*/
public String getRemoteStoragePolicy() {
if (tableProperty == null) {
return "";
}
return tableProperty.getRemoteStoragePolicy();
}
public void setEnableUniqueKeyMergeOnWrite(boolean speedup) {
if (tableProperty == null) {
tableProperty = new TableProperty(new HashMap<>());

18
fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
View File

@ -79,9 +79,6 @@ public class TableProperty implements Writable {
private DataSortInfo dataSortInfo = new DataSortInfo();
// remote storage policy, for cold data
private String remoteStoragePolicy;
public TableProperty(Map<String, String> properties) {
this.properties = properties;
}
@ -200,11 +197,6 @@ public class TableProperty implements Writable {
return this;
}
public TableProperty buildRemoteStoragePolicy() {
remoteStoragePolicy = properties.getOrDefault(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY, "");
return this;
}
public void modifyTableProperties(Map<String, String> modifyProperties) {
properties.putAll(modifyProperties);
removeDuplicateReplicaNumProperty();
@ -222,11 +214,6 @@ public class TableProperty implements Writable {
replicaAlloc.toCreateStmt());
}
public void setRemoteStoragePolicy(String remotePolicyName) {
this.remoteStoragePolicy = remotePolicyName;
properties.put(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY, remotePolicyName);
}
public ReplicaAllocation getReplicaAllocation() {
return replicaAlloc;
}
@ -269,10 +256,6 @@ public class TableProperty implements Writable {
return dataSortInfo;
}
public String getRemoteStoragePolicy() {
return remoteStoragePolicy;
}
public TCompressionType getCompressionType() {
return compressionType;
}
@ -324,7 +307,6 @@ public class TableProperty implements Writable {
.buildInMemory()
.buildStorageFormat()
.buildDataSortInfo()
.buildRemoteStoragePolicy()
.buildCompressionType()
.buildStoragePolicy()
.buildEnableLightSchemaChange();

4
fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
View File

@ -162,7 +162,7 @@ public class DynamicPartitionScheduler extends MasterDaemon {
idx = 0;
}
int hotPartitionNum = dynamicPartitionProperty.getHotPartitionNum();
String storagePolicyName = dynamicPartitionProperty.getRemoteStoragePolicy();
String storagePolicyName = dynamicPartitionProperty.getStoragePolicy();
for (; idx <= dynamicPartitionProperty.getEnd(); idx++) {
String prevBorder = DynamicPartitionUtil.getPartitionRangeString(
@ -267,7 +267,7 @@ public class DynamicPartitionScheduler extends MasterDaemon {
private void setStoragePolicyProperty(HashMap<String, String> partitionProperties,
DynamicPartitionProperty property, ZonedDateTime now, int offset,
String storagePolicyName) {
partitionProperties.put(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY, storagePolicyName);
partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY, storagePolicyName);
String baseTime = DynamicPartitionUtil.getPartitionRangeString(
property, now, offset, DynamicPartitionUtil.DATETIME_FORMAT);
partitionProperties.put(PropertyAnalyzer.PROPERTIES_DATA_BASE_TIME, baseTime);

7
fe/fe-core/src/main/java/org/apache/doris/common/Config.java
View File

@ -1918,5 +1918,12 @@ public class Config extends ConfigBase {
*/
@ConfField(mutable = true, masterOnly = true)
public static int max_same_name_catalog_trash_num = 3;
/**
* The storage policy is still under developement.
* Disable it by default.
*/
@ConfField(mutable = true, masterOnly = true)
public static boolean enable_storage_policy = false;
}

2
fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
View File

@ -288,7 +288,7 @@ public class PartitionsProcDir implements ProcDirInterface {
DataProperty dataProperty = tblPartitionInfo.getDataProperty(partitionId);
partitionInfo.add(dataProperty.getStorageMedium().name());
partitionInfo.add(TimeUtils.longToTimeString(dataProperty.getCooldownTimeMs()));
partitionInfo.add(dataProperty.getRemoteStoragePolicy());
partitionInfo.add(dataProperty.getStoragePolicy());
partitionInfo.add(TimeUtils.longToTimeString(partition.getLastCheckTime()));

16
fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View File

@ -346,22 +346,22 @@ public class DynamicPartitionUtil {
private static void checkRemoteStoragePolicy(String policyName) throws DdlException {
if (Strings.isNullOrEmpty(policyName)) {
LOG.info(DynamicPartitionProperty.REMOTE_STORAGE_POLICY + " is null, remove this key");
LOG.info(DynamicPartitionProperty.STORAGE_POLICY + " is null, remove this key");
return;
}
if (policyName.isEmpty()) {
throw new DdlException(DynamicPartitionProperty.REMOTE_STORAGE_POLICY + " is empty.");
throw new DdlException(DynamicPartitionProperty.STORAGE_POLICY + " is empty.");
}
StoragePolicy checkedPolicyCondition = StoragePolicy.ofCheck(policyName);
if (!Env.getCurrentEnv().getPolicyMgr().existPolicy(checkedPolicyCondition)) {
throw new DdlException(
DynamicPartitionProperty.REMOTE_STORAGE_POLICY + ": " + policyName + " doesn't exist.");
DynamicPartitionProperty.STORAGE_POLICY + ": " + policyName + " doesn't exist.");
}
StoragePolicy storagePolicy = (StoragePolicy) Env.getCurrentEnv().getPolicyMgr()
.getPolicy(checkedPolicyCondition);
if (Strings.isNullOrEmpty(storagePolicy.getCooldownTtl())) {
throw new DdlException("Storage policy cooldown type need to be cooldownTtl for properties "
+ DynamicPartitionProperty.REMOTE_STORAGE_POLICY + ": " + policyName);
+ DynamicPartitionProperty.STORAGE_POLICY + ": " + policyName);
}
}
@ -616,12 +616,12 @@ public class DynamicPartitionUtil {
properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS);
analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriods);
}
if (properties.containsKey(DynamicPartitionProperty.REMOTE_STORAGE_POLICY)) {
String remoteStoragePolicy = properties.get(DynamicPartitionProperty.REMOTE_STORAGE_POLICY);
if (properties.containsKey(DynamicPartitionProperty.STORAGE_POLICY)) {
String remoteStoragePolicy = properties.get(DynamicPartitionProperty.STORAGE_POLICY);
checkRemoteStoragePolicy(remoteStoragePolicy);
properties.remove(DynamicPartitionProperty.REMOTE_STORAGE_POLICY);
properties.remove(DynamicPartitionProperty.STORAGE_POLICY);
if (!Strings.isNullOrEmpty(remoteStoragePolicy)) {
analyzedProperties.put(DynamicPartitionProperty.REMOTE_STORAGE_POLICY, remoteStoragePolicy);
analyzedProperties.put(DynamicPartitionProperty.STORAGE_POLICY, remoteStoragePolicy);
}
}
return analyzedProperties;

81
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -46,7 +46,6 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -94,8 +93,6 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_INMEMORY = "in_memory";
public static final String PROPERTIES_REMOTE_STORAGE_POLICY = "remote_storage_policy";
public static final String PROPERTIES_TABLET_TYPE = "tablet_type";
public static final String PROPERTIES_STRICT_RANGE = "strict_range";
@ -147,12 +144,10 @@ public class PropertyAnalyzer {
}
TStorageMedium storageMedium = oldDataProperty.getStorageMedium();
long cooldownTimeStamp = oldDataProperty.getCooldownTimeMs();
String remoteStoragePolicy = oldDataProperty.getRemoteStoragePolicy();
long remoteCooldownTimeMs = oldDataProperty.getRemoteCooldownTimeMs();
long cooldownTimestamp = oldDataProperty.getCooldownTimeMs();
String newStoragePolicy = oldDataProperty.getStoragePolicy();
boolean hasStoragePolicy = false;
long dataBaseTimeMs = 0;
for (Map.Entry<String, String> entry : properties.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
@ -166,74 +161,66 @@ public class PropertyAnalyzer {
}
} else if (key.equalsIgnoreCase(PROPERTIES_STORAGE_COOLDOWN_TIME)) {
DateLiteral dateLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(Type.DATETIME));
cooldownTimeStamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone());
} else if (key.equalsIgnoreCase(PROPERTIES_REMOTE_STORAGE_POLICY)) {
remoteStoragePolicy = value;
} else if (key.equalsIgnoreCase(PROPERTIES_DATA_BASE_TIME)) {
DateLiteral dateLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(Type.DATETIME));
dataBaseTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone());
cooldownTimestamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone());
} else if (!hasStoragePolicy && key.equalsIgnoreCase(PROPERTIES_STORAGE_POLICY)) {
if (!Strings.isNullOrEmpty(value)) {
hasStoragePolicy = true;
newStoragePolicy = value;
}
}
} // end for properties
properties.remove(PROPERTIES_STORAGE_MEDIUM);
properties.remove(PROPERTIES_STORAGE_COOLDOWN_TIME);
properties.remove(PROPERTIES_REMOTE_STORAGE_POLICY);
properties.remove(PROPERTIES_STORAGE_POLICY);
properties.remove(PROPERTIES_DATA_BASE_TIME);
Preconditions.checkNotNull(storageMedium);
if (storageMedium == TStorageMedium.HDD) {
cooldownTimeStamp = DataProperty.MAX_COOLDOWN_TIME_MS;
cooldownTimestamp = DataProperty.MAX_COOLDOWN_TIME_MS;
LOG.info("Can not assign cool down timestamp to HDD storage medium, ignore user setting.");
}
boolean hasCooldown = cooldownTimeStamp != DataProperty.MAX_COOLDOWN_TIME_MS;
boolean hasRemoteStoragePolicy = StringUtils.isNotEmpty(remoteStoragePolicy);
boolean hasCooldown = cooldownTimestamp != DataProperty.MAX_COOLDOWN_TIME_MS;
long currentTimeMs = System.currentTimeMillis();
if (storageMedium == TStorageMedium.SSD && hasCooldown) {
if (cooldownTimeStamp <= currentTimeMs) {
throw new AnalysisException("Cool down time should later than now");
if (cooldownTimestamp <= currentTimeMs) {
throw new AnalysisException(
"Cool down time: " + cooldownTimestamp + " should later than now: " + currentTimeMs);
}
}
if (storageMedium == TStorageMedium.SSD && !hasCooldown) {
// set default cooldown time
cooldownTimeStamp = currentTimeMs + Config.storage_cooldown_second * 1000L;
cooldownTimestamp = currentTimeMs + Config.storage_cooldown_second * 1000L;
}
if (hasRemoteStoragePolicy) {
if (hasStoragePolicy) {
// check remote storage policy
StoragePolicy checkedPolicy = StoragePolicy.ofCheck(remoteStoragePolicy);
StoragePolicy checkedPolicy = StoragePolicy.ofCheck(newStoragePolicy);
Policy policy = Env.getCurrentEnv().getPolicyMgr().getPolicy(checkedPolicy);
if (!(policy instanceof StoragePolicy)) {
throw new AnalysisException("No PolicyStorage: " + remoteStoragePolicy);
throw new AnalysisException("No PolicyStorage: " + newStoragePolicy);
}
StoragePolicy storagePolicy = (StoragePolicy) policy;
// check remote storage cool down timestamp
if (storagePolicy.getCooldownTimestampMs() != -1) {
if (storagePolicy.getCooldownTimestampMs() <= currentTimeMs) {
throw new AnalysisException("Remote storage cool down time should later than now");
}
if (hasCooldown && storagePolicy.getCooldownTimestampMs() <= cooldownTimeStamp) {
throw new AnalysisException(
"`remote_storage_cooldown_time`" + " should later than `storage_cooldown_time`.");
"remote storage cool down time: " + storagePolicy.getCooldownTimestampMs()
+ " should later than now: " + currentTimeMs);
}
if (hasCooldown && storagePolicy.getCooldownTimestampMs() <= cooldownTimestamp) {
throw new AnalysisException(
"remote storage cool down time: " + storagePolicy.getCooldownTimestampMs()
+ " should later than storage cool down time: " + cooldownTimestamp);
}
remoteCooldownTimeMs = storagePolicy.getCooldownTimestampMs();
} else if (storagePolicy.getCooldownTtl() != null && dataBaseTimeMs > 0) {
remoteCooldownTimeMs = dataBaseTimeMs + storagePolicy.getCooldownTtlMs();
}
}
if (dataBaseTimeMs <= 0) {
remoteCooldownTimeMs = DataProperty.MAX_COOLDOWN_TIME_MS;
}
return new DataProperty(storageMedium, cooldownTimeStamp, remoteStoragePolicy, remoteCooldownTimeMs);
return new DataProperty(storageMedium, cooldownTimestamp, newStoragePolicy);
}
public static short analyzeShortKeyColumnCount(Map<String, String> properties) throws AnalysisException {
@ -560,28 +547,6 @@ public class PropertyAnalyzer {
return defaultVal;
}
/**
* analyze remote storage policy.
*
* @param properties property for table
* @return remote storage policy name
* @throws AnalysisException policy name doesn't exist
*/
public static String analyzeRemoteStoragePolicy(Map<String, String> properties) throws AnalysisException {
String remoteStoragePolicy = "";
if (properties != null && properties.containsKey(PROPERTIES_REMOTE_STORAGE_POLICY)) {
remoteStoragePolicy = properties.get(PROPERTIES_REMOTE_STORAGE_POLICY);
// check remote storage policy existence
StoragePolicy checkedStoragePolicy = StoragePolicy.ofCheck(remoteStoragePolicy);
Policy policy = Env.getCurrentEnv().getPolicyMgr().getPolicy(checkedStoragePolicy);
if (!(policy instanceof StoragePolicy)) {
throw new AnalysisException("StoragePolicy: " + remoteStoragePolicy + " does not exist.");
}
}
return remoteStoragePolicy;
}
public static String analyzeStoragePolicy(Map<String, String> properties) throws AnalysisException {
String storagePolicy = "";
if (properties != null && properties.containsKey(PROPERTIES_STORAGE_POLICY)) {
@ -833,3 +798,5 @@ public class PropertyAnalyzer {
}
}
}

6
fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View File

@ -1925,15 +1925,9 @@ public class InternalCatalog implements CatalogIf<Database> {
false);
olapTable.setIsInMemory(isInMemory);
// set remote storage
String remoteStoragePolicy = PropertyAnalyzer.analyzeRemoteStoragePolicy(properties);
olapTable.setRemoteStoragePolicy(remoteStoragePolicy);
// set storage policy
String storagePolicy = PropertyAnalyzer.analyzeStoragePolicy(properties);
Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(storagePolicy);
olapTable.setStoragePolicy(storagePolicy);
TTabletType tabletType;

17
fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
View File

@ -78,6 +78,7 @@ public class AlterTest {
FeConstants.default_scheduler_interval_millisecond = 100;
Config.dynamic_partition_check_interval_seconds = 1;
Config.disable_storage_medium_check = true;
Config.enable_storage_policy = true;
UtFrameUtils.createDorisCluster(runningDir);
be = Env.getCurrentSystemInfo().getIdToBackend().values().asList().get(0);
@ -165,7 +166,7 @@ public class AlterTest {
+ " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n"
+ " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00',\n"
+ " 'remote_storage_policy' = 'testPolicy'\n" + ");");
+ " 'storage_policy' = 'testPolicy'\n" + ");");
createTable("create table test.show_test (k1 int, k2 int) distributed by hash(k1) "
+ "buckets 1 properties(\"replication_num\" = \"1\");");
@ -512,13 +513,13 @@ public class AlterTest {
stmt = "alter table test.tbl4 modify partition (p3, p4) set ('storage_medium' = 'HDD')";
DateLiteral dateLiteral = new DateLiteral("2999-12-31 00:00:00", Type.DATETIME);
long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone());
DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "", -1);
DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "");
partitionList = Lists.newArrayList(p3, p4);
for (Partition partition : partitionList) {
Assert.assertEquals(oldDataProperty, tbl4.getPartitionInfo().getDataProperty(partition.getId()));
}
alterTable(stmt, false);
DataProperty newDataProperty = new DataProperty(TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "", -1);
DataProperty newDataProperty = new DataProperty(TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "");
for (Partition partition : partitionList) {
Assert.assertEquals(newDataProperty, tbl4.getPartitionInfo().getDataProperty(partition.getId()));
}
@ -531,7 +532,7 @@ public class AlterTest {
dateLiteral = new DateLiteral("2100-12-31 00:00:00", Type.DATETIME);
cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone());
DataProperty newDataProperty1 = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "", -1);
DataProperty newDataProperty1 = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "");
partitionList = Lists.newArrayList(p1, p2);
for (Partition partition : partitionList) {
Assert.assertEquals(newDataProperty1, tbl4.getPartitionInfo().getDataProperty(partition.getId()));
@ -559,7 +560,7 @@ public class AlterTest {
DateLiteral dateLiteral = new DateLiteral("2100-05-09 00:00:00", Type.DATETIME);
long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone());
DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy", -1);
DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy");
List<Partition> partitionList = Lists.newArrayList(p2, p3, p4);
for (Partition partition : partitionList) {
Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(partition.getId()));
@ -570,7 +571,7 @@ public class AlterTest {
alterTable(stmt, false);
DateLiteral newDateLiteral = new DateLiteral("2100-04-01 22:22:22", Type.DATETIME);
long newCooldownTimeMs = newDateLiteral.unixTimestamp(TimeUtils.getTimeZone());
DataProperty dataProperty2 = new DataProperty(TStorageMedium.SSD, newCooldownTimeMs, "testPolicy", -1);
DataProperty dataProperty2 = new DataProperty(TStorageMedium.SSD, newCooldownTimeMs, "testPolicy");
for (Partition partition : partitionList) {
Assert.assertEquals(dataProperty2, tblRemote.getPartitionInfo().getDataProperty(partition.getId()));
}
@ -580,14 +581,14 @@ public class AlterTest {
stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('storage_medium' = 'HDD')";
alterTable(stmt, false);
DataProperty dataProperty1 = new DataProperty(
TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "testPolicy", -1);
TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "testPolicy");
for (Partition partition : partitionList) {
Assert.assertEquals(dataProperty1, tblRemote.getPartitionInfo().getDataProperty(partition.getId()));
}
Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(p1.getId()));
// alter remote_storage
stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('remote_storage_policy' = 'testPolicy3')";
stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('storage_policy' = 'testPolicy3')";
alterTable(stmt, true);
Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(p1.getId()));

2
fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java
View File

@ -35,7 +35,7 @@ public class DataPropertyTest {
Assert.assertNotEquals(DataProperty.MAX_COOLDOWN_TIME_MS, dataProperty.getCooldownTimeMs());
long storageCooldownTimeMs = System.currentTimeMillis() + 24 * 3600 * 1000L;
dataProperty = new DataProperty(TStorageMedium.SSD, storageCooldownTimeMs, "", -1);
dataProperty = new DataProperty(TStorageMedium.SSD, storageCooldownTimeMs, "");
Assert.assertEquals(storageCooldownTimeMs, dataProperty.getCooldownTimeMs());
dataProperty = new DataProperty(TStorageMedium.HDD);

2
fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java
View File

@ -27,6 +27,7 @@ import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.AccessPrivilege;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
@ -51,6 +52,7 @@ public class PolicyTest extends TestWithFeService {
@Override
protected void runBeforeAll() throws Exception {
Config.enable_storage_policy = true;
FeConstants.runningUnitTest = true;
createDatabase("test");
useDatabase("test");

28
regression-test/conf/regression-conf.groovy
View File

@ -23,11 +23,11 @@ defaultDb = "regression_test"
// add useLocalSessionState so that the jdbc will not send
// init cmd like: select @@session.tx_read_only
// at each time we connect.
jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true"
jdbcUrl = "jdbc:mysql://127.0.0.1:9033/?useLocalSessionState=true"
jdbcUser = "root"
jdbcPassword = ""
feHttpAddress = "127.0.0.1:8030"
feHttpAddress = "127.0.0.1:8033"
feHttpUser = "root"
feHttpPassword = ""
@ -66,26 +66,26 @@ brokerName = "broker_name"
// broker load test config
enableBrokerLoad=true
ak=""
sk=""
ak="AKIDJTO4VheMnzwd5HEvNZWHmiPr41jCPN5Y"
sk="GH2Xu2uXu4zf5GDvNaXFivvgBRSDjz1o"
// jdbc connector test config
// To enable jdbc test, you need first start mysql/pg container.
// See `docker/thirdparties/start-thirdparties-docker.sh`
enableJdbcTest=false
mysql_57_port=3316
pg_14_port=5442
enableJdbcTest=true
mysql_57_port=3336
pg_14_port=5444
// hive catalog test config
// To enable jdbc test, you need first start hive container.
// See `docker/thirdparties/start-thirdparties-docker.sh`
enableHiveTest=false
hms_port=9183
hdfs_port=8120
enableHiveTest=true
hms_port=9184
hdfs_port=8121
// elasticsearch catalog test config
// See `docker/thirdparties/start-thirdparties-docker.sh`
enableEsTest=false
es_6_port=19200
es_7_port=29200
es_8_port=39200
enableEsTest=true
es_6_port=19222
es_7_port=29222
es_8_port=39222

6
regression-test/suites/cold_heat_separation/policy/alter.groovy
View File

@ -15,11 +15,9 @@
// specific language governing permissions and limitations
// under the License.
// The cases is copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions
// and modified by Doris.
suite("alter_policy") {
sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");"""
def has_resouce_policy_alter = sql """
SHOW RESOURCES WHERE NAME = "has_resouce_policy_alter";
"""

5
regression-test/suites/cold_heat_separation/policy/create.groovy
View File

@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
// The cases is copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions
// and modified by Doris.
suite("create_policy") {
sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");"""
def has_created_1 = sql """
SHOW RESOURCES WHERE NAME = "crete_policy_1";
"""

6
regression-test/suites/cold_heat_separation/policy/drop.groovy
View File

@ -15,11 +15,9 @@
// specific language governing permissions and limitations
// under the License.
// The cases is copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions
// and modified by Doris.
suite("drop_policy") {
sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");"""
def storage_exist = { name ->
def show_storage_policy = sql """
SHOW STORAGE POLICY;

6
regression-test/suites/cold_heat_separation/policy/show.groovy
View File

@ -15,13 +15,11 @@
// specific language governing permissions and limitations
// under the License.
// The cases is copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions
// and modified by Doris.
import groovy.json.JsonSlurper
suite("show_policy") {
sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");"""
def storage_exist = { name ->
def show_storage_policy = sql """
SHOW STORAGE POLICY;

6
regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy
View File

@ -15,11 +15,9 @@
// specific language governing permissions and limitations
// under the License.
// The cases is copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions
// and modified by Doris.
suite("add_table_policy_by_alter_table") {
sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");"""
def create_table_not_have_policy_result = try_sql """
CREATE TABLE IF NOT EXISTS create_table_not_have_policy
(

35
regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy
View File

@ -15,13 +15,18 @@
// specific language governing permissions and limitations
// under the License.
// The cases is copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions
// and modified by Doris.
import java.text.SimpleDateFormat;
import java.util.Date;
suite("create_table_use_partion_policy") {
def create_table_partion_use_not_create_policy = try_sql """
CREATE TABLE IF NOT EXISTS create_table_partion_use_not_create_policy
suite("create_table_use_partition_policy") {
sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");"""
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
Date date = new Date(System.currentTimeMillis() + 3600000)
def cooldownTime = format.format(date)
def create_table_partition_use_not_create_policy = try_sql """
CREATE TABLE IF NOT EXISTS create_table_partition_use_not_create_policy
(
k1 DATE,
k2 INT,
@ -33,7 +38,7 @@ suite("create_table_use_partion_policy") {
"""
// errCode = 2, detailMessage = Storage policy does not exist. name: not_exist_policy_1
assertEquals(create_table_partion_use_not_create_policy, null)
assertEquals(create_table_partition_use_not_create_policy, null)
def storage_exist = { name ->
def show_storage_policy = sql """
@ -64,7 +69,7 @@ suite("create_table_use_partion_policy") {
CREATE STORAGE POLICY test_create_table_partition_use_policy_1
PROPERTIES(
"storage_resource" = "test_create_table_partition_use_resource_1",
"cooldown_datetime" = "2022-06-08 00:00:00"
"cooldown_datetime" = "$cooldownTime"
);
"""
assertEquals(storage_exist.call("test_create_table_partition_use_policy_1"), true)
@ -88,7 +93,7 @@ suite("create_table_use_partion_policy") {
CREATE STORAGE POLICY test_create_table_partition_use_policy_2
PROPERTIES(
"storage_resource" = "test_create_table_partition_use_resource_2",
"cooldown_datetime" = "2022-06-08 00:00:00"
"cooldown_datetime" = "$cooldownTime"
);
"""
assertEquals(storage_exist.call("test_create_table_partition_use_policy_2"), true)
@ -96,7 +101,7 @@ suite("create_table_use_partion_policy") {
// success
def create_table_partition_use_created_policy = try_sql """
CREATE TABLE IF NOT EXISTS create_table_partion_use_created_policy
CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy
(
k1 DATE,
k2 INT,
@ -110,11 +115,11 @@ suite("create_table_use_partion_policy") {
assertEquals(create_table_partition_use_created_policy.size(), 1);
sql """
DROP TABLE create_table_partion_use_created_policy
DROP TABLE create_table_partition_use_created_policy
"""
def create_table_partition_use_created_policy_1 = try_sql """
CREATE TABLE IF NOT EXISTS create_table_partion_use_created_policy_1
CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy_1
(
k1 DATEV2,
k2 INT,
@ -128,11 +133,11 @@ suite("create_table_use_partion_policy") {
assertEquals(create_table_partition_use_created_policy_1.size(), 1);
sql """
DROP TABLE create_table_partion_use_created_policy_1
DROP TABLE create_table_partition_use_created_policy_1
"""
def create_table_partition_use_created_policy_2 = try_sql """
CREATE TABLE IF NOT EXISTS create_table_partion_use_created_policy_2
CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy_2
(
k1 DATETIMEV2(3),
k2 INT,
@ -146,6 +151,6 @@ suite("create_table_use_partion_policy") {
assertEquals(create_table_partition_use_created_policy_2.size(), 1);
sql """
DROP TABLE create_table_partion_use_created_policy_2
DROP TABLE create_table_partition_use_created_policy_2
"""
}

58
regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy
View File

@ -15,11 +15,16 @@
// specific language governing permissions and limitations
// under the License.
// The cases is copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions
// and modified by Doris.
import java.text.SimpleDateFormat;
import java.util.Date;
suite("create_table_use_policy") {
sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");"""
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
Date date = new Date(System.currentTimeMillis() + 3600000)
def cooldownTime = format.format(date)
def create_table_use_not_create_policy = try_sql """
CREATE TABLE IF NOT EXISTS create_table_use_not_create_policy
(
@ -50,28 +55,29 @@ suite("create_table_use_policy") {
return false;
}
if (!storage_exist.call("test_create_table_use_policy")) {
def create_s3_resource = try_sql """
CREATE RESOURCE "test_create_table_use_resource"
PROPERTIES(
"type"="s3",
"s3_region" = "bj",
"s3_endpoint" = "http://bj.s3.comaaaa",
"s3_root_path" = "path/to/rootaaaa",
"s3_secret_key" = "aaaa",
"s3_access_key" = "bbba",
"s3_bucket" = "test-bucket"
);
"""
def create_succ_1 = try_sql """
CREATE STORAGE POLICY test_create_table_use_policy
PROPERTIES(
"storage_resource" = "test_create_table_use_resource",
"cooldown_datetime" = "2022-06-08 00:00:00"
);
"""
assertEquals(storage_exist.call("test_create_table_use_policy"), true)
}
def create_s3_resource = try_sql """
CREATE RESOURCE IF NOT EXISTS "test_create_table_use_resource"
PROPERTIES(
"type"="s3",
"s3_region" = "bj",
"s3_endpoint" = "http://bj.s3.comaaaa",
"s3_root_path" = "path/to/rootaaaa",
"s3_secret_key" = "aaaa",
"s3_access_key" = "bbba",
"s3_bucket" = "test-bucket"
);
"""
def create_succ_1 = try_sql """
CREATE STORAGE POLICY IF NOT EXISTS test_create_table_use_policy
PROPERTIES(
"storage_resource" = "test_create_table_use_resource",
"cooldown_datetime" = "$cooldownTime"
);
"""
sql """ALTER STORAGE POLICY test_create_table_use_policy PROPERTIES("cooldown_datetime" = "$cooldownTime")"""
assertEquals(storage_exist.call("test_create_table_use_policy"), true)
// success
def create_table_use_created_policy = try_sql """
@ -92,7 +98,7 @@ suite("create_table_use_policy") {
assertEquals(create_table_use_created_policy.size(), 1);
sql """
DROP TABLE create_table_use_created_policy
DROP TABLE IF EXISTS create_table_use_created_policy
"""
}

57
regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy
View File

@ -15,11 +15,17 @@
// specific language governing permissions and limitations
// under the License.
// The cases is copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions
// and modified by Doris.
import java.text.SimpleDateFormat;
import java.util.Date;
suite("add_table_policy_by_modify_partition") {
sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");"""
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
Date date = new Date(System.currentTimeMillis() + 3600000)
def cooldownTime = format.format(date)
sql """DROP TABLE IF EXISTS create_table_partition"""
def create_table_partition_not_have_policy_result = try_sql """
CREATE TABLE IF NOT EXISTS `create_table_partition` (
`lo_orderkey` bigint(20) NOT NULL COMMENT "",
@ -72,28 +78,27 @@ suite("add_table_policy_by_modify_partition") {
return false;
}
if (!storage_exist.call("created_create_table_partition_alter_policy")) {
def create_s3_resource = try_sql """
CREATE RESOURCE "test_modify_partition_table_use_resource"
PROPERTIES(
"type"="s3",
"s3_region" = "bj",
"s3_endpoint" = "http://bj.s3.comaaaa",
"s3_root_path" = "path/to/rootaaaa",
"s3_secret_key" = "aaaa",
"s3_access_key" = "bbba",
"s3_bucket" = "test-bucket"
);
"""
def create_succ_1 = try_sql """
CREATE STORAGE POLICY created_create_table_partition_alter_policy
PROPERTIES(
"storage_resource" = "test_modify_partition_table_use_resource",
"cooldown_datetime" = "2022-06-08 00:00:00"
);
"""
assertEquals(storage_exist.call("created_create_table_partition_alter_policy"), true)
}
def create_s3_resource = try_sql """
CREATE RESOURCE IF NOT EXISTS "test_modify_partition_table_use_resource"
PROPERTIES(
"type"="s3",
"s3_region" = "bj",
"s3_endpoint" = "http://bj.s3.comaaaa",
"s3_root_path" = "path/to/rootaaaa",
"s3_secret_key" = "aaaa",
"s3_access_key" = "bbba",
"s3_bucket" = "test-bucket"
);
"""
def create_succ_1 = try_sql """
CREATE STORAGE POLICY IF NOT EXISTS created_create_table_partition_alter_policy
PROPERTIES(
"storage_resource" = "test_modify_partition_table_use_resource",
"cooldown_datetime" = "$cooldownTime"
);
"""
sql """ALTER STORAGE POLICY created_create_table_partition_alter_policy PROPERTIES("cooldown_datetime" = "$cooldownTime")"""
assertEquals(storage_exist.call("created_create_table_partition_alter_policy"), true)
def alter_table_partition_try_again_result = try_sql """
ALTER TABLE create_table_partition MODIFY PARTITION (*) SET("storage_policy"="created_create_table_partition_alter_policy");
@ -108,6 +113,6 @@ suite("add_table_policy_by_modify_partition") {
assertEquals(alter_table_when_table_partition_has_storage_policy_result, null);
sql """
DROP TABLE create_table_partition;
DROP TABLE IF EXISTS create_table_partition;
"""
}

6
regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy
View File

@ -15,11 +15,9 @@
// specific language governing permissions and limitations
// under the License.
// The cases is copied from https://github.com/trinodb/trino/tree/master
// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions
// and modified by Doris.
suite("use_default_storage_policy") {
sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");"""
def storage_exist = { name ->
def show_storage_policy = sql """
SHOW STORAGE POLICY;