From ca90253b09ddaa49e45f3d5a12d4709bc03e91ed Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Wed, 30 Nov 2022 10:04:33 +0800 Subject: [PATCH] [config](storage-policy) add a FE config to disable storage policy by default (#14655) the cold-hot separation feature is still under development. And seems there are some unsolved feature remains. So I add a fe config enable_storage_policy, and default is false, to disable the creation and usage of storage policy by default. So that user can aware that he is using an experimental feature on his own, and it will not be released formally in v1.2.0. Disable storage policy by default, user can not use or create storage policy. Configured by enable_storage_policy. Remove property remote_storage_policy, it is duplicate with storage_policy Change the persist field in DataProperty.java. And remove remoteCooldownTime from DataProperty, because it can be got from StoragePolicy. --- docs/en/docs/admin-manual/config/fe-config.md | 11 +++ .../docs/admin-manual/config/fe-config.md | 11 +++ .../doris/alter/SchemaChangeHandler.java | 5 +- .../doris/analysis/CreatePolicyStmt.java | 5 ++ .../analysis/ModifyTablePropertiesClause.java | 2 - .../apache/doris/catalog/DataProperty.java | 44 ++++------ .../catalog/DynamicPartitionProperty.java | 12 +-- .../java/org/apache/doris/catalog/Env.java | 6 -- .../org/apache/doris/catalog/OlapTable.java | 33 ++------ .../apache/doris/catalog/TableProperty.java | 18 ----- .../clone/DynamicPartitionScheduler.java | 4 +- .../java/org/apache/doris/common/Config.java | 7 ++ .../doris/common/proc/PartitionsProcDir.java | 2 +- .../common/util/DynamicPartitionUtil.java | 16 ++-- .../doris/common/util/PropertyAnalyzer.java | 81 ++++++------------- .../doris/datasource/InternalCatalog.java | 6 -- .../org/apache/doris/alter/AlterTest.java | 17 ++-- .../doris/catalog/DataPropertyTest.java | 2 +- .../org/apache/doris/policy/PolicyTest.java | 2 + regression-test/conf/regression-conf.groovy | 28 +++---- .../cold_heat_separation/policy/alter.groovy | 6 +- .../cold_heat_separation/policy/create.groovy | 5 +- .../cold_heat_separation/policy/drop.groovy | 6 +- .../cold_heat_separation/policy/show.groovy | 6 +- .../use_policy/alter_table_add_policy.groovy | 6 +- .../create_table_use_partition_policy.groovy | 35 ++++---- .../use_policy/create_table_use_policy.groovy | 58 +++++++------ .../modify_partition_add_policy.groovy | 57 +++++++------ .../use_default_storage_policy.groovy | 6 +- 29 files changed, 219 insertions(+), 278 deletions(-) diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index ab7215a561..339e4d836b 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -2347,3 +2347,14 @@ Default: 3 Is it possible to dynamically configure: true Is it a configuration item unique to the Master FE node: true + +### `enable_storage_policy` + +Whether to enable the Storage Policy feature. This feature allows users to separate hot and cold data. This feature is still under development. Recommended for test environments only. + +Default: false + +Is it possible to dynamically configure: true + +Is it a configuration item unique to the Master FE node: true + diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 05fd33e1d1..bf211caa1d 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2403,3 +2403,14 @@ hive partition 的最大缓存数量。 是否可以动态配置：true 是否为 Master FE 节点独有的配置项：true + +### `enable_storage_policy` + +是否开启 Storage Policy 功能。该功能用户冷热数据分离功能。该功能仍在开发中，不排除后续后功能修改或重构。仅建议测试环境使用。 + +默认值：false。即不开启 + +是否可以动态配置：true + +是否为 Master FE 节点独有的配置项：true + diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 834315752d..39395d0039 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1712,9 +1712,8 @@ public class SchemaChangeHandler extends AlterHandler { } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)) { Env.getCurrentEnv().modifyTableReplicaAllocation(db, olapTable, properties); return; - } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY)) { - olapTable.setRemoteStoragePolicy( - properties.get(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY)); + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)) { + olapTable.setStoragePolicy(properties.get(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)); return; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java index 98506c6066..178b7d4ed6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java @@ -18,6 +18,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; @@ -92,6 +93,10 @@ public class CreatePolicyStmt extends DdlStmt { super.analyze(analyzer); switch (type) { case STORAGE: + if (!Config.enable_storage_policy) { + throw new UserException("storage policy feature is disabled by default. " + + "Enable it by setting 'enable_storage_policy=true' in fe.conf"); + } break; case ROW: default: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java index 86ec09cbd4..528a024b2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java @@ -97,8 +97,6 @@ public class ModifyTablePropertiesClause extends AlterTableClause { } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY)) { this.needTableStable = false; this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; - } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY)) { - // do nothing, just check valid. } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_TABLET_TYPE)) { throw new AnalysisException("Alter tablet type not supported"); } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java index cc5035107a..0531d9d03a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java @@ -38,15 +38,12 @@ public class DataProperty implements Writable { ); public static final long MAX_COOLDOWN_TIME_MS = 253402271999000L; // 9999-12-31 23:59:59 - @SerializedName(value = "storageMedium") + @SerializedName(value = "storageMedium") private TStorageMedium storageMedium; - @SerializedName(value = "cooldownTimeMs") + @SerializedName(value = "cooldownTimeMs") private long cooldownTimeMs; - @SerializedName(value = "remoteStoragePolicy") - private String remoteStoragePolicy; - // cooldown time for remote storage - @SerializedName(value = "remoteCooldownTimeMs") - private long remoteCooldownTimeMs; + @SerializedName(value = "storagePolicy") + private String storagePolicy; private DataProperty() { // for persist @@ -60,8 +57,7 @@ public class DataProperty implements Writable { } else { this.cooldownTimeMs = MAX_COOLDOWN_TIME_MS; } - this.remoteStoragePolicy = ""; - this.remoteCooldownTimeMs = MAX_COOLDOWN_TIME_MS; + this.storagePolicy = ""; } /** @@ -69,18 +65,12 @@ public class DataProperty implements Writable { * * @param medium storage medium for the init storage of the table * @param cooldown cool down time for SSD->HDD - * @param remoteStoragePolicy remote storage policy for remote storage - * @param remoteCooldownTimeMs remote storage cooldown time + * @param storagePolicy remote storage policy for remote storage */ - public DataProperty(TStorageMedium medium, long cooldown, String remoteStoragePolicy, long remoteCooldownTimeMs) { + public DataProperty(TStorageMedium medium, long cooldown, String storagePolicy) { this.storageMedium = medium; this.cooldownTimeMs = cooldown; - this.remoteStoragePolicy = remoteStoragePolicy; - if (remoteCooldownTimeMs > 0) { - this.remoteCooldownTimeMs = remoteCooldownTimeMs; - } else { - this.remoteCooldownTimeMs = MAX_COOLDOWN_TIME_MS; - } + this.storagePolicy = storagePolicy; } public TStorageMedium getStorageMedium() { @@ -91,12 +81,8 @@ public class DataProperty implements Writable { return cooldownTimeMs; } - public String getRemoteStoragePolicy() { - return remoteStoragePolicy; - } - - public long getRemoteCooldownTimeMs() { - return remoteCooldownTimeMs; + public String getStoragePolicy() { + return storagePolicy; } public static DataProperty read(DataInput in) throws IOException { @@ -118,12 +104,12 @@ public class DataProperty implements Writable { public void readFields(DataInput in) throws IOException { storageMedium = TStorageMedium.valueOf(Text.readString(in)); cooldownTimeMs = in.readLong(); - remoteStoragePolicy = ""; + storagePolicy = ""; } @Override public int hashCode() { - return Objects.hash(storageMedium, cooldownTimeMs, remoteStoragePolicy, remoteCooldownTimeMs); + return Objects.hash(storageMedium, cooldownTimeMs, storagePolicy); } @Override @@ -140,8 +126,7 @@ public class DataProperty implements Writable { return this.storageMedium == other.storageMedium && this.cooldownTimeMs == other.cooldownTimeMs - && this.remoteStoragePolicy.equals(other.remoteStoragePolicy) - && this.remoteCooldownTimeMs == other.remoteCooldownTimeMs; + && this.storagePolicy.equals(other.storagePolicy); } @Override @@ -149,8 +134,7 @@ public class DataProperty implements Writable { StringBuilder sb = new StringBuilder(); sb.append("Storage medium[").append(this.storageMedium).append("]. "); sb.append("cool down[").append(TimeUtils.longToTimeString(cooldownTimeMs)).append("]. "); - sb.append("remote storage policy[").append(this.remoteStoragePolicy).append("]. "); - sb.append("remote cooldown time[").append(TimeUtils.longToTimeString(remoteCooldownTimeMs)).append("]. "); + sb.append("remote storage policy[").append(this.storagePolicy).append("]. "); return sb.toString(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index 5b76d43927..88446394e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -46,7 +46,7 @@ public class DynamicPartitionProperty { public static final String HISTORY_PARTITION_NUM = "dynamic_partition.history_partition_num"; public static final String HOT_PARTITION_NUM = "dynamic_partition.hot_partition_num"; public static final String RESERVED_HISTORY_PERIODS = "dynamic_partition.reserved_history_periods"; - public static final String REMOTE_STORAGE_POLICY = "dynamic_partition.remote_storage_policy"; + public static final String STORAGE_POLICY = "dynamic_partition.storage_policy"; public static final int MIN_START_OFFSET = Integer.MIN_VALUE; public static final int MAX_END_OFFSET = Integer.MAX_VALUE; @@ -73,7 +73,7 @@ public class DynamicPartitionProperty { // If not set, default is 0 private int hotPartitionNum; private String reservedHistoryPeriods; - private String remoteStoragePolicy; + private String storagePolicy; public DynamicPartitionProperty(Map properties) { if (properties != null && !properties.isEmpty()) { @@ -93,7 +93,7 @@ public class DynamicPartitionProperty { this.hotPartitionNum = Integer.parseInt(properties.getOrDefault(HOT_PARTITION_NUM, "0")); this.reservedHistoryPeriods = properties.getOrDefault( RESERVED_HISTORY_PERIODS, NOT_SET_RESERVED_HISTORY_PERIODS); - this.remoteStoragePolicy = properties.getOrDefault(REMOTE_STORAGE_POLICY, ""); + this.storagePolicy = properties.getOrDefault(STORAGE_POLICY, ""); createStartOfs(properties); } else { this.exist = false; @@ -173,8 +173,8 @@ public class DynamicPartitionProperty { return hotPartitionNum; } - public String getRemoteStoragePolicy() { - return remoteStoragePolicy; + public String getStoragePolicy() { + return storagePolicy; } public String getStartOfInfo() { @@ -220,7 +220,7 @@ public class DynamicPartitionProperty { + ",\n\"" + HISTORY_PARTITION_NUM + "\" = \"" + historyPartitionNum + "\"" + ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum + "\"" + ",\n\"" + RESERVED_HISTORY_PERIODS + "\" = \"" + reservedHistoryPeriods + "\"" - + ",\n\"" + REMOTE_STORAGE_POLICY + "\" = \"" + remoteStoragePolicy + "\""; + + ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\""; if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) { res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\""; } else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index cd83b98dfa..df08131e42 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -2971,12 +2971,6 @@ public class Env { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT).append("\" = \""); sb.append(olapTable.getStorageFormat()).append("\""); - // remote storage - String remoteStoragePolicy = olapTable.getRemoteStoragePolicy(); - if (!Strings.isNullOrEmpty(remoteStoragePolicy)) { - sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY).append("\" = \""); - sb.append(remoteStoragePolicy).append("\""); - } // compression type if (olapTable.getCompressionType() != TCompressionType.LZ4F) { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_COMPRESSION).append("\" = \""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 0af7cb87e8..91cd500ddf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -36,6 +36,7 @@ import org.apache.doris.catalog.Tablet.TabletStatus; import org.apache.doris.clone.TabletSchedCtx; import org.apache.doris.clone.TabletScheduler; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; @@ -68,6 +69,7 @@ import com.google.common.collect.Sets; import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.parquet.Strings; import java.io.DataInput; import java.io.DataOutput; @@ -1627,7 +1629,11 @@ public class OlapTable extends Table { tableProperty.buildEnableLightSchemaChange(); } - public void setStoragePolicy(String storagePolicy) { + public void setStoragePolicy(String storagePolicy) throws UserException { + if (!Config.enable_storage_policy && !Strings.isNullOrEmpty(storagePolicy)) { + throw new UserException("storage policy feature is disabled by default. " + + "Enable it by setting 'enable_storage_policy=true' in fe.conf"); + } if (tableProperty == null) { tableProperty = new TableProperty(new HashMap<>()); } @@ -1666,19 +1672,6 @@ public class OlapTable extends Table { tableProperty.buildDataSortInfo(); } - /** - * set remote storage policy for table. - * - * @param remoteStoragePolicy remote storage policy name - */ - public void setRemoteStoragePolicy(String remoteStoragePolicy) { - if (tableProperty == null) { - tableProperty = new TableProperty(new HashMap<>()); - } - tableProperty.setRemoteStoragePolicy(remoteStoragePolicy); - tableProperty.buildRemoteStoragePolicy(); - } - // return true if partition with given name already exist, both in partitions and temp partitions. // return false otherwise public boolean checkPartitionNameExist(String partitionName) { @@ -1846,18 +1839,6 @@ public class OlapTable extends Table { return tableProperty.getDataSortInfo(); } - /** - * get remote storage policy name. - * - * @return remote storage policy name for this table. - */ - public String getRemoteStoragePolicy() { - if (tableProperty == null) { - return ""; - } - return tableProperty.getRemoteStoragePolicy(); - } - public void setEnableUniqueKeyMergeOnWrite(boolean speedup) { if (tableProperty == null) { tableProperty = new TableProperty(new HashMap<>()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index cc426e70aa..860d404cd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -79,9 +79,6 @@ public class TableProperty implements Writable { private DataSortInfo dataSortInfo = new DataSortInfo(); - // remote storage policy, for cold data - private String remoteStoragePolicy; - public TableProperty(Map properties) { this.properties = properties; } @@ -200,11 +197,6 @@ public class TableProperty implements Writable { return this; } - public TableProperty buildRemoteStoragePolicy() { - remoteStoragePolicy = properties.getOrDefault(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY, ""); - return this; - } - public void modifyTableProperties(Map modifyProperties) { properties.putAll(modifyProperties); removeDuplicateReplicaNumProperty(); @@ -222,11 +214,6 @@ public class TableProperty implements Writable { replicaAlloc.toCreateStmt()); } - public void setRemoteStoragePolicy(String remotePolicyName) { - this.remoteStoragePolicy = remotePolicyName; - properties.put(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY, remotePolicyName); - } - public ReplicaAllocation getReplicaAllocation() { return replicaAlloc; } @@ -269,10 +256,6 @@ public class TableProperty implements Writable { return dataSortInfo; } - public String getRemoteStoragePolicy() { - return remoteStoragePolicy; - } - public TCompressionType getCompressionType() { return compressionType; } @@ -324,7 +307,6 @@ public class TableProperty implements Writable { .buildInMemory() .buildStorageFormat() .buildDataSortInfo() - .buildRemoteStoragePolicy() .buildCompressionType() .buildStoragePolicy() .buildEnableLightSchemaChange(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 9b21c9479c..2d8408cc06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -162,7 +162,7 @@ public class DynamicPartitionScheduler extends MasterDaemon { idx = 0; } int hotPartitionNum = dynamicPartitionProperty.getHotPartitionNum(); - String storagePolicyName = dynamicPartitionProperty.getRemoteStoragePolicy(); + String storagePolicyName = dynamicPartitionProperty.getStoragePolicy(); for (; idx <= dynamicPartitionProperty.getEnd(); idx++) { String prevBorder = DynamicPartitionUtil.getPartitionRangeString( @@ -267,7 +267,7 @@ public class DynamicPartitionScheduler extends MasterDaemon { private void setStoragePolicyProperty(HashMap partitionProperties, DynamicPartitionProperty property, ZonedDateTime now, int offset, String storagePolicyName) { - partitionProperties.put(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY, storagePolicyName); + partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY, storagePolicyName); String baseTime = DynamicPartitionUtil.getPartitionRangeString( property, now, offset, DynamicPartitionUtil.DATETIME_FORMAT); partitionProperties.put(PropertyAnalyzer.PROPERTIES_DATA_BASE_TIME, baseTime); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index f170da6643..c424350946 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1918,5 +1918,12 @@ public class Config extends ConfigBase { */ @ConfField(mutable = true, masterOnly = true) public static int max_same_name_catalog_trash_num = 3; + + /** + * The storage policy is still under developement. + * Disable it by default. + */ + @ConfField(mutable = true, masterOnly = true) + public static boolean enable_storage_policy = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 4c350c0294..cb78140a26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -288,7 +288,7 @@ public class PartitionsProcDir implements ProcDirInterface { DataProperty dataProperty = tblPartitionInfo.getDataProperty(partitionId); partitionInfo.add(dataProperty.getStorageMedium().name()); partitionInfo.add(TimeUtils.longToTimeString(dataProperty.getCooldownTimeMs())); - partitionInfo.add(dataProperty.getRemoteStoragePolicy()); + partitionInfo.add(dataProperty.getStoragePolicy()); partitionInfo.add(TimeUtils.longToTimeString(partition.getLastCheckTime())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index e87b48d1a0..c0df533032 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -346,22 +346,22 @@ public class DynamicPartitionUtil { private static void checkRemoteStoragePolicy(String policyName) throws DdlException { if (Strings.isNullOrEmpty(policyName)) { - LOG.info(DynamicPartitionProperty.REMOTE_STORAGE_POLICY + " is null, remove this key"); + LOG.info(DynamicPartitionProperty.STORAGE_POLICY + " is null, remove this key"); return; } if (policyName.isEmpty()) { - throw new DdlException(DynamicPartitionProperty.REMOTE_STORAGE_POLICY + " is empty."); + throw new DdlException(DynamicPartitionProperty.STORAGE_POLICY + " is empty."); } StoragePolicy checkedPolicyCondition = StoragePolicy.ofCheck(policyName); if (!Env.getCurrentEnv().getPolicyMgr().existPolicy(checkedPolicyCondition)) { throw new DdlException( - DynamicPartitionProperty.REMOTE_STORAGE_POLICY + ": " + policyName + " doesn't exist."); + DynamicPartitionProperty.STORAGE_POLICY + ": " + policyName + " doesn't exist."); } StoragePolicy storagePolicy = (StoragePolicy) Env.getCurrentEnv().getPolicyMgr() .getPolicy(checkedPolicyCondition); if (Strings.isNullOrEmpty(storagePolicy.getCooldownTtl())) { throw new DdlException("Storage policy cooldown type need to be cooldownTtl for properties " - + DynamicPartitionProperty.REMOTE_STORAGE_POLICY + ": " + policyName); + + DynamicPartitionProperty.STORAGE_POLICY + ": " + policyName); } } @@ -616,12 +616,12 @@ public class DynamicPartitionUtil { properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS); analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriods); } - if (properties.containsKey(DynamicPartitionProperty.REMOTE_STORAGE_POLICY)) { - String remoteStoragePolicy = properties.get(DynamicPartitionProperty.REMOTE_STORAGE_POLICY); + if (properties.containsKey(DynamicPartitionProperty.STORAGE_POLICY)) { + String remoteStoragePolicy = properties.get(DynamicPartitionProperty.STORAGE_POLICY); checkRemoteStoragePolicy(remoteStoragePolicy); - properties.remove(DynamicPartitionProperty.REMOTE_STORAGE_POLICY); + properties.remove(DynamicPartitionProperty.STORAGE_POLICY); if (!Strings.isNullOrEmpty(remoteStoragePolicy)) { - analyzedProperties.put(DynamicPartitionProperty.REMOTE_STORAGE_POLICY, remoteStoragePolicy); + analyzedProperties.put(DynamicPartitionProperty.STORAGE_POLICY, remoteStoragePolicy); } } return analyzedProperties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index ca872a5b3e..034c7cea39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -46,7 +46,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -94,8 +93,6 @@ public class PropertyAnalyzer { public static final String PROPERTIES_INMEMORY = "in_memory"; - public static final String PROPERTIES_REMOTE_STORAGE_POLICY = "remote_storage_policy"; - public static final String PROPERTIES_TABLET_TYPE = "tablet_type"; public static final String PROPERTIES_STRICT_RANGE = "strict_range"; @@ -147,12 +144,10 @@ public class PropertyAnalyzer { } TStorageMedium storageMedium = oldDataProperty.getStorageMedium(); - long cooldownTimeStamp = oldDataProperty.getCooldownTimeMs(); - String remoteStoragePolicy = oldDataProperty.getRemoteStoragePolicy(); - long remoteCooldownTimeMs = oldDataProperty.getRemoteCooldownTimeMs(); + long cooldownTimestamp = oldDataProperty.getCooldownTimeMs(); + String newStoragePolicy = oldDataProperty.getStoragePolicy(); boolean hasStoragePolicy = false; - long dataBaseTimeMs = 0; for (Map.Entry entry : properties.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); @@ -166,74 +161,66 @@ public class PropertyAnalyzer { } } else if (key.equalsIgnoreCase(PROPERTIES_STORAGE_COOLDOWN_TIME)) { DateLiteral dateLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(Type.DATETIME)); - cooldownTimeStamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - } else if (key.equalsIgnoreCase(PROPERTIES_REMOTE_STORAGE_POLICY)) { - remoteStoragePolicy = value; - } else if (key.equalsIgnoreCase(PROPERTIES_DATA_BASE_TIME)) { - DateLiteral dateLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(Type.DATETIME)); - dataBaseTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); + cooldownTimestamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); } else if (!hasStoragePolicy && key.equalsIgnoreCase(PROPERTIES_STORAGE_POLICY)) { if (!Strings.isNullOrEmpty(value)) { hasStoragePolicy = true; + newStoragePolicy = value; } } } // end for properties properties.remove(PROPERTIES_STORAGE_MEDIUM); properties.remove(PROPERTIES_STORAGE_COOLDOWN_TIME); - properties.remove(PROPERTIES_REMOTE_STORAGE_POLICY); + properties.remove(PROPERTIES_STORAGE_POLICY); properties.remove(PROPERTIES_DATA_BASE_TIME); Preconditions.checkNotNull(storageMedium); if (storageMedium == TStorageMedium.HDD) { - cooldownTimeStamp = DataProperty.MAX_COOLDOWN_TIME_MS; + cooldownTimestamp = DataProperty.MAX_COOLDOWN_TIME_MS; LOG.info("Can not assign cool down timestamp to HDD storage medium, ignore user setting."); } - boolean hasCooldown = cooldownTimeStamp != DataProperty.MAX_COOLDOWN_TIME_MS; - boolean hasRemoteStoragePolicy = StringUtils.isNotEmpty(remoteStoragePolicy); - + boolean hasCooldown = cooldownTimestamp != DataProperty.MAX_COOLDOWN_TIME_MS; long currentTimeMs = System.currentTimeMillis(); if (storageMedium == TStorageMedium.SSD && hasCooldown) { - if (cooldownTimeStamp <= currentTimeMs) { - throw new AnalysisException("Cool down time should later than now"); + if (cooldownTimestamp <= currentTimeMs) { + throw new AnalysisException( + "Cool down time: " + cooldownTimestamp + " should later than now: " + currentTimeMs); } } if (storageMedium == TStorageMedium.SSD && !hasCooldown) { // set default cooldown time - cooldownTimeStamp = currentTimeMs + Config.storage_cooldown_second * 1000L; + cooldownTimestamp = currentTimeMs + Config.storage_cooldown_second * 1000L; } - if (hasRemoteStoragePolicy) { + if (hasStoragePolicy) { // check remote storage policy - StoragePolicy checkedPolicy = StoragePolicy.ofCheck(remoteStoragePolicy); + StoragePolicy checkedPolicy = StoragePolicy.ofCheck(newStoragePolicy); Policy policy = Env.getCurrentEnv().getPolicyMgr().getPolicy(checkedPolicy); if (!(policy instanceof StoragePolicy)) { - throw new AnalysisException("No PolicyStorage: " + remoteStoragePolicy); + throw new AnalysisException("No PolicyStorage: " + newStoragePolicy); } StoragePolicy storagePolicy = (StoragePolicy) policy; // check remote storage cool down timestamp if (storagePolicy.getCooldownTimestampMs() != -1) { if (storagePolicy.getCooldownTimestampMs() <= currentTimeMs) { - throw new AnalysisException("Remote storage cool down time should later than now"); - } - if (hasCooldown && storagePolicy.getCooldownTimestampMs() <= cooldownTimeStamp) { throw new AnalysisException( - "`remote_storage_cooldown_time`" + " should later than `storage_cooldown_time`."); + "remote storage cool down time: " + storagePolicy.getCooldownTimestampMs() + + " should later than now: " + currentTimeMs); + } + if (hasCooldown && storagePolicy.getCooldownTimestampMs() <= cooldownTimestamp) { + throw new AnalysisException( + "remote storage cool down time: " + storagePolicy.getCooldownTimestampMs() + + " should later than storage cool down time: " + cooldownTimestamp); } - remoteCooldownTimeMs = storagePolicy.getCooldownTimestampMs(); - } else if (storagePolicy.getCooldownTtl() != null && dataBaseTimeMs > 0) { - remoteCooldownTimeMs = dataBaseTimeMs + storagePolicy.getCooldownTtlMs(); } } - if (dataBaseTimeMs <= 0) { - remoteCooldownTimeMs = DataProperty.MAX_COOLDOWN_TIME_MS; - } - return new DataProperty(storageMedium, cooldownTimeStamp, remoteStoragePolicy, remoteCooldownTimeMs); + return new DataProperty(storageMedium, cooldownTimestamp, newStoragePolicy); } public static short analyzeShortKeyColumnCount(Map properties) throws AnalysisException { @@ -560,28 +547,6 @@ public class PropertyAnalyzer { return defaultVal; } - /** - * analyze remote storage policy. - * - * @param properties property for table - * @return remote storage policy name - * @throws AnalysisException policy name doesn't exist - */ - public static String analyzeRemoteStoragePolicy(Map properties) throws AnalysisException { - String remoteStoragePolicy = ""; - if (properties != null && properties.containsKey(PROPERTIES_REMOTE_STORAGE_POLICY)) { - remoteStoragePolicy = properties.get(PROPERTIES_REMOTE_STORAGE_POLICY); - // check remote storage policy existence - StoragePolicy checkedStoragePolicy = StoragePolicy.ofCheck(remoteStoragePolicy); - Policy policy = Env.getCurrentEnv().getPolicyMgr().getPolicy(checkedStoragePolicy); - if (!(policy instanceof StoragePolicy)) { - throw new AnalysisException("StoragePolicy: " + remoteStoragePolicy + " does not exist."); - } - } - - return remoteStoragePolicy; - } - public static String analyzeStoragePolicy(Map properties) throws AnalysisException { String storagePolicy = ""; if (properties != null && properties.containsKey(PROPERTIES_STORAGE_POLICY)) { @@ -833,3 +798,5 @@ public class PropertyAnalyzer { } } } + + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index f59fc34215..31b1893e7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1925,15 +1925,9 @@ public class InternalCatalog implements CatalogIf { false); olapTable.setIsInMemory(isInMemory); - // set remote storage - String remoteStoragePolicy = PropertyAnalyzer.analyzeRemoteStoragePolicy(properties); - olapTable.setRemoteStoragePolicy(remoteStoragePolicy); - // set storage policy String storagePolicy = PropertyAnalyzer.analyzeStoragePolicy(properties); - Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(storagePolicy); - olapTable.setStoragePolicy(storagePolicy); TTabletType tabletType; diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java index 76453f69e0..346f6a4ddd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java @@ -78,6 +78,7 @@ public class AlterTest { FeConstants.default_scheduler_interval_millisecond = 100; Config.dynamic_partition_check_interval_seconds = 1; Config.disable_storage_medium_check = true; + Config.enable_storage_policy = true; UtFrameUtils.createDorisCluster(runningDir); be = Env.getCurrentSystemInfo().getIdToBackend().values().asList().get(0); @@ -165,7 +166,7 @@ public class AlterTest { + " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n" + " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00',\n" - + " 'remote_storage_policy' = 'testPolicy'\n" + ");"); + + " 'storage_policy' = 'testPolicy'\n" + ");"); createTable("create table test.show_test (k1 int, k2 int) distributed by hash(k1) " + "buckets 1 properties(\"replication_num\" = \"1\");"); @@ -512,13 +513,13 @@ public class AlterTest { stmt = "alter table test.tbl4 modify partition (p3, p4) set ('storage_medium' = 'HDD')"; DateLiteral dateLiteral = new DateLiteral("2999-12-31 00:00:00", Type.DATETIME); long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "", -1); + DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, ""); partitionList = Lists.newArrayList(p3, p4); for (Partition partition : partitionList) { Assert.assertEquals(oldDataProperty, tbl4.getPartitionInfo().getDataProperty(partition.getId())); } alterTable(stmt, false); - DataProperty newDataProperty = new DataProperty(TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "", -1); + DataProperty newDataProperty = new DataProperty(TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, ""); for (Partition partition : partitionList) { Assert.assertEquals(newDataProperty, tbl4.getPartitionInfo().getDataProperty(partition.getId())); } @@ -531,7 +532,7 @@ public class AlterTest { dateLiteral = new DateLiteral("2100-12-31 00:00:00", Type.DATETIME); cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - DataProperty newDataProperty1 = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "", -1); + DataProperty newDataProperty1 = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, ""); partitionList = Lists.newArrayList(p1, p2); for (Partition partition : partitionList) { Assert.assertEquals(newDataProperty1, tbl4.getPartitionInfo().getDataProperty(partition.getId())); @@ -559,7 +560,7 @@ public class AlterTest { DateLiteral dateLiteral = new DateLiteral("2100-05-09 00:00:00", Type.DATETIME); long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy", -1); + DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy"); List partitionList = Lists.newArrayList(p2, p3, p4); for (Partition partition : partitionList) { Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(partition.getId())); @@ -570,7 +571,7 @@ public class AlterTest { alterTable(stmt, false); DateLiteral newDateLiteral = new DateLiteral("2100-04-01 22:22:22", Type.DATETIME); long newCooldownTimeMs = newDateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - DataProperty dataProperty2 = new DataProperty(TStorageMedium.SSD, newCooldownTimeMs, "testPolicy", -1); + DataProperty dataProperty2 = new DataProperty(TStorageMedium.SSD, newCooldownTimeMs, "testPolicy"); for (Partition partition : partitionList) { Assert.assertEquals(dataProperty2, tblRemote.getPartitionInfo().getDataProperty(partition.getId())); } @@ -580,14 +581,14 @@ public class AlterTest { stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('storage_medium' = 'HDD')"; alterTable(stmt, false); DataProperty dataProperty1 = new DataProperty( - TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "testPolicy", -1); + TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "testPolicy"); for (Partition partition : partitionList) { Assert.assertEquals(dataProperty1, tblRemote.getPartitionInfo().getDataProperty(partition.getId())); } Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(p1.getId())); // alter remote_storage - stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('remote_storage_policy' = 'testPolicy3')"; + stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('storage_policy' = 'testPolicy3')"; alterTable(stmt, true); Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(p1.getId())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java index 1ea2bd3d56..5114142569 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java @@ -35,7 +35,7 @@ public class DataPropertyTest { Assert.assertNotEquals(DataProperty.MAX_COOLDOWN_TIME_MS, dataProperty.getCooldownTimeMs()); long storageCooldownTimeMs = System.currentTimeMillis() + 24 * 3600 * 1000L; - dataProperty = new DataProperty(TStorageMedium.SSD, storageCooldownTimeMs, "", -1); + dataProperty = new DataProperty(TStorageMedium.SSD, storageCooldownTimeMs, ""); Assert.assertEquals(storageCooldownTimeMs, dataProperty.getCooldownTimeMs()); dataProperty = new DataProperty(TStorageMedium.HDD); diff --git a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java index e5734c32a2..e57953af61 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java @@ -27,6 +27,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AccessPrivilege; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; @@ -51,6 +52,7 @@ public class PolicyTest extends TestWithFeService { @Override protected void runBeforeAll() throws Exception { + Config.enable_storage_policy = true; FeConstants.runningUnitTest = true; createDatabase("test"); useDatabase("test"); diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 4c64fc4b72..645952c330 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -23,11 +23,11 @@ defaultDb = "regression_test" // add useLocalSessionState so that the jdbc will not send // init cmd like: select @@session.tx_read_only // at each time we connect. -jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true" +jdbcUrl = "jdbc:mysql://127.0.0.1:9033/?useLocalSessionState=true" jdbcUser = "root" jdbcPassword = "" -feHttpAddress = "127.0.0.1:8030" +feHttpAddress = "127.0.0.1:8033" feHttpUser = "root" feHttpPassword = "" @@ -66,26 +66,26 @@ brokerName = "broker_name" // broker load test config enableBrokerLoad=true -ak="" -sk="" +ak="AKIDJTO4VheMnzwd5HEvNZWHmiPr41jCPN5Y" +sk="GH2Xu2uXu4zf5GDvNaXFivvgBRSDjz1o" // jdbc connector test config // To enable jdbc test, you need first start mysql/pg container. // See `docker/thirdparties/start-thirdparties-docker.sh` -enableJdbcTest=false -mysql_57_port=3316 -pg_14_port=5442 +enableJdbcTest=true +mysql_57_port=3336 +pg_14_port=5444 // hive catalog test config // To enable jdbc test, you need first start hive container. // See `docker/thirdparties/start-thirdparties-docker.sh` -enableHiveTest=false -hms_port=9183 -hdfs_port=8120 +enableHiveTest=true +hms_port=9184 +hdfs_port=8121 // elasticsearch catalog test config // See `docker/thirdparties/start-thirdparties-docker.sh` -enableEsTest=false -es_6_port=19200 -es_7_port=29200 -es_8_port=39200 +enableEsTest=true +es_6_port=19222 +es_7_port=29222 +es_8_port=39222 diff --git a/regression-test/suites/cold_heat_separation/policy/alter.groovy b/regression-test/suites/cold_heat_separation/policy/alter.groovy index d453ad4b52..b35732f364 100644 --- a/regression-test/suites/cold_heat_separation/policy/alter.groovy +++ b/regression-test/suites/cold_heat_separation/policy/alter.groovy @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - suite("alter_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def has_resouce_policy_alter = sql """ SHOW RESOURCES WHERE NAME = "has_resouce_policy_alter"; """ diff --git a/regression-test/suites/cold_heat_separation/policy/create.groovy b/regression-test/suites/cold_heat_separation/policy/create.groovy index 9cfe9a9ddb..2b646a502e 100644 --- a/regression-test/suites/cold_heat_separation/policy/create.groovy +++ b/regression-test/suites/cold_heat_separation/policy/create.groovy @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. suite("create_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def has_created_1 = sql """ SHOW RESOURCES WHERE NAME = "crete_policy_1"; """ diff --git a/regression-test/suites/cold_heat_separation/policy/drop.groovy b/regression-test/suites/cold_heat_separation/policy/drop.groovy index a696df0754..d762103bef 100644 --- a/regression-test/suites/cold_heat_separation/policy/drop.groovy +++ b/regression-test/suites/cold_heat_separation/policy/drop.groovy @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - suite("drop_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def storage_exist = { name -> def show_storage_policy = sql """ SHOW STORAGE POLICY; diff --git a/regression-test/suites/cold_heat_separation/policy/show.groovy b/regression-test/suites/cold_heat_separation/policy/show.groovy index cf0aeef76c..c0c793e2ea 100644 --- a/regression-test/suites/cold_heat_separation/policy/show.groovy +++ b/regression-test/suites/cold_heat_separation/policy/show.groovy @@ -15,13 +15,11 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - import groovy.json.JsonSlurper suite("show_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def storage_exist = { name -> def show_storage_policy = sql """ SHOW STORAGE POLICY; diff --git a/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy index 6221e7e573..b710f59011 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - suite("add_table_policy_by_alter_table") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def create_table_not_have_policy_result = try_sql """ CREATE TABLE IF NOT EXISTS create_table_not_have_policy ( diff --git a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy index cb231b10bf..ad6fcfee44 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy @@ -15,13 +15,18 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. +import java.text.SimpleDateFormat; +import java.util.Date; -suite("create_table_use_partion_policy") { - def create_table_partion_use_not_create_policy = try_sql """ - CREATE TABLE IF NOT EXISTS create_table_partion_use_not_create_policy +suite("create_table_use_partition_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + Date date = new Date(System.currentTimeMillis() + 3600000) + def cooldownTime = format.format(date) + + def create_table_partition_use_not_create_policy = try_sql """ + CREATE TABLE IF NOT EXISTS create_table_partition_use_not_create_policy ( k1 DATE, k2 INT, @@ -33,7 +38,7 @@ suite("create_table_use_partion_policy") { """ // errCode = 2, detailMessage = Storage policy does not exist. name: not_exist_policy_1 - assertEquals(create_table_partion_use_not_create_policy, null) + assertEquals(create_table_partition_use_not_create_policy, null) def storage_exist = { name -> def show_storage_policy = sql """ @@ -64,7 +69,7 @@ suite("create_table_use_partion_policy") { CREATE STORAGE POLICY test_create_table_partition_use_policy_1 PROPERTIES( "storage_resource" = "test_create_table_partition_use_resource_1", - "cooldown_datetime" = "2022-06-08 00:00:00" + "cooldown_datetime" = "$cooldownTime" ); """ assertEquals(storage_exist.call("test_create_table_partition_use_policy_1"), true) @@ -88,7 +93,7 @@ suite("create_table_use_partion_policy") { CREATE STORAGE POLICY test_create_table_partition_use_policy_2 PROPERTIES( "storage_resource" = "test_create_table_partition_use_resource_2", - "cooldown_datetime" = "2022-06-08 00:00:00" + "cooldown_datetime" = "$cooldownTime" ); """ assertEquals(storage_exist.call("test_create_table_partition_use_policy_2"), true) @@ -96,7 +101,7 @@ suite("create_table_use_partion_policy") { // success def create_table_partition_use_created_policy = try_sql """ - CREATE TABLE IF NOT EXISTS create_table_partion_use_created_policy + CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy ( k1 DATE, k2 INT, @@ -110,11 +115,11 @@ suite("create_table_use_partion_policy") { assertEquals(create_table_partition_use_created_policy.size(), 1); sql """ - DROP TABLE create_table_partion_use_created_policy + DROP TABLE create_table_partition_use_created_policy """ def create_table_partition_use_created_policy_1 = try_sql """ - CREATE TABLE IF NOT EXISTS create_table_partion_use_created_policy_1 + CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy_1 ( k1 DATEV2, k2 INT, @@ -128,11 +133,11 @@ suite("create_table_use_partion_policy") { assertEquals(create_table_partition_use_created_policy_1.size(), 1); sql """ - DROP TABLE create_table_partion_use_created_policy_1 + DROP TABLE create_table_partition_use_created_policy_1 """ def create_table_partition_use_created_policy_2 = try_sql """ - CREATE TABLE IF NOT EXISTS create_table_partion_use_created_policy_2 + CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy_2 ( k1 DATETIMEV2(3), k2 INT, @@ -146,6 +151,6 @@ suite("create_table_use_partion_policy") { assertEquals(create_table_partition_use_created_policy_2.size(), 1); sql """ - DROP TABLE create_table_partion_use_created_policy_2 + DROP TABLE create_table_partition_use_created_policy_2 """ } diff --git a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy index f847e36611..da5c2a1592 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy @@ -15,11 +15,16 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. +import java.text.SimpleDateFormat; +import java.util.Date; suite("create_table_use_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + Date date = new Date(System.currentTimeMillis() + 3600000) + def cooldownTime = format.format(date) + def create_table_use_not_create_policy = try_sql """ CREATE TABLE IF NOT EXISTS create_table_use_not_create_policy ( @@ -50,28 +55,29 @@ suite("create_table_use_policy") { return false; } - if (!storage_exist.call("test_create_table_use_policy")) { - def create_s3_resource = try_sql """ - CREATE RESOURCE "test_create_table_use_resource" - PROPERTIES( - "type"="s3", - "s3_region" = "bj", - "s3_endpoint" = "http://bj.s3.comaaaa", - "s3_root_path" = "path/to/rootaaaa", - "s3_secret_key" = "aaaa", - "s3_access_key" = "bbba", - "s3_bucket" = "test-bucket" - ); - """ - def create_succ_1 = try_sql """ - CREATE STORAGE POLICY test_create_table_use_policy - PROPERTIES( - "storage_resource" = "test_create_table_use_resource", - "cooldown_datetime" = "2022-06-08 00:00:00" - ); - """ - assertEquals(storage_exist.call("test_create_table_use_policy"), true) - } + def create_s3_resource = try_sql """ + CREATE RESOURCE IF NOT EXISTS "test_create_table_use_resource" + PROPERTIES( + "type"="s3", + "s3_region" = "bj", + "s3_endpoint" = "http://bj.s3.comaaaa", + "s3_root_path" = "path/to/rootaaaa", + "s3_secret_key" = "aaaa", + "s3_access_key" = "bbba", + "s3_bucket" = "test-bucket" + ); + """ + def create_succ_1 = try_sql """ + CREATE STORAGE POLICY IF NOT EXISTS test_create_table_use_policy + PROPERTIES( + "storage_resource" = "test_create_table_use_resource", + "cooldown_datetime" = "$cooldownTime" + ); + """ + + sql """ALTER STORAGE POLICY test_create_table_use_policy PROPERTIES("cooldown_datetime" = "$cooldownTime")""" + + assertEquals(storage_exist.call("test_create_table_use_policy"), true) // success def create_table_use_created_policy = try_sql """ @@ -92,7 +98,7 @@ suite("create_table_use_policy") { assertEquals(create_table_use_created_policy.size(), 1); sql """ - DROP TABLE create_table_use_created_policy + DROP TABLE IF EXISTS create_table_use_created_policy """ } diff --git a/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy index eb6eccde89..afe5d2e0c8 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy @@ -15,11 +15,17 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. +import java.text.SimpleDateFormat; +import java.util.Date; suite("add_table_policy_by_modify_partition") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + Date date = new Date(System.currentTimeMillis() + 3600000) + def cooldownTime = format.format(date) + + sql """DROP TABLE IF EXISTS create_table_partition""" def create_table_partition_not_have_policy_result = try_sql """ CREATE TABLE IF NOT EXISTS `create_table_partition` ( `lo_orderkey` bigint(20) NOT NULL COMMENT "", @@ -72,28 +78,27 @@ suite("add_table_policy_by_modify_partition") { return false; } - if (!storage_exist.call("created_create_table_partition_alter_policy")) { - def create_s3_resource = try_sql """ - CREATE RESOURCE "test_modify_partition_table_use_resource" - PROPERTIES( - "type"="s3", - "s3_region" = "bj", - "s3_endpoint" = "http://bj.s3.comaaaa", - "s3_root_path" = "path/to/rootaaaa", - "s3_secret_key" = "aaaa", - "s3_access_key" = "bbba", - "s3_bucket" = "test-bucket" - ); - """ - def create_succ_1 = try_sql """ - CREATE STORAGE POLICY created_create_table_partition_alter_policy - PROPERTIES( - "storage_resource" = "test_modify_partition_table_use_resource", - "cooldown_datetime" = "2022-06-08 00:00:00" - ); - """ - assertEquals(storage_exist.call("created_create_table_partition_alter_policy"), true) - } + def create_s3_resource = try_sql """ + CREATE RESOURCE IF NOT EXISTS "test_modify_partition_table_use_resource" + PROPERTIES( + "type"="s3", + "s3_region" = "bj", + "s3_endpoint" = "http://bj.s3.comaaaa", + "s3_root_path" = "path/to/rootaaaa", + "s3_secret_key" = "aaaa", + "s3_access_key" = "bbba", + "s3_bucket" = "test-bucket" + ); + """ + def create_succ_1 = try_sql """ + CREATE STORAGE POLICY IF NOT EXISTS created_create_table_partition_alter_policy + PROPERTIES( + "storage_resource" = "test_modify_partition_table_use_resource", + "cooldown_datetime" = "$cooldownTime" + ); + """ + sql """ALTER STORAGE POLICY created_create_table_partition_alter_policy PROPERTIES("cooldown_datetime" = "$cooldownTime")""" + assertEquals(storage_exist.call("created_create_table_partition_alter_policy"), true) def alter_table_partition_try_again_result = try_sql """ ALTER TABLE create_table_partition MODIFY PARTITION (*) SET("storage_policy"="created_create_table_partition_alter_policy"); @@ -108,6 +113,6 @@ suite("add_table_policy_by_modify_partition") { assertEquals(alter_table_when_table_partition_has_storage_policy_result, null); sql """ - DROP TABLE create_table_partition; + DROP TABLE IF EXISTS create_table_partition; """ } diff --git a/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy index 4a7c022842..16428ed32e 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - suite("use_default_storage_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def storage_exist = { name -> def show_storage_policy = sql """ SHOW STORAGE POLICY;