diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index ab7215a561..339e4d836b 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -2347,3 +2347,14 @@ Default: 3 Is it possible to dynamically configure: true Is it a configuration item unique to the Master FE node: true + +### `enable_storage_policy` + +Whether to enable the Storage Policy feature. This feature allows users to separate hot and cold data. This feature is still under development. Recommended for test environments only. + +Default: false + +Is it possible to dynamically configure: true + +Is it a configuration item unique to the Master FE node: true + diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 05fd33e1d1..bf211caa1d 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2403,3 +2403,14 @@ hive partition 的最大缓存数量。 是否可以动态配置：true 是否为 Master FE 节点独有的配置项：true + +### `enable_storage_policy` + +是否开启 Storage Policy 功能。该功能用户冷热数据分离功能。该功能仍在开发中，不排除后续后功能修改或重构。仅建议测试环境使用。 + +默认值：false。即不开启 + +是否可以动态配置：true + +是否为 Master FE 节点独有的配置项：true + diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 834315752d..39395d0039 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1712,9 +1712,8 @@ public class SchemaChangeHandler extends AlterHandler { } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)) { Env.getCurrentEnv().modifyTableReplicaAllocation(db, olapTable, properties); return; - } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY)) { - olapTable.setRemoteStoragePolicy( - properties.get(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY)); + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)) { + olapTable.setStoragePolicy(properties.get(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)); return; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java index 98506c6066..178b7d4ed6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreatePolicyStmt.java @@ -18,6 +18,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; @@ -92,6 +93,10 @@ public class CreatePolicyStmt extends DdlStmt { super.analyze(analyzer); switch (type) { case STORAGE: + if (!Config.enable_storage_policy) { + throw new UserException("storage policy feature is disabled by default. " + + "Enable it by setting 'enable_storage_policy=true' in fe.conf"); + } break; case ROW: default: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java index 86ec09cbd4..528a024b2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java @@ -97,8 +97,6 @@ public class ModifyTablePropertiesClause extends AlterTableClause { } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY)) { this.needTableStable = false; this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; - } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY)) { - // do nothing, just check valid. } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_TABLET_TYPE)) { throw new AnalysisException("Alter tablet type not supported"); } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java index cc5035107a..0531d9d03a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java @@ -38,15 +38,12 @@ public class DataProperty implements Writable { ); public static final long MAX_COOLDOWN_TIME_MS = 253402271999000L; // 9999-12-31 23:59:59 - @SerializedName(value = "storageMedium") + @SerializedName(value = "storageMedium") private TStorageMedium storageMedium; - @SerializedName(value = "cooldownTimeMs") + @SerializedName(value = "cooldownTimeMs") private long cooldownTimeMs; - @SerializedName(value = "remoteStoragePolicy") - private String remoteStoragePolicy; - // cooldown time for remote storage - @SerializedName(value = "remoteCooldownTimeMs") - private long remoteCooldownTimeMs; + @SerializedName(value = "storagePolicy") + private String storagePolicy; private DataProperty() { // for persist @@ -60,8 +57,7 @@ public class DataProperty implements Writable { } else { this.cooldownTimeMs = MAX_COOLDOWN_TIME_MS; } - this.remoteStoragePolicy = ""; - this.remoteCooldownTimeMs = MAX_COOLDOWN_TIME_MS; + this.storagePolicy = ""; } /** @@ -69,18 +65,12 @@ public class DataProperty implements Writable { * * @param medium storage medium for the init storage of the table * @param cooldown cool down time for SSD->HDD - * @param remoteStoragePolicy remote storage policy for remote storage - * @param remoteCooldownTimeMs remote storage cooldown time + * @param storagePolicy remote storage policy for remote storage */ - public DataProperty(TStorageMedium medium, long cooldown, String remoteStoragePolicy, long remoteCooldownTimeMs) { + public DataProperty(TStorageMedium medium, long cooldown, String storagePolicy) { this.storageMedium = medium; this.cooldownTimeMs = cooldown; - this.remoteStoragePolicy = remoteStoragePolicy; - if (remoteCooldownTimeMs > 0) { - this.remoteCooldownTimeMs = remoteCooldownTimeMs; - } else { - this.remoteCooldownTimeMs = MAX_COOLDOWN_TIME_MS; - } + this.storagePolicy = storagePolicy; } public TStorageMedium getStorageMedium() { @@ -91,12 +81,8 @@ public class DataProperty implements Writable { return cooldownTimeMs; } - public String getRemoteStoragePolicy() { - return remoteStoragePolicy; - } - - public long getRemoteCooldownTimeMs() { - return remoteCooldownTimeMs; + public String getStoragePolicy() { + return storagePolicy; } public static DataProperty read(DataInput in) throws IOException { @@ -118,12 +104,12 @@ public class DataProperty implements Writable { public void readFields(DataInput in) throws IOException { storageMedium = TStorageMedium.valueOf(Text.readString(in)); cooldownTimeMs = in.readLong(); - remoteStoragePolicy = ""; + storagePolicy = ""; } @Override public int hashCode() { - return Objects.hash(storageMedium, cooldownTimeMs, remoteStoragePolicy, remoteCooldownTimeMs); + return Objects.hash(storageMedium, cooldownTimeMs, storagePolicy); } @Override @@ -140,8 +126,7 @@ public class DataProperty implements Writable { return this.storageMedium == other.storageMedium && this.cooldownTimeMs == other.cooldownTimeMs - && this.remoteStoragePolicy.equals(other.remoteStoragePolicy) - && this.remoteCooldownTimeMs == other.remoteCooldownTimeMs; + && this.storagePolicy.equals(other.storagePolicy); } @Override @@ -149,8 +134,7 @@ public class DataProperty implements Writable { StringBuilder sb = new StringBuilder(); sb.append("Storage medium[").append(this.storageMedium).append("]. "); sb.append("cool down[").append(TimeUtils.longToTimeString(cooldownTimeMs)).append("]. "); - sb.append("remote storage policy[").append(this.remoteStoragePolicy).append("]. "); - sb.append("remote cooldown time[").append(TimeUtils.longToTimeString(remoteCooldownTimeMs)).append("]. "); + sb.append("remote storage policy[").append(this.storagePolicy).append("]. "); return sb.toString(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java index 5b76d43927..88446394e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DynamicPartitionProperty.java @@ -46,7 +46,7 @@ public class DynamicPartitionProperty { public static final String HISTORY_PARTITION_NUM = "dynamic_partition.history_partition_num"; public static final String HOT_PARTITION_NUM = "dynamic_partition.hot_partition_num"; public static final String RESERVED_HISTORY_PERIODS = "dynamic_partition.reserved_history_periods"; - public static final String REMOTE_STORAGE_POLICY = "dynamic_partition.remote_storage_policy"; + public static final String STORAGE_POLICY = "dynamic_partition.storage_policy"; public static final int MIN_START_OFFSET = Integer.MIN_VALUE; public static final int MAX_END_OFFSET = Integer.MAX_VALUE; @@ -73,7 +73,7 @@ public class DynamicPartitionProperty { // If not set, default is 0 private int hotPartitionNum; private String reservedHistoryPeriods; - private String remoteStoragePolicy; + private String storagePolicy; public DynamicPartitionProperty(Map properties) { if (properties != null && !properties.isEmpty()) { @@ -93,7 +93,7 @@ public class DynamicPartitionProperty { this.hotPartitionNum = Integer.parseInt(properties.getOrDefault(HOT_PARTITION_NUM, "0")); this.reservedHistoryPeriods = properties.getOrDefault( RESERVED_HISTORY_PERIODS, NOT_SET_RESERVED_HISTORY_PERIODS); - this.remoteStoragePolicy = properties.getOrDefault(REMOTE_STORAGE_POLICY, ""); + this.storagePolicy = properties.getOrDefault(STORAGE_POLICY, ""); createStartOfs(properties); } else { this.exist = false; @@ -173,8 +173,8 @@ public class DynamicPartitionProperty { return hotPartitionNum; } - public String getRemoteStoragePolicy() { - return remoteStoragePolicy; + public String getStoragePolicy() { + return storagePolicy; } public String getStartOfInfo() { @@ -220,7 +220,7 @@ public class DynamicPartitionProperty { + ",\n\"" + HISTORY_PARTITION_NUM + "\" = \"" + historyPartitionNum + "\"" + ",\n\"" + HOT_PARTITION_NUM + "\" = \"" + hotPartitionNum + "\"" + ",\n\"" + RESERVED_HISTORY_PERIODS + "\" = \"" + reservedHistoryPeriods + "\"" - + ",\n\"" + REMOTE_STORAGE_POLICY + "\" = \"" + remoteStoragePolicy + "\""; + + ",\n\"" + STORAGE_POLICY + "\" = \"" + storagePolicy + "\""; if (getTimeUnit().equalsIgnoreCase(TimeUnit.WEEK.toString())) { res += ",\n\"" + START_DAY_OF_WEEK + "\" = \"" + startOfWeek.dayOfWeek + "\""; } else if (getTimeUnit().equalsIgnoreCase(TimeUnit.MONTH.toString())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index cd83b98dfa..df08131e42 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -2971,12 +2971,6 @@ public class Env { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT).append("\" = \""); sb.append(olapTable.getStorageFormat()).append("\""); - // remote storage - String remoteStoragePolicy = olapTable.getRemoteStoragePolicy(); - if (!Strings.isNullOrEmpty(remoteStoragePolicy)) { - sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY).append("\" = \""); - sb.append(remoteStoragePolicy).append("\""); - } // compression type if (olapTable.getCompressionType() != TCompressionType.LZ4F) { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_COMPRESSION).append("\" = \""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 0af7cb87e8..91cd500ddf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -36,6 +36,7 @@ import org.apache.doris.catalog.Tablet.TabletStatus; import org.apache.doris.clone.TabletSchedCtx; import org.apache.doris.clone.TabletScheduler; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; @@ -68,6 +69,7 @@ import com.google.common.collect.Sets; import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.parquet.Strings; import java.io.DataInput; import java.io.DataOutput; @@ -1627,7 +1629,11 @@ public class OlapTable extends Table { tableProperty.buildEnableLightSchemaChange(); } - public void setStoragePolicy(String storagePolicy) { + public void setStoragePolicy(String storagePolicy) throws UserException { + if (!Config.enable_storage_policy && !Strings.isNullOrEmpty(storagePolicy)) { + throw new UserException("storage policy feature is disabled by default. " + + "Enable it by setting 'enable_storage_policy=true' in fe.conf"); + } if (tableProperty == null) { tableProperty = new TableProperty(new HashMap<>()); } @@ -1666,19 +1672,6 @@ public class OlapTable extends Table { tableProperty.buildDataSortInfo(); } - /** - * set remote storage policy for table. - * - * @param remoteStoragePolicy remote storage policy name - */ - public void setRemoteStoragePolicy(String remoteStoragePolicy) { - if (tableProperty == null) { - tableProperty = new TableProperty(new HashMap<>()); - } - tableProperty.setRemoteStoragePolicy(remoteStoragePolicy); - tableProperty.buildRemoteStoragePolicy(); - } - // return true if partition with given name already exist, both in partitions and temp partitions. // return false otherwise public boolean checkPartitionNameExist(String partitionName) { @@ -1846,18 +1839,6 @@ public class OlapTable extends Table { return tableProperty.getDataSortInfo(); } - /** - * get remote storage policy name. - * - * @return remote storage policy name for this table. - */ - public String getRemoteStoragePolicy() { - if (tableProperty == null) { - return ""; - } - return tableProperty.getRemoteStoragePolicy(); - } - public void setEnableUniqueKeyMergeOnWrite(boolean speedup) { if (tableProperty == null) { tableProperty = new TableProperty(new HashMap<>()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index cc426e70aa..860d404cd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -79,9 +79,6 @@ public class TableProperty implements Writable { private DataSortInfo dataSortInfo = new DataSortInfo(); - // remote storage policy, for cold data - private String remoteStoragePolicy; - public TableProperty(Map properties) { this.properties = properties; } @@ -200,11 +197,6 @@ public class TableProperty implements Writable { return this; } - public TableProperty buildRemoteStoragePolicy() { - remoteStoragePolicy = properties.getOrDefault(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY, ""); - return this; - } - public void modifyTableProperties(Map modifyProperties) { properties.putAll(modifyProperties); removeDuplicateReplicaNumProperty(); @@ -222,11 +214,6 @@ public class TableProperty implements Writable { replicaAlloc.toCreateStmt()); } - public void setRemoteStoragePolicy(String remotePolicyName) { - this.remoteStoragePolicy = remotePolicyName; - properties.put(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY, remotePolicyName); - } - public ReplicaAllocation getReplicaAllocation() { return replicaAlloc; } @@ -269,10 +256,6 @@ public class TableProperty implements Writable { return dataSortInfo; } - public String getRemoteStoragePolicy() { - return remoteStoragePolicy; - } - public TCompressionType getCompressionType() { return compressionType; } @@ -324,7 +307,6 @@ public class TableProperty implements Writable { .buildInMemory() .buildStorageFormat() .buildDataSortInfo() - .buildRemoteStoragePolicy() .buildCompressionType() .buildStoragePolicy() .buildEnableLightSchemaChange(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 9b21c9479c..2d8408cc06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -162,7 +162,7 @@ public class DynamicPartitionScheduler extends MasterDaemon { idx = 0; } int hotPartitionNum = dynamicPartitionProperty.getHotPartitionNum(); - String storagePolicyName = dynamicPartitionProperty.getRemoteStoragePolicy(); + String storagePolicyName = dynamicPartitionProperty.getStoragePolicy(); for (; idx <= dynamicPartitionProperty.getEnd(); idx++) { String prevBorder = DynamicPartitionUtil.getPartitionRangeString( @@ -267,7 +267,7 @@ public class DynamicPartitionScheduler extends MasterDaemon { private void setStoragePolicyProperty(HashMap partitionProperties, DynamicPartitionProperty property, ZonedDateTime now, int offset, String storagePolicyName) { - partitionProperties.put(PropertyAnalyzer.PROPERTIES_REMOTE_STORAGE_POLICY, storagePolicyName); + partitionProperties.put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY, storagePolicyName); String baseTime = DynamicPartitionUtil.getPartitionRangeString( property, now, offset, DynamicPartitionUtil.DATETIME_FORMAT); partitionProperties.put(PropertyAnalyzer.PROPERTIES_DATA_BASE_TIME, baseTime); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index f170da6643..c424350946 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1918,5 +1918,12 @@ public class Config extends ConfigBase { */ @ConfField(mutable = true, masterOnly = true) public static int max_same_name_catalog_trash_num = 3; + + /** + * The storage policy is still under developement. + * Disable it by default. + */ + @ConfField(mutable = true, masterOnly = true) + public static boolean enable_storage_policy = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 4c350c0294..cb78140a26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -288,7 +288,7 @@ public class PartitionsProcDir implements ProcDirInterface { DataProperty dataProperty = tblPartitionInfo.getDataProperty(partitionId); partitionInfo.add(dataProperty.getStorageMedium().name()); partitionInfo.add(TimeUtils.longToTimeString(dataProperty.getCooldownTimeMs())); - partitionInfo.add(dataProperty.getRemoteStoragePolicy()); + partitionInfo.add(dataProperty.getStoragePolicy()); partitionInfo.add(TimeUtils.longToTimeString(partition.getLastCheckTime())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index e87b48d1a0..c0df533032 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -346,22 +346,22 @@ public class DynamicPartitionUtil { private static void checkRemoteStoragePolicy(String policyName) throws DdlException { if (Strings.isNullOrEmpty(policyName)) { - LOG.info(DynamicPartitionProperty.REMOTE_STORAGE_POLICY + " is null, remove this key"); + LOG.info(DynamicPartitionProperty.STORAGE_POLICY + " is null, remove this key"); return; } if (policyName.isEmpty()) { - throw new DdlException(DynamicPartitionProperty.REMOTE_STORAGE_POLICY + " is empty."); + throw new DdlException(DynamicPartitionProperty.STORAGE_POLICY + " is empty."); } StoragePolicy checkedPolicyCondition = StoragePolicy.ofCheck(policyName); if (!Env.getCurrentEnv().getPolicyMgr().existPolicy(checkedPolicyCondition)) { throw new DdlException( - DynamicPartitionProperty.REMOTE_STORAGE_POLICY + ": " + policyName + " doesn't exist."); + DynamicPartitionProperty.STORAGE_POLICY + ": " + policyName + " doesn't exist."); } StoragePolicy storagePolicy = (StoragePolicy) Env.getCurrentEnv().getPolicyMgr() .getPolicy(checkedPolicyCondition); if (Strings.isNullOrEmpty(storagePolicy.getCooldownTtl())) { throw new DdlException("Storage policy cooldown type need to be cooldownTtl for properties " - + DynamicPartitionProperty.REMOTE_STORAGE_POLICY + ": " + policyName); + + DynamicPartitionProperty.STORAGE_POLICY + ": " + policyName); } } @@ -616,12 +616,12 @@ public class DynamicPartitionUtil { properties.remove(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS); analyzedProperties.put(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS, reservedHistoryPeriods); } - if (properties.containsKey(DynamicPartitionProperty.REMOTE_STORAGE_POLICY)) { - String remoteStoragePolicy = properties.get(DynamicPartitionProperty.REMOTE_STORAGE_POLICY); + if (properties.containsKey(DynamicPartitionProperty.STORAGE_POLICY)) { + String remoteStoragePolicy = properties.get(DynamicPartitionProperty.STORAGE_POLICY); checkRemoteStoragePolicy(remoteStoragePolicy); - properties.remove(DynamicPartitionProperty.REMOTE_STORAGE_POLICY); + properties.remove(DynamicPartitionProperty.STORAGE_POLICY); if (!Strings.isNullOrEmpty(remoteStoragePolicy)) { - analyzedProperties.put(DynamicPartitionProperty.REMOTE_STORAGE_POLICY, remoteStoragePolicy); + analyzedProperties.put(DynamicPartitionProperty.STORAGE_POLICY, remoteStoragePolicy); } } return analyzedProperties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index ca872a5b3e..034c7cea39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -46,7 +46,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -94,8 +93,6 @@ public class PropertyAnalyzer { public static final String PROPERTIES_INMEMORY = "in_memory"; - public static final String PROPERTIES_REMOTE_STORAGE_POLICY = "remote_storage_policy"; - public static final String PROPERTIES_TABLET_TYPE = "tablet_type"; public static final String PROPERTIES_STRICT_RANGE = "strict_range"; @@ -147,12 +144,10 @@ public class PropertyAnalyzer { } TStorageMedium storageMedium = oldDataProperty.getStorageMedium(); - long cooldownTimeStamp = oldDataProperty.getCooldownTimeMs(); - String remoteStoragePolicy = oldDataProperty.getRemoteStoragePolicy(); - long remoteCooldownTimeMs = oldDataProperty.getRemoteCooldownTimeMs(); + long cooldownTimestamp = oldDataProperty.getCooldownTimeMs(); + String newStoragePolicy = oldDataProperty.getStoragePolicy(); boolean hasStoragePolicy = false; - long dataBaseTimeMs = 0; for (Map.Entry entry : properties.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); @@ -166,74 +161,66 @@ public class PropertyAnalyzer { } } else if (key.equalsIgnoreCase(PROPERTIES_STORAGE_COOLDOWN_TIME)) { DateLiteral dateLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(Type.DATETIME)); - cooldownTimeStamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - } else if (key.equalsIgnoreCase(PROPERTIES_REMOTE_STORAGE_POLICY)) { - remoteStoragePolicy = value; - } else if (key.equalsIgnoreCase(PROPERTIES_DATA_BASE_TIME)) { - DateLiteral dateLiteral = new DateLiteral(value, ScalarType.getDefaultDateType(Type.DATETIME)); - dataBaseTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); + cooldownTimestamp = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); } else if (!hasStoragePolicy && key.equalsIgnoreCase(PROPERTIES_STORAGE_POLICY)) { if (!Strings.isNullOrEmpty(value)) { hasStoragePolicy = true; + newStoragePolicy = value; } } } // end for properties properties.remove(PROPERTIES_STORAGE_MEDIUM); properties.remove(PROPERTIES_STORAGE_COOLDOWN_TIME); - properties.remove(PROPERTIES_REMOTE_STORAGE_POLICY); + properties.remove(PROPERTIES_STORAGE_POLICY); properties.remove(PROPERTIES_DATA_BASE_TIME); Preconditions.checkNotNull(storageMedium); if (storageMedium == TStorageMedium.HDD) { - cooldownTimeStamp = DataProperty.MAX_COOLDOWN_TIME_MS; + cooldownTimestamp = DataProperty.MAX_COOLDOWN_TIME_MS; LOG.info("Can not assign cool down timestamp to HDD storage medium, ignore user setting."); } - boolean hasCooldown = cooldownTimeStamp != DataProperty.MAX_COOLDOWN_TIME_MS; - boolean hasRemoteStoragePolicy = StringUtils.isNotEmpty(remoteStoragePolicy); - + boolean hasCooldown = cooldownTimestamp != DataProperty.MAX_COOLDOWN_TIME_MS; long currentTimeMs = System.currentTimeMillis(); if (storageMedium == TStorageMedium.SSD && hasCooldown) { - if (cooldownTimeStamp <= currentTimeMs) { - throw new AnalysisException("Cool down time should later than now"); + if (cooldownTimestamp <= currentTimeMs) { + throw new AnalysisException( + "Cool down time: " + cooldownTimestamp + " should later than now: " + currentTimeMs); } } if (storageMedium == TStorageMedium.SSD && !hasCooldown) { // set default cooldown time - cooldownTimeStamp = currentTimeMs + Config.storage_cooldown_second * 1000L; + cooldownTimestamp = currentTimeMs + Config.storage_cooldown_second * 1000L; } - if (hasRemoteStoragePolicy) { + if (hasStoragePolicy) { // check remote storage policy - StoragePolicy checkedPolicy = StoragePolicy.ofCheck(remoteStoragePolicy); + StoragePolicy checkedPolicy = StoragePolicy.ofCheck(newStoragePolicy); Policy policy = Env.getCurrentEnv().getPolicyMgr().getPolicy(checkedPolicy); if (!(policy instanceof StoragePolicy)) { - throw new AnalysisException("No PolicyStorage: " + remoteStoragePolicy); + throw new AnalysisException("No PolicyStorage: " + newStoragePolicy); } StoragePolicy storagePolicy = (StoragePolicy) policy; // check remote storage cool down timestamp if (storagePolicy.getCooldownTimestampMs() != -1) { if (storagePolicy.getCooldownTimestampMs() <= currentTimeMs) { - throw new AnalysisException("Remote storage cool down time should later than now"); - } - if (hasCooldown && storagePolicy.getCooldownTimestampMs() <= cooldownTimeStamp) { throw new AnalysisException( - "`remote_storage_cooldown_time`" + " should later than `storage_cooldown_time`."); + "remote storage cool down time: " + storagePolicy.getCooldownTimestampMs() + + " should later than now: " + currentTimeMs); + } + if (hasCooldown && storagePolicy.getCooldownTimestampMs() <= cooldownTimestamp) { + throw new AnalysisException( + "remote storage cool down time: " + storagePolicy.getCooldownTimestampMs() + + " should later than storage cool down time: " + cooldownTimestamp); } - remoteCooldownTimeMs = storagePolicy.getCooldownTimestampMs(); - } else if (storagePolicy.getCooldownTtl() != null && dataBaseTimeMs > 0) { - remoteCooldownTimeMs = dataBaseTimeMs + storagePolicy.getCooldownTtlMs(); } } - if (dataBaseTimeMs <= 0) { - remoteCooldownTimeMs = DataProperty.MAX_COOLDOWN_TIME_MS; - } - return new DataProperty(storageMedium, cooldownTimeStamp, remoteStoragePolicy, remoteCooldownTimeMs); + return new DataProperty(storageMedium, cooldownTimestamp, newStoragePolicy); } public static short analyzeShortKeyColumnCount(Map properties) throws AnalysisException { @@ -560,28 +547,6 @@ public class PropertyAnalyzer { return defaultVal; } - /** - * analyze remote storage policy. - * - * @param properties property for table - * @return remote storage policy name - * @throws AnalysisException policy name doesn't exist - */ - public static String analyzeRemoteStoragePolicy(Map properties) throws AnalysisException { - String remoteStoragePolicy = ""; - if (properties != null && properties.containsKey(PROPERTIES_REMOTE_STORAGE_POLICY)) { - remoteStoragePolicy = properties.get(PROPERTIES_REMOTE_STORAGE_POLICY); - // check remote storage policy existence - StoragePolicy checkedStoragePolicy = StoragePolicy.ofCheck(remoteStoragePolicy); - Policy policy = Env.getCurrentEnv().getPolicyMgr().getPolicy(checkedStoragePolicy); - if (!(policy instanceof StoragePolicy)) { - throw new AnalysisException("StoragePolicy: " + remoteStoragePolicy + " does not exist."); - } - } - - return remoteStoragePolicy; - } - public static String analyzeStoragePolicy(Map properties) throws AnalysisException { String storagePolicy = ""; if (properties != null && properties.containsKey(PROPERTIES_STORAGE_POLICY)) { @@ -833,3 +798,5 @@ public class PropertyAnalyzer { } } } + + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index f59fc34215..31b1893e7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1925,15 +1925,9 @@ public class InternalCatalog implements CatalogIf { false); olapTable.setIsInMemory(isInMemory); - // set remote storage - String remoteStoragePolicy = PropertyAnalyzer.analyzeRemoteStoragePolicy(properties); - olapTable.setRemoteStoragePolicy(remoteStoragePolicy); - // set storage policy String storagePolicy = PropertyAnalyzer.analyzeStoragePolicy(properties); - Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(storagePolicy); - olapTable.setStoragePolicy(storagePolicy); TTabletType tabletType; diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java index 76453f69e0..346f6a4ddd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java @@ -78,6 +78,7 @@ public class AlterTest { FeConstants.default_scheduler_interval_millisecond = 100; Config.dynamic_partition_check_interval_seconds = 1; Config.disable_storage_medium_check = true; + Config.enable_storage_policy = true; UtFrameUtils.createDorisCluster(runningDir); be = Env.getCurrentSystemInfo().getIdToBackend().values().asList().get(0); @@ -165,7 +166,7 @@ public class AlterTest { + " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n" + " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00',\n" - + " 'remote_storage_policy' = 'testPolicy'\n" + ");"); + + " 'storage_policy' = 'testPolicy'\n" + ");"); createTable("create table test.show_test (k1 int, k2 int) distributed by hash(k1) " + "buckets 1 properties(\"replication_num\" = \"1\");"); @@ -512,13 +513,13 @@ public class AlterTest { stmt = "alter table test.tbl4 modify partition (p3, p4) set ('storage_medium' = 'HDD')"; DateLiteral dateLiteral = new DateLiteral("2999-12-31 00:00:00", Type.DATETIME); long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "", -1); + DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, ""); partitionList = Lists.newArrayList(p3, p4); for (Partition partition : partitionList) { Assert.assertEquals(oldDataProperty, tbl4.getPartitionInfo().getDataProperty(partition.getId())); } alterTable(stmt, false); - DataProperty newDataProperty = new DataProperty(TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "", -1); + DataProperty newDataProperty = new DataProperty(TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, ""); for (Partition partition : partitionList) { Assert.assertEquals(newDataProperty, tbl4.getPartitionInfo().getDataProperty(partition.getId())); } @@ -531,7 +532,7 @@ public class AlterTest { dateLiteral = new DateLiteral("2100-12-31 00:00:00", Type.DATETIME); cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - DataProperty newDataProperty1 = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "", -1); + DataProperty newDataProperty1 = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, ""); partitionList = Lists.newArrayList(p1, p2); for (Partition partition : partitionList) { Assert.assertEquals(newDataProperty1, tbl4.getPartitionInfo().getDataProperty(partition.getId())); @@ -559,7 +560,7 @@ public class AlterTest { DateLiteral dateLiteral = new DateLiteral("2100-05-09 00:00:00", Type.DATETIME); long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy", -1); + DataProperty oldDataProperty = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy"); List partitionList = Lists.newArrayList(p2, p3, p4); for (Partition partition : partitionList) { Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(partition.getId())); @@ -570,7 +571,7 @@ public class AlterTest { alterTable(stmt, false); DateLiteral newDateLiteral = new DateLiteral("2100-04-01 22:22:22", Type.DATETIME); long newCooldownTimeMs = newDateLiteral.unixTimestamp(TimeUtils.getTimeZone()); - DataProperty dataProperty2 = new DataProperty(TStorageMedium.SSD, newCooldownTimeMs, "testPolicy", -1); + DataProperty dataProperty2 = new DataProperty(TStorageMedium.SSD, newCooldownTimeMs, "testPolicy"); for (Partition partition : partitionList) { Assert.assertEquals(dataProperty2, tblRemote.getPartitionInfo().getDataProperty(partition.getId())); } @@ -580,14 +581,14 @@ public class AlterTest { stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('storage_medium' = 'HDD')"; alterTable(stmt, false); DataProperty dataProperty1 = new DataProperty( - TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "testPolicy", -1); + TStorageMedium.HDD, DataProperty.MAX_COOLDOWN_TIME_MS, "testPolicy"); for (Partition partition : partitionList) { Assert.assertEquals(dataProperty1, tblRemote.getPartitionInfo().getDataProperty(partition.getId())); } Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(p1.getId())); // alter remote_storage - stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('remote_storage_policy' = 'testPolicy3')"; + stmt = "alter table test.tbl_remote modify partition (p2, p3, p4) set ('storage_policy' = 'testPolicy3')"; alterTable(stmt, true); Assert.assertEquals(oldDataProperty, tblRemote.getPartitionInfo().getDataProperty(p1.getId())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java index 1ea2bd3d56..5114142569 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataPropertyTest.java @@ -35,7 +35,7 @@ public class DataPropertyTest { Assert.assertNotEquals(DataProperty.MAX_COOLDOWN_TIME_MS, dataProperty.getCooldownTimeMs()); long storageCooldownTimeMs = System.currentTimeMillis() + 24 * 3600 * 1000L; - dataProperty = new DataProperty(TStorageMedium.SSD, storageCooldownTimeMs, "", -1); + dataProperty = new DataProperty(TStorageMedium.SSD, storageCooldownTimeMs, ""); Assert.assertEquals(storageCooldownTimeMs, dataProperty.getCooldownTimeMs()); dataProperty = new DataProperty(TStorageMedium.HDD); diff --git a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java index e5734c32a2..e57953af61 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java @@ -27,6 +27,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AccessPrivilege; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; @@ -51,6 +52,7 @@ public class PolicyTest extends TestWithFeService { @Override protected void runBeforeAll() throws Exception { + Config.enable_storage_policy = true; FeConstants.runningUnitTest = true; createDatabase("test"); useDatabase("test"); diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 4c64fc4b72..645952c330 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -23,11 +23,11 @@ defaultDb = "regression_test" // add useLocalSessionState so that the jdbc will not send // init cmd like: select @@session.tx_read_only // at each time we connect. -jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true" +jdbcUrl = "jdbc:mysql://127.0.0.1:9033/?useLocalSessionState=true" jdbcUser = "root" jdbcPassword = "" -feHttpAddress = "127.0.0.1:8030" +feHttpAddress = "127.0.0.1:8033" feHttpUser = "root" feHttpPassword = "" @@ -66,26 +66,26 @@ brokerName = "broker_name" // broker load test config enableBrokerLoad=true -ak="" -sk="" +ak="AKIDJTO4VheMnzwd5HEvNZWHmiPr41jCPN5Y" +sk="GH2Xu2uXu4zf5GDvNaXFivvgBRSDjz1o" // jdbc connector test config // To enable jdbc test, you need first start mysql/pg container. // See `docker/thirdparties/start-thirdparties-docker.sh` -enableJdbcTest=false -mysql_57_port=3316 -pg_14_port=5442 +enableJdbcTest=true +mysql_57_port=3336 +pg_14_port=5444 // hive catalog test config // To enable jdbc test, you need first start hive container. // See `docker/thirdparties/start-thirdparties-docker.sh` -enableHiveTest=false -hms_port=9183 -hdfs_port=8120 +enableHiveTest=true +hms_port=9184 +hdfs_port=8121 // elasticsearch catalog test config // See `docker/thirdparties/start-thirdparties-docker.sh` -enableEsTest=false -es_6_port=19200 -es_7_port=29200 -es_8_port=39200 +enableEsTest=true +es_6_port=19222 +es_7_port=29222 +es_8_port=39222 diff --git a/regression-test/suites/cold_heat_separation/policy/alter.groovy b/regression-test/suites/cold_heat_separation/policy/alter.groovy index d453ad4b52..b35732f364 100644 --- a/regression-test/suites/cold_heat_separation/policy/alter.groovy +++ b/regression-test/suites/cold_heat_separation/policy/alter.groovy @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - suite("alter_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def has_resouce_policy_alter = sql """ SHOW RESOURCES WHERE NAME = "has_resouce_policy_alter"; """ diff --git a/regression-test/suites/cold_heat_separation/policy/create.groovy b/regression-test/suites/cold_heat_separation/policy/create.groovy index 9cfe9a9ddb..2b646a502e 100644 --- a/regression-test/suites/cold_heat_separation/policy/create.groovy +++ b/regression-test/suites/cold_heat_separation/policy/create.groovy @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. suite("create_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def has_created_1 = sql """ SHOW RESOURCES WHERE NAME = "crete_policy_1"; """ diff --git a/regression-test/suites/cold_heat_separation/policy/drop.groovy b/regression-test/suites/cold_heat_separation/policy/drop.groovy index a696df0754..d762103bef 100644 --- a/regression-test/suites/cold_heat_separation/policy/drop.groovy +++ b/regression-test/suites/cold_heat_separation/policy/drop.groovy @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - suite("drop_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def storage_exist = { name -> def show_storage_policy = sql """ SHOW STORAGE POLICY; diff --git a/regression-test/suites/cold_heat_separation/policy/show.groovy b/regression-test/suites/cold_heat_separation/policy/show.groovy index cf0aeef76c..c0c793e2ea 100644 --- a/regression-test/suites/cold_heat_separation/policy/show.groovy +++ b/regression-test/suites/cold_heat_separation/policy/show.groovy @@ -15,13 +15,11 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - import groovy.json.JsonSlurper suite("show_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def storage_exist = { name -> def show_storage_policy = sql """ SHOW STORAGE POLICY; diff --git a/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy index 6221e7e573..b710f59011 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - suite("add_table_policy_by_alter_table") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def create_table_not_have_policy_result = try_sql """ CREATE TABLE IF NOT EXISTS create_table_not_have_policy ( diff --git a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy index cb231b10bf..ad6fcfee44 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy @@ -15,13 +15,18 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. +import java.text.SimpleDateFormat; +import java.util.Date; -suite("create_table_use_partion_policy") { - def create_table_partion_use_not_create_policy = try_sql """ - CREATE TABLE IF NOT EXISTS create_table_partion_use_not_create_policy +suite("create_table_use_partition_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + Date date = new Date(System.currentTimeMillis() + 3600000) + def cooldownTime = format.format(date) + + def create_table_partition_use_not_create_policy = try_sql """ + CREATE TABLE IF NOT EXISTS create_table_partition_use_not_create_policy ( k1 DATE, k2 INT, @@ -33,7 +38,7 @@ suite("create_table_use_partion_policy") { """ // errCode = 2, detailMessage = Storage policy does not exist. name: not_exist_policy_1 - assertEquals(create_table_partion_use_not_create_policy, null) + assertEquals(create_table_partition_use_not_create_policy, null) def storage_exist = { name -> def show_storage_policy = sql """ @@ -64,7 +69,7 @@ suite("create_table_use_partion_policy") { CREATE STORAGE POLICY test_create_table_partition_use_policy_1 PROPERTIES( "storage_resource" = "test_create_table_partition_use_resource_1", - "cooldown_datetime" = "2022-06-08 00:00:00" + "cooldown_datetime" = "$cooldownTime" ); """ assertEquals(storage_exist.call("test_create_table_partition_use_policy_1"), true) @@ -88,7 +93,7 @@ suite("create_table_use_partion_policy") { CREATE STORAGE POLICY test_create_table_partition_use_policy_2 PROPERTIES( "storage_resource" = "test_create_table_partition_use_resource_2", - "cooldown_datetime" = "2022-06-08 00:00:00" + "cooldown_datetime" = "$cooldownTime" ); """ assertEquals(storage_exist.call("test_create_table_partition_use_policy_2"), true) @@ -96,7 +101,7 @@ suite("create_table_use_partion_policy") { // success def create_table_partition_use_created_policy = try_sql """ - CREATE TABLE IF NOT EXISTS create_table_partion_use_created_policy + CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy ( k1 DATE, k2 INT, @@ -110,11 +115,11 @@ suite("create_table_use_partion_policy") { assertEquals(create_table_partition_use_created_policy.size(), 1); sql """ - DROP TABLE create_table_partion_use_created_policy + DROP TABLE create_table_partition_use_created_policy """ def create_table_partition_use_created_policy_1 = try_sql """ - CREATE TABLE IF NOT EXISTS create_table_partion_use_created_policy_1 + CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy_1 ( k1 DATEV2, k2 INT, @@ -128,11 +133,11 @@ suite("create_table_use_partion_policy") { assertEquals(create_table_partition_use_created_policy_1.size(), 1); sql """ - DROP TABLE create_table_partion_use_created_policy_1 + DROP TABLE create_table_partition_use_created_policy_1 """ def create_table_partition_use_created_policy_2 = try_sql """ - CREATE TABLE IF NOT EXISTS create_table_partion_use_created_policy_2 + CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy_2 ( k1 DATETIMEV2(3), k2 INT, @@ -146,6 +151,6 @@ suite("create_table_use_partion_policy") { assertEquals(create_table_partition_use_created_policy_2.size(), 1); sql """ - DROP TABLE create_table_partion_use_created_policy_2 + DROP TABLE create_table_partition_use_created_policy_2 """ } diff --git a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy index f847e36611..da5c2a1592 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy @@ -15,11 +15,16 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. +import java.text.SimpleDateFormat; +import java.util.Date; suite("create_table_use_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + Date date = new Date(System.currentTimeMillis() + 3600000) + def cooldownTime = format.format(date) + def create_table_use_not_create_policy = try_sql """ CREATE TABLE IF NOT EXISTS create_table_use_not_create_policy ( @@ -50,28 +55,29 @@ suite("create_table_use_policy") { return false; } - if (!storage_exist.call("test_create_table_use_policy")) { - def create_s3_resource = try_sql """ - CREATE RESOURCE "test_create_table_use_resource" - PROPERTIES( - "type"="s3", - "s3_region" = "bj", - "s3_endpoint" = "http://bj.s3.comaaaa", - "s3_root_path" = "path/to/rootaaaa", - "s3_secret_key" = "aaaa", - "s3_access_key" = "bbba", - "s3_bucket" = "test-bucket" - ); - """ - def create_succ_1 = try_sql """ - CREATE STORAGE POLICY test_create_table_use_policy - PROPERTIES( - "storage_resource" = "test_create_table_use_resource", - "cooldown_datetime" = "2022-06-08 00:00:00" - ); - """ - assertEquals(storage_exist.call("test_create_table_use_policy"), true) - } + def create_s3_resource = try_sql """ + CREATE RESOURCE IF NOT EXISTS "test_create_table_use_resource" + PROPERTIES( + "type"="s3", + "s3_region" = "bj", + "s3_endpoint" = "http://bj.s3.comaaaa", + "s3_root_path" = "path/to/rootaaaa", + "s3_secret_key" = "aaaa", + "s3_access_key" = "bbba", + "s3_bucket" = "test-bucket" + ); + """ + def create_succ_1 = try_sql """ + CREATE STORAGE POLICY IF NOT EXISTS test_create_table_use_policy + PROPERTIES( + "storage_resource" = "test_create_table_use_resource", + "cooldown_datetime" = "$cooldownTime" + ); + """ + + sql """ALTER STORAGE POLICY test_create_table_use_policy PROPERTIES("cooldown_datetime" = "$cooldownTime")""" + + assertEquals(storage_exist.call("test_create_table_use_policy"), true) // success def create_table_use_created_policy = try_sql """ @@ -92,7 +98,7 @@ suite("create_table_use_policy") { assertEquals(create_table_use_created_policy.size(), 1); sql """ - DROP TABLE create_table_use_created_policy + DROP TABLE IF EXISTS create_table_use_created_policy """ } diff --git a/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy index eb6eccde89..afe5d2e0c8 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy @@ -15,11 +15,17 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. +import java.text.SimpleDateFormat; +import java.util.Date; suite("add_table_policy_by_modify_partition") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + Date date = new Date(System.currentTimeMillis() + 3600000) + def cooldownTime = format.format(date) + + sql """DROP TABLE IF EXISTS create_table_partition""" def create_table_partition_not_have_policy_result = try_sql """ CREATE TABLE IF NOT EXISTS `create_table_partition` ( `lo_orderkey` bigint(20) NOT NULL COMMENT "", @@ -72,28 +78,27 @@ suite("add_table_policy_by_modify_partition") { return false; } - if (!storage_exist.call("created_create_table_partition_alter_policy")) { - def create_s3_resource = try_sql """ - CREATE RESOURCE "test_modify_partition_table_use_resource" - PROPERTIES( - "type"="s3", - "s3_region" = "bj", - "s3_endpoint" = "http://bj.s3.comaaaa", - "s3_root_path" = "path/to/rootaaaa", - "s3_secret_key" = "aaaa", - "s3_access_key" = "bbba", - "s3_bucket" = "test-bucket" - ); - """ - def create_succ_1 = try_sql """ - CREATE STORAGE POLICY created_create_table_partition_alter_policy - PROPERTIES( - "storage_resource" = "test_modify_partition_table_use_resource", - "cooldown_datetime" = "2022-06-08 00:00:00" - ); - """ - assertEquals(storage_exist.call("created_create_table_partition_alter_policy"), true) - } + def create_s3_resource = try_sql """ + CREATE RESOURCE IF NOT EXISTS "test_modify_partition_table_use_resource" + PROPERTIES( + "type"="s3", + "s3_region" = "bj", + "s3_endpoint" = "http://bj.s3.comaaaa", + "s3_root_path" = "path/to/rootaaaa", + "s3_secret_key" = "aaaa", + "s3_access_key" = "bbba", + "s3_bucket" = "test-bucket" + ); + """ + def create_succ_1 = try_sql """ + CREATE STORAGE POLICY IF NOT EXISTS created_create_table_partition_alter_policy + PROPERTIES( + "storage_resource" = "test_modify_partition_table_use_resource", + "cooldown_datetime" = "$cooldownTime" + ); + """ + sql """ALTER STORAGE POLICY created_create_table_partition_alter_policy PROPERTIES("cooldown_datetime" = "$cooldownTime")""" + assertEquals(storage_exist.call("created_create_table_partition_alter_policy"), true) def alter_table_partition_try_again_result = try_sql """ ALTER TABLE create_table_partition MODIFY PARTITION (*) SET("storage_policy"="created_create_table_partition_alter_policy"); @@ -108,6 +113,6 @@ suite("add_table_policy_by_modify_partition") { assertEquals(alter_table_when_table_partition_has_storage_policy_result, null); sql """ - DROP TABLE create_table_partition; + DROP TABLE IF EXISTS create_table_partition; """ } diff --git a/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy index 4a7c022842..16428ed32e 100644 --- a/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy +++ b/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. -// The cases is copied from https://github.com/trinodb/trino/tree/master -// /testing/trino-product-tests/src/main/resources/sql-tests/testcases/window_functions -// and modified by Doris. - suite("use_default_storage_policy") { + sql """ADMIN SET FRONTEND CONFIG ("enable_storage_policy" = "true");""" + def storage_exist = { name -> def show_storage_policy = sql """ SHOW STORAGE POLICY;