diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 552ff2decb..7d0e5833fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -231,11 +231,24 @@ public class DynamicPartitionUtil { Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null); } - private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, Database db) throws DdlException { + private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, int hotPartitionNum, + Database db) throws DdlException { if (replicaAlloc.getTotalReplicaNum() <= 0) { ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_ZERO); } + Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null); + if (hotPartitionNum <= 0) { + return; + } + + try { + Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, TStorageMedium.SSD); + } catch (DdlException e) { + throw new DdlException("Failed to find enough backend for ssd storage medium. When setting " + + DynamicPartitionProperty.HOT_PARTITION_NUM + " > 0, the hot partitions will store " + + "in ssd. Please check the replication num,replication tag and storage medium."); + } } private static void checkHotPartitionNum(String val) throws DdlException { @@ -600,28 +613,33 @@ public class DynamicPartitionUtil { analyzedProperties.put(DynamicPartitionProperty.TIME_ZONE, val); } + int hotPartitionNum = 0; + if (properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) { + String val = properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM); + checkHotPartitionNum(val); + hotPartitionNum = Integer.parseInt(val); + properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM); + analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val); + } + // check replication_allocation first, then replciation_num + ReplicaAllocation replicaAlloc = null; if (properties.containsKey(DynamicPartitionProperty.REPLICATION_ALLOCATION)) { - ReplicaAllocation replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition"); - checkReplicaAllocation(replicaAlloc, db); + replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition"); properties.remove(DynamicPartitionProperty.REPLICATION_ALLOCATION); analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION, replicaAlloc.toCreateStmt()); } else if (properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM)) { String val = properties.get(DynamicPartitionProperty.REPLICATION_NUM); checkReplicationNum(val, db); properties.remove(DynamicPartitionProperty.REPLICATION_NUM); + replicaAlloc = new ReplicaAllocation(Short.valueOf(val)); analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION, - new ReplicaAllocation(Short.valueOf(val)).toCreateStmt()); + replicaAlloc.toCreateStmt()); } else { - checkReplicaAllocation(olapTable.getDefaultReplicaAllocation(), db); + replicaAlloc = olapTable.getDefaultReplicaAllocation(); } + checkReplicaAllocation(replicaAlloc, hotPartitionNum, db); - if (properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) { - String val = properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM); - checkHotPartitionNum(val); - properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM); - analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val); - } if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS)) { String reservedHistoryPeriods = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS); checkReservedHistoryPeriodValidate(reservedHistoryPeriods, diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 3609e56592..22c19fb159 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -28,6 +28,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.utframe.UtFrameUtils; @@ -82,6 +83,15 @@ public class DynamicPartitionTableTest { UtFrameUtils.cleanDorisFeDir(runningDir); } + private static void changeBeDisk(TStorageMedium storageMedium) { + List backends = Env.getCurrentSystemInfo().getAllBackends(); + for (Backend be : backends) { + for (DiskInfo diskInfo : be.getDisks().values()) { + diskInfo.setStorageMedium(storageMedium); + } + } + } + private static void createTable(String sql) throws Exception { CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); Env.getCurrentEnv().createTable(createTableStmt); @@ -970,6 +980,8 @@ public class DynamicPartitionTableTest { @Test public void testHotPartitionNum() throws Exception { + changeBeDisk(TStorageMedium.SSD); + Database testDb = Env.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test"); // 1. hour @@ -1188,9 +1200,11 @@ public class DynamicPartitionTableTest { } @Test(expected = DdlException.class) - public void testHotPartitionNumAbnormal() throws Exception { + public void testHotPartitionNumAbnormalLT0() throws Exception { + changeBeDisk(TStorageMedium.SSD); + // dynamic_partition.hot_partition_num must larger than 0. - String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1` (\n" + String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1_lt0` (\n" + " `k1` datetime NULL COMMENT \"\",\n" + " `k2` int NULL COMMENT \"\"\n" + ") ENGINE=OLAP\n" @@ -1211,6 +1225,32 @@ public class DynamicPartitionTableTest { createTable(createOlapTblStmt); } + @Test(expected = DdlException.class) + public void testHotPartitionNumAbnormalMissSSD() throws Exception { + changeBeDisk(TStorageMedium.HDD); + + // when dynamic_partition.hot_partition_num > 0, it require ssd storage medium. + String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1_miss_ssd` (\n" + + " `k1` datetime NULL COMMENT \"\",\n" + + " `k2` int NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "PARTITION BY RANGE(`k1`)\n" + + "()\n" + + "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"dynamic_partition.enable\" = \"true\",\n" + + "\"dynamic_partition.start\" = \"-3\",\n" + + "\"dynamic_partition.end\" = \"3\",\n" + + "\"dynamic_partition.create_history_partition\" = \"true\",\n" + + "\"dynamic_partition.time_unit\" = \"hour\",\n" + + "\"dynamic_partition.prefix\" = \"p\",\n" + + "\"dynamic_partition.buckets\" = \"1\",\n" + + "\"dynamic_partition.hot_partition_num\" = \"1\"\n" + + ");"; + createTable(createOlapTblStmt); + } + @Test public void testRuntimeInfo() throws Exception { DynamicPartitionScheduler scheduler = new DynamicPartitionScheduler("test", 10); diff --git a/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy b/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy index dfbea21562..3c195713ae 100644 --- a/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy +++ b/regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy @@ -218,7 +218,6 @@ suite("test_outer_join_with_with_window_function") { "dynamic_partition.buckets" = "4", "dynamic_partition.create_history_partition" = "true", "dynamic_partition.history_partition_num" = "50", - "dynamic_partition.hot_partition_num" = "2", "dynamic_partition.reserved_history_periods" = "NULL", "dynamic_partition.start_day_of_month" = "1", "in_memory" = "false", diff --git a/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy b/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy index 169e92c8ba..16abf0fb8a 100644 --- a/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy +++ b/regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy @@ -222,7 +222,6 @@ suite("test_outer_join_with_window_function_datev2") { "dynamic_partition.buckets" = "4", "dynamic_partition.create_history_partition" = "true", "dynamic_partition.history_partition_num" = "50", - "dynamic_partition.hot_partition_num" = "2", "dynamic_partition.reserved_history_periods" = "NULL", "dynamic_partition.start_day_of_month" = "1", "in_memory" = "false", diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy index 1a69dda4c7..f4680ff681 100644 --- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy +++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy @@ -149,4 +149,26 @@ suite("test_dynamic_partition") { exception "errCode = 2," } sql "drop table if exists dy_par_bad" + test { + sql """ + CREATE TABLE IF NOT EXISTS dy_par_bad + ( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL ) + AGGREGATE KEY(k1,k2) + PARTITION BY RANGE(k1) ( ) + DISTRIBUTED BY HASH(k1) BUCKETS 3 + PROPERTIES ( + "dynamic_partition.enable"="true", + "dynamic_partition.end"="3", + "dynamic_partition.start"="-3", + "dynamic_partition.prefix"="p", + "dynamic_partition.time_unit"="DAY", + "dynamic_partition.create_history_partition"="true", + "dynamic_partition.hot_partition_num" = "2", + "dynamic_partition.replication_allocation" = "tag.location.default: 1") + """ + // check exception message contains + // dynamic_partition.hot_partition_num > 0 will require ssd storage medium + exception "errCode = 2," + } + sql "drop table if exists dy_par_bad" }