database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](dynamic partition) fix create hot partition failed without error response (#20996)

Browse Source
This commit is contained in:
yujun
2023-07-18 10:56:37 +08:00
committed by GitHub
parent e24867e138
commit ebd2a4b707
5 changed files with 93 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View File

@ -231,11 +231,24 @@ public class DynamicPartitionUtil {
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null);
}
private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, Database db) throws DdlException {
private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, int hotPartitionNum,
Database db) throws DdlException {
if (replicaAlloc.getTotalReplicaNum() <= 0) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_ZERO);
}
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null);
if (hotPartitionNum <= 0) {
return;
}
try {
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, TStorageMedium.SSD);
} catch (DdlException e) {
throw new DdlException("Failed to find enough backend for ssd storage medium. When setting "
+ DynamicPartitionProperty.HOT_PARTITION_NUM + " > 0, the hot partitions will store "
+ "in ssd. Please check the replication num,replication tag and storage medium.");
}
}
private static void checkHotPartitionNum(String val) throws DdlException {
@ -600,28 +613,33 @@ public class DynamicPartitionUtil {
analyzedProperties.put(DynamicPartitionProperty.TIME_ZONE, val);
}
int hotPartitionNum = 0;
if (properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) {
String val = properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM);
checkHotPartitionNum(val);
hotPartitionNum = Integer.parseInt(val);
properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM);
analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val);
}
// check replication_allocation first, then replciation_num
ReplicaAllocation replicaAlloc = null;
if (properties.containsKey(DynamicPartitionProperty.REPLICATION_ALLOCATION)) {
ReplicaAllocation replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition");
checkReplicaAllocation(replicaAlloc, db);
replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition");
properties.remove(DynamicPartitionProperty.REPLICATION_ALLOCATION);
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION, replicaAlloc.toCreateStmt());
} else if (properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM)) {
String val = properties.get(DynamicPartitionProperty.REPLICATION_NUM);
checkReplicationNum(val, db);
properties.remove(DynamicPartitionProperty.REPLICATION_NUM);
replicaAlloc = new ReplicaAllocation(Short.valueOf(val));
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
new ReplicaAllocation(Short.valueOf(val)).toCreateStmt());
replicaAlloc.toCreateStmt());
} else {
checkReplicaAllocation(olapTable.getDefaultReplicaAllocation(), db);
replicaAlloc = olapTable.getDefaultReplicaAllocation();
}
checkReplicaAllocation(replicaAlloc, hotPartitionNum, db);
if (properties.containsKey(DynamicPartitionProperty.HOT_PARTITION_NUM)) {
String val = properties.get(DynamicPartitionProperty.HOT_PARTITION_NUM);
checkHotPartitionNum(val);
properties.remove(DynamicPartitionProperty.HOT_PARTITION_NUM);
analyzedProperties.put(DynamicPartitionProperty.HOT_PARTITION_NUM, val);
}
if (properties.containsKey(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS)) {
String reservedHistoryPeriods = properties.get(DynamicPartitionProperty.RESERVED_HISTORY_PERIODS);
checkReservedHistoryPeriodValidate(reservedHistoryPeriods,

44
fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
View File

@ -28,6 +28,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;
@ -82,6 +83,15 @@ public class DynamicPartitionTableTest {
UtFrameUtils.cleanDorisFeDir(runningDir);
}
private static void changeBeDisk(TStorageMedium storageMedium) {
List<Backend> backends = Env.getCurrentSystemInfo().getAllBackends();
for (Backend be : backends) {
for (DiskInfo diskInfo : be.getDisks().values()) {
diskInfo.setStorageMedium(storageMedium);
}
}
}
private static void createTable(String sql) throws Exception {
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Env.getCurrentEnv().createTable(createTableStmt);
@ -970,6 +980,8 @@ public class DynamicPartitionTableTest {
@Test
public void testHotPartitionNum() throws Exception {
changeBeDisk(TStorageMedium.SSD);
Database testDb =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test");
// 1. hour
@ -1188,9 +1200,11 @@ public class DynamicPartitionTableTest {
}
@Test(expected = DdlException.class)
public void testHotPartitionNumAbnormal() throws Exception {
public void testHotPartitionNumAbnormalLT0() throws Exception {
changeBeDisk(TStorageMedium.SSD);
// dynamic_partition.hot_partition_num must larger than 0.
String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1` (\n"
String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1_lt0` (\n"
+ " `k1` datetime NULL COMMENT \"\",\n"
+ " `k2` int NULL COMMENT \"\"\n"
+ ") ENGINE=OLAP\n"
@ -1211,6 +1225,32 @@ public class DynamicPartitionTableTest {
createTable(createOlapTblStmt);
}
@Test(expected = DdlException.class)
public void testHotPartitionNumAbnormalMissSSD() throws Exception {
changeBeDisk(TStorageMedium.HDD);
// when dynamic_partition.hot_partition_num > 0, it require ssd storage medium.
String createOlapTblStmt = "CREATE TABLE test.`hot_partition_hour_tbl1_miss_ssd` (\n"
+ " `k1` datetime NULL COMMENT \"\",\n"
+ " `k2` int NULL COMMENT \"\"\n"
+ ") ENGINE=OLAP\n"
+ "PARTITION BY RANGE(`k1`)\n"
+ "()\n"
+ "DISTRIBUTED BY HASH(`k2`) BUCKETS 3\n"
+ "PROPERTIES (\n"
+ "\"replication_num\" = \"1\",\n"
+ "\"dynamic_partition.enable\" = \"true\",\n"
+ "\"dynamic_partition.start\" = \"-3\",\n"
+ "\"dynamic_partition.end\" = \"3\",\n"
+ "\"dynamic_partition.create_history_partition\" = \"true\",\n"
+ "\"dynamic_partition.time_unit\" = \"hour\",\n"
+ "\"dynamic_partition.prefix\" = \"p\",\n"
+ "\"dynamic_partition.buckets\" = \"1\",\n"
+ "\"dynamic_partition.hot_partition_num\" = \"1\"\n"
+ ");";
createTable(createOlapTblStmt);
}
@Test
public void testRuntimeInfo() throws Exception {
DynamicPartitionScheduler scheduler = new DynamicPartitionScheduler("test", 10);

1
regression-test/suites/correctness_p0/test_outer_join_with_window_function.groovy
View File

@ -218,7 +218,6 @@ suite("test_outer_join_with_with_window_function") {
"dynamic_partition.buckets" = "4",
"dynamic_partition.create_history_partition" = "true",
"dynamic_partition.history_partition_num" = "50",
"dynamic_partition.hot_partition_num" = "2",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.start_day_of_month" = "1",
"in_memory" = "false",

1
regression-test/suites/correctness_p0/test_outer_join_with_window_function_datev2.groovy
View File

@ -222,7 +222,6 @@ suite("test_outer_join_with_window_function_datev2") {
"dynamic_partition.buckets" = "4",
"dynamic_partition.create_history_partition" = "true",
"dynamic_partition.history_partition_num" = "50",
"dynamic_partition.hot_partition_num" = "2",
"dynamic_partition.reserved_history_periods" = "NULL",
"dynamic_partition.start_day_of_month" = "1",
"in_memory" = "false",

22
regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition.groovy
View File

@ -149,4 +149,26 @@ suite("test_dynamic_partition") {
exception "errCode = 2,"
}
sql "drop table if exists dy_par_bad"
test {
sql """
CREATE TABLE IF NOT EXISTS dy_par_bad
( k1 date NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL )
AGGREGATE KEY(k1,k2)
PARTITION BY RANGE(k1) ( )
DISTRIBUTED BY HASH(k1) BUCKETS 3
PROPERTIES (
"dynamic_partition.enable"="true",
"dynamic_partition.end"="3",
"dynamic_partition.start"="-3",
"dynamic_partition.prefix"="p",
"dynamic_partition.time_unit"="DAY",
"dynamic_partition.create_history_partition"="true",
"dynamic_partition.hot_partition_num" = "2",
"dynamic_partition.replication_allocation" = "tag.location.default: 1")
"""
// check exception message contains
// dynamic_partition.hot_partition_num > 0 will require ssd storage medium
exception "errCode = 2,"
}
sql "drop table if exists dy_par_bad"
}