[fix](dynamic-partition) fix wrong check of replication num (#13755)

This commit is contained in:
Mingyu Chen
2022-11-02 12:55:33 +08:00
committed by GitHub
parent 667cfe5598
commit d5becdb4a1
8 changed files with 204 additions and 25 deletions

View File

@ -211,7 +211,7 @@ public class DynamicPartitionUtil {
}
}
private static void checkReplicationNum(String val) throws DdlException {
private static void checkReplicationNum(String val, Database db) throws DdlException {
if (Strings.isNullOrEmpty(val)) {
throw new DdlException("Invalid properties: " + DynamicPartitionProperty.REPLICATION_NUM);
}
@ -222,6 +222,8 @@ public class DynamicPartitionUtil {
} catch (NumberFormatException e) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_FORMAT, val);
}
ReplicaAllocation replicaAlloc = new ReplicaAllocation(Short.valueOf(val));
Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, db.getClusterName(), null);
}
private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, Database db) throws DdlException {
@ -585,19 +587,18 @@ public class DynamicPartitionUtil {
analyzedProperties.put(DynamicPartitionProperty.TIME_ZONE, val);
}
if (properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM)) {
String val = properties.get(DynamicPartitionProperty.REPLICATION_NUM);
checkReplicationNum(val);
properties.remove(DynamicPartitionProperty.REPLICATION_NUM);
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
new ReplicaAllocation(Short.valueOf(val)).toCreateStmt());
}
// check replication_allocation first, then replciation_num
if (properties.containsKey(DynamicPartitionProperty.REPLICATION_ALLOCATION)) {
ReplicaAllocation replicaAlloc = PropertyAnalyzer.analyzeReplicaAllocation(properties, "dynamic_partition");
checkReplicaAllocation(replicaAlloc, db);
properties.remove(DynamicPartitionProperty.REPLICATION_ALLOCATION);
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION, replicaAlloc.toCreateStmt());
} else if (properties.containsKey(DynamicPartitionProperty.REPLICATION_NUM)) {
String val = properties.get(DynamicPartitionProperty.REPLICATION_NUM);
checkReplicationNum(val, db);
properties.remove(DynamicPartitionProperty.REPLICATION_NUM);
analyzedProperties.put(DynamicPartitionProperty.REPLICATION_ALLOCATION,
new ReplicaAllocation(Short.valueOf(val)).toCreateStmt());
} else {
checkReplicaAllocation(olapTable.getDefaultReplicaAllocation(), db);
}

View File

@ -108,6 +108,28 @@ public class DynamicPartitionTableTest {
Env.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test");
OlapTable table = (OlapTable) db.getTableOrAnalysisException("dynamic_partition_normal");
Assert.assertTrue(table.getTableProperty().getDynamicPartitionProperty().getReplicaAllocation().isNotSet());
// test only set dynamic_partition.replication_num
createOlapTblStmt = "CREATE TABLE test.`dynamic_partition_normal2` (\n"
+ "`uuid` varchar(255) NULL,\n"
+ "`action_datetime` date NULL\n"
+ ")\n"
+ "DUPLICATE KEY(uuid)\n"
+ "PARTITION BY RANGE(action_datetime)()\n"
+ "DISTRIBUTED BY HASH(uuid) BUCKETS 3\n"
+ "PROPERTIES\n"
+ "(\n"
+ "\"dynamic_partition.enable\" = \"true\",\n"
+ "\"dynamic_partition.time_unit\" = \"DAY\",\n"
+ "\"dynamic_partition.end\" = \"3\",\n"
+ "\"dynamic_partition.prefix\" = \"p\",\n"
+ "\"dynamic_partition.buckets\" = \"32\",\n"
+ "\"dynamic_partition.replication_num\" = \"1\",\n"
+ "\"dynamic_partition.create_history_partition\"=\"true\",\n"
+ "\"dynamic_partition.start\" = \"-3\"\n"
+ ");\n"
+ "\n";
createTable(createOlapTblStmt);
}
@Test
@ -431,23 +453,17 @@ public class DynamicPartitionTableTest {
+ "PROPERTIES (\n" + "\"replication_num\" = \"1\",\n" + "\"dynamic_partition.enable\" = \"true\",\n"
+ "\"dynamic_partition.start\" = \"-3\",\n" + "\"dynamic_partition.end\" = \"3\",\n"
+ "\"dynamic_partition.time_unit\" = \"day\",\n" + "\"dynamic_partition.prefix\" = \"p\",\n"
+ "\"dynamic_partition.buckets\" = \"1\",\n" + "\"dynamic_partition.replication_num\" = \"2\"\n" + ");";
+ "\"dynamic_partition.buckets\" = \"1\",\n" + "\"dynamic_partition.replication_num\" = \"1\"\n" + ");";
createTable(createOlapTblStmt);
Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException("default_cluster:test");
OlapTable table = (OlapTable) db.getTableOrAnalysisException(tableName);
Assert.assertEquals(2,
table.getTableProperty().getDynamicPartitionProperty().getReplicaAllocation().getTotalReplicaNum());
String alter1 =
"alter table test.dynamic_partition_replication_num set ('dynamic_partition.replication_num' = '1')";
ExceptionChecker.expectThrowsNoException(() -> alterTable(alter1));
Assert.assertEquals(1,
table.getTableProperty().getDynamicPartitionProperty().getReplicaAllocation().getTotalReplicaNum());
String alter2 =
String alter1 =
"alter table test.dynamic_partition_replication_num set ('dynamic_partition.replication_num' = '0')";
ExceptionChecker.expectThrows(AnalysisException.class, () -> alterTable(alter2));
ExceptionChecker.expectThrows(AnalysisException.class, () -> alterTable(alter1));
Assert.assertEquals(1,
table.getTableProperty().getDynamicPartitionProperty().getReplicaAllocation().getTotalReplicaNum());
}

View File

@ -96,7 +96,7 @@ public class ModifyBackendTest {
+ "buckets 3 properties(\n" + " \"dynamic_partition.enable\" = \"true\",\n"
+ " \"dynamic_partition.time_unit\" = \"DAY\",\n" + " \"dynamic_partition.start\" = \"-3\",\n"
+ " \"dynamic_partition.end\" = \"3\",\n" + " \"dynamic_partition.prefix\" = \"p\",\n"
+ " \"dynamic_partition.buckets\" = \"1\",\n" + " \"dynamic_partition.replication_num\" = \"1\"\n"
+ " \"dynamic_partition.buckets\" = \"1\",\n" + " \"dynamic_partition.replication_num\" = \"3\"\n"
+ ");";
CreateTableStmt createStmt3 = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createStr, connectContext);
//partition create failed, because there is no BE with "default" tag

View File

@ -127,7 +127,7 @@ public class DorisAssert {
Env.getCurrentEnv().createMaterializedView(createMaterializedViewStmt);
checkAlterJob();
// waiting table state to normal
Thread.sleep(100);
Thread.sleep(1000);
return this;
}
@ -137,7 +137,7 @@ public class DorisAssert {
Env.getCurrentEnv().alterTable(alterTableStmt);
checkAlterJob();
// waiting table state to normal
Thread.sleep(100);
Thread.sleep(1000);
return this;
}