[feature](create-table) support setting replication num for creating table opertaion globally (#21848)
Add a new FE config `force_olap_table_replication_num`. If this config is larger than 0, when doing creating table operation, the replication num of table will forcibly be this value. Default is 0, which make no effect. This config will only effect the creating olap table operation, other operation such as `add partition`, `modify table properties` will not be effect. The motivation of this config is that the most regression test cases are creating table will single replica, this will be the regression test running well in p0, p1 pipeline. But we also need to run these cases in multi backend Doris cluster, so we need test cases will multi replicas. But it is hard to modify each test cases. So I add this config, so that we can simply set it to create all tables with specified replication number.
This commit is contained in:
@ -24,6 +24,7 @@ import org.apache.doris.catalog.Env;
|
||||
import org.apache.doris.catalog.Index;
|
||||
import org.apache.doris.catalog.KeysType;
|
||||
import org.apache.doris.catalog.PrimitiveType;
|
||||
import org.apache.doris.catalog.ReplicaAllocation;
|
||||
import org.apache.doris.catalog.Type;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
import org.apache.doris.common.Config;
|
||||
@ -535,6 +536,8 @@ public class CreateTableStmt extends DdlStmt {
|
||||
}
|
||||
|
||||
if (engineName.equals("olap")) {
|
||||
// before analyzing partition, handle the replication allocation info
|
||||
properties = rewriteReplicaAllocationProperties(properties);
|
||||
// analyze partition
|
||||
if (partitionDesc != null) {
|
||||
if (partitionDesc instanceof ListPartitionDesc || partitionDesc instanceof RangePartitionDesc
|
||||
@ -624,6 +627,34 @@ public class CreateTableStmt extends DdlStmt {
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> rewriteReplicaAllocationProperties(Map<String, String> properties) {
|
||||
if (Config.force_olap_table_replication_num <= 0) {
|
||||
return properties;
|
||||
}
|
||||
// if force_olap_table_replication_num is set, use this value to rewrite the replication_num or
|
||||
// replication_allocation properties
|
||||
Map<String, String> newProperties = properties;
|
||||
if (newProperties == null) {
|
||||
newProperties = Maps.newHashMap();
|
||||
}
|
||||
boolean rewrite = false;
|
||||
if (newProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM)) {
|
||||
newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
|
||||
String.valueOf(Config.force_olap_table_replication_num));
|
||||
rewrite = true;
|
||||
}
|
||||
if (newProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)) {
|
||||
newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
|
||||
new ReplicaAllocation((short) Config.force_olap_table_replication_num).toCreateStmt());
|
||||
rewrite = true;
|
||||
}
|
||||
if (!rewrite) {
|
||||
newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
|
||||
String.valueOf(Config.force_olap_table_replication_num));
|
||||
}
|
||||
return newProperties;
|
||||
}
|
||||
|
||||
private void analyzeEngineName() throws AnalysisException {
|
||||
if (Strings.isNullOrEmpty(engineName)) {
|
||||
engineName = "olap";
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
|
||||
package org.apache.doris.catalog;
|
||||
|
||||
import org.apache.doris.analysis.AlterTableStmt;
|
||||
import org.apache.doris.analysis.CreateDbStmt;
|
||||
import org.apache.doris.analysis.CreateTableStmt;
|
||||
import org.apache.doris.common.AnalysisException;
|
||||
@ -25,6 +26,7 @@ import org.apache.doris.common.ConfigBase;
|
||||
import org.apache.doris.common.ConfigException;
|
||||
import org.apache.doris.common.DdlException;
|
||||
import org.apache.doris.common.ExceptionChecker;
|
||||
import org.apache.doris.common.UserException;
|
||||
import org.apache.doris.qe.ConnectContext;
|
||||
import org.apache.doris.utframe.UtFrameUtils;
|
||||
|
||||
@ -67,6 +69,11 @@ public class CreateTableTest {
|
||||
Env.getCurrentEnv().createTable(createTableStmt);
|
||||
}
|
||||
|
||||
private static void alterTable(String sql) throws Exception {
|
||||
AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
|
||||
Env.getCurrentEnv().alterTable(alterTableStmt);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDuplicateCreateTable() throws Exception {
|
||||
// test
|
||||
@ -739,4 +746,27 @@ public class CreateTableTest {
|
||||
Assert.assertEquals(ScalarType.MAX_VARCHAR_LENGTH, tb.getColumn("k3").getStrLen());
|
||||
Assert.assertEquals(10, tb.getColumn("k4").getStrLen());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateTableWithForceReplica() throws DdlException {
|
||||
Config.force_olap_table_replication_num = 1;
|
||||
// no need to specify replication_num, the table can still be created.
|
||||
ExceptionChecker.expectThrowsNoException(() -> {
|
||||
createTable("create table test.test_replica\n" + "(k1 int, k2 int) partition by range(k1)\n" + "(\n"
|
||||
+ "partition p1 values less than(\"10\"),\n" + "partition p2 values less than(\"20\")\n" + ")\n"
|
||||
+ "distributed by hash(k2) buckets 1;");
|
||||
});
|
||||
|
||||
// can still set replication_num manually.
|
||||
ExceptionChecker.expectThrowsWithMsg(UserException.class, "Failed to find enough host with tag",
|
||||
() -> {
|
||||
alterTable("alter table test.test_replica modify partition p1 set ('replication_num' = '3')");
|
||||
});
|
||||
|
||||
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("default_cluster:test");
|
||||
OlapTable tb = (OlapTable) db.getTableOrDdlException("test_replica");
|
||||
Partition p1 = tb.getPartition("p1");
|
||||
Assert.assertEquals(1, tb.getPartitionInfo().getReplicaAllocation(p1.getId()).getTotalReplicaNum());
|
||||
Assert.assertEquals(1, tb.getTableProperty().getReplicaAllocation().getTotalReplicaNum());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user