[feature](create-table) support setting replication num for creating table opertaion globally (#21848)

Add a new FE config `force_olap_table_replication_num`.
If this config is larger than 0, when doing creating table operation, the replication num of table will
forcibly be this value.
Default is 0, which make no effect.
This config will only effect the creating olap table operation, other operation such as `add partition`,
`modify table properties` will not be effect.

The motivation of this config is that the most regression test cases are creating table will single replica,
this will be the regression test running well in p0, p1 pipeline.
But we also need to run these cases in multi backend Doris cluster, so we need test cases will multi replicas.
But it is hard to modify each test cases. So I add this config, so that we can simply set it to create all tables with
specified replication number.
This commit is contained in:
Mingyu Chen
2023-07-21 19:36:04 +08:00
committed by GitHub
parent e489b60ea3
commit 85cc044aaa
3 changed files with 72 additions and 0 deletions

View File

@ -24,6 +24,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@ -535,6 +536,8 @@ public class CreateTableStmt extends DdlStmt {
}
if (engineName.equals("olap")) {
// before analyzing partition, handle the replication allocation info
properties = rewriteReplicaAllocationProperties(properties);
// analyze partition
if (partitionDesc != null) {
if (partitionDesc instanceof ListPartitionDesc || partitionDesc instanceof RangePartitionDesc
@ -624,6 +627,34 @@ public class CreateTableStmt extends DdlStmt {
}
}
private Map<String, String> rewriteReplicaAllocationProperties(Map<String, String> properties) {
if (Config.force_olap_table_replication_num <= 0) {
return properties;
}
// if force_olap_table_replication_num is set, use this value to rewrite the replication_num or
// replication_allocation properties
Map<String, String> newProperties = properties;
if (newProperties == null) {
newProperties = Maps.newHashMap();
}
boolean rewrite = false;
if (newProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM)) {
newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
String.valueOf(Config.force_olap_table_replication_num));
rewrite = true;
}
if (newProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION)) {
newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
new ReplicaAllocation((short) Config.force_olap_table_replication_num).toCreateStmt());
rewrite = true;
}
if (!rewrite) {
newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
String.valueOf(Config.force_olap_table_replication_num));
}
return newProperties;
}
private void analyzeEngineName() throws AnalysisException {
if (Strings.isNullOrEmpty(engineName)) {
engineName = "olap";

View File

@ -17,6 +17,7 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.common.AnalysisException;
@ -25,6 +26,7 @@ import org.apache.doris.common.ConfigBase;
import org.apache.doris.common.ConfigException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
@ -67,6 +69,11 @@ public class CreateTableTest {
Env.getCurrentEnv().createTable(createTableStmt);
}
private static void alterTable(String sql) throws Exception {
AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Env.getCurrentEnv().alterTable(alterTableStmt);
}
@Test
public void testDuplicateCreateTable() throws Exception {
// test
@ -739,4 +746,27 @@ public class CreateTableTest {
Assert.assertEquals(ScalarType.MAX_VARCHAR_LENGTH, tb.getColumn("k3").getStrLen());
Assert.assertEquals(10, tb.getColumn("k4").getStrLen());
}
@Test
public void testCreateTableWithForceReplica() throws DdlException {
Config.force_olap_table_replication_num = 1;
// no need to specify replication_num, the table can still be created.
ExceptionChecker.expectThrowsNoException(() -> {
createTable("create table test.test_replica\n" + "(k1 int, k2 int) partition by range(k1)\n" + "(\n"
+ "partition p1 values less than(\"10\"),\n" + "partition p2 values less than(\"20\")\n" + ")\n"
+ "distributed by hash(k2) buckets 1;");
});
// can still set replication_num manually.
ExceptionChecker.expectThrowsWithMsg(UserException.class, "Failed to find enough host with tag",
() -> {
alterTable("alter table test.test_replica modify partition p1 set ('replication_num' = '3')");
});
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("default_cluster:test");
OlapTable tb = (OlapTable) db.getTableOrDdlException("test_replica");
Partition p1 = tb.getPartition("p1");
Assert.assertEquals(1, tb.getPartitionInfo().getReplicaAllocation(p1.getId()).getTotalReplicaNum());
Assert.assertEquals(1, tb.getTableProperty().getReplicaAllocation().getTotalReplicaNum());
}
}