database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhance](mtmv) Limit the number of partitions for table creation (#30867)

Browse Source 
- Creating too many partitions is time-consuming, so limiting the number of partitions
- add more case,such as `mor`,`mow`
This commit is contained in:
zhangdong
2024-02-07 11:40:18 +08:00
committed by yiguolei
parent b01625b7a9
commit 3017c0a6ff
14 changed files with 355 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionDesc.java
View File

@ -159,6 +159,15 @@ public class PartitionDesc {
throw new AnalysisException("No partition columns.");
}
int createTablePartitionMaxNum = ConnectContext.get().getSessionVariable().getCreateTablePartitionMaxNum();
if (singlePartitionDescs.size() > createTablePartitionMaxNum) {
throw new AnalysisException(String.format(
"The number of partitions to be created is [%s], exceeding the maximum value of [%s]. "
+ "Creating too many partitions can be time-consuming. If necessary, "
+ "You can set the session variable 'create_table_partition_max_num' to a larger value.",
singlePartitionDescs.size(), createTablePartitionMaxNum));
}
// `analyzeUniqueKeyMergeOnWrite` would modify `properties`, which will be used later,
// so we just clone a properties map here.
boolean enableUniqueKeyMergeOnWrite = false;

27
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.mtmv;
import org.apache.doris.analysis.AddPartitionClause;
import org.apache.doris.analysis.AllPartitionDesc;
import org.apache.doris.analysis.DropPartitionClause;
import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.analysis.SinglePartitionDesc;
@ -38,6 +39,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@ -106,6 +108,31 @@ public class MTMVPartitionUtil {
}
}
/**
* getPartitionDescsByRelatedTable when create MTMV
*
* @param relatedTable
* @param tableProperties
* @return
* @throws AnalysisException
*/
public static List<AllPartitionDesc> getPartitionDescsByRelatedTable(MTMVRelatedTableIf relatedTable,
Map<String, String> tableProperties) throws AnalysisException {
HashMap<String, String> partitionProperties = Maps.newHashMap();
List<AllPartitionDesc> res = Lists.newArrayList();
Map<Long, PartitionItem> relatedTableItems = relatedTable.getPartitionItems();
for (Entry<Long, PartitionItem> entry : relatedTableItems.entrySet()) {
PartitionKeyDesc oldPartitionKeyDesc = entry.getValue().toPartitionKeyDesc();
SinglePartitionDesc singlePartitionDesc = new SinglePartitionDesc(true,
generatePartitionName(oldPartitionKeyDesc),
oldPartitionKeyDesc, partitionProperties);
// mtmv can only has one partition col
singlePartitionDesc.analyze(1, tableProperties);
res.add(singlePartitionDesc);
}
return res;
}
public static List<String> getPartitionNamesByIds(MTMV mtmv, Collection<Long> ids) throws AnalysisException {
List<String> res = Lists.newArrayList();
for (Long partitionId : ids) {

4
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java
View File

@ -24,7 +24,6 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.job.exception.JobException;
import org.apache.doris.job.extensions.mtmv.MTMVTask;
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo;
import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo;
@ -85,9 +84,6 @@ public class MTMVService {
public void createMTMV(MTMV mtmv) throws DdlException, AnalysisException {
Objects.requireNonNull(mtmv);
LOG.info("createMTMV: " + mtmv.getName());
if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE) {
MTMVPartitionUtil.alignMvPartition(mtmv, mtmv.getMvPartitionInfo().getRelatedTable());
}
for (MTMVHookService mtmvHookService : hooks.values()) {
mtmvHookService.createMTMV(mtmv);
}

28
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
View File

@ -17,6 +17,7 @@
package org.apache.doris.nereids.trees.plans.commands.info;
import org.apache.doris.analysis.AllPartitionDesc;
import org.apache.doris.analysis.CreateMTMVStmt;
import org.apache.doris.analysis.KeysDesc;
import org.apache.doris.analysis.ListPartitionDesc;
@ -37,6 +38,7 @@ import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.mtmv.EnvInfo;
import org.apache.doris.mtmv.MTMVPartitionInfo;
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import org.apache.doris.mtmv.MTMVPartitionUtil;
import org.apache.doris.mtmv.MTMVPlanUtil;
import org.apache.doris.mtmv.MTMVRefreshInfo;
import org.apache.doris.mtmv.MTMVRelatedTableIf;
@ -241,7 +243,7 @@ public class CreateMTMVInfo {
}
getRelation(planner);
getColumns(plan);
analyzePartition(planner);
analyzePartition(planner, ctx);
}
private void getRelation(NereidsPlanner planner) {
@ -264,7 +266,7 @@ public class CreateMTMVInfo {
this.relation = MTMVPlanUtil.generateMTMVRelation(plan);
}
private void analyzePartition(NereidsPlanner planner) {
private void analyzePartition(NereidsPlanner planner, ConnectContext ctx) {
if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE) {
CascadesContext cascadesContext = planner.getCascadesContext();
@ -307,7 +309,7 @@ public class CreateMTMVInfo {
}
mvPartitionInfo.setRelatedTable(relatedTableInfo.get().getTableInfo());
mvPartitionInfo.setRelatedCol(relatedTableInfo.get().getColumn());
partitionDesc = generatePartitionDesc(mtmvBaseRealtedTable);
partitionDesc = generatePartitionDesc(mtmvBaseRealtedTable, ctx);
} finally {
// after operate, roll back the disable rules
sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules));
@ -316,15 +318,29 @@ public class CreateMTMVInfo {
}
}
private PartitionDesc generatePartitionDesc(MTMVRelatedTableIf relatedTable) {
private PartitionDesc generatePartitionDesc(MTMVRelatedTableIf relatedTable, ConnectContext ctx) {
List<AllPartitionDesc> allPartitionDescs = null;
try {
allPartitionDescs = MTMVPartitionUtil
.getPartitionDescsByRelatedTable(relatedTable, properties);
} catch (org.apache.doris.common.AnalysisException e) {
throw new AnalysisException("getPartitionDescsByRelatedTable failed", e);
}
if (allPartitionDescs.size() > ctx.getSessionVariable().getCreateTablePartitionMaxNum()) {
throw new AnalysisException(String.format(
"The number of partitions to be created is [%s], exceeding the maximum value of [%s]. "
+ "Creating too many partitions can be time-consuming. If necessary, "
+ "You can set the session variable 'create_table_partition_max_num' to a larger value.",
allPartitionDescs.size(), ctx.getSessionVariable().getCreateTablePartitionMaxNum()));
}
try {
PartitionType type = relatedTable.getPartitionType();
if (type == PartitionType.RANGE) {
return new RangePartitionDesc(Lists.newArrayList(mvPartitionInfo.getPartitionCol()),
Lists.newArrayList());
allPartitionDescs);
} else if (type == PartitionType.LIST) {
return new ListPartitionDesc(Lists.newArrayList(mvPartitionInfo.getPartitionCol()),
Lists.newArrayList());
allPartitionDescs);
} else {
return null;
}

11
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
View File

@ -813,6 +813,17 @@ public class CreateTableInfo {
? partitions.stream().map(PartitionDefinition::translateToCatalogStyle)
.collect(Collectors.toList())
: null;
int createTablePartitionMaxNum = ConnectContext.get().getSessionVariable().getCreateTablePartitionMaxNum();
if (partitionDescs != null && partitionDescs.size() > createTablePartitionMaxNum) {
throw new org.apache.doris.nereids.exceptions.AnalysisException(String.format(
"The number of partitions to be created is [%s], exceeding the maximum value of [%s]. "
+ "Creating too many partitions can be time-consuming. If necessary, "
+ "You can set the session variable 'create_table_partition_max_num' "
+ "to a larger value.",
partitionDescs.size(), createTablePartitionMaxNum));
}
try {
if (partitionType.equals(PartitionType.RANGE.name())) {
if (isAutoPartition) {

12
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -488,6 +488,9 @@ public class SessionVariable implements Serializable, Writable {
public static final String MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_EXTERNAL_TABLE
= "materialized_view_rewrite_enable_contain_external_table";
public static final String CREATE_TABLE_PARTITION_MAX_NUM
= "create_table_partition_max_num";
public static final String ENABLE_PUSHDOWN_MINMAX_ON_UNIQUE = "enable_pushdown_minmax_on_unique";
public static final String ENABLE_PUSHDOWN_STRING_MINMAX = "enable_pushdown_string_minmax";
@ -1539,6 +1542,11 @@ public class SessionVariable implements Serializable, Writable {
+ "when using rewriting based on struct info"})
public boolean materializedViewRewriteEnableContainExternalTable = false;
@VariableMgr.VarAttr(name = CREATE_TABLE_PARTITION_MAX_NUM, needForward = true,
description = {"建表时创建分区的最大数量",
"The maximum number of partitions created during table creation"})
public int createTablePartitionMaxNum = 10000;
@VariableMgr.VarAttr(name = FORCE_JNI_SCANNER,
description = {"强制使用jni方式读取外表", "Force the use of jni mode to read external table"})
private boolean forceJniScanner = false;
@ -3293,6 +3301,10 @@ public class SessionVariable implements Serializable, Writable {
return materializedViewRewriteEnableContainExternalTable;
}
public int getCreateTablePartitionMaxNum() {
return createTablePartitionMaxNum;
}
public boolean isIgnoreStorageDataDistribution() {
return ignoreStorageDataDistribution && getEnablePipelineXEngine() && enableLocalShuffle
&& enableNereidsPlanner;