database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](mtmv) mtmv support workload group (#29595)

Browse Source 
MTMV supports controlling the resource usage of refresh tasks by setting the name of workload group
about workload group : https://doris.apache.org/zh-CN/docs/dev/admin-manual/workload-group
This commit is contained in:
zhangdong
2024-02-04 11:11:58 +08:00
committed by yiguolei
parent e10defeaba
commit b275cb0f44
8 changed files with 135 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -40,6 +40,7 @@ import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -47,6 +48,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@ -190,6 +192,14 @@ public class MTMV extends OlapTable {
}
}
public Optional<String> getWorkloadGroup() {
if (mvProperties.containsKey(PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP) && !StringUtils
.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP))) {
return Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP));
}
return Optional.empty();
}
public int getRefreshPartitionNum() {
if (mvProperties.containsKey(PropertyAnalyzer.PROPERTIES_REFRESH_PARTITION_NUM)) {
int value = Integer.parseInt(mvProperties.get(PropertyAnalyzer.PROPERTIES_REFRESH_PARTITION_NUM));

1
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
View File

@ -164,6 +164,7 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_GRACE_PERIOD = "grace_period";
public static final String PROPERTIES_EXCLUDED_TRIGGER_TABLES = "excluded_trigger_tables";
public static final String PROPERTIES_REFRESH_PARTITION_NUM = "refresh_partition_num";
public static final String PROPERTIES_WORKLOAD_GROUP = "workload_group";
// For unique key data model, the feature Merge-on-Write will leverage a primary
// key index and a delete-bitmap to mark duplicate keys as deleted in load stage,
// which can avoid the merging cost in read stage, and accelerate the aggregation

5
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
View File

@ -43,6 +43,7 @@ import org.apache.doris.qe.SessionVariable;
import com.google.common.collect.Sets;
import java.util.List;
import java.util.Optional;
import java.util.Set;
public class MTMVPlanUtil {
@ -59,6 +60,10 @@ public class MTMVPlanUtil {
ctx.changeDefaultCatalog(catalog.getName());
ctx.setDatabase(catalog.getDbOrAnalysisException(mtmv.getEnvInfo().getDbId()).getFullName());
ctx.getSessionVariable().enableFallbackToOriginalPlanner = false;
Optional<String> workloadGroup = mtmv.getWorkloadGroup();
if (workloadGroup.isPresent()) {
ctx.getSessionVariable().setWorkloadGroup(workloadGroup.get());
}
ctx.getSessionVariable().enableNereidsDML = true;
return ctx;
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterMTMVPropertyInfo.java
View File

@ -20,9 +20,12 @@ package org.apache.doris.nereids.trees.plans.commands.info;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.qe.ConnectContext;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
import java.util.Objects;
@ -71,6 +74,16 @@ public class AlterMTMVPropertyInfo extends AlterMTMVInfo {
}
} else if (PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES.equals(key)) {
// nothing
} else if (PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP.equals(key)) {
String workloadGroup = properties.get(PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP);
if (!StringUtils.isEmpty(workloadGroup) && !Env.getCurrentEnv().getAccessManager()
.checkWorkloadGroupPriv(ConnectContext.get(), workloadGroup, PrivPredicate.USAGE)) {
String message = String
.format("Access denied; you need (at least one of) "
+ "the %s privilege(s) to use workload group '%s'.",
"USAGE/ADMIN", workloadGroup);
throw new AnalysisException(message);
}
} else {
throw new org.apache.doris.nereids.exceptions.AnalysisException("illegal key:" + key);
}

13
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
View File

@ -202,6 +202,19 @@ public class CreateMTMVInfo {
mvProperties.put(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES, excludedTriggerTables);
properties.remove(PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES);
}
if (properties.containsKey(PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP)) {
String workloadGroup = properties.get(PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP);
if (!Env.getCurrentEnv().getAccessManager()
.checkWorkloadGroupPriv(ConnectContext.get(), workloadGroup, PrivPredicate.USAGE)) {
String message = String
.format("Access denied;"
+ " you need (at least one of) the %s privilege(s) to use workload group '%s'.",
"USAGE/ADMIN", workloadGroup);
throw new AnalysisException(message);
}
mvProperties.put(PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP, workloadGroup);
properties.remove(PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP);
}
}
/**