database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](resource-group) Supports memory hard isolation of resource group (#19526)

Browse Source
This commit is contained in:
luozenglin
2023-05-15 22:45:46 +08:00
committed by GitHub
parent 276e631e9c
commit 6c9c9e9765
19 changed files with 441 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -871,10 +871,6 @@ public class StmtExecutor {
|| (parsedStmt instanceof InsertStmt && !((InsertStmt) parsedStmt).needLoadManager())
|| parsedStmt instanceof CreateTableAsSelectStmt
|| parsedStmt instanceof InsertOverwriteTableStmt) {
if (Config.enable_resource_group && context.sessionVariable.enablePipelineEngine()) {
analyzer.setResourceGroups(analyzer.getEnv().getResourceGroupMgr()
.getResourceGroup(context.sessionVariable.resourceGroup));
}
Map<Long, TableIf> tableMap = Maps.newTreeMap();
QueryStmt queryStmt;
Set<String> parentViewNameSet = Sets.newHashSet();
@ -1060,6 +1056,11 @@ public class StmtExecutor {
parsedStmt.setIsExplain(explainOptions);
}
}
if (parsedStmt instanceof QueryStmt && Config.enable_resource_group
&& context.sessionVariable.enablePipelineEngine()) {
analyzer.setResourceGroups(analyzer.getEnv().getResourceGroupMgr()
.getResourceGroup(context.sessionVariable.resourceGroup));
}
}
profile.getSummaryProfile().setQueryAnalysisFinishTime();
planner = new OriginalPlanner(analyzer);

43
fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/ResourceGroup.java
View File

@ -30,6 +30,8 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
@ -38,14 +40,17 @@ import java.util.HashMap;
import java.util.Map;
public class ResourceGroup implements Writable {
private static final Logger LOG = LogManager.getLogger(ResourceGroup.class);
public static final String CPU_SHARE = "cpu_share";
public static final String MEMORY_LIMIT = "memory_limit";
private static final ImmutableSet<String> REQUIRED_PROPERTIES_NAME = new ImmutableSet.Builder<String>().add(
CPU_SHARE).build();
CPU_SHARE).add(MEMORY_LIMIT).build();
private static final ImmutableSet<String> ALL_PROPERTIES_NAME = new ImmutableSet.Builder<String>().add(
CPU_SHARE).build();
CPU_SHARE).add(MEMORY_LIMIT).build();
@SerializedName(value = "id")
private long id;
@ -60,11 +65,10 @@ public class ResourceGroup implements Writable {
@SerializedName(value = "version")
private long version;
private double memoryLimitPercent;
private ResourceGroup(long id, String name, Map<String, String> properties) {
this.id = id;
this.name = name;
this.properties = properties;
this.version = 0;
this(id, name, properties, 0);
}
private ResourceGroup(long id, String name, Map<String, String> properties, long version) {
@ -72,6 +76,8 @@ public class ResourceGroup implements Writable {
this.name = name;
this.properties = properties;
this.version = version;
String memoryLimitString = properties.get(MEMORY_LIMIT);
this.memoryLimitPercent = Double.parseDouble(memoryLimitString.substring(0, memoryLimitString.length() - 1));
}
public static ResourceGroup create(String name, Map<String, String> properties) throws DdlException {
@ -79,10 +85,9 @@ public class ResourceGroup implements Writable {
return new ResourceGroup(Env.getCurrentEnv().getNextId(), name, properties);
}
public static ResourceGroup create(ResourceGroup resourceGroup, Map<String, String> updateProperties)
public static ResourceGroup copyAndUpdate(ResourceGroup resourceGroup, Map<String, String> updateProperties)
throws DdlException {
Map<String, String> newProperties = new HashMap<>();
newProperties.putAll(resourceGroup.getProperties());
Map<String, String> newProperties = new HashMap<>(resourceGroup.getProperties());
for (Map.Entry<String, String> kv : updateProperties.entrySet()) {
if (!Strings.isNullOrEmpty(kv.getValue())) {
newProperties.put(kv.getKey(), kv.getValue());
@ -108,7 +113,21 @@ public class ResourceGroup implements Writable {
String cpuSchedulingWeight = properties.get(CPU_SHARE);
if (!StringUtils.isNumeric(cpuSchedulingWeight) || Long.parseLong(cpuSchedulingWeight) <= 0) {
throw new DdlException(CPU_SHARE + " requires a positive integer.");
throw new DdlException(CPU_SHARE + " " + cpuSchedulingWeight + " requires a positive integer.");
}
String memoryLimit = properties.get(MEMORY_LIMIT);
if (!memoryLimit.endsWith("%")) {
throw new DdlException(MEMORY_LIMIT + " " + memoryLimit + " requires a percentage and ends with a '%'");
}
String memLimitErr = MEMORY_LIMIT + " " + memoryLimit + " requires a positive floating point number.";
try {
if (Double.parseDouble(memoryLimit.substring(0, memoryLimit.length() - 1)) <= 0) {
throw new DdlException(memLimitErr);
}
} catch (NumberFormatException e) {
LOG.debug(memLimitErr, e);
throw new DdlException(memLimitErr);
}
}
@ -128,6 +147,10 @@ public class ResourceGroup implements Writable {
return version;
}
public double getMemoryLimitPercent() {
return memoryLimitPercent;
}
public void getProcNodeData(BaseProcResult result) {
for (Map.Entry<String, String> entry : properties.entrySet()) {
result.addRow(Lists.newArrayList(String.valueOf(id), name, entry.getKey(), entry.getValue()));

23
fe/fe-core/src/main/java/org/apache/doris/resource/resourcegroup/ResourceGroupMgr.java
View File

@ -46,6 +46,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.locks.ReentrantReadWriteLock;
public class ResourceGroupMgr implements Writable, GsonPostProcessable {
@ -122,6 +123,7 @@ public class ResourceGroupMgr implements Writable, GsonPostProcessable {
}
Map<String, String> properties = Maps.newHashMap();
properties.put(ResourceGroup.CPU_SHARE, "10");
properties.put(ResourceGroup.MEMORY_LIMIT, "100%");
defaultResourceGroup = ResourceGroup.create(DEFAULT_GROUP_NAME, properties);
nameToResourceGroup.put(DEFAULT_GROUP_NAME, defaultResourceGroup);
idToResourceGroup.put(defaultResourceGroup.getId(), defaultResourceGroup);
@ -141,12 +143,14 @@ public class ResourceGroupMgr implements Writable, GsonPostProcessable {
String resourceGroupName = resourceGroup.getName();
writeLock();
try {
if (nameToResourceGroup.putIfAbsent(resourceGroupName, resourceGroup) != null) {
if (nameToResourceGroup.containsKey(resourceGroupName)) {
if (stmt.isIfNotExists()) {
return;
}
throw new DdlException("Resource group " + resourceGroupName + " already exist");
}
checkGlobalUnlock(resourceGroup, null);
nameToResourceGroup.put(resourceGroupName, resourceGroup);
idToResourceGroup.put(resourceGroup.getId(), resourceGroup);
Env.getCurrentEnv().getEditLog().logCreateResourceGroup(resourceGroup);
} finally {
@ -155,6 +159,18 @@ public class ResourceGroupMgr implements Writable, GsonPostProcessable {
LOG.info("Create resource group success: {}", resourceGroup);
}
private void checkGlobalUnlock(ResourceGroup resourceGroup, ResourceGroup old) throws DdlException {
double totalMemoryLimit = idToResourceGroup.values().stream().mapToDouble(ResourceGroup::getMemoryLimitPercent)
.sum() + resourceGroup.getMemoryLimitPercent();
if (!Objects.isNull(old)) {
totalMemoryLimit -= old.getMemoryLimitPercent();
}
if (totalMemoryLimit > 100.0 + 1e-6) {
throw new DdlException(
"The sum of all resource group " + ResourceGroup.MEMORY_LIMIT + " cannot be greater than 100.0%.");
}
}
public void alterResourceGroup(AlterResourceGroupStmt stmt) throws DdlException {
checkResourceGroupEnabled();
@ -167,7 +183,8 @@ public class ResourceGroupMgr implements Writable, GsonPostProcessable {
throw new DdlException("Resource Group(" + resourceGroupName + ") does not exist.");
}
ResourceGroup resourceGroup = nameToResourceGroup.get(resourceGroupName);
newResourceGroup = ResourceGroup.create(resourceGroup, properties);
newResourceGroup = ResourceGroup.copyAndUpdate(resourceGroup, properties);
checkGlobalUnlock(newResourceGroup, resourceGroup);
nameToResourceGroup.put(resourceGroupName, newResourceGroup);
idToResourceGroup.put(newResourceGroup.getId(), newResourceGroup);
Env.getCurrentEnv().getEditLog().logAlterResourceGroup(newResourceGroup);
@ -181,7 +198,7 @@ public class ResourceGroupMgr implements Writable, GsonPostProcessable {
checkResourceGroupEnabled();
String resourceGroupName = stmt.getResourceGroupName();
if (resourceGroupName == DEFAULT_GROUP_NAME) {
if (DEFAULT_GROUP_NAME.equals(resourceGroupName)) {
throw new DdlException("Dropping default resource group " + resourceGroupName + " is not allowed");
}