database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](user-property) Support user level exec_mem_limit and load_mem_limit (#8365)

Browse Source 
```
SET PROPERTY FOR 'jack' 'exec_mem_limit' = '2147483648';
SET PROPERTY FOR 'jack' 'load_mem_limit' = '2147483648';
```
The user level property will overwrite the value in session variables.
This commit is contained in:
Mingyu Chen
2022-03-11 17:20:09 +08:00
committed by GitHub
parent 68dd799796
commit e403dbc38c
7 changed files with 139 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java
View File

@ -47,6 +47,12 @@ public class CommonUserProperties implements Writable {
// The tag of the resource that the user is allowed to use
@SerializedName("resourceTags")
private Set<Tag> resourceTags = Sets.newHashSet();
// user level exec_mem_limit, if > 0, will overwrite the exec_mem_limit in session variable
@SerializedName("execMemLimit")
private long execMemLimit = -1;
// user level load_mem_limit, if > 0, will overwrite the load_mem_limit in session variable
@SerializedName("loadMemLimit")
private long loadMemLimit = -1;
private String[] sqlBlockRulesSplit = {};
@ -100,6 +106,22 @@ public class CommonUserProperties implements Writable {
return resourceTags;
}
public long getExecMemLimit() {
return execMemLimit;
}
public void setExecMemLimit(long execMemLimit) {
this.execMemLimit = execMemLimit;
}
public long getLoadMemLimit() {
return loadMemLimit;
}
public void setLoadMemLimit(long loadMemLimit) {
this.loadMemLimit = loadMemLimit;
}
public static CommonUserProperties read(DataInput in) throws IOException {
String json = Text.readString(in);
CommonUserProperties commonUserProperties = GsonUtils.GSON.fromJson(json, CommonUserProperties.class);

18
fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java
View File

@ -1172,6 +1172,24 @@ public class PaloAuth implements Writable {
}
}
public long getExecMemLimit(String qualifiedUser) {
readLock();
try {
return propertyMgr.getExecMemLimit(qualifiedUser);
} finally {
readUnlock();
}
}
public long getLoadMemLimit(String qualifiedUser) {
readLock();
try {
return propertyMgr.getLoadMemLimit(qualifiedUser);
} finally {
readUnlock();
}
}
public void getAllDomains(Set<String> allDomains) {
readLock();
try {

40
fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java
View File

@ -63,6 +63,8 @@ public class UserProperty implements Writable {
private static final String PROP_RESOURCE = "resource";
private static final String PROP_SQL_BLOCK_RULES = "sql_block_rules";
private static final String PROP_CPU_RESOURCE_LIMIT = "cpu_resource_limit";
private static final String PROP_EXEC_MEM_LIMIT = "exec_mem_limit";
private static final String PROP_LOAD_MEM_LIMIT = "load_mem_limit";
// advanced properties end
private static final String PROP_LOAD_CLUSTER = "load_cluster";
@ -107,6 +109,8 @@ public class UserProperty implements Writable {
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_SQL_BLOCK_RULES + "$", Pattern.CASE_INSENSITIVE));
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_CPU_RESOURCE_LIMIT + "$", Pattern.CASE_INSENSITIVE));
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_RESOURCE_TAGS + "$", Pattern.CASE_INSENSITIVE));
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_EXEC_MEM_LIMIT + "$", Pattern.CASE_INSENSITIVE));
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_LOAD_MEM_LIMIT + "$", Pattern.CASE_INSENSITIVE));
COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_QUOTA + ".", Pattern.CASE_INSENSITIVE));
COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_DEFAULT_LOAD_CLUSTER + "$", Pattern.CASE_INSENSITIVE));
@ -149,6 +153,14 @@ public class UserProperty implements Writable {
return Sets.newHashSet(this.commonProperties.getResourceTags());
}
public long getExecMemLimit() {
return commonProperties.getExecMemLimit();
}
public long getLoadMemLimit() {
return commonProperties.getLoadMemLimit();
}
public void setPasswordForDomain(String domain, byte[] password, boolean errOnExist) throws DdlException {
if (errOnExist && whiteList.containsDomain(domain)) {
throw new DdlException("Domain " + domain + " of user " + qualifiedUser + " already exists");
@ -170,6 +182,8 @@ public class UserProperty implements Writable {
String sqlBlockRules = this.commonProperties.getSqlBlockRules();
int cpuResourceLimit = this.commonProperties.getCpuResourceLimit();
Set<Tag> resourceTags = this.commonProperties.getResourceTags();
long execMemLimit = this.commonProperties.getExecMemLimit();
long loadMemLimit = this.commonProperties.getLoadMemLimit();
UserResource newResource = resource.getCopiedUserResource();
String newDefaultLoadCluster = defaultLoadCluster;
@ -298,6 +312,11 @@ public class UserProperty implements Writable {
throw new DdlException(PROP_RESOURCE_TAGS + " parse failed: " + e.getMessage());
}
}
} else if (keyArr[0].equalsIgnoreCase(PROP_EXEC_MEM_LIMIT)) {
// set property "exec_mem_limit" = "2147483648";
execMemLimit = getLongProperty(key, value, keyArr, PROP_EXEC_MEM_LIMIT);
} else if (keyArr[0].equalsIgnoreCase(PROP_LOAD_MEM_LIMIT)) {
loadMemLimit = getLongProperty(key, value, keyArr, PROP_LOAD_MEM_LIMIT);
} else {
throw new DdlException("Unknown user property(" + key + ")");
}
@ -309,6 +328,8 @@ public class UserProperty implements Writable {
this.commonProperties.setSqlBlockRules(sqlBlockRules);
this.commonProperties.setCpuResourceLimit(cpuResourceLimit);
this.commonProperties.setResourceTags(resourceTags);
this.commonProperties.setExecMemLimit(execMemLimit);
this.commonProperties.setExecMemLimit(loadMemLimit);
resource = newResource;
if (newDppConfigs.containsKey(newDefaultLoadCluster)) {
defaultLoadCluster = newDefaultLoadCluster;
@ -318,6 +339,25 @@ public class UserProperty implements Writable {
clusterToDppConfig = newDppConfigs;
}
private long getLongProperty(String key, String value, String[] keyArr, String propName) throws DdlException {
// eg: set property "load_mem_limit" = "2147483648";
if (keyArr.length != 1) {
throw new DdlException(propName + " format error");
}
long limit = -1;
try {
limit = Long.parseLong(value);
} catch (NumberFormatException e) {
throw new DdlException(key + " is not number");
}
// -1 means unlimited
if (limit <= 0 && limit != -1) {
throw new DdlException(key + " is not valid. Should not larger than 0 or equal to -1");
}
return limit;
}
private Set<Tag> parseLocationResoureTags(String value) throws AnalysisException {
Set<Tag> tags = Sets.newHashSet();
String[] parts = value.replaceAll(" ", "").split(",");

22
fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java
View File

@ -29,12 +29,12 @@ import org.apache.doris.resource.Tag;
import org.apache.doris.thrift.TAgentServiceVersion;
import org.apache.doris.thrift.TFetchResourceResult;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@ -250,6 +250,22 @@ public class UserPropertyMgr implements Writable {
return propertyMap.get(qualifiedUserName);
}
public long getExecMemLimit(String qualifiedUser) {
UserProperty existProperty = propertyMap.get(qualifiedUser);
if (existProperty == null) {
return -1;
}
return existProperty.getExecMemLimit();
}
public long getLoadMemLimit(String qualifiedUser) {
UserProperty existProperty = propertyMap.get(qualifiedUser);
if (existProperty == null) {
return -1;
}
return existProperty.getLoadMemLimit();
}
public static UserPropertyMgr read(DataInput in) throws IOException {
UserPropertyMgr userPropertyMgr = new UserPropertyMgr();
userPropertyMgr.readFields(in);

23
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -274,15 +274,30 @@ public class Coordinator {
}
private void setFromUserProperty(Analyzer analyzer) {
// set cpu resource limit
String qualifiedUser = analyzer.getQualifiedUser();
int limit = Catalog.getCurrentCatalog().getAuth().getCpuResourceLimit(qualifiedUser);
if (limit > 0) {
// set cpu resource limit
int cpuLimit = Catalog.getCurrentCatalog().getAuth().getCpuResourceLimit(qualifiedUser);
if (cpuLimit > 0) {
// overwrite the cpu resource limit from session variable;
TResourceLimit resourceLimit = new TResourceLimit();
resourceLimit.setCpuLimit(limit);
resourceLimit.setCpuLimit(cpuLimit);
this.queryOptions.setResourceLimit(resourceLimit);
}
// set exec mem limit
long memLimit = Catalog.getCurrentCatalog().getAuth().getExecMemLimit(qualifiedUser);
if (memLimit > 0) {
// overwrite the exec_mem_limit from session variable;
this.queryOptions.setMemLimit(memLimit);
this.queryOptions.setMaxReservation(memLimit);
this.queryOptions.setInitialReservationTotalClaims(memLimit);
this.queryOptions.setBufferPoolLimit(memLimit);
}
// set load mem limit
memLimit = Catalog.getCurrentCatalog().getAuth().getLoadMemLimit(qualifiedUser);
if (memLimit > 0) {
// overwrite the load_mem_limit from session variable;
this.queryOptions.setLoadMemLimit(memLimit);
}
}
public long getJobId() {