database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](user-property) Support user level exec_mem_limit and load_mem_limit (#8365)

Browse Source 
```
SET PROPERTY FOR 'jack' 'exec_mem_limit' = '2147483648';
SET PROPERTY FOR 'jack' 'load_mem_limit' = '2147483648';
```
The user level property will overwrite the value in session variables.
This commit is contained in:
Mingyu Chen
2022-03-11 17:20:09 +08:00
committed by GitHub
parent 68dd799796
commit e403dbc38c
7 changed files with 139 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/en/sql-reference/sql-statements/Account Management/SET PROPERTY.md
View File

@ -42,10 +42,14 @@ max_user_connections: Maximum number of connections.
max_query_instances: Maximum number of query instance user can use when query.
sql_block_rules: set sql block rules。After setting, if the query user execute match the rules, it will be rejected.
cpu_resource_limit: limit the cpu resource usage of a query. See session variable `cpu_resource_limit`.
exec_mem_limit: Limit the memory usage of the query. See the description of the session variable `exec_mem_limit` for details. -1 means not set.
load_mem_limit: Limit memory usage for imports. See the introduction of the session variable `load_mem_limit` for details. -1 means not set.
resource.cpu_share: cpu resource assignment.(Derepcated)
Load_cluster. {cluster_name}. priority: assigns priority to a specified cluster, which can be HIGH or NORMAL
resource_tags: Specify the user's resource tag permissions.
> Notice: The `cpu_resource_limit`, `exec_mem_limit`, and `load_mem_limit` properties default to the values in the session variables if they are not set.
Ordinary user rights:
Quota.normal: Resource allocation at the normal level.
Quota.high: Resource allocation at the high level.
@ -93,6 +97,12 @@ SET PROPERTY FOR 'jack' 'cpu_resource_limit' = '2';
11. Modify user's resource tag permission
SET PROPERTY FOR 'jack' 'resource_tags.location' = 'group_a, group_b';
12. modify the user's query memory usage limit in bytes
SET PROPERTY FOR 'jack' 'exec_mem_limit' = '2147483648';
13. modify the user's import memory usage limit in bytes
SET PROPERTY FOR 'jack' 'load_mem_limit' = '2147483648';
## keyword
SET, PROPERTY

12
docs/zh-CN/sql-reference/sql-statements/Account Management/SET PROPERTY.md
View File

@ -41,11 +41,15 @@ under the License.
max_user_connections: 最大连接数。
max_query_instances: 用户同一时间点执行查询可以使用的instance个数。
sql_block_rules: 设置 sql block rules。设置后，该用户发送的查询如果匹配规则，则会被拒绝。
cpu_resource_limit: 限制查询的cpu资源。详见会话变量 `cpu_resource_limit` 的介绍。
cpu_resource_limit: 限制查询的cpu资源。详见会话变量 `cpu_resource_limit` 的介绍。-1 表示未设置。
exec_mem_limit: 限制查询的内存使用。详见会话变量 `exec_mem_limit` 的介绍。-1 表示未设置。
load_mem_limit: 限制导入的内存使用。详见会话变量 `load_mem_limit` 的介绍。-1 表示未设置。
resource.cpu_share: cpu资源分配。（已废弃）
load_cluster.{cluster_name}.priority: 为指定的cluster分配优先级，可以为 HIGH 或 NORMAL
resource_tags：指定用户的资源标签权限。
注：`cpu_resource_limit`, `exec_mem_limit`, `load_mem_limit` 三个属性如果未设置，则默认使用会话变量中值。
普通用户权限：
quota.normal: normal级别的资源分配。
quota.high: high级别的资源分配。
@ -93,6 +97,12 @@ under the License.
11. 修改用户的资源标签权限
SET PROPERTY FOR 'jack' 'resource_tags.location' = 'group_a, group_b';
12. 修改用户的查询内存使用限制，单位字节
SET PROPERTY FOR 'jack' 'exec_mem_limit' = '2147483648';
13. 修改用户的导入内存使用限制，单位字节
SET PROPERTY FOR 'jack' 'load_mem_limit' = '2147483648';
## keyword
SET, PROPERTY

22
fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java
View File

@ -47,6 +47,12 @@ public class CommonUserProperties implements Writable {
// The tag of the resource that the user is allowed to use
@SerializedName("resourceTags")
private Set<Tag> resourceTags = Sets.newHashSet();
// user level exec_mem_limit, if > 0, will overwrite the exec_mem_limit in session variable
@SerializedName("execMemLimit")
private long execMemLimit = -1;
// user level load_mem_limit, if > 0, will overwrite the load_mem_limit in session variable
@SerializedName("loadMemLimit")
private long loadMemLimit = -1;
private String[] sqlBlockRulesSplit = {};
@ -100,6 +106,22 @@ public class CommonUserProperties implements Writable {
return resourceTags;
}
public long getExecMemLimit() {
return execMemLimit;
}
public void setExecMemLimit(long execMemLimit) {
this.execMemLimit = execMemLimit;
}
public long getLoadMemLimit() {
return loadMemLimit;
}
public void setLoadMemLimit(long loadMemLimit) {
this.loadMemLimit = loadMemLimit;
}
public static CommonUserProperties read(DataInput in) throws IOException {
String json = Text.readString(in);
CommonUserProperties commonUserProperties = GsonUtils.GSON.fromJson(json, CommonUserProperties.class);

18
fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/PaloAuth.java
View File

@ -1172,6 +1172,24 @@ public class PaloAuth implements Writable {
}
}
public long getExecMemLimit(String qualifiedUser) {
readLock();
try {
return propertyMgr.getExecMemLimit(qualifiedUser);
} finally {
readUnlock();
}
}
public long getLoadMemLimit(String qualifiedUser) {
readLock();
try {
return propertyMgr.getLoadMemLimit(qualifiedUser);
} finally {
readUnlock();
}
}
public void getAllDomains(Set<String> allDomains) {
readLock();
try {

40
fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserProperty.java
View File

@ -63,6 +63,8 @@ public class UserProperty implements Writable {
private static final String PROP_RESOURCE = "resource";
private static final String PROP_SQL_BLOCK_RULES = "sql_block_rules";
private static final String PROP_CPU_RESOURCE_LIMIT = "cpu_resource_limit";
private static final String PROP_EXEC_MEM_LIMIT = "exec_mem_limit";
private static final String PROP_LOAD_MEM_LIMIT = "load_mem_limit";
// advanced properties end
private static final String PROP_LOAD_CLUSTER = "load_cluster";
@ -107,6 +109,8 @@ public class UserProperty implements Writable {
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_SQL_BLOCK_RULES + "$", Pattern.CASE_INSENSITIVE));
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_CPU_RESOURCE_LIMIT + "$", Pattern.CASE_INSENSITIVE));
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_RESOURCE_TAGS + "$", Pattern.CASE_INSENSITIVE));
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_EXEC_MEM_LIMIT + "$", Pattern.CASE_INSENSITIVE));
ADVANCED_PROPERTIES.add(Pattern.compile("^" + PROP_LOAD_MEM_LIMIT + "$", Pattern.CASE_INSENSITIVE));
COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_QUOTA + ".", Pattern.CASE_INSENSITIVE));
COMMON_PROPERTIES.add(Pattern.compile("^" + PROP_DEFAULT_LOAD_CLUSTER + "$", Pattern.CASE_INSENSITIVE));
@ -149,6 +153,14 @@ public class UserProperty implements Writable {
return Sets.newHashSet(this.commonProperties.getResourceTags());
}
public long getExecMemLimit() {
return commonProperties.getExecMemLimit();
}
public long getLoadMemLimit() {
return commonProperties.getLoadMemLimit();
}
public void setPasswordForDomain(String domain, byte[] password, boolean errOnExist) throws DdlException {
if (errOnExist && whiteList.containsDomain(domain)) {
throw new DdlException("Domain " + domain + " of user " + qualifiedUser + " already exists");
@ -170,6 +182,8 @@ public class UserProperty implements Writable {
String sqlBlockRules = this.commonProperties.getSqlBlockRules();
int cpuResourceLimit = this.commonProperties.getCpuResourceLimit();
Set<Tag> resourceTags = this.commonProperties.getResourceTags();
long execMemLimit = this.commonProperties.getExecMemLimit();
long loadMemLimit = this.commonProperties.getLoadMemLimit();
UserResource newResource = resource.getCopiedUserResource();
String newDefaultLoadCluster = defaultLoadCluster;
@ -298,6 +312,11 @@ public class UserProperty implements Writable {
throw new DdlException(PROP_RESOURCE_TAGS + " parse failed: " + e.getMessage());
}
}
} else if (keyArr[0].equalsIgnoreCase(PROP_EXEC_MEM_LIMIT)) {
// set property "exec_mem_limit" = "2147483648";
execMemLimit = getLongProperty(key, value, keyArr, PROP_EXEC_MEM_LIMIT);
} else if (keyArr[0].equalsIgnoreCase(PROP_LOAD_MEM_LIMIT)) {
loadMemLimit = getLongProperty(key, value, keyArr, PROP_LOAD_MEM_LIMIT);
} else {
throw new DdlException("Unknown user property(" + key + ")");
}
@ -309,6 +328,8 @@ public class UserProperty implements Writable {
this.commonProperties.setSqlBlockRules(sqlBlockRules);
this.commonProperties.setCpuResourceLimit(cpuResourceLimit);
this.commonProperties.setResourceTags(resourceTags);
this.commonProperties.setExecMemLimit(execMemLimit);
this.commonProperties.setExecMemLimit(loadMemLimit);
resource = newResource;
if (newDppConfigs.containsKey(newDefaultLoadCluster)) {
defaultLoadCluster = newDefaultLoadCluster;
@ -318,6 +339,25 @@ public class UserProperty implements Writable {
clusterToDppConfig = newDppConfigs;
}
private long getLongProperty(String key, String value, String[] keyArr, String propName) throws DdlException {
// eg: set property "load_mem_limit" = "2147483648";
if (keyArr.length != 1) {
throw new DdlException(propName + " format error");
}
long limit = -1;
try {
limit = Long.parseLong(value);
} catch (NumberFormatException e) {
throw new DdlException(key + " is not number");
}
// -1 means unlimited
if (limit <= 0 && limit != -1) {
throw new DdlException(key + " is not valid. Should not larger than 0 or equal to -1");
}
return limit;
}
private Set<Tag> parseLocationResoureTags(String value) throws AnalysisException {
Set<Tag> tags = Sets.newHashSet();
String[] parts = value.replaceAll(" ", "").split(",");

22
fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java
View File

@ -29,12 +29,12 @@ import org.apache.doris.resource.Tag;
import org.apache.doris.thrift.TAgentServiceVersion;
import org.apache.doris.thrift.TFetchResourceResult;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@ -250,6 +250,22 @@ public class UserPropertyMgr implements Writable {
return propertyMap.get(qualifiedUserName);
}
public long getExecMemLimit(String qualifiedUser) {
UserProperty existProperty = propertyMap.get(qualifiedUser);
if (existProperty == null) {
return -1;
}
return existProperty.getExecMemLimit();
}
public long getLoadMemLimit(String qualifiedUser) {
UserProperty existProperty = propertyMap.get(qualifiedUser);
if (existProperty == null) {
return -1;
}
return existProperty.getLoadMemLimit();
}
public static UserPropertyMgr read(DataInput in) throws IOException {
UserPropertyMgr userPropertyMgr = new UserPropertyMgr();
userPropertyMgr.readFields(in);

23
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -274,15 +274,30 @@ public class Coordinator {
}
private void setFromUserProperty(Analyzer analyzer) {
// set cpu resource limit
String qualifiedUser = analyzer.getQualifiedUser();
int limit = Catalog.getCurrentCatalog().getAuth().getCpuResourceLimit(qualifiedUser);
if (limit > 0) {
// set cpu resource limit
int cpuLimit = Catalog.getCurrentCatalog().getAuth().getCpuResourceLimit(qualifiedUser);
if (cpuLimit > 0) {
// overwrite the cpu resource limit from session variable;
TResourceLimit resourceLimit = new TResourceLimit();
resourceLimit.setCpuLimit(limit);
resourceLimit.setCpuLimit(cpuLimit);
this.queryOptions.setResourceLimit(resourceLimit);
}
// set exec mem limit
long memLimit = Catalog.getCurrentCatalog().getAuth().getExecMemLimit(qualifiedUser);
if (memLimit > 0) {
// overwrite the exec_mem_limit from session variable;
this.queryOptions.setMemLimit(memLimit);
this.queryOptions.setMaxReservation(memLimit);
this.queryOptions.setInitialReservationTotalClaims(memLimit);
this.queryOptions.setBufferPoolLimit(memLimit);
}
// set load mem limit
memLimit = Catalog.getCurrentCatalog().getAuth().getLoadMemLimit(qualifiedUser);
if (memLimit > 0) {
// overwrite the load_mem_limit from session variable;
this.queryOptions.setLoadMemLimit(memLimit);
}
}
public long getJobId() {