database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](ranger) modify and enhance the feature of ranger access controller (#34392) (#34426)

Browse Source 
bp #34392
This commit is contained in:
Mingyu Chen
2024-05-06 17:08:47 +08:00
committed by GitHub
parent ab5ee81811
commit 3cb0deae9c
8 changed files with 297 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java
View File

@ -68,7 +68,6 @@ public abstract class RangerAccessController implements CatalogAccessController
}
}
public static void checkRequestResults(Collection<RangerAccessResult> results, String name)
throws AuthorizationException {
for (RangerAccessResult result : results) {
@ -82,7 +81,8 @@ public abstract class RangerAccessController implements CatalogAccessController
throw new AuthorizationException(String.format(
"Permission denied: user [%s] does not have privilege for [%s] command on [%s]",
result.getAccessRequest().getUser(), name,
result.getAccessRequest().getResource().getAsString().replaceAll("/", ".")));
Optional.ofNullable(result.getAccessRequest().getResource().getAsString())
.orElse("unknown resource").replaceAll("/", ".")));
}
}
}
@ -135,12 +135,27 @@ public abstract class RangerAccessController implements CatalogAccessController
if (StringUtils.isEmpty(maskType)) {
return Optional.empty();
}
String transformer = policy.getMaskTypeDef().getTransformer();
if (StringUtils.isEmpty(transformer)) {
return Optional.empty();
switch (maskType) {
case "MASK_NULL":
return Optional.of(new RangerDataMaskPolicy(currentUser, ctl, db, tbl, col, policy.getPolicyId(),
policy.getPolicyVersion(), maskType, "NULL"));
case "MASK_NONE":
return Optional.empty();
case "CUSTOM":
String maskedValue = policy.getMaskedValue();
if (StringUtils.isEmpty(maskedValue)) {
return Optional.empty();
}
return Optional.of(new RangerDataMaskPolicy(currentUser, ctl, db, tbl, col, policy.getPolicyId(),
policy.getPolicyVersion(), maskType, maskedValue.replace("{col}", col)));
default:
String transformer = policy.getMaskTypeDef().getTransformer();
if (StringUtils.isEmpty(transformer)) {
return Optional.empty();
}
return Optional.of(new RangerDataMaskPolicy(currentUser, ctl, db, tbl, col, policy.getPolicyId(),
policy.getPolicyVersion(), maskType, transformer.replace("{col}", col)));
}
return Optional.of(new RangerDataMaskPolicy(currentUser, ctl, db, tbl, col, policy.getPolicyId(),
policy.getPolicyVersion(), maskType, transformer.replace("{col}", col)));
}
protected abstract RangerAccessRequestImpl createRequest(UserIdentity currentUser);

6
fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/DorisAccessType.java
View File

@ -46,9 +46,11 @@ public enum DorisAccessType {
} else if (priv == PrivPredicate.SHOW_VIEW) {
return SHOW_VIEW;
} else if (priv == PrivPredicate.SHOW_RESOURCES) {
return SHOW_RESOURCES;
// For Ranger, there is only USAGE priv for RESOURCE and WORKLOAD_GROUP.
// So when checking SHOW_XXX priv, convert it to USAGE priv and pass to Ranger.
return USAGE;
} else if (priv == PrivPredicate.SHOW_WORKLOAD_GROUP) {
return SHOW_WORKLOAD_GROUP;
return USAGE;
} else if (priv == PrivPredicate.GRANT) {
return GRANT;
} else if (priv == PrivPredicate.ADMIN) {

27
fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/RangerDorisAccessController.java
View File

@ -23,7 +23,9 @@ import org.apache.doris.catalog.authorizer.ranger.RangerAccessController;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AuthorizationException;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr;
import com.google.common.annotations.VisibleForTesting;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
@ -41,7 +43,7 @@ import java.util.stream.Collectors;
public class RangerDorisAccessController extends RangerAccessController {
private static final Logger LOG = LogManager.getLogger(RangerDorisAccessController.class);
private RangerDorisPlugin dorisPlugin;
private RangerBasePlugin dorisPlugin;
// private static ScheduledThreadPoolExecutor logFlushTimer = ThreadPoolManager.newDaemonScheduledThreadPool(1,
// "ranger-doris-audit-log-flusher-timer", true);
// private RangerHiveAuditHandler auditHandler;
@ -53,6 +55,11 @@ public class RangerDorisAccessController extends RangerAccessController {
// logFlushTimer.scheduleAtFixedRate(new RangerHiveAuditLogFlusher(auditHandler), 10, 20L, TimeUnit.SECONDS);
}
@VisibleForTesting
public RangerDorisAccessController(RangerBasePlugin plugin) {
dorisPlugin = plugin;
}
private RangerAccessRequestImpl createRequest(UserIdentity currentUser, DorisAccessType accessType) {
RangerAccessRequestImpl request = createRequest(currentUser);
request.setAction(accessType.name());
@ -117,6 +124,10 @@ public class RangerDorisAccessController extends RangerAccessController {
@Override
public boolean checkDbPriv(UserIdentity currentUser, String ctl, String db, PrivPredicate wanted) {
boolean res = checkCtlPriv(currentUser, ctl, wanted);
if (res) {
return true;
}
RangerDorisResource resource = new RangerDorisResource(DorisObjectType.DATABASE, ctl,
ClusterNamespace.getNameFromFullName(db));
return checkPrivilege(currentUser, DorisAccessType.toAccessType(wanted), resource);
@ -124,6 +135,11 @@ public class RangerDorisAccessController extends RangerAccessController {
@Override
public boolean checkTblPriv(UserIdentity currentUser, String ctl, String db, String tbl, PrivPredicate wanted) {
boolean res = checkDbPriv(currentUser, ctl, db, wanted);
if (res) {
return true;
}
RangerDorisResource resource = new RangerDorisResource(DorisObjectType.TABLE,
ctl, ClusterNamespace.getNameFromFullName(db), tbl);
return checkPrivilege(currentUser, DorisAccessType.toAccessType(wanted), resource);
@ -132,6 +148,11 @@ public class RangerDorisAccessController extends RangerAccessController {
@Override
public void checkColsPriv(UserIdentity currentUser, String ctl, String db, String tbl, Set<String> cols,
PrivPredicate wanted) throws AuthorizationException {
boolean res = checkTblPriv(currentUser, ctl, db, tbl, wanted);
if (res) {
return;
}
List<RangerDorisResource> resources = new ArrayList<>();
for (String col : cols) {
RangerDorisResource resource = new RangerDorisResource(DorisObjectType.COLUMN,
@ -150,6 +171,10 @@ public class RangerDorisAccessController extends RangerAccessController {
@Override
public boolean checkWorkloadGroupPriv(UserIdentity currentUser, String workloadGroupName, PrivPredicate wanted) {
// For compatibility with older versions, it is not needed to check the privileges of the default group.
if (WorkloadGroupMgr.DEFAULT_GROUP_NAME.equals(workloadGroupName)) {
return true;
}
RangerDorisResource resource = new RangerDorisResource(DorisObjectType.WORKLOAD_GROUP, workloadGroupName);
return checkPrivilege(currentUser, DorisAccessType.toAccessType(wanted), resource);
}

4
fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java
View File

@ -184,7 +184,9 @@ public class RangerHiveAccessController extends RangerAccessController {
@Override
public boolean checkWorkloadGroupPriv(UserIdentity currentUser, String workloadGroupName, PrivPredicate wanted) {
return false;
// Not support workload group privilege in ranger hive plugin.
// So always return true to pass the check
return true;
}
@Override

11
fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
View File

@ -55,8 +55,15 @@ public class PrintableMap<K, V> {
SENSITIVE_KEY.add("bos_secret_accesskey");
SENSITIVE_KEY.add("jdbc.password");
SENSITIVE_KEY.add("elasticsearch.password");
SENSITIVE_KEY.addAll(Arrays.asList(S3Properties.SECRET_KEY, ObsProperties.SECRET_KEY, OssProperties.SECRET_KEY,
GCSProperties.SECRET_KEY, CosProperties.SECRET_KEY, GlueProperties.SECRET_KEY, MCProperties.SECRET_KEY,
SENSITIVE_KEY.addAll(Arrays.asList(
S3Properties.SECRET_KEY,
S3Properties.Env.SECRET_KEY,
ObsProperties.SECRET_KEY,
OssProperties.SECRET_KEY,
GCSProperties.SECRET_KEY,
CosProperties.SECRET_KEY,
GlueProperties.SECRET_KEY,
MCProperties.SECRET_KEY,
DLFProperties.SECRET_KEY));
HIDDEN_KEY = Sets.newHashSet();
HIDDEN_KEY.addAll(S3Properties.Env.FS_KEYS);

3
fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java
View File

@ -386,12 +386,13 @@ public class Auth implements Writable {
public boolean checkWorkloadGroupPriv(UserIdentity currentUser, String workloadGroupName, PrivPredicate wanted) {
readLock();
try {
Set<Role> roles = getRolesByUserWithLdap(currentUser);
// currently stream load not support ip based auth, so normal should not auth temporary
// need remove later
if (WorkloadGroupMgr.DEFAULT_GROUP_NAME.equals(workloadGroupName)) {
return true;
}
Set<Role> roles = getRolesByUserWithLdap(currentUser);
for (Role role : roles) {
if (role.checkWorkloadGroupPriv(workloadGroupName, wanted)) {
return true;

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java
View File

@ -68,7 +68,7 @@ public class DateTrunc extends ScalarFunction
final String constParam = ((VarcharLiteral) getArgument(1)).getStringValue().toLowerCase();
if (!Lists.newArrayList("year", "quarter", "month", "week", "day", "hour", "minute", "second")
.contains(constParam)) {
throw new AnalysisException("date_trunc function second param only support argument is"
throw new AnalysisException("date_trunc function second param only support argument is "
+ "year|quarter|month|week|day|hour|minute|second");
}
}

230
fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/RangerTest.java Normal file
View File

@ -0,0 +1,230 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.mysql.privilege;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.authorizer.ranger.doris.RangerDorisAccessController;
import org.apache.doris.catalog.authorizer.ranger.doris.RangerDorisResource;
import org.apache.doris.common.AuthorizationException;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
import org.apache.ranger.plugin.policyengine.RangerAccessResource;
import org.apache.ranger.plugin.policyengine.RangerAccessResult;
import org.apache.ranger.plugin.policyengine.RangerAccessResultProcessor;
import org.apache.ranger.plugin.service.RangerBasePlugin;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.Set;
public class RangerTest {
public static class DorisTestPlugin extends RangerBasePlugin {
public DorisTestPlugin(String serviceName) {
super(serviceName, null, null);
// super.init();
}
@Override
public Collection<RangerAccessResult> isAccessAllowed(Collection<RangerAccessRequest> requests) {
List<RangerAccessResult> results = Lists.newArrayList();
for (RangerAccessRequest request : requests) {
RangerAccessResult result = isAccessAllowed(request);
if (result != null) {
results.add(result);
}
}
return results;
}
@Override
public RangerAccessResult isAccessAllowed(RangerAccessRequest request) {
RangerAccessResource resource = request.getResource();
String ctl = (String) resource.getValue(RangerDorisResource.KEY_CATALOG);
String db = (String) resource.getValue(RangerDorisResource.KEY_DATABASE);
String tbl = (String) resource.getValue(RangerDorisResource.KEY_TABLE);
String col = (String) resource.getValue(RangerDorisResource.KEY_COLUMN);
String rs = (String) resource.getValue(RangerDorisResource.KEY_RESOURCE);
String wg = (String) resource.getValue(RangerDorisResource.KEY_WORKLOAD_GROUP);
String user = request.getUser();
return returnAccessResult(request, ctl, db, tbl, col, rs, wg, user);
}
@Override
public RangerAccessResult evalDataMaskPolicies(RangerAccessRequest request,
RangerAccessResultProcessor resultProcessor) {
RangerAccessResource resource = request.getResource();
String ctl = (String) resource.getValue(RangerDorisResource.KEY_CATALOG);
String db = (String) resource.getValue(RangerDorisResource.KEY_DATABASE);
String tbl = (String) resource.getValue(RangerDorisResource.KEY_TABLE);
String col = (String) resource.getValue(RangerDorisResource.KEY_COLUMN);
RangerAccessResult result = new RangerAccessResult(2, "test", null, request);
result.setPolicyVersion(1L);
if ("ctl1".equals(ctl) && "db1".equals(db) && "tbl1".equals(tbl) && "col1".equals(col)) {
result.addAdditionalInfo("maskType", "MASK_NULL");
} else if ("ctl1".equals(ctl) && "db1".equals(db) && "tbl1".equals(tbl) && "col2".equals(col)) {
result.addAdditionalInfo("maskType", "MASK_NONE");
} else if ("ctl1".equals(ctl) && "db1".equals(db) && "tbl1".equals(tbl) && "col3".equals(col)) {
result.addAdditionalInfo("maskType", "CUSTOM");
result.addAdditionalInfo("maskedValue", "hex({col})");
} else {
// Unable to mock other mask type
result.addAdditionalInfo("maskType", "");
}
return result;
}
private RangerAccessResult returnAccessResult(
RangerAccessRequest request, String ctl, String db, String tbl,
String col, String rs, String wg, String user) {
RangerAccessResult result = new RangerAccessResult(1, "test", null, request);
if (!Strings.isNullOrEmpty(wg)) {
result.setIsAllowed(wg.equals("wg1"));
} else if (!Strings.isNullOrEmpty(rs)) {
result.setIsAllowed(wg.equals("rs1"));
} else if (!Strings.isNullOrEmpty(col)) {
boolean res = ("ctl1".equals(ctl) && "db1".equals(db) && "tbl1".equals(tbl) && "col1".equals(col))
|| ("ctl1".equals(ctl) && "db1".equals(db) && "tbl1".equals(tbl) && "col2".equals(col));
result.setIsAllowed(res);
} else if (!Strings.isNullOrEmpty(tbl)) {
result.setIsAllowed("ctl2".equals(ctl) && "db2".equals(db) && "tbl2".equals(tbl));
} else if (!Strings.isNullOrEmpty(db)) {
result.setIsAllowed("ctl3".equals(ctl) && "db3".equals(db));
} else if (!Strings.isNullOrEmpty(ctl)) {
result.setIsAllowed("ctl4".equals(ctl));
} else {
result.setIsAllowed(false);
}
return result;
}
}
// Does not have priv on ctl1.db1.tbl1.col3
@Test(expected = AuthorizationException.class)
public void testNoAuthCol() throws AuthorizationException {
DorisTestPlugin plugin = new DorisTestPlugin("test");
RangerDorisAccessController ac = new RangerDorisAccessController(plugin);
UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%");
Set<String> cols = Sets.newHashSet();
cols.add("col1");
cols.add("col3");
ac.checkColsPriv(ui, "ctl1", "db1", "tbl1", cols, PrivPredicate.SELECT);
}
// Have priv on ctl1.db1.tbl1.col1 & col2
@Test
public void testAuthCol() throws AuthorizationException {
DorisTestPlugin plugin = new DorisTestPlugin("test");
RangerDorisAccessController ac = new RangerDorisAccessController(plugin);
UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%");
Set<String> cols = Sets.newHashSet();
cols.add("col1");
cols.add("col2");
ac.checkColsPriv(ui, "ctl1", "db1", "tbl1", cols, PrivPredicate.SELECT);
}
// Have priv on ctl2.db2.tbl2, so when checking auth on col1 & col2, can pass
@Test
public void testUsingTableAuthAsColAuth() throws AuthorizationException {
DorisTestPlugin plugin = new DorisTestPlugin("test");
RangerDorisAccessController ac = new RangerDorisAccessController(plugin);
UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%");
Set<String> cols = Sets.newHashSet();
cols.add("col1");
cols.add("col2");
ac.checkColsPriv(ui, "ctl2", "db2", "tbl2", cols, PrivPredicate.SELECT);
}
// Does not have priv on ctl2.db2.tbl3, so when checking auth on col1 & col2, can not pass
@Test(expected = AuthorizationException.class)
public void testUsingNoTableAuthAsColAuth() throws AuthorizationException {
DorisTestPlugin plugin = new DorisTestPlugin("test");
RangerDorisAccessController ac = new RangerDorisAccessController(plugin);
UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%");
Set<String> cols = Sets.newHashSet();
cols.add("col1");
cols.add("col2");
ac.checkColsPriv(ui, "ctl2", "db2", "tbl3", cols, PrivPredicate.SELECT);
}
// Have priv on ctl3.db3, so when checking auth on tbl1 and (tbl1.col1 & tbl1.col2), can pass
@Test
public void testUsingDbAuthAsColAndTableAuth() throws AuthorizationException {
DorisTestPlugin plugin = new DorisTestPlugin("test");
RangerDorisAccessController ac = new RangerDorisAccessController(plugin);
UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%");
Set<String> cols = Sets.newHashSet();
cols.add("col1");
cols.add("col2");
ac.checkColsPriv(ui, "ctl3", "db3", "tbl1", cols, PrivPredicate.SELECT);
ac.checkTblPriv(ui, "ctl3", "db3", "tbl1", PrivPredicate.SELECT);
}
// Does not have priv on ctl2.db3, so when checking auth on col1 & col2, can not pass
@Test(expected = AuthorizationException.class)
public void testNoDbAuthAsColAndTableAuth() throws AuthorizationException {
DorisTestPlugin plugin = new DorisTestPlugin("test");
RangerDorisAccessController ac = new RangerDorisAccessController(plugin);
UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%");
Set<String> cols = Sets.newHashSet();
cols.add("col1");
cols.add("col2");
ac.checkColsPriv(ui, "ctl2", "db3", "tbl3", cols, PrivPredicate.SELECT);
}
// Have priv on ctl4, so when checking auth on objs under ctl4, can pass
@Test
public void testUsingCtlAuthAsColAndTableAndDbAuth() throws AuthorizationException {
DorisTestPlugin plugin = new DorisTestPlugin("test");
RangerDorisAccessController ac = new RangerDorisAccessController(plugin);
UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%");
Set<String> cols = Sets.newHashSet();
cols.add("col1");
cols.add("col2");
ac.checkColsPriv(ui, "ctl4", "db1", "tbl1", cols, PrivPredicate.SELECT);
ac.checkTblPriv(ui, "ctl4", "db2", "tbl2", PrivPredicate.SELECT);
ac.checkDbPriv(ui, "ctl4", "db3", PrivPredicate.SELECT);
}
@Test
public void testDataMask() {
DorisTestPlugin plugin = new DorisTestPlugin("test");
RangerDorisAccessController ac = new RangerDorisAccessController(plugin);
UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%");
// MASK_NULL
Optional<DataMaskPolicy> policy = ac.evalDataMaskPolicy(ui, "ctl1", "db1", "tbl1", "col1");
Assertions.assertEquals("NULL", policy.get().getMaskTypeDef());
// MASK_NONE
policy = ac.evalDataMaskPolicy(ui, "ctl1", "db1", "tbl1", "col2");
Assertions.assertTrue(!policy.isPresent());
// CUSTOM
policy = ac.evalDataMaskPolicy(ui, "ctl1", "db1", "tbl1", "col3");
Assertions.assertEquals("hex(col3)", policy.get().getMaskTypeDef());
// Others
policy = ac.evalDataMaskPolicy(ui, "ctl1", "db1", "tbl1", "col4");
Assertions.assertTrue(!policy.isPresent());
}
}