database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](filesystem)Use simple authentication directly in S3FileSystem for 2.1 (#43636) (#44238)

Browse Source 
bp: #43636
This commit is contained in:
wuwenchi
2024-11-22 11:45:56 +08:00
committed by GitHub
parent cd223327c8
commit 7d123edcf8
6 changed files with 261 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-common/pom.xml
View File

@ -137,6 +137,12 @@ under the License.
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aliyun</artifactId>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>${commons-collections.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<finalName>doris-fe-common</finalName>

48
fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
View File

@ -17,10 +17,14 @@
package org.apache.doris.common.security.authentication;
import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
public abstract class AuthenticationConfig {
private static final Logger LOG = LogManager.getLogger(AuthenticationConfig.class);
public static String HADOOP_USER_NAME = "hadoop.username";
public static String HADOOP_KERBEROS_PRINCIPAL = "hadoop.kerberos.principal";
public static String HADOOP_KERBEROS_KEYTAB = "hadoop.kerberos.keytab";
@ -42,6 +46,10 @@ public abstract class AuthenticationConfig {
return AuthenticationConfig.getKerberosConfig(conf, HADOOP_KERBEROS_PRINCIPAL, HADOOP_KERBEROS_KEYTAB);
}
public static AuthenticationConfig getSimpleAuthenticationConfig(Configuration conf) {
return AuthenticationConfig.createSimpleAuthenticationConfig(conf);
}
/**
* get kerberos config from hadoop conf
* @param conf config
@ -54,17 +62,35 @@ public abstract class AuthenticationConfig {
String krbKeytabKey) {
String authentication = conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, null);
if (AuthType.KERBEROS.getDesc().equals(authentication)) {
KerberosAuthenticationConfig krbConfig = new KerberosAuthenticationConfig();
krbConfig.setKerberosPrincipal(conf.get(krbPrincipalKey));
krbConfig.setKerberosKeytab(conf.get(krbKeytabKey));
krbConfig.setConf(conf);
krbConfig.setPrintDebugLog(Boolean.parseBoolean(conf.get(DORIS_KRB5_DEBUG, "false")));
return krbConfig;
} else {
// AuthType.SIMPLE
SimpleAuthenticationConfig simpleAuthenticationConfig = new SimpleAuthenticationConfig();
simpleAuthenticationConfig.setUsername(conf.get(HADOOP_USER_NAME));
return simpleAuthenticationConfig;
String principalKey = conf.get(krbPrincipalKey);
String keytabKey = conf.get(krbKeytabKey);
if (!Strings.isNullOrEmpty(principalKey) && !Strings.isNullOrEmpty(keytabKey)) {
KerberosAuthenticationConfig krbConfig = new KerberosAuthenticationConfig();
krbConfig.setKerberosPrincipal(principalKey);
krbConfig.setKerberosKeytab(keytabKey);
krbConfig.setConf(conf);
krbConfig.setPrintDebugLog(Boolean.parseBoolean(conf.get(DORIS_KRB5_DEBUG, "false")));
return krbConfig;
} else {
// Due to some historical reasons, `core-size.xml` may be stored in path:`fe/conf`,
// but this file may only contain `hadoop.security.authentication configuration`,
// and no krbPrincipalKey and krbKeytabKey,
// which will cause kerberos initialization failure.
// Now:
// if kerberos is needed, the relevant configuration can be written in the catalog properties,
// if kerberos is not needed, to prevent the influence of historical reasons,
// the following simple authentication method needs to be used.
LOG.warn("{} or {} is null or empty, fallback to simple authentication",
krbPrincipalKey, krbKeytabKey);
}
}
return createSimpleAuthenticationConfig(conf);
}
private static AuthenticationConfig createSimpleAuthenticationConfig(Configuration conf) {
// AuthType.SIMPLE
SimpleAuthenticationConfig simpleAuthenticationConfig = new SimpleAuthenticationConfig();
simpleAuthenticationConfig.setUsername(conf.get(HADOOP_USER_NAME));
return simpleAuthenticationConfig;
}
}

45
fe/fe-common/src/test/java/org/apache/doris/common/security/authentication/AuthenticationTest.java Normal file
View File

@ -0,0 +1,45 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common.security.authentication;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.junit.Assert;
import org.junit.Test;
public class AuthenticationTest {
@Test
public void testAuthConf() {
Configuration conf = new Configuration();
AuthenticationConfig conf1 = AuthenticationConfig.getKerberosConfig(conf);
Assert.assertEquals(SimpleAuthenticationConfig.class, conf1.getClass());
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
AuthenticationConfig conf2 = AuthenticationConfig.getKerberosConfig(conf);
Assert.assertEquals(SimpleAuthenticationConfig.class, conf2.getClass());
conf.set(AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL, "principal");
conf.set(AuthenticationConfig.HADOOP_KERBEROS_KEYTAB, "keytab");
AuthenticationConfig conf3 = AuthenticationConfig.getKerberosConfig(conf);
Assert.assertEquals(KerberosAuthenticationConfig.class, conf3.getClass());
}
}