f7b5f36da4
At present, Doris can only access the hadoop cluster with kerberos authentication enabled by broker, but Doris BE itself does not supports access to a kerberos-authenticated HDFS file. This PR hope solve the problem. When create hive external table, users just specify following properties to access the hdfs data with kerberos authentication enabled: ```sql CREATE EXTERNAL TABLE t_hive ( k1 int NOT NULL COMMENT "", k2 char(10) NOT NULL COMMENT "", k3 datetime NOT NULL COMMENT "", k5 varchar(20) NOT NULL COMMENT "", k6 double NOT NULL COMMENT "" ) ENGINE=HIVE COMMENT "HIVE" PROPERTIES ( 'hive.metastore.uris' = 'thrift://192.168.0.1:9083', 'database' = 'hive_db', 'table' = 'hive_table', 'dfs.nameservices'='hacluster', 'dfs.ha.namenodes.hacluster'='n1,n2', 'dfs.namenode.rpc-address.hacluster.n1'='192.168.0.1:8020', 'dfs.namenode.rpc-address.hacluster.n2'='192.168.0.2:8020', 'dfs.client.failover.proxy.provider.hacluster'='org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider', 'dfs.namenode.kerberos.principal'='hadoop/_HOST@REALM.COM' 'hadoop.security.authentication'='kerberos', 'hadoop.kerberos.principal'='doris_test@REALM.COM', 'hadoop.kerberos.keytab'='/path/to/doris_test.keytab' ); ``` If you want to `select into outfile` to HDFS that kerberos authentication enable, you can refer to the following SQL statement： ```sql select * from test into outfile "hdfs://tmp/outfile1" format as csv properties ( 'fs.defaultFS'='hdfs://hacluster/', 'dfs.nameservices'='hacluster', 'dfs.ha.namenodes.hacluster'='n1,n2', 'dfs.namenode.rpc-address.hacluster.n1'='192.168.0.1:8020', 'dfs.namenode.rpc-address.hacluster.n2'='192.168.0.2:8020', 'dfs.client.failover.proxy.provider.hacluster'='org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider', 'dfs.namenode.kerberos.principal'='hadoop/_HOST@REALM.COM' 'hadoop.security.authentication'='kerberos', 'hadoop.kerberos.principal'='doris_test@REALM.COM', 'hadoop.kerberos.keytab'='/path/to/doris_test.keytab' ); ```
46 lines
1.6 KiB
C++
46 lines
1.6 KiB
C++
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
#include "io/hdfs_file_reader.h"
|
|
|
|
#include <gtest/gtest.h>
|
|
|
|
#include "io/hdfs_reader_writer.h"
|
|
|
|
namespace doris {
|
|
|
|
class HdfsFileReaderTest : public testing::Test {};
|
|
|
|
TEST_F(HdfsFileReaderTest, test_connect_fail) {
|
|
THdfsParams hdfsParams;
|
|
hdfsParams.__set_fs_name("hdfs://127.0.0.9:8888"); // An invalid address
|
|
hdfsParams.__set_hdfs_kerberos_principal("somebody@TEST.COM");
|
|
hdfsParams.__set_hdfs_kerberos_keytab("/etc/keytab/doris.keytab");
|
|
std::vector<THdfsConf> confs;
|
|
THdfsConf item;
|
|
item.key = "dfs.ha.namenodes.service1";
|
|
item.value = "n1,n2";
|
|
confs.push_back(item);
|
|
hdfsParams.__set_hdfs_conf(confs);
|
|
HdfsFileReader hdfs_file_reader(hdfsParams, "/user/foo/test.data", 0);
|
|
Status status = hdfs_file_reader.open();
|
|
EXPECT_EQ(TStatusCode::INTERNAL_ERROR, status.code());
|
|
hdfs_file_reader.close();
|
|
}
|
|
|
|
} // end namespace doris
|