[deps] libhdfs3 build enable kerberos support (#9524)

Currently, the libhdfs3 library integrated by doris BE does not support accessing the cluster with kerberos authentication enabled, and found that kerberos-related dependencies（gsasl and krb5） were not added when build libhdfs3. so, this pr will enable kerberos support and rebuild libhdfs3 with dependencies gsasl and krb5: - gsasl version: 1.8.0 - krb5 version: 1.19
2022-05-22 20:58:19 +08:00
parent 97fad7a2ff
commit b3a2a92bf5
7 changed files with 139 additions and 11 deletions
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@ -85,6 +85,7 @@ set(BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(ENV{DORIS_HOME} "${BASE_DIR}/../")
 set(BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(THIRDPARTY_DIR "$ENV{DORIS_THIRDPARTY}/installed/")
+set(HDFS3_KRB5_INSTALL_DIR "$ENV{DORIS_THIRDPARTY}/installed/libhdfs_with_kerberos/")
 set(GENSRC_DIR "${BASE_DIR}/../gensrc/build/")
 set(SRC_DIR "${BASE_DIR}/src/")
 set(TEST_DIR "${CMAKE_SOURCE_DIR}/test/")
@ -97,6 +98,7 @@ else()
 endif()
 message(STATUS "make test: ${MAKE_TEST}")
 option(WITH_MYSQL "Support access MySQL" ON)
+option(WITH_KERBEROS "Enable or disable Kereberos support" ${WITH_KERBEROS})

 # Check gcc
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@ -328,17 +330,40 @@ set_target_properties(minizip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
 add_library(idn STATIC IMPORTED)
 set_target_properties(idn PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libidn.a)

-add_library(gsasl STATIC IMPORTED)
-set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
-
 add_library(breakpad STATIC IMPORTED)
 set_target_properties(breakpad PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libbreakpad_client.a)

-if (ARCH_AMD64)
-    # libhdfs3 only support x86 or amd64
+if (WITH_KERBEROS)
+    # kerberos lib for libhdfs3 
+    add_library(gsasl STATIC IMPORTED)
+    set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgsasl.a)
+
+    add_library(krb5support STATIC IMPORTED)
+    set_target_properties(krb5support PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5support.a)
+    
+    add_library(krb5 STATIC IMPORTED)
+    set_target_properties(krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5.a)
+
+    add_library(com_err STATIC IMPORTED)
+    set_target_properties(com_err PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libcom_err.a)
+
+    add_library(k5crypto STATIC IMPORTED)
+    set_target_properties(k5crypto PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libk5crypto.a)
+
+    add_library(gssapi_krb5 STATIC IMPORTED)
+    set_target_properties(gssapi_krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgssapi_krb5.a)
+
+    add_library(hdfs3 STATIC IMPORTED)
+    set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libhdfs3.a)
+else()
+    add_library(gsasl STATIC IMPORTED)
+    set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
+
    add_library(hdfs3 STATIC IMPORTED)
    set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a)
+endif()

+if (ARCH_AMD64)
    add_library(xml2 STATIC IMPORTED)
    set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a)

@ -507,6 +532,14 @@ endif()
 set(WL_START_GROUP "-Wl,--start-group")
 set(WL_END_GROUP "-Wl,--end-group")

+
+set(KRB5_LIBS
+    krb5support
+    krb5
+    com_err
+    gssapi_krb5
+    k5crypto)
+
 set(AWS_LIBS
    aws-sdk-s3
    aws-sdk-core
@ -640,6 +673,12 @@ if (WITH_MYSQL)
        )
 endif()

+if (WITH_KERBEROS)
+    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
+    	${KRB5_LIBS}
+        )
+endif()
+
 set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})

 message(STATUS "DORIS_DEPENDENCIES is ${DORIS_DEPENDENCIES}")
@ -651,6 +690,7 @@ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
    -static-libstdc++
    -static-libgcc
    -lstdc++fs
+    -lresolv
 )

 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
--- a/build.sh
+++ b/build.sh
@ -175,7 +175,9 @@ if [[ ${HELP} -eq 1 ]]; then
    usage
    exit
 fi
-
+if [[ -z ${WITH_KERBEROS} ]]; then
+    WITH_KERBEROS=ON
+fi
 # build thirdparty libraries if necessary
 if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a ]]; then
    echo "Thirdparty libraries need to be build ..."
@ -228,6 +230,7 @@ echo "Get params:
    PARALLEL            -- $PARALLEL
    CLEAN               -- $CLEAN
    WITH_MYSQL          -- $WITH_MYSQL
+    WITH_KERBEROS       -- $WITH_KERBEROS
    WITH_LZO            -- $WITH_LZO
    GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY
    USE_AVX2            -- $USE_AVX2
@ -289,6 +292,7 @@ if [ ${BUILD_BE} -eq 1 ] ; then
            -DMAKE_TEST=OFF \
            ${CMAKE_USE_CCACHE} \
            -DWITH_MYSQL=${WITH_MYSQL} \
+            -DWITH_KERBEROS=${WITH_KERBEROS} \
            -DWITH_LZO=${WITH_LZO} \
            -DUSE_LIBCPP=${USE_LIBCPP} \
            -DBUILD_META_TOOL=${BUILD_META_TOOL} \
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@ -136,6 +136,7 @@ ${CMAKE_CMD} -G "${GENERATOR}" \
    -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
    -DBUILD_META_TOOL=OFF \
    -DWITH_MYSQL=OFF \
+    -DWITH_KERBEROS=OFF \
    -DUSE_DWARF=${USE_DWARF} \
    ${CMAKE_USE_CCACHE} ../
 ${BUILD_SYSTEM} -j ${PARALLEL}
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@ -904,16 +904,46 @@ build_gsasl() {
    make -j $PARALLEL && make install
 }

+# build_gsasl2 just for libgsasl1.8.0
+build_gsasl2() {
+    check_if_source_exist $GSASL2_SOURCE
+    cd $TP_SOURCE_DIR/$GSASL2_SOURCE
+    mkdir -p $BUILD_DIR && cd $BUILD_DIR
+    ../configure --prefix=$HDFS3_KRB5_INSTALL_DIR --with-gssapi-impl=mit --enable-shared=no --with-pic --with-libidn-prefix=$TP_INSTALL_DIR
+    make -j $PARALLEL && make install
+}
+
+# krb5
+build_krb5() {
+    check_if_source_exist $KRB5_SOURCE
+    cd $TP_SOURCE_DIR/$KRB5_SOURCE/src
+    mkdir -p $BUILD_DIR && cd $BUILD_DIR
+    CFLAGS="-fcommon" \
+    ../configure --prefix=$HDFS3_KRB5_INSTALL_DIR --disable-shared --enable-static
+    make -j $PARALLEL && make install
+}
+
 # hdfs3
 build_hdfs3() {
    check_if_source_exist $HDFS3_SOURCE
    cd $TP_SOURCE_DIR/$HDFS3_SOURCE
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
-    ../bootstrap --dependency=$TP_INSTALL_DIR --prefix=$TP_INSTALL_DIR
+    mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
+    # build libhdfs3 without kerberos
+    ../bootstrap --dependency="$TP_INSTALL_DIR" --prefix=$TP_INSTALL_DIR
    make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
    make install
 }

+# hdfs3_with_kerberos
+build_hdfs3_with_kerberos() {
+    check_if_source_exist $HDFS3_SOURCE
+    cd $TP_SOURCE_DIR/$HDFS3_SOURCE
+    mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
+    # build libhdfs3 with kerberos support
+    ../bootstrap --dependency="$HDFS3_KRB5_INSTALL_DIR:$TP_INSTALL_DIR -DWITH_KERBEROS=true" --prefix=$HDFS3_KRB5_INSTALL_DIR
+    make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
+    make install
+}
 # benchmark
 build_benchmark() {
    check_if_source_exist $BENCHMARK_SOURCE
@ -999,7 +1029,10 @@ build_lzma
 build_xml2
 build_idn
 build_gsasl
+build_gsasl2
+build_krb5
 build_hdfs3
+build_hdfs3_with_kerberos
 build_benchmark
 build_breakpad
 build_simdjson
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@ -156,7 +156,7 @@ do
 done
 echo "===== Downloading thirdparty archives...done"

-# check if all tp archievs exists
+# check if all tp archives exists
 echo "===== Checking all thirdpart archives..."
 for TP_ARCH in ${TP_ARCHIVES[*]}
 do
@ -271,6 +271,16 @@ fi
 cd -
 echo "Finished patching $S2_SOURCE"

+# gsasl2 patch to fix link error such as mutilple func defination
+# when link target with kerberos
+cd $TP_SOURCE_DIR/$GSASL2_SOURCE
+if [ ! -f $PATCHED_MARK ]; then
+    patch -p1 < $TP_PATCH_DIR/libgsasl-1.8.0.patch
+    touch $PATCHED_MARK
+fi
+cd -
+echo "Finished patching $GSASL2_SOURCE"
+
 # hdfs3 patch to fix compile error
 cd $TP_SOURCE_DIR/$HDFS3_SOURCE
 if [ ! -f $PATCHED_MARK ]; then
--- a/thirdparty/patches/libgsasl-1.8.0.patch
+++ b/thirdparty/patches/libgsasl-1.8.0.patch
@ -0,0 +1,24 @@
+--- a/config.h.in	2012-05-29 01:11:28.000000000 +0800
+++ b/config.h.in	2022-05-11 18:06:02.000000000 +0800
+@@ -176,7 +176,7 @@
+ #undef HAVE_GSSAPI_H
+
+ /* Define to 1 if you have the `GSS_C_NT_HOSTBASED_SERVICE' function. */
+-#undef HAVE_GSS_C_NT_HOSTBASED_SERVICE
+#define HAVE_GSS_C_NT_HOSTBASED_SERVICE 1
+
+ /* Define to 1 if you have the `gss_decapsulate_token' function. */
+ #undef HAVE_GSS_DECAPSULATE_TOKEN
+@@ -185,10 +185,10 @@
+ #undef HAVE_GSS_ENCAPSULATE_TOKEN
+
+ /* Define to 1 if you have the `gss_inquire_mech_for_saslname' function. */
+-#undef HAVE_GSS_INQUIRE_MECH_FOR_SASLNAME
+#define HAVE_GSS_INQUIRE_MECH_FOR_SASLNAME 1
+
+ /* Define to 1 if you have the `gss_oid_equal' function. */
+-#undef HAVE_GSS_OID_EQUAL
+#define HAVE_GSS_OID_EQUAL 1
+
+ /* Define if you have the iconv() function and it works. */
+ #undef HAVE_ICONV
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@ -32,6 +32,9 @@ export TP_SOURCE_DIR=$TP_DIR/src
 # thirdparties will be installed to here
 export TP_INSTALL_DIR=$TP_DIR/installed

+# libhdfs3-with-kerberos will be installed to here
+export HDFS3_KRB5_INSTALL_DIR=$TP_INSTALL_DIR/libhdfs_with_kerberos
+
 # patches for all thirdparties
 export TP_PATCH_DIR=$TP_DIR/patches

@ -44,8 +47,8 @@ export TP_LIB_DIR=$TP_INSTALL_DIR/lib
 # all java libraries will be unpacked to here
 export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar

-# source of all dependencies
-export REPOSITORY_URL=https://doris-thirdparty-repo.bj.bcebos.com/thirdparty
+# source of all dependencies, default unuse it
+# export REPOSITORY_URL=https://doris-thirdparty-repo.bj.bcebos.com/thirdparty

 #####################################################
 # Download url, filename and unpaced filename
@ -326,6 +329,17 @@ GSASL_NAME="libgsasl-1.10.0.tar.gz"
 GSASL_SOURCE="libgsasl-1.10.0"
 GSASL_MD5SUM="9c8fc632da4ce108fb7581b33de2a5ce"

+GSASL2_DOWNLOAD="https://ftp.gnu.org/gnu/gsasl/libgsasl-1.8.0.tar.gz"
+GSASL2_NAME="libgsasl-1.8.0.tar.gz"
+GSASL2_SOURCE="libgsasl-1.8.0"
+GSASL2_MD5SUM="5dbdf859f6e60e05813370e2b193b92b"
+
+# krb5
+KRB5_DOWNLOAD="https://kerberos.org/dist/krb5/1.19/krb5-1.19.tar.gz"
+KRB5_NAME="krb5-1.19.tar.gz"
+KRB5_SOURCE="krb5-1.19"
+KRB5_MD5SUM="aaf18447a5a014aa3b7e81814923f4c9"
+
 # hdfs3
 HDFS3_DOWNLOAD="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/libhdfs3-master.zip"
 HDFS3_NAME="libhdfs3-master.zip"
@ -423,6 +437,8 @@ LZMA
 XML2
 IDN
 GSASL
+GSASL2
+KRB5
 HDFS3
 LIBDIVIDE
 PDQSORT