[Thirdparty] Fix build thirdparty may be failed (#5187)

1. fix build thirdparty may be failed  in some os, because of default lib path is `lib` or`lib64` or `arrow` bulld failed by `brotil` and `zstd`
2. fix canot extract `.tar.bz2` file

be/CMakeLists.txt

@@ -64,15 +64,9 @@ option(WITH_MYSQL "Support access MySQL" ON)
 
 # Check gcc
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8.2")
-        message(FATAL_ERROR "Need GCC version at least 4.8.2")
-    endif()
-
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "7.3.0")
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7.3.0")
+        message(FATAL_ERROR "Need GCC version at least 7.3.0")
         message(STATUS "GCC version is greater than 7.3.0, disable -Werror. Be careful with compile warnings.")
-    else()
-        #   -Werror: compile warnings should be errors when using the toolchain compiler.
-        set(CXX_GCC_FLAGS "${CXX_GCC_FLAGS} -Werror")
     endif()
 elseif (NOT APPLE)
     message(FATAL_ERROR "Compiler should be GNU")

be/src/common/daemon.cpp

@@ -17,6 +17,7 @@
 
 #include "common/daemon.h"
 
+#include <signal.h>
 #include <gflags/gflags.h>
 #include <gperftools/malloc_extension.h>
 

build.sh

@@ -171,12 +171,6 @@ if [ ${BUILD_BE} -eq 1 ] ; then
     fi
     mkdir -p ${CMAKE_BUILD_DIR}
     cd ${CMAKE_BUILD_DIR}
-    GENERATOR="Unix Makefiles"
-    BUILD_SYSTEM="make"
-    if ninja --version 2>/dev/null; then
-        GENERATOR="Ninja"
-        BUILD_SYSTEM="ninja"
-    fi
     ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DMAKE_TEST=OFF -DWITH_MYSQL=${WITH_MYSQL} -DWITH_LZO=${WITH_LZO} ../
     ${BUILD_SYSTEM} -j${PARALLEL}
     ${BUILD_SYSTEM} install

docs/en/installing/compilation.md

@@ -93,7 +93,7 @@ You can try to compile Doris directly in your own Linux environment.
 
 1. System Dependence
 
-    `GCC 5.3.1+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake 3.11+`
+    `GCC 7.3+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake 3.11+ Bison 3.0+`
 
     If you are using Ubuntu 16.04 or newer, you can use the following command to install the dependencies
 
@@ -101,7 +101,7 @@ You can try to compile Doris directly in your own Linux environment.
     
     If you are using CentOS you can use the following command to install the dependencies
    
-   `sudo yum groupinstall 'Development Tools' && sudo yum install maven cmake byacc flex automake libtool bison binutils-devel zip unzip ncurses-devel curl git wget python2 glibc-static libstdc++-static`
+   `sudo yum groupinstall 'Development Tools' && sudo yum install maven cmake byacc flex automake libtool bison binutils-devel zip unzip ncurses-devel curl git wget python2 glibc-static libstdc++-static java-1.8.0-openjdk`
 
     After installation, set environment variables `PATH`, `JAVA_HOME`, etc.
 

docs/zh-CN/installing/compilation.md

@@ -93,13 +93,15 @@ under the License.
 
 1. 系统依赖
 
-    `GCC 5.3.1+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake 3.11+`
+    `GCC 7.3+, Oracle JDK 1.8+, Python 2.7+, Apache Maven 3.5+, CMake 3.11+     Bison 3.0+`
 
     如果使用Ubuntu 16.04 及以上系统 可以执行以下命令来安装依赖
     
     `sudo apt-get install build-essential openjdk-8-jdk maven cmake byacc flex automake libtool-bin bison binutils-dev libiberty-dev zip unzip libncurses5-dev curl git ninja-build python`
+
     如果是CentOS 可以执行以下命令
-    `sudo yum groupinstall 'Development Tools' && sudo yum install maven cmake byacc flex automake libtool bison binutils-devel zip unzip ncurses-devel curl git wget python2 glibc-static libstdc++-static`
+
+    `sudo yum groupinstall 'Development Tools' && sudo yum install maven cmake byacc flex automake libtool bison binutils-devel zip unzip ncurses-devel curl git wget python2 glibc-static libstdc++-static java-1.8.0-openjdk`
 
     安装完成后，自行设置环境变量 `PATH`, `JAVA_HOME` 等。
     

env.sh

@@ -55,7 +55,7 @@ if [[ -z ${DORIS_GCC_HOME} ]]; then
 fi
 
 gcc_ver=`${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion`
-required_ver="5.3.1"
+required_ver="7.3.0"
 if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$required_ver" ]]; then 
     echo "Error: GCC version (${gcc_ver}) must be greater than or equal to ${required_ver}"
     exit 1
@@ -100,3 +100,12 @@ if ! ${CMAKE_CMD} --version; then
     exit 1
 fi
 export CMAKE_CMD
+
+GENERATOR="Unix Makefiles"
+BUILD_SYSTEM="make"
+if ninja --version 2>/dev/null; then
+    GENERATOR="Ninja"
+    BUILD_SYSTEM="ninja"
+fi
+export GENERATOR
+export BUILD_SYSTEM

run-be-ut.sh

@@ -107,13 +107,6 @@ if [ ! -d ${CMAKE_BUILD_DIR} ]; then
 fi
 
 cd ${CMAKE_BUILD_DIR}
-GENERATOR="Unix Makefiles"
-BUILD_SYSTEM="make"
-if ninja --version 2>/dev/null; then
-    GENERATOR="Ninja"
-    BUILD_SYSTEM="ninja"
-fi
-
 ${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_MYSQL=OFF -DMAKE_TEST=ON -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
 ${BUILD_SYSTEM} -j${PARALLEL}
 

thirdparty/build-thirdparty.sh

@@ -68,7 +68,10 @@ else
 fi
 
 # prepare installed prefix
-mkdir -p ${TP_DIR}/installed
+mkdir -p ${TP_DIR}/installed/lib64
+pushd  ${TP_DIR}/installed/
+ln -sf lib64 lib
+popd
 
 check_prerequest() {
     local CMD=$1
@@ -178,12 +181,7 @@ build_openssl() {
     CFLAGS="-fPIC" \
     LIBDIR="lib" \
     ./Configure --prefix=$TP_INSTALL_DIR -zlib -shared ${OPENSSL_PLATFORM}
-    make -j$PARALLEL && make install
-    if [ -f $TP_INSTALL_DIR/lib64/libcrypto.a ]; then
-        mkdir -p $TP_INSTALL_DIR/lib && \
-        cp $TP_INSTALL_DIR/lib64/libcrypto.a $TP_INSTALL_DIR/lib/libcrypto.a && \
-        cp $TP_INSTALL_DIR/lib64/libssl.a $TP_INSTALL_DIR/lib/libssl.a
-    fi
+    make -j$PARALLEL && make install_sw
     # NOTE(zc): remove this dynamic library files to make libcurl static link.
     # If I don't remove this files, I don't known how to make libcurl link static library
     if [ -f $TP_INSTALL_DIR/lib64/libcrypto.so ]; then
@@ -192,12 +190,6 @@ build_openssl() {
     if [ -f $TP_INSTALL_DIR/lib64/libssl.so ]; then
         rm -rf $TP_INSTALL_DIR/lib64/libssl.so*
     fi
-    if [ -f $TP_INSTALL_DIR/lib/libcrypto.so ]; then
-        rm -rf $TP_INSTALL_DIR/lib/libcrypto.so*
-    fi
-    if [ -f $TP_INSTALL_DIR/lib/libssl.so ]; then
-        rm -rf $TP_INSTALL_DIR/lib/libssl.so*
-    fi
 }
 
 # thrift
@@ -248,8 +240,8 @@ build_llvm() {
     mkdir -p llvm-build && cd llvm-build
     rm -rf CMakeCache.txt CMakeFiles/
     LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
-    $CMAKE_CMD -DLLVM_REQUIRES_RTTI:Bool=True -DLLVM_TARGETS_TO_BUILD=${LLVM_TARGET} -DLLVM_ENABLE_TERMINFO=OFF LLVM_BUILD_LLVM_DYLIB:BOOL=OFF -DLLVM_ENABLE_PIC=true -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE="RELEASE" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/llvm ../$LLVM_SOURCE
-    make -j$PARALLEL REQUIRES_RTTI=1 && make install
+    ${CMAKE_CMD} -G "${GENERATOR}" -DLLVM_REQUIRES_RTTI:Bool=True -DLLVM_TARGETS_TO_BUILD=${LLVM_TARGET} -DLLVM_ENABLE_TERMINFO=OFF LLVM_BUILD_LLVM_DYLIB:BOOL=OFF -DLLVM_ENABLE_PIC=true -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE="RELEASE" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/llvm ../$LLVM_SOURCE
+    ${BUILD_SYSTEM} -j$PARALLEL REQUIRES_RTTI=1 && ${BUILD_SYSTEM} install
 }
 
 # protobuf
@@ -275,9 +267,9 @@ build_gflags() {
     cd $TP_SOURCE_DIR/$GFLAGS_SOURCE
     mkdir -p $BUILD_DIR && cd $BUILD_DIR
     rm -rf CMakeCache.txt CMakeFiles/
-    $CMAKE_CMD -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
+    ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
     -DCMAKE_POSITION_INDEPENDENT_CODE=On ../
-    make -j$PARALLEL && make install
+    ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
 }
 
 # glog
@@ -303,9 +295,9 @@ build_gtest() {
     cd $TP_SOURCE_DIR/$GTEST_SOURCE
     mkdir -p $BUILD_DIR && cd $BUILD_DIR
     rm -rf CMakeCache.txt CMakeFiles/
-    $CMAKE_CMD -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
+    ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
     -DCMAKE_POSITION_INDEPENDENT_CODE=On ../
-    make -j$PARALLEL && make install
+    ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
 }
 
 # rapidjson
@@ -323,23 +315,17 @@ build_snappy() {
 
     mkdir -p $BUILD_DIR && cd $BUILD_DIR
     rm -rf CMakeCache.txt CMakeFiles/
-    CFLAGS="-O3" CXXFLAGS="-O3" $CMAKE_CMD -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-    -DCMAKE_INSTALL_LIBDIR=lib64 \
+    CFLAGS="-O3" CXXFLAGS="-O3" ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
     -DCMAKE_POSITION_INDEPENDENT_CODE=On \
     -DCMAKE_INSTALL_INCLUDEDIR=$TP_INCLUDE_DIR/snappy \
     -DSNAPPY_BUILD_TESTS=0 ../
-    make -j$PARALLEL && make install
-    if [ -f $TP_INSTALL_DIR/lib64/libsnappy.a ]; then
-        mkdir -p $TP_INSTALL_DIR/lib && cp $TP_INSTALL_DIR/lib64/libsnappy.a $TP_INSTALL_DIR/lib/libsnappy.a
-
-    fi
+    ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
 
     #build for libarrow.a
     cp $TP_INCLUDE_DIR/snappy/snappy-c.h  $TP_INCLUDE_DIR/snappy-c.h && \
     cp $TP_INCLUDE_DIR/snappy/snappy-sinksource.h  $TP_INCLUDE_DIR/snappy-sinksource.h && \
     cp $TP_INCLUDE_DIR/snappy/snappy-stubs-public.h  $TP_INCLUDE_DIR/snappy-stubs-public.h && \
-    cp $TP_INCLUDE_DIR/snappy/snappy.h  $TP_INCLUDE_DIR/snappy.h && \
-    cp $TP_INSTALL_DIR/lib/libsnappy.a $TP_INSTALL_DIR/libsnappy.a
+    cp $TP_INCLUDE_DIR/snappy/snappy.h  $TP_INCLUDE_DIR/snappy.h
 }
 
 # gperftools
@@ -403,6 +389,37 @@ build_lzo2() {
     make -j$PARALLEL && make install
 }
 
+# brotli
+build_brotli() {
+    check_if_source_exist $BROTLI_SOURCE
+    cd $TP_SOURCE_DIR/$BROTLI_SOURCE
+    mkdir -p $BUILD_DIR && cd $BUILD_DIR
+    rm -rf CMakeCache.txt CMakeFiles/
+    ${CMAKE_CMD} .. -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
+    ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
+    if [ -f $TP_INSTALL_DIR/lib64/libbrotlidec.so ]; then
+        rm -rf $TP_INSTALL_DIR/lib64/llibbrotlidec.so*
+    fi
+    if [ -f $TP_INSTALL_DIR/lib64/libbrotlicommon.so ]; then
+        rm -rf $TP_INSTALL_DIR/lib64/libbrotlicommon.so*
+    fi
+    if [ -f $TP_INSTALL_DIR/lib64/libbrotlienc.so ]; then
+        rm -rf $TP_INSTALL_DIR/lib64/libbrotlienc.so*
+    fi
+}
+
+#jemalloc
+build_jemalloc() {
+    check_if_source_exist $JEMALLOC_SOURCE
+    cd $TP_SOURCE_DIR/$JEMALLOC_SOURCE
+
+    CPPFLAGS="-I${TP_INCLUDE_DIR} -fPIC" \
+    LDFLAGS="-L${TP_LIB_DIR}" \
+    CFLAGS="-fPIC" \
+    ./configure --prefix=$TP_INSTALL_DIR
+    make -j$PARALLEL && make install_lib_static
+}
+
 # curl
 build_curl() {
     check_if_source_exist $CURL_SOURCE
@@ -421,8 +438,8 @@ build_re2() {
     check_if_source_exist $RE2_SOURCE
     cd $TP_SOURCE_DIR/$RE2_SOURCE
 
-    $CMAKE_CMD -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
-    make -j$PARALLEL install
+    ${CMAKE_CMD} -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
+    ${BUILD_SYSTEM} -j$PARALLEL install
 }
 
 # boost
@@ -448,11 +465,11 @@ build_mysql() {
         cp -rf $TP_SOURCE_DIR/$BOOST_FOR_MYSQL_SOURCE ./
     fi
 
-    $CMAKE_CMD ../ -DWITH_BOOST=`pwd`/$BOOST_FOR_MYSQL_SOURCE -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/mysql/ \
-    -DCMAKE_INCLUDE_PATH=$TP_INCLUDE_DIR -DCMAKE_LIBRARY_PATH=$TP_LIB_DIR -DWITHOUT_SERVER=1 \
+    ${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_BOOST=`pwd`/$BOOST_FOR_MYSQL_SOURCE -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/mysql/ \
+    -DCMAKE_INCLUDE_PATH=$TP_INCLUDE_DIR -DWITHOUT_SERVER=1 \
     -DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O3 -g -fabi-version=2 -fno-omit-frame-pointer -fno-strict-aliasing -std=gnu++11" \
     -DDISABLE_SHARED=1 -DBUILD_SHARED_LIBS=0
-    make -j$PARALLEL mysqlclient
+    ${BUILD_SYSTEM} -j$PARALLEL mysqlclient
 
     # copy headers manually
     rm -rf ../../../installed/include/mysql/
@@ -484,15 +501,10 @@ build_brpc() {
     mkdir -p $BUILD_DIR && cd $BUILD_DIR
     rm -rf CMakeCache.txt CMakeFiles/
     LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
-    $CMAKE_CMD -v -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
+    ${CMAKE_CMD} -G "${GENERATOR}" -v -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
     -DBRPC_WITH_GLOG=ON -DWITH_GLOG=ON -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \
-    -DCMAKE_LIBRARY_PATH="$TP_INSTALL_DIR/lib;$TP_INSTALL_DIR/lib64" \
-    -DPROTOBUF_PROTOC_EXECUTABLE=$TP_INSTALL_DIR/bin/protoc \
-    -DProtobuf_PROTOC_EXECUTABLE=$TP_INSTALL_DIR/bin/protoc ..
-    make -j$PARALLEL && make install
-    if [ -f $TP_INSTALL_DIR/lib/libbrpc.a ]; then
-        mkdir -p $TP_INSTALL_DIR/lib64 && cp $TP_INSTALL_DIR/lib/libbrpc.a $TP_INSTALL_DIR/lib64/libbrpc.a
-    fi
+    -DPROTOBUF_PROTOC_EXECUTABLE=$TP_INSTALL_DIR/bin/protoc .. 
+    ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
 }
 
 # rocksdb
@@ -541,8 +553,8 @@ build_flatbuffers() {
   rm -rf CMakeCache.txt CMakeFiles/
   CXXFLAGS="-fPIC -Wno-class-memaccess" \
   LDFLAGS="-static-libstdc++ -static-libgcc" \
-  ${CMAKE_CMD} ..
-  make -j$PARALLEL
+  ${CMAKE_CMD} -G "${GENERATOR}" ..
+  ${BUILD_SYSTEM} -j$PARALLEL
   cp flatc  ../../../installed/bin/flatc
   cp -r ../include/flatbuffers  ../../../installed/include/flatbuffers
   cp libflatbuffers.a ../../../installed/lib/libflatbuffers.a
@@ -559,32 +571,28 @@ build_arrow() {
     export ARROW_FLATBUFFERS_URL=${TP_SOURCE_DIR}/${FLATBUFFERS_NAME}
     export ARROW_ZSTD_URL=${TP_SOURCE_DIR}/${ZSTD_NAME}
     export ARROW_JEMALLOC_URL=${TP_SOURCE_DIR}/${JEMALLOC_NAME}
+    export ARROW_Thrift_URL=${TP_SOURCE_DIR}/${THRIFT_NAME}
     export LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
+    export export ZSTD_STATIC_LIB=${TP_SOURCE_DIR}/lib/libzstd.a
 
-    ${CMAKE_CMD} -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_USE_GLOG=off -DARROW_BUILD_SHARED=OFF \
+    ${CMAKE_CMD} -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_USE_GLOG=off \
+    -DARROW_BUILD_SHARED=OFF -DARROW_BUILD_STATIC=ON -DARROW_WITH_ZSTD=ON \
     -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
     -DCMAKE_INSTALL_LIBDIR=lib64 \
-    -DARROW_BOOST_USE_SHARED=OFF -DARROW_GFLAGS_USE_SHARED=OFF -DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=$TP_INSTALL_DIR \
+    -DARROW_BOOST_USE_SHARED=OFF -DARROW_GFLAGS_USE_SHARED=OFF \
+    -DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=$TP_INSTALL_DIR \
+    -DPARQUET_ARROW_LINKAGE=static \
     -Dgflags_ROOT=$TP_INSTALL_DIR/ \
     -DSnappy_ROOT=$TP_INSTALL_DIR/ \
     -DGLOG_ROOT=$TP_INSTALL_DIR/ \
     -DLZ4_ROOT=$TP_INSTALL_DIR/ \
-    -DThrift_ROOT=$TP_INSTALL_DIR/ \
     -DZSTD_SOURCE=BUNDLED \
-    -Ddouble-conversion_SOURCE=BUNDLED ..
+    -DJEMALLOC_HOME=$TP_INSTALL_DIR/ \
+    -DBROTLI_ROOT=$TP_INSTALL_DIR/ \
+    -Ddouble-conversion_SOURCE=BUNDLED \
+    -DThrift_ROOT=$TP_INSTALL_DIR/ ..
 
-    make -j$PARALLEL && make install
-    #copy dep libs
-    cp -rf ./jemalloc_ep-prefix/src/jemalloc_ep/dist/lib/libjemalloc_pic.a $TP_INSTALL_DIR/lib64/libjemalloc.a
-    cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a $TP_INSTALL_DIR/lib64/libbrotlienc.a
-    cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a $TP_INSTALL_DIR/lib64/libbrotlidec.a
-    cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a $TP_INSTALL_DIR/lib64/libbrotlicommon.a
-    if [ -f ./zstd_ep-install/lib64/libzstd.a ]; then
-        cp -rf ./zstd_ep-install/lib64/libzstd.a $TP_INSTALL_DIR/lib64/libzstd.a
-    else
-        cp -rf ./zstd_ep-install/lib/libzstd.a $TP_INSTALL_DIR/lib64/libzstd.a
-    fi
-    cp -rf ./double-conversion_ep/src/double-conversion_ep/lib/libdouble-conversion.a $TP_INSTALL_DIR/lib64/libdouble-conversion.a
+    ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
 }
 
 # s2
@@ -595,15 +603,14 @@ build_s2() {
     rm -rf CMakeCache.txt CMakeFiles/
     CXXFLAGS="-O3" \
     LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
-    $CMAKE_CMD -v -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
+    ${CMAKE_CMD} -G "${GENERATOR}" -v -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
     -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \
     -DBUILD_SHARED_LIBS=OFF \
     -DGFLAGS_ROOT_DIR="$TP_INSTALL_DIR/include" \
     -DWITH_GFLAGS=ON \
     -DGLOG_ROOT_DIR="$TP_INSTALL_DIR/include" \
-    -DWITH_GLOG=ON \
-    -DCMAKE_LIBRARY_PATH="$TP_INSTALL_DIR/lib;$TP_INSTALL_DIR/lib64" ..
-    make -j$PARALLEL && make install
+    -DWITH_GLOG=ON ..
+    ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
 }
 
 # bitshuffle
@@ -665,11 +672,10 @@ build_croaringbitmap() {
     rm -rf CMakeCache.txt CMakeFiles/
     CXXFLAGS="-O3" \
     LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
-    $CMAKE_CMD -v -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
+    ${CMAKE_CMD} -G "${GENERATOR}" -v -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
     -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \
-    -DENABLE_ROARING_TESTS=OFF \
-    -DCMAKE_LIBRARY_PATH="$TP_INSTALL_DIR/lib;$TP_INSTALL_DIR/lib64" ..
-    make -j$PARALLEL && make install
+    -DENABLE_ROARING_TESTS=OFF ..
+    ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
 }
 #orc
 build_orc() {
@@ -678,7 +684,7 @@ build_orc() {
     mkdir -p $BUILD_DIR && cd $BUILD_DIR
     rm -rf CMakeCache.txt CMakeFiles/
     CXXFLAGS="-O3 -Wno-array-bounds" \
-    $CMAKE_CMD ../ -DBUILD_JAVA=OFF \
+    ${CMAKE_CMD} -G "${GENERATOR}" ../ -DBUILD_JAVA=OFF \
     -DPROTOBUF_HOME=$TP_INSTALL_DIR \
     -DSNAPPY_HOME=$TP_INSTALL_DIR \
     -DGTEST_HOME=$TP_INSTALL_DIR \
@@ -689,7 +695,7 @@ build_orc() {
     -DBUILD_CPP_TESTS=OFF \
     -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR 
 
-    make -j$PARALLEL && make install
+    ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
 }
 
 #cctz
@@ -728,6 +734,8 @@ build_zlib
 build_lz4
 build_bzip
 build_lzo2
+build_brotli
+build_jemalloc
 build_openssl
 build_boost # must before thrift
 build_protobuf

thirdparty/download-thirdparty.sh

@@ -174,6 +174,7 @@ UNZIP_CMD="unzip"
 SUFFIX_TGZ="\.(tar\.gz|tgz)$"
 SUFFIX_XZ="\.tar\.xz$"
 SUFFIX_ZIP="\.zip$"
+SUFFIX_BZ2="\.tar\.bz2$"
 for TP_ARCH in ${TP_ARCHIVES[*]}
 do
     NAME=$TP_ARCH"_NAME"
@@ -203,6 +204,11 @@ do
                 echo "Failed to unzip ${!NAME}"
                 exit 1
             fi
+        elif [[ "${!NAME}" =~ $SUFFIX_BZ2 ]]; then
+            if ! $TAR_CMD xf "$TP_SOURCE_DIR/${!NAME}" -C "$TP_SOURCE_DIR/"; then
+                echo "Failed to untar ${!NAME}"
+                exit 1
+            fi
         fi
     else
         echo "${!SOURCE} already unpacked."