[chore][fix](librdkafka) disable librdkafka assert and update some thirdparty (#8425)

1. comment librdkafka `rd_assert(thrd_is_current(rkb->rkb_thread));` to avoid core dump 2. upgrade arrow to 7.0.0 3. upgrade aws sdk to 1.9 4. upgrade orc to 1.7.2
2022-03-12 22:09:06 +08:00
parent 316d7c6e86
commit f3c44bcd75
11 changed files with 128 additions and 155 deletions
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@ -272,9 +272,25 @@ set_target_properties(aws-sdk-core PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR
 add_library(aws-sdk-s3 STATIC IMPORTED)
 set_target_properties(aws-sdk-s3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-s3.a)

+add_library(aws-sdk-transfer STATIC IMPORTED)
+set_target_properties(aws-sdk-transfer PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-transfer.a)
+
+add_library(aws-sdk-s3-crt STATIC IMPORTED)
+set_target_properties(aws-sdk-s3-crt PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-s3-crt.a)
+
+
+add_library(aws-crt-cpp STATIC IMPORTED)
+set_target_properties(aws-crt-cpp PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-crt-cpp.a)
+
 add_library(aws-c-cal STATIC IMPORTED)
 set_target_properties(aws-c-cal PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-cal.a)

+add_library(aws-c-auth STATIC IMPORTED)
+set_target_properties(aws-c-auth PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-auth.a)
+
+add_library(aws-c-compression STATIC IMPORTED)
+set_target_properties(aws-c-compression PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-compression.a)
+
 add_library(aws-c-common STATIC IMPORTED)
 set_target_properties(aws-c-common PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-common.a)

@ -284,9 +300,18 @@ set_target_properties(aws-c-event-stream PROPERTIES IMPORTED_LOCATION ${THIRDPAR
 add_library(aws-c-io STATIC IMPORTED)
 set_target_properties(aws-c-io PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-io.a)

+add_library(aws-c-http STATIC IMPORTED)
+set_target_properties(aws-c-http PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-http.a)
+
+add_library(aws-c-mqtt STATIC IMPORTED)
+set_target_properties(aws-c-mqtt PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-mqtt.a)
+
 add_library(aws-checksums STATIC IMPORTED)
 set_target_properties(aws-checksums PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-checksums.a)

+add_library(aws-c-s3 STATIC IMPORTED)
+set_target_properties(aws-c-s3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-s3.a)
+
 add_library(aws-s2n STATIC IMPORTED)
 set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libs2n.a)

@ -352,11 +377,11 @@ if (COMPILER_GCC)
    # https://stackoverflow.com/questions/67584073/gcc-11-false-array-subscript-is-partly-outside-array-bounds-warning
    # https://stackoverflow.com/questions/69426070/gcc-11-order-of-arguments-triggers-false-positive-wstringop-overflow-is-this-bu
    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-array-bounds -Wno-stringop-overread")
-    add_compile_options(-Wno-stringop-overflow)
+    add_compile_options(-Wno-stringop-overflow -fdiagnostics-color=always)
 endif ()

 if (COMPILER_CLANG)
-
+    add_compile_options (-fcolor-diagnostics)
    if(MAKE_TEST STREQUAL "OFF")
        add_compile_options(-Qunused-arguments)
    endif()
@ -432,12 +457,6 @@ SET(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")

 message(STATUS "Compiler Flags: ${CMAKE_CXX_FLAGS}")

-if (CMAKE_GENERATOR STREQUAL "Ninja" AND NOT DISABLE_COLORED_BUILD)
-    # Turn on colored output. https://github.com/ninja-build/ninja/wiki/FAQ
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
-    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
-endif ()
-
 # Thrift requires these two definitions for some types that we use
 add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H)

@ -452,15 +471,28 @@ include_directories(
    ${GENSRC_DIR}/
    ${THIRDPARTY_DIR}/include
    ${GPERFTOOLS_HOME}/include
-    ${THIRDPARTY_DIR}/include/thrift/
-    ${THIRDPARTY_DIR}/include/event/
    ${THIRDPARTY_DIR}/include/breakpad/
 )

 set(WL_START_GROUP "-Wl,--start-group")
 set(WL_END_GROUP "-Wl,--end-group")

-set(AWS_LIBS aws-sdk-s3 aws-sdk-core aws-checksums aws-c-io aws-c-event-stream aws-c-common aws-c-cal aws-s2n)
+set(AWS_LIBS
+    aws-sdk-s3
+    aws-sdk-core
+    aws-checksums
+    aws-c-io
+    aws-c-event-stream
+    aws-c-common
+    aws-c-cal
+    aws-s2n
+    aws-c-s3
+    aws-c-auth
+    aws-crt-cpp
+    aws-c-compression
+    aws-c-http
+    aws-c-mqtt
+    aws-sdk-s3-crt)

 # Set Doris libraries
 set(DORIS_LINK_LIBS
--- a/be/src/exec/parquet_writer.cpp
+++ b/be/src/exec/parquet_writer.cpp
@ -139,7 +139,7 @@ void ParquetWriterWrapper::parse_properties(
            if (property_value == "v1") {
                builder.version(parquet::ParquetVersion::PARQUET_1_0);
            } else {
-                builder.version(parquet::ParquetVersion::PARQUET_2_0);
+                builder.version(parquet::ParquetVersion::PARQUET_2_LATEST);
            }
        }
    }
--- a/be/src/util/arrow/row_batch.cpp
+++ b/be/src/util/arrow/row_batch.cpp
@ -27,8 +27,9 @@
 #include <arrow/record_batch.h>
 #include <arrow/status.h>
 #include <arrow/type.h>
+#include <arrow/visit_array_inline.h>
+#include <arrow/visit_type_inline.h>
 #include <arrow/visitor.h>
-#include <arrow/visitor_inline.h>

 #include <cstdlib>
 #include <ctime>
--- a/be/src/util/arrow/row_block.cpp
+++ b/be/src/util/arrow/row_block.cpp
@ -23,7 +23,8 @@
 #include <arrow/record_batch.h>
 #include <arrow/type.h>
 #include <arrow/type_fwd.h>
-#include <arrow/visitor_inline.h>
+#include <arrow/visit_array_inline.h>
+#include <arrow/visit_type_inline.h>

 #include "gutil/strings/substitute.h"
 #include "olap/column_block.h"
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -53,6 +53,9 @@ ENV REPOSITORY_URL="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/" \
    JAVA_HOME="/usr/lib/jvm/java-11" \
    PATH="/var/local/ldb-toolchain/bin/:$PATH"

+# disable auto enable ccache
+RUN rm -f /etc/profile.d/ccache.*
+
 # clone lastest source code, download and build third party
 COPY incubator-doris ${DEFAULT_DIR}/incubator-doris
 RUN cd ${DEFAULT_DIR}/incubator-doris && /bin/bash thirdparty/build-thirdparty.sh \
--- a/fe/pom.xml
+++ b/fe/pom.xml
@ -139,7 +139,7 @@ under the License.
        <java-cup.version>0.11-a-czt02-cdh</java-cup.version>
        <javassist.version>3.18.2-GA</javassist.version>
        <javax.servlet-api.version>3.0.1</javax.servlet-api.version>
-        <je.version>7.3.7</je.version>
+        <je.version>18.3.12</je.version>
        <jetty.version>6.1.14</jetty.version>
        <jflex.version>1.4.3</jflex.version>
        <jmockit.version>1.49</jmockit.version>
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@ -2,6 +2,13 @@

 This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/incubator-doris, and the tag is `build-env-${version}`

+## v20220310
+- Modified: arrow 5.0.0 -> 7.0.0
+- Modified: aws-sdk-cpp 1.8.108 -> 1.9.211
+- Modified: orc 1.6.6 -> 1.7.2
+
+- Removed: aws-c-common: 0.4.63,aws-c-event-stream: 0.2.6, aws-checksums: 0.1.10, aws-c-io-0.7.0 aws-s2n: 0.10.0, aws-c-cal: 0.4.5; those libs are managed by aws-sdk-cpp now
+
 ## v20220211

 - Added: simdjson 1.0.2
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@ -499,7 +499,7 @@ build_mysql() {
    CFLAGS="-static -pthread -lrt" CXXFLAGS="-static -pthread -lrt" \
    ${CMAKE_CMD} -G "${GENERATOR}" ../ -DCMAKE_LINK_SEARCH_END_STATIC=1 \
    -DWITH_BOOST=`pwd`/$BOOST_SOURCE -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/mysql/ \
-    -DCMAKE_INCLUDE_PATH=$TP_INCLUDE_DIR -DWITHOUT_SERVER=1 -DWITH_ZLIB=1 -DZLIB_ROOT=$TP_INSTALL_DIR \
+    -DWITHOUT_SERVER=1 -DWITH_ZLIB=1 -DZLIB_ROOT=$TP_INSTALL_DIR \
    -DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O3 -g -fabi-version=2 -fno-omit-frame-pointer -fno-strict-aliasing -std=gnu++11" \
    -DDISABLE_SHARED=1 -DBUILD_SHARED_LIBS=0 -DZLIB_LIBRARY=$TP_INSTALL_DIR/lib/libz.a -DENABLE_DTRACE=0
    ${BUILD_SYSTEM} -v -j $PARALLEL mysqlclient
@ -644,6 +644,7 @@ build_arrow() {
    -Dzstd_SOURCE=SYSTEM \
    -DSnappy_LIB=$TP_INSTALL_DIR/lib/libsnappy.a -DSnappy_INCLUDE_DIR=$TP_INSTALL_DIR/include \
    -DSnappy_SOURCE=SYSTEM \
+    -DBoost_INCLUDE_DIR=$TP_INSTALL_DIR/include \
    -DThrift_ROOT=$TP_INSTALL_DIR ..

    ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
@ -735,7 +736,7 @@ build_croaringbitmap() {
    CXXFLAGS="-O3" \
    LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
    ${CMAKE_CMD} -G "${GENERATOR}" -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-    -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" -DENABLE_ROARING_TESTS=OFF ..
+    -DENABLE_ROARING_TESTS=OFF ..
    ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
 }

@ -780,13 +781,11 @@ build_orc() {
    ${CMAKE_CMD} -G "${GENERATOR}" ../ -DBUILD_JAVA=OFF \
    -DPROTOBUF_HOME=$TP_INSTALL_DIR \
    -DSNAPPY_HOME=$TP_INSTALL_DIR \
-    -DGTEST_HOME=$TP_INSTALL_DIR \
    -DLZ4_HOME=$TP_INSTALL_DIR \
    -DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR/include/lz4 \
    -DZLIB_HOME=$TP_INSTALL_DIR \
    -DZSTD_HOME=$TP_INSTALL_DIR \
    -DZSTD_INCLUDE_DIR=$TP_INSTALL_DIR/include \
-    -DZSTD_LIBRARIES=$TP_INSTALL_DIR/lib/libzstd.a \
    -DBUILD_LIBHDFSPP=OFF \
    -DBUILD_CPP_TESTS=OFF \
    -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
@ -828,76 +827,17 @@ build_tsan_header() {
    cp $TSAN_HEADER_FILE $TP_INSTALL_DIR/include/sanitizer/
 }

-# aws-c-common
-build_aws_c_common() {
-    check_if_source_exist $AWS_C_COMMON_SOURCE
-    cd $TP_SOURCE_DIR/$AWS_C_COMMON_SOURCE
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
-    cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
-    -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_uninitialized $warning_option_ignored"
-    ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
-}
-
-# aws-c-event-stream
-build_aws_c_event_stream() {
-    check_if_source_exist $AWS_C_EVENT_STREAM_SOURCE
-    cd $TP_SOURCE_DIR/$AWS_C_EVENT_STREAM_SOURCE
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
-    cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR  -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
-    -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_option_ignored"
-    ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
-}
-
-# aws-checksums
-build_aws_checksums() {
-    check_if_source_exist $AWS_CHECKSUMS_SOURCE
-    cd $TP_SOURCE_DIR/$AWS_CHECKSUMS_SOURCE
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
-    cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
-    -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_option_ignored"
-    ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
-}
-
-# aws-c-io
-build_aws_c_io() {
-    check_if_source_exist $AWS_C_IO_SOURCE
-    cd $TP_SOURCE_DIR/$AWS_C_IO_SOURCE
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
-    cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
-    -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_option_ignored"
-    ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
-}
-
-# aws-s2n
-build_aws_s2n() {
-    check_if_source_exist $AWS_S2N_SOURCE
-    cd $TP_SOURCE_DIR/$AWS_S2N_SOURCE
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
-    cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
-    -DCMAKE_C_FLAGS="$warning_array_parameter"
-    ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
-}
-
-# aws-c-cal
-build_aws_c_cal() {
-    check_if_source_exist $AWS_C_CAL_SOURCE
-    cd $TP_SOURCE_DIR/$AWS_C_CAL_SOURCE
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
-    cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
-    -DBUILD_TESTING=OFF -DCMAKE_C_FLAGS="$warning_option_ignored"
-    ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
-}
-
 # aws_sdk
 build_aws_sdk() {
    check_if_source_exist $AWS_SDK_SOURCE
    cd $TP_SOURCE_DIR/$AWS_SDK_SOURCE
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
+    rm -rf $BUILD_DIR
    # -Wno-nonnull gcc-11
-    $CMAKE_CMD -G "${GENERATOR}" .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-    -DBUILD_DEPS=OFF -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF \
-    -DCMAKE_MODULE_PATH=$TP_INSTALL_DIR/lib64/cmake -DBUILD_ONLY="s3" \
-    -DCMAKE_CXX_FLAGS="-Wno-nonnull"
+    $CMAKE_CMD -G "${GENERATOR}" -B$BUILD_DIR -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
+    -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF \
+    -DCURL_LIBRARY_RELEASE=${TP_INSTALL_DIR}/lib/libcurl.a -DZLIB_LIBRARY_RELEASE=${TP_INSTALL_DIR}/lib/libz.a \
+    -DBUILD_ONLY="core;s3;s3-crt;transfer" -DCMAKE_CXX_FLAGS="-Wno-nonnull" -DCPP_STANDARD=17
+    cd $BUILD_DIR
    ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
 }

@ -1033,12 +973,6 @@ build_orc
 build_cctz
 build_tsan_header
 build_mysql
-build_aws_c_common
-build_aws_s2n
-build_aws_c_cal
-build_aws_c_io
-build_aws_checksums
-build_aws_c_event_stream
 build_aws_sdk
 build_js_and_css
 build_lzma
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@ -302,3 +302,28 @@ if [ $ROCKSDB_SOURCE == "rocksdb-5.14.2" ]; then
    cd -
 fi
 echo "Finished patching $ROCKSDB_SOURCE"
+
+# patch librdkafka to avoid crash
+if [ $LIBRDKAFKA_SOURCE = "librdkafka-1.8.2" ]; then
+    cd $TP_SOURCE_DIR/$LIBRDKAFKA_SOURCE
+    if [ ! -f $PATCHED_MARK ]; then
+        patch -p0 < $TP_PATCH_DIR/librdkafka-1.8.2.patch
+        touch $PATCHED_MARK
+    fi
+    cd -
+fi
+echo "Finished patching $LIBRDKAFKA_SOURCE"
+
+cd $TP_SOURCE_DIR/$AWS_SDK_SOURCE
+if [ ! -f $PATCHED_MARK ]; then
+    if [ $AWS_SDK_SOURCE == "aws-sdk-cpp-1.9.211" ]; then
+         curl -L https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/aws-crt-cpp-1.9.211.tar.gz | tar -zx 
+    else
+        bash ./prefetch_crt_dependency.sh
+    fi
+    touch $PATCHED_MARK
+fi
+cd -
+echo "Finished patching $AWS_SDK_SOURCE"
+
+
--- a/thirdparty/patches/librdkafka-1.8.2.patch
+++ b/thirdparty/patches/librdkafka-1.8.2.patch
@ -0,0 +1,13 @@
+--- src/rdkafka_broker.c	2021-10-12 04:15:44.000000000 +0800
+++ src/rdkafka_broker.c	2022-03-10 10:31:45.141882467 +0800
+@@ -5464,7 +5464,9 @@
+  */
+ void rd_kafka_broker_destroy_final (rd_kafka_broker_t *rkb) {
+ 
+-        rd_assert(thrd_is_current(rkb->rkb_thread));
+        // To avoid the error describe in https://github.com/edenhill/librdkafka/issues/3608
+        // comment this line to fix it temporarily.
+        // rd_assert(thrd_is_current(rkb->rkb_thread));
+         rd_assert(TAILQ_EMPTY(&rkb->rkb_monitors));
+         rd_assert(TAILQ_EMPTY(&rkb->rkb_outbufs.rkbq_bufs));
+         rd_assert(TAILQ_EMPTY(&rkb->rkb_waitresps.rkbq_bufs));
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@ -191,11 +191,11 @@ CYRUS_SASL_NAME=cyrus-sasl-2.1.27.tar.gz
 CYRUS_SASL_SOURCE=cyrus-sasl-2.1.27
 CYRUS_SASL_MD5SUM="a33820c66e0622222c5aefafa1581083"

-# librdkafka-1.8.0
-LIBRDKAFKA_DOWNLOAD="https://github.com/edenhill/librdkafka/archive/v1.8.0.tar.gz"
-LIBRDKAFKA_NAME=librdkafka-1.8.0.tar.gz
-LIBRDKAFKA_SOURCE=librdkafka-1.8.0
-LIBRDKAFKA_MD5SUM="f31bd3b7a91a486d65b740a720b925dc"
+# librdkafka-1.8.2
+LIBRDKAFKA_DOWNLOAD="https://github.com/edenhill/librdkafka/archive/refs/tags/v1.8.2.tar.gz"
+LIBRDKAFKA_NAME=librdkafka-1.8.2.tar.gz
+LIBRDKAFKA_SOURCE=librdkafka-1.8.2
+LIBRDKAFKA_MD5SUM="0abec0888d10c9553cdcbcbf9172d558"

 # zstd
 ZSTD_DOWNLOAD="https://github.com/facebook/zstd/archive/v1.5.0.tar.gz"
@ -216,10 +216,10 @@ FLATBUFFERS_SOURCE=flatbuffers-2.0.0
 FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"

 # arrow
-ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/apache-arrow-5.0.0.tar.gz"
-ARROW_NAME="arrow-apache-arrow-5.0.0.tar.gz"
-ARROW_SOURCE="arrow-apache-arrow-5.0.0"
-ARROW_MD5SUM="9caf5dbd36ef4972c3a591bcfeaf59c8"
+ARROW_DOWNLOAD="https://dlcdn.apache.org/arrow/arrow-7.0.0/apache-arrow-7.0.0.tar.gz"
+ARROW_NAME="apache-arrow-7.0.0.tar.gz"
+ARROW_SOURCE="apache-arrow-7.0.0"
+ARROW_MD5SUM="316ade159901646849b3b4760fa52816"

 # S2
 S2_DOWNLOAD="https://github.com/google/s2geometry/archive/v0.9.0.tar.gz"
@ -252,10 +252,10 @@ PARALLEL_HASHMAP_SOURCE="parallel-hashmap-1.33"
 PARALLEL_HASHMAP_MD5SUM="7626b5215f745c4ce59b5a4e41d16235"

 # orc
-ORC_DOWNLOAD="https://archive.apache.org/dist/orc/orc-1.6.6/orc-1.6.6.tar.gz"
-ORC_NAME=orc-1.6.6.tar.gz
-ORC_SOURCE=orc-1.6.6
-ORC_MD5SUM="26c94135111d312fb1ea4fc80d776c5f"
+ORC_DOWNLOAD="https://archive.apache.org/dist/orc/orc-1.7.2/orc-1.7.2.tar.gz"
+ORC_NAME=orc-1.7.2.tar.gz
+ORC_SOURCE=orc-1.7.2
+ORC_MD5SUM="6cab37935eacdec7d078d327746a8578"

 # jemalloc
 JEMALLOC_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
@ -290,47 +290,11 @@ BOOTSTRAP_TABLE_CSS_NAME="bootstrap-table.min.css"
 BOOTSTRAP_TABLE_CSS_FILE="bootstrap-table.min.css"
 BOOTSTRAP_TABLE_CSS_MD5SUM="23389d4456da412e36bae30c469a766a"

-# aws-c-common
-AWS_C_COMMON_DOWNLOAD="https://github.com/awslabs/aws-c-common/archive/v0.4.63.tar.gz"
-AWS_C_COMMON_NAME="aws-c-common-0.4.63.tar.gz"
-AWS_C_COMMON_SOURCE="aws-c-common-0.4.63"
-AWS_C_COMMON_MD5SUM="8298e00a0fb64779b7cf660592d50ab6"
-
-# aws-c-event-stream
-AWS_C_EVENT_STREAM_DOWNLOAD="https://github.com/awslabs/aws-c-event-stream/archive/v0.2.6.tar.gz"
-AWS_C_EVENT_STREAM_NAME="aws-c-event-stream-0.2.6.tar.gz"
-AWS_C_EVENT_STREAM_SOURCE="aws-c-event-stream-0.2.6"
-AWS_C_EVENT_STREAM_MD5SUM="fceedde198ddbf38ffdaed08d1435f7f"
-
-# aws-checksums
-AWS_CHECKSUMS_DOWNLOAD="https://github.com/awslabs/aws-checksums/archive/v0.1.10.tar.gz"
-AWS_CHECKSUMS_NAME="aws-checksums-0.1.10.tar.gz"
-AWS_CHECKSUMS_SOURCE="aws-checksums-0.1.10"
-AWS_CHECKSUMS_MD5SUM="2383c66f6250fa0238edbd1d779b49d3"
-
-# aws-c-io
-AWS_C_IO_DOWNLOAD="https://github.com/awslabs/aws-c-io/archive/v0.7.0.tar.gz"
-AWS_C_IO_NAME="aws-c-io-0.7.0.tar.gz"
-AWS_C_IO_SOURCE="aws-c-io-0.7.0"
-AWS_C_IO_MD5SUM="b95a6f9d20500727231dd726c957276b"
-
-# aws-s2n
-AWS_S2N_DOWNLOAD="https://github.com/awslabs/s2n/archive/v0.10.0.tar.gz"
-AWS_S2N_NAME="s2n-0.10.0.tar.gz"
-AWS_S2N_SOURCE="s2n-tls-0.10.0"
-AWS_S2N_MD5SUM="345aa5d2f9e82347bb3e568c22104d0e"
-
-# aws-c-cal
-AWS_C_CAL_DOWNLOAD="https://github.com/awslabs/aws-c-cal/archive/v0.4.5.tar.gz"
-AWS_C_CAL_NAME="aws-c-cal-0.4.5.tar.gz"
-AWS_C_CAL_SOURCE="aws-c-cal-0.4.5"
-AWS_C_CAL_MD5SUM="317f3dbafae551a0fc7d70f31434e216"
-
 # aws sdk
-AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/1.8.108.tar.gz"
-AWS_SDK_NAME="aws-sdk-cpp-1.8.108.tar.gz"
-AWS_SDK_SOURCE="aws-sdk-cpp-1.8.108"
-AWS_SDK_MD5SUM="76d8855406e7da61f1f996c11c0b93d7"
+AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.9.211.tar.gz"
+AWS_SDK_NAME="aws-sdk-cpp-1.9.211.tar.gz"
+AWS_SDK_SOURCE="aws-sdk-cpp-1.9.211"
+AWS_SDK_MD5SUM="667b8e08baf0b9967c19224198e33160"

 # tsan_header
 TSAN_HEADER_DOWNLOAD="https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libsanitizer/include/sanitizer/tsan_interface_atomic.h;hb=refs/heads/releases/gcc-7"
@ -395,11 +359,11 @@ BREAKPAD_SOURCE=breakpad-src-38ee0be-with-lss
 BREAKPAD_MD5SUM="fd8c4f6f5cf8b5e03a4c3c39fde83368"

 # xsimd
-# for arrow-5.0.0, if arrow upgrade, this version may also need to be changed
-XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/e9234cd6e6f4428fc260073b2c34ffe86fda1f34.tar.gz"
-XSIMD_NAME=xsimd-e9234cd6e6f4428fc260073b2c34ffe86fda1f34.tar.gz
-XSIMD_SOURCE=xsimd-e9234cd6e6f4428fc260073b2c34ffe86fda1f34
-XSIMD_MD5SUM="9f230757cf4acd3d544c4a79a020c9dc"
+# for arrow-7.0.0, if arrow upgrade, this version may also need to be changed
+XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz"
+XSIMD_NAME=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2.tar.gz
+XSIMD_SOURCE=xsimd-aeec9c872c8b475dedd7781336710f2dd2666cb2
+XSIMD_MD5SUM="d024855f71c0a2837a6918c0f8f66245"

 # simdjson
 SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v1.0.2.tar.gz"
@ -448,13 +412,6 @@ DATATABLES
 BOOTSTRAP_TABLE_JS
 BOOTSTRAP_TABLE_CSS
 TSAN_HEADER
-AWS_C_COMMON
-AWS_C_EVENT_STREAM
-AWS_C_IO
-AWS_C_CAL
-AWS_C_IO
-AWS_CHECKSUMS
-AWS_S2N
 AWS_SDK
 LZMA
 XML2