From 8a7b0e54856bfdf4c5521d2e7564782e0cca4f03 Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:12:10 +0800 Subject: [PATCH] [upgrade](thirdparty) upgrade librdkafka from 1.8.2 to 2.0.2 (#28210) Some error happen when using routine load ``` [INTERNAL_ERROR]Message at offset XXX might be too large to fetch, try increasing receive.message.max.bytes ``` Refer to https://github.com/confluentinc/librdkafka/issues/2993, we should upgrade librdkafka version to avoid this bug. --- thirdparty/CHANGELOG.md | 3 + thirdparty/download-thirdparty.sh | 4 +- ...fka-1.8.2.patch => librdkafka-2.0.2.patch} | 55 +++++++++---------- thirdparty/vars.sh | 10 ++-- 4 files changed, 37 insertions(+), 35 deletions(-) rename thirdparty/patches/{librdkafka-1.8.2.patch => librdkafka-2.0.2.patch} (75%) diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 8c29d29016..a323454a4b 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,9 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## v20231210 +- Modified: librdkafka 1.8.2 -> 2.0.2 + ## v20231020 - Added: avx2neon -> 1.0.0 diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index e8c45fa86b..4702defb81 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -327,10 +327,10 @@ fi echo "Finished patching ${ARROW_SOURCE}" # patch librdkafka to avoid crash -if [[ "${LIBRDKAFKA_SOURCE}" == "librdkafka-1.8.2" ]]; then +if [[ "${LIBRDKAFKA_SOURCE}" == "librdkafka-2.0.2" ]]; then cd "${TP_SOURCE_DIR}/${LIBRDKAFKA_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p0 <"${TP_PATCH_DIR}/librdkafka-1.8.2.patch" + patch -p0 <"${TP_PATCH_DIR}/librdkafka-2.0.2.patch" touch "${PATCHED_MARK}" fi cd - diff --git a/thirdparty/patches/librdkafka-1.8.2.patch b/thirdparty/patches/librdkafka-2.0.2.patch similarity index 75% rename from thirdparty/patches/librdkafka-1.8.2.patch rename to thirdparty/patches/librdkafka-2.0.2.patch index d632540476..87fb7a0da6 100644 --- a/thirdparty/patches/librdkafka-1.8.2.patch +++ b/thirdparty/patches/librdkafka-2.0.2.patch @@ -1,20 +1,17 @@ ---- src/rdkafka_broker.c 2021-10-12 04:15:44.000000000 +0800 -+++ src/rdkafka_broker.c 2022-03-10 10:31:45.141882467 +0800 -@@ -5464,7 +5464,9 @@ - */ - void rd_kafka_broker_destroy_final (rd_kafka_broker_t *rkb) { +--- lds-gen.py ++++ lds-gen.py +@@ -58,7 +58,7 @@ if __name__ == '__main__': -- rd_assert(thrd_is_current(rkb->rkb_thread)); -+ // To avoid the error describe in https://github.com/edenhill/librdkafka/issues/3608 -+ // comment this line to fix it temporarily. -+ // rd_assert(thrd_is_current(rkb->rkb_thread)); - rd_assert(TAILQ_EMPTY(&rkb->rkb_monitors)); - rd_assert(TAILQ_EMPTY(&rkb->rkb_outbufs.rkbq_bufs)); - rd_assert(TAILQ_EMPTY(&rkb->rkb_waitresps.rkbq_bufs)); - ---- mklove/modules/configure.base 2022-11-21 03:47:14.072094876 +0800 -+++ mklove/modules/configure.base 2022-11-21 03:47:01.360075164 +0800 -@@ -1716,7 +1716,7 @@ + # Special symbols not covered by above matches or not exposed in + # the public header files. +- funcs.append('rd_ut_coverage_check') ++ # funcs.append('rd_ut_coverage_check') + + print('# Automatically generated by lds-gen.py - DO NOT EDIT') + print('{\n global:') +--- mklove/modules/configure.base ++++ mklove/modules/configure.base +@@ -1741,7 +1741,7 @@ function mkl_pkg_config_check { mkl_check_begin "$cname" "$2" "no-cache" "$1 (by pkg-config)" && return $? local cflags= @@ -23,7 +20,7 @@ mkl_dbg "pkg-config check $libname for CFLAGS ($2): $cmd" cflags=$($cmd 2>&1) -@@ -1738,11 +1738,11 @@ +@@ -1764,11 +1764,11 @@ $cflags" fi local libs= @@ -37,15 +34,17 @@ # Clear define name ($2): caller may have additional checks mkl_check_failed "$cname" "" "$3" "pkg-config --libs failed" return 1 +--- src/rdkafka_broker.c ++++ src/rdkafka_broker.c +@@ -4613,7 +4613,9 @@ static int rd_kafka_broker_thread_main(void *arg) { + */ + void rd_kafka_broker_destroy_final(rd_kafka_broker_t *rkb) { + +- rd_assert(thrd_is_current(rkb->rkb_thread)); ++ // To avoid the error describe in https://github.com/edenhill/librdkafka/issues/3608 ++ // comment this line to fix it temporarily. ++ // rd_assert(thrd_is_current(rkb->rkb_thread)); + rd_assert(TAILQ_EMPTY(&rkb->rkb_monitors)); + rd_assert(TAILQ_EMPTY(&rkb->rkb_outbufs.rkbq_bufs)); + rd_assert(TAILQ_EMPTY(&rkb->rkb_waitresps.rkbq_bufs)); ---- lds-gen.py 2023-11-19 15:47:14.083929584 +0800 -+++ lds-gen.py 2023-11-19 15:47:51.759274031 +0800 -@@ -53,7 +53,7 @@ - - # Special symbols not covered by above matches or not exposed in - # the public header files. -- funcs.append('rd_ut_coverage_check') -+ # funcs.append('rd_ut_coverage_check') - - print('# Automatically generated by lds-gen.py - DO NOT EDIT') - print('{\n global:') diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index d66b7c5b7f..84f22508e3 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -215,11 +215,11 @@ CYRUS_SASL_NAME=cyrus-sasl-2.1.27.tar.gz CYRUS_SASL_SOURCE=cyrus-sasl-2.1.27 CYRUS_SASL_MD5SUM="a33820c66e0622222c5aefafa1581083" -# librdkafka-1.8.2 -LIBRDKAFKA_DOWNLOAD="https://github.com/edenhill/librdkafka/archive/refs/tags/v1.8.2.tar.gz" -LIBRDKAFKA_NAME=librdkafka-1.8.2.tar.gz -LIBRDKAFKA_SOURCE=librdkafka-1.8.2 -LIBRDKAFKA_MD5SUM="0abec0888d10c9553cdcbcbf9172d558" +# librdkafka-2.0.2 +LIBRDKAFKA_DOWNLOAD="https://github.com/confluentinc/librdkafka/archive/refs/tags/v2.0.2.tar.gz" +LIBRDKAFKA_NAME=librdkafka-2.0.2.tar.gz +LIBRDKAFKA_SOURCE=librdkafka-2.0.2 +LIBRDKAFKA_MD5SUM="c0120dc32acc129bfb4656fe17568da1" # zstd ZSTD_DOWNLOAD="https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"