[deps](compression) Opt gzip decompress by libdeflate on X86 and X86_64 platforms: 1. Add libdeflate lib. (#27542)
Test result: - env: 1 node(16 cores, 64G). - parquet column: 100 million rows of char(255) column. - result: 9.09 s -> 6.04 s.
This commit is contained in:
15
thirdparty/build-thirdparty.sh
vendored
15
thirdparty/build-thirdparty.sh
vendored
@ -1691,6 +1691,20 @@ build_avx2neon() {
|
||||
cp -r ./* "${TP_INSTALL_DIR}/include/avx2neon/"
|
||||
}
|
||||
|
||||
# libdeflate
|
||||
build_libdeflate() {
|
||||
check_if_source_exist "${LIBDEFLATE_SOURCE}"
|
||||
cd "${TP_SOURCE_DIR}/${LIBDEFLATE_SOURCE}"
|
||||
|
||||
rm -rf "${BUILD_DIR}"
|
||||
mkdir -p "${BUILD_DIR}"
|
||||
cd "${BUILD_DIR}"
|
||||
|
||||
"${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release ..
|
||||
"${BUILD_SYSTEM}" -j "${PARALLEL}"
|
||||
"${BUILD_SYSTEM}" install
|
||||
}
|
||||
|
||||
if [[ "${#packages[@]}" -eq 0 ]]; then
|
||||
packages=(
|
||||
libunixodbc
|
||||
@ -1754,6 +1768,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
|
||||
libunwind
|
||||
dragonbox
|
||||
avx2neon
|
||||
libdeflate
|
||||
)
|
||||
if [[ "$(uname -s)" == 'Darwin' ]]; then
|
||||
read -r -a packages <<<"binutils gettext ${packages[*]}"
|
||||
|
||||
7
thirdparty/vars.sh
vendored
7
thirdparty/vars.sh
vendored
@ -478,6 +478,12 @@ AVX2NEON_NAME=v1.0.0.tar.gz
|
||||
AVX2NEON_SOURCE=AvxToNeon-1.0.0
|
||||
AVX2NEON_MD5SUM="692d0e0f8b885a86ebc5172a9d8ee8db"
|
||||
|
||||
# libdeflate
|
||||
LIBDEFLATE_DOWNLOAD="https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.tar.gz"
|
||||
LIBDEFLATE_NAME=libdeflate-1.19.tar.gz
|
||||
LIBDEFLATE_SOURCE=libdeflate-1.19
|
||||
LIBDEFLATE_MD5SUM="c69e9193d2975a729068ffa862c81fb6"
|
||||
|
||||
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
|
||||
export TP_ARCHIVES=(
|
||||
'LIBEVENT'
|
||||
@ -548,6 +554,7 @@ export TP_ARCHIVES=(
|
||||
'HADOOP_LIBS'
|
||||
'DRAGONBOX'
|
||||
'AVX2NEON'
|
||||
'LIBDEFLATE'
|
||||
)
|
||||
|
||||
if [[ "$(uname -s)" == 'Darwin' ]]; then
|
||||
|
||||
Reference in New Issue
Block a user