[deps](compression) Opt gzip decompress by libdeflate on X86 and X86_64 platforms: 1. Add libdeflate lib. (#27542)

Test result:

- env: 1 node(16 cores, 64G).
- parquet column: 100 million rows of char(255) column.
- result: 9.09 s -> 6.04 s.
This commit is contained in:
Qi Chen
2023-11-28 10:14:48 +08:00
committed by GitHub
parent b48c40ed31
commit 65126459bd
2 changed files with 22 additions and 0 deletions

View File

@ -1691,6 +1691,20 @@ build_avx2neon() {
cp -r ./* "${TP_INSTALL_DIR}/include/avx2neon/"
}
# libdeflate
build_libdeflate() {
check_if_source_exist "${LIBDEFLATE_SOURCE}"
cd "${TP_SOURCE_DIR}/${LIBDEFLATE_SOURCE}"
rm -rf "${BUILD_DIR}"
mkdir -p "${BUILD_DIR}"
cd "${BUILD_DIR}"
"${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release ..
"${BUILD_SYSTEM}" -j "${PARALLEL}"
"${BUILD_SYSTEM}" install
}
if [[ "${#packages[@]}" -eq 0 ]]; then
packages=(
libunixodbc
@ -1754,6 +1768,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
libunwind
dragonbox
avx2neon
libdeflate
)
if [[ "$(uname -s)" == 'Darwin' ]]; then
read -r -a packages <<<"binutils gettext ${packages[*]}"

7
thirdparty/vars.sh vendored
View File

@ -478,6 +478,12 @@ AVX2NEON_NAME=v1.0.0.tar.gz
AVX2NEON_SOURCE=AvxToNeon-1.0.0
AVX2NEON_MD5SUM="692d0e0f8b885a86ebc5172a9d8ee8db"
# libdeflate
LIBDEFLATE_DOWNLOAD="https://github.com/ebiggers/libdeflate/archive/refs/tags/v1.19.tar.gz"
LIBDEFLATE_NAME=libdeflate-1.19.tar.gz
LIBDEFLATE_SOURCE=libdeflate-1.19
LIBDEFLATE_MD5SUM="c69e9193d2975a729068ffa862c81fb6"
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES=(
'LIBEVENT'
@ -548,6 +554,7 @@ export TP_ARCHIVES=(
'HADOOP_LIBS'
'DRAGONBOX'
'AVX2NEON'
'LIBDEFLATE'
)
if [[ "$(uname -s)" == 'Darwin' ]]; then