diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index ce7b5fee60..c85f87e8a4 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -687,6 +687,9 @@ set(DORIS_LINK_LIBS ${WL_END_GROUP} ) +set(absl_DIR ${THIRDPARTY_DIR}/lib/cmake/absl) +find_package(absl) + # COMMON_THIRDPARTY are thirdparty dependencies that can run on all platform # When adding new dependencies, If you don’t know if it can run on all platforms, # add it here first. @@ -753,6 +756,14 @@ set(COMMON_THIRDPARTY simdjson ) +if (absl_FOUND) + set(COMMON_THIRDPARTY + ${COMMON_THIRDPARTY} + absl::flat_hash_set + absl::str_format + ) +endif() + if (OS_MACOSX) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} diff --git a/be/src/common/config.h b/be/src/common/config.h index 0f3a04ccd5..b4feaef618 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -864,6 +864,8 @@ CONF_Bool(enable_fuzzy_mode, "false"); CONF_Int32(pipeline_executor_size, "0"); +CONF_Double(s2geo_eps, "0.000000001"); + #ifdef BE_TEST // test s3 CONF_String(test_s3_resource, "resource"); diff --git a/be/src/geo/geo_functions.cpp b/be/src/geo/geo_functions.cpp index bda65a1346..0502432f35 100644 --- a/be/src/geo/geo_functions.cpp +++ b/be/src/geo/geo_functions.cpp @@ -24,10 +24,7 @@ namespace doris { -void GeoFunctions::init() { - // set s2debug to false to avoid crash - FLAGS_s2debug = false; -} +void GeoFunctions::init() {} DoubleVal GeoFunctions::st_distance_sphere(FunctionContext* ctx, const DoubleVal& x_lng, const DoubleVal& x_lat, const DoubleVal& y_lng, diff --git a/be/src/geo/geo_types.cpp b/be/src/geo/geo_types.cpp index 3a0ab43ee0..4c6422b619 100644 --- a/be/src/geo/geo_types.cpp +++ b/be/src/geo/geo_types.cpp @@ -29,7 +29,9 @@ #include #include +#include +#include "common/config.h" #include "geo/wkt_parse.h" namespace doris { @@ -74,7 +76,7 @@ static bool is_loop_closed(const std::vector& points) { if (points.empty()) { return false; } - if (points[0] != points[points.size() - 1]) { + if (!points[0].aequal(points[points.size() - 1], config::s2geo_eps)) { return false; } return true; @@ -85,7 +87,7 @@ static void remove_duplicate_points(std::vector* points) { int lhs = 0; int rhs = 1; for (; rhs < points->size(); ++rhs) { - if ((*points)[rhs] != (*points)[lhs]) { + if (!(*points)[rhs].aequal((*points)[lhs], config::s2geo_eps)) { lhs++; if (lhs != rhs) { (*points)[lhs] = (*points)[rhs]; @@ -147,6 +149,37 @@ static GeoParseStatus to_s2polyline(const GeoCoordinateList& coords, return GEO_PARSE_OK; } +// remove those compatibility codes when we finish upgrade s2geo. +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#elif defined(__GNUC__) || defined(__GNUG__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif +template +constexpr bool is_pointer_argument() { + return true; +} + +constexpr bool is_pointer_argument(...) { + return false; +} + +template +bool adapt_contains(const T* lhs, const T* rhs) { + if constexpr (is_pointer_argument()) { + return lhs->Contains(rhs); + } else { + return lhs->Contains(*rhs); + } +} +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) || defined(__GNUG__) +#pragma GCC diagnostic pop +#endif + static GeoParseStatus to_s2polygon(const GeoCoordinateListList& coords_list, std::unique_ptr* polygon) { std::vector> loops(coords_list.list.size()); @@ -155,7 +188,7 @@ static GeoParseStatus to_s2polygon(const GeoCoordinateListList& coords_list, if (res != GEO_PARSE_OK) { return res; } - if (i != 0 && !loops[0]->Contains(loops[i].get())) { + if (i != 0 && !adapt_contains(loops[0].get(), loops[i].get())) { return GEO_PARSE_POLYGON_NOT_HOLE; } } @@ -355,7 +388,7 @@ bool GeoPolygon::contains(const GeoShape* rhs) const { } case GEO_SHAPE_POLYGON: { const GeoPolygon* other = (const GeoPolygon*)rhs; - return _polygon->Contains(other->polygon()); + return adapt_contains(_polygon.get(), other->polygon()); } default: return false; diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt index 4bdd6cc537..98b11f2166 100644 --- a/dist/LICENSE-dist.txt +++ b/dist/LICENSE-dist.txt @@ -1518,8 +1518,9 @@ The Apache Software License, Version 2.0 * brpc: 1.1.0 * rocksdb: 5.14.2 * arrow: 7.0.0 - * S2: 0.9.0 + * S2: 0.10.0 * croaringbitmap: 0.4.0 + * Abseil LTS 20220623 * parallel-hashmap: 1.3.8 * orc: 1.7.2 * cctz: 2.3 diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index b3a3938c57..11e0bb4de0 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,10 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## v20221212 +- Modified: s2geometry 0.9.0 -> 0.10.0 +- Added: Abseil, it is dependency of new s2geometry + ## v20221209 - Modified: update parallel-hashmap from 1.33 to 1.3.8 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 316f27c9f7..1c83eb73ba 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -969,6 +969,23 @@ build_arrow() { strip_lib libparquet.a } +# abseil +build_abseil() { + check_if_source_exist "${ABSEIL_SOURCE}" + cd "${TP_SOURCE_DIR}/${ABSEIL_SOURCE}" + + CXXFLAGS="-O3" \ + LDFLAGS="-L${TP_LIB_DIR}" \ + "${CMAKE_CMD}" -B "${BUILD_DIR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DABSL_ENABLE_INSTALL=ON \ + -DBUILD_DEPS=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_CXX_STANDARD=11 + + cmake --build "${BUILD_DIR}" + cmake --install "${BUILD_DIR}" --prefix "${TP_INSTALL_DIR}" +} + # s2 build_s2() { check_if_source_exist "${S2_SOURCE}" @@ -979,23 +996,12 @@ build_s2() { rm -rf CMakeCache.txt CMakeFiles/ - if [[ "${KERNEL}" != 'Darwin' ]]; then - ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" - else - ldflags="-L${TP_LIB_DIR}" - fi - CXXFLAGS="-O3" \ - LDFLAGS="${ldflags}" \ - "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ - -DCMAKE_INCLUDE_PATH="${TP_INSTALL_DIR}/include" \ + LDFLAGS="-L${TP_LIB_DIR}" \ + ${CMAKE_CMD} -G "${GENERATOR}" -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \ -DBUILD_SHARED_LIBS=OFF \ - -DGFLAGS_ROOT_DIR="${TP_INSTALL_DIR}/include" \ - -DWITH_GFLAGS=ON \ - -DGLOG_ROOT_DIR="${TP_INSTALL_DIR}/include" \ - -DCMAKE_LIBRARY_PATH="${TP_INSTALL_DIR}/lib64" \ - -DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}/include" \ - -DWITH_GLOG=ON .. + -DCMAKE_LIBRARY_PATH="${TP_INSTALL_DIR}" .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install @@ -1558,6 +1564,7 @@ build_librdkafka build_flatbuffers build_orc build_arrow +build_abseil build_s2 build_bitshuffle build_croaringbitmap diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index c267e748c8..48a036b5fc 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -257,15 +257,6 @@ fi cd - echo "Finished patching ${LIBEVENT_SOURCE}" -# s2 patch to disable shared library -cd "${TP_SOURCE_DIR}/${S2_SOURCE}" -if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p1 <"${TP_PATCH_DIR}/s2geometry-0.9.0.patch" - touch "${PATCHED_MARK}" -fi -cd - -echo "Finished patching ${S2_SOURCE}" - # gsasl2 patch to fix link error such as mutilple func defination # when link target with kerberos cd "${TP_SOURCE_DIR}/${GSASL_SOURCE}" diff --git a/thirdparty/patches/s2geometry-0.9.0.patch b/thirdparty/patches/s2geometry-0.9.0.patch deleted file mode 100644 index 5d7618df13..0000000000 --- a/thirdparty/patches/s2geometry-0.9.0.patch +++ /dev/null @@ -1,129 +0,0 @@ -diff -uprN a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt 2021-09-28 17:39:01.747718492 +0800 -+++ b/CMakeLists.txt 2021-09-28 17:44:47.934137768 +0800 -@@ -531,6 +531,6 @@ if (BUILD_EXAMPLES) - add_subdirectory("doc/examples" examples) - endif() - --if (${SWIG_FOUND} AND ${PYTHONLIBS_FOUND}) -- add_subdirectory("src/python" python) --endif() -+#if (${SWIG_FOUND} AND ${PYTHONLIBS_FOUND}) -+# add_subdirectory("src/python" python) -+#endif() -diff -uprN a/src/s2/third_party/absl/base/internal/unaligned_access.h b/src/s2/third_party/absl/base/internal/unaligned_access.h ---- a/src/s2/third_party/absl/base/internal/unaligned_access.h 2021-09-28 17:39:01.782720861 +0800 -+++ b/src/s2/third_party/absl/base/internal/unaligned_access.h 2021-09-28 17:42:50.863217996 +0800 -@@ -80,7 +80,7 @@ inline uint32_t UnalignedLoad32(const vo - return __sanitizer_unaligned_load32(p); - } - --inline uint64 UnalignedLoad64(const void *p) { -+inline uint64_t UnalignedLoad64(const void *p) { - return __sanitizer_unaligned_load64(p); - } - -@@ -92,7 +92,7 @@ inline void UnalignedStore32(void *p, ui - __sanitizer_unaligned_store32(p, v); - } - --inline void UnalignedStore64(void *p, uint64 v) { -+inline void UnalignedStore64(void *p, uint64_t v) { - __sanitizer_unaligned_store64(p, v); - } - -@@ -130,8 +130,8 @@ inline uint32_t UnalignedLoad32(const vo - return t; - } - --inline uint64 UnalignedLoad64(const void *p) { -- uint64 t; -+inline uint64_t UnalignedLoad64(const void *p) { -+ uint64_t t; - memcpy(&t, p, sizeof t); - return t; - } -@@ -140,7 +140,7 @@ inline void UnalignedStore16(void *p, ui - - inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); } - --inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); } -+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } - - } // namespace base_internal - } // namespace absl -@@ -172,14 +172,14 @@ inline void UnalignedStore64(void *p, ui - #define ABSL_INTERNAL_UNALIGNED_LOAD32(_p) \ - (*reinterpret_cast(_p)) - #define ABSL_INTERNAL_UNALIGNED_LOAD64(_p) \ -- (*reinterpret_cast(_p)) -+ (*reinterpret_cast(_p)) - - #define ABSL_INTERNAL_UNALIGNED_STORE16(_p, _val) \ - (*reinterpret_cast(_p) = (_val)) - #define ABSL_INTERNAL_UNALIGNED_STORE32(_p, _val) \ - (*reinterpret_cast(_p) = (_val)) - #define ABSL_INTERNAL_UNALIGNED_STORE64(_p, _val) \ -- (*reinterpret_cast(_p) = (_val)) -+ (*reinterpret_cast(_p) = (_val)) - - #elif defined(__arm__) && \ - !defined(__ARM_ARCH_5__) && \ -@@ -246,13 +246,13 @@ struct Unaligned32Struct { - namespace absl { - namespace base_internal { - --inline uint64 UnalignedLoad64(const void *p) { -- uint64 t; -+inline uint64_t UnalignedLoad64(const void *p) { -+ uint64_t t; - memcpy(&t, p, sizeof t); - return t; - } - --inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); } -+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } - - } // namespace base_internal - } // namespace absl -@@ -286,8 +286,8 @@ inline uint32_t UnalignedLoad32(const vo - return t; - } - --inline uint64 UnalignedLoad64(const void *p) { -- uint64 t; -+inline uint64_t UnalignedLoad64(const void *p) { -+ uint64_t t; - memcpy(&t, p, sizeof t); - return t; - } -@@ -296,7 +296,7 @@ inline void UnalignedStore16(void *p, ui - - inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); } - --inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); } -+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } - - } // namespace base_internal - } // namespace absl -diff -uprN a/src/s2/util/coding/coder.h b/src/s2/util/coding/coder.h ---- a/src/s2/util/coding/coder.h 2021-09-28 17:39:01.762719508 +0800 -+++ b/src/s2/util/coding/coder.h 2021-09-28 17:44:11.903700328 +0800 -@@ -155,7 +155,7 @@ class Encoder { - class Decoder { - public: - // Empty constructor to create uninitialized decoder -- inline Decoder() { } -+ inline Decoder() = default; - - // NOTE: for efficiency reasons, this is not virtual. so don't add - // any members that really need to be destructed, and be careful about -@@ -461,7 +461,7 @@ inline void DecoderExtensions::FillArray - "Decoder must be trivially copy-assignable"); - static_assert(absl::is_trivially_destructible::value, - "Decoder must be trivially destructible"); -- std::memset(array, 0, num_decoders * sizeof(Decoder)); -+ std::memset(static_cast(array), 0, num_decoders * sizeof(Decoder)); - } - - inline void Encoder::put8(unsigned char v) { diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 4ff2e64195..ceec530203 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -245,11 +245,17 @@ ARROW_NAME="apache-arrow-7.0.0.tar.gz" ARROW_SOURCE="apache-arrow-7.0.0" ARROW_MD5SUM="316ade159901646849b3b4760fa52816" +# Abseil +ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz" +ABSEIL_NAME=abseil-cpp-20220623.1.tar.gz +ABSEIL_SOURCE=abseil-cpp-20220623.1 +ABSEIL_MD5SUM="2aea7c1171c4c280f755de170295afd6" + # S2 -S2_DOWNLOAD="https://github.com/google/s2geometry/archive/v0.9.0.tar.gz" -S2_NAME=s2geometry-0.9.0.tar.gz -S2_SOURCE=s2geometry-0.9.0 -S2_MD5SUM="293552c7646193b8b4a01556808fe155" +S2_DOWNLOAD="https://github.com/google/s2geometry/archive/refs/tags/v0.10.0.tar.gz" +S2_NAME=s2geometry-0.10.0.tar.gz +S2_SOURCE=s2geometry-0.10.0 +S2_MD5SUM="c68f3c5d326dde9255681b9201393a9f" # bitshuffle BITSHUFFLE_DOWNLOAD="https://github.com/kiyo-masui/bitshuffle/archive/0.3.5.tar.gz" @@ -467,6 +473,7 @@ export TP_ARCHIVES=( 'ARROW' 'BROTLI' 'ZSTD' + 'ABSEIL' 'S2' 'BITSHUFFLE' 'CROARINGBITMAP'