database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Chore](s2geo) upgrade s2geo to 0.10.0 (#15002)

Browse Source 
upgrade s2geo to 0.10.0
This commit is contained in:
Pxl
2022-12-13 10:34:51 +08:00
committed by GitHub
parent 1200b22fd2
commit decabbb933
10 changed files with 90 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
be/CMakeLists.txt
View File

@ -687,6 +687,9 @@ set(DORIS_LINK_LIBS
${WL_END_GROUP}
)
set(absl_DIR ${THIRDPARTY_DIR}/lib/cmake/absl)
find_package(absl)
# COMMON_THIRDPARTY are thirdparty dependencies that can run on all platform
# When adding new dependencies, If you don’t know if it can run on all platforms,
# add it here first.
@ -753,6 +756,14 @@ set(COMMON_THIRDPARTY
simdjson
)
if (absl_FOUND)
set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
absl::flat_hash_set
absl::str_format
)
endif()
if (OS_MACOSX)
set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}

2
be/src/common/config.h
View File

@ -864,6 +864,8 @@ CONF_Bool(enable_fuzzy_mode, "false");
CONF_Int32(pipeline_executor_size, "0");
CONF_Double(s2geo_eps, "0.000000001");
#ifdef BE_TEST
// test s3
CONF_String(test_s3_resource, "resource");

5
be/src/geo/geo_functions.cpp
View File

@ -24,10 +24,7 @@
namespace doris {
void GeoFunctions::init() {
// set s2debug to false to avoid crash
FLAGS_s2debug = false;
}
void GeoFunctions::init() {}
DoubleVal GeoFunctions::st_distance_sphere(FunctionContext* ctx, const DoubleVal& x_lng,
const DoubleVal& x_lat, const DoubleVal& y_lng,

41
be/src/geo/geo_types.cpp
View File

@ -29,7 +29,9 @@
#include <iomanip>
#include <sstream>
#include <type_traits>
#include "common/config.h"
#include "geo/wkt_parse.h"
namespace doris {
@ -74,7 +76,7 @@ static bool is_loop_closed(const std::vector<S2Point>& points) {
if (points.empty()) {
return false;
}
if (points[0] != points[points.size() - 1]) {
if (!points[0].aequal(points[points.size() - 1], config::s2geo_eps)) {
return false;
}
return true;
@ -85,7 +87,7 @@ static void remove_duplicate_points(std::vector<S2Point>* points) {
int lhs = 0;
int rhs = 1;
for (; rhs < points->size(); ++rhs) {
if ((*points)[rhs] != (*points)[lhs]) {
if (!(*points)[rhs].aequal((*points)[lhs], config::s2geo_eps)) {
lhs++;
if (lhs != rhs) {
(*points)[lhs] = (*points)[rhs];
@ -147,6 +149,37 @@ static GeoParseStatus to_s2polyline(const GeoCoordinateList& coords,
return GEO_PARSE_OK;
}
// remove those compatibility codes when we finish upgrade s2geo.
#if defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
#elif defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
template <typename T, bool (T::*)(const T*) const = &T::Contains>
constexpr bool is_pointer_argument() {
return true;
}
constexpr bool is_pointer_argument(...) {
return false;
}
template <typename T>
bool adapt_contains(const T* lhs, const T* rhs) {
if constexpr (is_pointer_argument<T>()) {
return lhs->Contains(rhs);
} else {
return lhs->Contains(*rhs);
}
}
#if defined(__clang__)
#pragma clang diagnostic pop
#elif defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
static GeoParseStatus to_s2polygon(const GeoCoordinateListList& coords_list,
std::unique_ptr<S2Polygon>* polygon) {
std::vector<std::unique_ptr<S2Loop>> loops(coords_list.list.size());
@ -155,7 +188,7 @@ static GeoParseStatus to_s2polygon(const GeoCoordinateListList& coords_list,
if (res != GEO_PARSE_OK) {
return res;
}
if (i != 0 && !loops[0]->Contains(loops[i].get())) {
if (i != 0 && !adapt_contains(loops[0].get(), loops[i].get())) {
return GEO_PARSE_POLYGON_NOT_HOLE;
}
}
@ -355,7 +388,7 @@ bool GeoPolygon::contains(const GeoShape* rhs) const {
}
case GEO_SHAPE_POLYGON: {
const GeoPolygon* other = (const GeoPolygon*)rhs;
return _polygon->Contains(other->polygon());
return adapt_contains(_polygon.get(), other->polygon());
}
default:
return false;

3
dist/LICENSE-dist.txt vendored
View File

@ -1518,8 +1518,9 @@ The Apache Software License, Version 2.0
* brpc: 1.1.0
* rocksdb: 5.14.2
* arrow: 7.0.0
* S2: 0.9.0
* S2: 0.10.0
* croaringbitmap: 0.4.0
* Abseil LTS 20220623
* parallel-hashmap: 1.3.8
* orc: 1.7.2
* cctz: 2.3

4
thirdparty/CHANGELOG.md vendored
View File

@ -2,6 +2,10 @@
This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`
## v20221212
- Modified: s2geometry 0.9.0 -> 0.10.0
- Added: Abseil, it is dependency of new s2geometry
## v20221209
- Modified: update parallel-hashmap from 1.33 to 1.3.8

37
thirdparty/build-thirdparty.sh vendored
View File

@ -969,6 +969,23 @@ build_arrow() {
strip_lib libparquet.a
}
# abseil
build_abseil() {
check_if_source_exist "${ABSEIL_SOURCE}"
cd "${TP_SOURCE_DIR}/${ABSEIL_SOURCE}"
CXXFLAGS="-O3" \
LDFLAGS="-L${TP_LIB_DIR}" \
"${CMAKE_CMD}" -B "${BUILD_DIR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
-DABSL_ENABLE_INSTALL=ON \
-DBUILD_DEPS=ON \
-DBUILD_SHARED_LIBS=OFF \
-DCMAKE_CXX_STANDARD=11
cmake --build "${BUILD_DIR}"
cmake --install "${BUILD_DIR}" --prefix "${TP_INSTALL_DIR}"
}
# s2
build_s2() {
check_if_source_exist "${S2_SOURCE}"
@ -979,23 +996,12 @@ build_s2() {
rm -rf CMakeCache.txt CMakeFiles/
if [[ "${KERNEL}" != 'Darwin' ]]; then
ldflags="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
else
ldflags="-L${TP_LIB_DIR}"
fi
CXXFLAGS="-O3" \
LDFLAGS="${ldflags}" \
"${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
-DCMAKE_INCLUDE_PATH="${TP_INSTALL_DIR}/include" \
LDFLAGS="-L${TP_LIB_DIR}" \
${CMAKE_CMD} -G "${GENERATOR}" -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
-DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" \
-DBUILD_SHARED_LIBS=OFF \
-DGFLAGS_ROOT_DIR="${TP_INSTALL_DIR}/include" \
-DWITH_GFLAGS=ON \
-DGLOG_ROOT_DIR="${TP_INSTALL_DIR}/include" \
-DCMAKE_LIBRARY_PATH="${TP_INSTALL_DIR}/lib64" \
-DOPENSSL_ROOT_DIR="${TP_INSTALL_DIR}/include" \
-DWITH_GLOG=ON ..
-DCMAKE_LIBRARY_PATH="${TP_INSTALL_DIR}" ..
"${BUILD_SYSTEM}" -j "${PARALLEL}"
"${BUILD_SYSTEM}" install
@ -1558,6 +1564,7 @@ build_librdkafka
build_flatbuffers
build_orc
build_arrow
build_abseil
build_s2
build_bitshuffle
build_croaringbitmap

9
thirdparty/download-thirdparty.sh vendored
View File

@ -257,15 +257,6 @@ fi
cd -
echo "Finished patching ${LIBEVENT_SOURCE}"
# s2 patch to disable shared library
cd "${TP_SOURCE_DIR}/${S2_SOURCE}"
if [[ ! -f "${PATCHED_MARK}" ]]; then
patch -p1 <"${TP_PATCH_DIR}/s2geometry-0.9.0.patch"
touch "${PATCHED_MARK}"
fi
cd -
echo "Finished patching ${S2_SOURCE}"
# gsasl2 patch to fix link error such as mutilple func defination
# when link target with kerberos
cd "${TP_SOURCE_DIR}/${GSASL_SOURCE}"

129
thirdparty/patches/s2geometry-0.9.0.patch vendored
View File

@ -1,129 +0,0 @@
diff -uprN a/CMakeLists.txt b/CMakeLists.txt
--- a/CMakeLists.txt 2021-09-28 17:39:01.747718492 +0800
+++ b/CMakeLists.txt 2021-09-28 17:44:47.934137768 +0800
@@ -531,6 +531,6 @@ if (BUILD_EXAMPLES)
add_subdirectory("doc/examples" examples)
endif()
-if (${SWIG_FOUND} AND ${PYTHONLIBS_FOUND})
- add_subdirectory("src/python" python)
-endif()
+#if (${SWIG_FOUND} AND ${PYTHONLIBS_FOUND})
+# add_subdirectory("src/python" python)
+#endif()
diff -uprN a/src/s2/third_party/absl/base/internal/unaligned_access.h b/src/s2/third_party/absl/base/internal/unaligned_access.h
--- a/src/s2/third_party/absl/base/internal/unaligned_access.h 2021-09-28 17:39:01.782720861 +0800
+++ b/src/s2/third_party/absl/base/internal/unaligned_access.h 2021-09-28 17:42:50.863217996 +0800
@@ -80,7 +80,7 @@ inline uint32_t UnalignedLoad32(const vo
return __sanitizer_unaligned_load32(p);
}
-inline uint64 UnalignedLoad64(const void *p) {
+inline uint64_t UnalignedLoad64(const void *p) {
return __sanitizer_unaligned_load64(p);
}
@@ -92,7 +92,7 @@ inline void UnalignedStore32(void *p, ui
__sanitizer_unaligned_store32(p, v);
}
-inline void UnalignedStore64(void *p, uint64 v) {
+inline void UnalignedStore64(void *p, uint64_t v) {
__sanitizer_unaligned_store64(p, v);
}
@@ -130,8 +130,8 @@ inline uint32_t UnalignedLoad32(const vo
return t;
}
-inline uint64 UnalignedLoad64(const void *p) {
- uint64 t;
+inline uint64_t UnalignedLoad64(const void *p) {
+ uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
@@ -140,7 +140,7 @@ inline void UnalignedStore16(void *p, ui
inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); }
-inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); }
+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
} // namespace base_internal
} // namespace absl
@@ -172,14 +172,14 @@ inline void UnalignedStore64(void *p, ui
#define ABSL_INTERNAL_UNALIGNED_LOAD32(_p) \
(*reinterpret_cast<const uint32_t *>(_p))
#define ABSL_INTERNAL_UNALIGNED_LOAD64(_p) \
- (*reinterpret_cast<const uint64 *>(_p))
+ (*reinterpret_cast<const uint64_t *>(_p))
#define ABSL_INTERNAL_UNALIGNED_STORE16(_p, _val) \
(*reinterpret_cast<uint16_t *>(_p) = (_val))
#define ABSL_INTERNAL_UNALIGNED_STORE32(_p, _val) \
(*reinterpret_cast<uint32_t *>(_p) = (_val))
#define ABSL_INTERNAL_UNALIGNED_STORE64(_p, _val) \
- (*reinterpret_cast<uint64 *>(_p) = (_val))
+ (*reinterpret_cast<uint64_t *>(_p) = (_val))
#elif defined(__arm__) && \
!defined(__ARM_ARCH_5__) && \
@@ -246,13 +246,13 @@ struct Unaligned32Struct {
namespace absl {
namespace base_internal {
-inline uint64 UnalignedLoad64(const void *p) {
- uint64 t;
+inline uint64_t UnalignedLoad64(const void *p) {
+ uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
-inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); }
+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
} // namespace base_internal
} // namespace absl
@@ -286,8 +286,8 @@ inline uint32_t UnalignedLoad32(const vo
return t;
}
-inline uint64 UnalignedLoad64(const void *p) {
- uint64 t;
+inline uint64_t UnalignedLoad64(const void *p) {
+ uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
@@ -296,7 +296,7 @@ inline void UnalignedStore16(void *p, ui
inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); }
-inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); }
+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
} // namespace base_internal
} // namespace absl
diff -uprN a/src/s2/util/coding/coder.h b/src/s2/util/coding/coder.h
--- a/src/s2/util/coding/coder.h 2021-09-28 17:39:01.762719508 +0800
+++ b/src/s2/util/coding/coder.h 2021-09-28 17:44:11.903700328 +0800
@@ -155,7 +155,7 @@ class Encoder {
class Decoder {
public:
// Empty constructor to create uninitialized decoder
- inline Decoder() { }
+ inline Decoder() = default;
// NOTE: for efficiency reasons, this is not virtual. so don't add
// any members that really need to be destructed, and be careful about
@@ -461,7 +461,7 @@ inline void DecoderExtensions::FillArray
"Decoder must be trivially copy-assignable");
static_assert(absl::is_trivially_destructible<Decoder>::value,
"Decoder must be trivially destructible");
- std::memset(array, 0, num_decoders * sizeof(Decoder));
+ std::memset(static_cast<void*>(array), 0, num_decoders * sizeof(Decoder));
}
inline void Encoder::put8(unsigned char v) {

15
thirdparty/vars.sh vendored
View File

@ -245,11 +245,17 @@ ARROW_NAME="apache-arrow-7.0.0.tar.gz"
ARROW_SOURCE="apache-arrow-7.0.0"
ARROW_MD5SUM="316ade159901646849b3b4760fa52816"
# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz"
ABSEIL_NAME=abseil-cpp-20220623.1.tar.gz
ABSEIL_SOURCE=abseil-cpp-20220623.1
ABSEIL_MD5SUM="2aea7c1171c4c280f755de170295afd6"
# S2
S2_DOWNLOAD="https://github.com/google/s2geometry/archive/v0.9.0.tar.gz"
S2_NAME=s2geometry-0.9.0.tar.gz
S2_SOURCE=s2geometry-0.9.0
S2_MD5SUM="293552c7646193b8b4a01556808fe155"
S2_DOWNLOAD="https://github.com/google/s2geometry/archive/refs/tags/v0.10.0.tar.gz"
S2_NAME=s2geometry-0.10.0.tar.gz
S2_SOURCE=s2geometry-0.10.0
S2_MD5SUM="c68f3c5d326dde9255681b9201393a9f"
# bitshuffle
BITSHUFFLE_DOWNLOAD="https://github.com/kiyo-masui/bitshuffle/archive/0.3.5.tar.gz"
@ -467,6 +473,7 @@ export TP_ARCHIVES=(
'ARROW'
'BROTLI'
'ZSTD'
'ABSEIL'
'S2'
'BITSHUFFLE'
'CROARINGBITMAP'