diff --git a/.rat-excludes b/.rat-excludes index 1dd0e07864..a277d321e4 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -11,6 +11,7 @@ DISCLAIMER-WIP LICENSE NOTICE gutil/* +glibc-compatibility/* manifest patches/* data/* diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index c87bb6c7ae..5d624ace66 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 2.8.10) +cmake_minimum_required(VERSION 3.12.0) # set CMAKE_C_COMPILER, this must set before project command if (DEFINED ENV{DORIS_GCC_HOME}) @@ -28,6 +28,31 @@ endif() project(doris CXX C) +# set platforms + +if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64") + set (ARCH_AMD64 1) +endif () +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)") + set (ARCH_AARCH64 1) +endif () +if (ARCH_AARCH64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm") + set (ARCH_ARM 1) +endif () +if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386") + set (ARCH_I386 1) +endif () +if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386) + message (FATAL_ERROR "32bit platforms are not supported") +endif () + +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)") + set (ARCH_PPC64LE 1) +endif () + +option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON) +message(STATUS "GLIBC_COMPATIBILITY is ${GLIBC_COMPATIBILITY}") + # set CMAKE_BUILD_TYPE if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RELEASE) @@ -501,13 +526,15 @@ else() message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}") endif() -add_subdirectory(${SRC_DIR}/glibc-compatibility) - set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -lrt -l:libbfd.a -liberty -lc -lm -ldl -pthread - glibc-compatibility ) +if (GLIBC_COMPATIBILITY) + add_subdirectory(${SRC_DIR}/glibc-compatibility) + set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} glibc-compatibility) +endif() + # Set libraries for test set (TEST_LINK_LIBS ${DORIS_LINK_LIBS} ${WL_START_GROUP} diff --git a/be/src/glibc-compatibility/CMakeLists.txt b/be/src/glibc-compatibility/CMakeLists.txt index 92860fbdb5..19b1cafb5c 100644 --- a/be/src/glibc-compatibility/CMakeLists.txt +++ b/be/src/glibc-compatibility/CMakeLists.txt @@ -1,55 +1,65 @@ -enable_language(ASM) -include(CheckIncludeFile) +if (GLIBC_COMPATIBILITY) + enable_language(ASM) + include(CheckIncludeFile) -check_include_file("sys/random.h" HAVE_SYS_RANDOM_H) + check_include_file("sys/random.h" HAVE_SYS_RANDOM_H) -if(COMPILER_CLANG) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-builtin-requires-header") -endif() + if(COMPILER_CLANG) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-builtin-requires-header") + endif() -if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.12") - macro(add_glob cur_list) - file(GLOB __tmp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS ${ARGN}) - list(APPEND ${cur_list} ${__tmp}) + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.12") + macro(add_glob cur_list) + file(GLOB __tmp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} CONFIGURE_DEPENDS ${ARGN}) + list(APPEND ${cur_list} ${__tmp}) + endmacro() + else () + macro(add_glob cur_list) + file(GLOB __tmp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${ARGN}) + list(APPEND ${cur_list} ${__tmp}) + endmacro() + endif () + + macro(add_headers_and_sources prefix common_path) + add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) + add_glob(${prefix}_sources ${common_path}/*.cpp ${common_path}/*.c ${common_path}/*.h) endmacro() -else () - macro(add_glob cur_list) - file(GLOB __tmp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${ARGN}) - list(APPEND ${cur_list} ${__tmp}) + + macro(add_headers_only prefix common_path) + add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) endmacro() + + add_headers_and_sources(glibc_compatibility .) + add_headers_and_sources(glibc_compatibility musl) + if (ARCH_ARM) + list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s musl/aarch64/longjmp.s) + set (musl_arch_include_dir musl/aarch64) + elseif (ARCH_AMD64) + list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s FastMemcpy.c) + set (musl_arch_include_dir musl/x86_64) + else () + message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.") + endif () + + list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c) + if(HAVE_SYS_RANDOM_H) + list(APPEND glibc_compatibility_sources musl/getentropy.c) + endif() + + # Need to omit frame pointers to match the performance of glibc + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer") + + add_library(glibc-compatibility STATIC ${glibc_compatibility_sources}) + + if (COMPILER_CLANG) + target_compile_options(glibc-compatibility PRIVATE -Wno-unused-command-line-argument) + elseif (COMPILER_GCC) + target_compile_options(glibc-compatibility PRIVATE -Wno-unused-but-set-variable) + endif () + + target_include_directories(glibc-compatibility PRIVATE ${musl_arch_include_dir}) + + message (STATUS "Some symbols from glibc will be replaced for compatibility") +else() + message (STATUS "not------- compatibility") endif () - -macro(add_headers_and_sources prefix common_path) - add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) - add_glob(${prefix}_sources ${common_path}/*.cpp ${common_path}/*.c ${common_path}/*.h) -endmacro() - -macro(add_headers_only prefix common_path) - add_glob(${prefix}_headers ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) -endmacro() - -add_headers_and_sources(glibc_compatibility .) -add_headers_and_sources(glibc_compatibility musl) - -list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s FastMemcpy.c) -set (musl_arch_include_dir musl/x86_64) - -list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c) -if(HAVE_SYS_RANDOM_H) - list(APPEND glibc_compatibility_sources musl/getentropy.c) -endif() - -# Need to omit frame pointers to match the performance of glibc -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer") - -add_library(glibc-compatibility STATIC ${glibc_compatibility_sources}) - -if (COMPILER_CLANG) - target_compile_options(glibc-compatibility PRIVATE -Wno-unused-command-line-argument) -elseif (COMPILER_GCC) - target_compile_options(glibc-compatibility PRIVATE -Wno-unused-but-set-variable) -endif () - -target_include_directories(glibc-compatibility PRIVATE ${musl_arch_include_dir}) - -message (STATUS "Some symbols from glibc will be replaced for compatibility") diff --git a/be/src/glibc-compatibility/musl/lgamma.c b/be/src/glibc-compatibility/musl/lgamma.c index b0e4f3aa53..fb9d105d0f 100644 --- a/be/src/glibc-compatibility/musl/lgamma.c +++ b/be/src/glibc-compatibility/musl/lgamma.c @@ -24,7 +24,7 @@ * = log(6.3*5.3) + lgamma(5.3) * = log(6.3*5.3*4.3*3.3*2.3) + lgamma(2.3) * 2. Polynomial approximation of lgamma around its - * minimun ymin=1.461632144968362245 to maintain monotonicity. + * minimum ymin=1.461632144968362245 to maintain monotonicity. * On [ymin-0.23, ymin+0.27] (i.e., [1.23164,1.73163]), use * Let z = x-ymin; * lgamma(x) = -1.214862905358496078218 + z^2*poly(z) @@ -258,11 +258,3 @@ double lgamma_r(double x, int *signgamp) r = nadj - r; return r; } - - -int signgam; - -double lgamma(double x) -{ - return lgamma_r(x, &signgam); -} diff --git a/be/src/glibc-compatibility/musl/powl.c b/be/src/glibc-compatibility/musl/powl.c index 5b6da07b2e..70cc3fd02a 100644 --- a/be/src/glibc-compatibility/musl/powl.c +++ b/be/src/glibc-compatibility/musl/powl.c @@ -191,6 +191,9 @@ static const volatile long double twom10000 = 0x1p-10000L; static long double reducl(long double); static long double powil(long double, int); +long double __polevll(long double x, const long double *P, int n); +long double __p1evll(long double x, const long double *P, int n); + long double powl(long double x, long double y) { /* double F, Fa, Fb, G, Ga, Gb, H, Ha, Hb */ @@ -199,7 +202,7 @@ long double powl(long double x, long double y) volatile long double z=0; long double w=0, W=0, Wa=0, Wb=0, ya=0, yb=0, u=0; - /* make sure no invalid exception is raised by nan comparision */ + /* make sure no invalid exception is raised by nan comparison */ if (isnan(x)) { if (!isnan(y) && y == 0.0) return 1.0; diff --git a/be/src/glibc-compatibility/musl/sched_getcpu.c b/be/src/glibc-compatibility/musl/sched_getcpu.c index 4ec5eaf679..04e19fa484 100644 --- a/be/src/glibc-compatibility/musl/sched_getcpu.c +++ b/be/src/glibc-compatibility/musl/sched_getcpu.c @@ -4,6 +4,28 @@ #include "syscall.h" #include "atomic.h" +#ifndef __NR_getcpu +#if defined(__x86_64__) +#define __NR_getcpu 309 +#elif defined(__i386__) +#define __NR_getcpu 318 +#elif defined(__aarch64__) +#define __NR_getcpu 168 +#endif +#endif + +#ifndef SYS_getcpu +#ifdef __NR_getcpu +#define SYS_getcpu __NR_getcpu +#endif +#endif + +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#include +#endif +#endif + #ifdef VDSO_GETCPU_SYM static void *volatile vdso_func; @@ -25,7 +47,7 @@ static void *volatile vdso_func = (void *)getcpu_init; int sched_getcpu(void) { int r; - unsigned cpu; + unsigned cpu = 0; #ifdef VDSO_GETCPU_SYM getcpu_f f = (getcpu_f)vdso_func; @@ -37,6 +59,13 @@ int sched_getcpu(void) #endif r = __syscall(SYS_getcpu, &cpu, 0, 0); - if (!r) return cpu; + if (!r) { +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) + __msan_unpoison(&cpu, sizeof(cpu)); +#endif +#endif + return cpu; + } return __syscall_ret(r); } diff --git a/be/src/glibc-compatibility/musl/syscall.h b/be/src/glibc-compatibility/musl/syscall.h index 70b4688f64..3160357f25 100644 --- a/be/src/glibc-compatibility/musl/syscall.h +++ b/be/src/glibc-compatibility/musl/syscall.h @@ -13,3 +13,11 @@ long __syscall(syscall_arg_t, ...); __attribute__((visibility("hidden"))) void *__vdsosym(const char *, const char *); + +#define syscall(...) __syscall_ret(__syscall(__VA_ARGS__)) + +#define socketcall(...) __syscall_ret(__socketcall(__VA_ARGS__)) + +#define __socketcall(nm,a,b,c,d,e,f) __syscall(SYS_##nm, a, b, c, d, e, f) + +#define socketcall_cp socketcall diff --git a/be/src/glibc-compatibility/musl/vdso.c b/be/src/glibc-compatibility/musl/vdso.c index c0dd0f33e4..b108c4ef75 100644 --- a/be/src/glibc-compatibility/musl/vdso.c +++ b/be/src/glibc-compatibility/musl/vdso.c @@ -40,24 +40,10 @@ static int checkver(Verdef *def, int vsym, const char *vername, char *strings) #define OK_TYPES (1<e_phoff); size_t *dynv=0, base=-1; diff --git a/build.sh b/build.sh index 6c0422b200..8f91718ac8 100755 --- a/build.sh +++ b/build.sh @@ -134,6 +134,9 @@ fi if [[ -z ${WITH_MYSQL} ]]; then WITH_MYSQL=OFF fi +if [[ -z ${GLIBC_COMPATIBILITY} ]]; then + GLIBC_COMPATIBILITY=OFF +fi if [[ -z ${WITH_LZO} ]]; then WITH_LZO=OFF fi @@ -147,6 +150,7 @@ echo "Get params: RUN_UT -- $RUN_UT WITH_MYSQL -- $WITH_MYSQL WITH_LZO -- $WITH_LZO + GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY " # Clean and build generated code @@ -171,8 +175,9 @@ if [ ${BUILD_BE} -eq 1 ] ; then fi mkdir -p ${CMAKE_BUILD_DIR} cd ${CMAKE_BUILD_DIR} - ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DMAKE_TEST=OFF -DWITH_MYSQL=${WITH_MYSQL} -DWITH_LZO=${WITH_LZO} ../ - ${BUILD_SYSTEM} -j${PARALLEL} + ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DMAKE_TEST=OFF \ + -DWITH_MYSQL=${WITH_MYSQL} -DWITH_LZO=${WITH_LZO} -DGLIBC_COMPATIBILITY=${GLIBC_COMPATIBILITY} ../ + ${BUILD_SYSTEM} -j ${PARALLEL} ${BUILD_SYSTEM} install cd ${DORIS_HOME} fi diff --git a/run-be-ut.sh b/run-be-ut.sh index dfdbf18748..ff78176256 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -106,9 +106,14 @@ if [ ! -d ${CMAKE_BUILD_DIR} ]; then mkdir -p ${CMAKE_BUILD_DIR} fi +if [[ -z ${GLIBC_COMPATIBILITY} ]]; then + GLIBC_COMPATIBILITY=OFF +fi + cd ${CMAKE_BUILD_DIR} -${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_MYSQL=OFF -DMAKE_TEST=ON -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -${BUILD_SYSTEM} -j${PARALLEL} +${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_MYSQL=OFF -DMAKE_TEST=ON -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DGLIBC_COMPATIBILITY=${GLIBC_COMPATIBILITY} +${BUILD_SYSTEM} -j ${PARALLEL} if [ ${RUN} -ne 1 ]; then echo "Finished"