From 2209b714d17f31478a7f78b130aedb926d25e6bf Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Wed, 12 Apr 2023 10:37:50 +0800 Subject: [PATCH] [chore](orc) Update orc lib to third party lib(1.8.3) using git submodule. (#18531) --- .gitmodules | 4 +++ be/CMakeLists.txt | 28 ++++++++++++++++--- be/src/apache-orc | 1 + build.sh | 5 ++++ docker/README.md | 2 ++ .../developer-guide/be-vscode-dev.md | 6 ++-- .../community/developer-guide/docker-dev.md | 2 ++ .../how-to-contribute/pull-request.md | 3 +- docs/en/docs/ecosystem/hive-bitmap-udf.md | 2 ++ .../developer-guide/be-vscode-dev.md | 2 ++ .../community/developer-guide/docker-dev.md | 2 ++ .../how-to-contribute/pull-request.md | 2 ++ docs/zh-CN/docs/ecosystem/hive-bitmap-udf.md | 2 ++ 13 files changed, 54 insertions(+), 7 deletions(-) create mode 160000 be/src/apache-orc diff --git a/.gitmodules b/.gitmodules index 145e050d54..06213cbb75 100644 --- a/.gitmodules +++ b/.gitmodules @@ -25,3 +25,7 @@ [submodule ".github/actions/clang-tidy-review"] path = .github/actions/clang-tidy-review url = https://github.com/ZedThree/clang-tidy-review.git +[submodule "be/src/apache-orc"] + path = be/src/apache-orc + url = https://github.com/apache/doris-thirdparty.git + branch = orc diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 0ca8ae7b4f..d54498e78c 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -301,9 +301,6 @@ set_target_properties(roaring PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib add_library(fmt STATIC IMPORTED) set_target_properties(fmt PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libfmt.a) -add_library(orc STATIC IMPORTED) -set_target_properties(orc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/liborc.a) - add_library(cctz STATIC IMPORTED) set_target_properties(cctz PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libcctz.a) @@ -437,6 +434,21 @@ if (OS_MACOSX) set_target_properties(intl PROPERTIES IMPORTED_LOCATION "${THIRDPARTY_DIR}/lib/libintl.a") endif() +option(BUILD_JAVA OFF) +option(BUILD_CPP_TESTS OFF) +option(STOP_BUILD_ON_WARNING OFF) +option(BUILD_LIBHDFSPP OFF) +SET(PROTOBUF_HOME "$ENV{DORIS_THIRDPARTY}/installed") +SET(SNAPPY_HOME "$ENV{DORIS_THIRDPARTY}/installed") +SET(LZ4_HOME "$ENV{DORIS_THIRDPARTY}/installed") +SET(LZ4_INCLUDE_DIR "$ENV{DORIS_THIRDPARTY}/installed/include/lz4") +SET(ZLIB_HOME "$ENV{DORIS_THIRDPARTY}/installed") +SET(ZSTD_HOME "$ENV{DORIS_THIRDPARTY}/installed") +SET(ZSTD_INCLUDE_DIR "$ENV{DORIS_THIRDPARTY}/installed/include/zstd") + +add_subdirectory(${SRC_DIR}/apache-orc EXCLUDE_FROM_ALL) +target_compile_options(orc PRIVATE -Wno-implicit-fallthrough) + # Check if functions are supported in this platform. All flags will generated # in gensrc/build/common/env_config.h. # You can check funcion here which depends on platform. Don't forget add this @@ -621,6 +633,11 @@ set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS} ${EXTRA_CXX_FLAGS}") set(CMAKE_C_FLAGS ${CMAKE_CXX_FLAGS}) # Set include dirs +include_directories( + ${SRC_DIR}/apache-orc/c++/include + ${CMAKE_CURRENT_BINARY_DIR}/src/apache-orc/c++/include +) + include_directories( ${SRC_DIR}/ ${TEST_DIR}/ @@ -744,7 +761,6 @@ set(COMMON_THIRDPARTY zstd arrow parquet - orc odbc cctz minizip @@ -829,6 +845,8 @@ if (WITH_MYSQL) set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} mysql) endif() +set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} orc) + set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP}) # Add all external dependencies. They should come after the palo libs. @@ -851,6 +869,7 @@ else() ) endif() + if (USE_JEMALLOC) set(MALLOCLIB jemalloc) else () @@ -1004,3 +1023,4 @@ get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTO foreach(dir ${dirs}) message(STATUS "dir='${dir}'") endforeach() + diff --git a/be/src/apache-orc b/be/src/apache-orc new file mode 160000 index 0000000000..baf9e30475 --- /dev/null +++ b/be/src/apache-orc @@ -0,0 +1 @@ +Subproject commit baf9e30475baa924bde1e4a135e6517845a08d53 diff --git a/build.sh b/build.sh index dc5bcfc5ce..301f8496da 100755 --- a/build.sh +++ b/build.sh @@ -247,6 +247,11 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a" ]]; then fi fi +if [[ ! -f "${DORIS_HOME}/be/src/apache-orc/README.md " ]]; then + echo "apache-orc not exists, need to update submodules ..." + git submodule update --init --recursive +fi + if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && "${BUILD_SPARK_DPP}" -eq 0 ]]; then clean_gensrc clean_be diff --git a/docker/README.md b/docker/README.md index bb52cb0d80..72143e6400 100644 --- a/docker/README.md +++ b/docker/README.md @@ -26,6 +26,8 @@ under the License. ```console $ cd /to/your/workspace/ $ git clone https://github.com/apache/doris.git + $ cd doris + $ git submodule update --init --recursive ``` You can remove the `.git` dir in `doris/` to make the dir size smaller. diff --git a/docs/en/community/developer-guide/be-vscode-dev.md b/docs/en/community/developer-guide/be-vscode-dev.md index f2e65ac483..bf70f93c2c 100644 --- a/docs/en/community/developer-guide/be-vscode-dev.md +++ b/docs/en/community/developer-guide/be-vscode-dev.md @@ -57,10 +57,12 @@ sudo apt install -y openssl libssl-dev The following steps are carried out in the /home/workspace directory -1. dowload source +1. download source ``` -git clone https://github.com/apache/doris.git +git clone https://github.com/apache/doris.git +cd doris +git submodule update --init --recursive ``` 2. Compile third-party dependency packages diff --git a/docs/en/community/developer-guide/docker-dev.md b/docs/en/community/developer-guide/docker-dev.md index a48ca0c7fe..7d5f70ba33 100644 --- a/docs/en/community/developer-guide/docker-dev.md +++ b/docs/en/community/developer-guide/docker-dev.md @@ -112,6 +112,8 @@ create directory and download doris su mkdir code && cd code git clone https://github.com/apache/doris.git +cd doris +git submodule update --init --recursive ``` ## Compile diff --git a/docs/en/community/how-to-contribute/pull-request.md b/docs/en/community/how-to-contribute/pull-request.md index f7cf3ac73e..9670d9f91a 100644 --- a/docs/en/community/how-to-contribute/pull-request.md +++ b/docs/en/community/how-to-contribute/pull-request.md @@ -40,6 +40,8 @@ Go to the [github page](https://github.com/apache/doris) of apache/doris , and c ``` git clone https://github.com//doris.git +cd doris +git submodule update --init --recursive ``` Note: Please replace your GitHub name with your yourgithubname. @@ -49,7 +51,6 @@ When clone is completed, origin defaults to the remote fork address on github. #### (2) Add apache/doris to the remote branch upstream of the local warehouse: ``` -cd doris git remote add upstream https://github.com/apache/doris.git ``` diff --git a/docs/en/docs/ecosystem/hive-bitmap-udf.md b/docs/en/docs/ecosystem/hive-bitmap-udf.md index cc23a29af5..6629b9d6a2 100644 --- a/docs/en/docs/ecosystem/hive-bitmap-udf.md +++ b/docs/en/docs/ecosystem/hive-bitmap-udf.md @@ -62,6 +62,8 @@ CREATE TABLE IF NOT EXISTS `hive_table`( ```sql --clone doris code git clone https://github.com/apache/doris.git +cd doris +git submodule update --init --recursive --install thrift --Enter the fe directory cd fe diff --git a/docs/zh-CN/community/developer-guide/be-vscode-dev.md b/docs/zh-CN/community/developer-guide/be-vscode-dev.md index 418d629be1..7a18a186f3 100644 --- a/docs/zh-CN/community/developer-guide/be-vscode-dev.md +++ b/docs/zh-CN/community/developer-guide/be-vscode-dev.md @@ -59,6 +59,8 @@ sudo apt install -y openssl libssl-dev ``` git clone https://github.com/apache/doris.git +cd doris +git submodule update --init --recursive ``` 2. 编译第三方依赖包 diff --git a/docs/zh-CN/community/developer-guide/docker-dev.md b/docs/zh-CN/community/developer-guide/docker-dev.md index 3a36b5a6f2..5a81820196 100644 --- a/docs/zh-CN/community/developer-guide/docker-dev.md +++ b/docs/zh-CN/community/developer-guide/docker-dev.md @@ -113,6 +113,8 @@ plugins=(git zsh-autosuggestions zsh-syntax-highlighting) su mkdir code && cd code git clone https://github.com/apache/doris.git +cd doris +git submodule update --init --recursive ``` ## 编译 diff --git a/docs/zh-CN/community/how-to-contribute/pull-request.md b/docs/zh-CN/community/how-to-contribute/pull-request.md index d8c0e08407..04fb9065e2 100644 --- a/docs/zh-CN/community/how-to-contribute/pull-request.md +++ b/docs/zh-CN/community/how-to-contribute/pull-request.md @@ -40,6 +40,8 @@ under the License. ``` git clone https://github.com//doris.git +cd doris +git submodule update --init --recursive ``` 注意：请将 \ 替换为您的 github 名字。 diff --git a/docs/zh-CN/docs/ecosystem/hive-bitmap-udf.md b/docs/zh-CN/docs/ecosystem/hive-bitmap-udf.md index c4b3c984b6..7f886352ed 100644 --- a/docs/zh-CN/docs/ecosystem/hive-bitmap-udf.md +++ b/docs/zh-CN/docs/ecosystem/hive-bitmap-udf.md @@ -64,6 +64,8 @@ Hive Bitmap UDF 需要在 Hive/Spark 中使用，首先需要编译fe得到hive- ```sql --clone doris源码 git clone https://github.com/apache/doris.git +cd doris +git submodule update --init --recursive --安装thrift --进入fe目录 cd fe