diff --git a/.github/workflows/build-extension.yml b/.github/workflows/build-extension.yml index 5e43699db9..44889371ec 100644 --- a/.github/workflows/build-extension.yml +++ b/.github/workflows/build-extension.yml @@ -65,11 +65,22 @@ jobs: - name: Setup thrift run: | - sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y thrift-compiler=0.13.0-2build2 + pushd thirdparty + branch="${{ github.base_ref }}" + if [[ -z "${branch}" ]] || [[ "${branch}" == 'master' ]]; then + curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz \ + -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz + else + curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-${branch/branch-/}/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" \ + -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz + fi + tar -xvf doris-thirdparty-prebuilt-linux-x86_64.tar.xz + popd + export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}" + thrift --version - mkdir -p thirdparty/installed/bin/ - cd thirdparty/installed/bin/ && ln -s /usr/bin/thrift thrift + # mkdir -p thirdparty/installed/bin/ + # cd thirdparty/installed/bin/ && ln -s /usr/bin/thrift thrift - name: Build broker run: | cd fs_brokers/apache_hdfs_broker/ && /bin/bash build.sh diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt index f4a95b0e2b..3e86ed865f 100644 --- a/dist/LICENSE-dist.txt +++ b/dist/LICENSE-dist.txt @@ -683,22 +683,17 @@ The Apache Software License, Version 2.0 - org.apache.hadoop:hadoop-aws:2.7.3 (no url defined) - org.apache.hadoop:hadoop-aws:2.8.0 (no url defined) * Apache Hadoop Annotations: - - org.apache.hadoop:hadoop-annotations:2.6.5 (no url defined) - - org.apache.hadoop:hadoop-annotations:2.7.3 (no url defined) - - org.apache.hadoop:hadoop-annotations:2.8.0 (no url defined) + - org.apache.hadoop:hadoop-annotations:2.10.2 (no url defined) * Apache Hadoop Auth: - - org.apache.hadoop:hadoop-auth:2.7.3 (no url defined) - - org.apache.hadoop:hadoop-auth:2.8.0 (no url defined) + - org.apache.hadoop:hadoop-auth:2.10.2 (no url defined) * Apache Hadoop Client: - - org.apache.hadoop:hadoop-client:2.6.5 (no url defined) + - org.apache.hadoop:hadoop-client:2.10.2 (no url defined) * Apache Hadoop Common: - - org.apache.hadoop:hadoop-common:2.7.3 (no url defined) - - org.apache.hadoop:hadoop-common:2.8.0 (no url defined) + - org.apache.hadoop:hadoop-common:2.10.2 (no url defined) * Apache Hadoop HDFS: - - org.apache.hadoop:hadoop-hdfs:2.7.3 (no url defined) - - org.apache.hadoop:hadoop-hdfs:2.8.0 (no url defined) + - org.apache.hadoop:hadoop-hdfs:2.10.2 (no url defined) * Apache Hadoop HDFS Client: - - org.apache.hadoop:hadoop-hdfs-client:2.8.0 (no url defined) + - org.apache.hadoop:hadoop-hdfs-client:2.10.2 (no url defined) * Apache HttpClient: - org.apache.httpcomponents:httpclient:4.5.13 (http://hc.apache.org/httpcomponents-client) * Apache HttpCore: @@ -985,6 +980,14 @@ The Apache Software License, Version 2.0 - org.lz4:lz4-java:1.4.0 (https://github.com/lz4/lz4-java) * Metrics Core: - io.dropwizard.metrics:metrics-core:4.0.2 (http://metrics.dropwizard.io/metrics-core) + * avatica: + - org.apache.calcite.avatica:avatica:1.22.0 (https://calcite.apache.org/avatica/) + * calcite-core: + - org.apache.calcite:calcite-core:1.32.0 (https://calcite.apache.org/avatica/) + * calcite-core: + - org.apache.calcite:calcite-linq4j:1.32.0 (https://calcite.apache.org/avatica/) + * calcite-core: + - org.apache.calcite.:calcite-druid:1.32.0 (https://calcite.apache.org/avatica/) * Netty: - io.netty:netty:3.9.9.Final (http://netty.io/) * Netty Reactive Streams HTTP support: @@ -992,8 +995,7 @@ The Apache Software License, Version 2.0 * Netty Reactive Streams Implementation: - com.typesafe.netty:netty-reactive-streams:2.0.4 (https://github.com/playframework/netty-reactive-streams/netty-reactive-streams) * Netty/All-in-One: - - io.netty:netty-all:4.0.23.Final (http://netty.io/netty-all/) - - io.netty:netty-all:4.1.42.Final (https://netty.io/netty-all/) + - io.netty:netty-all:4.1.65.Final (https://netty.io/netty-all/) * Netty/Buffer: - io.netty:netty-buffer:4.1.72.Final (https://netty.io/netty-buffer/) * Netty/Codec: @@ -1190,7 +1192,7 @@ The Apache Software License, Version 2.0 * io.grpc:grpc-stub: - io.grpc:grpc-stub:1.30.0 (https://github.com/grpc/grpc-java) * jackson-databind: - - com.fasterxml.jackson.core:jackson-databind:2.12.1 (http://github.com/FasterXML/jackson) + - com.fasterxml.jackson.core:jackson-databind:2.14.1 (http://github.com/FasterXML/jackson) * jackson-module-scala: - com.fasterxml.jackson.module:jackson-module-scala_2.12:2.13.1 (http://wiki.fasterxml.com/JacksonModuleScala) - com.fasterxml.jackson.module:jackson-module-scala_2.12:2.6.7.1 (http://wiki.fasterxml.com/JacksonModuleScala) diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index e6b2280656..fe4530290f 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -85,7 +85,7 @@ under the License. org.apache.hadoop hadoop-mapreduce-client - 2.7.4 + ${hadoop.version} compile @@ -100,6 +100,12 @@ under the License. ${project.groupId} spark-dpp ${project.version} + + + netty-all + io.netty + + org.apache.logging.log4j @@ -438,6 +444,12 @@ under the License. org.apache.spark spark-core_2.12 + + + netty-all + io.netty + + @@ -464,6 +476,10 @@ under the License. ch.qos.logback logback-classic + + netty-all + io.netty + @@ -480,6 +496,10 @@ under the License. ch.qos.logback logback-classic + + netty-all + io.netty + @@ -607,6 +627,18 @@ under the License. org.apache.hive hive-metastore provided + + + netty-all + io.netty + + + + + + io.netty + netty-all + ${netty.version} org.apache.hive @@ -618,20 +650,89 @@ under the License. org.apache.velocity velocity + + commons-httpclient + commons-httpclient + + + calcite-core + org.apache.calcite + + + calcite-druid + org.apache.calcite + + + calcite-linq4j + org.apache.calcite + + + avatica + org.apache.calcite.avatica + - - - org.apache.velocity - velocity-engine-core - 2.3 - + + + + org.apache.calcite.avatica + avatica + ${avatica.version} + + + + org.apache.calcite + calcite-core + ${calcite.version} + + + + org.apache.calcite + calcite-linq4j + ${calcite.version} + + + + org.apache.calcite + calcite-druid + ${calcite.version} + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + org.apache.velocity + velocity-engine-core + 2.3 + org.apache.hadoop hadoop-hdfs provided + + + netty-all + io.netty + + + jackson-databind + com.fasterxml.jackson.core + + + + + com.fasterxml.jackson.core + jackson-databind + 2.14.1 + + + org.apache.hadoop + hadoop-auth + ${hadoop.version} + provided - io.opentelemetry @@ -682,6 +783,16 @@ under the License. org.apache.hudi hudi-common + + + commons-httpclient + commons-httpclient + + + netty-all + io.netty + + diff --git a/fe/hive-udf/pom.xml b/fe/hive-udf/pom.xml index 564b399062..17994353e1 100644 --- a/fe/hive-udf/pom.xml +++ b/fe/hive-udf/pom.xml @@ -48,8 +48,46 @@ under the License. org.apache.velocity velocity + + commons-httpclient + commons-httpclient + + + avatica + org.apache.calcite.avatica + + + + org.apache.calcite.avatica + avatica + ${avatica.version} + + + + org.apache.calcite + calcite-core + ${calcite.version} + + + + org.apache.calcite + calcite-linq4j + ${calcite.version} + + + + org.apache.calcite + calcite-druid + ${calcite.version} + + + + org.apache.httpcomponents + httpclient + 4.5.13 + org.apache.velocity velocity-engine-core diff --git a/fe/java-udf/pom.xml b/fe/java-udf/pom.xml index d8fc0da715..f65234957f 100644 --- a/fe/java-udf/pom.xml +++ b/fe/java-udf/pom.xml @@ -44,6 +44,16 @@ under the License. ${project.groupId} fe-core ${project.version} + + + netty-all + io.netty + + + commons-httpclient + commons-httpclient + + org.apache.hive @@ -58,6 +68,10 @@ under the License. org.apache.velocity velocity + + commons-httpclient + commons-httpclient + @@ -65,6 +79,11 @@ under the License. velocity-engine-core 2.3 + + org.apache.httpcomponents + httpclient + 4.5.13 + org.junit.jupiter diff --git a/fe/pom.xml b/fe/pom.xml index 143d18114c..6fed5b9fbc 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -220,6 +220,9 @@ under the License. 2.4.6 2.3.7 2.8.0 + 4.1.65.Final + 1.32.0 + 1.22.0 @@ -239,7 +242,7 @@ under the License. 1.11.95 3.0.4 0.2.14 - + 2.10.2 1.0-SNAPSHOT github @@ -308,7 +311,7 @@ under the License. org.apache.hadoop hadoop-client - 2.8.0 + ${hadoop.version} compile diff --git a/fe/spark-dpp/pom.xml b/fe/spark-dpp/pom.xml index 1d7bff6aae..8e792980d7 100644 --- a/fe/spark-dpp/pom.xml +++ b/fe/spark-dpp/pom.xml @@ -87,7 +87,24 @@ under the License. org.apache.spark spark-core_2.12 provided + + + netty-all + io.netty + + + netty + io.netty + + + + + io.netty + netty-all + ${netty.version} + + org.apache.spark diff --git a/fs_brokers/apache_hdfs_broker/pom.xml b/fs_brokers/apache_hdfs_broker/pom.xml index 6d1147b8c1..bea0443d56 100644 --- a/fs_brokers/apache_hdfs_broker/pom.xml +++ b/fs_brokers/apache_hdfs_broker/pom.xml @@ -69,7 +69,8 @@ under the License. 1.8 2.18.0 github - 2.9.1 + 2.10.2 + 4.1.65.Final @@ -211,7 +212,29 @@ under the License. org.apache.hadoop hadoop-hdfs ${hadoop.version} + + + netty-all + io.netty + + + jackson-databind + com.fasterxml.jackson.core + + + + com.fasterxml.jackson.core + jackson-databind + 2.14.1 + + + + io.netty + netty-all + ${netty.version} + + org.apache.htrace @@ -285,6 +308,12 @@ under the License. org.apache.hadoop hadoop-huaweicloud 2.8.3 + + + jackson-databind + com.fasterxml.jackson.core + + @@ -292,6 +321,7 @@ under the License. hadoop-aliyun ${hadoop.version} + com.qcloud.cos