[deps](fe)upgrade deps version (#15262)

upgrade hadoop version to 2.10.2 jackson-databind to 2.14.1
2022-12-24 22:18:10 +08:00
parent 82d316b419
commit d48abd91df
8 changed files with 260 additions and 29 deletions
--- a/.github/workflows/build-extension.yml
+++ b/.github/workflows/build-extension.yml
@ -65,11 +65,22 @@ jobs:

      - name: Setup thrift
        run: |
-          sudo apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y thrift-compiler=0.13.0-2build2
+          pushd thirdparty
+          branch="${{ github.base_ref }}"
+          if [[ -z "${branch}" ]] || [[ "${branch}" == 'master' ]]; then
+            curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz \
+              -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz
+          else
+            curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-${branch/branch-/}/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" \
+              -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz
+          fi
+          tar -xvf doris-thirdparty-prebuilt-linux-x86_64.tar.xz
+          popd
+          export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
+          
          thrift --version
-          mkdir -p thirdparty/installed/bin/
-          cd thirdparty/installed/bin/ && ln -s /usr/bin/thrift thrift
+          # mkdir -p thirdparty/installed/bin/
+          # cd thirdparty/installed/bin/ && ln -s /usr/bin/thrift thrift
      - name: Build broker
        run: |
          cd fs_brokers/apache_hdfs_broker/ && /bin/bash build.sh
--- a/dist/LICENSE-dist.txt
+++ b/dist/LICENSE-dist.txt
@ -683,22 +683,17 @@ The Apache Software License, Version 2.0
        - org.apache.hadoop:hadoop-aws:2.7.3 (no url defined)
        - org.apache.hadoop:hadoop-aws:2.8.0 (no url defined)
    * Apache Hadoop Annotations:
-        - org.apache.hadoop:hadoop-annotations:2.6.5 (no url defined)
-        - org.apache.hadoop:hadoop-annotations:2.7.3 (no url defined)
-        - org.apache.hadoop:hadoop-annotations:2.8.0 (no url defined)
+        - org.apache.hadoop:hadoop-annotations:2.10.2 (no url defined)
    * Apache Hadoop Auth:
-        - org.apache.hadoop:hadoop-auth:2.7.3 (no url defined)
-        - org.apache.hadoop:hadoop-auth:2.8.0 (no url defined)
+        - org.apache.hadoop:hadoop-auth:2.10.2 (no url defined)
    * Apache Hadoop Client:
-        - org.apache.hadoop:hadoop-client:2.6.5 (no url defined)
+        - org.apache.hadoop:hadoop-client:2.10.2 (no url defined)
    * Apache Hadoop Common:
-        - org.apache.hadoop:hadoop-common:2.7.3 (no url defined)
-        - org.apache.hadoop:hadoop-common:2.8.0 (no url defined)
+        - org.apache.hadoop:hadoop-common:2.10.2 (no url defined)
    * Apache Hadoop HDFS:
-        - org.apache.hadoop:hadoop-hdfs:2.7.3 (no url defined)
-        - org.apache.hadoop:hadoop-hdfs:2.8.0 (no url defined)
+        - org.apache.hadoop:hadoop-hdfs:2.10.2 (no url defined)
    * Apache Hadoop HDFS Client:
-        - org.apache.hadoop:hadoop-hdfs-client:2.8.0 (no url defined)
+        - org.apache.hadoop:hadoop-hdfs-client:2.10.2 (no url defined)
    * Apache HttpClient:
        - org.apache.httpcomponents:httpclient:4.5.13 (http://hc.apache.org/httpcomponents-client)
    * Apache HttpCore:
@ -985,6 +980,14 @@ The Apache Software License, Version 2.0
        - org.lz4:lz4-java:1.4.0 (https://github.com/lz4/lz4-java)
    * Metrics Core:
        - io.dropwizard.metrics:metrics-core:4.0.2 (http://metrics.dropwizard.io/metrics-core)
+    * avatica:
+        - org.apache.calcite.avatica:avatica:1.22.0 (https://calcite.apache.org/avatica/)        
+    * calcite-core:
+        - org.apache.calcite:calcite-core:1.32.0 (https://calcite.apache.org/avatica/)        
+    * calcite-core:
+        - org.apache.calcite:calcite-linq4j:1.32.0 (https://calcite.apache.org/avatica/)        
+    * calcite-core:
+        - org.apache.calcite.:calcite-druid:1.32.0 (https://calcite.apache.org/avatica/)        
    * Netty:
        - io.netty:netty:3.9.9.Final (http://netty.io/)
    * Netty Reactive Streams HTTP support:
@ -992,8 +995,7 @@ The Apache Software License, Version 2.0
    * Netty Reactive Streams Implementation:
        - com.typesafe.netty:netty-reactive-streams:2.0.4 (https://github.com/playframework/netty-reactive-streams/netty-reactive-streams)
    * Netty/All-in-One:
-        - io.netty:netty-all:4.0.23.Final (http://netty.io/netty-all/)
-        - io.netty:netty-all:4.1.42.Final (https://netty.io/netty-all/)
+        - io.netty:netty-all:4.1.65.Final (https://netty.io/netty-all/)
    * Netty/Buffer:
        - io.netty:netty-buffer:4.1.72.Final (https://netty.io/netty-buffer/)
    * Netty/Codec:
@ -1190,7 +1192,7 @@ The Apache Software License, Version 2.0
    * io.grpc:grpc-stub:
        - io.grpc:grpc-stub:1.30.0 (https://github.com/grpc/grpc-java)
    * jackson-databind:
-        - com.fasterxml.jackson.core:jackson-databind:2.12.1 (http://github.com/FasterXML/jackson)
+        - com.fasterxml.jackson.core:jackson-databind:2.14.1 (http://github.com/FasterXML/jackson)
    * jackson-module-scala:
        - com.fasterxml.jackson.module:jackson-module-scala_2.12:2.13.1 (http://wiki.fasterxml.com/JacksonModuleScala)
        - com.fasterxml.jackson.module:jackson-module-scala_2.12:2.6.7.1 (http://wiki.fasterxml.com/JacksonModuleScala)
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@ -85,7 +85,7 @@ under the License.
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client</artifactId>
-                <version>2.7.4</version>
+                <version>${hadoop.version}</version>
                <scope>compile</scope>
            </dependency>
        </dependencies>
@ -100,6 +100,12 @@ under the License.
            <groupId>${project.groupId}</groupId>
            <artifactId>spark-dpp</artifactId>
            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
+            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
@ -438,6 +444,12 @@ under the License.
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
+            <exclusions>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
+            </exclusions>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-launcher_2.12 -->
        <dependency>
@ -464,6 +476,10 @@ under the License.
                    <groupId>ch.qos.logback</groupId>
                    <artifactId>logback-classic</artifactId>
                </exclusion>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
            </exclusions>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.alibaba.otter/canal.protocol -->
@ -480,6 +496,10 @@ under the License.
                    <groupId>ch.qos.logback</groupId>
                    <artifactId>logback-classic</artifactId>
                </exclusion>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
@ -607,6 +627,18 @@ under the License.
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-metastore</artifactId>
            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-all</artifactId>
+            <version>${netty.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hive</groupId>
@ -618,20 +650,89 @@ under the License.
                    <groupId>org.apache.velocity</groupId>
                    <artifactId>velocity</artifactId>
                </exclusion>
+                <exclusion>
+                    <artifactId>commons-httpclient</artifactId>
+                    <groupId>commons-httpclient</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>calcite-core</artifactId>
+                    <groupId>org.apache.calcite</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>calcite-druid</artifactId>
+                    <groupId>org.apache.calcite</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>calcite-linq4j</artifactId>
+                    <groupId>org.apache.calcite</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>avatica</artifactId>
+                    <groupId>org.apache.calcite.avatica</groupId>
+                </exclusion>
            </exclusions>
-    </dependency>
-    <dependency>
-        <groupId>org.apache.velocity</groupId>
-        <artifactId>velocity-engine-core</artifactId>
-        <version>2.3</version>
-    </dependency>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.calcite.avatica/avatica -->
+        <dependency>
+            <groupId>org.apache.calcite.avatica</groupId>
+            <artifactId>avatica</artifactId>
+            <version>${avatica.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.calcite/calcite-core -->
+        <dependency>
+            <groupId>org.apache.calcite</groupId>
+            <artifactId>calcite-core</artifactId>
+            <version>${calcite.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.calcite/calcite-linq4j -->
+        <dependency>
+            <groupId>org.apache.calcite</groupId>
+            <artifactId>calcite-linq4j</artifactId>
+            <version>${calcite.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.calcite/calcite-druid -->
+        <dependency>
+            <groupId>org.apache.calcite</groupId>
+            <artifactId>calcite-druid</artifactId>
+            <version>${calcite.version}</version>
+        </dependency>
+        <dependency>
+           <groupId>org.apache.httpcomponents</groupId>
+           <artifactId>httpclient</artifactId>
+           <version>4.5.13</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.velocity</groupId>
+            <artifactId>velocity-engine-core</artifactId>
+            <version>2.3</version>
+        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
+                 <exclusion>
+                    <artifactId>jackson-databind</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>               
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.14.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-auth</artifactId>
+            <version>${hadoop.version}</version>
+            <scope>provided</scope>
        </dependency>
-
        <!-- https://mvnrepository.com/artifact/io.opentelemetry/opentelemetry-api -->
        <dependency>
            <groupId>io.opentelemetry</groupId>
@ -682,6 +783,16 @@ under the License.
        <dependency>
            <groupId>org.apache.hudi</groupId>
            <artifactId>hudi-common</artifactId>
+            <exclusions>
+                <exclusion>
+                    <artifactId>commons-httpclient</artifactId>
+                    <groupId>commons-httpclient</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
+            </exclusions>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.hudi/hudi-hadoop-mr -->
--- a/fe/hive-udf/pom.xml
+++ b/fe/hive-udf/pom.xml
@ -48,8 +48,46 @@ under the License.
                    <groupId>org.apache.velocity</groupId>
                    <artifactId>velocity</artifactId>
                </exclusion>
+                <exclusion>
+                    <artifactId>commons-httpclient</artifactId>
+                    <groupId>commons-httpclient</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>avatica</artifactId>
+                    <groupId>org.apache.calcite.avatica</groupId>
+                </exclusion>
            </exclusions>
        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.calcite.avatica/avatica -->
+        <dependency>
+            <groupId>org.apache.calcite.avatica</groupId>
+            <artifactId>avatica</artifactId>
+            <version>${avatica.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.calcite/calcite-core -->
+        <dependency>
+            <groupId>org.apache.calcite</groupId>
+            <artifactId>calcite-core</artifactId>
+            <version>${calcite.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.calcite/calcite-linq4j -->
+        <dependency>
+            <groupId>org.apache.calcite</groupId>
+            <artifactId>calcite-linq4j</artifactId>
+            <version>${calcite.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.calcite/calcite-druid -->
+        <dependency>
+            <groupId>org.apache.calcite</groupId>
+            <artifactId>calcite-druid</artifactId>
+            <version>${calcite.version}</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.13</version>
+        </dependency>
        <dependency>
            <groupId>org.apache.velocity</groupId>
            <artifactId>velocity-engine-core</artifactId>
--- a/fe/java-udf/pom.xml
+++ b/fe/java-udf/pom.xml
@ -44,6 +44,16 @@ under the License.
            <groupId>${project.groupId}</groupId>
            <artifactId>fe-core</artifactId>
            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>commons-httpclient</artifactId>
+                    <groupId>commons-httpclient</groupId>
+                </exclusion>
+            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.apache.hive</groupId>
@ -58,6 +68,10 @@ under the License.
                    <groupId>org.apache.velocity</groupId>
                    <artifactId>velocity</artifactId>
                </exclusion>
+                <exclusion>
+                    <artifactId>commons-httpclient</artifactId>
+                    <groupId>commons-httpclient</groupId>
+                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
@ -65,6 +79,11 @@ under the License.
            <artifactId>velocity-engine-core</artifactId>
            <version>2.3</version>
        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.13</version>
+        </dependency>
        <!-- https://mvnrepository.com/artifact/org.junit.jupiter/junit-jupiter-engine -->
        <dependency>
            <groupId>org.junit.jupiter</groupId>
--- a/fe/pom.xml
+++ b/fe/pom.xml
@ -220,6 +220,9 @@ under the License.
        <spark.version>2.4.6</spark.version>
        <hive.version>2.3.7</hive.version>
        <hadoop.version>2.8.0</hadoop.version>
+        <netty.version>4.1.65.Final</netty.version>
+        <calcite.version>1.32.0</calcite.version>
+        <avatica.version>1.22.0</avatica.version>
        <!-- ATTN: avro version must be consistent with Iceberg version -->
        <!-- Please modify iceberg.version and avro.version together,
         you can find avro version info in iceberg mvn repository -->
@ -239,7 +242,7 @@ under the License.
        <aws-java-sdk-s3.version>1.11.95</aws-java-sdk-s3.version>
        <mariadb-java-client.version>3.0.4</mariadb-java-client.version>
        <dlf-metastore-client-hive2.version>0.2.14</dlf-metastore-client-hive2.version>
-
+        <hadoop.version>2.10.2</hadoop.version>
        <revision>1.0-SNAPSHOT</revision>
        <project.scm.id>github</project.scm.id>
    </properties>
@ -308,7 +311,7 @@ under the License.
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-client</artifactId>
-                <version>2.8.0</version>
+                <version>${hadoop.version}</version>
                <scope>compile</scope>
            </dependency>
            <dependency>
--- a/fe/spark-dpp/pom.xml
+++ b/fe/spark-dpp/pom.xml
@ -87,7 +87,24 @@ under the License.
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>netty</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
+            </exclusions>
        </dependency>
+        <!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-all</artifactId>
+            <version>${netty.version}</version>
+        </dependency>
+
        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.12 -->
        <dependency>
            <groupId>org.apache.spark</groupId>
--- a/fs_brokers/apache_hdfs_broker/pom.xml
+++ b/fs_brokers/apache_hdfs_broker/pom.xml
@ -69,7 +69,8 @@ under the License.
        <maven.compiler.target>1.8</maven.compiler.target>
        <log4j2.version>2.18.0</log4j2.version>
        <project.scm.id>github</project.scm.id>
-        <hadoop.version>2.9.1</hadoop.version>
+        <hadoop.version>2.10.2</hadoop.version>
+        <netty.version>4.1.65.Final</netty.version>
    </properties>
    <profiles>
        <!-- for custom internal repository -->
@ -211,7 +212,29 @@ under the License.
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>netty-all</artifactId>
+                    <groupId>io.netty</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jackson-databind</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>
+            </exclusions>
        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.14.1</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-all</artifactId>
+            <version>${netty.version}</version>
+        </dependency>
+
        <!-- https://mvnrepository.com/artifact/org.apache.htrace/htrace-core -->
        <dependency>
            <groupId>org.apache.htrace</groupId>
@ -285,6 +308,12 @@ under the License.
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-huaweicloud</artifactId>
            <version>2.8.3</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>jackson-databind</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>
+            </exclusions>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aliyun -->
        <dependency>
@ -292,6 +321,7 @@ under the License.
            <artifactId>hadoop-aliyun</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
+
        <!-- https://mvnrepository.com/artifact/com.qcloud.cos/hadoop-cos -->
        <dependency>
            <groupId>com.qcloud.cos</groupId>