database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[branch-2.1](thirdparty) upgrade arrow to 17.0.0 (#39773)

Browse Source 
pick #38572
This commit is contained in:
Xinyi Zou
2024-08-22 18:35:32 +08:00
committed by GitHub
parent 13b882a4cc
commit 40500f2b10
6 changed files with 43 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/pom.xml
View File

@ -691,6 +691,10 @@ under the License.
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty</artifactId>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty-buffer-patch</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>

2
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java
View File

@ -49,7 +49,7 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TMinMaxRuntimeFilterType;
import org.apache.doris.thrift.TRuntimeFilterType;
import cfjd.com.google.common.base.Preconditions;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;

160
fe/pom.xml
View File

@ -266,13 +266,15 @@ under the License.
<slf4j.version>2.0.6</slf4j.version>
<metrics-core.version>4.0.2</metrics-core.version>
<!--Netty 4.1.94 is not compatible with arrow flight.-->
<netty-all.version>4.1.104.Final</netty-all.version>
<!--Need to ensure that the version is the same as in arrow/java/pom.xml or compatible with it.-->
<netty-all.version>4.1.110.Final</netty-all.version>
<!--The dependence of transitive dependence cannot be ruled out, only Saving the nation through twisted ways.-->
<netty-3-test.version>3.10.6.Final</netty-3-test.version>
<objenesis.version>2.1</objenesis.version>
<!-- NOTE: Using grpc-java whose version is newer than 1.34.0 will break the build on CentOS 6 due to the obsolete GLIBC -->
<grpc-java.version>1.34.0</grpc-java.version>
<grpc.version>1.60.1</grpc.version>
<!--Need to ensure that the version is the same as in arrow/java/pom.xml or compatible with it.-->
<grpc.version>1.63.0</grpc.version>
<check.freamework.version>3.43.0</check.freamework.version>
<protobuf.version>3.24.3</protobuf.version>
<!-- we use protoc-jar-maven-plugin to generate protobuf generated code -->
@ -312,7 +314,7 @@ under the License.
<iceberg.version>1.4.3</iceberg.version>
<maxcompute.version>0.45.2-public</maxcompute.version>
<avro.version>1.11.3</avro.version>
<arrow.version>15.0.2</arrow.version>
<arrow.version>17.0.0</arrow.version>
<!-- hudi -->
<hudi.version>0.14.1</hudi.version>
<presto.hadoop.version>2.7.4-11</presto.hadoop.version>
@ -404,6 +406,20 @@ under the License.
</profiles>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>${netty-all.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-bom</artifactId>
<version>${grpc.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.apache.ivy</groupId>
<artifactId>ivy</artifactId>
@ -865,109 +881,6 @@ under the License.
<artifactId>metrics-core</artifactId>
<version>${metrics-core.version}</version>
</dependency>
<!-- netty dependencies -->
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
<scope>test</scope>
<version>${netty-3-test.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-udt</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-sctp</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-sctp</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-classes-kqueue</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-epoll</artifactId>
<classifier>linux-x86_64</classifier>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-rxtx</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-memcache</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-mqtt</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http2</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-classes-epoll</artifactId>
<version>${netty-all.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-unix-common</artifactId>
<version>${netty-all.version}</version>
</dependency>
<!-- velocity-engine-core dependency-->
<dependency>
<groupId>org.apache.velocity</groupId>
@ -1024,21 +937,6 @@ under the License.
<artifactId>objenesis</artifactId>
<version>${objenesis.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-protobuf</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-core</artifactId>
<version>${grpc.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
<dependency>
<groupId>com.google.protobuf</groupId>
@ -1602,25 +1500,10 @@ under the License.
<artifactId>tomcat-embed-core</artifactId>
<version>${tomcat-embed-core.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-context</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-api</artifactId>
<version>${grpc.version}</version>
</dependency>
<dependency>
<groupId>com.google.flatbuffers</groupId>
<artifactId>flatbuffers-java</artifactId>
@ -1636,6 +1519,11 @@ under the License.
<artifactId>arrow-memory-netty</artifactId>
<version>${arrow.version}</version>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty-buffer-patch</artifactId>
<version>${arrow.version}</version>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>flight-core</artifactId>

2
regression-test/framework/pom.xml
View File

@ -74,7 +74,7 @@ under the License.
<groovy.version>4.0.19</groovy.version>
<antlr.version>4.9.3</antlr.version>
<hadoop.version>2.8.0</hadoop.version>
<arrow.version>15.0.2</arrow.version>
<arrow.version>17.0.0</arrow.version>
</properties>
<build>
<plugins>

4
thirdparty/CHANGELOG.md vendored
View File

@ -2,6 +2,10 @@
This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`
## 20240815
- Modified: arrow 15.0.2 -> 17.0.0
## 20240325
- Modified: arrow 13.0.0 -> 15.0.2

18
thirdparty/vars.sh vendored
View File

@ -253,10 +253,10 @@ GRPC_SOURCE=grpc-1.54.3
GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de"
# arrow
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-15.0.2.tar.gz"
ARROW_NAME="apache-arrow-15.0.2.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-15.0.2"
ARROW_MD5SUM="d5d051b2698be901e5ad84e36d64e7ab"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-17.0.0.tar.gz"
ARROW_NAME="apache-arrow-17.0.0.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-17.0.0"
ARROW_MD5SUM="ba18bf83e2164abd34b9ac4cb164f0f0"
# Abseil
ABSEIL_DOWNLOAD="https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz"
@ -412,11 +412,11 @@ BENCHMARK_SOURCE=benchmark-1.8.0
BENCHMARK_MD5SUM="8ddf8571d3f6198d37852bcbd964f817"
# xsimd
# for arrow-13.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/9.0.1.tar.gz"
XSIMD_NAME="xsimd-9.0.1.tar.gz"
XSIMD_SOURCE=xsimd-9.0.1
XSIMD_MD5SUM="59f38fe3364acd7ed137771258812d6c"
# for arrow-17.0.0, if arrow upgrade, this version may also need to be changed
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.0.0.tar.gz"
XSIMD_NAME="13.0.0.tar.gz"
XSIMD_SOURCE=xsimd-13.0.0
XSIMD_MD5SUM="c661deb91836e82d3070f81032014fe6"
# simdjson
SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz"