database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor] update parent pom version and optimize build scripts (#7548)

Browse Source
This commit is contained in:
Zhengguo Yang
2022-01-05 10:45:11 +08:00
committed by GitHub
parent 9ddcf0625c
commit 738d2d2e07
23 changed files with 472 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/build-extension.yml vendored
View File

@ -58,15 +58,23 @@ jobs:
- name: Build spark connector v2
run: |
thrift --version
cd extension/spark-doris-connector/ && /bin/bash build.sh 2
cd extension/spark-doris-connector/ && /bin/bash build.sh 2.3.4 2.11
- name: Build spark connector v3
run: |
cd extension/spark-doris-connector/ && /bin/bash build.sh 3
cd extension/spark-doris-connector/ && /bin/bash build.sh 3.1.2 2.12
- name: Build flink connector
- name: Build flink connector 1.11
run: |
cd extension/flink-doris-connector/ && /bin/bash build.sh
cd extension/flink-doris-connector/ && /bin/bash build.sh 1.11.6 2.12
- name: Build flink connector 1.12
run: |
cd extension/flink-doris-connector/ && /bin/bash build.sh 1.12.7 2.12
- name: Build flink connector 1.13
run: |
cd extension/flink-doris-connector/ && /bin/bash build.sh 1.13.5 2.12
- name: Build docs
run: |

1
docs/.vuepress/sidebar/en.js
View File

@ -683,6 +683,7 @@ module.exports = [
"subscribe-mail-list",
"feedback",
"how-to-contribute",
"how-to-deploy-to-maven",
"committer-guide",
"commit-format-specification",
"pull-request",

1
docs/.vuepress/sidebar/zh-CN.js
View File

@ -687,6 +687,7 @@ module.exports = [
"subscribe-mail-list",
"feedback",
"how-to-contribute",
"how-to-deploy-to-maven",
"committer-guide",
"commit-format-specification",
"pull-request",

107
docs/en/community/how-to-deploy-to-maven.md Normal file
View File

@ -0,0 +1,107 @@
---
{
"title": "How to deploy to Maven Central Repository",
"language": "en"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
## Prepare
### 0. Requirements
1. apache account ID
2. apache password
3. gpg key
### 1. Prepare the local maven environment
1. Generate the master password: `mvn --encrypt-master-password <password>` This password is only used to encrypt other subsequent passwords, and the output is similar to `{VSb+6+76djkH/43...}` and then create` ~/.m2/settings-security.xml` file, the content is
```
<settingsSecurity>
<master>{VSb+6+76djkH/43...}</master>
</settingsSecurity>
```
2. Encrypt apache password: `mvn --encrypt-password <password>` This password is the password of the apache account. The output is similar to the above `{GRKbCylpwysHfV...}` and added in `~/.m2/settings.xml`
```
<servers>
<!-- To publish a snapshot of your project -->
<server>
<id>apache.snapshots.https</id>
<username>yangzhg</username>
<password>{GRKbCylpwysHfV...}</password>
</server>
<!-- To stage a release of your project -->
<server>
<id>apache.releases.https</id>
<username>yangzhg</username>
<password>{GRKbCylpwysHfV...}</password>
</server>
</servers>
```
3. Optional, (you can also pass -Darguments="-Dgpg.passphrase=xxxx" during deployment), add the following content in `~/.m2/settings.xml`, if the profiles tag already exists, this is only required Just add profile to profiles, activeProfiles is the same as above, xxxx is the passphrase of the gpg key
```
<profiles>
<profile>
<id>gpg</id>
<properties>
<gpg.executable>gpg</gpg.executable>
<gpg.passphrase>xxxx</gpg.passphrase>
</properties>
</profile>
</profiles>
<activeProfiles>
<activeProfile>gpg</activeProfile>
</activeProfiles>
```
### Publish to SNAPSHOT
### 1. Deploy flink connector
Switch to the flink connector directory, let’s take flink version 1.11.6 and scalar 2.12 as examples
```
cd incubator-doris/extension/flink-doris-connector
export DORIS_HOME=$PWD/../../
source ${DORIS_HOME}/env.sh
if [ -f ${DORIS_HOME}/custom_env.sh ]; then source ${DORIS_HOME}/custom_env.sh; fi
export FLINK_VERSION="1.11.6"
export SCALA_VERSION="2.12"
mvn deploy
```
### 2. Deploy Spark connector
Switch to the spark connector directory, let’s take spark version 2.3.4 and scalar 2.11 as examples
```
cd incubator-doris/extension/spark-doris-connector
export DORIS_HOME=$PWD/../../
source ${DORIS_HOME}/env.sh
if [ -f ${DORIS_HOME}/custom_env.sh ]; then source ${DORIS_HOME}/custom_env.sh; fi
export SPARK_VERSION="2.3.4"
export SCALA_VERSION="2.11"
mvn deploy
```

22
docs/en/extending-doris/flink-doris-connector.md
View File

@ -34,8 +34,9 @@ Flink Doris Connector can support read and write data stored in Doris through Fl
| Connector | Flink | Doris | Java | Scala |
| --------- | ----- | ------ | ---- | ----- |
| 1.0.0 | 1.11.2 | 0.13+ | 8 | 2.12 |
| 1.0.0 | 1.13.x | 0.13.+ | 8 | 2.12 |
| 1.11.6-2.12-xx | 1.11.x | 0.13+ | 8 | 2.12 |
| 1.12.7-2.12-xx | 1.12.x | 0.13.+ | 8 | 2.12 |
| 1.13.5-2.12-xx | 1.13.x | 0.13.+ | 8 | 2.12 |
**For Flink 1.13.x version adaptation issues**
@ -63,7 +64,7 @@ Execute following command in dir `extension/flink-doris-connector/`:
2. It is recommended to compile under the docker compile environment `apache/incubator-doris:build-env-1.2` of doris, because the JDK version below 1.3 is 11, there will be compilation problems.
```bash
sh build.sh
sh build.sh 1.11.6 2.12 # flink 1.11.6 scala 2.12
```
After successful compilation, the file `doris-flink-1.0.0-SNAPSHOT.jar` will be generated in the `output/` directory. Copy this file to `ClassPath` in `Flink` to use `Flink-Doris-Connector`. For example, `Flink` running in `Local` mode, put this file in the `jars/` folder. `Flink` running in `Yarn` cluster mode, put this file in the pre-deployment package.
@ -78,6 +79,21 @@ conf/fe.conf
```
enable_http_server_v2 = true
```
## Using Maven
Add Dependency
```
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>doris-flink-connector</artifactId>
<version>1.11.6-2.12-SNAPSHOT</version>
</dependency>
```
**Remarks**
`1.11.6 ` can be substitute with `1.12.7` or `1.13.5` base on flink version you are using
## How to use

29
docs/en/extending-doris/spark-doris-connector.md
View File

@ -35,10 +35,11 @@ Spark Doris Connector can support reading data stored in Doris and writing data
## Version Compatibility
| Connector | Spark | Doris | Java | Scala |
| --------- | ----- | ------ | ---- | ----- |
| 1.0.0 | 2.x | 0.12+ | 8 | 2.11 |
| 1.0.0 | 3.x | 0.12.+ | 8 | 2.12 |
| Connector | Spark | Doris | Java | Scala |
|---------------| ----- | ------ | ---- | ----- |
| 2.3.4-2.11.xx | 2.x | 0.12+ | 8 | 2.11 |
| 3.1.2-2.12.xx | 3.x | 0.12.+ | 8 | 2.12 |
## Build and Install
@ -51,12 +52,28 @@ Execute following command in dir `extension/spark-doris-connector/`:
2. It is recommended to compile under the docker compile environment `apache/incubator-doris:build-env-1.2` of doris, because the JDK version below 1.3 is 11, there will be compilation problems.
```bash
sh build.sh 3 ## spark 3.x version, the default is 3.1.2
sh build.sh 2 ## soark 2.x version, the default is 2.3.4
sh build.sh 2.3.4 2.11 ## spark 2.3.4 version, and scala 2.11
sh build.sh 3.1.2 2.12 ## spark 3.1.2 version, and scala 2.12
```
After successful compilation, the file `doris-spark-1.0.0-SNAPSHOT.jar` will be generated in the `output/` directory. Copy this file to `ClassPath` in `Spark` to use `Spark-Doris-Connector`. For example, `Spark` running in `Local` mode, put this file in the `jars/` folder. `Spark` running in `Yarn` cluster mode, put this file in the pre-deployment package.
## Using Maven
Add dependency
```
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>doris-spark-connector</artifactId>
<version>2.3.4-2.11-SNAPSHOT</version>
</dependency>
```
**Remark**
`2.3.4-2.11` can be repacled with `3.1.2-2.12` base on you spark and scala version
## Example
### Read

109
docs/zh-CN/community/how-to-deploy-to-maven.md Normal file
View File

@ -0,0 +1,109 @@
---
{
"title": "如何部署到Maven 仓库",
"language": "zh-CN"
}
---
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
## 准备
### 0. Requirements
1. apache 账号ID
2. apache 账号密码
3. gpg key
### 1. 准备本地maven 环境
1. 生成主密码： `mvn --encrypt-master-password <password>` 这个密码仅用作加密后续的其他密码使用, 输出类似 `{VSb+6+76djkH/43...}` 之后创建 `~/.m2/settings-security.xml` 文件，内容是
```
<settingsSecurity>
<master>{VSb+6+76djkH/43...}</master>
</settingsSecurity>
```
2. 加密 apache 密码： `mvn --encrypt-password <password>` 这个密码是apache 账号的密码 输出和上面类似`{GRKbCylpwysHfV...}` 在`~/.m2/settings.xml` 中加入
```
<servers>
<!-- To publish a snapshot of your project -->
<server>
<id>apache.snapshots.https</id>
<username>yangzhg</username>
<password>{GRKbCylpwysHfV...}</password>
</server>
<!-- To stage a release of your project -->
<server>
<id>apache.releases.https</id>
<username>yangzhg</username>
<password>{GRKbCylpwysHfV...}</password>
</server>
</servers>
```
3. 可选，（也可以在部署时传递-Darguments="-Dgpg.passphrase=xxxx"），在`~/.m2/settings.xml` 中加入如下内容，如果已经存在profiles 标签， 这只需要将profile 加入profiles 中即可，activeProfiles 同上， xxxx 是gpg 密钥的passphrase
```
<profiles>
<profile>
<id>gpg</id>
<properties>
<gpg.executable>gpg</gpg.executable>
<gpg.passphrase>xxxx</gpg.passphrase>
</properties>
</profile>
</profiles>
<activeProfiles>
<activeProfile>gpg</activeProfile>
</activeProfiles>
```
### 发布到SNAPSHOT
### 1. 部署 flink connector
切换到 flink connector 目录， 我们以 flink 版本 1.11.6， scalar 2.12 为例
```
cd incubator-doris/extension/flink-doris-connector
export DORIS_HOME=$PWD/../../
source ${DORIS_HOME}/env.sh
if [ -f ${DORIS_HOME}/custom_env.sh ]; then source ${DORIS_HOME}/custom_env.sh; fi
export FLINK_VERSION="1.11.6"
export SCALA_VERSION="2.12"
mvn deploy
```
### 2. 部署 Spark connector
切换到 spark connector 目录， 我们以 spark 版本 2.3.4， scalar 2.11 为例
```
cd incubator-doris/extension/spark-doris-connector
export DORIS_HOME=$PWD/../../
source ${DORIS_HOME}/env.sh
if [ -f ${DORIS_HOME}/custom_env.sh ]; then source ${DORIS_HOME}/custom_env.sh; fi
export SPARK_VERSION="2.3.4"
export SCALA_VERSION="2.11"
mvn deploy
```

28
docs/zh-CN/extending-doris/flink-doris-connector.md
View File

@ -36,8 +36,9 @@ Flink Doris Connector 可以支持通过 Flink 读写 Doris 中存储的数据
| Connector | Flink | Doris | Java | Scala |
| --------- | ----- | ------ | ---- | ----- |
| 1.0.0 | 1.11.x , 1.12.x | 0.13+ | 8 | 2.12 |
| 1.0.0 | 1.13.x | 0.13.+ | 8 | 2.12 |
| 1.11.6-2.12-xx | 1.11.x | 0.13+ | 8 | 2.12 |
| 1.12.7-2.12-xx | 1.12.x | 0.13.+ | 8 | 2.12 |
| 1.13.5-2.12-xx | 1.13.x | 0.13.+ | 8 | 2.12 |
**针对Flink 1.13.x版本适配问题**
@ -65,10 +66,10 @@ Flink Doris Connector 可以支持通过 Flink 读写 Doris 中存储的数据
2. 建议在 doris 的 docker 编译环境 `apache/incubator-doris:build-env-1.2` 下进行编译，因为 1.3 下面的JDK 版本是 11，会存在编译问题。
```bash
sh build.sh
sh build.sh 1.11.6 2.12 # flink 1.11.6 scala 2.12
```
编译成功后，会在 `output/` 目录下生成文件 `doris-flink-1.0.0-SNAPSHOT.jar` 。将此文件复制到 `Flink` 的 `ClassPath` 中即可使用 `Flink-Doris-Connector` 。例如， `Local` 模式运行的 `Flink` ，将此文件放入 `jars/` 文件夹下。 `Yarn` 集群模式运行的 `Flink` ，则将此文件放入预部署包中。：
编译成功后，会在 `output/` 目录下生成文件 `doris-flink-1.0.0-SNAPSHOT.jar` 。将此文件复制到 `Flink` 的 `ClassPath` 中即可使用 `Flink-Doris-Connector` 。例如， `Local` 模式运行的 `Flink` ，将此文件放入 `jars/` 文件夹下。 `Yarn` 集群模式运行的 `Flink` ，则将此文件放入预部署包中。
**备注**
@ -81,6 +82,24 @@ conf/fe.conf
enable_http_server_v2 = true
```
## 使用Maven 管理
添加 maven 依赖
```
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>doris-flink-connector</artifactId>
<version>1.11.6-2.12-SNAPSHOT</version>
</dependency>
```
**备注**
`1.11.6 ` 可以根据flink 版本替换成替换成 `1.12.7` 或者 `1.13.5`
## 使用方法
Flink读写Doris数据主要有三种方式
@ -322,3 +341,4 @@ outputFormat.close();
| DECIMALV2 | DECIMAL |
| TIME | DOUBLE |
| HLL | Unsupported datatype |

28
docs/zh-CN/extending-doris/spark-doris-connector.md
View File

@ -35,10 +35,10 @@ Spark Doris Connector 可以支持通过 Spark 读取 Doris 中存储的数据
## 版本兼容
| Connector | Spark | Doris | Java | Scala |
| --------- | ----- | ------ | ---- | ----- |
| 1.0.0 | 2.x | 0.12+ | 8 | 2.11 |
| 1.0.0 | 3.x | 0.12.+ | 8 | 2.12 |
| Connector | Spark | Doris | Java | Scala |
|---------------| ----- | ------ | ---- | ----- |
| 2.3.4-2.11.xx | 2.x | 0.12+ | 8 | 2.11 |
| 3.1.2-2.12.xx | 3.x | 0.12.+ | 8 | 2.12 |
## 编译与安装
@ -51,12 +51,28 @@ Spark Doris Connector 可以支持通过 Spark 读取 Doris 中存储的数据
2. 建议在 doris 的 docker 编译环境 `apache/incubator-doris:build-env-1.2` 下进行编译，因为 1.3 下面的JDK 版本是 11，会存在编译问题。
```bash
sh build.sh 3 ## spark 3.x版本，默认是3.1.2
sh build.sh 2 ## spark 2.x版本，默认是2.3.4
sh build.sh 2.3.4 2.11 ## spark 2.3.4, scala 2.11
sh build.sh 3.1.2 2.12 ## spark 3.1.2, scala 2.12
```
编译成功后，会在 `output/` 目录下生成文件 `doris-spark-1.0.0-SNAPSHOT.jar`。将此文件复制到 `Spark` 的 `ClassPath` 中即可使用 `Spark-Doris-Connector`。例如，`Local` 模式运行的 `Spark`，将此文件放入 `jars/` 文件夹下。`Yarn`集群模式运行的`Spark`，则将此文件放入预部署包中。
## 使用Maven管理
添加依赖
```
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>doris-spark-connector</artifactId>
<version>2.3.4-2.11-SNAPSHOT</version>
</dependency>
```
**注意**
`2.3.4-2.11` 可以根据spark和scala 版本替换成 `3.1.2-2.12`
## 使用示例
### 读取

38
extension/flink-doris-connector/build.sh
View File

@ -25,23 +25,49 @@
set -eo pipefail
ROOT=`dirname "$0"`
ROOT=`cd "$ROOT"; pwd`
usage() {
echo "
Usage:
$0 flink_version scala_version
e.g.:
$0 1.11.6 2.12
$0 1.12.7 2.12
$0 1.13.5 2.12
"
exit 1
}
if [ $# -ne 2 ]; then
usage
fi
ROOT=$(dirname "$0")
ROOT=$(
cd "$ROOT"
pwd
)
export DORIS_HOME=${ROOT}/../../
. ${DORIS_HOME}/env.sh
. "${DORIS_HOME}"/env.sh
# include custom environment variables
if [[ -f ${DORIS_HOME}/custom_env.sh ]]; then
. ${DORIS_HOME}/custom_env.sh
. "${DORIS_HOME}"/custom_env.sh
fi
# check maven
MVN_CMD=mvn
if [[ ! -z ${CUSTOM_MVN} ]]; then
if [[ -n ${CUSTOM_MVN} ]]; then
MVN_CMD=${CUSTOM_MVN}
fi
if [ -z "$1" ]; then
export FLINK_VERSION="$1"
fi
if [ -z "$2" ]; then
export SCALA_VERSION="$2"
fi
if ! ${MVN_CMD} --version; then
echo "Error: mvn is not found"
exit 1
@ -50,7 +76,6 @@ export MVN_CMD
rm -rf output/
${MVN_CMD} clean package
mkdir -p output/
cp target/doris-flink-*.jar ./output/
@ -59,4 +84,3 @@ echo "Successfully build Flink-Doris-Connector"
echo "*****************************************"
exit 0

38
extension/flink-doris-connector/pom.xml
View File

@ -19,18 +19,18 @@ specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>18</version>
<version>23</version>
</parent>
<groupId>org.apache.doris</groupId>
<artifactId>doris-flink-connector</artifactId>
<version>flink-${flink.version}-${scala.version}-SNAPSHOT</version>
<version>${flink.version}-${scala.version}-1.0.0-SNAPSHOT</version>
<name>Doris Flink Connector</name>
<url>https://doris.apache.org/</url>
<licenses>
@ -67,8 +67,8 @@ under the License.
</mailingList>
</mailingLists>
<properties>
<scala.version>2.12</scala.version>
<flink.version>1.11.2</flink.version>
<scala.version>${env.SCALA_VERSION}</scala.version>
<flink.version>${env.FLINK_VERSION}</flink.version>
<libthrift.version>0.13.0</libthrift.version>
<arrow.version>5.0.0</arrow.version>
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
@ -113,6 +113,30 @@ under the License.
</pluginRepository>
</pluginRepositories>
</profile>
<profile>
<id>fink-version</id>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>!env.FLINK_VERSION</name>
</property>
</activation>
<properties>
<env.FLINK_VERSION>1.11.6</env.FLINK_VERSION>
</properties>
</profile>
<profile>
<id>scala-version</id>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>!env.SCALA_VERSION</name>
</property>
</activation>
<properties>
<env.SCALA_VERSION>2.12</env.SCALA_VERSION>
</properties>
</profile>
<!-- for general repository -->
<profile>
@ -391,9 +415,9 @@ under the License.
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven-javadoc-plugin.version}</version>
<configuration>
<skip>true</skip>
<source>8</source>
<failOnError>false</failOnError>
<aggregate>true</aggregate>
</configuration>
<executions>
<execution>

49
extension/spark-doris-connector/build.sh
View File

@ -25,23 +25,38 @@
set -eo pipefail
ROOT=`dirname "$0"`
ROOT=`cd "$ROOT"; pwd`
usage() {
echo "
Usage:
$0 spark_version scala_version
e.g.:
$0 2.3.4 2.11
$0 3.1.2 2.12
"
exit 1
}
if [ $# -ne 2 ]; then
usage
fi
ROOT=$(dirname "$0")
ROOT=$(cd "$ROOT"; pwd)
export DORIS_HOME=${ROOT}/../../
export PATH=${DORIS_THIRDPARTY}/installed/bin:$PATH
. ${DORIS_HOME}/env.sh
. "${DORIS_HOME}"/env.sh
# include custom environment variables
if [[ -f ${DORIS_HOME}/custom_env.sh ]]; then
. ${DORIS_HOME}/custom_env.sh
. "${DORIS_HOME}"/custom_env.sh
fi
# check maven
MVN_CMD=mvn
if [[ ! -z ${CUSTOM_MVN} ]]; then
if [[ -n ${CUSTOM_MVN} ]]; then
MVN_CMD=${CUSTOM_MVN}
fi
if ! ${MVN_CMD} --version; then
@ -50,29 +65,15 @@ if ! ${MVN_CMD} --version; then
fi
export MVN_CMD
usage() {
echo "
Eg.
$0 2 build with spark 2.x
$0 3 build with spark 3.x
"
exit 1
}
if [ $# == 0 ]; then
usage
fi
rm -rf output/
if [ "$1"x == "3x" ]
then
${MVN_CMD} clean package -f pom_3.0.xml
if [ -z "$1" ]; then
export SPARK_VERSION="$1"
fi
if [ "$1"x == "2x" ]
then
${MVN_CMD} clean package
if [ -z "$2" ]; then
export SCALA_VERSION="$2"
fi
${MVN_CMD} clean package
mkdir -p output/
cp target/doris-spark-*.jar ./output/

39
extension/spark-doris-connector/pom.xml
View File

@ -19,18 +19,18 @@
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>18</version>
<version>23</version>
</parent>
<groupId>org.apache.doris</groupId>
<artifactId>doris-spark-connector</artifactId>
<version>${spark.version}-${scala.version}-SNAPSHOT</version>
<version>${spark.version}-${scala.version}-1.0.0-SNAPSHOT</version>
<name>Doris Spark Connector</name>
<url>https://doris.apache.org/</url>
<licenses>
@ -68,8 +68,8 @@
</mailingLists>
<properties>
<scala.version>2.11</scala.version>
<spark.version>2.3.4</spark.version>
<scala.version>${env.SCALA_VERSION}</scala.version>
<spark.version>${env.SPARK_VERSION}</spark.version>
<libthrift.version>0.13.0</libthrift.version>
<arrow.version>5.0.0</arrow.version>
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
@ -103,7 +103,30 @@
</pluginRepository>
</pluginRepositories>
</profile>
<profile>
<id>spark-version</id>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>!env.SPARK_VERSION</name>
</property>
</activation>
<properties>
<env.SPARK_VERSION>2.3.4</env.SPARK_VERSION>
</properties>
</profile>
<profile>
<id>scala-version</id>
<activation>
<activeByDefault>true</activeByDefault>
<property>
<name>!env.SCALA_VERSION</name>
</property>
</activation>
<properties>
<env.SCALA_VERSION>2.11</env.SCALA_VERSION>
</properties>
</profile>
<!-- for general repository -->
<profile>
<id>general-env</id>
@ -338,9 +361,9 @@
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven-javadoc-plugin.version}</version>
<configuration>
<skip>true</skip>
<source>8</source>
<failOnError>false</failOnError>
<aggregate>true</aggregate>
</configuration>
<executions>
<execution>

372
extension/spark-doris-connector/pom_3.0.xml
View File

@ -1,372 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>18</version>
</parent>
<groupId>org.apache</groupId>
<artifactId>doris-spark-connector</artifactId>
<version>${spark.version}-${scala.version}-SNAPSHOT</version>
<name>Doris Spark Connector</name>
<url>https://doris.apache.org/</url>
<licenses>
<license>
<name>Apache 2.0 License</name>
<url>https://www.apache.org/licenses/LICENSE-2.0.html</url>
<distribution>repo</distribution>
</license>
</licenses>
<scm>
<connection>scm:git:git@github.com:apache/incubator-doris.git</connection>
<developerConnection>scm:git:git@github.com:apache/incubator-doris.git</developerConnection>
<url>scm:git:git@github.com:apache/incubator-doris.git</url>
<tag>HEAD</tag>
</scm>
<issueManagement>
<system>GitHub</system>
<url>https://github.com/apache/incubator-doris/issues</url>
</issueManagement>
<mailingLists>
<mailingList>
<name>Dev Mailing List</name>
<post>dev@doris.apache.org</post>
<subscribe>dev-subscribe@doris.apache.org</subscribe>
<unsubscribe>dev-unsubscribe@doris.apache.org</unsubscribe>
</mailingList>
<mailingList>
<name>Commits Mailing List</name>
<post>commits@doris.apache.org</post>
<subscribe>commits-subscribe@doris.apache.org</subscribe>
<unsubscribe>commits-unsubscribe@doris.apache.org</unsubscribe>
</mailingList>
</mailingLists>
<properties>
<scala.version>2.12</scala.version>
<spark.version>3.1.2</spark.version>
<libthrift.version>0.13.0</libthrift.version>
<arrow.version>5.0.0</arrow.version>
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
<maven-javadoc-plugin.version>3.3.0</maven-javadoc-plugin.version>
<maven-source-plugin.version>3.2.1</maven-source-plugin.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<doris.thirdparty>${env.DORIS_THIRDPARTY}</doris.thirdparty>
</properties>
<profiles>
<!-- for custom internal repository -->
<profile>
<id>custom-env</id>
<activation>
<property>
<name>env.CUSTOM_MAVEN_REPO</name>
</property>
</activation>
<repositories>
<repository>
<id>custom-nexus</id>
<url>${env.CUSTOM_MAVEN_REPO}</url>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>custom-nexus</id>
<url>${env.CUSTOM_MAVEN_REPO}</url>
</pluginRepository>
</pluginRepositories>
</profile>
<!-- for general repository -->
<profile>
<id>general-env</id>
<activation>
<property>
<name>!env.CUSTOM_MAVEN_REPO</name>
</property>
</activation>
<repositories>
<repository>
<id>central</id>
<name>central maven repo https</name>
<url>https://repo.maven.apache.org/maven2</url>
</repository>
</repositories>
</profile>
</profiles>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
<version>${libthrift.version}</version>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<version>${arrow.version}</version>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty</artifactId>
<version>${arrow.version}</version>
<scope>runtime</scope>
</dependency>
<!--Test-->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-scala_${scala.version}</artifactId>
<version>1.4.7</version>
<exclusions>
<exclusion>
<artifactId>hamcrest-core</artifactId>
<groupId>org.hamcrest</groupId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<exclusions>
<exclusion>
<artifactId>hamcrest-core</artifactId>
<groupId>org.hamcrest</groupId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.10.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.10.0</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.27.Final</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<!-- add gensrc java build src dir -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.2.0</version>
<executions>
<execution>
<id>add-source</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<!-- add arbitrary num of src dirs here -->
<source>${project.build.directory}/generated-sources/thrift/</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.thrift.tools</groupId>
<artifactId>maven-thrift-plugin</artifactId>
<version>0.1.11</version>
<configuration>
<thriftExecutable>${doris.thirdparty}/installed/bin/thrift</thriftExecutable>
<generator>java:fullcamel</generator>
</configuration>
<executions>
<execution>
<id>thrift-sources</id>
<phase>generate-sources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.2.1</version>
<executions>
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<args>
<arg>-feature</arg>
</args>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.1</version>
<configuration>
<artifactSet>
<excludes>
<exclude>com.google.code.findbugs:*</exclude>
<exclude>org.slf4j:*</exclude>
</excludes>
</artifactSet>
<relocations>
<relocation>
<pattern>org.apache.arrow</pattern>
<shadedPattern>org.apache.doris.shaded.org.apache.arrow</shadedPattern>
</relocation>
<relocation>
<pattern>io.netty</pattern>
<shadedPattern>org.apache.doris.shaded.io.netty</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson</pattern>
<shadedPattern>org.apache.doris.shaded.com.fasterxml.jackson</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons.codec</pattern>
<shadedPattern>org.apache.doris.shaded.org.apache.commons.codec</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.flatbuffers</pattern>
<shadedPattern>org.apache.doris.shaded.com.google.flatbuffers</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.thrift</pattern>
<shadedPattern>org.apache.doris.shaded.org.apache.thrift</shadedPattern>
</relocation>
</relocations>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>${maven-javadoc-plugin.version}</version>
<configuration>
<source>8</source>
<failOnError>false</failOnError>
<aggregate>true</aggregate>
</configuration>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${maven-source-plugin.version}</version>
<configuration>
<attach>true</attach>
</configuration>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

11
fe/fe-common/pom.xml
View File

@ -19,8 +19,8 @@ specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@ -164,6 +164,13 @@ under the License.
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<!-- for FE java code style checking -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>

11
fe/fe-core/pom.xml
View File

@ -19,7 +19,7 @@ specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@ -665,7 +665,7 @@ under the License.
<classifier>core</classifier>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -739,6 +739,13 @@ under the License.
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<!-- protobuf -->
<plugin>

4
fe/pom.xml
View File

@ -19,13 +19,13 @@ specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>18</version>
<version>23</version>
</parent>
<groupId>org.apache.doris</groupId>

14
fe/spark-dpp/pom.xml
View File

@ -19,8 +19,8 @@ specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@ -103,7 +103,7 @@ under the License.
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.12 -->
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.12 -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.12</artifactId>
@ -308,7 +308,13 @@ under the License.
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
<pluginManagement>

2
fe_plugins/auditdemo/pom.xml
View File

@ -36,7 +36,7 @@ under the License.
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>fe</artifactId>
<artifactId>fe-core</artifactId>
<version>${doris.version}</version>
</dependency>

2
fe_plugins/auditloader/pom.xml
View File

@ -36,7 +36,7 @@ under the License.
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>fe</artifactId>
<artifactId>fe-core</artifactId>
<version>${doris.version}</version>
</dependency>
<dependency>

19
fe_plugins/pom.xml
View File

@ -25,7 +25,7 @@ under the License.
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>18</version>
<version>23</version>
</parent>
<groupId>org.apache.doris</groupId>
@ -73,9 +73,8 @@ under the License.
</modules>
<properties>
<doris.home>${env.DORIS_HOME}</doris.home>
<log4j2.version>2.17.1</log4j2.version>
<doris.version>>0.15-SNAPSHOT</doris.version>
<doris.version>0.15-SNAPSHOT</doris.version>
</properties>
<profiles>
<!-- for custom internal repository -->
@ -105,20 +104,6 @@ under the License.
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>fe</artifactId>
<version>0.15-SNAPSHOT</version>
<scope>system</scope>
<systemPath>${doris.home}/fe/fe-core/target/palo-fe.jar</systemPath>
</dependency>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>fe-common</artifactId>
<scope>system</scope>
<systemPath>${doris.home}/fe/fe-common/target/doris-fe-common.jar</systemPath>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>

2
fs_brokers/apache_hdfs_broker/pom.xml
View File

@ -25,7 +25,7 @@ under the License.
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>18</version>
<version>23</version>
</parent>
<groupId>org.apache.doris</groupId>

2
samples/connect/java/client/pom.xml
View File

@ -24,7 +24,7 @@ under the License.
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache</groupId>
<groupId>org.apache.doris</groupId>
<artifactId>doris-client</artifactId>
<version>0.0.1</version>
<packaging>jar</packaging>