0e79f6908b
This CL mainly changes:
1. Add 2 new FE modules
1. fe-common
save all common classes for other modules, currently only `jmockit`
2. spark-dpp
The Spark DPP application for Spark Load. And I removed all dpp related classes to this module, including unit tests.
2. Change the `build.sh`
Add a new param `--spark-dpp` to compile the `spark-dpp` alone. And `--fe` will compile all FE modules.
the output of `spark-dpp` module is `spark-dpp-1.0.0-jar-with-dependencies.jar`, and it will be installed to `output/fe/spark-dpp/`.
3. Modify some bugs of spark load
431 lines
18 KiB
XML
431 lines
18 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
|
|
<!--
|
|
Licensed to the Apache Software Foundation (ASF) under one
|
|
or more contributor license agreements. See the NOTICE file
|
|
distributed with this work for additional information
|
|
regarding copyright ownership. The ASF licenses this file
|
|
to you under the Apache License, Version 2.0 (the
|
|
"License"); you may not use this file except in compliance
|
|
with the License. You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing,
|
|
software distributed under the License is distributed on an
|
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
KIND, either express or implied. See the License for the
|
|
specific language governing permissions and limitations
|
|
under the License.
|
|
-->
|
|
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<parent>
|
|
<groupId>org.apache</groupId>
|
|
<artifactId>doris-fe</artifactId>
|
|
<version>3.4.0</version>
|
|
<relativePath>../pom.xml</relativePath>
|
|
</parent>
|
|
|
|
<artifactId>spark-dpp</artifactId>
|
|
<version>1.0.0</version>
|
|
<packaging>jar</packaging>
|
|
|
|
<properties>
|
|
<doris.home>${basedir}/../../</doris.home>
|
|
<fe_ut_parallel>1</fe_ut_parallel>
|
|
</properties>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache</groupId>
|
|
<artifactId>fe-common</artifactId>
|
|
</dependency>
|
|
|
|
<!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
|
|
<dependency>
|
|
<groupId>commons-codec</groupId>
|
|
<artifactId>commons-codec</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
|
|
<dependency>
|
|
<groupId>org.apache.commons</groupId>
|
|
<artifactId>commons-lang3</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
|
|
<dependency>
|
|
<groupId>com.google.code.gson</groupId>
|
|
<artifactId>gson</artifactId>
|
|
</dependency>
|
|
|
|
<!-- https://mvnrepository.com/artifact/org.jmockit/jmockit -->
|
|
<dependency>
|
|
<groupId>org.jmockit</groupId>
|
|
<artifactId>jmockit</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<!-- https://mvnrepository.com/artifact/joda-time/joda-time -->
|
|
<dependency>
|
|
<groupId>joda-time</groupId>
|
|
<artifactId>joda-time</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<!-- https://mvnrepository.com/artifact/junit/junit -->
|
|
<dependency>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<!-- https://mvnrepository.com/artifact/log4j/log4j -->
|
|
<dependency>
|
|
<groupId>log4j</groupId>
|
|
<artifactId>log4j</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.roaringbitmap</groupId>
|
|
<artifactId>RoaringBitmap</artifactId>
|
|
</dependency>
|
|
|
|
<!-- spark -->
|
|
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12 -->
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_2.12</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.12 -->
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_2.12</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-column</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-hadoop</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-common</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>commons-collections</groupId>
|
|
<artifactId>commons-collections</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-library</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.esotericsoftware</groupId>
|
|
<artifactId>kryo-shaded</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-catalyst_2.12</artifactId>
|
|
<version>2.4.5</version>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.google.guava</groupId>
|
|
<artifactId>guava</artifactId>
|
|
</dependency>
|
|
</dependencies>
|
|
|
|
<build>
|
|
<finalName>spark-dpp-${version}</finalName>
|
|
|
|
<plugins>
|
|
<!-- jmockit -->
|
|
<plugin>
|
|
<artifactId>maven-surefire-plugin</artifactId>
|
|
<version>2.22.2</version>
|
|
<configuration>
|
|
<!-->set larger, eg, 3, to reduce the time or running FE unit tests<-->
|
|
<forkCount>${fe_ut_parallel}</forkCount>
|
|
<!-->not reuse forked jvm, so that each unit test will run in separate jvm. to avoid singleton confict<-->
|
|
<reuseForks>false</reuseForks>
|
|
<argLine>
|
|
-javaagent:${settings.localRepository}/org/jmockit/jmockit/1.48/jmockit-1.48.jar
|
|
</argLine>
|
|
</configuration>
|
|
</plugin>
|
|
|
|
<!-- copy all dependency libs to target lib dir -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-dependency-plugin</artifactId>
|
|
<version>3.1.1</version>
|
|
<executions>
|
|
<execution>
|
|
<id>copy-dependencies</id>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>copy-dependencies</goal>
|
|
</goals>
|
|
<configuration>
|
|
<outputDirectory>${project.build.directory}/lib</outputDirectory>
|
|
<overWriteReleases>false</overWriteReleases>
|
|
<overWriteSnapshots>false</overWriteSnapshots>
|
|
<overWriteIfNewer>true</overWriteIfNewer>
|
|
<skip>${skip.plugin}</skip>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<plugin>
|
|
<artifactId>maven-assembly-plugin</artifactId>
|
|
<configuration>
|
|
<archive>
|
|
<manifest>
|
|
<mainClass>org.apache.doris.load.loadv2.etl.SparkEtlJob</mainClass>
|
|
</manifest>
|
|
</archive>
|
|
<descriptorRefs>
|
|
<descriptorRef>jar-with-dependencies</descriptorRef>
|
|
</descriptorRefs>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<id>make-assembly</id> <!-- this is used for inheritance merges -->
|
|
<phase>package</phase> <!-- bind to the packaging phase -->
|
|
<goals>
|
|
<goal>single</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<plugin>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>cobertura-maven-plugin</artifactId>
|
|
<version>2.7</version>
|
|
<configuration>
|
|
<check>
|
|
<maxmem>1024m</maxmem>
|
|
</check>
|
|
</configuration>
|
|
</plugin>
|
|
|
|
<!-- for FE java code style checking -->
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-checkstyle-plugin</artifactId>
|
|
<version>3.1.0</version>
|
|
<configuration>
|
|
<configLocation>checkstyle.xml</configLocation>
|
|
<encoding>UTF-8</encoding>
|
|
<consoleOutput>true</consoleOutput>
|
|
<failsOnError>true</failsOnError>
|
|
<linkXRef>false</linkXRef>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<id>validate</id>
|
|
<phase>validate</phase>
|
|
<goals>
|
|
<goal>check</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<!-- clean fe/target dir before building -->
|
|
<plugin>
|
|
<artifactId>maven-clean-plugin</artifactId>
|
|
<version>3.1.0</version>
|
|
<executions>
|
|
<execution>
|
|
<id>auto-clean</id>
|
|
<phase>initialize</phase>
|
|
<goals>
|
|
<goal>clean</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-shade-plugin</artifactId>
|
|
<version>3.2.1</version>
|
|
<configuration>
|
|
<artifactSet>
|
|
<excludes>
|
|
<exclude>com.google.code.findbugs:*</exclude>
|
|
<exclude>org.slf4j:*</exclude>
|
|
</excludes>
|
|
</artifactSet>
|
|
<relocations>
|
|
<relocation>
|
|
<pattern>org.roaringbitmap</pattern>
|
|
<shadedPattern>org.apache.doris.shaded.org.roaringbitmap</shadedPattern>
|
|
<pattern>com.google.guava</pattern>
|
|
<shadedPattern>org.apache.doris.shaded.com.google.guava</shadedPattern>
|
|
</relocation>
|
|
</relocations>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>shade</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
</plugins>
|
|
|
|
<pluginManagement>
|
|
<plugins>
|
|
<!--set for ecplise lifecycle -->
|
|
<plugin>
|
|
<groupId>org.eclipse.m2e</groupId>
|
|
<artifactId>lifecycle-mapping</artifactId>
|
|
<version>1.0.0</version>
|
|
<configuration>
|
|
<lifecycleMappingMetadata>
|
|
<pluginExecutions>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>net.sourceforge.czt.dev</groupId>
|
|
<artifactId>cup-maven-plugin</artifactId>
|
|
<versionRange>[1.6,)</versionRange>
|
|
<goals>
|
|
<goal>generate</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>de.jflex</groupId>
|
|
<artifactId>maven-jflex-plugin</artifactId>
|
|
<versionRange>[1.4.3,)</versionRange>
|
|
<goals>
|
|
<goal>generate</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>exec-maven-plugin</artifactId>
|
|
<versionRange>[1.6,)</versionRange>
|
|
<goals>
|
|
<goal>exec</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>build-helper-maven-plugin</artifactId>
|
|
<versionRange>[1.7,)</versionRange>
|
|
<goals>
|
|
<goal>add-source</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-dependency-plugin</artifactId>
|
|
<versionRange>[3.1.1,)</versionRange>
|
|
<goals>
|
|
<goal>copy-dependencies</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-compiler-plugin</artifactId>
|
|
<versionRange>[3.1,)</versionRange>
|
|
<goals>
|
|
<goal>compile</goal>
|
|
<goal>testCompile</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
<pluginExecution>
|
|
<pluginExecutionFilter>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-resources-plugin</artifactId>
|
|
<versionRange>[2.6,)</versionRange>
|
|
<goals>
|
|
<goal>resources</goal>
|
|
<goal>testResources</goal>
|
|
</goals>
|
|
</pluginExecutionFilter>
|
|
<action>
|
|
<ignore></ignore>
|
|
</action>
|
|
</pluginExecution>
|
|
</pluginExecutions>
|
|
|
|
</lifecycleMappingMetadata>
|
|
</configuration>
|
|
</plugin>
|
|
|
|
</plugins>
|
|
</pluginManagement>
|
|
|
|
</build>
|
|
</project>
|