From d8202ca9cc7a9229de78ff16b83c5cd86854824b Mon Sep 17 00:00:00 2001 From: copperybean Date: Thu, 4 Feb 2021 13:41:03 +0800 Subject: [PATCH] [Enhancement] move common codes from fe-core to fe-common and remove log4j1 (#5317) (#5318) The io related codes may be used by new modules, so It's better to move them to fe-common. The modification to fe-core is frequent, but there are many generated java files by thrift will slow down the compilation, so It's better to move thrift generation process to fe-common. Currently both log4j1 and log4j2 are used, which leads to logs are written to wrong files. Our modification will remove log4j1 from dependency, use slf4j + slf4j -> log4j2 instead. --- fe/fe-common/pom.xml | 122 +++++++++++++++++- .../org/apache/doris/common/FeConstants.java | 0 .../apache/doris/common/FeMetaVersion.java | 0 .../doris/common/io/DataInputBuffer.java | 0 .../doris/common/io/DataOutputBuffer.java | 0 .../org/apache/doris/common/io/DeepCopy.java | 7 +- .../common/io/FastByteArrayInputStream.java | 0 .../common/io/FastByteArrayOutputStream.java | 0 .../org/apache/doris/common/io/IOUtils.java | 2 +- .../apache/doris/common/io/InputBuffer.java | 0 .../doris/common/io/LimitInputStream.java | 6 +- .../doris/common/io/LimitOutputStream.java | 6 +- .../apache/doris/common/io/OutputBuffer.java | 0 .../java/org/apache/doris/common/io/Text.java | 6 +- .../org/apache/doris/common/io/Writable.java | 0 .../org/apache/doris/meta/MetaContext.java | 0 fe/fe-core/pom.xml | 47 +------ .../analysis/CreateRoutineLoadStmtTest.java | 6 +- .../planner/MaterializedViewSelectorTest.java | 14 +- fe/pom.xml | 33 ++--- fe/spark-dpp/pom.xml | 9 +- .../doris/load/loadv2/dpp/ColumnParser.java | 6 +- .../load/loadv2/dpp/GlobalDictBuilder.java | 6 +- .../doris/load/loadv2/dpp/SparkDpp.java | 10 +- .../doris/load/loadv2/etl/SparkEtlJob.java | 6 +- fe_plugins/auditloader/pom.xml | 4 + fe_plugins/pom.xml | 7 + 27 files changed, 199 insertions(+), 98 deletions(-) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/FeConstants.java (100%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/FeMetaVersion.java (100%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/DataInputBuffer.java (100%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/DataOutputBuffer.java (100%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/DeepCopy.java (95%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/FastByteArrayInputStream.java (100%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/FastByteArrayOutputStream.java (100%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/IOUtils.java (99%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/InputBuffer.java (100%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/LimitInputStream.java (95%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/LimitOutputStream.java (95%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/OutputBuffer.java (100%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/Text.java (99%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/common/io/Writable.java (100%) rename fe/{fe-core => fe-common}/src/main/java/org/apache/doris/meta/MetaContext.java (100%) diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index 225182c8c6..576598fe11 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -30,16 +30,136 @@ under the License. ../pom.xml - fe-common + doris-fe-common 1.0.0 jar ${basedir}/../../ + ${basedir}/../../thirdparty + + + thirdparty + + + env.DORIS_THIRDPARTY + + + + ${env.DORIS_THIRDPARTY} + + + + + + + javax.annotation + javax.annotation-api + provided + + + + + com.google.guava + guava + + + + + org.apache.thrift + libfb303 + pom + + + + org.apache.thrift + libthrift + + + + org.slf4j + slf4j-api + + + + + org.jmockit + jmockit + test + + + + junit + junit + test + + + + doris-fe-common + + + + org.apache.thrift.tools + maven-thrift-plugin + 0.1.11 + + ${doris.thirdparty}/installed/bin/thrift + ${doris.home}/gensrc/thrift + java:fullcamel + + + + thrift-sources + generate-sources + + compile + + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.1.0 + + true + + + + create-source-jar + + jar-no-fork + test-jar-no-fork + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.1.2 + + + prepare-test-jar + test-compile + + test-jar + + + + log4j.properties + + + + + + org.apache.maven.plugins diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeConstants.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java rename to fe/fe-common/src/main/java/org/apache/doris/common/FeConstants.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java rename to fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/DataInputBuffer.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/DataInputBuffer.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/DataInputBuffer.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/DataInputBuffer.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/DataOutputBuffer.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/DataOutputBuffer.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/DataOutputBuffer.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/DataOutputBuffer.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/DeepCopy.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/DeepCopy.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/DeepCopy.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/DeepCopy.java index bdeac68975..2f9bfda721 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/io/DeepCopy.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/io/DeepCopy.java @@ -18,9 +18,8 @@ package org.apache.doris.common.io; import org.apache.doris.meta.MetaContext; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.DataInput; import java.io.DataInputStream; @@ -31,7 +30,7 @@ import java.lang.reflect.Method; * This class is for deep copying a writable instance. */ public class DeepCopy { - private static final Logger LOG = LogManager.getLogger(DeepCopy.class); + private static final Logger LOG = LoggerFactory.getLogger(DeepCopy.class); public static final String READ_FIELDS_METHOD_NAME = "readFields"; public static final String READ_METHOD_NAME = "read"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/FastByteArrayInputStream.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/FastByteArrayInputStream.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/FastByteArrayInputStream.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/FastByteArrayInputStream.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/FastByteArrayOutputStream.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/FastByteArrayOutputStream.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/FastByteArrayOutputStream.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/FastByteArrayOutputStream.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/IOUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/IOUtils.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/IOUtils.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/IOUtils.java index 464905aecb..819a41020a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/io/IOUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/io/IOUtils.java @@ -18,7 +18,7 @@ package org.apache.doris.common.io; import com.google.common.base.Strings; -import org.apache.logging.log4j.Logger; +import org.slf4j.Logger; import java.io.DataInput; import java.io.DataOutput; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/InputBuffer.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/InputBuffer.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/InputBuffer.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/InputBuffer.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/LimitInputStream.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/LimitInputStream.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/LimitInputStream.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/LimitInputStream.java index dbe836f1c6..d098c1892d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/io/LimitInputStream.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/io/LimitInputStream.java @@ -17,8 +17,8 @@ package org.apache.doris.common.io; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; @@ -27,7 +27,7 @@ import java.io.InputStream; * A LimitInputStream that the InputStream is limited . */ public class LimitInputStream extends InputStream { - private static final Logger LOG = LogManager.getLogger(LimitInputStream.class); + private static final Logger LOG = LoggerFactory.getLogger(LimitInputStream.class); /** * The input stream to be limited. diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/LimitOutputStream.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/LimitOutputStream.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/LimitOutputStream.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/LimitOutputStream.java index 96516efa83..5533a84d0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/io/LimitOutputStream.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/io/LimitOutputStream.java @@ -17,8 +17,8 @@ package org.apache.doris.common.io; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.OutputStream; @@ -27,7 +27,7 @@ import java.io.OutputStream; * A LimitOutputStream that the OutputStream is limited . */ public class LimitOutputStream extends OutputStream { - private static final Logger LOG = LogManager.getLogger(LimitOutputStream.class.getName()); + private static final Logger LOG = LoggerFactory.getLogger(LimitOutputStream.class.getName()); /** * The output stream to be limited. diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/OutputBuffer.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/OutputBuffer.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/OutputBuffer.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/OutputBuffer.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/Text.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/Text.java similarity index 99% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/Text.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/Text.java index c72da27874..aae99564ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/io/Text.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/io/Text.java @@ -17,8 +17,8 @@ package org.apache.doris.common.io; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.DataInput; @@ -46,7 +46,7 @@ import java.text.StringCharacterIterator; * length of an encoded string. */ public class Text implements Writable { - private static final Logger LOG = LogManager.getLogger(Text.class); + private static final Logger LOG = LoggerFactory.getLogger(Text.class); private static ThreadLocal ENCODER_FACTORY = new ThreadLocal() { protected CharsetEncoder initialValue() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/io/Writable.java b/fe/fe-common/src/main/java/org/apache/doris/common/io/Writable.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/common/io/Writable.java rename to fe/fe-common/src/main/java/org/apache/doris/common/io/Writable.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/meta/MetaContext.java b/fe/fe-common/src/main/java/org/apache/doris/meta/MetaContext.java similarity index 100% rename from fe/fe-core/src/main/java/org/apache/doris/meta/MetaContext.java rename to fe/fe-common/src/main/java/org/apache/doris/meta/MetaContext.java diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index a8549bddad..3218be1424 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -37,7 +37,7 @@ under the License. ${basedir}/../../ 1 - ${basedir}/../../thirdparty + ${basedir}/../../thirdparty @@ -49,7 +49,7 @@ under the License. - ${env.DORIS_THIRDPARTY} + ${env.DORIS_THIRDPARTY} @@ -66,6 +66,10 @@ under the License. + + org.apache + doris-fe-common + org.apache.logging.log4j @@ -257,19 +261,6 @@ under the License. test - - - org.apache.thrift - libfb303 - pom - - - - - org.apache.thrift - libthrift - - org.apache.logging.log4j @@ -421,12 +412,6 @@ under the License. zjsonpatch - - - log4j - log4j - - org.slf4j @@ -581,26 +566,6 @@ under the License. palo-fe - - - org.apache.thrift.tools - maven-thrift-plugin - 0.1.11 - - ${doris.thridparty}/installed/bin/thrift - ${doris.home}/gensrc/thrift - java:fullcamel - - - - thrift-sources - generate-sources - - compile - - - - net.sourceforge.czt.dev diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java index 41dc52a1ca..d9645022a1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateRoutineLoadStmtTest.java @@ -28,11 +28,11 @@ import org.apache.doris.load.routineload.LoadDataSourceType; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -46,7 +46,7 @@ import mockit.Mocked; public class CreateRoutineLoadStmtTest { - private static final Logger LOG = LogManager.getLogger(CreateRoutineLoadStmtTest.class); + private static final Logger LOG = LoggerFactory.getLogger(CreateRoutineLoadStmtTest.class); @Mocked Database database; diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java index 1c8ef7c78b..2a18f2b704 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java @@ -39,6 +39,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.doris.thrift.TStorageType; import org.junit.Assert; import org.junit.Test; @@ -355,24 +356,27 @@ public class MaterializedViewSelectorTest { @Test public void testCompensateIndex(@Injectable SelectStmt selectStmt, @Injectable Analyzer analyzer, @Injectable OlapTable table) { - Map> candidateIndexIdToSchema = Maps.newHashMap(); - Map> allVisibleIndexes = Maps.newHashMap(); + Map candidateIndexIdToSchema = Maps.newHashMap(); + Map allVisibleIndexes = Maps.newHashMap(); List index1Columns = Lists.newArrayList(); Column index1Column1 = new Column("c2", Type.INT, true, AggregateType.SUM, true, "", ""); index1Columns.add(index1Column1); - allVisibleIndexes.put(new Long(1), index1Columns); + allVisibleIndexes.put(new Long(1), new MaterializedIndexMeta( + 0, index1Columns, 0, 0, (short) 0, TStorageType.COLUMN, KeysType.AGG_KEYS, null)); List index2Columns = Lists.newArrayList(); Column index2Column1 = new Column("c1", Type.INT, true, null, true, "", ""); index2Columns.add(index2Column1); Column index2Column2 = new Column("c2", Type.INT, false, AggregateType.SUM, true, "", ""); index2Columns.add(index2Column2); - allVisibleIndexes.put(new Long(2), index2Columns); + allVisibleIndexes.put(new Long(2), new MaterializedIndexMeta( + 0, index2Columns, 0, 0, (short) 0, TStorageType.COLUMN, KeysType.AGG_KEYS, null)); List index3Columns = Lists.newArrayList(); Column index3Column1 = new Column("c1", Type.INT, true, null, true, "", ""); index3Columns.add(index3Column1); Column index3Column2 = new Column("c3", Type.INT, false, AggregateType.SUM, true, "", ""); index3Columns.add(index3Column2); - allVisibleIndexes.put(new Long(3), index3Columns); + allVisibleIndexes.put(new Long(3), new MaterializedIndexMeta( + 0, index3Columns, 0, 0, (short) 0, TStorageType.COLUMN, KeysType.AGG_KEYS, null)); List keyColumns = Lists.newArrayList(); keyColumns.add(index2Column1); new Expectations() { diff --git a/fe/pom.xml b/fe/pom.xml index 1d92bdef03..7cd488f2c3 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -122,7 +122,7 @@ under the License. org.apache - fe-common + doris-fe-common 1.0.0 @@ -513,13 +513,6 @@ under the License. 0.2.3 - - - log4j - log4j - 1.2.17 - - org.slf4j @@ -581,6 +574,16 @@ under the License. org.apache.spark spark-core_2.12 2.4.5 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + @@ -604,49 +607,49 @@ under the License. 2.6.5 provided - + org.apache.parquet parquet-column 1.10.1 provided - + org.apache.parquet parquet-hadoop 1.10.1 provided - + org.apache.parquet parquet-common 1.10.1 provided - + commons-collections commons-collections 3.2.1 compile - + org.scala-lang scala-library 2.12.10 provided - + com.esotericsoftware kryo-shaded 4.0.2 compile - + org.apache.spark spark-catalyst_2.12 diff --git a/fe/spark-dpp/pom.xml b/fe/spark-dpp/pom.xml index 3945348800..0a0a042430 100644 --- a/fe/spark-dpp/pom.xml +++ b/fe/spark-dpp/pom.xml @@ -42,7 +42,7 @@ under the License. org.apache - fe-common + doris-fe-common @@ -86,11 +86,10 @@ under the License. test - - log4j - log4j - provided + org.slf4j + slf4j-log4j12 + test diff --git a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/ColumnParser.java b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/ColumnParser.java index 1547191cf8..a7b8d3df5b 100644 --- a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/ColumnParser.java +++ b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/ColumnParser.java @@ -19,10 +19,10 @@ package org.apache.doris.load.loadv2.dpp; import org.apache.doris.common.SparkDppException; import org.apache.doris.load.loadv2.etl.EtlJobConfig; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.Serializable; import java.math.BigDecimal; @@ -31,7 +31,7 @@ import java.math.BigInteger; // Parser to validate value for different type public abstract class ColumnParser implements Serializable { - protected static final Logger LOG = LogManager.getLogger(ColumnParser.class); + protected static final Logger LOG = LoggerFactory.getLogger(ColumnParser.class); // thread safe formatter public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd"); diff --git a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/GlobalDictBuilder.java b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/GlobalDictBuilder.java index 8275edf61e..c19f73fca1 100644 --- a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/GlobalDictBuilder.java +++ b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/GlobalDictBuilder.java @@ -27,8 +27,8 @@ import org.apache.spark.sql.catalog.Column; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; @@ -63,7 +63,7 @@ import java.util.stream.Collectors; public class GlobalDictBuilder { - protected static final Logger LOG = LogManager.getLogger(GlobalDictBuilder.class); + protected static final Logger LOG = LoggerFactory.getLogger(GlobalDictBuilder.class); // name of the column in doris table which need to build global dict // for example: some dict columns a,b,c diff --git a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java index b30d102003..f31da201fc 100644 --- a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java +++ b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java @@ -17,8 +17,6 @@ package org.apache.doris.load.loadv2.dpp; -import scala.Tuple2; - import org.apache.doris.common.SparkDppException; import org.apache.doris.load.loadv2.etl.EtlJobConfig; import com.google.common.base.Strings; @@ -31,8 +29,6 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.metadata.CompressionCodecName; @@ -58,6 +54,8 @@ import org.apache.spark.sql.types.StructType; import org.apache.spark.storage.StorageLevel; import org.apache.spark.util.LongAccumulator; import org.apache.spark.util.SerializableConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -77,6 +75,8 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import scala.Tuple2; + // This class is a Spark-based data preprocessing program, // which will make use of the distributed compute framework of spark to // do ETL job/sort/preaggregate jobs in spark job @@ -89,7 +89,7 @@ import java.util.Set; // 3. process aggregation if needed // 4. write data to parquet file public final class SparkDpp implements java.io.Serializable { - private static final Logger LOG = LogManager.getLogger(SparkDpp.class); + private static final Logger LOG = LoggerFactory.getLogger(SparkDpp.class); private static final String NULL_FLAG = "\\N"; private static final String DPP_RESULT_FILE = "dpp_result.json"; diff --git a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/etl/SparkEtlJob.java b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/etl/SparkEtlJob.java index 86bae31490..66fb4ce35b 100644 --- a/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/etl/SparkEtlJob.java +++ b/fe/spark-dpp/src/main/java/org/apache/doris/load/loadv2/etl/SparkEtlJob.java @@ -27,12 +27,12 @@ import org.apache.doris.load.loadv2.etl.EtlJobConfig.EtlIndex; import org.apache.doris.load.loadv2.etl.EtlJobConfig.EtlTable; import org.apache.commons.collections.map.MultiValueMap; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import org.apache.spark.SparkConf; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.functions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -50,7 +50,7 @@ import java.util.Set; * 4. dpp (data partition, data sort and data aggregation) */ public class SparkEtlJob { - private static final Logger LOG = LogManager.getLogger(SparkEtlJob.class); + private static final Logger LOG = LoggerFactory.getLogger(SparkEtlJob.class); private static final String BITMAP_DICT_FUNC = "bitmap_dict"; private static final String TO_BITMAP_FUNC = "to_bitmap"; diff --git a/fe_plugins/auditloader/pom.xml b/fe_plugins/auditloader/pom.xml index 655ac69507..bdd433eeb6 100644 --- a/fe_plugins/auditloader/pom.xml +++ b/fe_plugins/auditloader/pom.xml @@ -17,6 +17,10 @@ org.apache doris-fe + + org.apache + doris-fe-common + diff --git a/fe_plugins/pom.xml b/fe_plugins/pom.xml index bd26f41e66..f9238d88fd 100644 --- a/fe_plugins/pom.xml +++ b/fe_plugins/pom.xml @@ -50,6 +50,13 @@ system ${doris.home}/fe/fe-core/target/palo-fe.jar + + org.apache + doris-fe-common + 1.0.0 + system + ${doris.home}/fe/fe-common/target/doris-fe-common.jar +