From 57656b24599e4df53be692945c877aa8ea6abe92 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Tue, 13 Jun 2023 09:41:22 +0800 Subject: [PATCH] [Enhancement](java-udf) java-udf module split to sub modules (#20185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The java-udf module has become increasingly large and difficult to manage, making it inconvenient to package and use as needed. It needs to be split into multiple sub-modules, such as : java-commom、java-udf、jdbc-scanner、hudi-scanner、 paimon-scanner. Co-authored-by: lexluo --- be/src/util/jni-util.cpp | 8 +- be/src/util/jni_native_method.h | 2 +- be/src/vec/exec/jni_connector.cpp | 10 +- be/src/vec/exec/jni_connector.h | 8 +- be/src/vec/exec/scan/jni_reader.cpp | 4 +- .../vec/exec/scan/max_compute_jni_reader.cpp | 4 +- be/src/vec/exec/scan/paimon_reader.cpp | 4 +- be/src/vec/exec/vjdbc_connector.cpp | 2 +- build.sh | 96 +++++++++------- .../hudi-scanner}/pom.xml | 103 ++--------------- .../apache/doris/hudi/HudiColumnValue.java | 3 +- .../org/apache/doris/hudi/HudiJniScanner.java | 9 +- .../org/apache/doris/hudi/HudiScanParam.java | 3 +- .../org/apache/doris/hudi/HudiScanUtils.java | 0 .../java/org/apache/doris/hudi}/Utils.java | 2 +- .../src/main/resources/package.xml | 0 .../apache/doris/hudi/HudiJniScannerTest.java | 0 fe/be-java-extensions/java-common/pom.xml | 60 ++++++++++ .../common/exception}/InternalException.java | 2 +- .../exception}/UdfRuntimeException.java | 2 +- .../apache/doris/common}/jni/JniScanner.java | 11 +- .../doris/common}/jni/MockJniScanner.java | 9 +- .../doris/common/jni/utils}/JMXJsonUtil.java | 2 +- .../common/jni/utils}/JNINativeMethod.java | 2 +- .../doris/common/jni/utils}/JniUtil.java | 3 +- .../common/jni/utils}/JvmPauseMonitor.java | 2 +- .../doris/common}/jni/utils/OffHeap.java | 3 +- .../common}/jni/utils/TypeNativeBytes.java | 2 +- .../doris/common/jni/utils}/UdfUtils.java | 7 +- .../doris/common}/jni/vec/ColumnType.java | 2 +- .../doris/common}/jni/vec/ColumnValue.java | 2 +- .../doris/common}/jni/vec/ScanPredicate.java | 9 +- .../doris/common}/jni/vec/VectorColumn.java | 9 +- .../doris/common}/jni/vec/VectorTable.java | 7 +- .../src/main/resources/log4j.properties | 0 .../src/main/resources/package.xml | 41 +++++++ .../doris/common}/jni/JniScannerTest.java | 7 +- fe/be-java-extensions/java-udf/pom.xml | 74 +++++++++++++ .../org/apache/doris/udf/BaseExecutor.java | 5 +- .../org/apache/doris/udf/UdafExecutor.java | 4 +- .../org/apache/doris/udf/UdfExecutor.java | 4 +- .../java-udf/src/main/resources/package.xml | 41 +++++++ .../org/apache/doris/udf/ConstantOneUdf.java | 0 .../org/apache/doris/udf/DateTimeUdf.java | 0 .../java/org/apache/doris/udf/DecimalUdf.java | 0 .../org/apache/doris/udf/LargeIntUdf.java | 0 .../org/apache/doris/udf/SimpleAddUdf.java | 0 .../org/apache/doris/udf/StringConcatUdf.java | 0 .../org/apache/doris/udf/UdfExecutorTest.java | 1 + fe/be-java-extensions/jdbc-scanner/pom.xml | 90 +++++++++++++++ .../apache/doris/jdbc}/JdbcDataSource.java | 2 +- .../org/apache/doris/jdbc}/JdbcExecutor.java | 12 +- .../src/main/resources/package.xml | 41 +++++++ .../max-compute-scanner/pom.xml | 104 ++++++++++++++++++ .../maxcompute}/MaxComputeColumnValue.java | 4 +- .../maxcompute}/MaxComputeJniScanner.java | 8 +- .../src/main/resources/package.xml | 41 +++++++ fe/be-java-extensions/paimon-scanner/pom.xml | 100 +++++++++++++++++ .../doris/paimon}/PaimonColumnValue.java | 4 +- .../doris/paimon}/PaimonJniScanner.java | 10 +- .../src/main/resources/package.xml | 41 +++++++ fe/be-java-extensions/pom.xml | 46 ++++++++ fe/fe-core/pom.xml | 5 +- fe/pom.xml | 49 ++++++++- 64 files changed, 908 insertions(+), 218 deletions(-) rename fe/{java-udf => be-java-extensions/hudi-scanner}/pom.xml (57%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/java/org/apache/doris/hudi/HudiColumnValue.java (98%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/java/org/apache/doris/hudi/HudiJniScanner.java (97%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/java/org/apache/doris/hudi/HudiScanParam.java (99%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/java/org/apache/doris/hudi/HudiScanUtils.java (100%) rename fe/{java-udf/src/main/java/org/apache/doris/jni/utils => be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi}/Utils.java (99%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/resources/package.xml (100%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java (100%) create mode 100644 fe/be-java-extensions/java-common/pom.xml rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception}/InternalException.java (95%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception}/UdfRuntimeException.java (95%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/JniScanner.java (92%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/MockJniScanner.java (96%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/JMXJsonUtil.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/JNINativeMethod.java (96%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/JniUtil.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/JvmPauseMonitor.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/utils/OffHeap.java (98%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/utils/TypeNativeBytes.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/UdfUtils.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/ColumnType.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/ColumnValue.java (97%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/ScanPredicate.java (97%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/VectorColumn.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/VectorTable.java (96%) rename fe/{java-udf => be-java-extensions/java-common}/src/main/resources/log4j.properties (100%) create mode 100644 fe/be-java-extensions/java-common/src/main/resources/package.xml rename fe/{java-udf/src/test/java/org/apache/doris => be-java-extensions/java-common/src/test/java/org/apache/doris/common}/jni/JniScannerTest.java (94%) create mode 100644 fe/be-java-extensions/java-udf/pom.xml rename fe/{ => be-java-extensions}/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java (99%) rename fe/{ => be-java-extensions}/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java (98%) rename fe/{ => be-java-extensions}/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java (98%) create mode 100644 fe/be-java-extensions/java-udf/src/main/resources/package.xml rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java (99%) create mode 100644 fe/be-java-extensions/jdbc-scanner/pom.xml rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc}/JdbcDataSource.java (97%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc}/JdbcExecutor.java (99%) create mode 100644 fe/be-java-extensions/jdbc-scanner/src/main/resources/package.xml create mode 100644 fe/be-java-extensions/max-compute-scanner/pom.xml rename fe/{java-udf/src/main/java/org/apache/doris/jni/vec => be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute}/MaxComputeColumnValue.java (98%) rename fe/{java-udf/src/main/java/org/apache/doris/jni => be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute}/MaxComputeJniScanner.java (98%) create mode 100644 fe/be-java-extensions/max-compute-scanner/src/main/resources/package.xml create mode 100644 fe/be-java-extensions/paimon-scanner/pom.xml rename fe/{java-udf/src/main/java/org/apache/doris/jni/vec => be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon}/PaimonColumnValue.java (97%) rename fe/{java-udf/src/main/java/org/apache/doris/jni => be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon}/PaimonJniScanner.java (97%) create mode 100644 fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml create mode 100644 fe/be-java-extensions/pom.xml diff --git a/be/src/util/jni-util.cpp b/be/src/util/jni-util.cpp index 5ee95e7ea5..7b1843490a 100644 --- a/be/src/util/jni-util.cpp +++ b/be/src/util/jni-util.cpp @@ -267,7 +267,7 @@ Status JniUtil::Init() { RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); if (env == NULL) return Status::InternalError("Failed to get/create JVM"); // Find JniUtil class and create a global ref. - jclass local_jni_util_cl = env->FindClass("org/apache/doris/udf/JniUtil"); + jclass local_jni_util_cl = env->FindClass("org/apache/doris/common/jni/utils/JniUtil"); if (local_jni_util_cl == NULL) { if (env->ExceptionOccurred()) env->ExceptionDescribe(); return Status::InternalError("Failed to find JniUtil class."); @@ -283,7 +283,8 @@ Status JniUtil::Init() { } // Find InternalException class and create a global ref. - jclass local_internal_exc_cl = env->FindClass("org/apache/doris/udf/InternalException"); + jclass local_internal_exc_cl = + env->FindClass("org/apache/doris/common/exception/InternalException"); if (local_internal_exc_cl == NULL) { if (env->ExceptionOccurred()) env->ExceptionDescribe(); return Status::InternalError("Failed to find JniUtil class."); @@ -299,7 +300,8 @@ Status JniUtil::Init() { } // Find JNINativeMethod class and create a global ref. - jclass local_jni_native_exc_cl = env->FindClass("org/apache/doris/udf/JNINativeMethod"); + jclass local_jni_native_exc_cl = + env->FindClass("org/apache/doris/common/jni/utils/JNINativeMethod"); if (local_jni_native_exc_cl == nullptr) { if (env->ExceptionOccurred()) { env->ExceptionDescribe(); diff --git a/be/src/util/jni_native_method.h b/be/src/util/jni_native_method.h index d208cb0d95..e3dc3fb3dd 100644 --- a/be/src/util/jni_native_method.h +++ b/be/src/util/jni_native_method.h @@ -24,7 +24,7 @@ namespace doris { /** - * Java native methods for org.apache.doris.udf.JNINativeMethod. + * Java native methods for org.apache.doris.common.jni.utils.JNINativeMethod. */ struct JavaNativeMethods { /** diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp index fd5cf52e2b..178bcd8bb7 100644 --- a/be/src/vec/exec/jni_connector.cpp +++ b/be/src/vec/exec/jni_connector.cpp @@ -71,7 +71,7 @@ Status JniConnector::open(RuntimeState* state, RuntimeProfile* profile) { return Status::InternalError("Failed to get/create JVM"); } RETURN_IF_ERROR(_init_jni_scanner(env, state->batch_size())); - // Call org.apache.doris.jni.JniScanner#open + // Call org.apache.doris.common.jni.JniScanner#open env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_open); RETURN_ERROR_IF_EXC(env); return Status::OK(); @@ -82,7 +82,7 @@ Status JniConnector::init( _generate_predicates(colname_to_value_range); if (_predicates_length != 0 && _predicates != nullptr) { int64_t predicates_address = (int64_t)_predicates.get(); - // We can call org.apache.doris.jni.vec.ScanPredicate#parseScanPredicates to parse the + // We can call org.apache.doris.common.jni.vec.ScanPredicate#parseScanPredicates to parse the // serialized predicates in java side. _scanner_params.emplace("push_down_predicates", std::to_string(predicates_address)); } @@ -90,7 +90,7 @@ Status JniConnector::init( } Status JniConnector::get_nex_block(Block* block, size_t* read_rows, bool* eof) { - // Call org.apache.doris.jni.JniScanner#getNextBatchMeta + // Call org.apache.doris.common.jni.JniScanner#getNextBatchMeta // return the address of meta information JNIEnv* env = nullptr; RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); @@ -123,7 +123,7 @@ Status JniConnector::close() { JNIEnv* env = nullptr; RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); // _fill_block may be failed and returned, we should release table in close. - // org.apache.doris.jni.JniScanner#releaseTable is idempotent + // org.apache.doris.common.jni.JniScanner#releaseTable is idempotent env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_release_table); env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_close); env->DeleteGlobalRef(_jni_scanner_obj); @@ -199,7 +199,7 @@ Status JniConnector::_fill_column(ColumnPtr& doris_column, DataTypePtr& data_typ TypeIndex logical_type = remove_nullable(data_type)->get_type_id(); void* null_map_ptr = _next_meta_as_ptr(); if (null_map_ptr == nullptr) { - // org.apache.doris.jni.vec.ColumnType.Type#UNSUPPORTED will set column address as 0 + // org.apache.doris.common.jni.vec.ColumnType.Type#UNSUPPORTED will set column address as 0 return Status::InternalError("Unsupported type {} in java side", getTypeName(logical_type)); } MutableColumnPtr data_column; diff --git a/be/src/vec/exec/jni_connector.h b/be/src/vec/exec/jni_connector.h index 90d5fbcd0a..77be2329e0 100644 --- a/be/src/vec/exec/jni_connector.h +++ b/be/src/vec/exec/jni_connector.h @@ -55,13 +55,13 @@ struct Decimal; namespace doris::vectorized { /** - * Connector to java jni scanner, which should extend org.apache.doris.jni.JniScanner + * Connector to java jni scanner, which should extend org.apache.doris.common.jni.JniScanner */ class JniConnector { public: /** * The predicates that can be pushed down to java side. - * Reference to java class org.apache.doris.jni.vec.ScanPredicate + * Reference to java class org.apache.doris.common.jni.vec.ScanPredicate */ template struct ScanPredicate { @@ -102,7 +102,7 @@ public: /** * The value ranges can be stored as byte array as following format: * number_filters(4) | length(4) | column_name | op(4) | scale(4) | num_values(4) | value_length(4) | value | ... - * The read method is implemented in org.apache.doris.jni.vec.ScanPredicate#parseScanPredicates + * The read method is implemented in org.apache.doris.common.jni.vec.ScanPredicate#parseScanPredicates */ int write(std::unique_ptr& predicates, int origin_length) { int num_filters = 0; @@ -232,7 +232,7 @@ private: std::unique_ptr _predicates = nullptr; /** - * Set the address of meta information, which is returned by org.apache.doris.jni.JniScanner#getNextBatchMeta + * Set the address of meta information, which is returned by org.apache.doris.common.jni.JniScanner#getNextBatchMeta */ void _set_meta(long meta_addr) { _meta_ptr = static_cast(reinterpret_cast(meta_addr)); diff --git a/be/src/vec/exec/scan/jni_reader.cpp b/be/src/vec/exec/scan/jni_reader.cpp index bbf9a5f8d0..99858c1c42 100644 --- a/be/src/vec/exec/scan/jni_reader.cpp +++ b/be/src/vec/exec/scan/jni_reader.cpp @@ -58,8 +58,8 @@ MockJniReader::MockJniReader(const std::vector& file_slot_descs std::map params = {{"mock_rows", "10240"}, {"required_fields", required_fields.str()}, {"columns_types", columns_types.str()}}; - _jni_connector = std::make_unique("org/apache/doris/jni/MockJniScanner", params, - column_names); + _jni_connector = std::make_unique("org/apache/doris/common/jni/MockJniScanner", + params, column_names); } Status MockJniReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { diff --git a/be/src/vec/exec/scan/max_compute_jni_reader.cpp b/be/src/vec/exec/scan/max_compute_jni_reader.cpp index f5182931d1..9edd8bfc51 100644 --- a/be/src/vec/exec/scan/max_compute_jni_reader.cpp +++ b/be/src/vec/exec/scan/max_compute_jni_reader.cpp @@ -70,8 +70,8 @@ MaxComputeJniReader::MaxComputeJniReader(const MaxComputeTableDescriptor* mc_des {"split_size", std::to_string(_range.size)}, {"required_fields", required_fields.str()}, {"columns_types", columns_types.str()}}; - _jni_connector = std::make_unique("org/apache/doris/jni/MaxComputeJniScanner", - params, column_names); + _jni_connector = std::make_unique( + "org/apache/doris/maxcompute/MaxComputeJniScanner", params, column_names); } Status MaxComputeJniReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { diff --git a/be/src/vec/exec/scan/paimon_reader.cpp b/be/src/vec/exec/scan/paimon_reader.cpp index 906973d838..9348ad6b01 100644 --- a/be/src/vec/exec/scan/paimon_reader.cpp +++ b/be/src/vec/exec/scan/paimon_reader.cpp @@ -55,8 +55,8 @@ PaimonJniReader::PaimonJniReader(const std::vector& file_slot_d params["length_byte"] = range.table_format_params.paimon_params.length_byte; params["split_byte"] = std::to_string((int64_t)range.table_format_params.paimon_params.paimon_split.data()); - _jni_connector = std::make_unique("org/apache/doris/jni/PaimonJniScanner", params, - column_names); + _jni_connector = std::make_unique("org/apache/doris/paimon/PaimonJniScanner", + params, column_names); } Status PaimonJniReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index 055ea1e90d..f9743584ab 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -61,7 +61,7 @@ namespace doris { namespace vectorized { -const char* JDBC_EXECUTOR_CLASS = "org/apache/doris/udf/JdbcExecutor"; +const char* JDBC_EXECUTOR_CLASS = "org/apache/doris/jdbc/JdbcExecutor"; const char* JDBC_EXECUTOR_CTOR_SIGNATURE = "([B)V"; const char* JDBC_EXECUTOR_WRITE_SIGNATURE = "(Ljava/lang/String;)I"; const char* JDBC_EXECUTOR_STMT_WRITE_SIGNATURE = "(Ljava/util/Map;)I"; diff --git a/build.sh b/build.sh index 6095a595a9..381e22e3e6 100755 --- a/build.sh +++ b/build.sh @@ -38,23 +38,23 @@ usage() { echo " Usage: $0 Optional options: - [no option] build all components - --fe build Frontend and Spark DPP application. Default ON. - --be build Backend. Default ON. - --meta-tool build Backend meta tool. Default OFF. - --broker build Broker. Default ON. - --audit build audit loader. Default ON. - --spark-dpp build Spark DPP application. Default ON. - --hive-udf build Hive UDF library for Spark Load. Default ON. - --java-udf build Java UDF. Default ON. - --clean clean and build target - --output specify the output directory - -j build Backend parallel + [no option] build all components + --fe build Frontend and Spark DPP application. Default ON. + --be build Backend. Default ON. + --meta-tool build Backend meta tool. Default OFF. + --broker build Broker. Default ON. + --audit build audit loader. Default ON. + --spark-dpp build Spark DPP application. Default ON. + --hive-udf build Hive UDF library for Spark Load. Default ON. + --be-java-extensions build Backend java extensions. Default ON. + --clean clean and build target + --output specify the output directory + -j build Backend parallel Environment variables: USE_AVX2 If the CPU does not support AVX2 instruction set, please set USE_AVX2=0. Default is ON. STRIP_DEBUG_INFO If set STRIP_DEBUG_INFO=ON, the debug information in the compiled binaries will be stored separately in the 'be/lib/debug_info' directory. Default is OFF. - DISABLE_JAVA_UDF If set DISABLE_JAVA_UDF=ON, we will do not build binary with java-udf. Default is OFF. + DISABLE_BE_JAVA_EXTENSIONS If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will do not build binary with java-udf,hudi-scanner,jdbc-scanner and so on Default is OFF. DISABLE_JAVA_CHECK_STYLE If set DISABLE_JAVA_CHECK_STYLE=ON, it will skip style check of java code in FE. Eg. $0 build all @@ -119,7 +119,7 @@ if ! OPTS="$(getopt \ -l 'meta-tool' \ -l 'spark-dpp' \ -l 'hive-udf' \ - -l 'java-udf' \ + -l 'be-java-extensions' \ -l 'clean' \ -l 'coverage' \ -l 'help' \ @@ -138,7 +138,7 @@ BUILD_BROKER=0 BUILD_AUDIT=0 BUILD_META_TOOL='OFF' BUILD_SPARK_DPP=0 -BUILD_JAVA_UDF=0 +BUILD_BE_JAVA_EXTENSIONS=0 BUILD_HIVE_UDF=0 CLEAN=0 HELP=0 @@ -154,7 +154,7 @@ if [[ "$#" == 1 ]]; then BUILD_META_TOOL='OFF' BUILD_SPARK_DPP=1 BUILD_HIVE_UDF=1 - BUILD_JAVA_UDF=1 + BUILD_BE_JAVA_EXTENSIONS=1 CLEAN=0 else while true; do @@ -163,12 +163,12 @@ else BUILD_FE=1 BUILD_SPARK_DPP=1 BUILD_HIVE_UDF=1 - BUILD_JAVA_UDF=1 + BUILD_BE_JAVA_EXTENSIONS=1 shift ;; --be) BUILD_BE=1 - BUILD_JAVA_UDF=1 + BUILD_BE_JAVA_EXTENSIONS=1 shift ;; --broker) @@ -191,8 +191,8 @@ else BUILD_HIVE_UDF=1 shift ;; - --java-udf) - BUILD_JAVA_UDF=1 + --be-java-extensions) + BUILD_BE_JAVA_EXTENSIONS=1 shift ;; --clean) @@ -239,7 +239,7 @@ else BUILD_META_TOOL='ON' BUILD_SPARK_DPP=1 BUILD_HIVE_UDF=1 - BUILD_JAVA_UDF=1 + BUILD_BE_JAVA_EXTENSIONS=1 CLEAN=0 fi fi @@ -343,8 +343,8 @@ if [[ -z "${OUTPUT_BE_BINARY}" ]]; then OUTPUT_BE_BINARY=${BUILD_BE} fi -if [[ -z "${DISABLE_JAVA_UDF}" ]]; then - DISABLE_JAVA_UDF='OFF' +if [[ -z "${BUILD_BE_JAVA_EXTENSIONS}" ]]; then + BUILD_BE_JAVA_EXTENSIONS='OFF' fi if [[ -z "${DISABLE_JAVA_CHECK_STYLE}" ]]; then @@ -355,7 +355,7 @@ if [[ -z "${RECORD_COMPILER_SWITCHES}" ]]; then RECORD_COMPILER_SWITCHES='OFF' fi -if [[ "${BUILD_JAVA_UDF}" -eq 1 && "$(uname -s)" == 'Darwin' ]]; then +if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 && "$(uname -s)" == 'Darwin' ]]; then if [[ -z "${JAVA_HOME}" ]]; then CAUSE='the environment variable JAVA_HOME is not set' else @@ -369,13 +369,13 @@ if [[ "${BUILD_JAVA_UDF}" -eq 1 && "$(uname -s)" == 'Darwin' ]]; then if [[ -n "${CAUSE}" ]]; then echo -e "\033[33;1mWARNNING: \033[37;1mSkip building with Java UDF due to ${CAUSE}.\033[0m" - BUILD_JAVA_UDF=0 - DISABLE_JAVA_UDF_IN_CONF=1 + BUILD_BE_JAVA_EXTENSIONS=0 + BUILD_BE_JAVA_EXTENSIONS_IN_CONF=1 fi fi -if [[ "${DISABLE_JAVA_UDF}" == "ON" ]]; then - BUILD_JAVA_UDF=0 +if [[ "${BUILD_BE_JAVA_EXTENSIONS}" == "ON" ]]; then + BUILD_BE_JAVA_EXTENSIONS=0 fi echo "Get params: @@ -385,7 +385,7 @@ echo "Get params: BUILD_AUDIT -- ${BUILD_AUDIT} BUILD_META_TOOL -- ${BUILD_META_TOOL} BUILD_SPARK_DPP -- ${BUILD_SPARK_DPP} - BUILD_JAVA_UDF -- ${BUILD_JAVA_UDF} + BUILD_BE_JAVA_EXTENSIONS -- ${BUILD_BE_JAVA_EXTENSIONS} BUILD_HIVE_UDF -- ${BUILD_HIVE_UDF} PARALLEL -- ${PARALLEL} CLEAN -- ${CLEAN} @@ -424,14 +424,19 @@ if [[ "${BUILD_SPARK_DPP}" -eq 1 ]]; then modules+=("fe-common") modules+=("spark-dpp") fi -if [[ "${BUILD_JAVA_UDF}" -eq 1 ]]; then - modules+=("fe-common") - modules+=("java-udf") -fi if [[ "${BUILD_HIVE_UDF}" -eq 1 ]]; then modules+=("fe-common") modules+=("hive-udf") fi +if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then + modules+=("fe-common") + modules+=("be-java-extensions/hudi-scanner") + modules+=("be-java-extensions/java-common") + modules+=("be-java-extensions/java-udf") + modules+=("be-java-extensions/jdbc-scanner") + modules+=("be-java-extensions/paimon-scanner") + modules+=("be-java-extensions/max-compute-scanner") +fi FE_MODULES="$( IFS=',' echo "${modules[*]}" @@ -596,11 +601,11 @@ if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then rm -rf "${DORIS_OUTPUT}/be/lib/hadoop_hdfs/native/" fi - if [[ "${DISABLE_JAVA_UDF_IN_CONF}" -eq 1 ]]; then + if [[ "${BUILD_BE_JAVA_EXTENSIONS_IN_CONF}" -eq 1 ]]; then echo -e "\033[33;1mWARNNING: \033[37;1mDisable Java UDF support in be.conf due to the BE was built without Java UDF.\033[0m" cat >>"${DORIS_OUTPUT}/be/conf/be.conf" < - 4.0.0 - + be-java-extensions org.apache.doris ${revision} - fe - ../pom.xml - - java-udf - jar + 4.0.0 + hudi-scanner ${basedir}/../../ 1 - 2.7.4-11 - 3.0.0-8 - 0.4-SNAPSHOT - 0.13.0 - com.vesoft - client - 3.0.0 - - - org.apache.paimon - paimon-bundle - ${paimon.version} - - - org.apache.paimon - paimon-hive-connector-2.3 - ${paimon.version} - - - hive-common - org.apache.hive - 2.3.9 - - - com.fasterxml.jackson.core - jackson-core - - - org.apache.velocity - velocity-engine-core - - - org.apache.httpcomponents - httpclient - - - - com.oracle.database.jdbc - ojdbc6 - - - - com.alibaba - druid - - - com.clickhouse - clickhouse-jdbc - all - - - com.aliyun.odps - odps-sdk-core - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - - - - - org.apache.arrow - arrow-vector - 9.0.0 - - - com.fasterxml.jackson.core - jackson-databind - - - - - org.apache.arrow - arrow-memory-unsafe - 9.0.0 + org.apache.doris + java-common + ${project.version} + org.apache.hudi hudi-hadoop-mr-bundle @@ -167,16 +88,14 @@ under the License. - - org.apache.doris - fe-common - ${project.version} + commons-io + commons-io - + - java-udf + hudi-scanner org.apache.maven.plugins diff --git a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiColumnValue.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/hudi/HudiColumnValue.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java index bb7272eede..4b4a7bbcf7 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiColumnValue.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java @@ -17,7 +17,8 @@ package org.apache.doris.hudi; -import org.apache.doris.jni.vec.ColumnValue; + +import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; diff --git a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiJniScanner.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/hudi/HudiJniScanner.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java index a913329214..d7078dc9ca 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiJniScanner.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java @@ -17,9 +17,9 @@ package org.apache.doris.hudi; -import org.apache.doris.jni.JniScanner; -import org.apache.doris.jni.utils.Utils; -import org.apache.doris.jni.vec.ColumnValue; + +import org.apache.doris.common.jni.JniScanner; +import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.serde2.Deserializer; @@ -163,7 +163,8 @@ public class HudiJniScanner extends JniScanner { executorService.scheduleAtFixedRate(() -> { if (!isKilled.get()) { synchronized (HudiJniScanner.class) { - List pids = Utils.getChildProcessIds(Utils.getCurrentProcId()); + List pids = Utils.getChildProcessIds( + Utils.getCurrentProcId()); for (long pid : pids) { String cmd = Utils.getCommandLine(pid); if (cmd != null && cmd.contains("org.openjdk.jol.vm.sa.AttachMain")) { diff --git a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanParam.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanParam.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanParam.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanParam.java index 68a0b88e0a..4343dd3f49 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanParam.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanParam.java @@ -17,7 +17,8 @@ package org.apache.doris.hudi; -import org.apache.doris.jni.vec.ColumnType; + +import org.apache.doris.common.jni.vec.ColumnType; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.serde.serdeConstants; diff --git a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanUtils.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanUtils.java similarity index 100% rename from fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanUtils.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanUtils.java diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/Utils.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/jni/utils/Utils.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java index 53805e96f1..c7f3ecf9da 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/Utils.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.utils; +package org.apache.doris.hudi; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; diff --git a/fe/java-udf/src/main/resources/package.xml b/fe/be-java-extensions/hudi-scanner/src/main/resources/package.xml similarity index 100% rename from fe/java-udf/src/main/resources/package.xml rename to fe/be-java-extensions/hudi-scanner/src/main/resources/package.xml diff --git a/fe/java-udf/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java b/fe/be-java-extensions/hudi-scanner/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java rename to fe/be-java-extensions/hudi-scanner/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java diff --git a/fe/be-java-extensions/java-common/pom.xml b/fe/be-java-extensions/java-common/pom.xml new file mode 100644 index 0000000000..eedd7c00ad --- /dev/null +++ b/fe/be-java-extensions/java-common/pom.xml @@ -0,0 +1,60 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + java-common + + + 8 + 8 + + + + + com.vesoft + client + + + org.apache.doris + fe-common + ${project.version} + + + com.fasterxml.jackson.core + jackson-core + + + org.apache.velocity + velocity-engine-core + + + org.apache.httpcomponents + httpclient + + + + diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/InternalException.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/InternalException.java similarity index 95% rename from fe/java-udf/src/main/java/org/apache/doris/udf/InternalException.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/InternalException.java index 1b1e335ae2..ddd94e5048 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/InternalException.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/InternalException.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.exception; public class InternalException extends Exception { public InternalException(String msg, Throwable cause) { diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfRuntimeException.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/UdfRuntimeException.java similarity index 95% rename from fe/java-udf/src/main/java/org/apache/doris/udf/UdfRuntimeException.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/UdfRuntimeException.java index ef02ae5c36..861b4f46fe 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfRuntimeException.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/UdfRuntimeException.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.exception; public class UdfRuntimeException extends Exception { public UdfRuntimeException(String msg, Throwable cause) { diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/JniScanner.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java similarity index 92% rename from fe/java-udf/src/main/java/org/apache/doris/jni/JniScanner.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java index 3d0223ba75..7e7391520e 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/JniScanner.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.common.jni; -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.ColumnValue; -import org.apache.doris.jni.vec.ScanPredicate; -import org.apache.doris.jni.vec.VectorTable; + +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ColumnValue; +import org.apache.doris.common.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.vec.VectorTable; import java.io.IOException; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/MockJniScanner.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java similarity index 96% rename from fe/java-udf/src/main/java/org/apache/doris/jni/MockJniScanner.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java index c4c4c5f80b..b559f2a0af 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/MockJniScanner.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.common.jni; -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.ColumnValue; -import org.apache.doris.jni.vec.ScanPredicate; + +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ColumnValue; +import org.apache.doris.common.jni.vec.ScanPredicate; import org.apache.log4j.Logger; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JMXJsonUtil.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JMXJsonUtil.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JMXJsonUtil.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JMXJsonUtil.java index 5089b45a07..02cb53232d 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JMXJsonUtil.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JMXJsonUtil.java @@ -18,7 +18,7 @@ // https://github.com/apache/impala/blob/branch-4.0.0/fe/src/main/java/org/apache/impala/util/JMXJsonUtil.java // and modified by Doris -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JNINativeMethod.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JNINativeMethod.java similarity index 96% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JNINativeMethod.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JNINativeMethod.java index c496bd5f0f..50820c2ecb 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JNINativeMethod.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JNINativeMethod.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; /** * Native method in doris::JavaNativeMethods. diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java index af1cf6e3b0..1a91df84be 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java @@ -15,8 +15,9 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; +import org.apache.doris.common.exception.InternalException; import org.apache.doris.thrift.TGetJMXJsonResponse; import org.apache.doris.thrift.TGetJvmMemoryMetricsResponse; import org.apache.doris.thrift.TGetJvmThreadsInfoRequest; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JvmPauseMonitor.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JvmPauseMonitor.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JvmPauseMonitor.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JvmPauseMonitor.java index 1129c7ac49..85af333156 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JvmPauseMonitor.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JvmPauseMonitor.java @@ -18,7 +18,7 @@ // https://github.com/apache/impala/blob/branch-4.0.0/fe/src/main/java/org/apache/impala/util/JvmPauseMonitor.java // and modified by Doris -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; import com.google.common.base.Joiner; import com.google.common.base.Stopwatch; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/OffHeap.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/OffHeap.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/jni/utils/OffHeap.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/OffHeap.java index 2ad77b7a7e..268f5f6787 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/OffHeap.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/OffHeap.java @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.utils; +package org.apache.doris.common.jni.utils; -import org.apache.doris.udf.JNINativeMethod; import sun.misc.Unsafe; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/TypeNativeBytes.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/jni/utils/TypeNativeBytes.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java index 666f968329..bd2eb79cd2 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/TypeNativeBytes.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.utils; +package org.apache.doris.common.jni.utils; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfUtils.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/UdfUtils.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java index c1248f87ca..09e0dc3804 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfUtils.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java @@ -15,13 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.Pair; +import org.apache.doris.common.exception.InternalException; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.TScalarType; import org.apache.doris.thrift.TTypeDesc; @@ -193,7 +194,7 @@ public class UdfUtils { } } - protected static Pair fromThrift(TTypeDesc typeDesc, int nodeIdx) throws InternalException { + public static Pair fromThrift(TTypeDesc typeDesc, int nodeIdx) throws InternalException { TTypeNode node = typeDesc.getTypes().get(nodeIdx); Type type = null; switch (node.getType()) { @@ -238,7 +239,7 @@ public class UdfUtils { return Pair.of(type, nodeIdx); } - protected static long getAddressAtOffset(long base, int offset) { + public static long getAddressAtOffset(long base, int offset) { return base + 8L * offset; } diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnType.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java index a5eee7e43e..eba69e51c8 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnType.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; import java.util.ArrayList; import java.util.Arrays; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnValue.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnValue.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java index 8d190aa212..0495179260 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnValue.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ScanPredicate.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/ScanPredicate.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java index e02107f143..f9b35b2352 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ScanPredicate.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.utils.TypeNativeBytes; -import org.apache.doris.jni.vec.ColumnType.Type; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.utils.TypeNativeBytes; +import org.apache.doris.common.jni.vec.ColumnType.Type; import org.apache.commons.lang3.StringUtils; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorColumn.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index 4996191776..70b358a87f 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.utils.TypeNativeBytes; -import org.apache.doris.jni.vec.ColumnType.Type; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.utils.TypeNativeBytes; +import org.apache.doris.common.jni.vec.ColumnType.Type; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorTable.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java similarity index 96% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorTable.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java index 3f86ebc7a2..e70d8f683f 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorTable.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.vec.ColumnType.Type; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.ColumnType.Type; /** * Store a batch of data as vector table. diff --git a/fe/java-udf/src/main/resources/log4j.properties b/fe/be-java-extensions/java-common/src/main/resources/log4j.properties similarity index 100% rename from fe/java-udf/src/main/resources/log4j.properties rename to fe/be-java-extensions/java-common/src/main/resources/log4j.properties diff --git a/fe/be-java-extensions/java-common/src/main/resources/package.xml b/fe/be-java-extensions/java-common/src/main/resources/package.xml new file mode 100644 index 0000000000..4bbb261060 --- /dev/null +++ b/fe/be-java-extensions/java-common/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/java-udf/src/test/java/org/apache/doris/jni/JniScannerTest.java b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java similarity index 94% rename from fe/java-udf/src/test/java/org/apache/doris/jni/JniScannerTest.java rename to fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java index 6be67ecba3..e8a6d49df7 100644 --- a/fe/java-udf/src/test/java/org/apache/doris/jni/JniScannerTest.java +++ b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.common.jni; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.vec.VectorTable; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.VectorTable; import org.junit.Assert; import org.junit.Test; diff --git a/fe/be-java-extensions/java-udf/pom.xml b/fe/be-java-extensions/java-udf/pom.xml new file mode 100644 index 0000000000..bf05aeafc2 --- /dev/null +++ b/fe/be-java-extensions/java-udf/pom.xml @@ -0,0 +1,74 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + java-udf + + + 8 + 8 + + + + + + org.apache.doris + java-common + ${project.version} + + + + + java-udf + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java rename to fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java index f1d2e3a9a7..3dbe10ca27 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java @@ -19,8 +19,11 @@ package org.apache.doris.udf; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; +import org.apache.doris.common.exception.InternalException; +import org.apache.doris.common.exception.UdfRuntimeException; +import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType; import org.apache.doris.thrift.TJavaUdfExecutorCtorParams; -import org.apache.doris.udf.UdfUtils.JavaUdfDataType; import org.apache.log4j.Logger; import org.apache.thrift.TDeserializer; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java rename to fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java index 76e9a09c3d..e2d8ab1b75 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java @@ -19,8 +19,10 @@ package org.apache.doris.udf; import org.apache.doris.catalog.Type; import org.apache.doris.common.Pair; +import org.apache.doris.common.exception.UdfRuntimeException; +import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType; import org.apache.doris.thrift.TJavaUdfExecutorCtorParams; -import org.apache.doris.udf.UdfUtils.JavaUdfDataType; import com.google.common.base.Joiner; import com.google.common.collect.Lists; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java rename to fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java index b888279db3..18eaaf8c45 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java @@ -19,8 +19,10 @@ package org.apache.doris.udf; import org.apache.doris.catalog.Type; import org.apache.doris.common.Pair; +import org.apache.doris.common.exception.UdfRuntimeException; +import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType; import org.apache.doris.thrift.TJavaUdfExecutorCtorParams; -import org.apache.doris.udf.UdfUtils.JavaUdfDataType; import com.google.common.base.Joiner; import com.google.common.collect.Lists; diff --git a/fe/be-java-extensions/java-udf/src/main/resources/package.xml b/fe/be-java-extensions/java-udf/src/main/resources/package.xml new file mode 100644 index 0000000000..4bbb261060 --- /dev/null +++ b/fe/be-java-extensions/java-udf/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java similarity index 99% rename from fe/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java index 7330c9b083..7c725da50e 100644 --- a/fe/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java +++ b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java @@ -17,6 +17,7 @@ package org.apache.doris.udf; +import org.apache.doris.common.jni.utils.UdfUtils; import org.apache.doris.thrift.TFunction; import org.apache.doris.thrift.TFunctionBinaryType; import org.apache.doris.thrift.TFunctionName; diff --git a/fe/be-java-extensions/jdbc-scanner/pom.xml b/fe/be-java-extensions/jdbc-scanner/pom.xml new file mode 100644 index 0000000000..037804bab7 --- /dev/null +++ b/fe/be-java-extensions/jdbc-scanner/pom.xml @@ -0,0 +1,90 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + jdbc-scanner + + + 8 + 8 + + + + + org.apache.doris + java-common + ${project.version} + + + + + com.oracle.database.jdbc + ojdbc6 + + + + com.alibaba + druid + + + com.clickhouse + clickhouse-jdbc + all + + + + + jdbc-scanner + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + + diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcDataSource.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JdbcDataSource.java rename to fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java index 4d6f4f59e6..95b3dac585 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcDataSource.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.jdbc; import com.alibaba.druid.pool.DruidDataSource; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java rename to fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java index 4f8eb2199d..11bf3d6d79 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.jdbc; -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.VectorColumn; -import org.apache.doris.jni.vec.VectorTable; +import org.apache.doris.common.exception.InternalException; +import org.apache.doris.common.exception.UdfRuntimeException; +import org.apache.doris.common.jni.utils.JNINativeMethod; +import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.VectorColumn; +import org.apache.doris.common.jni.vec.VectorTable; import org.apache.doris.thrift.TJdbcExecutorCtorParams; import org.apache.doris.thrift.TJdbcOperation; import org.apache.doris.thrift.TOdbcTableType; diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/resources/package.xml b/fe/be-java-extensions/jdbc-scanner/src/main/resources/package.xml new file mode 100644 index 0000000000..4bbb261060 --- /dev/null +++ b/fe/be-java-extensions/jdbc-scanner/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/be-java-extensions/max-compute-scanner/pom.xml b/fe/be-java-extensions/max-compute-scanner/pom.xml new file mode 100644 index 0000000000..8e057da807 --- /dev/null +++ b/fe/be-java-extensions/max-compute-scanner/pom.xml @@ -0,0 +1,104 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + max-compute-scanner + + + 8 + 8 + + + + + org.apache.doris + java-common + ${project.version} + + + com.aliyun.odps + odps-sdk-core + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + org.apache.arrow + arrow-memory-unsafe + ${arrow.version} + + + + + max-compute-scanner + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + + diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/MaxComputeColumnValue.java b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/MaxComputeColumnValue.java rename to fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java index 0945f1f326..4c3aff97b4 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/MaxComputeColumnValue.java +++ b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.maxcompute; + +import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.DateDayVector; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/MaxComputeJniScanner.java b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/jni/MaxComputeJniScanner.java rename to fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java index 65aa2ccded..95664c93c3 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/MaxComputeJniScanner.java +++ b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.maxcompute; -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.MaxComputeColumnValue; -import org.apache.doris.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.JniScanner; +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ScanPredicate; import com.aliyun.odps.Column; import com.aliyun.odps.Odps; diff --git a/fe/be-java-extensions/max-compute-scanner/src/main/resources/package.xml b/fe/be-java-extensions/max-compute-scanner/src/main/resources/package.xml new file mode 100644 index 0000000000..4bbb261060 --- /dev/null +++ b/fe/be-java-extensions/max-compute-scanner/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/be-java-extensions/paimon-scanner/pom.xml b/fe/be-java-extensions/paimon-scanner/pom.xml new file mode 100644 index 0000000000..9a61869d65 --- /dev/null +++ b/fe/be-java-extensions/paimon-scanner/pom.xml @@ -0,0 +1,100 @@ + + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + paimon-scanner + + + 8 + 8 + + + + + org.apache.doris + java-common + ${project.version} + + + + hive-common + org.apache.hive + + + + org.apache.paimon + paimon-bundle + ${paimon.version} + + + org.apache.paimon + paimon-hive-connector-2.3 + ${paimon.version} + + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + + org.apache.hadoop + hadoop-mapreduce-client-common + + + + + + paimon-scanner + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/PaimonColumnValue.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/PaimonColumnValue.java rename to fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index 8e3744b1f1..e0a89c9db2 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.paimon; + +import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.paimon.data.columnar.ColumnarRow; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/jni/PaimonJniScanner.java rename to fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 18d8d1dc4b..190bdb829a 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.paimon; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.PaimonColumnValue; -import org.apache.doris.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.JniScanner; +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ScanPredicate; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.log4j.Logger; diff --git a/fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml b/fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml new file mode 100644 index 0000000000..4bbb261060 --- /dev/null +++ b/fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml new file mode 100644 index 0000000000..aee6321093 --- /dev/null +++ b/fe/be-java-extensions/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + hudi-scanner + java-common + java-udf + jdbc-scanner + paimon-scanner + max-compute-scanner + + + + org.apache.doris + ${revision} + fe + ../pom.xml + + + be-java-extensions + pom + + ${basedir}/../../ + 1 + + + diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 2c264e146b..cc35349f59 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -34,7 +34,6 @@ under the License. 1 4.9.3 2.17.257 - 0.4-SNAPSHOT @@ -499,6 +498,10 @@ under the License. org.apache.hadoop hadoop-hdfs + + org.apache.hadoop + hadoop-client + org.apache.hadoop hadoop-auth diff --git a/fe/pom.xml b/fe/pom.xml index 5cb5f36482..4d7b077ab8 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -186,7 +186,7 @@ under the License. spark-dpp fe-core hive-udf - java-udf + be-java-extensions @@ -263,6 +263,8 @@ under the License. 0.8.13 2.4.6 3.1.3 + 2.3.9 + 2.10.1 1.33.0 1.22.0 @@ -270,9 +272,13 @@ under the License. you can find avro version info in iceberg mvn repository --> 1.1.0 0.43.3-public + 9.0.0 1.11.1 0.13.0 + 2.7.4-11 + 3.0.0-8 + 1.13.0 3.2.2 1.22 @@ -299,6 +305,9 @@ under the License. 6.5.1 2.0.3 1.5.4 + 3.0.0 + + 0.4.0-incubating @@ -460,6 +469,12 @@ under the License. org.apache.hadoop hadoop-client ${hadoop.version} + + + javax.servlet + servlet-api + + org.apache.hadoop @@ -1090,6 +1105,12 @@ under the License. org.apache.hadoop hadoop-mapreduce-client ${hadoop.version} + + + javax.servlet + servlet-api + + @@ -1269,6 +1290,27 @@ under the License. metastore-client-hive3 ${dlf-metastore-client-hive.version} + + org.apache.hive + hive-common + ${hive.common.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${mapreduce.client.version} + + + javax.servlet + servlet-api + + + + + org.apache.hadoop + hadoop-mapreduce-client-common + ${mapreduce.client.version} + com.alibaba druid @@ -1297,6 +1339,11 @@ under the License. tomcat-embed-core ${tomcat-embed-core.version} + + com.vesoft + client + ${vesoft.client.version} +