diff --git a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h index 816576046f..37888b35ac 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h +++ b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h @@ -37,6 +37,7 @@ #include "vec/common/string_ref.h" #include "vec/core/field.h" #include "vec/core/types.h" +#include "vec/exec/jni_connector.h" #include "vec/io/io_helper.h" namespace doris::vectorized { @@ -45,10 +46,10 @@ const char* UDAF_EXECUTOR_CLASS = "org/apache/doris/udf/UdafExecutor"; const char* UDAF_EXECUTOR_CTOR_SIGNATURE = "([B)V"; const char* UDAF_EXECUTOR_CLOSE_SIGNATURE = "()V"; const char* UDAF_EXECUTOR_DESTROY_SIGNATURE = "()V"; -const char* UDAF_EXECUTOR_ADD_SIGNATURE = "(ZJJ)V"; +const char* UDAF_EXECUTOR_ADD_SIGNATURE = "(ZIIJILjava/util/Map;)V"; const char* UDAF_EXECUTOR_SERIALIZE_SIGNATURE = "(J)[B"; const char* UDAF_EXECUTOR_MERGE_SIGNATURE = "(J[B)V"; -const char* UDAF_EXECUTOR_RESULT_SIGNATURE = "(JJ)Z"; +const char* UDAF_EXECUTOR_GET_SIGNATURE = "(JLjava/util/Map;)J"; const char* UDAF_EXECUTOR_RESET_SIGNATURE = "(J)V"; // Calling Java method about those signature means: "(argument-types)return-type" // https://www.iitk.ac.in/esc101/05Aug/tutorial/native1.1/implementing/method.html @@ -60,10 +61,14 @@ public: ~AggregateJavaUdafData() { JNIEnv* env; - Status status; - RETURN_IF_STATUS_ERROR(status, JniUtil::GetJNIEnv(&env)); + if (!JniUtil::GetJNIEnv(&env).ok()) { + LOG(WARNING) << "Failed to get JNIEnv"; + } env->CallNonvirtualVoidMethod(executor_obj, executor_cl, executor_close_id); - RETURN_IF_STATUS_ERROR(status, JniUtil::GetJniExceptionMsg(env)); + Status st = JniUtil::GetJniExceptionMsg(env); + if (!st.ok()) { + LOG(WARNING) << "Failed to close JAVA UDAF: " << st.to_string(); + } env->DeleteGlobalRef(executor_cl); env->DeleteGlobalRef(executor_obj); } @@ -103,126 +108,24 @@ public: int place_offset) { JNIEnv* env = nullptr; RETURN_NOT_OK_STATUS_WITH_WARN(JniUtil::GetJNIEnv(&env), "Java-Udaf add function"); - jclass obj_class = env->FindClass("[Ljava/lang/Object;"); - jobjectArray arg_objects = env->NewObjectArray(argument_size, obj_class, nullptr); - int64_t nullmap_address = 0; - for (int arg_idx = 0; arg_idx < argument_size; ++arg_idx) { - bool arg_column_nullable = false; - auto data_col = columns[arg_idx]; - if (auto* nullable = check_and_get_column(*columns[arg_idx])) { - arg_column_nullable = true; - auto null_col = nullable->get_null_map_column_ptr(); - data_col = nullable->get_nested_column_ptr(); - nullmap_address = reinterpret_cast( - check_and_get_column>(null_col)->get_data().data()); - } - // convert argument column data into java type - jobjectArray arr_obj = nullptr; - if (data_col->is_numeric() || data_col->is_column_decimal()) { - arr_obj = (jobjectArray)env->CallObjectMethod( - executor_obj, executor_convert_basic_argument_id, arg_idx, - arg_column_nullable, row_num_start, row_num_end, nullmap_address, - reinterpret_cast(data_col->get_raw_data().data), 0); - } else if (data_col->is_column_string()) { - const ColumnString* str_col = assert_cast(data_col); - arr_obj = (jobjectArray)env->CallObjectMethod( - executor_obj, executor_convert_basic_argument_id, arg_idx, - arg_column_nullable, row_num_start, row_num_end, nullmap_address, - reinterpret_cast(str_col->get_chars().data()), - reinterpret_cast(str_col->get_offsets().data())); - } else if (data_col->is_column_array()) { - const ColumnArray* array_col = assert_cast(data_col); - const ColumnNullable& array_nested_nullable = - assert_cast(array_col->get_data()); - auto data_column_null_map = array_nested_nullable.get_null_map_column_ptr(); - auto data_column = array_nested_nullable.get_nested_column_ptr(); - auto offset_address = reinterpret_cast( - array_col->get_offsets_column().get_raw_data().data); - auto nested_nullmap_address = reinterpret_cast( - check_and_get_column>(data_column_null_map) - ->get_data() - .data()); - int64_t nested_data_address = 0, nested_offset_address = 0; - // array type need pass address: [nullmap_address], offset_address, nested_nullmap_address, nested_data_address/nested_char_address,nested_offset_address - if (data_column->is_column_string()) { - const ColumnString* col = assert_cast(data_column.get()); - nested_data_address = reinterpret_cast(col->get_chars().data()); - nested_offset_address = reinterpret_cast(col->get_offsets().data()); - } else { - nested_data_address = - reinterpret_cast(data_column->get_raw_data().data); - } - arr_obj = (jobjectArray)env->CallObjectMethod( - executor_obj, executor_convert_array_argument_id, arg_idx, - arg_column_nullable, row_num_start, row_num_end, nullmap_address, - offset_address, nested_nullmap_address, nested_data_address, - nested_offset_address); - } else if (data_col->is_column_map()) { - const ColumnMap* map_col = assert_cast(data_col); - auto offset_address = reinterpret_cast( - map_col->get_offsets_column().get_raw_data().data); - const ColumnNullable& map_key_column_nullable = - assert_cast(map_col->get_keys()); - auto key_data_column_null_map = map_key_column_nullable.get_null_map_column_ptr(); - auto key_data_column = map_key_column_nullable.get_nested_column_ptr(); - - auto key_nested_nullmap_address = reinterpret_cast( - check_and_get_column>(key_data_column_null_map) - ->get_data() - .data()); - int64_t key_nested_data_address = 0, key_nested_offset_address = 0; - if (key_data_column->is_column_string()) { - const ColumnString* col = - assert_cast(key_data_column.get()); - key_nested_data_address = reinterpret_cast(col->get_chars().data()); - key_nested_offset_address = - reinterpret_cast(col->get_offsets().data()); - } else { - key_nested_data_address = - reinterpret_cast(key_data_column->get_raw_data().data); - } - - const ColumnNullable& map_value_column_nullable = - assert_cast(map_col->get_values()); - auto value_data_column_null_map = - map_value_column_nullable.get_null_map_column_ptr(); - auto value_data_column = map_value_column_nullable.get_nested_column_ptr(); - auto value_nested_nullmap_address = reinterpret_cast( - check_and_get_column>(value_data_column_null_map) - ->get_data() - .data()); - int64_t value_nested_data_address = 0, value_nested_offset_address = 0; - if (value_data_column->is_column_string()) { - const ColumnString* col = - assert_cast(value_data_column.get()); - value_nested_data_address = reinterpret_cast(col->get_chars().data()); - value_nested_offset_address = - reinterpret_cast(col->get_offsets().data()); - } else { - value_nested_data_address = - reinterpret_cast(value_data_column->get_raw_data().data); - } - arr_obj = (jobjectArray)env->CallObjectMethod( - executor_obj, executor_convert_map_argument_id, arg_idx, - arg_column_nullable, row_num_start, row_num_end, nullmap_address, - offset_address, key_nested_nullmap_address, key_nested_data_address, - key_nested_offset_address, value_nested_nullmap_address, - value_nested_data_address, value_nested_offset_address); - } else { - return Status::InvalidArgument( - strings::Substitute("Java UDAF doesn't support type is $0 now !", - argument_types[arg_idx]->get_name())); - } - env->SetObjectArrayElement(arg_objects, arg_idx, arr_obj); - env->DeleteLocalRef(arr_obj); + Block input_block; + for (size_t i = 0; i < argument_size; ++i) { + input_block.insert(ColumnWithTypeAndName(columns[i]->get_ptr(), argument_types[i], + std::to_string(i))); } - RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); + std::unique_ptr input_table; + RETURN_IF_ERROR(JniConnector::to_java_table(&input_block, input_table)); + auto input_table_schema = JniConnector::parse_table_schema(&input_block); + std::map input_params = { + {"meta_address", std::to_string((long)input_table.get())}, + {"required_fields", input_table_schema.first}, + {"columns_types", input_table_schema.second}}; + jobject input_map = JniUtil::convert_to_java_map(env, input_params); // invoke add batch env->CallObjectMethod(executor_obj, executor_add_batch_id, is_single_place, row_num_start, - row_num_end, places_address, place_offset, arg_objects); - env->DeleteLocalRef(arg_objects); - env->DeleteLocalRef(obj_class); + row_num_end, places_address, place_offset, input_map); + env->DeleteLocalRef(input_map); return JniUtil::GetJniExceptionMsg(env); } @@ -275,160 +178,33 @@ public: } Status get(IColumn& to, const DataTypePtr& result_type, int64_t place) const { - to.insert_default(); JNIEnv* env = nullptr; RETURN_NOT_OK_STATUS_WITH_WARN(JniUtil::GetJNIEnv(&env), "Java-Udaf get value function"); - int64_t nullmap_address = 0; - if (result_type->is_nullable()) { - auto& nullable = assert_cast(to); - nullmap_address = - reinterpret_cast(nullable.get_null_map_column().get_raw_data().data); - auto& data_col = nullable.get_nested_column(); - RETURN_IF_ERROR(get_result(to, result_type, place, env, data_col, nullmap_address)); - } else { - nullmap_address = -1; - auto& data_col = to; - RETURN_IF_ERROR(get_result(to, result_type, place, env, data_col, nullmap_address)); - } - return JniUtil::GetJniExceptionMsg(env); + + Block output_block; + output_block.insert(ColumnWithTypeAndName(to.get_ptr(), result_type, "_result_")); + auto output_table_schema = JniConnector::parse_table_schema(&output_block); + std::string output_nullable = result_type->is_nullable() ? "true" : "false"; + std::map output_params = {{"is_nullable", output_nullable}, + {"required_fields", output_table_schema.first}, + {"columns_types", output_table_schema.second}}; + jobject output_map = JniUtil::convert_to_java_map(env, output_params); + long output_address = + env->CallLongMethod(executor_obj, executor_get_value_id, place, output_map); + RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); + env->DeleteLocalRef(output_map); + + return JniConnector::fill_block(&output_block, {0}, output_address); } private: - Status get_result(IColumn& to, const DataTypePtr& return_type, int64_t place, JNIEnv* env, - IColumn& data_col, int64_t nullmap_address) const { - jobject result_obj = env->CallNonvirtualObjectMethod(executor_obj, executor_cl, - executor_get_value_id, place); - bool result_nullable = return_type->is_nullable(); - if (data_col.is_column_string()) { - const ColumnString* str_col = check_and_get_column(data_col); - ColumnString::Chars& chars = const_cast(str_col->get_chars()); - ColumnString::Offsets& offsets = - const_cast(str_col->get_offsets()); - int increase_buffer_size = 0; - int64_t buffer_size = JniUtil::IncreaseReservedBufferSize(increase_buffer_size); - chars.resize(buffer_size); - env->CallNonvirtualVoidMethod( - executor_obj, executor_cl, executor_copy_basic_result_id, result_obj, - to.size() - 1, nullmap_address, reinterpret_cast(chars.data()), - reinterpret_cast(&chars), reinterpret_cast(offsets.data())); - } else if (data_col.is_numeric() || data_col.is_column_decimal()) { - env->CallNonvirtualVoidMethod(executor_obj, executor_cl, executor_copy_basic_result_id, - result_obj, to.size() - 1, nullmap_address, - reinterpret_cast(data_col.get_raw_data().data), - 0, 0); - } else if (data_col.is_column_array()) { - jclass arraylist_class = env->FindClass("Ljava/util/ArrayList;"); - ColumnArray* array_col = assert_cast(&data_col); - ColumnNullable& array_nested_nullable = - assert_cast(array_col->get_data()); - auto data_column_null_map = array_nested_nullable.get_null_map_column_ptr(); - auto data_column = array_nested_nullable.get_nested_column_ptr(); - auto& offset_column = array_col->get_offsets_column(); - auto offset_address = reinterpret_cast(offset_column.get_raw_data().data); - auto& null_map_data = - assert_cast*>(data_column_null_map.get())->get_data(); - auto nested_nullmap_address = reinterpret_cast(null_map_data.data()); - jmethodID list_size = env->GetMethodID(arraylist_class, "size", "()I"); - - size_t has_put_element_size = array_col->get_offsets().back(); - size_t arrar_list_size = env->CallIntMethod(result_obj, list_size); - size_t element_size = has_put_element_size + arrar_list_size; - array_nested_nullable.resize(element_size); - memset(null_map_data.data() + has_put_element_size, 0, arrar_list_size); - int64_t nested_data_address = 0, nested_offset_address = 0; - if (data_column->is_column_string()) { - ColumnString* str_col = assert_cast(data_column.get()); - ColumnString::Chars& chars = - assert_cast(str_col->get_chars()); - ColumnString::Offsets& offsets = - assert_cast(str_col->get_offsets()); - nested_data_address = reinterpret_cast(&chars); - nested_offset_address = reinterpret_cast(offsets.data()); - } else { - nested_data_address = reinterpret_cast(data_column->get_raw_data().data); - } - int row = to.size() - 1; - env->CallNonvirtualVoidMethod(executor_obj, executor_cl, executor_copy_array_result_id, - has_put_element_size, result_nullable, row, result_obj, - nullmap_address, offset_address, nested_nullmap_address, - nested_data_address, nested_offset_address); - env->DeleteLocalRef(arraylist_class); - } else if (data_col.is_column_map()) { - jclass hashmap_class = env->FindClass("Ljava/util/HashMap;"); - ColumnMap* map_col = assert_cast(&data_col); - auto& offset_column = map_col->get_offsets_column(); - auto offset_address = reinterpret_cast(offset_column.get_raw_data().data); - ColumnNullable& map_key_column_nullable = - assert_cast(map_col->get_keys()); - auto key_data_column_null_map = map_key_column_nullable.get_null_map_column_ptr(); - auto key_data_column = map_key_column_nullable.get_nested_column_ptr(); - auto& key_null_map_data = - assert_cast*>(key_data_column_null_map.get())->get_data(); - auto key_nested_nullmap_address = reinterpret_cast(key_null_map_data.data()); - ColumnNullable& map_value_column_nullable = - assert_cast(map_col->get_values()); - auto value_data_column_null_map = map_value_column_nullable.get_null_map_column_ptr(); - auto value_data_column = map_value_column_nullable.get_nested_column_ptr(); - auto& value_null_map_data = - assert_cast*>(value_data_column_null_map.get())->get_data(); - auto value_nested_nullmap_address = - reinterpret_cast(value_null_map_data.data()); - jmethodID map_size = env->GetMethodID(hashmap_class, "size", "()I"); - size_t has_put_element_size = map_col->get_offsets().back(); - size_t hashmap_size = env->CallIntMethod(result_obj, map_size); - size_t element_size = has_put_element_size + hashmap_size; - map_key_column_nullable.resize(element_size); - memset(key_null_map_data.data() + has_put_element_size, 0, hashmap_size); - map_value_column_nullable.resize(element_size); - memset(value_null_map_data.data() + has_put_element_size, 0, hashmap_size); - - int64_t key_nested_data_address = 0, key_nested_offset_address = 0; - if (key_data_column->is_column_string()) { - ColumnString* str_col = assert_cast(key_data_column.get()); - ColumnString::Chars& chars = - assert_cast(str_col->get_chars()); - ColumnString::Offsets& offsets = - assert_cast(str_col->get_offsets()); - key_nested_data_address = reinterpret_cast(&chars); - key_nested_offset_address = reinterpret_cast(offsets.data()); - } else { - key_nested_data_address = - reinterpret_cast(key_data_column->get_raw_data().data); - } - - int64_t value_nested_data_address = 0, value_nested_offset_address = 0; - if (value_data_column->is_column_string()) { - ColumnString* str_col = assert_cast(value_data_column.get()); - ColumnString::Chars& chars = - assert_cast(str_col->get_chars()); - ColumnString::Offsets& offsets = - assert_cast(str_col->get_offsets()); - value_nested_data_address = reinterpret_cast(&chars); - value_nested_offset_address = reinterpret_cast(offsets.data()); - } else { - value_nested_data_address = - reinterpret_cast(value_data_column->get_raw_data().data); - } - int row = to.size() - 1; - env->CallNonvirtualVoidMethod(executor_obj, executor_cl, executor_copy_map_result_id, - has_put_element_size, result_nullable, row, result_obj, - nullmap_address, offset_address, - key_nested_nullmap_address, key_nested_data_address, - key_nested_offset_address, value_nested_nullmap_address, - value_nested_data_address, value_nested_offset_address); - env->DeleteLocalRef(hashmap_class); - } else { - return Status::InvalidArgument(strings::Substitute( - "Java UDAF doesn't support return type is $0 now !", return_type->get_name())); - } - return Status::OK(); - } - Status register_func_id(JNIEnv* env) { auto register_id = [&](const char* func_name, const char* func_sign, jmethodID& func_id) { func_id = env->GetMethodID(executor_cl, func_name, func_sign); Status s = JniUtil::GetJniExceptionMsg(env); if (!s.ok()) { + LOG(WARNING) << "Failed to register function " << func_name << ": " + << s.to_string(); return Status::InternalError(strings::Substitute( "Java-Udaf register_func_id meet error and error is $0", s.to_string())); } @@ -440,27 +216,12 @@ private: RETURN_IF_ERROR(register_id("merge", UDAF_EXECUTOR_MERGE_SIGNATURE, executor_merge_id)); RETURN_IF_ERROR( register_id("serialize", UDAF_EXECUTOR_SERIALIZE_SIGNATURE, executor_serialize_id)); - RETURN_IF_ERROR(register_id("getValue", "(J)Ljava/lang/Object;", executor_get_value_id)); + RETURN_IF_ERROR( + register_id("getValue", UDAF_EXECUTOR_GET_SIGNATURE, executor_get_value_id)); RETURN_IF_ERROR( register_id("destroy", UDAF_EXECUTOR_DESTROY_SIGNATURE, executor_destroy_id)); - RETURN_IF_ERROR(register_id("convertBasicArguments", "(IZIIJJJ)[Ljava/lang/Object;", - executor_convert_basic_argument_id)); - RETURN_IF_ERROR(register_id("convertArrayArguments", "(IZIIJJJJJ)[Ljava/lang/Object;", - executor_convert_array_argument_id)); - RETURN_IF_ERROR(register_id("convertMapArguments", "(IZIIJJJJJJJJ)[Ljava/lang/Object;", - executor_convert_map_argument_id)); - - RETURN_IF_ERROR(register_id("copyTupleBasicResult", "(Ljava/lang/Object;IJJJJ)V", - executor_copy_basic_result_id)); - - RETURN_IF_ERROR(register_id("copyTupleArrayResult", "(JZILjava/lang/Object;JJJJJ)V", - executor_copy_array_result_id)); - - RETURN_IF_ERROR(register_id("copyTupleMapResult", "(JZILjava/lang/Object;JJJJJJJJ)V", - executor_copy_map_result_id)); - RETURN_IF_ERROR( - register_id("addBatch", "(ZIIJI[Ljava/lang/Object;)V", executor_add_batch_id)); + register_id("addBatch", UDAF_EXECUTOR_ADD_SIGNATURE, executor_add_batch_id)); return Status::OK(); } @@ -478,12 +239,6 @@ private: jmethodID executor_reset_id; jmethodID executor_close_id; jmethodID executor_destroy_id; - jmethodID executor_convert_basic_argument_id; - jmethodID executor_convert_array_argument_id; - jmethodID executor_convert_map_argument_id; - jmethodID executor_copy_basic_result_id; - jmethodID executor_copy_array_result_id; - jmethodID executor_copy_map_result_id; int argument_size = 0; std::string serialize_data; }; diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp index 2c2a9625a6..6756ba47e5 100644 --- a/be/src/vec/exec/jni_connector.cpp +++ b/be/src/vec/exec/jni_connector.cpp @@ -311,8 +311,8 @@ Status JniConnector::_fill_column(TableMetaAddress& address, ColumnPtr& doris_co } MutableColumnPtr data_column; if (doris_column->is_nullable()) { - auto* nullable_column = reinterpret_cast( - (*std::move(doris_column)).mutate().get()); + auto* nullable_column = + reinterpret_cast(doris_column->assume_mutable().get()); data_column = nullable_column->get_nested_column_ptr(); NullMap& null_map = nullable_column->get_null_map_data(); size_t origin_size = null_map.size(); diff --git a/be/src/vec/functions/function_java_udf.cpp b/be/src/vec/functions/function_java_udf.cpp index 3687734e2e..ca69fbb0bf 100644 --- a/be/src/vec/functions/function_java_udf.cpp +++ b/be/src/vec/functions/function_java_udf.cpp @@ -138,9 +138,9 @@ Status JavaFunctionCall::execute_impl(FunctionContext* context, Block& block, jobject output_map = JniUtil::convert_to_java_map(env, output_params); long output_address = env->CallLongMethod(jni_ctx->executor, jni_env->executor_evaluate_id, input_map, output_map); + RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); env->DeleteLocalRef(input_map); env->DeleteLocalRef(output_map); - RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); return JniConnector::fill_block(&block, {result}, output_address); } diff --git a/be/src/vec/functions/function_java_udf.h b/be/src/vec/functions/function_java_udf.h index 748b1fb221..425f05ccfd 100644 --- a/be/src/vec/functions/function_java_udf.h +++ b/be/src/vec/functions/function_java_udf.h @@ -147,9 +147,10 @@ private: return; } env->CallNonvirtualVoidMethodA(executor, executor_cl_, executor_close_id_, NULL); + env->DeleteGlobalRef(executor); + env->DeleteGlobalRef(executor_cl_); Status s = JniUtil::GetJniExceptionMsg(env); if (!s.ok()) LOG(WARNING) << s; - env->DeleteGlobalRef(executor); is_closed = true; } }; diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java index 5485a10dfd..5ebafb5708 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java @@ -38,7 +38,6 @@ import java.net.URL; import java.net.URLClassLoader; import java.security.AccessController; import java.security.PrivilegedAction; -import java.time.DateTimeException; import java.time.LocalDate; import java.time.LocalDateTime; import java.util.Set; @@ -150,6 +149,7 @@ public class UdfUtils { * Sets the argument types of a Java UDF or UDAF. Returns true if the argument types specified * in the UDF are compatible with the argument types of the evaluate() function loaded * from the associated JAR file. + * * @throws InternalException */ public static Pair setArgTypes(Type[] parameterTypes, Class[] udfArgTypes, @@ -193,191 +193,6 @@ public class UdfUtils { return Pair.of(true, inputArgTypes); } - public static Object convertDateTimeV2ToJavaDateTime(long date, Class clz) { - int year = (int) (date >> 46); - int yearMonth = (int) (date >> 42); - int yearMonthDay = (int) (date >> 37); - - int month = (yearMonth & 0XF); - int day = (yearMonthDay & 0X1F); - - int hour = (int) ((date >> 32) & 0X1F); - int minute = (int) ((date >> 26) & 0X3F); - int second = (int) ((date >> 20) & 0X3F); - //here don't need those bits are type = ((minus_type_neg >> 1) & 0x7); - - if (LocalDateTime.class.equals(clz)) { - return convertToLocalDateTime(year, month, day, hour, minute, second); - } else if (org.joda.time.DateTime.class.equals(clz)) { - return convertToJodaDateTime(year, month, day, hour, minute, second); - } else if (org.joda.time.LocalDateTime.class.equals(clz)) { - return convertToJodaLocalDateTime(year, month, day, hour, minute, second); - } else { - return null; - } - } - - /** - * input is a 64bit num from backend, and then get year, month, day, hour, minus, second by the order of bits. - */ - public static Object convertDateTimeToJavaDateTime(long date, Class clz) { - int year = (int) (date >> 48); - int yearMonth = (int) (date >> 40); - int yearMonthDay = (int) (date >> 32); - - int month = (yearMonth & 0XFF); - int day = (yearMonthDay & 0XFF); - - int hourMinuteSecond = (int) (date % (1 << 31)); - int minuteTypeNeg = (hourMinuteSecond % (1 << 16)); - - int hour = (hourMinuteSecond >> 24); - int minute = ((hourMinuteSecond >> 16) & 0XFF); - int second = (minuteTypeNeg >> 4); - //here don't need those bits are type = ((minus_type_neg >> 1) & 0x7); - - if (LocalDateTime.class.equals(clz)) { - return convertToLocalDateTime(year, month, day, hour, minute, second); - } else if (org.joda.time.DateTime.class.equals(clz)) { - return convertToJodaDateTime(year, month, day, hour, minute, second); - } else if (org.joda.time.LocalDateTime.class.equals(clz)) { - return convertToJodaLocalDateTime(year, month, day, hour, minute, second); - } else { - return null; - } - } - - public static Object convertDateV2ToJavaDate(int date, Class clz) { - int year = date >> 9; - int month = (date >> 5) & 0XF; - int day = date & 0X1F; - if (LocalDate.class.equals(clz)) { - return convertToLocalDate(year, month, day); - } else if (java.util.Date.class.equals(clz)) { - return convertToJavaDate(year, month, day); - } else if (org.joda.time.LocalDate.class.equals(clz)) { - return convertToJodaDate(year, month, day); - } else { - return null; - } - } - - public static LocalDateTime convertToLocalDateTime(int year, int month, int day, - int hour, int minute, int second) { - LocalDateTime value = null; - try { - value = LocalDateTime.of(year, month, day, hour, minute, second); - } catch (DateTimeException e) { - LOG.warn("Error occurs when parsing date time value: {}", e); - } - return value; - } - - public static org.joda.time.DateTime convertToJodaDateTime(int year, int month, int day, - int hour, int minute, int second) { - try { - return new org.joda.time.DateTime(year, month, day, hour, minute, second); - } catch (Exception e) { - return null; - } - } - - public static org.joda.time.LocalDateTime convertToJodaLocalDateTime(int year, int month, int day, - int hour, int minute, int second) { - try { - return new org.joda.time.LocalDateTime(year, month, day, hour, minute, second); - } catch (Exception e) { - return null; - } - } - - public static Object convertDateToJavaDate(long date, Class clz) { - int year = (int) (date >> 48); - int yearMonth = (int) (date >> 40); - int yearMonthDay = (int) (date >> 32); - - int month = (yearMonth & 0XFF); - int day = (yearMonthDay & 0XFF); - if (LocalDate.class.equals(clz)) { - return convertToLocalDate(year, month, day); - } else if (java.util.Date.class.equals(clz)) { - return convertToJavaDate(year, month, day); - } else if (org.joda.time.LocalDate.class.equals(clz)) { - return convertToJodaDate(year, month, day); - } else { - return null; - } - } - - /** - * a 64bit num convertToDate. - */ - public static LocalDate convertToLocalDate(int year, int month, int day) { - LocalDate value = null; - try { - value = LocalDate.of(year, month, day); - } catch (DateTimeException e) { - LOG.warn("Error occurs when parsing date value: {}", e); - } - return value; - } - - public static org.joda.time.LocalDate convertToJodaDate(int year, int month, int day) { - try { - return new org.joda.time.LocalDate(year, month, day); - } catch (Exception e) { - return null; - } - } - - public static java.util.Date convertToJavaDate(int year, int month, int day) { - try { - return new java.util.Date(year - 1900, month - 1, day); - } catch (Exception e) { - return null; - } - } - - /** - * input is the second, minute, hours, day , month and year respectively. - * and then combining all num to a 64bit value return to backend; - */ - public static long convertToDateTime(Object obj, Class clz) { - if (LocalDateTime.class.equals(clz)) { - LocalDateTime date = (LocalDateTime) obj; - return convertToDateTime(date.getYear(), date.getMonthValue(), date.getDayOfMonth(), date.getHour(), - date.getMinute(), date.getSecond(), false); - } else if (org.joda.time.DateTime.class.equals(clz)) { - org.joda.time.DateTime date = (org.joda.time.DateTime) obj; - return convertToDateTime(date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(), - date.getMinuteOfHour(), date.getSecondOfMinute(), false); - } else if (org.joda.time.LocalDateTime.class.equals(clz)) { - org.joda.time.LocalDateTime date = (org.joda.time.LocalDateTime) obj; - return convertToDateTime(date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(), - date.getMinuteOfHour(), date.getSecondOfMinute(), false); - } else { - return 0; - } - } - - public static long convertToDate(Object obj, Class clz) { - if (LocalDate.class.equals(clz)) { - LocalDate date = (LocalDate) obj; - return convertToDateTime(date.getYear(), date.getMonthValue(), date.getDayOfMonth(), 0, - 0, 0, true); - } else if (java.util.Date.class.equals(clz)) { - java.util.Date date = (java.util.Date) obj; - return convertToDateTime(date.getYear() + 1900, date.getMonth() + 1, date.getDay(), 0, - 0, 0, true); - } else if (org.joda.time.LocalDate.class.equals(clz)) { - org.joda.time.LocalDate date = (org.joda.time.LocalDate) obj; - return convertToDateTime(date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), 0, - 0, 0, true); - } else { - return 0; - } - } - public static long convertToDateTime(int year, int month, int day, int hour, int minute, int second, boolean isDate) { long time = 0; @@ -394,54 +209,16 @@ public class UdfUtils { return time; } - public static long convertToDateTimeV2(int year, int month, int day, int hour, int minute, int second) { - return (long) second << 20 | (long) minute << 26 | (long) hour << 32 - | (long) day << 37 | (long) month << 42 | (long) year << 46; - } - public static long convertToDateTimeV2( int year, int month, int day, int hour, int minute, int second, int microsecond) { return (long) microsecond | (long) second << 20 | (long) minute << 26 | (long) hour << 32 | (long) day << 37 | (long) month << 42 | (long) year << 46; } - public static long convertToDateTimeV2(Object obj, Class clz) { - if (LocalDateTime.class.equals(clz)) { - LocalDateTime date = (LocalDateTime) obj; - return convertToDateTimeV2(date.getYear(), date.getMonthValue(), date.getDayOfMonth(), date.getHour(), - date.getMinute(), date.getSecond()); - } else if (org.joda.time.DateTime.class.equals(clz)) { - org.joda.time.DateTime date = (org.joda.time.DateTime) obj; - return convertToDateTimeV2(date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(), - date.getMinuteOfHour(), date.getSecondOfMinute(), date.getMillisOfSecond() * 1000); - } else if (org.joda.time.LocalDateTime.class.equals(clz)) { - org.joda.time.LocalDateTime date = (org.joda.time.LocalDateTime) obj; - return convertToDateTimeV2(date.getYear(), date.getMonthOfYear(), date.getDayOfMonth(), date.getHourOfDay(), - date.getMinuteOfHour(), date.getSecondOfMinute(), date.getMillisOfSecond() * 1000); - } else { - return 0; - } - } - public static int convertToDateV2(int year, int month, int day) { return (int) (day | (long) month << 5 | (long) year << 9); } - public static int convertToDateV2(Object obj, Class clz) { - if (LocalDate.class.equals(clz)) { - LocalDate date = (LocalDate) obj; - return convertToDateV2(date.getYear(), date.getMonthValue(), date.getDayOfMonth()); - } else if (java.util.Date.class.equals(clz)) { - java.util.Date date = (java.util.Date) obj; - return convertToDateV2(date.getYear(), date.getMonth(), date.getDay()); - } else if (org.joda.time.LocalDate.class.equals(clz)) { - org.joda.time.LocalDate date = (org.joda.time.LocalDate) obj; - return convertToDateV2(date.getYear(), date.getDayOfMonth(), date.getDayOfMonth()); - } else { - return 0; - } - } - /** * Change the order of the bytes, Because JVM is Big-Endian , x86 is Little-Endian. */ diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java index 3330fddbb1..76d21072c8 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java @@ -171,6 +171,11 @@ public class ColumnType { return type == Type.DATETIMEV2; } + public boolean isPrimitive() { + return type == Type.BOOLEAN || type == Type.BYTE || type == Type.TINYINT || type == Type.SMALLINT + || type == Type.INT || type == Type.BIGINT || type == Type.FLOAT || type == Type.DOUBLE; + } + public Type getType() { return type; } diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index ecbf8b12a8..b03cbd9cdc 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -1301,7 +1301,11 @@ public class VectorColumn { } } - private Object[] newObjectContainerArray(ColumnType.Type type, int size) { + public Object[] newObjectContainerArray(int size) { + return newObjectContainerArray(columnType.getType(), size); + } + + public Object[] newObjectContainerArray(ColumnType.Type type, int size) { switch (type) { case BOOLEAN: return new Boolean[size]; diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java index 8f6c6b8b64..a8849d1fa2 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java @@ -138,13 +138,13 @@ public class VectorTable { * @param converters A map of converters. Convert the column values if the type is not defined in ColumnType. * The map key is the field ID in VectorTable. */ - public Object[][] getMaterializedData(Map converters) { + public Object[][] getMaterializedData(int start, int end, Map converters) { if (columns.length == 0) { return new Object[0][0]; } Object[][] data = new Object[columns.length][]; for (int j = 0; j < columns.length; ++j) { - Object[] columnData = columns[j].getObjectColumn(0, columns[j].numRows()); + Object[] columnData = columns[j].getObjectColumn(start, end); if (converters.containsKey(j)) { data[j] = converters.get(j).convert(columnData); } else { @@ -154,6 +154,10 @@ public class VectorTable { return data; } + public Object[][] getMaterializedData(Map converters) { + return getMaterializedData(0, getNumRows(), converters); + } + public Object[][] getMaterializedData() { return getMaterializedData(Collections.emptyMap()); } @@ -166,6 +170,10 @@ public class VectorTable { return columns[fieldId]; } + public ColumnType getColumnType(int fieldId) { + return columnTypes[fieldId]; + } + public ColumnType[] getColumnTypes() { return columnTypes; } diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java index 70c6dcb937..1e996b720e 100644 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java @@ -17,37 +17,25 @@ package org.apache.doris.udf; -import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.exception.InternalException; import org.apache.doris.common.exception.UdfRuntimeException; -import org.apache.doris.common.jni.utils.JNINativeMethod; import org.apache.doris.common.jni.utils.JavaUdfDataType; -import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.vec.ColumnValueConverter; import org.apache.doris.thrift.TFunction; import org.apache.doris.thrift.TJavaUdfExecutorCtorParams; +import org.apache.doris.thrift.TPrimitiveType; import com.esotericsoftware.reflectasm.MethodAccess; -import com.google.common.base.Preconditions; import org.apache.log4j.Logger; import org.apache.thrift.TDeserializer; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import java.io.IOException; -import java.lang.reflect.Array; -import java.lang.reflect.Method; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; import java.net.URLClassLoader; -import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; public abstract class BaseExecutor { private static final Logger LOG = Logger.getLogger(BaseExecutor.class); @@ -104,32 +92,38 @@ public abstract class BaseExecutor { } public String debugString() { - String res = ""; + StringBuilder res = new StringBuilder(); for (JavaUdfDataType type : argTypes) { - res = res + type.toString(); + res.append(type.toString()); if (type.getItemType() != null) { - res = res + " item: " + type.getItemType().toString() + " sql: " + type.getItemType().toSql(); + res.append(" item: ").append(type.getItemType().toString()).append(" sql: ") + .append(type.getItemType().toSql()); } if (type.getKeyType() != null) { - res = res + " key: " + type.getKeyType().toString() + " sql: " + type.getKeyType().toSql(); + res.append(" key: ").append(type.getKeyType().toString()).append(" sql: ") + .append(type.getKeyType().toSql()); } if (type.getValueType() != null) { - res = res + " key: " + type.getValueType().toString() + " sql: " + type.getValueType().toSql(); + res.append(" key: ").append(type.getValueType().toString()).append(" sql: ") + .append(type.getValueType().toSql()); } } - res = res + " return type: " + retType.toString(); + res.append(" return type: ").append(retType.toString()); if (retType.getItemType() != null) { - res = res + " item: " + retType.getItemType().toString() + " sql: " + retType.getItemType().toSql(); + res.append(" item: ").append(retType.getItemType().toString()).append(" sql: ") + .append(retType.getItemType().toSql()); } if (retType.getKeyType() != null) { - res = res + " key: " + retType.getKeyType().toString() + " sql: " + retType.getKeyType().toSql(); + res.append(" key: ").append(retType.getKeyType().toString()).append(" sql: ") + .append(retType.getKeyType().toSql()); } if (retType.getValueType() != null) { - res = res + " key: " + retType.getValueType().toString() + " sql: " + retType.getValueType().toSql(); + res.append(" key: ").append(retType.getValueType().toString()).append(" sql: ") + .append(retType.getValueType().toSql()); } - res = res + " methodAccess: " + methodAccess.toString(); - res = res + " fn.toString(): " + fn.toString(); - return res; + res.append(" methodAccess: ").append(methodAccess.toString()); + res.append(" fn.toString(): ").append(fn.toString()); + return res.toString(); } protected abstract void init(TJavaUdfExecutorCtorParams request, String jarPath, @@ -152,940 +146,145 @@ public abstract class BaseExecutor { classLoader = null; } - public void copyTupleBasicResult(Object obj, long row, Class retClass, - long outputBufferBase, long charsAddress, long offsetsAddr, JavaUdfDataType retType) - throws UdfRuntimeException { - switch (retType.getPrimitiveType()) { - case BOOLEAN: { - boolean val = (boolean) obj; - UdfUtils.UNSAFE.putByte(outputBufferBase + row * retType.getLen(), - val ? (byte) 1 : 0); - break; - } - case TINYINT: { - UdfUtils.UNSAFE.putByte(outputBufferBase + row * retType.getLen(), - (byte) obj); - break; - } - case SMALLINT: { - UdfUtils.UNSAFE.putShort(outputBufferBase + row * retType.getLen(), - (short) obj); - break; - } - case INT: { - UdfUtils.UNSAFE.putInt(outputBufferBase + row * retType.getLen(), - (int) obj); - break; - } - case BIGINT: { - UdfUtils.UNSAFE.putLong(outputBufferBase + row * retType.getLen(), - (long) obj); - break; - } - case FLOAT: { - UdfUtils.UNSAFE.putFloat(outputBufferBase + row * retType.getLen(), - (float) obj); - break; - } - case DOUBLE: { - UdfUtils.UNSAFE.putDouble(outputBufferBase + row * retType.getLen(), - (double) obj); - break; - } - case DATE: { - long time = UdfUtils.convertToDate(obj, retClass); - UdfUtils.UNSAFE.putLong(outputBufferBase + row * retType.getLen(), time); - break; - } - case DATETIME: { - long time = UdfUtils.convertToDateTime(obj, retClass); - UdfUtils.UNSAFE.putLong(outputBufferBase + row * retType.getLen(), time); - break; - } + protected ColumnValueConverter getInputConverter(TPrimitiveType primitiveType, Class clz) { + switch (primitiveType) { + case DATE: case DATEV2: { - int time = UdfUtils.convertToDateV2(obj, retClass); - UdfUtils.UNSAFE.putInt(outputBufferBase + row * retType.getLen(), time); + if (java.util.Date.class.equals(clz)) { + return (Object[] columnData) -> { + Object[] result = new java.util.Date[columnData.length]; + for (int i = 0; i < columnData.length; ++i) { + if (columnData[i] != null) { + LocalDate v = (LocalDate) columnData[i]; + result[i] = new java.util.Date(v.getYear() - 1900, v.getMonthValue() - 1, + v.getDayOfMonth()); + } + } + return result; + }; + } else if (org.joda.time.LocalDate.class.equals(clz)) { + return (Object[] columnData) -> { + Object[] result = new org.joda.time.LocalDate[columnData.length]; + for (int i = 0; i < columnData.length; ++i) { + if (columnData[i] != null) { + LocalDate v = (LocalDate) columnData[i]; + result[i] = new org.joda.time.LocalDate(v.getYear(), v.getMonthValue(), + v.getDayOfMonth()); + } + } + return result; + }; + } else if (!LocalDate.class.equals(clz)) { + throw new RuntimeException("Unsupported date type: " + clz.getCanonicalName()); + } break; } - case DATETIMEV2: { - long time = UdfUtils.convertToDateTimeV2(obj, retClass); - UdfUtils.UNSAFE.putLong(outputBufferBase + row * retType.getLen(), time); - break; - } - case LARGEINT: { - BigInteger data = (BigInteger) obj; - byte[] bytes = UdfUtils.convertByteOrder(data.toByteArray()); - - // here value is 16 bytes, so if result data greater than the maximum of 16 - // bytesit will return a wrong num to backend; - byte[] value = new byte[16]; - // check data is negative - if (data.signum() == -1) { - Arrays.fill(value, (byte) -1); - } - for (int index = 0; index < Math.min(bytes.length, value.length); ++index) { - value[index] = bytes[index]; - } - - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, - outputBufferBase + row * retType.getLen(), value.length); - break; - } - case DECIMALV2: { - BigDecimal retValue = ((BigDecimal) obj).setScale(9, RoundingMode.HALF_EVEN); - BigInteger data = retValue.unscaledValue(); - byte[] bytes = UdfUtils.convertByteOrder(data.toByteArray()); - // TODO: here is maybe overflow also, and may find a better way to handle - byte[] value = new byte[16]; - if (data.signum() == -1) { - Arrays.fill(value, (byte) -1); - } - - for (int index = 0; index < Math.min(bytes.length, value.length); ++index) { - value[index] = bytes[index]; - } - - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, - outputBufferBase + row * retType.getLen(), value.length); - break; - } - case DECIMAL32: - case DECIMAL64: - case DECIMAL128I: { - BigDecimal retValue = ((BigDecimal) obj).setScale(retType.getScale(), RoundingMode.HALF_EVEN); - BigInteger data = retValue.unscaledValue(); - byte[] bytes = UdfUtils.convertByteOrder(data.toByteArray()); - // TODO: here is maybe overflow also, and may find a better way to handle - byte[] value = new byte[retType.getLen()]; - if (data.signum() == -1) { - Arrays.fill(value, (byte) -1); - } - - for (int index = 0; index < Math.min(bytes.length, value.length); ++index) { - value[index] = bytes[index]; - } - - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, - outputBufferBase + row * retType.getLen(), value.length); - break; - } - case CHAR: - case VARCHAR: - case STRING: { - byte[] bytes = ((String) obj).getBytes(StandardCharsets.UTF_8); - long offset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * (row - 1)); - int needLen = (int) (offset + bytes.length); - outputBufferBase = JNINativeMethod.resizeStringColumn(charsAddress, needLen); - offset += bytes.length; - UdfUtils.UNSAFE.putInt(null, offsetsAddr + 4L * row, Integer.parseUnsignedInt(String.valueOf(offset))); - UdfUtils.copyMemory(bytes, UdfUtils.BYTE_ARRAY_OFFSET, null, outputBufferBase + offset - bytes.length, - bytes.length); - break; - } - case ARRAY: - default: - throw new UdfRuntimeException("Unsupported return type: " + retType); - } - } - - public Object[] convertBasicArg(boolean isUdf, int argIdx, boolean isNullable, int rowStart, int rowEnd, - long nullMapAddr, long columnAddr, long strOffsetAddr) { - switch (argTypes[argIdx].getPrimitiveType()) { - case BOOLEAN: - return UdfConvert.convertBooleanArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr); - case TINYINT: - return UdfConvert.convertTinyIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr); - case SMALLINT: - return UdfConvert.convertSmallIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr); - case INT: - return UdfConvert.convertIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr); - case BIGINT: - return UdfConvert.convertBigIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr); - case LARGEINT: - return UdfConvert.convertLargeIntArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr); - case FLOAT: - return UdfConvert.convertFloatArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr); - case DOUBLE: - return UdfConvert.convertDoubleArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr); - case CHAR: - case VARCHAR: - case STRING: - return UdfConvert - .convertStringArg(isNullable, rowStart, rowEnd, nullMapAddr, columnAddr, strOffsetAddr); - case DATE: // udaf maybe argClass[i + argClassOffset] need add +1 - return UdfConvert - .convertDateArg(isUdf ? argClass[argIdx] : argClass[argIdx + 1], isNullable, rowStart, rowEnd, - nullMapAddr, columnAddr); case DATETIME: - return UdfConvert - .convertDateTimeArg(isUdf ? argClass[argIdx] : argClass[argIdx + 1], isNullable, rowStart, - rowEnd, nullMapAddr, columnAddr); - case DATEV2: - return UdfConvert - .convertDateV2Arg(isUdf ? argClass[argIdx] : argClass[argIdx + 1], isNullable, rowStart, rowEnd, - nullMapAddr, columnAddr); - case DATETIMEV2: - return UdfConvert - .convertDateTimeV2Arg(isUdf ? argClass[argIdx] : argClass[argIdx + 1], isNullable, rowStart, - rowEnd, nullMapAddr, columnAddr); - case DECIMALV2: - case DECIMAL128I: - return UdfConvert - .convertDecimalArg(argTypes[argIdx].getScale(), 16L, isNullable, rowStart, rowEnd, nullMapAddr, - columnAddr); - case DECIMAL32: - return UdfConvert - .convertDecimalArg(argTypes[argIdx].getScale(), 4L, isNullable, rowStart, rowEnd, nullMapAddr, - columnAddr); - case DECIMAL64: - return UdfConvert - .convertDecimalArg(argTypes[argIdx].getScale(), 8L, isNullable, rowStart, rowEnd, nullMapAddr, - columnAddr); - default: { - LOG.info("Not support type: " + argTypes[argIdx].toString()); - Preconditions.checkState(false, "Not support type: " + argTypes[argIdx].toString()); + case DATETIMEV2: { + if (org.joda.time.DateTime.class.equals(clz)) { + return (Object[] columnData) -> { + Object[] result = new org.joda.time.DateTime[columnData.length]; + for (int i = 0; i < columnData.length; ++i) { + if (columnData[i] != null) { + LocalDateTime v = (LocalDateTime) columnData[i]; + result[i] = new org.joda.time.DateTime(v.getYear(), v.getMonthValue(), + v.getDayOfMonth(), v.getHour(), + v.getMinute(), v.getSecond(), v.getNano() / 1000000); + } + } + return result; + }; + } else if (org.joda.time.LocalDateTime.class.equals(clz)) { + return (Object[] columnData) -> { + Object[] result = new org.joda.time.LocalDateTime[columnData.length]; + for (int i = 0; i < columnData.length; ++i) { + if (columnData[i] != null) { + LocalDateTime v = (LocalDateTime) columnData[i]; + result[i] = new org.joda.time.LocalDateTime(v.getYear(), v.getMonthValue(), + v.getDayOfMonth(), v.getHour(), + v.getMinute(), v.getSecond(), v.getNano() / 1000000); + } + } + return result; + }; + } else if (!LocalDateTime.class.equals(clz)) { + throw new RuntimeException("Unsupported date type: " + clz.getCanonicalName()); + } break; } + default: + break; } return null; } - public Object[] convertArrayArg(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr, - long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) { - Object[] argument = (Object[]) Array.newInstance(ArrayList.class, rowEnd - rowStart); - for (int row = rowStart; row < rowEnd; ++row) { - long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row - 1)); - long offsetEnd = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row)); - int currentRowNum = (int) (offsetEnd - offsetStart); - switch (argTypes[argIdx].getItemType().getPrimitiveType()) { - case BOOLEAN: { - argument[row - rowStart] = UdfConvert - .convertArrayBooleanArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; + protected ColumnValueConverter getOutputConverter(TPrimitiveType primitiveType, Class clz) { + switch (primitiveType) { + case DATE: + case DATEV2: { + if (java.util.Date.class.equals(clz)) { + return (Object[] columnData) -> { + Object[] result = new LocalDate[columnData.length]; + for (int i = 0; i < columnData.length; ++i) { + if (columnData[i] != null) { + java.util.Date v = (java.util.Date) columnData[i]; + result[i] = LocalDate.of(v.getYear() + 1900, v.getMonth() + 1, v.getDate()); + } + } + return result; + }; + } else if (org.joda.time.LocalDate.class.equals(clz)) { + return (Object[] columnData) -> { + Object[] result = new LocalDate[columnData.length]; + for (int i = 0; i < columnData.length; ++i) { + if (columnData[i] != null) { + org.joda.time.LocalDate v = (org.joda.time.LocalDate) columnData[i]; + result[i] = LocalDate.of(v.getYear(), v.getMonthOfYear(), v.getDayOfMonth()); + } + } + return result; + }; + } else if (!LocalDate.class.equals(clz)) { + throw new RuntimeException("Unsupported date type: " + clz.getCanonicalName()); } - case TINYINT: { - argument[row - rowStart] = UdfConvert - .convertArrayTinyIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case SMALLINT: { - argument[row - rowStart] = UdfConvert - .convertArraySmallIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case INT: { - argument[row - rowStart] = UdfConvert - .convertArrayIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case BIGINT: { - argument[row - rowStart] = UdfConvert - .convertArrayBigIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case LARGEINT: { - argument[row - rowStart] = UdfConvert - .convertArrayLargeIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case FLOAT: { - argument[row - rowStart] = UdfConvert - .convertArrayFloatArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DOUBLE: { - argument[row - rowStart] = UdfConvert - .convertArrayDoubleArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case CHAR: - case VARCHAR: - case STRING: { - argument[row - rowStart] = UdfConvert - .convertArrayStringArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr, strOffsetAddr); - break; - } - case DATE: { - argument[row - rowStart] = UdfConvert - .convertArrayDateArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DATETIME: { - argument[row - rowStart] = UdfConvert - .convertArrayDateTimeArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DATEV2: { - argument[row - rowStart] = UdfConvert - .convertArrayDateV2Arg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DATETIMEV2: { - argument[row - rowStart] = UdfConvert - .convertArrayDateTimeV2Arg(row, currentRowNum, offsetStart, isNullable, - nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMALV2: - case DECIMAL128: { - argument[row - rowStart] = UdfConvert - .convertArrayDecimalArg(argTypes[argIdx].getScale(), 16L, row, currentRowNum, - offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMAL32: { - argument[row - rowStart] = UdfConvert - .convertArrayDecimalArg(argTypes[argIdx].getScale(), 4L, row, currentRowNum, - offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMAL64: { - argument[row - rowStart] = UdfConvert - .convertArrayDecimalArg(argTypes[argIdx].getScale(), 8L, row, currentRowNum, - offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - default: { - LOG.info("Not support: " + argTypes[argIdx]); - Preconditions.checkState(false, "Not support type " + argTypes[argIdx].toString()); - break; - } - } - } - return argument; - } - - public Object[] convertMapArg(PrimitiveType type, int argIdx, boolean isNullable, int rowStart, int rowEnd, - long nullMapAddr, - long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr, int scale) { - Object[] argument = (Object[]) Array.newInstance(ArrayList.class, rowEnd - rowStart); - for (int row = rowStart; row < rowEnd; ++row) { - long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row - 1)); - long offsetEnd = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row)); - int currentRowNum = (int) (offsetEnd - offsetStart); - switch (type) { - case BOOLEAN: { - argument[row - - rowStart] = UdfConvert - .convertArrayBooleanArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case TINYINT: { - argument[row - rowStart] = UdfConvert - .convertArrayTinyIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case SMALLINT: { - argument[row - rowStart] = UdfConvert - .convertArraySmallIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case INT: { - argument[row - rowStart] = UdfConvert - .convertArrayIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case BIGINT: { - argument[row - rowStart] = UdfConvert - .convertArrayBigIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case LARGEINT: { - argument[row - rowStart] = UdfConvert - .convertArrayLargeIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case FLOAT: { - argument[row - rowStart] = UdfConvert - .convertArrayFloatArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DOUBLE: { - argument[row - rowStart] = UdfConvert - .convertArrayDoubleArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case CHAR: - case VARCHAR: - case STRING: { - argument[row - rowStart] = UdfConvert - .convertArrayStringArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr, strOffsetAddr); - break; - } - case DATE: { - argument[row - rowStart] = UdfConvert - .convertArrayDateArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DATETIME: { - argument[row - rowStart] = UdfConvert - .convertArrayDateTimeArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DATEV2: { - argument[row - rowStart] = UdfConvert - .convertArrayDateV2Arg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DATETIMEV2: { - argument[row - rowStart] = UdfConvert - .convertArrayDateTimeV2Arg(row, currentRowNum, offsetStart, isNullable, - nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMALV2: - case DECIMAL128: { - argument[row - rowStart] = UdfConvert - .convertArrayDecimalArg(scale, 16L, row, currentRowNum, - offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMAL32: { - argument[row - rowStart] = UdfConvert - .convertArrayDecimalArg(scale, 4L, row, currentRowNum, - offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMAL64: { - argument[row - rowStart] = UdfConvert - .convertArrayDecimalArg(scale, 8L, row, currentRowNum, - offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - default: { - LOG.info("Not support: " + argTypes[argIdx]); - Preconditions.checkState(false, "Not support type " + argTypes[argIdx].toString()); - break; - } - } - } - return argument; - } - - public Object[] buildHashMap(PrimitiveType keyType, PrimitiveType valueType, Object[] keyCol, Object[] valueCol) { - switch (keyType) { - case BOOLEAN: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case TINYINT: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case SMALLINT: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case INT: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case BIGINT: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case LARGEINT: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case FLOAT: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case DOUBLE: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case CHAR: - case VARCHAR: - case STRING: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case DATEV2: - case DATE: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case DATETIMEV2: - case DATETIME: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - case DECIMAL32: - case DECIMAL64: - case DECIMALV2: - case DECIMAL128: { - return new HashMapBuilder().get(keyCol, valueCol, valueType); - } - default: { - LOG.info("Not support: " + keyType); - Preconditions.checkState(false, "Not support type " + keyType.toString()); break; } + case DATETIME: + case DATETIMEV2: { + if (org.joda.time.DateTime.class.equals(clz)) { + return (Object[] columnData) -> { + Object[] result = new LocalDateTime[columnData.length]; + for (int i = 0; i < columnData.length; ++i) { + if (columnData[i] != null) { + org.joda.time.DateTime v = (org.joda.time.DateTime) columnData[i]; + result[i] = LocalDateTime.of(v.getYear(), v.getMonthOfYear(), v.getDayOfMonth(), + v.getHourOfDay(), + v.getMinuteOfHour(), v.getSecondOfMinute(), v.getMillisOfSecond() * 1000000); + } + } + return result; + }; + } else if (org.joda.time.LocalDateTime.class.equals(clz)) { + return (Object[] columnData) -> { + Object[] result = new LocalDateTime[columnData.length]; + for (int i = 0; i < columnData.length; ++i) { + if (columnData[i] != null) { + org.joda.time.LocalDateTime v = (org.joda.time.LocalDateTime) columnData[i]; + result[i] = LocalDateTime.of(v.getYear(), v.getMonthOfYear(), v.getDayOfMonth(), + v.getHourOfDay(), + v.getMinuteOfHour(), v.getSecondOfMinute(), v.getMillisOfSecond() * 1000000); + } + } + return result; + }; + } else if (!LocalDateTime.class.equals(clz)) { + throw new RuntimeException("Unsupported date type: " + clz.getCanonicalName()); + } + break; + } + default: + break; } return null; } - - public static class HashMapBuilder { - public Object[] get(Object[] keyCol, Object[] valueCol, PrimitiveType valueType) { - switch (valueType) { - case BOOLEAN: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case TINYINT: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case SMALLINT: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case INT: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case BIGINT: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case LARGEINT: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case FLOAT: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case DOUBLE: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case CHAR: - case VARCHAR: - case STRING: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case DATEV2: - case DATE: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case DATETIMEV2: - case DATETIME: { - return new BuildMapFromType().get(keyCol, valueCol); - } - case DECIMAL32: - case DECIMAL64: - case DECIMALV2: - case DECIMAL128: { - return new BuildMapFromType().get(keyCol, valueCol); - } - default: { - LOG.info("Not support: " + valueType); - Preconditions.checkState(false, "Not support type " + valueType.toString()); - break; - } - } - return null; - } - } - - public static class BuildMapFromType { - public Object[] get(Object[] keyCol, Object[] valueCol) { - Object[] retHashMap = new HashMap[keyCol.length]; - for (int colIdx = 0; colIdx < keyCol.length; colIdx++) { - HashMap hashMap = new HashMap<>(); - ArrayList keys = (ArrayList) (keyCol[colIdx]); - ArrayList values = (ArrayList) (valueCol[colIdx]); - for (int i = 0; i < keys.size(); i++) { - T1 key = keys.get(i); - T2 value = values.get(i); - if (!hashMap.containsKey(key)) { - hashMap.put(key, value); - } - } - retHashMap[colIdx] = hashMap; - } - return retHashMap; - } - } - - public void copyBatchBasicResultImpl(boolean isNullable, int numRows, Object[] result, long nullMapAddr, - long resColumnAddr, long strOffsetAddr, Method method) { - switch (retType.getPrimitiveType()) { - case BOOLEAN: { - UdfConvert.copyBatchBooleanResult(isNullable, numRows, (Boolean[]) result, nullMapAddr, resColumnAddr); - break; - } - case TINYINT: { - UdfConvert.copyBatchTinyIntResult(isNullable, numRows, (Byte[]) result, nullMapAddr, resColumnAddr); - break; - } - case SMALLINT: { - UdfConvert.copyBatchSmallIntResult(isNullable, numRows, (Short[]) result, nullMapAddr, resColumnAddr); - break; - } - case INT: { - UdfConvert.copyBatchIntResult(isNullable, numRows, (Integer[]) result, nullMapAddr, resColumnAddr); - break; - } - case BIGINT: { - UdfConvert.copyBatchBigIntResult(isNullable, numRows, (Long[]) result, nullMapAddr, resColumnAddr); - break; - } - case LARGEINT: { - UdfConvert.copyBatchLargeIntResult(isNullable, numRows, (BigInteger[]) result, nullMapAddr, - resColumnAddr); - break; - } - case FLOAT: { - UdfConvert.copyBatchFloatResult(isNullable, numRows, (Float[]) result, nullMapAddr, resColumnAddr); - break; - } - case DOUBLE: { - UdfConvert.copyBatchDoubleResult(isNullable, numRows, (Double[]) result, nullMapAddr, resColumnAddr); - break; - } - case CHAR: - case VARCHAR: - case STRING: { - UdfConvert.copyBatchStringResult(isNullable, numRows, (String[]) result, nullMapAddr, resColumnAddr, - strOffsetAddr); - break; - } - case DATE: { - UdfConvert.copyBatchDateResult(method.getReturnType(), isNullable, numRows, result, - nullMapAddr, resColumnAddr); - break; - } - case DATETIME: { - UdfConvert - .copyBatchDateTimeResult(method.getReturnType(), isNullable, numRows, result, - nullMapAddr, - resColumnAddr); - break; - } - case DATEV2: { - UdfConvert.copyBatchDateV2Result(method.getReturnType(), isNullable, numRows, result, - nullMapAddr, - resColumnAddr); - break; - } - case DATETIMEV2: { - UdfConvert.copyBatchDateTimeV2Result(method.getReturnType(), isNullable, numRows, - result, nullMapAddr, - resColumnAddr); - break; - } - case DECIMALV2: - case DECIMAL128I: { - UdfConvert.copyBatchDecimal128Result(retType.getScale(), isNullable, numRows, (BigDecimal[]) result, - nullMapAddr, - resColumnAddr); - break; - } - case DECIMAL32: { - UdfConvert.copyBatchDecimal32Result(retType.getScale(), isNullable, numRows, (BigDecimal[]) result, - nullMapAddr, - resColumnAddr); - break; - } - case DECIMAL64: { - UdfConvert.copyBatchDecimal64Result(retType.getScale(), isNullable, numRows, (BigDecimal[]) result, - nullMapAddr, - resColumnAddr); - break; - } - default: { - LOG.info("Not support return type: " + retType); - Preconditions.checkState(false, "Not support type: " + retType.toString()); - break; - } - } - } - - public void copyBatchArrayResultImpl(boolean isNullable, int numRows, Object[] result, long nullMapAddr, - long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr, - PrimitiveType type, int scale) { - long hasPutElementNum = 0; - for (int row = 0; row < numRows; ++row) { - hasPutElementNum = copyTupleArrayResultImpl(hasPutElementNum, isNullable, row, result[row], nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr, strOffsetAddr, type, scale); - } - } - - public long copyTupleArrayResultImpl(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, - long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr, - PrimitiveType type, int scale) { - switch (type) { - case BOOLEAN: { - hasPutElementNum = UdfConvert - .copyBatchArrayBooleanResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case TINYINT: { - hasPutElementNum = UdfConvert - .copyBatchArrayTinyIntResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case SMALLINT: { - hasPutElementNum = UdfConvert - .copyBatchArraySmallIntResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case INT: { - hasPutElementNum = UdfConvert - .copyBatchArrayIntResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case BIGINT: { - hasPutElementNum = UdfConvert - .copyBatchArrayBigIntResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case LARGEINT: { - hasPutElementNum = UdfConvert - .copyBatchArrayLargeIntResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case FLOAT: { - hasPutElementNum = UdfConvert - .copyBatchArrayFloatResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case DOUBLE: { - hasPutElementNum = UdfConvert - .copyBatchArrayDoubleResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case CHAR: - case VARCHAR: - case STRING: { - hasPutElementNum = UdfConvert - .copyBatchArrayStringResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr, strOffsetAddr); - break; - } - case DATE: { - hasPutElementNum = UdfConvert - .copyBatchArrayDateResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case DATETIME: { - hasPutElementNum = UdfConvert - .copyBatchArrayDateTimeResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case DATEV2: { - hasPutElementNum = UdfConvert - .copyBatchArrayDateV2Result(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case DATETIMEV2: { - hasPutElementNum = UdfConvert - .copyBatchArrayDateTimeV2Result(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMALV2: { - hasPutElementNum = UdfConvert - .copyBatchArrayDecimalResult(hasPutElementNum, isNullable, row, result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMAL32: { - hasPutElementNum = UdfConvert - .copyBatchArrayDecimalV3Result(scale, 4L, hasPutElementNum, isNullable, row, - result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMAL64: { - hasPutElementNum = UdfConvert - .copyBatchArrayDecimalV3Result(scale, 8L, hasPutElementNum, isNullable, row, - result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMAL128: { - hasPutElementNum = UdfConvert - .copyBatchArrayDecimalV3Result(scale, 16L, hasPutElementNum, isNullable, row, - result, nullMapAddr, - offsetsAddr, nestedNullMapAddr, dataAddr); - break; - } - default: { - Preconditions.checkState(false, "Not support type in array: " + retType); - break; - } - } - return hasPutElementNum; - } - - public void buildArrayListFromHashMap(Object[] result, PrimitiveType keyType, PrimitiveType valueType, - Object[] keyCol, Object[] valueCol) { - switch (keyType) { - case BOOLEAN: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case TINYINT: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case SMALLINT: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case INT: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case BIGINT: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case LARGEINT: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case FLOAT: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case DOUBLE: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case CHAR: - case VARCHAR: - case STRING: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case DATEV2: - case DATE: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case DATETIMEV2: - case DATETIME: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - case DECIMAL32: - case DECIMAL64: - case DECIMALV2: - case DECIMAL128: { - new ArrayListBuilder().get(result, keyCol, valueCol, valueType); - break; - } - default: { - LOG.info("Not support: " + keyType); - Preconditions.checkState(false, "Not support type " + keyType.toString()); - break; - } - } - } - - public static class ArrayListBuilder { - public void get(Object[] map, Object[] keyCol, Object[] valueCol, PrimitiveType valueType) { - switch (valueType) { - case BOOLEAN: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case TINYINT: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case SMALLINT: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case INT: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case BIGINT: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case LARGEINT: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case FLOAT: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case DOUBLE: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case CHAR: - case VARCHAR: - case STRING: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case DATEV2: - case DATE: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case DATETIMEV2: - case DATETIME: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - case DECIMAL32: - case DECIMAL64: - case DECIMALV2: - case DECIMAL128: { - new BuildArrayFromType().get(map, keyCol, valueCol); - break; - } - default: { - LOG.info("Not support: " + valueType); - Preconditions.checkState(false, "Not support type " + valueType.toString()); - break; - } - } - } - } - - public static class BuildArrayFromType { - public void get(Object[] map, Object[] keyCol, Object[] valueCol) { - for (int colIdx = 0; colIdx < map.length; colIdx++) { - HashMap hashMap = (HashMap) map[colIdx]; - ArrayList keys = new ArrayList<>(); - ArrayList values = new ArrayList<>(); - for (Map.Entry entry : hashMap.entrySet()) { - keys.add(entry.getKey()); - values.add(entry.getValue()); - } - keyCol[colIdx] = keys; - valueCol[colIdx] = values; - } - } - } } diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java index 1679125b6e..caae5ef820 100644 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java @@ -17,12 +17,14 @@ package org.apache.doris.udf; -import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.Pair; import org.apache.doris.common.exception.UdfRuntimeException; import org.apache.doris.common.jni.utils.JavaUdfDataType; +import org.apache.doris.common.jni.utils.OffHeap; import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.vec.ColumnValueConverter; +import org.apache.doris.common.jni.vec.VectorTable; import org.apache.doris.thrift.TJavaUdfExecutorCtorParams; import com.esotericsoftware.reflectasm.MethodAccess; @@ -34,12 +36,13 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.net.MalformedURLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; +import java.util.Map; /** * udaf executor. @@ -52,6 +55,7 @@ public class UdafExecutor extends BaseExecutor { private HashMap stateObjMap; private Class retClass; private int addIndex; + private VectorTable outputTable = null; /** * Constructor to create an object. @@ -65,102 +69,90 @@ public class UdafExecutor extends BaseExecutor { */ @Override public void close() { - allMethods = null; + if (outputTable != null) { + outputTable.close(); + } super.close(); } - public Object[] convertBasicArguments(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr, - long columnAddr, long strOffsetAddr) { - return convertBasicArg(false, argIdx, isNullable, rowStart, rowEnd, nullMapAddr, columnAddr, strOffsetAddr); + private Map getInputConverters(int numColumns) { + Map converters = new HashMap<>(); + for (int j = 0; j < numColumns; ++j) { + ColumnValueConverter converter = getInputConverter(argTypes[j].getPrimitiveType(), argClass[j + 1]); + if (converter != null) { + converters.put(j, converter); + } + } + return converters; } - public Object[] convertArrayArguments(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr, - long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) { - return convertArrayArg(argIdx, isNullable, rowStart, rowEnd, nullMapAddr, offsetsAddr, nestedNullMapAddr, - dataAddr, strOffsetAddr); + private ColumnValueConverter getOutputConverter() { + return getOutputConverter(retType.getPrimitiveType(), retClass); } - public Object[] convertMapArguments(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr, - long offsetsAddr, long keyNestedNullMapAddr, long keyDataAddr, long keyStrOffsetAddr, - long valueNestedNullMapAddr, long valueDataAddr, long valueStrOffsetAddr) { - PrimitiveType keyType = argTypes[argIdx].getKeyType().getPrimitiveType(); - PrimitiveType valueType = argTypes[argIdx].getValueType().getPrimitiveType(); - Object[] keyCol = convertMapArg(keyType, argIdx, isNullable, rowStart, rowEnd, nullMapAddr, offsetsAddr, - keyNestedNullMapAddr, keyDataAddr, - keyStrOffsetAddr, argTypes[argIdx].getKeyScale()); - Object[] valueCol = convertMapArg(valueType, argIdx, isNullable, rowStart, rowEnd, nullMapAddr, offsetsAddr, - valueNestedNullMapAddr, - valueDataAddr, - valueStrOffsetAddr, argTypes[argIdx].getValueScale()); - return buildHashMap(keyType, valueType, keyCol, valueCol); - } - - public void addBatch(boolean isSinglePlace, int rowStart, int rowEnd, long placeAddr, int offset, Object[] column) - throws UdfRuntimeException { - if (isSinglePlace) { - addBatchSingle(rowStart, rowEnd, placeAddr, column); - } else { - addBatchPlaces(rowStart, rowEnd, placeAddr, offset, column); + public void addBatch(boolean isSinglePlace, int rowStart, int rowEnd, long placeAddr, int offset, + Map inputParams) throws UdfRuntimeException { + try { + VectorTable inputTable = VectorTable.createReadableTable(inputParams); + Object[][] inputs = inputTable.getMaterializedData(rowStart, rowEnd, + getInputConverters(inputTable.getNumColumns())); + if (isSinglePlace) { + addBatchSingle(rowStart, rowEnd, placeAddr, inputs); + } else { + addBatchPlaces(rowStart, rowEnd, placeAddr, offset, inputs); + } + } catch (Exception e) { + LOG.warn("evaluate exception: " + debugString(), e); + throw new UdfRuntimeException("UDAF failed to evaluate", e); } } - public void addBatchSingle(int rowStart, int rowEnd, long placeAddr, Object[] column) throws UdfRuntimeException { - try { - Long curPlace = placeAddr; - Object[] inputArgs = new Object[argTypes.length + 1]; + public void addBatchSingle(int rowStart, int rowEnd, long placeAddr, Object[][] inputs) throws UdfRuntimeException { + Long curPlace = placeAddr; + Object[] inputArgs = new Object[argTypes.length + 1]; + Object state = stateObjMap.get(curPlace); + if (state != null) { + inputArgs[0] = state; + } else { + Object newState = createAggState(); + stateObjMap.put(curPlace, newState); + inputArgs[0] = newState; + } + int numColumns = inputs.length; + int numRows = rowEnd - rowStart; + for (int i = 0; i < numRows; ++i) { + for (int j = 0; j < numColumns; ++j) { + inputArgs[j + 1] = inputs[j][i]; + } + methodAccess.invoke(udf, addIndex, inputArgs); + } + } + + public void addBatchPlaces(int rowStart, int rowEnd, long placeAddr, int offset, Object[][] inputs) + throws UdfRuntimeException { + int numColumns = inputs.length; + int numRows = rowEnd - rowStart; + Object[] placeState = new Object[numRows]; + for (int row = rowStart; row < rowEnd; ++row) { + Long curPlace = OffHeap.UNSAFE.getLong(null, placeAddr + (8L * row)) + offset; Object state = stateObjMap.get(curPlace); if (state != null) { - inputArgs[0] = state; + placeState[row - rowStart] = state; } else { Object newState = createAggState(); stateObjMap.put(curPlace, newState); - inputArgs[0] = newState; + placeState[row - rowStart] = newState; } - - Object[][] inputs = (Object[][]) column; - for (int i = 0; i < (rowEnd - rowStart); ++i) { - for (int j = 0; j < column.length; ++j) { - inputArgs[j + 1] = inputs[j][i]; - } - methodAccess.invoke(udf, addIndex, inputArgs); - } - } catch (Exception e) { - LOG.info("evaluate exception debug: " + debugString()); - LOG.info("invoke add function meet some error: " + e.getCause().toString()); - throw new UdfRuntimeException("UDAF failed to addBatchSingle: ", e); } - } + //spilt into two for loop - public void addBatchPlaces(int rowStart, int rowEnd, long placeAddr, int offset, Object[] column) - throws UdfRuntimeException { - try { - Object[][] inputs = (Object[][]) column; - ArrayList placeState = new ArrayList<>(rowEnd - rowStart); - for (int row = rowStart; row < rowEnd; ++row) { - Long curPlace = UdfUtils.UNSAFE.getLong(null, placeAddr + (8L * row)) + offset; - Object state = stateObjMap.get(curPlace); - if (state != null) { - placeState.add(state); - } else { - Object newState = createAggState(); - stateObjMap.put(curPlace, newState); - placeState.add(newState); - } + Object[] inputArgs = new Object[argTypes.length + 1]; + for (int row = 0; row < numRows; ++row) { + inputArgs[0] = placeState[row]; + for (int j = 0; j < numColumns; ++j) { + inputArgs[j + 1] = inputs[j][row]; } - //spilt into two for loop - - Object[] inputArgs = new Object[argTypes.length + 1]; - for (int row = 0; row < (rowEnd - rowStart); ++row) { - inputArgs[0] = placeState.get(row); - for (int j = 0; j < column.length; ++j) { - inputArgs[j + 1] = inputs[j][row]; - } - methodAccess.invoke(udf, addIndex, inputArgs); - } - } catch (Exception e) { - LOG.info("evaluate exception debug: " + debugString()); - LOG.info("invoke add function meet some error: " + Arrays.toString(e.getStackTrace())); - throw new UdfRuntimeException("UDAF failed to addBatchPlaces: ", e); + methodAccess.invoke(udf, addIndex, inputArgs); } } @@ -171,7 +163,7 @@ public class UdafExecutor extends BaseExecutor { try { return allMethods.get(UDAF_CREATE_FUNCTION).invoke(udf, null); } catch (Exception e) { - LOG.warn("invoke createAggState function meet some error: " + e.getCause().toString()); + LOG.warn("invoke createAggState function meet some error: ", e); throw new UdfRuntimeException("UDAF failed to create: ", e); } } @@ -186,7 +178,7 @@ public class UdafExecutor extends BaseExecutor { } stateObjMap.clear(); } catch (Exception e) { - LOG.warn("invoke destroy function meet some error: " + e.getCause().toString()); + LOG.warn("invoke destroy function meet some error: ", e); throw new UdfRuntimeException("UDAF failed to destroy: ", e); } } @@ -198,31 +190,31 @@ public class UdafExecutor extends BaseExecutor { try { Object[] args = new Object[2]; ByteArrayOutputStream baos = new ByteArrayOutputStream(); - args[0] = stateObjMap.get((Long) place); + args[0] = stateObjMap.get(place); args[1] = new DataOutputStream(baos); allMethods.get(UDAF_SERIALIZE_FUNCTION).invoke(udf, args); return baos.toByteArray(); } catch (Exception e) { LOG.info("evaluate exception debug: " + debugString()); - LOG.warn("invoke serialize function meet some error: " + e.getCause().toString()); + LOG.warn("invoke serialize function meet some error: ", e); throw new UdfRuntimeException("UDAF failed to serialize: ", e); } } - /* + /** * invoke reset function and reset the state to init. */ public void reset(long place) throws UdfRuntimeException { try { Object[] args = new Object[1]; - args[0] = stateObjMap.get((Long) place); + args[0] = stateObjMap.get(place); if (args[0] == null) { return; } allMethods.get(UDAF_RESET_FUNCTION).invoke(udf, args); } catch (Exception e) { LOG.info("evaluate exception debug: " + debugString()); - LOG.warn("invoke reset function meet some error: " + e.getCause().toString()); + LOG.warn("invoke reset function meet some error: ", e); throw new UdfRuntimeException("UDAF failed to reset: ", e); } } @@ -251,7 +243,7 @@ public class UdafExecutor extends BaseExecutor { allMethods.get(UDAF_MERGE_FUNCTION).invoke(udf, args); } catch (Exception e) { LOG.info("evaluate exception debug: " + debugString()); - LOG.warn("invoke merge function meet some error: " + e.getCause().toString()); + LOG.warn("invoke merge function meet some error: ", e); throw new UdfRuntimeException("UDAF failed to merge: ", e); } } @@ -259,75 +251,32 @@ public class UdafExecutor extends BaseExecutor { /** * invoke getValue to return finally result. */ - - public Object getValue(long place) throws UdfRuntimeException { + public long getValue(long place, Map outputParams) throws UdfRuntimeException { try { + if (outputTable != null) { + outputTable.close(); + } + outputTable = VectorTable.createWritableTable(outputParams, 1); if (stateObjMap.get(place) == null) { stateObjMap.put(place, createAggState()); } - return allMethods.get(UDAF_RESULT_FUNCTION).invoke(udf, stateObjMap.get((Long) place)); + Object value = allMethods.get(UDAF_RESULT_FUNCTION).invoke(udf, stateObjMap.get(place)); + // If the return type is primitive, we can't cast the array of primitive type as array of Object, + // so we have to new its wrapped Object. + Object[] result = outputTable.getColumnType(0).isPrimitive() + ? outputTable.getColumn(0).newObjectContainerArray(1) + : (Object[]) Array.newInstance(retClass, 1); + result[0] = value; + boolean isNullable = Boolean.parseBoolean(outputParams.getOrDefault("is_nullable", "true")); + outputTable.appendData(0, result, getOutputConverter(), isNullable); + return outputTable.getMetaAddress(); } catch (Exception e) { LOG.info("evaluate exception debug: " + debugString()); - LOG.warn("invoke getValue function meet some error: " + e.getCause().toString()); + LOG.warn("invoke getValue function meet some error: ", e); throw new UdfRuntimeException("UDAF failed to result", e); } } - public void copyTupleBasicResult(Object result, int row, long outputNullMapPtr, long outputBufferBase, - long charsAddress, - long offsetsAddr) throws UdfRuntimeException { - if (result == null) { - // put null obj - if (outputNullMapPtr == -1) { - throw new UdfRuntimeException("UDAF failed to store null data to not null column"); - } else { - UdfUtils.UNSAFE.putByte(outputNullMapPtr + row, (byte) 1); - } - return; - } - try { - if (outputNullMapPtr != -1) { - UdfUtils.UNSAFE.putByte(outputNullMapPtr + row, (byte) 0); - } - copyTupleBasicResult(result, row, retClass, outputBufferBase, charsAddress, - offsetsAddr, retType); - } catch (UdfRuntimeException e) { - LOG.info(e.toString()); - } - } - - public void copyTupleArrayResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, - long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) throws UdfRuntimeException { - if (nullMapAddr > 0) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 0); - } - copyTupleArrayResultImpl(hasPutElementNum, isNullable, row, result, nullMapAddr, offsetsAddr, nestedNullMapAddr, - dataAddr, strOffsetAddr, retType.getItemType().getPrimitiveType(), retType.getScale()); - } - - public void copyTupleMapResult(long hasPutElementNum, boolean isNullable, int row, Object result, long nullMapAddr, - long offsetsAddr, - long keyNsestedNullMapAddr, long keyDataAddr, - long keyStrOffsetAddr, - long valueNsestedNullMapAddr, long valueDataAddr, long valueStrOffsetAddr) throws UdfRuntimeException { - if (nullMapAddr > 0) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 0); - } - PrimitiveType keyType = retType.getKeyType().getPrimitiveType(); - PrimitiveType valueType = retType.getValueType().getPrimitiveType(); - Object[] keyCol = new Object[1]; - Object[] valueCol = new Object[1]; - Object[] resultArr = new Object[1]; - resultArr[0] = result; - buildArrayListFromHashMap(resultArr, keyType, valueType, keyCol, valueCol); - copyTupleArrayResultImpl(hasPutElementNum, isNullable, row, - valueCol[0], nullMapAddr, offsetsAddr, - valueNsestedNullMapAddr, valueDataAddr, valueStrOffsetAddr, valueType, retType.getKeyScale()); - copyTupleArrayResultImpl(hasPutElementNum, isNullable, row, keyCol[0], nullMapAddr, offsetsAddr, - keyNsestedNullMapAddr, keyDataAddr, keyStrOffsetAddr, keyType, retType.getValueScale()); - } - @Override protected void init(TJavaUdfExecutorCtorParams request, String jarPath, Type funcRetType, Type... parameterTypes) throws UdfRuntimeException { @@ -406,7 +355,9 @@ public class UdafExecutor extends BaseExecutor { return; } StringBuilder sb = new StringBuilder(); - sb.append("Unable to find evaluate function with the correct signature: ").append(className + ".evaluate(") + sb.append("Unable to find evaluate function with the correct signature: ") + .append(className) + .append(".evaluate(") .append(Joiner.on(", ").join(parameterTypes)).append(")\n").append("UDF contains: \n ") .append(Joiner.on("\n ").join(signatures)); throw new UdfRuntimeException(sb.toString()); diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfConvert.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfConvert.java deleted file mode 100644 index 7b3a151f00..0000000000 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfConvert.java +++ /dev/null @@ -1,1774 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.udf; - -import org.apache.doris.common.jni.utils.JNINativeMethod; -import org.apache.doris.common.jni.utils.OffHeap; -import org.apache.doris.common.jni.utils.UdfUtils; - -import com.google.common.base.Preconditions; -import org.apache.log4j.Logger; - -import java.lang.reflect.Array; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.nio.charset.StandardCharsets; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Arrays; - -public class UdfConvert { - private static final Logger LOG = Logger.getLogger(UdfConvert.class); - - public static Object[] convertBooleanArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr, - long columnAddr) { - Boolean[] argument = new Boolean[rowsEnd - rowsStart]; - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getBoolean(null, columnAddr + i); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getBoolean(null, columnAddr + i); - } - } - return argument; - } - - public static Object[] convertTinyIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr, - long columnAddr) { - Byte[] argument = new Byte[rowsEnd - rowsStart]; - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getByte(null, columnAddr + i); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getByte(null, columnAddr + i); - } - } - return argument; - } - - public static Object[] convertSmallIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr, - long columnAddr) { - Short[] argument = new Short[rowsEnd - rowsStart]; - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getShort(null, columnAddr + (i * 2L)); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getShort(null, columnAddr + (i * 2L)); - } - } - return argument; - } - - public static Object[] convertIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr, - long columnAddr) { - Integer[] argument = new Integer[rowsEnd - rowsStart]; - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L)); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L)); - } - } - return argument; - } - - public static Object[] convertBigIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr, - long columnAddr) { - Long[] argument = new Long[rowsEnd - rowsStart]; - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L)); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L)); - } - } - return argument; - } - - public static Object[] convertFloatArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr, - long columnAddr) { - Float[] argument = new Float[rowsEnd - rowsStart]; - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getFloat(null, columnAddr + (i * 4L)); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getFloat(null, columnAddr + (i * 4L)); - } - } - return argument; - } - - public static Object[] convertDoubleArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr, - long columnAddr) { - Double[] argument = new Double[rowsEnd - rowsStart]; - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getDouble(null, columnAddr + (i * 8L)); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - argument[i - rowsStart] = UdfUtils.UNSAFE.getDouble(null, columnAddr + (i * 8L)); - } - } - return argument; - } - - public static Object[] convertDateArg(Class argTypeClass, boolean isNullable, int rowsStart, int rowsEnd, - long nullMapAddr, long columnAddr) { - Object[] argument = (Object[]) Array.newInstance(argTypeClass, rowsEnd - rowsStart); - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L)); - argument[i - rowsStart] = UdfUtils.convertDateToJavaDate(value, argTypeClass); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L)); - argument[i - rowsStart] = UdfUtils.convertDateToJavaDate(value, argTypeClass); - } - } - return argument; - } - - public static Object[] convertDateTimeArg(Class argTypeClass, boolean isNullable, int rowsStart, int rowsEnd, - long nullMapAddr, long columnAddr) { - Object[] argument = (Object[]) Array.newInstance(argTypeClass, rowsEnd - rowsStart); - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L)); - argument[i - rowsStart] = UdfUtils - .convertDateTimeToJavaDateTime(value, argTypeClass); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L)); - argument[i - rowsStart] = UdfUtils.convertDateTimeToJavaDateTime(value, argTypeClass); - } - } - return argument; - } - - public static Object[] convertDateV2Arg(Class argTypeClass, boolean isNullable, int rowsStart, int rowsEnd, - long nullMapAddr, long columnAddr) { - Object[] argument = (Object[]) Array.newInstance(argTypeClass, rowsEnd - rowsStart); - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - int value = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L)); - argument[i - rowsStart] = UdfUtils.convertDateV2ToJavaDate(value, argTypeClass); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - int value = UdfUtils.UNSAFE.getInt(null, columnAddr + (i * 4L)); - argument[i - rowsStart] = UdfUtils.convertDateV2ToJavaDate(value, argTypeClass); - } - } - return argument; - } - - public static Object[] convertDateTimeV2Arg(Class argTypeClass, boolean isNullable, int rowsStart, int rowsEnd, - long nullMapAddr, long columnAddr) { - Object[] argument = (Object[]) Array.newInstance(argTypeClass, rowsEnd - rowsStart); - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(null, nullMapAddr + i) == 0) { - long value = UdfUtils.UNSAFE.getLong(columnAddr + (i * 8L)); - argument[i - rowsStart] = UdfUtils - .convertDateTimeV2ToJavaDateTime(value, argTypeClass); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - long value = UdfUtils.UNSAFE.getLong(null, columnAddr + (i * 8L)); - argument[i - rowsStart] = UdfUtils - .convertDateTimeV2ToJavaDateTime(value, argTypeClass); - } - } - return argument; - } - - public static Object[] convertLargeIntArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr, - long columnAddr) { - BigInteger[] argument = new BigInteger[rowsEnd - rowsStart]; - byte[] bytes = new byte[16]; - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - UdfUtils.copyMemory(null, columnAddr + (i * 16L), bytes, UdfUtils.BYTE_ARRAY_OFFSET, 16); - argument[i - rowsStart] = new BigInteger(UdfUtils.convertByteOrder(bytes)); - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - UdfUtils.copyMemory(null, columnAddr + (i * 16L), bytes, UdfUtils.BYTE_ARRAY_OFFSET, 16); - argument[i - rowsStart] = new BigInteger(UdfUtils.convertByteOrder(bytes)); - } - } - return argument; - } - - public static Object[] convertDecimalArg(int scale, long typeLen, boolean isNullable, int rowsStart, int rowsEnd, - long nullMapAddr, long columnAddr) { - BigDecimal[] argument = new BigDecimal[rowsEnd - rowsStart]; - byte[] bytes = new byte[(int) typeLen]; - if (isNullable) { - for (int i = rowsStart; i < rowsEnd; ++i) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + i) == 0) { - UdfUtils.copyMemory(null, columnAddr + (i * typeLen), bytes, UdfUtils.BYTE_ARRAY_OFFSET, typeLen); - BigInteger bigInteger = new BigInteger(UdfUtils.convertByteOrder(bytes)); - argument[i - rowsStart] = new BigDecimal(bigInteger, scale); //show to pass scale info - } // else is the current row is null - } - } else { - for (int i = rowsStart; i < rowsEnd; ++i) { - UdfUtils.copyMemory(null, columnAddr + (i * typeLen), bytes, UdfUtils.BYTE_ARRAY_OFFSET, typeLen); - BigInteger bigInteger = new BigInteger(UdfUtils.convertByteOrder(bytes)); - argument[i - rowsStart] = new BigDecimal(bigInteger, scale); - } - } - return argument; - } - - public static Object[] convertStringArg(boolean isNullable, int rowsStart, int rowsEnd, long nullMapAddr, - long charsAddr, long offsetsAddr) { - String[] argument = new String[rowsEnd - rowsStart]; - Preconditions.checkState(UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * (0 - 1)) == 0, - "offsetsAddr[-1] should be 0;"); - final int totalLen = UdfUtils.UNSAFE.getInt(null, offsetsAddr + (rowsEnd - 1) * 4L); - byte[] bytes = new byte[totalLen]; - UdfUtils.copyMemory(null, charsAddr, bytes, UdfUtils.BYTE_ARRAY_OFFSET, totalLen); - if (isNullable) { - for (int row = rowsStart; row < rowsEnd; ++row) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - int prevOffset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * (row - 1)); - int currOffset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + row * 4L); - argument[row - rowsStart] = new String(bytes, prevOffset, currOffset - prevOffset, - StandardCharsets.UTF_8); - } // else is the current row is null - } - } else { - for (int row = rowsStart; row < rowsEnd; ++row) { - int prevOffset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * (row - 1)); - int currOffset = UdfUtils.UNSAFE.getInt(null, offsetsAddr + 4L * row); - argument[row - rowsStart] = new String(bytes, prevOffset, currOffset - prevOffset, - StandardCharsets.UTF_8); - } - } - return argument; - } - - /////////////////////////////////////////copyBatch////////////////////////////////////////////////////////////// - public static void copyBatchBooleanResult(boolean isNullable, int numRows, Boolean[] result, long nullMapAddr, - long resColumnAddr) { - byte[] dataArr = new byte[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = result[i] ? (byte) 1 : 0; - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = result[i] ? (byte) 1 : 0; - } - } - UdfUtils.copyMemory(dataArr, UdfUtils.BYTE_ARRAY_OFFSET, null, resColumnAddr, numRows); - } - - public static void copyBatchTinyIntResult(boolean isNullable, int numRows, Byte[] result, long nullMapAddr, - long resColumnAddr) { - byte[] dataArr = new byte[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(dataArr, UdfUtils.BYTE_ARRAY_OFFSET, null, resColumnAddr, numRows); - } - - public static void copyBatchSmallIntResult(boolean isNullable, int numRows, Short[] result, long nullMapAddr, - long resColumnAddr) { - short[] dataArr = new short[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(dataArr, OffHeap.SHORT_ARRAY_OFFSET, null, resColumnAddr, numRows * 2L); - } - - public static void copyBatchIntResult(boolean isNullable, int numRows, Integer[] result, long nullMapAddr, - long resColumnAddr) { - int[] dataArr = new int[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(dataArr, UdfUtils.INT_ARRAY_OFFSET, null, resColumnAddr, numRows * 4L); - } - - public static void copyBatchBigIntResult(boolean isNullable, int numRows, Long[] result, long nullMapAddr, - long resColumnAddr) { - long[] dataArr = new long[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(dataArr, OffHeap.LONG_ARRAY_OFFSET, null, resColumnAddr, numRows * 8L); - } - - public static void copyBatchFloatResult(boolean isNullable, int numRows, Float[] result, long nullMapAddr, - long resColumnAddr) { - float[] dataArr = new float[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(dataArr, OffHeap.FLOAT_ARRAY_OFFSET, null, resColumnAddr, numRows * 4L); - } - - public static void copyBatchDoubleResult(boolean isNullable, int numRows, Double[] result, long nullMapAddr, - long resColumnAddr) { - double[] dataArr = new double[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = result[i]; - } - } - UdfUtils.copyMemory(dataArr, OffHeap.DOUBLE_ARRAY_OFFSET, null, resColumnAddr, numRows * 8L); - } - - public static void copyBatchDateResult(Class retClass, boolean isNullable, int numRows, Object[] result, - long nullMapAddr, - long resColumnAddr) { - long[] dataArr = new long[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = UdfUtils.convertToDate(result[i], retClass); - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = UdfUtils.convertToDate(result[i], retClass); - } - } - UdfUtils.copyMemory(dataArr, OffHeap.LONG_ARRAY_OFFSET, null, resColumnAddr, numRows * 8L); - } - - public static void copyBatchDateV2Result(Class retClass, boolean isNullable, int numRows, Object[] result, - long nullMapAddr, - long resColumnAddr) { - int[] dataArr = new int[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = UdfUtils.convertToDateV2(result[i], retClass); - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = UdfUtils.convertToDateV2(result[i], retClass); - } - } - UdfUtils.copyMemory(dataArr, OffHeap.INT_ARRAY_OFFSET, null, resColumnAddr, numRows * 4L); - } - - public static void copyBatchDateTimeResult(Class retClass, boolean isNullable, int numRows, Object[] result, - long nullMapAddr, long resColumnAddr) { - long[] dataArr = new long[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = UdfUtils.convertToDateTime(result[i], retClass); - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = UdfUtils.convertToDateTime(result[i], retClass); - } - } - UdfUtils.copyMemory(dataArr, OffHeap.LONG_ARRAY_OFFSET, null, resColumnAddr, numRows * 8L); - } - - public static void copyBatchDateTimeV2Result(Class retClass, boolean isNullable, int numRows, - Object[] result, long nullMapAddr, long resColumnAddr) { - long[] dataArr = new long[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - dataArr[i] = UdfUtils.convertToDateTimeV2(result[i], retClass); - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - dataArr[i] = UdfUtils.convertToDateTimeV2(result[i], retClass); - } - } - UdfUtils.copyMemory(dataArr, OffHeap.LONG_ARRAY_OFFSET, null, resColumnAddr, numRows * 8L); - } - - public static void copyBatchLargeIntResult(boolean isNullable, int numRows, BigInteger[] result, long nullMapAddr, - long resColumnAddr) { - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - byte[] bytes = UdfUtils.convertByteOrder(result[i].toByteArray()); - byte[] value = new byte[16]; - if (result[i].signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, resColumnAddr + (i * 16L), 16); - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - byte[] bytes = UdfUtils.convertByteOrder(result[i].toByteArray()); - byte[] value = new byte[16]; - if (result[i].signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, resColumnAddr + (i * 16L), 16); - } - } - } - - public static void copyBatchDecimal32Result(int scale, boolean isNullable, int numRows, BigDecimal[] result, - long nullMapAddr, - long columnAddr) { - BigInteger[] data = new BigInteger[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - data[i] = result[i].setScale(scale, RoundingMode.HALF_EVEN).unscaledValue(); - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - data[i] = result[i].setScale(scale, RoundingMode.HALF_EVEN).unscaledValue(); - } - } - copyBatchDecimalResult(4, isNullable, numRows, data, columnAddr); - } - - public static void copyBatchDecimal64Result(int scale, boolean isNullable, int numRows, BigDecimal[] result, - long nullMapAddr, - long columnAddr) { - BigInteger[] data = new BigInteger[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - data[i] = result[i].setScale(scale, RoundingMode.HALF_EVEN).unscaledValue(); - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - data[i] = result[i].setScale(scale, RoundingMode.HALF_EVEN).unscaledValue(); - } - } - copyBatchDecimalResult(8, isNullable, numRows, data, columnAddr); - } - - - public static void copyBatchDecimal128Result(int scale, boolean isNullable, int numRows, BigDecimal[] result, - long nullMapAddr, - long columnAddr) { - BigInteger[] data = new BigInteger[numRows]; - if (isNullable) { - byte[] nulls = new byte[numRows]; - for (int i = 0; i < numRows; i++) { - if (result[i] == null) { - nulls[i] = 1; - } else { - data[i] = result[i].setScale(scale, RoundingMode.HALF_EVEN).unscaledValue(); - } - } - UdfUtils.copyMemory(nulls, UdfUtils.BYTE_ARRAY_OFFSET, null, nullMapAddr, numRows); - } else { - for (int i = 0; i < numRows; i++) { - data[i] = result[i].setScale(scale, RoundingMode.HALF_EVEN).unscaledValue(); - } - } - copyBatchDecimalResult(16, isNullable, numRows, data, columnAddr); - } - - private static void copyBatchDecimalResult(long typeLen, boolean isNullable, int numRows, BigInteger[] result, - long resColumnAddr) { - if (isNullable) { - for (int i = 0; i < numRows; i++) { - if (result[i] != null) { - byte[] bytes = UdfUtils.convertByteOrder(result[i].toByteArray()); - byte[] value = new byte[(int) typeLen]; - if (result[i].signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, resColumnAddr + (i * typeLen), - value.length); - } - } - } else { - for (int i = 0; i < numRows; i++) { - byte[] bytes = UdfUtils.convertByteOrder(result[i].toByteArray()); - byte[] value = new byte[(int) typeLen]; - if (result[i].signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, resColumnAddr + (i * typeLen), - value.length); - } - } - } - - private static final byte[] emptyBytes = new byte[0]; - - public static void copyBatchStringResult(boolean isNullable, int numRows, String[] strResult, long nullMapAddr, - long charsAddr, long offsetsAddr) { - int[] offsets = new int[numRows]; - byte[][] byteRes = new byte[numRows][]; - int offset = 0; - if (isNullable) { - for (int i = 0; i < numRows; i++) { - if (strResult[i] == null) { - byteRes[i] = emptyBytes; - UdfUtils.UNSAFE.putByte(nullMapAddr + i, (byte) 1); - } else { - byteRes[i] = ((String) strResult[i]).getBytes(StandardCharsets.UTF_8); - } - offset += byteRes[i].length; - offsets[i] = offset; - } - } else { - for (int i = 0; i < numRows; i++) { - byteRes[i] = ((String) strResult[i]).getBytes(StandardCharsets.UTF_8); - offset += byteRes[i].length; - offsets[i] = offset; - } - } - byte[] bytes = new byte[offsets[numRows - 1]]; - long bytesAddr = JNINativeMethod.resizeStringColumn(charsAddr, offsets[numRows - 1]); - int dst = 0; - for (int i = 0; i < numRows; i++) { - for (int j = 0; j < byteRes[i].length; j++) { - bytes[dst++] = byteRes[i][j]; - } - } - - UdfUtils.copyMemory(offsets, UdfUtils.INT_ARRAY_OFFSET, null, offsetsAddr, numRows * 4L); - UdfUtils.copyMemory(bytes, UdfUtils.BYTE_ARRAY_OFFSET, null, bytesAddr, offsets[numRows - 1]); - } - - - //////////////////////////////////// copyBatchArray////////////////////////////////////////////////////////// - - public static long copyBatchArrayBooleanResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Boolean value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putByte(dataAddr + ((hasPutElementNum + i)), value ? (byte) 1 : 0); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Boolean value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putByte(dataAddr + ((hasPutElementNum + i)), value ? (byte) 1 : 0); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayTinyIntResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Byte value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putByte(dataAddr + ((hasPutElementNum + i)), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Byte value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putByte(dataAddr + ((hasPutElementNum + i)), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArraySmallIntResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Short value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putShort(dataAddr + ((hasPutElementNum + i) * 2L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Short value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putShort(dataAddr + ((hasPutElementNum + i) * 2L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayIntResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Integer value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putInt(dataAddr + ((hasPutElementNum + i) * 4L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Integer value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putInt(dataAddr + ((hasPutElementNum + i) * 4L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayBigIntResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Long value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putLong(dataAddr + ((hasPutElementNum + i) * 8L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Long value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putLong(dataAddr + ((hasPutElementNum + i) * 8L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayFloatResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Float value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putFloat(dataAddr + ((hasPutElementNum + i) * 4L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Float value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putFloat(dataAddr + ((hasPutElementNum + i) * 4L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayDoubleResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Double value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putDouble(dataAddr + ((hasPutElementNum + i) * 8L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - Double value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - UdfUtils.UNSAFE.putDouble(dataAddr + ((hasPutElementNum + i) * 8L), value); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayDateResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - LocalDate value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - long time = UdfUtils.convertToDate(value, LocalDate.class); - UdfUtils.UNSAFE.putLong(dataAddr + ((hasPutElementNum + i) * 8L), time); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - LocalDate value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - long time = UdfUtils.convertToDate(value, LocalDate.class); - UdfUtils.UNSAFE.putLong(dataAddr + ((hasPutElementNum + i) * 8L), time); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayDateTimeResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - LocalDateTime value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - long time = UdfUtils.convertToDateTime(value, LocalDateTime.class); - UdfUtils.UNSAFE.putLong(dataAddr + ((hasPutElementNum + i) * 8L), time); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - LocalDateTime value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - long time = UdfUtils.convertToDateTime(value, LocalDateTime.class); - UdfUtils.UNSAFE.putLong(dataAddr + ((hasPutElementNum + i) * 8L), time); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayDateV2Result(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - LocalDate value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - int time = UdfUtils.convertToDateV2(value, LocalDate.class); - UdfUtils.UNSAFE.putInt(dataAddr + ((hasPutElementNum + i) * 4L), time); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - LocalDate value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - int time = UdfUtils.convertToDateV2(value, LocalDate.class); - UdfUtils.UNSAFE.putInt(dataAddr + ((hasPutElementNum + i) * 4L), time); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayDateTimeV2Result(long hasPutElementNum, boolean isNullable, int row, - Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - LocalDateTime value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - long time = UdfUtils.convertToDateTimeV2(value, LocalDateTime.class); - UdfUtils.UNSAFE.putLong(dataAddr + ((hasPutElementNum + i) * 8L), time); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - LocalDateTime value = data.get(i); - if (value == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - long time = UdfUtils.convertToDateTimeV2(value, LocalDateTime.class); - UdfUtils.UNSAFE.putLong(dataAddr + ((hasPutElementNum + i) * 8L), time); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayLargeIntResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - BigInteger bigInteger = data.get(i); - if (bigInteger == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - byte[] bytes = UdfUtils.convertByteOrder(bigInteger.toByteArray()); - byte[] value = new byte[16]; - // check data is negative - if (bigInteger.signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, - dataAddr + ((hasPutElementNum + i) * 16L), 16); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - BigInteger bigInteger = data.get(i); - if (bigInteger == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - byte[] bytes = UdfUtils.convertByteOrder(bigInteger.toByteArray()); - byte[] value = new byte[16]; - // check data is negative - if (bigInteger.signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, - dataAddr + ((hasPutElementNum + i) * 16L), 16); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayDecimalResult(long hasPutElementNum, boolean isNullable, int row, Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - BigDecimal bigDecimal = data.get(i); - if (bigDecimal == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - BigInteger bigInteger = bigDecimal.setScale(9, RoundingMode.HALF_EVEN).unscaledValue(); - byte[] bytes = UdfUtils.convertByteOrder(bigInteger.toByteArray()); - byte[] value = new byte[16]; - // check data is negative - if (bigInteger.signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, - dataAddr + ((hasPutElementNum + i) * 16L), 16); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - BigDecimal bigDecimal = data.get(i); - if (bigDecimal == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - BigInteger bigInteger = bigDecimal.setScale(9, RoundingMode.HALF_EVEN).unscaledValue(); - byte[] bytes = UdfUtils.convertByteOrder(bigInteger.toByteArray()); - byte[] value = new byte[16]; - // check data is negative - if (bigInteger.signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, - dataAddr + ((hasPutElementNum + i) * 16L), 16); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayDecimalV3Result(int scale, long typeLen, long hasPutElementNum, boolean isNullable, - int row, - Object result, - long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - BigDecimal bigDecimal = data.get(i); - if (bigDecimal == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - BigInteger bigInteger = bigDecimal.setScale(scale, RoundingMode.HALF_EVEN).unscaledValue(); - byte[] bytes = UdfUtils.convertByteOrder(bigInteger.toByteArray()); - byte[] value = new byte[(int) typeLen]; - // check data is negative - if (bigInteger.signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, - dataAddr + ((hasPutElementNum + i) * typeLen), typeLen); - } - } - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - for (int i = 0; i < num; ++i) { - BigDecimal bigDecimal = data.get(i); - if (bigDecimal == null) { - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - BigInteger bigInteger = bigDecimal.setScale(scale, RoundingMode.HALF_EVEN).unscaledValue(); - byte[] bytes = UdfUtils.convertByteOrder(bigInteger.toByteArray()); - byte[] value = new byte[(int) typeLen]; - // check data is negative - if (bigInteger.signum() == -1) { - Arrays.fill(value, (byte) -1); - } - System.arraycopy(bytes, 0, value, 0, Math.min(bytes.length, value.length)); - UdfUtils.copyMemory(value, UdfUtils.BYTE_ARRAY_OFFSET, null, - dataAddr + ((hasPutElementNum + i) * typeLen), typeLen); - } - } - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - public static long copyBatchArrayStringResult(long hasPutElementNum, boolean isNullable, int row, - Object result, long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr, - long strOffsetAddr) { - ArrayList data = (ArrayList) result; - if (isNullable) { - if (data == null) { - UdfUtils.UNSAFE.putByte(nullMapAddr + row, (byte) 1); - } else { - int num = data.size(); - int[] offsets = new int[num]; - byte[][] byteRes = new byte[num][]; - int oldOffsetNum = UdfUtils.UNSAFE.getInt(null, strOffsetAddr + ((hasPutElementNum - 1) * 4L)); - int offset = oldOffsetNum; - for (int i = 0; i < num; ++i) { - String value = data.get(i); - if (value == null) { - byteRes[i] = emptyBytes; - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - byteRes[i] = value.getBytes(StandardCharsets.UTF_8); - } - offset += byteRes[i].length; - offsets[i] = offset; - } - int oldSzie = 0; - if (num > 0) { - oldSzie = offsets[num - 1]; - } - byte[] bytes = new byte[oldSzie - oldOffsetNum]; - long bytesAddr = JNINativeMethod.resizeStringColumn(dataAddr, oldSzie); - int dst = 0; - for (int i = 0; i < num; i++) { - for (int j = 0; j < byteRes[i].length; j++) { - bytes[dst++] = byteRes[i][j]; - } - } - UdfUtils.copyMemory(offsets, UdfUtils.INT_ARRAY_OFFSET, null, strOffsetAddr + (4L * hasPutElementNum), - num * 4L); - UdfUtils.copyMemory(bytes, UdfUtils.BYTE_ARRAY_OFFSET, null, bytesAddr + oldOffsetNum, - oldSzie - oldOffsetNum); - hasPutElementNum = hasPutElementNum + num; - } - } else { - int num = data.size(); - int[] offsets = new int[num]; - byte[][] byteRes = new byte[num][]; - int offset = 0; - for (int i = 0; i < num; ++i) { - String value = data.get(i); - if (value == null) { - byteRes[i] = emptyBytes; - UdfUtils.UNSAFE.putByte(nestedNullMapAddr + row, (byte) 1); - } else { - byteRes[i] = value.getBytes(StandardCharsets.UTF_8); - } - offset += byteRes[i].length; - offsets[i] = offset; - } - int oldOffsetNum = UdfUtils.UNSAFE.getInt(null, strOffsetAddr + ((hasPutElementNum - 1) * 4L)); - int oldSzie = 0; - if (num > 0) { - oldSzie = offsets[num - 1]; - } - byte[] bytes = new byte[oldSzie]; - long bytesAddr = JNINativeMethod.resizeStringColumn(dataAddr, oldOffsetNum + oldSzie); - int dst = 0; - for (int i = 0; i < num; i++) { - for (int j = 0; j < byteRes[i].length; j++) { - bytes[dst++] = byteRes[i][j]; - } - } - UdfUtils.copyMemory(offsets, UdfUtils.INT_ARRAY_OFFSET, null, strOffsetAddr + (4L * oldOffsetNum), - num * 4L); - UdfUtils.copyMemory(bytes, UdfUtils.BYTE_ARRAY_OFFSET, null, bytesAddr + oldOffsetNum, - oldSzie); - hasPutElementNum = hasPutElementNum + num; - } - UdfUtils.UNSAFE.putLong(null, offsetsAddr + 8L * row, hasPutElementNum); - return hasPutElementNum; - } - - //////////////////////////////////////////convertArray/////////////////////////////////////////////////////////// - public static ArrayList convertArrayBooleanArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - boolean value = UdfUtils.UNSAFE.getBoolean(null, dataAddr + (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - boolean value = UdfUtils.UNSAFE.getBoolean(null, dataAddr + (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayTinyIntArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - byte value = UdfUtils.UNSAFE.getByte(null, dataAddr + (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - byte value = UdfUtils.UNSAFE.getByte(null, dataAddr + (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArraySmallIntArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - short value = UdfUtils.UNSAFE.getShort(null, dataAddr + 2L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - short value = UdfUtils.UNSAFE.getShort(null, dataAddr + 2L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayIntArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - int value = UdfUtils.UNSAFE.getInt(null, dataAddr + 4L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - int value = UdfUtils.UNSAFE.getInt(null, dataAddr + 4L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayBigIntArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - long value = UdfUtils.UNSAFE.getLong(null, dataAddr + 8L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - long value = UdfUtils.UNSAFE.getLong(null, dataAddr + 8L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayFloatArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - float value = UdfUtils.UNSAFE.getFloat(null, dataAddr + 4L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - float value = UdfUtils.UNSAFE.getFloat(null, dataAddr + 4L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayDoubleArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - double value = UdfUtils.UNSAFE.getDouble(null, dataAddr + 8L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - double value = UdfUtils.UNSAFE.getDouble(null, dataAddr + 8L * (offsetRow + offsetStart)); - data.add(value); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayDateArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - long value = UdfUtils.UNSAFE.getLong(null, dataAddr + 8L * (offsetRow + offsetStart)); - LocalDate obj = (LocalDate) UdfUtils.convertDateToJavaDate(value, LocalDate.class); - data.add(obj); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - long value = UdfUtils.UNSAFE.getLong(null, dataAddr + 8L * (offsetRow + offsetStart)); - // TODO: now argClass[argIdx + argClassOffset] is java.util.ArrayList, can't get - // nested class type, so don't know the date type class is LocalDate or others - // LocalDate obj = UdfUtils.convertDateToJavaDate(value, argClass[argIdx + - // argClassOffset]); - LocalDate obj = (LocalDate) UdfUtils.convertDateToJavaDate(value, LocalDate.class); - data.add(obj); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayDateTimeArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - long value = UdfUtils.UNSAFE.getLong(null, dataAddr + 8L * (offsetRow + offsetStart)); - LocalDateTime obj = (LocalDateTime) UdfUtils - .convertDateTimeToJavaDateTime(value, LocalDateTime.class); - data.add(obj); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - long value = UdfUtils.UNSAFE.getLong(null, dataAddr + 8L * (offsetRow + offsetStart)); - LocalDateTime obj = (LocalDateTime) UdfUtils - .convertDateTimeToJavaDateTime(value, LocalDateTime.class); - data.add(obj); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayDateV2Arg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - int value = UdfUtils.UNSAFE.getInt(null, dataAddr + 4L * (offsetRow + offsetStart)); - LocalDate obj = (LocalDate) UdfUtils.convertDateV2ToJavaDate(value, LocalDate.class); - data.add(obj); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - int value = UdfUtils.UNSAFE.getInt(null, dataAddr + 4L * (offsetRow + offsetStart)); - LocalDate obj = (LocalDate) UdfUtils.convertDateV2ToJavaDate(value, LocalDate.class); - data.add(obj); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayDateTimeV2Arg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - long value = UdfUtils.UNSAFE.getLong(null, dataAddr + 8L * (offsetRow + offsetStart)); - LocalDateTime obj = (LocalDateTime) UdfUtils - .convertDateTimeV2ToJavaDateTime(value, LocalDateTime.class); - data.add(obj); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - long value = UdfUtils.UNSAFE.getLong(null, dataAddr + 8L * (offsetRow + offsetStart)); - LocalDateTime obj = (LocalDateTime) UdfUtils - .convertDateTimeV2ToJavaDateTime(value, LocalDateTime.class); - data.add(obj); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayLargeIntArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - byte[] bytes = new byte[16]; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - UdfUtils.copyMemory(null, dataAddr + 16L * (offsetRow + offsetStart), bytes, - UdfUtils.BYTE_ARRAY_OFFSET, 16); - data.add(new BigInteger(UdfUtils.convertByteOrder(bytes))); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - UdfUtils.copyMemory(null, dataAddr + 16L * (offsetRow + offsetStart), bytes, - UdfUtils.BYTE_ARRAY_OFFSET, 16); - data.add(new BigInteger(UdfUtils.convertByteOrder(bytes))); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayDecimalArg(int scale, long typeLen, int row, int currentRowNum, - long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr) { - ArrayList data = null; - byte[] bytes = new byte[16]; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - UdfUtils.copyMemory(null, dataAddr + typeLen * (offsetRow + offsetStart), bytes, - UdfUtils.BYTE_ARRAY_OFFSET, typeLen); - BigInteger bigInteger = new BigInteger(UdfUtils.convertByteOrder(bytes)); - data.add(new BigDecimal(bigInteger, scale)); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // else is current array row is null - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - UdfUtils.copyMemory(null, dataAddr + typeLen * (offsetRow + offsetStart), bytes, - UdfUtils.BYTE_ARRAY_OFFSET, typeLen); - BigInteger bigInteger = new BigInteger(UdfUtils.convertByteOrder(bytes)); - data.add(new BigDecimal(bigInteger, scale)); - } else { // in the array row, current offset is null - data.add(null); - } - } // for loop - } // end for all current row - return data; - } - - public static ArrayList convertArrayStringArg(int row, int currentRowNum, long offsetStart, - boolean isNullable, long nullMapAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) { - ArrayList data = null; - if (isNullable) { - if (UdfUtils.UNSAFE.getByte(nullMapAddr + row) == 0) { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) - == 0)) { - int offset = UdfUtils.UNSAFE - .getInt(null, strOffsetAddr + (offsetRow + offsetStart) * 4L); - int numBytes = offset - UdfUtils.UNSAFE - .getInt(null, strOffsetAddr + 4L * ((offsetRow + offsetStart) - 1)); - long base = dataAddr + offset - numBytes; - byte[] bytes = new byte[numBytes]; - UdfUtils.copyMemory(null, base, bytes, UdfUtils.BYTE_ARRAY_OFFSET, numBytes); - data.add(new String(bytes, StandardCharsets.UTF_8)); - } else { - data.add(null); - } - } - } - } else { - data = new ArrayList<>(currentRowNum); - for (int offsetRow = 0; offsetRow < currentRowNum; ++offsetRow) { - if ((UdfUtils.UNSAFE.getByte(null, nestedNullMapAddr + (offsetStart + offsetRow)) == 0)) { - int offset = UdfUtils.UNSAFE - .getInt(null, strOffsetAddr + (offsetRow + offsetStart) * 4L); - int numBytes = offset - UdfUtils.UNSAFE - .getInt(null, strOffsetAddr + 4L * ((offsetRow + offsetStart) - 1)); - long base = dataAddr + offset - numBytes; - byte[] bytes = new byte[numBytes]; - UdfUtils.copyMemory(null, base, bytes, UdfUtils.BYTE_ARRAY_OFFSET, numBytes); - data.add(new String(bytes, StandardCharsets.UTF_8)); - } else { - data.add(null); - } - } - } - return data; - } -} diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java index 2d22cfda6a..beb05a10f0 100644 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java @@ -25,7 +25,6 @@ import org.apache.doris.common.jni.utils.UdfUtils; import org.apache.doris.common.jni.vec.ColumnValueConverter; import org.apache.doris.common.jni.vec.VectorTable; import org.apache.doris.thrift.TJavaUdfExecutorCtorParams; -import org.apache.doris.thrift.TPrimitiveType; import com.esotericsoftware.reflectasm.MethodAccess; import com.google.common.base.Joiner; @@ -36,15 +35,11 @@ import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.net.MalformedURLException; -import java.time.LocalDate; -import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; public class UdfExecutor extends BaseExecutor { - // private static final java.util.logging.Logger LOG = - // Logger.getLogger(UdfExecutor.class); public static final Logger LOG = Logger.getLogger(UdfExecutor.class); // setup by init() and cleared by close() private Method method; @@ -76,78 +71,6 @@ public class UdfExecutor extends BaseExecutor { super.close(); } - private ColumnValueConverter getInputConverter(TPrimitiveType primitiveType, Class clz) { - switch (primitiveType) { - case DATE: - case DATEV2: { - if (java.util.Date.class.equals(clz)) { - return (Object[] columnData) -> { - Object[] result = new java.util.Date[columnData.length]; - for (int i = 0; i < columnData.length; ++i) { - if (columnData[i] != null) { - LocalDate v = (LocalDate) columnData[i]; - result[i] = new java.util.Date(v.getYear() - 1900, v.getMonthValue() - 1, - v.getDayOfMonth()); - } - } - return result; - }; - } else if (org.joda.time.LocalDate.class.equals(clz)) { - return (Object[] columnData) -> { - Object[] result = new org.joda.time.LocalDate[columnData.length]; - for (int i = 0; i < columnData.length; ++i) { - if (columnData[i] != null) { - LocalDate v = (LocalDate) columnData[i]; - result[i] = new org.joda.time.LocalDate(v.getYear(), v.getMonthValue(), - v.getDayOfMonth()); - } - } - return result; - }; - } else if (!LocalDate.class.equals(clz)) { - throw new RuntimeException("Unsupported date type: " + clz.getCanonicalName()); - } - break; - } - case DATETIME: - case DATETIMEV2: { - if (org.joda.time.DateTime.class.equals(clz)) { - return (Object[] columnData) -> { - Object[] result = new org.joda.time.DateTime[columnData.length]; - for (int i = 0; i < columnData.length; ++i) { - if (columnData[i] != null) { - LocalDateTime v = (LocalDateTime) columnData[i]; - result[i] = new org.joda.time.DateTime(v.getYear(), v.getMonthValue(), - v.getDayOfMonth(), v.getHour(), - v.getMinute(), v.getSecond(), v.getNano() / 1000000); - } - } - return result; - }; - } else if (org.joda.time.LocalDateTime.class.equals(clz)) { - return (Object[] columnData) -> { - Object[] result = new org.joda.time.LocalDateTime[columnData.length]; - for (int i = 0; i < columnData.length; ++i) { - if (columnData[i] != null) { - LocalDateTime v = (LocalDateTime) columnData[i]; - result[i] = new org.joda.time.LocalDateTime(v.getYear(), v.getMonthValue(), - v.getDayOfMonth(), v.getHour(), - v.getMinute(), v.getSecond(), v.getNano() / 1000000); - } - } - return result; - }; - } else if (!LocalDateTime.class.equals(clz)) { - throw new RuntimeException("Unsupported date type: " + clz.getCanonicalName()); - } - break; - } - default: - break; - } - return null; - } - private Map getInputConverters(int numColumns) { Map converters = new HashMap<>(); for (int j = 0; j < numColumns; ++j) { @@ -160,74 +83,7 @@ public class UdfExecutor extends BaseExecutor { } private ColumnValueConverter getOutputConverter() { - Class clz = method.getReturnType(); - switch (retType.getPrimitiveType()) { - case DATE: - case DATEV2: { - if (java.util.Date.class.equals(clz)) { - return (Object[] columnData) -> { - Object[] result = new LocalDate[columnData.length]; - for (int i = 0; i < columnData.length; ++i) { - if (columnData[i] != null) { - java.util.Date v = (java.util.Date) columnData[i]; - result[i] = LocalDate.of(v.getYear() + 1900, v.getMonth() + 1, v.getDate()); - } - } - return result; - }; - } else if (org.joda.time.LocalDate.class.equals(clz)) { - return (Object[] columnData) -> { - Object[] result = new LocalDate[columnData.length]; - for (int i = 0; i < columnData.length; ++i) { - if (columnData[i] != null) { - org.joda.time.LocalDate v = (org.joda.time.LocalDate) columnData[i]; - result[i] = LocalDate.of(v.getYear(), v.getMonthOfYear(), v.getDayOfMonth()); - } - } - return result; - }; - } else if (!LocalDate.class.equals(clz)) { - throw new RuntimeException("Unsupported date type: " + clz.getCanonicalName()); - } - break; - } - case DATETIME: - case DATETIMEV2: { - if (org.joda.time.DateTime.class.equals(clz)) { - return (Object[] columnData) -> { - Object[] result = new LocalDateTime[columnData.length]; - for (int i = 0; i < columnData.length; ++i) { - if (columnData[i] != null) { - org.joda.time.DateTime v = (org.joda.time.DateTime) columnData[i]; - result[i] = LocalDateTime.of(v.getYear(), v.getMonthOfYear(), v.getDayOfMonth(), - v.getHourOfDay(), - v.getMinuteOfHour(), v.getSecondOfMinute(), v.getMillisOfSecond() * 1000000); - } - } - return result; - }; - } else if (org.joda.time.LocalDateTime.class.equals(clz)) { - return (Object[] columnData) -> { - Object[] result = new LocalDateTime[columnData.length]; - for (int i = 0; i < columnData.length; ++i) { - if (columnData[i] != null) { - org.joda.time.LocalDateTime v = (org.joda.time.LocalDateTime) columnData[i]; - result[i] = LocalDateTime.of(v.getYear(), v.getMonthOfYear(), v.getDayOfMonth(), - v.getHourOfDay(), - v.getMinuteOfHour(), v.getSecondOfMinute(), v.getMillisOfSecond() * 1000000); - } - } - return result; - }; - } else if (!LocalDateTime.class.equals(clz)) { - throw new RuntimeException("Unsupported date type: " + clz.getCanonicalName()); - } - break; - } - default: - break; - } - return null; + return getOutputConverter(retType.getPrimitiveType(), method.getReturnType()); } public long evaluate(Map inputParams, Map outputParams) throws UdfRuntimeException { @@ -235,7 +91,16 @@ public class UdfExecutor extends BaseExecutor { VectorTable inputTable = VectorTable.createReadableTable(inputParams); int numRows = inputTable.getNumRows(); int numColumns = inputTable.getNumColumns(); - Object[] result = (Object[]) Array.newInstance(method.getReturnType(), numRows); + if (outputTable != null) { + outputTable.close(); + } + outputTable = VectorTable.createWritableTable(outputParams, numRows); + + // If the return type is primitive, we can't cast the array of primitive type as array of Object, + // so we have to new its wrapped Object. + Object[] result = outputTable.getColumnType(0).isPrimitive() + ? outputTable.getColumn(0).newObjectContainerArray(numRows) + : (Object[]) Array.newInstance(method.getReturnType(), numRows); Object[][] inputs = inputTable.getMaterializedData(getInputConverters(numColumns)); Object[] parameters = new Object[numColumns]; for (int i = 0; i < numRows; ++i) { @@ -244,13 +109,7 @@ public class UdfExecutor extends BaseExecutor { } result[i] = methodAccess.invoke(udf, evaluateIndex, parameters); } - - if (outputTable != null) { - outputTable.close(); - } - boolean isNullable = Boolean.parseBoolean(outputParams.getOrDefault("is_nullable", "true")); - outputTable = VectorTable.createWritableTable(outputParams, numRows); outputTable.appendData(0, result, getOutputConverter(), isNullable); return outputTable.getMetaAddress(); } catch (Exception e) { @@ -336,7 +195,8 @@ public class UdfExecutor extends BaseExecutor { StringBuilder sb = new StringBuilder(); sb.append("Unable to find evaluate function with the correct signature: ") - .append(className + ".evaluate(") + .append(className) + .append(".evaluate(") .append(Joiner.on(", ").join(parameterTypes)) .append(")\n") .append("UDF contains: \n ") diff --git a/regression-test/suites/javaudf_p0/test_javaudf_agg_map.groovy b/regression-test/suites/javaudf_p0/test_javaudf_agg_map.groovy index 03f84e5b34..526ddfd895 100644 --- a/regression-test/suites/javaudf_p0/test_javaudf_agg_map.groovy +++ b/regression-test/suites/javaudf_p0/test_javaudf_agg_map.groovy @@ -32,9 +32,9 @@ suite("test_javaudf_agg_map") { CREATE TABLE IF NOT EXISTS db_agg_map( `id` INT NULL COMMENT "", `i` INT NULL COMMENT "", - `d` Double NULL COMMENT "", - `mii` Map NULL COMMENT "", - `mid` Map NULL COMMENT "" + `d` Double NULL COMMENT "", + `mii` Map NULL COMMENT "", + `mid` Map NULL COMMENT "" ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1