diff --git a/be/src/exprs/anyval_util.cpp b/be/src/exprs/anyval_util.cpp index 1099c1297e..9d923bb4a7 100644 --- a/be/src/exprs/anyval_util.cpp +++ b/be/src/exprs/anyval_util.cpp @@ -81,6 +81,7 @@ AnyVal* create_any_val(ObjectPool* pool, const TypeDescriptor& type) { return pool->add(new FloatVal); case TYPE_TIME: + case TYPE_TIMEV2: case TYPE_DOUBLE: return pool->add(new DoubleVal); @@ -150,6 +151,7 @@ FunctionContext::TypeDesc AnyValUtil::column_type_to_type_desc(const TypeDescrip out.type = FunctionContext::TYPE_FLOAT; break; case TYPE_TIME: + case TYPE_TIMEV2: case TYPE_DOUBLE: out.type = FunctionContext::TYPE_DOUBLE; break; diff --git a/be/src/exprs/anyval_util.h b/be/src/exprs/anyval_util.h index d30565e0b9..742d44214e 100644 --- a/be/src/exprs/anyval_util.h +++ b/be/src/exprs/anyval_util.h @@ -405,6 +405,7 @@ public: *reinterpret_cast(slot); return; case TYPE_TIME: + case TYPE_TIMEV2: case TYPE_DOUBLE: reinterpret_cast(dst)->val = *reinterpret_cast(slot); diff --git a/be/src/exprs/expr.cpp b/be/src/exprs/expr.cpp index f113733597..22684e60aa 100644 --- a/be/src/exprs/expr.cpp +++ b/be/src/exprs/expr.cpp @@ -123,6 +123,7 @@ Expr::Expr(const TypeDescriptor& type) case TYPE_FLOAT: case TYPE_DOUBLE: case TYPE_TIME: + case TYPE_TIMEV2: _node_type = (TExprNodeType::FLOAT_LITERAL); break; @@ -185,6 +186,7 @@ Expr::Expr(const TypeDescriptor& type, bool is_slotref) case TYPE_FLOAT: case TYPE_DOUBLE: case TYPE_TIME: + case TYPE_TIMEV2: _node_type = (TExprNodeType::FLOAT_LITERAL); break; @@ -682,7 +684,8 @@ doris_udf::AnyVal* Expr::get_const_val(ExprContext* context) { break; } case TYPE_DOUBLE: - case TYPE_TIME: { + case TYPE_TIME: + case TYPE_TIMEV2: { _constant_val.reset(new DoubleVal(get_double_val(context, nullptr))); break; } diff --git a/be/src/exprs/expr_context.cpp b/be/src/exprs/expr_context.cpp index a76fbb936e..3f63c93d2c 100644 --- a/be/src/exprs/expr_context.cpp +++ b/be/src/exprs/expr_context.cpp @@ -223,6 +223,7 @@ void* ExprContext::get_value(Expr* e, TupleRow* row, int precision, int scale) { return &_result.float_val; } case TYPE_TIME: + case TYPE_TIMEV2: case TYPE_DOUBLE: { doris_udf::DoubleVal v = e->get_double_val(this, row); if (v.is_null) { diff --git a/be/src/exprs/literal.cpp b/be/src/exprs/literal.cpp index e107831739..6d2d381c04 100644 --- a/be/src/exprs/literal.cpp +++ b/be/src/exprs/literal.cpp @@ -75,6 +75,7 @@ Literal::Literal(const TExprNode& node) : Expr(node) { break; case TYPE_DOUBLE: case TYPE_TIME: + case TYPE_TIMEV2: DCHECK_EQ(node.node_type, TExprNodeType::FLOAT_LITERAL); DCHECK(node.__isset.float_literal); _value.double_val = node.float_literal.value; @@ -199,7 +200,8 @@ FloatVal Literal::get_float_val(ExprContext* context, TupleRow* row) { } DoubleVal Literal::get_double_val(ExprContext* context, TupleRow* row) { - DCHECK(_type.type == TYPE_DOUBLE || _type.type == TYPE_TIME) << _type; + DCHECK(_type.type == TYPE_DOUBLE || _type.type == TYPE_TIME || _type.type == TYPE_TIMEV2) + << _type; return DoubleVal(_value.double_val); } diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index 2695e31f25..5983b11fcc 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -65,6 +65,7 @@ TExprNodeType::type get_expr_node_type(PrimitiveType type) { case TYPE_FLOAT: case TYPE_DOUBLE: case TYPE_TIME: + case TYPE_TIMEV2: return TExprNodeType::FLOAT_LITERAL; break; diff --git a/be/src/exprs/scalar_fn_call.cpp b/be/src/exprs/scalar_fn_call.cpp index bb710b81b7..3eae4e6f41 100644 --- a/be/src/exprs/scalar_fn_call.cpp +++ b/be/src/exprs/scalar_fn_call.cpp @@ -435,7 +435,7 @@ FloatVal ScalarFnCall::get_float_val(ExprContext* context, TupleRow* row) { } DoubleVal ScalarFnCall::get_double_val(ExprContext* context, TupleRow* row) { - DCHECK(_type.type == TYPE_DOUBLE || _type.type == TYPE_TIME); + DCHECK(_type.type == TYPE_DOUBLE || _type.type == TYPE_TIME || _type.type == TYPE_TIMEV2); DCHECK(context != nullptr); if (_scalar_fn_wrapper == nullptr) { return interpret_eval(context, row); diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h index 973db8f31c..03362acd65 100644 --- a/be/src/runtime/primitive_type.h +++ b/be/src/runtime/primitive_type.h @@ -132,6 +132,11 @@ struct PrimitiveTypeTraits { using ColumnType = vectorized::ColumnFloat64; }; template <> +struct PrimitiveTypeTraits { + using CppType = double; + using ColumnType = vectorized::ColumnFloat64; +}; +template <> struct PrimitiveTypeTraits { using CppType = double; using ColumnType = vectorized::ColumnFloat64; diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp index 249d008205..e2d1e9d9f3 100644 --- a/be/src/vec/data_types/data_type_factory.cpp +++ b/be/src/vec/data_types/data_type_factory.cpp @@ -93,6 +93,7 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo nested = std::make_shared(); break; case TYPE_TIME: + case TYPE_TIMEV2: case TYPE_DOUBLE: nested = std::make_shared(); break; diff --git a/be/src/vec/exprs/vliteral.cpp b/be/src/vec/exprs/vliteral.cpp index e20597391d..8db2e85df5 100644 --- a/be/src/vec/exprs/vliteral.cpp +++ b/be/src/vec/exprs/vliteral.cpp @@ -83,6 +83,7 @@ void VLiteral::init(const TExprNode& node) { break; } case TYPE_TIME: + case TYPE_TIMEV2: case TYPE_DOUBLE: { DCHECK_EQ(node.node_type, TExprNodeType::FLOAT_LITERAL); DCHECK(node.__isset.float_literal); @@ -212,6 +213,7 @@ std::string VLiteral::debug_string() const { break; } case TYPE_TIME: + case TYPE_TIMEV2: case TYPE_DOUBLE: { out << *(reinterpret_cast(ref.data)); break; diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h index 24cb18e984..8dd9991f18 100644 --- a/be/src/vec/functions/function.h +++ b/be/src/vec/functions/function.h @@ -302,6 +302,17 @@ public: ? ((DataTypeNullable*)get_return_type(arguments).get()) ->get_nested_type() : get_return_type(arguments))) || + // For some date functions such as str_to_date(string, string), return_type will + // be datetimev2 if users enable datev2 but get_return_type(arguments) will still + // return datetime. We need keep backward compatibility here. + (is_date_v2_or_datetime_v2( + return_type->is_nullable() + ? ((DataTypeNullable*)return_type.get())->get_nested_type() + : return_type) && + is_date_or_datetime(get_return_type(arguments)->is_nullable() + ? ((DataTypeNullable*)get_return_type(arguments).get()) + ->get_nested_type() + : get_return_type(arguments))) || (is_decimal(return_type->is_nullable() ? ((DataTypeNullable*)return_type.get())->get_nested_type() : return_type) && diff --git a/be/src/vec/functions/function_convert_tz.cpp b/be/src/vec/functions/function_convert_tz.cpp index 7dc2c1762c..00745d1c6d 100644 --- a/be/src/vec/functions/function_convert_tz.cpp +++ b/be/src/vec/functions/function_convert_tz.cpp @@ -22,7 +22,15 @@ namespace doris::vectorized { void register_function_convert_tz(SimpleFunctionFactory& factory) { - factory.register_function(); + factory.register_function< + FunctionConvertTZ, DataTypeDateTimeV2>, + DataTypeDateTimeV2>>(); + factory.register_function, + DataTypeDateTime>>(); + factory.register_function, DataTypeDateV2>, DataTypeDateV2>>(); + factory.register_function< + FunctionConvertTZ, DataTypeDate>>(); } } // namespace doris::vectorized diff --git a/be/src/vec/functions/function_convert_tz.h b/be/src/vec/functions/function_convert_tz.h index 6a9549188d..fd7c33756c 100644 --- a/be/src/vec/functions/function_convert_tz.h +++ b/be/src/vec/functions/function_convert_tz.h @@ -21,10 +21,69 @@ #include "vec/common/string_ref.h" #include "vec/core/types.h" #include "vec/data_types/data_type_date_time.h" +#include "vec/data_types/data_type_string.h" #include "vec/utils/util.hpp" namespace doris::vectorized { +template +struct ConvertTZImpl { + using ColumnType = std::conditional_t< + std::is_same_v, DateValueType>, ColumnDateV2, + std::conditional_t, DateValueType>, + ColumnDateTimeV2, ColumnDateTime>>; + using NativeType = std::conditional_t< + std::is_same_v, DateValueType>, UInt32, + std::conditional_t, DateValueType>, + UInt64, Int64>>; + using ReturnDateType = std::conditional_t, + VecDateTimeValue, DateV2Value>; + using ReturnNativeType = + std::conditional_t, Int64, UInt64>; + using ReturnColumnType = std::conditional_t, + ColumnDateTime, ColumnDateTimeV2>; + + static void execute(FunctionContext* context, const ColumnType* date_column, + const ColumnString* from_tz_column, const ColumnString* to_tz_column, + ReturnColumnType* result_column, NullMap& result_null_map, + size_t input_rows_count) { + for (size_t i = 0; i < input_rows_count; i++) { + if (result_null_map[i]) { + result_column->insert_default(); + continue; + } + + StringRef from_tz = from_tz_column->get_data_at(i); + StringRef to_tz = to_tz_column->get_data_at(i); + + DateValueType ts_value = + binary_cast(date_column->get_element(i)); + int64_t timestamp; + + if (!ts_value.unix_timestamp(×tamp, from_tz.to_string())) { + result_null_map[i] = true; + result_column->insert_default(); + continue; + } + + ReturnDateType ts_value2; + if (!ts_value2.from_unixtime(timestamp, to_tz.to_string())) { + result_null_map[i] = true; + result_column->insert_default(); + continue; + } + + result_column->insert(binary_cast(ts_value2)); + } + } + + static DataTypes get_variadic_argument_types() { + return {std::make_shared(), std::make_shared(), + std::make_shared()}; + } +}; + +template class FunctionConvertTZ : public IFunction { public: static constexpr auto name = "convert_tz"; @@ -36,7 +95,18 @@ public: size_t get_number_of_arguments() const override { return 3; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - return make_nullable(std::make_shared()); + if constexpr (std::is_same_v || + std::is_same_v) { + return make_nullable(std::make_shared()); + } else { + return make_nullable(std::make_shared()); + } + } + + bool is_variadic() const override { return true; } + + DataTypes get_variadic_argument_types_impl() const override { + return Transform::get_variadic_argument_types(); } bool use_default_implementation_for_constants() const override { return true; } @@ -44,7 +114,6 @@ public: Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) override { - auto result_column = ColumnDateTime::create(); auto result_null_map_column = ColumnUInt8::create(input_rows_count, 0); ColumnPtr argument_columns[3]; @@ -62,51 +131,42 @@ public: } } - execute_straight(context, assert_cast(argument_columns[0].get()), - assert_cast(argument_columns[1].get()), - assert_cast(argument_columns[2].get()), - assert_cast(result_column.get()), - assert_cast(result_null_map_column.get())->get_data(), - input_rows_count); - - block.get_by_position(result).column = - ColumnNullable::create(std::move(result_column), std::move(result_null_map_column)); - return Status::OK(); - } - -private: - void execute_straight(FunctionContext* context, const ColumnDateTime* date_column, - const ColumnString* from_tz_column, const ColumnString* to_tz_column, - ColumnDateTime* result_column, NullMap& result_null_map, - size_t input_rows_count) { - for (size_t i = 0; i < input_rows_count; i++) { - if (result_null_map[i]) { - result_column->insert_default(); - continue; - } - - StringRef from_tz = from_tz_column->get_data_at(i); - StringRef to_tz = to_tz_column->get_data_at(i); - - VecDateTimeValue ts_value = - binary_cast(date_column->get_element(i)); - int64_t timestamp; - - if (!ts_value.unix_timestamp(×tamp, from_tz.to_string())) { - result_null_map[i] = true; - result_column->insert_default(); - continue; - } - - VecDateTimeValue ts_value2; - if (!ts_value2.from_unixtime(timestamp, to_tz.to_string())) { - result_null_map[i] = true; - result_column->insert_default(); - continue; - } - - result_column->insert(binary_cast(ts_value2)); + if constexpr (std::is_same_v || + std::is_same_v) { + auto result_column = ColumnDateTime::create(); + Transform::execute(context, + assert_cast(argument_columns[0].get()), + assert_cast(argument_columns[1].get()), + assert_cast(argument_columns[2].get()), + assert_cast(result_column.get()), + assert_cast(result_null_map_column.get())->get_data(), + input_rows_count); + block.get_by_position(result).column = ColumnNullable::create( + std::move(result_column), std::move(result_null_map_column)); + } else if constexpr (std::is_same_v) { + auto result_column = ColumnDateTimeV2::create(); + Transform::execute(context, assert_cast(argument_columns[0].get()), + assert_cast(argument_columns[1].get()), + assert_cast(argument_columns[2].get()), + assert_cast(result_column.get()), + assert_cast(result_null_map_column.get())->get_data(), + input_rows_count); + block.get_by_position(result).column = ColumnNullable::create( + std::move(result_column), std::move(result_null_map_column)); + } else { + auto result_column = ColumnDateTimeV2::create(); + Transform::execute(context, + assert_cast(argument_columns[0].get()), + assert_cast(argument_columns[1].get()), + assert_cast(argument_columns[2].get()), + assert_cast(result_column.get()), + assert_cast(result_null_map_column.get())->get_data(), + input_rows_count); + block.get_by_position(result).column = ColumnNullable::create( + std::move(result_column), std::move(result_null_map_column)); } + + return Status::OK(); } }; diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h b/be/src/vec/functions/function_date_or_datetime_computation.h index 8915a8dfaf..48d34e22c7 100644 --- a/be/src/vec/functions/function_date_or_datetime_computation.h +++ b/be/src/vec/functions/function_date_or_datetime_computation.h @@ -603,13 +603,29 @@ struct CurrentDateTimeImpl { static constexpr auto name = FunctionName::name; static Status execute(FunctionContext* context, Block& block, size_t result, size_t input_rows_count) { - auto col_to = ColumnVector::create(); - VecDateTimeValue dtv; + WhichDataType which(block.get_by_position(result).type); + if (which.is_date_time_v2()) { + return executeImpl, UInt64>(context, block, result, + input_rows_count); + } else if (which.is_date_v2()) { + return executeImpl, UInt32>(context, block, result, + input_rows_count); + } else { + return executeImpl(context, block, result, input_rows_count); + } + } + + template + static Status executeImpl(FunctionContext* context, Block& block, size_t result, + size_t input_rows_count) { + auto col_to = ColumnVector::create(); + DateValueType dtv; if (dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000, context->impl()->state()->timezone_obj())) { - reinterpret_cast(&dtv)->set_type(TIME_DATETIME); - auto date_packed_int = binary_cast( - *reinterpret_cast(&dtv)); + if constexpr (std::is_same_v) { + reinterpret_cast(&dtv)->set_type(TIME_DATETIME); + } + auto date_packed_int = binary_cast(dtv); for (int i = 0; i < input_rows_count; i++) { col_to->insert_data( const_cast(reinterpret_cast(&date_packed_int)), 0); @@ -705,12 +721,30 @@ struct UtcTimestampImpl { static constexpr auto name = "utc_timestamp"; static Status execute(FunctionContext* context, Block& block, size_t result, size_t input_rows_count) { + WhichDataType which(block.get_by_position(result).type); + if (which.is_date_time_v2()) { + return executeImpl, UInt64>(context, block, result, + input_rows_count); + } else if (which.is_date_v2()) { + return executeImpl, UInt32>(context, block, result, + input_rows_count); + } else { + return executeImpl(context, block, result, input_rows_count); + } + } + + template + static Status executeImpl(FunctionContext* context, Block& block, size_t result, + size_t input_rows_count) { auto col_to = ColumnVector::create(); - VecDateTimeValue dtv; + DateValueType dtv; if (dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000, "+00:00")) { - reinterpret_cast(&dtv)->set_type(TIME_DATETIME); - auto date_packed_int = binary_cast( - *reinterpret_cast(&dtv)); + if constexpr (std::is_same_v) { + reinterpret_cast(&dtv)->set_type(TIME_DATETIME); + } + + auto date_packed_int = + binary_cast(*reinterpret_cast(&dtv)); for (int i = 0; i < input_rows_count; i++) { col_to->insert_data( const_cast(reinterpret_cast(&date_packed_int)), 0); diff --git a/be/src/vec/functions/function_date_or_datetime_to_something.h b/be/src/vec/functions/function_date_or_datetime_to_something.h index 634f6064d1..0762855fb2 100644 --- a/be/src/vec/functions/function_date_or_datetime_to_something.h +++ b/be/src/vec/functions/function_date_or_datetime_to_something.h @@ -91,9 +91,6 @@ public: Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) override { - const IDataType* from_type = block.get_by_position(arguments[0]).type.get(); - WhichDataType which(from_type); - return DateTimeTransformImpl::execute(block, arguments, result, input_rows_count); diff --git a/be/src/vec/functions/function_datetime_floor_ceil.cpp b/be/src/vec/functions/function_datetime_floor_ceil.cpp index 45d1514b49..98519a3e32 100644 --- a/be/src/vec/functions/function_datetime_floor_ceil.cpp +++ b/be/src/vec/functions/function_datetime_floor_ceil.cpp @@ -23,9 +23,19 @@ namespace doris::vectorized { -template +template class FunctionDateTimeFloorCeil : public IFunction { public: + using ReturnDataType = std::conditional_t< + std::is_same_v, DataTypeDateTime, + std::conditional_t>, + DataTypeDateV2, DataTypeDateTimeV2>>; + using NativeType = std::conditional_t< + std::is_same_v, Int64, + std::conditional_t>, UInt32, + UInt64>>; + using DeltaDataType = + std::conditional_t, DataTypeInt32, DataTypeInt64>; static constexpr auto name = Impl::name; static FunctionPtr create() { return std::make_shared(); } @@ -39,43 +49,86 @@ public: bool is_variadic() const override { return true; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - return make_nullable(std::make_shared()); + return make_nullable(std::make_shared()); + } + + DataTypes get_variadic_argument_types_impl() const override { + if constexpr (std::is_same_v && ArgNum == 1) { + return {std::make_shared()}; + } else if constexpr (std::is_same_v> && + ArgNum == 1) { + return {std::make_shared()}; + } else if constexpr (std::is_same_v> && + ArgNum == 1) { + return {std::make_shared()}; + } else if constexpr (std::is_same_v && ArgNum == 2 && + UseDelta) { + return {std::make_shared(), std::make_shared()}; + } else if constexpr (std::is_same_v> && + ArgNum == 2 && UseDelta) { + return {std::make_shared(), std::make_shared()}; + } else if constexpr (std::is_same_v> && + ArgNum == 2 && UseDelta) { + return {std::make_shared(), std::make_shared()}; + } else if constexpr (std::is_same_v && ArgNum == 2 && + !UseDelta) { + return {std::make_shared(), std::make_shared()}; + } else if constexpr (std::is_same_v> && + ArgNum == 2 && !UseDelta) { + return {std::make_shared(), std::make_shared()}; + } else if constexpr (std::is_same_v> && + ArgNum == 2 && !UseDelta) { + return {std::make_shared(), std::make_shared()}; + } else if constexpr (std::is_same_v && ArgNum == 3) { + return {std::make_shared(), std::make_shared(), + std::make_shared()}; + } else if constexpr (std::is_same_v> && + ArgNum == 3) { + return {std::make_shared(), std::make_shared(), + std::make_shared()}; + } else { + return {std::make_shared(), std::make_shared(), + std::make_shared()}; + } } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) override { const ColumnPtr source_col = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); - if (const auto* sources = check_and_get_column>(source_col.get())) { - auto col_to = ColumnVector::create(); + if (const auto* sources = + check_and_get_column>(source_col.get())) { + auto col_to = ColumnVector::create(); col_to->resize(input_rows_count); auto null_map = ColumnVector::create(); null_map->get_data().resize_fill(input_rows_count, false); - if (arguments.size() == 1) { - Impl::vector(sources->get_data(), col_to->get_data(), null_map->get_data()); - } else if (arguments.size() == 2) { + if constexpr (ArgNum == 1) { + Impl::template vector(sources->get_data(), col_to->get_data(), + null_map->get_data()); + } else if constexpr (ArgNum == 2) { const IColumn& delta_column = *block.get_by_position(arguments[1]).column; if (const auto* delta_const_column = typeid_cast(&delta_column)) { if (block.get_by_position(arguments[1]).type->get_type_id() != TypeIndex::Int32) { - Impl::vector_constant(sources->get_data(), - delta_const_column->get_field().get(), - col_to->get_data(), null_map->get_data()); + Impl::template vector_constant( + sources->get_data(), + delta_const_column->get_field().get(), + col_to->get_data(), null_map->get_data()); } else { - Impl::vector_constant(sources->get_data(), - delta_const_column->get_field().get(), - col_to->get_data(), null_map->get_data()); + Impl::template vector_constant_delta( + sources->get_data(), delta_const_column->get_field().get(), + col_to->get_data(), null_map->get_data()); } } else { if (const auto* delta_vec_column0 = - check_and_get_column>(delta_column)) { + check_and_get_column>(delta_column)) { Impl::vector_vector(sources->get_data(), delta_vec_column0->get_data(), col_to->get_data(), null_map->get_data()); } else { const auto* delta_vec_column1 = - check_and_get_column>(delta_column); + check_and_get_column>(delta_column); DCHECK(delta_vec_column1 != nullptr); Impl::vector_vector(sources->get_data(), delta_vec_column1->get_data(), col_to->get_data(), null_map->get_data()); @@ -88,14 +141,14 @@ public: .column->convert_to_full_column_if_const(); const auto arg1_column = - check_and_get_column>(*arg1_column_ptr); + check_and_get_column>(*arg1_column_ptr); const auto arg2_column = - check_and_get_column>(*arg2_column_ptr); + check_and_get_column>(*arg2_column_ptr); DCHECK(arg1_column != nullptr); DCHECK(arg2_column != nullptr); - Impl::vector_vector(sources->get_data(), arg1_column->get_data(), - arg2_column->get_data(), col_to->get_data(), - null_map->get_data()); + Impl::template vector_vector( + sources->get_data(), arg1_column->get_data(), arg2_column->get_data(), + col_to->get_data(), null_map->get_data()); } block.get_by_position(result).column = @@ -113,47 +166,98 @@ template struct FloorCeilImpl { static constexpr auto name = Impl::name; - static void vector(const PaddedPODArray& dates, PaddedPODArray& res, + template + static void vector(const PaddedPODArray& dates, PaddedPODArray& res, NullMap& null_map) { - vector_constant(dates, Int32(1), res, null_map); + vector_constant_delta(dates, Int32(1), res, null_map); } - static void vector_constant(const PaddedPODArray& dates, Int64 origin_date, - PaddedPODArray& res, NullMap& null_map) { + template + static void vector_constant(const PaddedPODArray& dates, NativeType origin_date, + PaddedPODArray& res, NullMap& null_map) { for (int i = 0; i < dates.size(); ++i) { - Impl::time_round(dates[i], Int32(1), origin_date, res[i], null_map[i]); + if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], Int32(1), origin_date, res[i], null_map[i]); + } else if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], Int32(1), origin_date, res[i], null_map[i]); + } else { + Impl::template time_round(dates[i], Int32(1), origin_date, + res[i], null_map[i]); + } } } - static void vector_constant(const PaddedPODArray& dates, Int32 period, - PaddedPODArray& res, NullMap& null_map) { + template + static void vector_constant_delta(const PaddedPODArray& dates, DeltaType period, + PaddedPODArray& res, NullMap& null_map) { for (int i = 0; i < dates.size(); ++i) { - Impl::time_round(dates[i], period, res[i], null_map[i]); + if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], period, res[i], null_map[i]); + } else if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], period, res[i], null_map[i]); + } else { + Impl::template time_round(dates[i], period, res[i], + null_map[i]); + } } } - static void vector_vector(const PaddedPODArray& dates, - const PaddedPODArray& origin_dates, PaddedPODArray& res, - NullMap& null_map) { + template + static void vector_vector(const PaddedPODArray& dates, + const PaddedPODArray& origin_dates, + PaddedPODArray& res, NullMap& null_map) { for (int i = 0; i < dates.size(); ++i) { - Impl::time_round(dates[i], Int32(1), origin_dates[i], res[i], null_map[i]); + if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], Int32(1), origin_dates[i], res[i], null_map[i]); + } else if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], Int32(1), origin_dates[i], res[i], null_map[i]); + } else { + Impl::template time_round( + dates[i], Int32(1), origin_dates[i], res[i], null_map[i]); + } } } - static void vector_vector(const PaddedPODArray& dates, - const PaddedPODArray& periods, PaddedPODArray& res, - NullMap& null_map) { + template + static void vector_vector(const PaddedPODArray& dates, + const PaddedPODArray& periods, + PaddedPODArray& res, NullMap& null_map) { for (int i = 0; i < dates.size(); ++i) { - Impl::time_round(dates[i], periods[i], res[i], null_map[i]); + if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], periods[i], res[i], null_map[i]); + } else if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], periods[i], res[i], null_map[i]); + } else { + Impl::template time_round(dates[i], periods[i], res[i], + null_map[i]); + } } } - static void vector_vector(const PaddedPODArray& dates, - const PaddedPODArray& periods, - const PaddedPODArray& origin_dates, PaddedPODArray& res, - NullMap& null_map) { + template + static void vector_vector(const PaddedPODArray& dates, + const PaddedPODArray& periods, + const PaddedPODArray& origin_dates, + PaddedPODArray& res, NullMap& null_map) { for (int i = 0; i < dates.size(); ++i) { - Impl::time_round(dates[i], periods[i], origin_dates[i], res[i], null_map[i]); + if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], periods[i], origin_dates[i], res[i], null_map[i]); + } else if constexpr (std::is_same_v) { + Impl::template time_round>( + dates[i], periods[i], origin_dates[i], res[i], null_map[i]); + } else { + Impl::template time_round( + dates[i], periods[i], origin_dates[i], res[i], null_map[i]); + } } } }; @@ -166,8 +270,15 @@ struct TimeRound { static constexpr int8_t FLOOR = 0; static constexpr int8_t CEIL = 1; - static void time_round(const doris::vectorized::VecDateTimeValue& ts2, Int32 period, - doris::vectorized::VecDateTimeValue& ts1, UInt8& is_null) { + static constexpr uint32_t MASK_YEAR_FOR_DATEV2 = -1 >> 23; + static constexpr uint32_t MASK_YEAR_MONTH_FOR_DATEV2 = -1 >> 27; + + static constexpr uint64_t MASK_YEAR_FOR_DATETIMEV2 = -1 >> 18; + static constexpr uint64_t MASK_YEAR_MONTH_FOR_DATETIMEV2 = -1 >> 22; + + template + static void time_round(const DateValueType& ts2, Int32 period, DateValueType& ts1, + UInt8& is_null) { if (period < 1) { is_null = true; return; @@ -176,48 +287,123 @@ struct TimeRound { int64_t diff; int64_t trivial_part_ts1; int64_t trivial_part_ts2; - if constexpr (Impl::Unit == YEAR) { - diff = (ts2.year() - ts1.year()); - trivial_part_ts2 = ts2.to_int64() % 10000000000; - trivial_part_ts1 = ts1.to_int64() % 10000000000; - } - if constexpr (Impl::Unit == MONTH) { - diff = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month()); - trivial_part_ts2 = ts2.to_int64() % 100000000; - trivial_part_ts1 = ts1.to_int64() % 100000000; - } - if constexpr (Impl::Unit == MONTH) { - diff = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month()); - trivial_part_ts2 = ts2.to_int64() % 100000000; - trivial_part_ts1 = ts1.to_int64() % 100000000; - } - if constexpr (Impl::Unit == WEEK) { - diff = ts2.daynr() / 7 - ts1.daynr() / 7; - trivial_part_ts2 = ts2.daynr() % 7 * 24 * 3600 + ts2.hour() * 3600 + ts2.minute() * 60 + - ts2.second(); - trivial_part_ts1 = ts1.daynr() % 7 * 24 * 3600 + ts1.hour() * 3600 + ts1.minute() * 60 + - ts1.second(); - } - if constexpr (Impl::Unit == DAY) { - diff = ts2.daynr() - ts1.daynr(); - trivial_part_ts2 = ts2.hour() * 3600 + ts2.minute() * 60 + ts2.second(); - trivial_part_ts1 = ts1.hour() * 3600 + ts1.minute() * 60 + ts1.second(); - } - if constexpr (Impl::Unit == HOUR) { - diff = (ts2.daynr() - ts1.daynr()) * 24 + (ts2.hour() - ts1.hour()); - trivial_part_ts2 = ts2.minute() * 60 + ts2.second(); - trivial_part_ts1 = ts1.minute() * 60 + ts1.second(); - } - if constexpr (Impl::Unit == MINUTE) { - diff = (ts2.daynr() - ts1.daynr()) * 24 * 60 + (ts2.hour() - ts1.hour()) * 60 + - (ts2.minute() - ts1.minute()); - trivial_part_ts2 = ts2.second(); - trivial_part_ts1 = ts1.second(); - } - if constexpr (Impl::Unit == SECOND) { - diff = ts2.second_diff(ts1); - trivial_part_ts1 = 0; - trivial_part_ts2 = 0; + if constexpr (std::is_same_v) { + if constexpr (Impl::Unit == YEAR) { + diff = (ts2.year() - ts1.year()); + trivial_part_ts2 = ts2.to_int64() % 10000000000; + trivial_part_ts1 = ts1.to_int64() % 10000000000; + } + if constexpr (Impl::Unit == MONTH) { + diff = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month()); + trivial_part_ts2 = ts2.to_int64() % 100000000; + trivial_part_ts1 = ts1.to_int64() % 100000000; + } + if constexpr (Impl::Unit == WEEK) { + diff = ts2.daynr() / 7 - ts1.daynr() / 7; + trivial_part_ts2 = ts2.daynr() % 7 * 24 * 3600 + ts2.hour() * 3600 + + ts2.minute() * 60 + ts2.second(); + trivial_part_ts1 = ts1.daynr() % 7 * 24 * 3600 + ts1.hour() * 3600 + + ts1.minute() * 60 + ts1.second(); + } + if constexpr (Impl::Unit == DAY) { + diff = ts2.daynr() - ts1.daynr(); + trivial_part_ts2 = ts2.hour() * 3600 + ts2.minute() * 60 + ts2.second(); + trivial_part_ts1 = ts1.hour() * 3600 + ts1.minute() * 60 + ts1.second(); + } + if constexpr (Impl::Unit == HOUR) { + diff = (ts2.daynr() - ts1.daynr()) * 24 + (ts2.hour() - ts1.hour()); + trivial_part_ts2 = ts2.minute() * 60 + ts2.second(); + trivial_part_ts1 = ts1.minute() * 60 + ts1.second(); + } + if constexpr (Impl::Unit == MINUTE) { + diff = (ts2.daynr() - ts1.daynr()) * 24 * 60 + (ts2.hour() - ts1.hour()) * 60 + + (ts2.minute() - ts1.minute()); + trivial_part_ts2 = ts2.second(); + trivial_part_ts1 = ts1.second(); + } + if constexpr (Impl::Unit == SECOND) { + diff = ts2.second_diff(ts1); + trivial_part_ts1 = 0; + trivial_part_ts2 = 0; + } + } else if constexpr (std::is_same_v>) { + if constexpr (Impl::Unit == YEAR) { + diff = (ts2.year() - ts1.year()); + trivial_part_ts2 = ts2.to_date_int_val() & MASK_YEAR_FOR_DATEV2; + trivial_part_ts1 = ts1.to_date_int_val() & MASK_YEAR_FOR_DATEV2; + } + if constexpr (Impl::Unit == MONTH) { + diff = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month()); + trivial_part_ts2 = ts2.to_date_int_val() & MASK_YEAR_MONTH_FOR_DATEV2; + trivial_part_ts1 = ts1.to_date_int_val() & MASK_YEAR_MONTH_FOR_DATEV2; + } + if constexpr (Impl::Unit == WEEK) { + diff = ts2.daynr() / 7 - ts1.daynr() / 7; + trivial_part_ts2 = ts2.daynr() % 7 * 24 * 3600 + ts2.hour() * 3600 + + ts2.minute() * 60 + ts2.second(); + trivial_part_ts1 = ts1.daynr() % 7 * 24 * 3600 + ts1.hour() * 3600 + + ts1.minute() * 60 + ts1.second(); + } + if constexpr (Impl::Unit == DAY) { + diff = ts2.daynr() - ts1.daynr(); + trivial_part_ts2 = ts2.hour() * 3600 + ts2.minute() * 60 + ts2.second(); + trivial_part_ts1 = ts1.hour() * 3600 + ts1.minute() * 60 + ts1.second(); + } + if constexpr (Impl::Unit == HOUR) { + diff = (ts2.daynr() - ts1.daynr()) * 24 + (ts2.hour() - ts1.hour()); + trivial_part_ts2 = ts2.minute() * 60 + ts2.second(); + trivial_part_ts1 = ts1.minute() * 60 + ts1.second(); + } + if constexpr (Impl::Unit == MINUTE) { + diff = (ts2.daynr() - ts1.daynr()) * 24 * 60 + (ts2.hour() - ts1.hour()) * 60 + + (ts2.minute() - ts1.minute()); + trivial_part_ts2 = ts2.second(); + trivial_part_ts1 = ts1.second(); + } + if constexpr (Impl::Unit == SECOND) { + diff = ts2.second_diff(ts1); + trivial_part_ts1 = 0; + trivial_part_ts2 = 0; + } + } else if constexpr (std::is_same_v>) { + if constexpr (Impl::Unit == YEAR) { + diff = (ts2.year() - ts1.year()); + trivial_part_ts2 = ts2.to_date_int_val() & MASK_YEAR_FOR_DATETIMEV2; + trivial_part_ts1 = ts1.to_date_int_val() & MASK_YEAR_FOR_DATETIMEV2; + } + if constexpr (Impl::Unit == MONTH) { + diff = (ts2.year() - ts1.year()) * 12 + (ts2.month() - ts1.month()); + trivial_part_ts2 = ts2.to_date_int_val() & MASK_YEAR_MONTH_FOR_DATETIMEV2; + trivial_part_ts1 = ts1.to_date_int_val() & MASK_YEAR_MONTH_FOR_DATETIMEV2; + } + if constexpr (Impl::Unit == WEEK) { + diff = ts2.daynr() / 7 - ts1.daynr() / 7; + trivial_part_ts2 = ts2.daynr() % 7 * 24 * 3600 + ts2.hour() * 3600 + + ts2.minute() * 60 + ts2.second(); + trivial_part_ts1 = ts1.daynr() % 7 * 24 * 3600 + ts1.hour() * 3600 + + ts1.minute() * 60 + ts1.second(); + } + if constexpr (Impl::Unit == DAY) { + diff = ts2.daynr() - ts1.daynr(); + trivial_part_ts2 = ts2.hour() * 3600 + ts2.minute() * 60 + ts2.second(); + trivial_part_ts1 = ts1.hour() * 3600 + ts1.minute() * 60 + ts1.second(); + } + if constexpr (Impl::Unit == HOUR) { + diff = (ts2.daynr() - ts1.daynr()) * 24 + (ts2.hour() - ts1.hour()); + trivial_part_ts2 = ts2.minute() * 60 + ts2.second(); + trivial_part_ts1 = ts1.minute() * 60 + ts1.second(); + } + if constexpr (Impl::Unit == MINUTE) { + diff = (ts2.daynr() - ts1.daynr()) * 24 * 60 + (ts2.hour() - ts1.hour()) * 60 + + (ts2.minute() - ts1.minute()); + trivial_part_ts2 = ts2.second(); + trivial_part_ts1 = ts1.second(); + } + if constexpr (Impl::Unit == SECOND) { + diff = ts2.second_diff(ts1); + trivial_part_ts1 = 0; + trivial_part_ts2 = 0; + } } //round down/up to specific time-unit(HOUR/DAY/MONTH...) by increase/decrease diff variable @@ -239,22 +425,24 @@ struct TimeRound { : count); bool is_neg = step < 0; TimeInterval interval(Impl::Unit, is_neg ? -step : step, is_neg); - is_null = !ts1.date_add_interval(interval); + is_null = !ts1.template date_add_interval(interval); return; } - static void time_round(Int64 date, Int32 period, Int64 origin_date, Int64& res, + template + static void time_round(NativeType date, Int32 period, NativeType origin_date, NativeType& res, UInt8& is_null) { res = origin_date; - auto ts2 = binary_cast(date); - auto& ts1 = (doris::vectorized::VecDateTimeValue&)(res); + auto ts2 = binary_cast(date); + auto& ts1 = (DateValueType&)(res); - time_round(ts2, period, ts1, is_null); + TimeRound::template time_round(ts2, period, ts1, is_null); } - static void time_round(Int64 date, Int32 period, Int64& res, UInt8& is_null) { - auto ts2 = binary_cast(date); - auto& ts1 = (doris::vectorized::VecDateTimeValue&)(res); + template + static void time_round(NativeType date, Int32 period, NativeType& res, UInt8& is_null) { + auto ts2 = binary_cast(date); + auto& ts1 = (DateValueType&)(res); if constexpr (Impl::Unit != WEEK) { ts1.from_olap_datetime(FIRST_DAY); } else { @@ -262,17 +450,57 @@ struct TimeRound { ts1.from_olap_datetime(FIRST_SUNDAY); } - time_round(ts2, period, ts1, is_null); + TimeRound::template time_round(ts2, period, ts1, is_null); } }; -#define TIME_ROUND(CLASS, NAME, UNIT, TYPE) \ - struct CLASS { \ - static constexpr auto name = #NAME; \ - static constexpr TimeUnit Unit = UNIT; \ - static constexpr auto Type = TYPE; \ - }; \ - using Function##CLASS = FunctionDateTimeFloorCeil>>; +#define TIME_ROUND_WITH_DELTA_TYPE(CLASS, NAME, UNIT, TYPE, DELTA) \ + using FunctionOneArg##CLASS##DELTA = \ + FunctionDateTimeFloorCeil>, VecDateTimeValue, DELTA, 1, \ + false>; \ + using FunctionTwoArg##CLASS##DELTA = \ + FunctionDateTimeFloorCeil>, VecDateTimeValue, DELTA, 2, \ + false>; \ + using FunctionThreeArg##CLASS##DELTA = \ + FunctionDateTimeFloorCeil>, VecDateTimeValue, DELTA, 3, \ + false>; \ + using FunctionDateV2OneArg##CLASS##DELTA = \ + FunctionDateTimeFloorCeil>, \ + DateV2Value, DELTA, 1, false>; \ + using FunctionDateV2TwoArg##CLASS##DELTA = \ + FunctionDateTimeFloorCeil>, \ + DateV2Value, DELTA, 2, false>; \ + using FunctionDateV2ThreeArg##CLASS##DELTA = \ + FunctionDateTimeFloorCeil>, \ + DateV2Value, DELTA, 3, false>; \ + using FunctionDateTimeV2OneArg##CLASS##DELTA = \ + FunctionDateTimeFloorCeil>, \ + DateV2Value, DELTA, 1, false>; \ + using FunctionDateTimeV2TwoArg##CLASS##DELTA = \ + FunctionDateTimeFloorCeil>, \ + DateV2Value, DELTA, 2, false>; \ + using FunctionDateTimeV2ThreeArg##CLASS##DELTA = \ + FunctionDateTimeFloorCeil>, \ + DateV2Value, DELTA, 3, false>; + +#define TIME_ROUND(CLASS, NAME, UNIT, TYPE) \ + struct CLASS { \ + static constexpr auto name = #NAME; \ + static constexpr TimeUnit Unit = UNIT; \ + static constexpr auto Type = TYPE; \ + }; \ + \ + TIME_ROUND_WITH_DELTA_TYPE(CLASS, NAME, UNIT, TYPE, Int32) \ + TIME_ROUND_WITH_DELTA_TYPE(CLASS, NAME, UNIT, TYPE, Int64) \ + using FunctionDateTimeV2TwoArg##CLASS = \ + FunctionDateTimeFloorCeil>, \ + DateV2Value, Int32, 2, true>; \ + using FunctionDateV2TwoArg##CLASS = \ + FunctionDateTimeFloorCeil>, \ + DateV2Value, Int32, 2, true>; \ + using FunctionDateTimeTwoArg##CLASS = \ + FunctionDateTimeFloorCeil>, VecDateTimeValue, Int32, 2, \ + true>; TIME_ROUND(YearFloor, year_floor, YEAR, false); TIME_ROUND(MonthFloor, month_floor, MONTH, false); @@ -291,20 +519,38 @@ TIME_ROUND(MinuteCeil, minute_ceil, MINUTE, true); TIME_ROUND(SecondCeil, second_ceil, SECOND, true); void register_function_datetime_floor_ceil(SimpleFunctionFactory& factory) { - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); +#define REGISTER_FUNC_WITH_DELTA_TYPE(CLASS, DELTA) \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); \ + factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); - factory.register_function(); +#define REGISTER_FUNC(CLASS) \ + REGISTER_FUNC_WITH_DELTA_TYPE(CLASS, Int32) \ + REGISTER_FUNC_WITH_DELTA_TYPE(CLASS, Int64) + + REGISTER_FUNC(YearFloor); + REGISTER_FUNC(MonthFloor); + REGISTER_FUNC(WeekFloor); + REGISTER_FUNC(DayFloor); + REGISTER_FUNC(HourFloor); + REGISTER_FUNC(MinuteFloor); + REGISTER_FUNC(SecondFloor); + + REGISTER_FUNC(YearCeil); + REGISTER_FUNC(MonthCeil); + REGISTER_FUNC(WeekCeil); + REGISTER_FUNC(DayCeil); + REGISTER_FUNC(HourCeil); + REGISTER_FUNC(MinuteCeil); + REGISTER_FUNC(SecondCeil); } } // namespace doris::vectorized diff --git a/be/src/vec/functions/function_timestamp.cpp b/be/src/vec/functions/function_timestamp.cpp index ea6defb0dd..de06f993d9 100644 --- a/be/src/vec/functions/function_timestamp.cpp +++ b/be/src/vec/functions/function_timestamp.cpp @@ -29,17 +29,70 @@ namespace doris::vectorized { -template struct StrToDate { static constexpr auto name = "str_to_date"; - using ReturnType = DateType; - using ColumnType = ColumnVector; - static void vector_vector(FunctionContext* context, const ColumnString::Chars& ldata, - const ColumnString::Offsets& loffsets, - const ColumnString::Chars& rdata, - const ColumnString::Offsets& roffsets, - PaddedPODArray& res, NullMap& null_map) { + static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) { + auto null_map = ColumnUInt8::create(input_rows_count, 0); + ColumnPtr argument_columns[2]; + + // focus convert const to full column to simply execute logic + // handle + for (int i = 0; i < 2; ++i) { + argument_columns[i] = + block.get_by_position(arguments[i]).column->convert_to_full_column_if_const(); + if (auto* nullable = check_and_get_column(*argument_columns[i])) { + // Danger: Here must dispose the null map data first! Because + // argument_columns[i]=nullable->get_nested_column_ptr(); will release the mem + // of column nullable mem of null map + VectorizedUtils::update_null_map(null_map->get_data(), + nullable->get_null_map_data()); + argument_columns[i] = nullable->get_nested_column_ptr(); + } + } + + auto specific_str_column = assert_cast(argument_columns[0].get()); + auto specific_char_column = assert_cast(argument_columns[1].get()); + + auto& ldata = specific_str_column->get_chars(); + auto& loffsets = specific_str_column->get_offsets(); + + auto& rdata = specific_char_column->get_chars(); + auto& roffsets = specific_char_column->get_offsets(); + + ColumnPtr res = nullptr; + WhichDataType which(remove_nullable(block.get_by_position(result).type)); + if (which.is_date_time_v2()) { + res = ColumnVector::create(); + executeImpl, UInt64>( + context, ldata, loffsets, rdata, roffsets, + static_cast*>(res->assume_mutable().get())->get_data(), + null_map->get_data()); + } else if (which.is_date_v2()) { + res = ColumnVector::create(); + executeImpl, UInt32>( + context, ldata, loffsets, rdata, roffsets, + static_cast*>(res->assume_mutable().get())->get_data(), + null_map->get_data()); + } else { + res = ColumnVector::create(); + executeImpl( + context, ldata, loffsets, rdata, roffsets, + static_cast*>(res->assume_mutable().get())->get_data(), + null_map->get_data()); + } + + block.get_by_position(result).column = + ColumnNullable::create(std::move(res), std::move(null_map)); + return Status::OK(); + } + + template + static void executeImpl(FunctionContext* context, const ColumnString::Chars& ldata, + const ColumnString::Offsets& loffsets, const ColumnString::Chars& rdata, + const ColumnString::Offsets& roffsets, PaddedPODArray& res, + NullMap& null_map) { size_t size = loffsets.size(); res.resize(size); for (size_t i = 0; i < size; ++i) { @@ -49,43 +102,76 @@ struct StrToDate { const char* r_raw_str = reinterpret_cast(&rdata[roffsets[i - 1]]); int r_str_size = roffsets[i] - roffsets[i - 1] - 1; - if constexpr (std::is_same_v || - std::is_same_v) { - auto& ts_val = *reinterpret_cast(&res[i]); - if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) { - null_map[i] = 1; - } + auto& ts_val = *reinterpret_cast(&res[i]); + if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) { + null_map[i] = 1; + } + if constexpr (std::is_same_v) { if (context->impl()->get_return_type().type == doris_udf::FunctionContext::Type::TYPE_DATETIME) { ts_val.to_datetime(); } else { ts_val.cast_to_date(); } - } else { - auto& ts_val = *reinterpret_cast*>(&res[i]); - if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) { - null_map[i] = 1; - } } } } }; -struct NameMakeDate { - static constexpr auto name = "makedate"; -}; - -template struct MakeDateImpl { - using ResultDataType = ResultDateType; - using LeftDataColumnType = ColumnVector; - using RightDataColumnType = ColumnVector; - using ColumnType = ColumnVector; + static constexpr auto name = "makedate"; - static void vector_vector(const PaddedPODArray& ldata, - const PaddedPODArray& rdata, - PaddedPODArray& res, NullMap& null_map) { + static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) { + auto null_map = ColumnUInt8::create(input_rows_count, 0); + DCHECK_EQ(arguments.size(), 2); + ColumnPtr argument_columns[2]; + for (int i = 0; i < 2; ++i) { + argument_columns[i] = + block.get_by_position(arguments[i]).column->convert_to_full_column_if_const(); + if (auto* nullable = check_and_get_column(*argument_columns[i])) { + // Danger: Here must dispose the null map data first! Because + // argument_columns[i]=nullable->get_nested_column_ptr(); will release the mem + // of column nullable mem of null map + VectorizedUtils::update_null_map(null_map->get_data(), + nullable->get_null_map_data()); + argument_columns[i] = nullable->get_nested_column_ptr(); + } + } + + ColumnPtr res = nullptr; + WhichDataType which(remove_nullable(block.get_by_position(result).type)); + if (which.is_date_v2()) { + res = ColumnVector::create(); + executeImpl, UInt32>( + static_cast*>(argument_columns[0].get())->get_data(), + static_cast*>(argument_columns[1].get())->get_data(), + static_cast*>(res->assume_mutable().get())->get_data(), + null_map->get_data()); + } else if (which.is_date_time_v2()) { + res = ColumnVector::create(); + executeImpl, UInt64>( + static_cast*>(argument_columns[0].get())->get_data(), + static_cast*>(argument_columns[1].get())->get_data(), + static_cast*>(res->assume_mutable().get())->get_data(), + null_map->get_data()); + } else { + res = ColumnVector::create(); + executeImpl( + static_cast*>(argument_columns[0].get())->get_data(), + static_cast*>(argument_columns[1].get())->get_data(), + static_cast*>(res->assume_mutable().get())->get_data(), + null_map->get_data()); + } + + block.get_by_position(result).column = + ColumnNullable::create(std::move(res), std::move(null_map)); + return Status::OK(); + } + + template + static void executeImpl(const PaddedPODArray& ldata, const PaddedPODArray& rdata, + PaddedPODArray& res, NullMap& null_map) { auto len = ldata.size(); res.resize(len); @@ -97,10 +183,8 @@ struct MakeDateImpl { continue; } - if constexpr (std::is_same_v || - std::is_same_v) { - auto& res_val = *reinterpret_cast(&res[i]); - + auto& res_val = *reinterpret_cast(&res[i]); + if constexpr (std::is_same_v) { VecDateTimeValue ts_value = VecDateTimeValue(); ts_value.set_time(l, 1, 1, 0, 0, 0); @@ -113,16 +197,15 @@ struct MakeDateImpl { TimeInterval interval(DAY, r - 1, false); res_val = VecDateTimeValue::from_datetime_val(ts_val); - if (!res_val.date_add_interval(interval)) { + if (!res_val.template date_add_interval(interval)) { null_map[i] = 1; continue; } res_val.cast_to_date(); } else { - DateV2Value* value = new (&res[i]) DateV2Value(); - value->set_time(l, 1, 1); + res_val.set_time(l, 1, 1, 0, 0, 0, 0); TimeInterval interval(DAY, r - 1, false); - if (!value->date_add_interval(interval, *value)) { + if (!res_val.template date_add_interval(interval)) { null_map[i] = 1; } } @@ -130,7 +213,6 @@ struct MakeDateImpl { } }; -template class FromDays : public IFunction { public: static constexpr auto name = "from_days"; @@ -146,48 +228,65 @@ public: bool use_default_implementation_for_nulls() const override { return true; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - return make_nullable(std::make_shared()); + return make_nullable(std::make_shared()); } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) override { auto null_map = ColumnUInt8::create(input_rows_count, 0); - auto res_column = ColumnInt64::create(input_rows_count); - auto& res_data = assert_cast(*res_column).get_data(); + ColumnPtr argument_column = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); - auto data_col = assert_cast*>(argument_column.get()); + + ColumnPtr res_column; + WhichDataType which(remove_nullable(block.get_by_position(result).type)); + if (which.is_date()) { + res_column = ColumnInt64::create(input_rows_count); + execute_straight( + input_rows_count, null_map->get_data(), data_col->get_data(), + static_cast*>(res_column->assume_mutable().get()) + ->get_data()); + } else { + res_column = ColumnVector::create(input_rows_count); + execute_straight, UInt32>( + input_rows_count, null_map->get_data(), data_col->get_data(), + static_cast*>(res_column->assume_mutable().get()) + ->get_data()); + } + + block.replace_by_position( + result, ColumnNullable::create(std::move(res_column), std::move(null_map))); + return Status::OK(); + } + +private: + template + void execute_straight(size_t input_rows_count, NullMap& null_map, + const PaddedPODArray& data_col, + PaddedPODArray& res_data) { for (int i = 0; i < input_rows_count; i++) { - if constexpr (std::is_same_v) { - const auto& cur_data = data_col->get_data()[i]; - auto& ts_value = *reinterpret_cast(&res_data[i]); + if constexpr (std::is_same_v) { + const auto& cur_data = data_col[i]; + auto& ts_value = *reinterpret_cast(&res_data[i]); if (!ts_value.from_date_daynr(cur_data)) { - null_map->get_data()[i] = 1; + null_map[i] = 1; continue; } DateTimeVal ts_val; ts_value.to_datetime_val(&ts_val); ts_value = VecDateTimeValue::from_datetime_val(ts_val); } else { - const auto& cur_data = data_col->get_data()[i]; - auto& ts_value = *reinterpret_cast*>(&res_data[i]); + const auto& cur_data = data_col[i]; + auto& ts_value = *reinterpret_cast(&res_data[i]); if (!ts_value.get_date_from_daynr(cur_data)) { - null_map->get_data()[i] = 1; + null_map[i] = 1; } } } - block.replace_by_position( - result, ColumnNullable::create(std::move(res_column), std::move(null_map))); - return Status::OK(); } }; -using FunctionStrToDate = FunctionBinaryStringOperateToNullType>; - -using FunctionMakeDate = FunctionBinaryToNullType; - struct UnixTimeStampImpl { static Int32 trim_timestamp(Int64 timestamp) { if (timestamp < 0 || timestamp > INT_MAX) { @@ -258,7 +357,7 @@ struct UnixTimeStampDateImpl { null_map_data[i] = false; col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } - } else { + } else if constexpr (std::is_same_v) { const DateV2Value& ts_value = reinterpret_cast&>(*source.data); int64_t timestamp; @@ -269,6 +368,17 @@ struct UnixTimeStampDateImpl { null_map_data[i] = false; col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } + } else { + const DateV2Value& ts_value = + reinterpret_cast&>(*source.data); + int64_t timestamp; + if (!ts_value.unix_timestamp(×tamp, + context->impl()->state()->timezone_obj())) { + null_map_data[i] = true; + } else { + null_map_data[i] = false; + col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); + } } } @@ -367,16 +477,45 @@ public: } }; +template +class FunctionOtherTypesToDateType : public IFunction { +public: + static constexpr auto name = Impl::name; + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { return 2; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return make_nullable(std::make_shared()); + } + + bool use_default_implementation_for_constants() const override { return true; } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) override { + return Impl::execute(context, block, arguments, result, input_rows_count); + } +}; + +using FunctionStrToDate = FunctionOtherTypesToDateType; + +using FunctionMakeDate = FunctionOtherTypesToDateType; + void register_function_timestamp(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); - factory.register_function>(); + factory.register_function(); factory.register_function>(); factory.register_function>>(); factory.register_function>>(); + factory.register_function>>(); factory.register_function>>(); factory.register_function>>(); + factory.register_function< + FunctionUnixTimestamp>>(); factory.register_function>(); } diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index 9056e44c46..c9d5ff9d1c 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -2333,7 +2333,8 @@ template uint32_t DateV2Value::year_week(uint8_t mode) const { uint16_t year = 0; // The range of the week in the year_week is 1-53, so the mode WEEK_YEAR is always true. - uint8_t week = calc_week(this->daynr(), this->year(), this->month(), this->day(), mode, &year); + uint8_t week = + calc_week(this->daynr(), this->year(), this->month(), this->day(), mode | 2, &year); // When the mode WEEK_FIRST_WEEKDAY is not set, // the week in which the last three days of the year fall may belong to the following year. if (week == 53 && day() >= 29 && !(mode & 4)) { diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp index a0c7dbb02a..04436dcd68 100644 --- a/be/src/vec/sink/vmysql_result_writer.cpp +++ b/be/src/vec/sink/vmysql_result_writer.cpp @@ -214,7 +214,7 @@ Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr, if constexpr (type == TYPE_DOUBLE) { buf_ret = _buffer.push_double(data[i]); } - if constexpr (type == TYPE_TIME) { + if constexpr (type == TYPE_TIME || type == TYPE_TIMEV2) { buf_ret = _buffer.push_time(data[i]); } if constexpr (type == TYPE_DATETIME) { @@ -502,6 +502,14 @@ Status VMysqlResultWriter::append_block(Block& input_block) { } break; } + case TYPE_TIMEV2: { + if (type_ptr->is_nullable()) { + status = _add_one_column(column_ptr, result); + } else { + status = _add_one_column(column_ptr, result); + } + break; + } case TYPE_STRING: case TYPE_CHAR: case TYPE_VARCHAR: { diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp index 40bcd4fb5f..021965e2b0 100644 --- a/be/test/vec/function/function_time_test.cpp +++ b/be/test/vec/function/function_time_test.cpp @@ -539,7 +539,7 @@ TEST(VTimestampFunctionsTest, convert_tz_test) { {{DATETIME("2019-08-01 13:21:03"), STRING("+08:00"), STRING("America/Los_Angeles")}, str_to_date_time("2019-07-31 22:21:03", true)}}; - check_function(func_name, input_types, data_set); + check_function(func_name, input_types, data_set); } TEST(VTimestampFunctionsTest, weekday_test) { @@ -1817,4 +1817,19 @@ TEST(VTimestampFunctionsTest, seconds_sub_v2_test) { } } +TEST(VTimestampFunctionsTest, convert_tz_v2_test) { + std::string func_name = "convert_tz"; + + InputTypeSet input_types = {TypeIndex::DateTimeV2, TypeIndex::String, TypeIndex::String}; + + DataSet data_set = { + {{DATETIME("2019-08-01 13:21:03"), STRING("Asia/Shanghai"), + STRING("America/Los_Angeles")}, + str_to_datetime_v2("2019-07-31 22:21:03", "%Y-%m-%d %H:%i:%s.%f")}, + {{DATETIME("2019-08-01 13:21:03"), STRING("+08:00"), STRING("America/Los_Angeles")}, + str_to_datetime_v2("2019-07-31 22:21:03", "%Y-%m-%d %H:%i:%s.%f")}}; + + check_function(func_name, input_types, data_set); +} + } // namespace doris::vectorized diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index ae70ca3297..ee65f35469 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -32,6 +32,7 @@ import org.apache.doris.catalog.ScalarFunction; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.util.VectorizedUtil; @@ -1026,10 +1027,12 @@ public class FunctionCallExpr extends Expr { for (int i = 0; i < argTypes.length; ++i) { // For varargs, we must compare with the last type in callArgs.argTypes. int ix = Math.min(args.length - 1, i); - if (!argTypes[i].matchesType(args[ix]) && !( + if (!argTypes[i].matchesType(args[ix]) && Config.use_date_v2_by_default + && !argTypes[i].isDateType() && (args[ix].isDate() || args[ix].isDatetime())) { + uncheckedCastChild(DateLiteral.getDefaultDateType(args[ix]), i); + } else if (!argTypes[i].matchesType(args[ix]) && !( argTypes[i].isDateType() && args[ix].isDateType())) { uncheckedCastChild(args[ix], i); - //if (argTypes[i] != args[ix]) castChild(args[ix], i); } } } @@ -1075,6 +1078,22 @@ public class FunctionCallExpr extends Expr { this.type = fn.getReturnType(); } + Type[] childTypes = collectChildReturnTypes(); + if ((this.type.isDate() || this.type.isDatetime()) && Config.use_date_v2_by_default + && fn.getArgs().length == childTypes.length) { + boolean implicitCastToDate = false; + for (int i = 0; i < fn.getArgs().length; i++) { + implicitCastToDate = Type.canCastTo(childTypes[i], fn.getArgs()[i]); + if (implicitCastToDate) { + break; + } + } + if (implicitCastToDate) { + this.type = DateLiteral.getDefaultDateType(fn.getReturnType()); + fn.setReturnType(DateLiteral.getDefaultDateType(fn.getReturnType())); + } + } + if (this.type.isDecimalV3()) { // DECIMAL need to pass precision and scale to be if (DECIMAL_FUNCTION_SET.contains(fn.getFunctionName().getFunction()) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 63a4cb4a6f..136c97a822 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -367,9 +367,6 @@ visible_functions = [ [['now', 'current_timestamp', 'localtime', 'localtimestamp'], 'DATETIME', [], '_ZN5doris18TimestampFunctions3nowEPN9doris_udf15FunctionContextE', '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], - [['now', 'current_timestamp', 'localtime', 'localtimestamp'], 'DATETIMEV2', [], - '_ZN5doris18TimestampFunctions3nowEPN9doris_udf15FunctionContextE', - '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], [['curtime', 'current_time'], 'TIME', [], '_ZN5doris18TimestampFunctions7curtimeEPN9doris_udf15FunctionContextE', '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], @@ -436,7 +433,7 @@ visible_functions = [ '_ZN5doris18TimestampFunctions6secondEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', '', '', 'vec', 'ALWAYS_NULLABLE'], - [['makedate'], 'DATETIME', ['INT', 'INT'], + [['makedate'], 'DATE', ['INT', 'INT'], '_ZN5doris18TimestampFunctions9make_dateEPN9doris_udf15FunctionContextERKNS1_6IntValES6_', '', '', 'vec', 'ALWAYS_NULLABLE'], [['years_add'], 'DATETIME', ['DATETIME', 'INT'], @@ -739,19 +736,10 @@ visible_functions = [ '_ZN5doris18TimestampFunctions11second_ceilEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_6IntValES6_', '', '', 'vec', 'ALWAYS_NULLABLE'], - [['curdate', 'current_date'], 'DATEV2', [], - '_ZN5doris18TimestampFunctions7curdateEPN9doris_udf15FunctionContextE', - '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], - [['utc_timestamp'], 'DATETIMEV2', [], - '_ZN5doris18TimestampFunctions13utc_timestampEPN9doris_udf15FunctionContextE', - '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], [['timestamp'], 'DATETIMEV2', ['DATETIMEV2'], '_ZN5doris18TimestampFunctions9timestampEPN9doris_udf15FunctionContextERKNS1_11DateTimeV2ValE', '', '', 'vec', 'ALWAYS_NULLABLE'], - [['from_days'], 'DATEV2', ['INT'], - '_ZN5doris18TimestampFunctions9from_daysEPN9doris_udf15FunctionContextERKNS1_6IntValE', - '', '', 'vec', 'ALWAYS_NULLABLE'], [['to_days'], 'INT', ['DATEV2'], '_ZN5doris18TimestampFunctions7to_daysEPN9doris_udf15FunctionContextERKNS1_11DateTimeV2ValE', '', '', 'vec', 'ALWAYS_NULLABLE'], @@ -848,9 +836,6 @@ visible_functions = [ '_ZN5doris18TimestampFunctions6secondEPN9doris_udf15FunctionContextERKNS1_11DateV2ValE', '', '', 'vec', 'ALWAYS_NULLABLE'], - [['makedate'], 'DATETIMEV2', ['INT', 'INT'], - '_ZN5doris18TimestampFunctions9make_dateEPN9doris_udf15FunctionContextERKNS1_6IntValES6_', - '', '', 'vec', 'ALWAYS_NULLABLE'], [['years_add'], 'DATETIMEV2', ['DATETIMEV2', 'INT'], '_ZN5doris18TimestampFunctions9years_addEPN9doris_udf' '15FunctionContextERKNS1_11DateTimeV2ValERKNS1_6IntValE', @@ -916,9 +901,6 @@ visible_functions = [ '15FunctionContextERKNS1_11DateTimeV2ValERKNS1_6IntValE', '', '', '', ''], - [['makedate'], 'DATEV2', ['INT', 'INT'], - '_ZN5doris18TimestampFunctions9make_dateEPN9doris_udf15FunctionContextERKNS1_6IntValES6_', - '', '', 'vec', 'ALWAYS_NULLABLE'], [['years_add'], 'DATEV2', ['DATEV2', 'INT'], '_ZN5doris18TimestampFunctions9years_addEPN9doris_udf' '15FunctionContextERKNS1_11DateTimeV2ValERKNS1_6IntValE', @@ -1026,12 +1008,6 @@ visible_functions = [ '_ZN5doris18TimestampFunctions9time_diffEPN9doris_udf' '15FunctionContextERKNS1_11DateTimeV2ValES6_', '', '', 'vec', 'ALWAYS_NULLABLE'], - [['str_to_date'], 'DATETIMEV2', ['VARCHAR', 'VARCHAR'], - '_ZN5doris18TimestampFunctions11str_to_dateEPN9doris_udf' - '15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', 'ALWAYS_NULLABLE'], - [['str_to_date'], 'DATETIMEV2', ['STRING', 'STRING'], - '_ZN5doris18TimestampFunctions11str_to_dateEPN9doris_udf' - '15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', 'ALWAYS_NULLABLE'], [['date_format'], 'VARCHAR', ['DATETIMEV2', 'VARCHAR'], '_ZN5doris18TimestampFunctions11date_formatEPN9doris_udf' '15FunctionContextERKNS1_11DateTimeV2ValERKNS1_9StringValE',