// Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common/status.h" #include "runtime/decimalv2_value.h" #include "runtime/define_primitive_type.h" #include "runtime/runtime_state.h" #include "runtime/types.h" #include "udf/udf.h" #include "util/binary_cast.hpp" #include "util/datetype_cast.hpp" #include "util/time.h" #include "util/time_lut.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/columns/column.h" #include "vec/columns/column_const.h" #include "vec/columns/column_nullable.h" #include "vec/columns/column_string.h" #include "vec/columns/column_vector.h" #include "vec/columns/columns_number.h" #include "vec/common/assert_cast.h" #include "vec/common/pod_array_fwd.h" #include "vec/common/string_ref.h" #include "vec/core/block.h" #include "vec/core/column_numbers.h" #include "vec/core/column_with_type_and_name.h" #include "vec/core/columns_with_type_and_name.h" #include "vec/core/types.h" #include "vec/data_types/data_type.h" #include "vec/data_types/data_type_date.h" #include "vec/data_types/data_type_date_time.h" #include "vec/data_types/data_type_decimal.h" #include "vec/data_types/data_type_nullable.h" #include "vec/data_types/data_type_number.h" #include "vec/data_types/data_type_string.h" #include "vec/data_types/data_type_time_v2.h" #include "vec/functions/function.h" #include "vec/functions/simple_function_factory.h" #include "vec/runtime/vdatetime_value.h" #include "vec/utils/util.hpp" namespace doris::vectorized { template struct StrToDate { static constexpr auto name = "str_to_date"; static bool is_variadic() { return false; } static DataTypes get_variadic_argument_types() { return {std::make_shared(), std::make_shared()}; } static DataTypePtr get_return_type_impl(const DataTypes& arguments) { if constexpr (IsDataTypeDateTimeV2) { // max scale return make_nullable(std::make_shared(6)); } return make_nullable(std::make_shared()); } static StringRef rewrite_specific_format(const char* raw_str, size_t str_size) { const static std::string specific_format_strs[3] = {"yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}; const static std::string specific_format_rewrite[3] = {"%Y%m%d", "%Y-%m-%d", "%Y-%m-%d %H:%i:%s"}; for (int i = 0; i < 3; i++) { const StringRef specific_format {specific_format_strs[i].data(), specific_format_strs[i].size()}; if (specific_format == StringRef {raw_str, str_size}) { return {specific_format_rewrite[i].data(), specific_format_rewrite[i].size()}; } } return {raw_str, str_size}; } static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) { auto null_map = ColumnUInt8::create(input_rows_count, 0); const auto& col0 = block.get_by_position(arguments[0]).column; bool col_const[2] = {is_column_const(*col0)}; ColumnPtr argument_columns[2] = { col_const[0] ? static_cast(*col0).convert_to_full_column() : col0}; check_set_nullable(argument_columns[0], null_map, col_const[0]); //TODO: when we set default implementation for nullable, the check_set_nullable for arguments is useless. consider to remove it. std::tie(argument_columns[1], col_const[1]) = unpack_if_const(block.get_by_position(arguments[1]).column); check_set_nullable(argument_columns[1], null_map, col_const[1]); auto specific_str_column = assert_cast(argument_columns[0].get()); auto specific_char_column = assert_cast(argument_columns[1].get()); auto& ldata = specific_str_column->get_chars(); auto& loffsets = specific_str_column->get_offsets(); auto& rdata = specific_char_column->get_chars(); auto& roffsets = specific_char_column->get_offsets(); // Because of we cant distinguish by return_type when we find function. so the return_type may NOT be same with real return type // which decided by FE. that's found by which. ColumnPtr res = nullptr; WhichDataType which(remove_nullable(block.get_by_position(result).type)); if (which.is_date_time_v2()) { res = ColumnDateTimeV2::create(); if (col_const[1]) { execute_impl_const_right( context, ldata, loffsets, specific_char_column->get_data_at(0), static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } else { execute_impl( context, ldata, loffsets, rdata, roffsets, static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } } else if (which.is_date_v2()) { res = ColumnDateV2::create(); if (col_const[1]) { execute_impl_const_right( context, ldata, loffsets, specific_char_column->get_data_at(0), static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } else { execute_impl( context, ldata, loffsets, rdata, roffsets, static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } } else { res = ColumnDateTime::create(); if (col_const[1]) { execute_impl_const_right( context, ldata, loffsets, specific_char_column->get_data_at(0), static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } else { execute_impl( context, ldata, loffsets, rdata, roffsets, static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } } block.get_by_position(result).column = ColumnNullable::create(res, std::move(null_map)); return Status::OK(); } private: template , typename NativeType = date_cast::TypeToColumnV> static void execute_impl(FunctionContext* context, const ColumnString::Chars& ldata, const ColumnString::Offsets& loffsets, const ColumnString::Chars& rdata, const ColumnString::Offsets& roffsets, PaddedPODArray& res, NullMap& null_map) { size_t size = loffsets.size(); res.resize(size); for (size_t i = 0; i < size; ++i) { const char* l_raw_str = reinterpret_cast(&ldata[loffsets[i - 1]]); size_t l_str_size = loffsets[i] - loffsets[i - 1]; const char* r_raw_str = reinterpret_cast(&rdata[roffsets[i - 1]]); size_t r_str_size = roffsets[i] - roffsets[i - 1]; const StringRef format_str = rewrite_specific_format(r_raw_str, r_str_size); _execute_inner_loop(l_raw_str, l_str_size, format_str.data, format_str.size, context, res, null_map, i); } } template , typename NativeType = date_cast::TypeToColumnV> static void execute_impl_const_right(FunctionContext* context, const ColumnString::Chars& ldata, const ColumnString::Offsets& loffsets, const StringRef& rdata, PaddedPODArray& res, NullMap& null_map) { size_t size = loffsets.size(); res.resize(size); const StringRef format_str = rewrite_specific_format(rdata.data, rdata.size); for (size_t i = 0; i < size; ++i) { const char* l_raw_str = reinterpret_cast(&ldata[loffsets[i - 1]]); size_t l_str_size = loffsets[i] - loffsets[i - 1]; _execute_inner_loop(l_raw_str, l_str_size, format_str.data, format_str.size, context, res, null_map, i); } } template static void _execute_inner_loop(const char* l_raw_str, size_t l_str_size, const char* r_raw_str, size_t r_str_size, FunctionContext* context, PaddedPODArray& res, NullMap& null_map, size_t index) { auto& ts_val = *reinterpret_cast(&res[index]); if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) { null_map[index] = 1; } else { if constexpr (std::is_same_v) { if (context->get_return_type().type == doris::PrimitiveType::TYPE_DATETIME) { ts_val.to_datetime(); } else { ts_val.cast_to_date(); } } } } }; struct MakeDateImpl { static constexpr auto name = "makedate"; static bool is_variadic() { return false; } static DataTypes get_variadic_argument_types() { return {}; } static DataTypePtr get_return_type_impl(const DataTypes& arguments) { return make_nullable(std::make_shared()); } static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) { auto null_map = ColumnUInt8::create(input_rows_count, 0); DCHECK_EQ(arguments.size(), 2); const auto& col0 = block.get_by_position(arguments[0]).column; bool col_const[2] = {is_column_const(*col0)}; ColumnPtr argument_columns[2] = { col_const[0] ? static_cast(*col0).convert_to_full_column() : col0}; check_set_nullable(argument_columns[0], null_map, col_const[0]); std::tie(argument_columns[1], col_const[1]) = unpack_if_const(block.get_by_position(arguments[1]).column); check_set_nullable(argument_columns[1], null_map, col_const[1]); ColumnPtr res = nullptr; WhichDataType which(remove_nullable(block.get_by_position(result).type)); if (which.is_date_v2()) { res = ColumnDateV2::create(); if (col_const[1]) { execute_impl_right_const( static_cast*>(argument_columns[0].get()) ->get_data(), static_cast*>(argument_columns[1].get()) ->get_element(0), static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } else { execute_impl( static_cast*>(argument_columns[0].get()) ->get_data(), static_cast*>(argument_columns[1].get()) ->get_data(), static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } } else if (which.is_date_time_v2()) { res = ColumnDateTimeV2::create(); if (col_const[1]) { execute_impl_right_const( static_cast*>(argument_columns[0].get()) ->get_data(), static_cast*>(argument_columns[1].get()) ->get_element(0), static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } else { execute_impl( static_cast*>(argument_columns[0].get()) ->get_data(), static_cast*>(argument_columns[1].get()) ->get_data(), static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } } else { res = ColumnDateTime::create(); if (col_const[1]) { execute_impl_right_const( static_cast*>(argument_columns[0].get()) ->get_data(), static_cast*>(argument_columns[1].get()) ->get_element(0), static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } else { execute_impl( static_cast*>(argument_columns[0].get()) ->get_data(), static_cast*>(argument_columns[1].get()) ->get_data(), static_cast(res->assume_mutable().get())->get_data(), null_map->get_data()); } } block.get_by_position(result).column = ColumnNullable::create(res, std::move(null_map)); return Status::OK(); } private: template , typename ReturnType = date_cast::TypeToColumnV> static void execute_impl(const PaddedPODArray& ldata, const PaddedPODArray& rdata, PaddedPODArray& res, NullMap& null_map) { auto len = ldata.size(); res.resize(len); for (size_t i = 0; i < len; ++i) { const auto& l = ldata[i]; const auto& r = rdata[i]; if (r <= 0 || l < 0 || l > 9999) { null_map[i] = 1; continue; } _execute_inner_loop(l, r, res, null_map, i); } } template , typename ReturnType = date_cast::TypeToColumnV> static void execute_impl_right_const(const PaddedPODArray& ldata, Int32 rdata, PaddedPODArray& res, NullMap& null_map) { auto len = ldata.size(); res.resize(len); const auto& r = rdata; for (size_t i = 0; i < len; ++i) { const auto& l = ldata[i]; if (r <= 0 || l < 0 || l > 9999) { null_map[i] = 1; continue; } _execute_inner_loop(l, r, res, null_map, i); } } template static void _execute_inner_loop(const int& l, const int& r, PaddedPODArray& res, NullMap& null_map, size_t index) { auto& res_val = *reinterpret_cast(&res[index]); if constexpr (std::is_same_v) { VecDateTimeValue ts_value = VecDateTimeValue(); ts_value.set_time(l, 1, 1, 0, 0, 0); TimeInterval interval(DAY, r - 1, false); res_val = ts_value; if (!res_val.template date_add_interval(interval)) { null_map[index] = 1; return; } res_val.cast_to_date(); } else { res_val.set_time(l, 1, 1, 0, 0, 0, 0); TimeInterval interval(DAY, r - 1, false); if (!res_val.template date_add_interval(interval)) { null_map[index] = 1; } } } }; struct DateTruncState { using Callback_function = std::function; Callback_function callback_function; }; template struct DateTrunc { static constexpr auto name = "date_trunc"; using ColumnType = date_cast::TypeToColumnV; using DateValueType = date_cast::TypeToValueTypeV; using ArgType = date_cast::ValueTypeOfColumnV; static bool is_variadic() { return true; } static DataTypes get_variadic_argument_types() { return {std::make_shared(), std::make_shared()}; } static DataTypePtr get_return_type_impl(const DataTypes& arguments) { return make_nullable(std::make_shared()); } static Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) { if (scope != FunctionContext::THREAD_LOCAL) { return Status::OK(); } if (!context->is_col_constant(1)) { return Status::InvalidArgument( "date_trunc function of time unit argument must be constant."); } const auto& data_str = context->get_constant_col(1)->column_ptr->get_data_at(0); std::string lower_str(data_str.data, data_str.size); std::transform(lower_str.begin(), lower_str.end(), lower_str.begin(), [](unsigned char c) { return std::tolower(c); }); std::shared_ptr state = std::make_shared(); if (std::strncmp("year", lower_str.data(), 4) == 0) { state->callback_function = &execute_impl_right_const; } else if (std::strncmp("quarter", lower_str.data(), 7) == 0) { state->callback_function = &execute_impl_right_const; } else if (std::strncmp("month", lower_str.data(), 5) == 0) { state->callback_function = &execute_impl_right_const; } else if (std::strncmp("week", lower_str.data(), 4) == 0) { state->callback_function = &execute_impl_right_const; } else if (std::strncmp("day", lower_str.data(), 3) == 0) { state->callback_function = &execute_impl_right_const; } else if (std::strncmp("hour", lower_str.data(), 4) == 0) { state->callback_function = &execute_impl_right_const; } else if (std::strncmp("minute", lower_str.data(), 6) == 0) { state->callback_function = &execute_impl_right_const; } else if (std::strncmp("second", lower_str.data(), 6) == 0) { state->callback_function = &execute_impl_right_const; } else { return Status::RuntimeError( "Illegal second argument column of function date_trunc. now only support " "[second,minute,hour,day,week,month,quarter,year]"); } context->set_function_state(scope, state); return Status::OK(); } static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) { DCHECK_EQ(arguments.size(), 2); auto null_map = ColumnUInt8::create(input_rows_count, 0); const auto& datetime_column = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); ColumnPtr res = ColumnType::create(input_rows_count); auto* state = reinterpret_cast( context->get_function_state(FunctionContext::THREAD_LOCAL)); DCHECK(state != nullptr); state->callback_function(datetime_column, res, null_map->get_data(), input_rows_count); block.get_by_position(result).column = ColumnNullable::create(res, std::move(null_map)); return Status::OK(); } private: template static void execute_impl_right_const(const ColumnPtr& datetime_column, ColumnPtr& result_column, NullMap& null_map, size_t input_rows_count) { auto& data = static_cast(datetime_column.get())->get_data(); auto& res = static_cast(result_column->assume_mutable().get())->get_data(); for (size_t i = 0; i < input_rows_count; ++i) { auto dt = binary_cast(data[i]); null_map[i] = !dt.template datetime_trunc(); res[i] = binary_cast(dt); } } }; class FromDays : public IFunction { public: static constexpr auto name = "from_days"; static FunctionPtr create() { return std::make_shared(); } String get_name() const override { return name; } size_t get_number_of_arguments() const override { return 1; } bool use_default_implementation_for_nulls() const override { return true; } DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { return make_nullable(std::make_shared()); } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const override { auto null_map = ColumnUInt8::create(input_rows_count, 0); ColumnPtr& argument_column = block.get_by_position(arguments[0]).column; auto data_col = assert_cast*>(argument_column.get()); ColumnPtr res_column; WhichDataType which(remove_nullable(block.get_by_position(result).type)); if (which.is_date()) { res_column = ColumnInt64::create(input_rows_count); execute_straight( input_rows_count, null_map->get_data(), data_col->get_data(), static_cast(res_column->assume_mutable().get())->get_data()); } else { res_column = ColumnDateV2::create(input_rows_count); execute_straight, UInt32>( input_rows_count, null_map->get_data(), data_col->get_data(), static_cast(res_column->assume_mutable().get())->get_data()); } block.replace_by_position( result, ColumnNullable::create(std::move(res_column), std::move(null_map))); return Status::OK(); } private: template void execute_straight(size_t input_rows_count, NullMap& null_map, const PaddedPODArray& data_col, PaddedPODArray& res_data) const { for (int i = 0; i < input_rows_count; i++) { if constexpr (std::is_same_v) { const auto& cur_data = data_col[i]; auto& ts_value = *reinterpret_cast(&res_data[i]); if (!ts_value.from_date_daynr(cur_data)) { null_map[i] = 1; continue; } } else { const auto& cur_data = data_col[i]; auto& ts_value = *reinterpret_cast(&res_data[i]); if (!ts_value.get_date_from_daynr(cur_data)) { null_map[i] = 1; } } } } }; struct UnixTimeStampImpl { static Int32 trim_timestamp(Int64 timestamp) { if (timestamp < 0 || timestamp > INT_MAX) { timestamp = 0; } return (Int32)timestamp; } static std::pair trim_timestamp(std::pair timestamp) { if (timestamp.first < 0 || timestamp.first > INT_MAX) { return {0, 0}; } return std::make_pair((Int32)timestamp.first, (Int32)timestamp.second); } static DataTypes get_variadic_argument_types() { return {}; } static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) { return std::make_shared(); } static Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) { auto col_result = ColumnVector::create(); col_result->resize(1); col_result->get_data()[0] = context->state()->timestamp_ms() / 1000; auto col_const = ColumnConst::create(std::move(col_result), input_rows_count); block.replace_by_position(result, std::move(col_const)); return Status::OK(); } }; template struct UnixTimeStampDateImpl { static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) { if constexpr (std::is_same_v) { if (arguments[0].type->is_nullable()) { UInt32 scale = static_cast(arguments[0].type.get()) ->get_nested_type() ->get_scale(); return make_nullable( std::make_shared>(10 + scale, scale)); } UInt32 scale = arguments[0].type->get_scale(); return std::make_shared>(10 + scale, scale); } else { if (arguments[0].type->is_nullable()) { return make_nullable(std::make_shared()); } return std::make_shared(); } } static Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) { const ColumnPtr& col = block.get_by_position(arguments[0]).column; DCHECK(!col->is_nullable()); if constexpr (std::is_same_v || std::is_same_v) { const auto* col_source = assert_cast(col.get()); auto col_result = ColumnVector::create(); auto& col_result_data = col_result->get_data(); col_result->resize(input_rows_count); for (int i = 0; i < input_rows_count; i++) { StringRef source = col_source->get_data_at(i); const auto& ts_value = reinterpret_cast(*source.data); int64_t timestamp {}; ts_value.unix_timestamp(×tamp, context->state()->timezone_obj()); col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } block.replace_by_position(result, std::move(col_result)); } else if constexpr (std::is_same_v) { const auto* col_source = assert_cast(col.get()); auto col_result = ColumnVector::create(); auto& col_result_data = col_result->get_data(); col_result->resize(input_rows_count); for (int i = 0; i < input_rows_count; i++) { StringRef source = col_source->get_data_at(i); const auto& ts_value = reinterpret_cast&>(*source.data); int64_t timestamp {}; const auto valid = ts_value.unix_timestamp(×tamp, context->state()->timezone_obj()); DCHECK(valid); col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } block.replace_by_position(result, std::move(col_result)); } else { // DatetimeV2 const auto* col_source = assert_cast(col.get()); UInt32 scale = block.get_by_position(arguments[0]).type->get_scale(); auto col_result = ColumnDecimal::create(input_rows_count, scale); auto& col_result_data = col_result->get_data(); col_result->resize(input_rows_count); for (int i = 0; i < input_rows_count; i++) { StringRef source = col_source->get_data_at(i); const auto& ts_value = reinterpret_cast&>(*source.data); std::pair timestamp {}; const auto valid = ts_value.unix_timestamp(×tamp, context->state()->timezone_obj()); DCHECK(valid); auto& [sec, ms] = timestamp; sec = UnixTimeStampImpl::trim_timestamp(sec); auto ms_str = std::to_string(ms).substr(0, scale); if (ms_str.empty()) { ms_str = "0"; } col_result_data[i] = Decimal64::from_int_frac(sec, std::stoll(ms_str), scale).value; } block.replace_by_position(result, std::move(col_result)); } return Status::OK(); } }; template struct UnixTimeStampDatetimeImpl : public UnixTimeStampDateImpl { static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } }; template struct UnixTimeStampDateImplOld { static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) { RETURN_REAL_TYPE_FOR_DATEV2_FUNCTION(DataTypeInt32); } static Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) { const ColumnPtr& col_source = block.get_by_position(arguments[0]).column; auto col_result = ColumnVector::create(); auto null_map = ColumnVector::create(); auto& col_result_data = col_result->get_data(); col_result->resize(input_rows_count); if constexpr (std::is_same_v) { null_map->resize(input_rows_count); auto& null_map_data = null_map->get_data(); for (int i = 0; i < input_rows_count; i++) { if (col_source->is_null_at(i)) { null_map_data[i] = true; continue; } StringRef source = col_source->get_data_at(i); const VecDateTimeValue& ts_value = reinterpret_cast(*source.data); int64_t timestamp {}; if (!ts_value.unix_timestamp(×tamp, context->state()->timezone_obj())) { null_map_data[i] = true; } else { null_map_data[i] = false; col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } } block.replace_by_position( result, ColumnNullable::create(std::move(col_result), std::move(null_map))); } else if constexpr (std::is_same_v) { const auto is_nullable = block.get_by_position(arguments[0]).type->is_nullable(); if (is_nullable) { null_map->resize(input_rows_count); auto& null_map_data = null_map->get_data(); for (int i = 0; i < input_rows_count; i++) { if (col_source->is_null_at(i)) { DCHECK(is_nullable); null_map_data[i] = true; continue; } StringRef source = col_source->get_data_at(i); const DateV2Value& ts_value = reinterpret_cast&>(*source.data); int64_t timestamp {}; if (!ts_value.unix_timestamp(×tamp, context->state()->timezone_obj())) { null_map_data[i] = true; } else { null_map_data[i] = false; col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } } block.replace_by_position( result, ColumnNullable::create(std::move(col_result), std::move(null_map))); } else { for (int i = 0; i < input_rows_count; i++) { DCHECK(!col_source->is_null_at(i)); StringRef source = col_source->get_data_at(i); const DateV2Value& ts_value = reinterpret_cast&>(*source.data); int64_t timestamp {}; const auto valid = ts_value.unix_timestamp(×tamp, context->state()->timezone_obj()); DCHECK(valid); col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } block.replace_by_position(result, std::move(col_result)); } } else { const auto is_nullable = block.get_by_position(arguments[0]).type->is_nullable(); if (is_nullable) { null_map->resize(input_rows_count); auto& null_map_data = null_map->get_data(); for (int i = 0; i < input_rows_count; i++) { if (col_source->is_null_at(i)) { DCHECK(is_nullable); null_map_data[i] = true; continue; } StringRef source = col_source->get_data_at(i); const DateV2Value& ts_value = reinterpret_cast&>(*source.data); int64_t timestamp {}; if (!ts_value.unix_timestamp(×tamp, context->state()->timezone_obj())) { null_map_data[i] = true; } else { null_map_data[i] = false; col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } } block.replace_by_position( result, ColumnNullable::create(std::move(col_result), std::move(null_map))); } else { for (int i = 0; i < input_rows_count; i++) { DCHECK(!col_source->is_null_at(i)); StringRef source = col_source->get_data_at(i); const DateV2Value& ts_value = reinterpret_cast&>(*source.data); int64_t timestamp {}; const auto valid = ts_value.unix_timestamp(×tamp, context->state()->timezone_obj()); DCHECK(valid); col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } block.replace_by_position(result, std::move(col_result)); } } return Status::OK(); } }; template struct UnixTimeStampDatetimeImplOld : public UnixTimeStampDateImplOld { static DataTypes get_variadic_argument_types() { return {std::make_shared()}; } }; // This impl doesn't use default impl to deal null value. struct UnixTimeStampStrImpl { static DataTypes get_variadic_argument_types() { return {std::make_shared(), std::make_shared()}; } static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) { return make_nullable(std::make_shared>(16, 6)); } static Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) { ColumnPtr col_left = nullptr, col_right = nullptr; bool source_const = false, format_const = false; std::tie(col_left, source_const) = unpack_if_const(block.get_by_position(arguments[0]).column); std::tie(col_right, format_const) = unpack_if_const(block.get_by_position(arguments[1]).column); auto col_result = ColumnDecimal::create(input_rows_count, 6); auto null_map = ColumnVector::create(input_rows_count); auto& col_result_data = col_result->get_data(); auto& null_map_data = null_map->get_data(); check_set_nullable(col_left, null_map, source_const); check_set_nullable(col_right, null_map, format_const); const auto* col_source = assert_cast(col_left.get()); const auto* col_format = assert_cast(col_right.get()); for (int i = 0; i < input_rows_count; i++) { StringRef source = col_source->get_data_at(index_check_const(i, source_const)); StringRef fmt = col_format->get_data_at(index_check_const(i, format_const)); DateV2Value ts_value; if (!ts_value.from_date_format_str(fmt.data, fmt.size, source.data, source.size)) { null_map_data[i] = true; continue; } std::pair timestamp {}; if (!ts_value.unix_timestamp(×tamp, context->state()->timezone_obj())) { null_map_data[i] = true; // impossible now } else { null_map_data[i] = false; auto [sec, ms] = UnixTimeStampImpl::trim_timestamp(timestamp); // trailing ms auto ms_str = std::to_string(ms).substr(0, 6); if (ms_str.empty()) { ms_str = "0"; } col_result_data[i] = Decimal64::from_int_frac(sec, std::stoll(ms_str), 6).value; } } block.replace_by_position( result, ColumnNullable::create(std::move(col_result), std::move(null_map))); return Status::OK(); } }; struct UnixTimeStampStrImplOld { static DataTypes get_variadic_argument_types() { return {std::make_shared(), std::make_shared()}; } static DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) { return make_nullable(std::make_shared()); } static Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) { const ColumnPtr col_source = block.get_by_position(arguments[0]).column; const ColumnPtr col_format = block.get_by_position(arguments[1]).column; auto col_result = ColumnVector::create(); auto null_map = ColumnVector::create(); col_result->resize(input_rows_count); null_map->resize(input_rows_count); auto& col_result_data = col_result->get_data(); auto& null_map_data = null_map->get_data(); for (int i = 0; i < input_rows_count; i++) { if (col_source->is_null_at(i) || col_format->is_null_at(i)) { null_map_data[i] = true; continue; } StringRef source = col_source->get_data_at(i); StringRef fmt = col_format->get_data_at(i); VecDateTimeValue ts_value; if (!ts_value.from_date_format_str(fmt.data, fmt.size, source.data, source.size)) { null_map_data[i] = true; continue; } int64_t timestamp {}; if (!ts_value.unix_timestamp(×tamp, context->state()->timezone_obj())) { null_map_data[i] = true; } else { null_map_data[i] = false; col_result_data[i] = UnixTimeStampImpl::trim_timestamp(timestamp); } } block.replace_by_position( result, ColumnNullable::create(std::move(col_result), std::move(null_map))); return Status::OK(); } }; template class FunctionUnixTimestamp : public IFunction { public: static constexpr auto name = "unix_timestamp"; static FunctionPtr create() { return std::make_shared>(); } String get_name() const override { return name; } size_t get_number_of_arguments() const override { return get_variadic_argument_types_impl().size(); } DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { return Impl::get_return_type_impl(arguments); } DataTypes get_variadic_argument_types_impl() const override { return Impl::get_variadic_argument_types(); } bool use_default_implementation_for_nulls() const override { return !static_cast(std::is_same_v); } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const override { return Impl::execute_impl(context, block, arguments, result, input_rows_count); } }; struct MicroSec { static constexpr auto name = "microsecond_timestamp"; static constexpr Int64 ratio = 1000000; }; struct MilliSec { static constexpr auto name = "millisecond_timestamp"; static constexpr Int64 ratio = 1000; }; struct Sec { static constexpr auto name = "second_timestamp"; static constexpr Int64 ratio = 1; }; template class DateTimeToTimestamp : public IFunction { public: using ReturnType = Int64; static constexpr Int64 ratio_to_micro = (1000 * 1000) / Impl::ratio; static constexpr auto name = Impl::name; static FunctionPtr create() { return std::make_shared>(); } String get_name() const override { return name; } size_t get_number_of_arguments() const override { return 1; } DataTypePtr get_return_type_impl(const ColumnsWithTypeAndName& arguments) const override { if (arguments[0].type->is_nullable()) { return make_nullable(std::make_shared()); } return std::make_shared(); } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const override { const auto& arg_col = block.get_by_position(arguments[0]).column; const auto& column_data = assert_cast(*arg_col); auto res_col = ColumnInt64::create(); auto& res_data = res_col->get_data(); res_col->get_data().resize_fill(input_rows_count, 0); for (int i = 0; i < input_rows_count; i++) { StringRef source = column_data.get_data_at(i); const auto& dt = reinterpret_cast&>(*source.data); const cctz::time_zone& time_zone = context->state()->timezone_obj(); int64_t timestamp {0}; auto ret = dt.unix_timestamp(×tamp, time_zone); // ret must be true DCHECK(ret); auto microsecond = dt.microsecond(); timestamp = timestamp * Impl::ratio + microsecond / ratio_to_micro; res_data[i] = timestamp; } block.replace_by_position(result, std::move(res_col)); return Status::OK(); } }; template