diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp index ebcb6eb378..8cbeebe4c3 100644 --- a/be/src/runtime/types.cpp +++ b/be/src/runtime/types.cpp @@ -46,7 +46,7 @@ TypeDescriptor::TypeDescriptor(const std::vector& types, int* idx) DCHECK(scalar_type.__isset.len); len = scalar_type.len; } else if (type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 || - type == TYPE_DECIMAL128I || type == TYPE_DATETIMEV2) { + type == TYPE_DECIMAL128I || type == TYPE_DATETIMEV2 || type == TYPE_TIMEV2) { DCHECK(scalar_type.__isset.precision); DCHECK(scalar_type.__isset.scale); precision = scalar_type.precision; diff --git a/be/src/util/date_func.cpp b/be/src/util/date_func.cpp index e7ae30e003..6ca199d630 100644 --- a/be/src/util/date_func.cpp +++ b/be/src/util/date_func.cpp @@ -86,6 +86,7 @@ uint64_t timestamp_from_datetime_v2(const std::string& date_str) { } // refer to https://dev.mysql.com/doc/refman/5.7/en/time.html // the time value between '-838:59:59' and '838:59:59' +/// TODO: Why is the time type stored as double? Can we directly use int64 and remove the time limit? int32_t time_to_buffer_from_double(double time, char* buffer) { char* begin = buffer; if (time < 0) { @@ -115,6 +116,52 @@ int32_t time_to_buffer_from_double(double time, char* buffer) { return buffer - begin; } +int32_t timev2_to_buffer_from_double(double time, char* buffer, int scale) { + static int pow10[7] = {1, 10, 100, 1000, 10000, 100000, 1000000}; + char* begin = buffer; + if (time < 0) { + time = -time; + *buffer++ = '-'; + } + int64_t m_time = time; + // m_time = hour * 3600 * 1000 * 1000 + minute * 60 * 1000 * 1000 + second * 1000 * 1000 + microsecond + int64_t hour = m_time / ((int64_t)3600 * 1000 * 1000); + if (hour >= 100) { + auto f = fmt::format_int(hour); + memcpy(buffer, f.data(), f.size()); + buffer = buffer + f.size(); + } else { + *buffer++ = (char)('0' + (hour / 10)); + *buffer++ = (char)('0' + (hour % 10)); + } + *buffer++ = ':'; + m_time %= (int64_t)3600 * 1000 * 1000; + int64_t minute = m_time / (60 * 1000 * 1000); + *buffer++ = (char)('0' + (minute / 10)); + *buffer++ = (char)('0' + (minute % 10)); + *buffer++ = ':'; + m_time %= 60 * 1000 * 1000; + int32_t second = m_time / (1000 * 1000); + *buffer++ = (char)('0' + (second / 10)); + *buffer++ = (char)('0' + (second % 10)); + m_time %= 1000 * 1000; + if (scale == 0) { + return buffer - begin; + } + *buffer++ = '.'; + memset(buffer, '0', scale); + buffer += scale; + int32_t micosecond = m_time % (1000 * 1000); + micosecond /= pow10[6 - scale]; + auto it = buffer - 1; + while (micosecond) { + *it = (char)('0' + (micosecond % 10)); + micosecond /= 10; + it--; + } + return buffer - begin; +} + std::string time_to_buffer_from_double(double time) { fmt::memory_buffer buffer; if (time < 0) { @@ -136,4 +183,51 @@ std::string time_to_buffer_from_double(double time) { return fmt::to_string(buffer); } +std::string timev2_to_buffer_from_double(double time, int scale) { + static int pow10[7] = {1, 10, 100, 1000, 10000, 100000, 1000000}; + fmt::memory_buffer buffer; + if (time < 0) { + time = -time; + fmt::format_to(buffer, "-"); + } + int64_t m_time = time; + // m_time = hour * 3600 * 1000 * 1000 + minute * 60 * 1000 * 1000 + second * 1000 * 1000 + microsecond + int64_t hour = m_time / ((int64_t)3600 * 1000 * 1000); + if (hour >= 100) { + fmt::format_to(buffer, fmt::format("{}", hour)); + } else { + fmt::format_to(buffer, fmt::format("{:02d}", hour)); + } + m_time %= (int64_t)3600 * 1000 * 1000; + int64_t minute = m_time / (60 * 1000 * 1000); + m_time %= 60 * 1000 * 1000; + int32_t second = m_time / (1000 * 1000); + int32_t micosecond = m_time % (1000 * 1000); + micosecond /= pow10[6 - scale]; + switch (scale) { + case 0: + fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}", minute, second, micosecond)); + break; + case 1: + fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:01d}", minute, second, micosecond)); + break; + case 2: + fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:02d}", minute, second, micosecond)); + break; + case 3: + fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:03d}", minute, second, micosecond)); + break; + case 4: + fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:04d}", minute, second, micosecond)); + break; + case 5: + fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:05d}", minute, second, micosecond)); + break; + case 6: + fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}.{:06d}", minute, second, micosecond)); + break; + } + + return fmt::to_string(buffer); +} } // namespace doris diff --git a/be/src/util/date_func.h b/be/src/util/date_func.h index cc9aa7b6da..405132c3f0 100644 --- a/be/src/util/date_func.h +++ b/be/src/util/date_func.h @@ -26,9 +26,10 @@ namespace doris { uint64_t timestamp_from_datetime(const std::string& datetime_str); uint32_t timestamp_from_date(const std::string& date_str); int32_t time_to_buffer_from_double(double time, char* buffer); +int32_t timev2_to_buffer_from_double(double time, char* buffer, int scale); uint32_t timestamp_from_date_v2(const std::string& date_str); uint64_t timestamp_from_datetime_v2(const std::string& date_str); std::string time_to_buffer_from_double(double time); - +std::string timev2_to_buffer_from_double(double time, int scale); } // namespace doris diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp index a8d2674073..00193481b7 100644 --- a/be/src/util/mysql_row_buffer.cpp +++ b/be/src/util/mysql_row_buffer.cpp @@ -195,6 +195,13 @@ static char* add_time(double data, char* pos, bool dynamic_mode) { return pos + length; } +static char* add_timev2(double data, char* pos, bool dynamic_mode, int scale) { + int length = timev2_to_buffer_from_double(data, pos + !dynamic_mode, scale); + if (!dynamic_mode) { + int1store(pos++, length); + } + return pos + length; +} template static char* add_datetime(const DateType& data, char* pos, bool dynamic_mode) { int length = data.to_buffer(pos + !dynamic_mode); @@ -422,6 +429,25 @@ int MysqlRowBuffer::push_time(double data) { return 0; } +template +int MysqlRowBuffer::push_timev2(double data, int scale) { + if (is_binary_format && !_dynamic_mode) { + char buff[8]; + _field_pos++; + float8store(buff, data); + return append(buff, 8); + } + // 1 for string trail, 1 for length, other for time str + int ret = reserve(2 + MAX_TIME_WIDTH); + + if (0 != ret) { + LOG(ERROR) << "mysql row buffer reserve failed."; + return ret; + } + + _pos = add_timev2(data, _pos, _dynamic_mode, scale); + return 0; +} template template int MysqlRowBuffer::push_vec_datetime(DateType& data) { diff --git a/be/src/util/mysql_row_buffer.h b/be/src/util/mysql_row_buffer.h index 2df739450f..6a6c2b0ad0 100644 --- a/be/src/util/mysql_row_buffer.h +++ b/be/src/util/mysql_row_buffer.h @@ -72,6 +72,7 @@ public: int push_float(float data); int push_double(double data); int push_time(double data); + int push_timev2(double data, int scale); template int push_datetime(const DateType& data); int push_decimal(const DecimalV2Value& data, int round_scale); diff --git a/be/src/vec/core/call_on_type_index.h b/be/src/vec/core/call_on_type_index.h index bc822d9902..7e6ea4118c 100644 --- a/be/src/vec/core/call_on_type_index.h +++ b/be/src/vec/core/call_on_type_index.h @@ -205,6 +205,8 @@ bool call_on_index_and_data_type(TypeIndex number, F&& f) { return f(TypePair, T>()); case TypeIndex::Time: return f(TypePair()); + case TypeIndex::TimeV2: + return f(TypePair()); case TypeIndex::Decimal32: return f(TypePair, T>()); case TypeIndex::Decimal64: diff --git a/be/src/vec/data_types/data_type.cpp b/be/src/vec/data_types/data_type.cpp index cf66b64f95..7ab7a99ca3 100644 --- a/be/src/vec/data_types/data_type.cpp +++ b/be/src/vec/data_types/data_type.cpp @@ -171,6 +171,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) { return PGenericType::TIME; case TypeIndex::AggState: return PGenericType::AGG_STATE; + case TypeIndex::TimeV2: + return PGenericType::TIMEV2; default: return PGenericType::UNKNOWN; } diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp index 366cc04c89..48dbbd3270 100644 --- a/be/src/vec/data_types/data_type_factory.cpp +++ b/be/src/vec/data_types/data_type_factory.cpp @@ -144,9 +144,11 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo nested = std::make_shared(); break; case TYPE_TIME: - case TYPE_TIMEV2: nested = std::make_shared(); break; + case TYPE_TIMEV2: + nested = std::make_shared(col_desc.scale); + break; case TYPE_DOUBLE: nested = std::make_shared(); break; @@ -314,7 +316,7 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeIndex& type_index, bool nested = std::make_shared(); break; case TypeIndex::TimeV2: - nested = std::make_shared(); + nested = std::make_shared(); break; default: DCHECK(false) << "invalid typeindex:" << getTypeName(type_index); @@ -518,6 +520,10 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) { nested = std::make_shared(); break; } + case PGenericType::TIMEV2: { + nested = std::make_shared(pcolumn.decimal_param().scale()); + break; + } case PGenericType::AGG_STATE: { DataTypes sub_types; for (auto child : pcolumn.children()) { diff --git a/be/src/vec/data_types/data_type_time.cpp b/be/src/vec/data_types/data_type_time.cpp index 7881124f3e..d9c2af440e 100644 --- a/be/src/vec/data_types/data_type_time.cpp +++ b/be/src/vec/data_types/data_type_time.cpp @@ -20,6 +20,8 @@ #include "vec/data_types/data_type_time.h" +#include + #include #include @@ -56,8 +58,34 @@ void DataTypeTime::to_string(const IColumn& column, size_t row_num, BufferWritab ostr.write(value.data(), value.size()); } +void DataTypeTimeV2::to_pb_column_meta(PColumnMeta* col_meta) const { + IDataType::to_pb_column_meta(col_meta); + col_meta->mutable_decimal_param()->set_scale(_scale); +} + MutableColumnPtr DataTypeTime::create_column() const { return DataTypeNumberBase::create_column(); } +bool DataTypeTimeV2::equals(const IDataType& rhs) const { + return typeid(rhs) == typeid(*this); +} + +std::string DataTypeTimeV2::to_string(const IColumn& column, size_t row_num) const { + auto result = check_column_const_set_readability(column, row_num); + ColumnPtr ptr = result.first; + row_num = result.second; + + auto value = assert_cast(*ptr).get_element(row_num); + return timev2_to_buffer_from_double(value, _scale); +} + +void DataTypeTimeV2::to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const { + std::string value = to_string(column, row_num); + ostr.write(value.data(), value.size()); +} + +MutableColumnPtr DataTypeTimeV2::create_column() const { + return DataTypeNumberBase::create_column(); +} } // namespace doris::vectorized diff --git a/be/src/vec/data_types/data_type_time.h b/be/src/vec/data_types/data_type_time.h index 0bc1b1aad8..47311804e9 100644 --- a/be/src/vec/data_types/data_type_time.h +++ b/be/src/vec/data_types/data_type_time.h @@ -69,6 +69,42 @@ public: DataTypeSerDeSPtr get_serde() const override { return std::make_shared(); }; TypeIndex get_type_id() const override { return TypeIndex::Time; } + const char* get_family_name() const override { return "time"; } }; +class DataTypeTimeV2 final : public DataTypeNumberBase { +public: + DataTypeTimeV2(int scale = 0) : _scale(scale) { + if (UNLIKELY(scale > 6)) { + LOG(FATAL) << fmt::format("Scale {} is out of bounds", scale); + } + } + bool equals(const IDataType& rhs) const override; + + std::string to_string(const IColumn& column, size_t row_num) const override; + PrimitiveType get_type_as_primitive_type() const override { return TYPE_TIMEV2; } + TPrimitiveType::type get_type_as_tprimitive_type() const override { + return TPrimitiveType::TIMEV2; + } + + void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override; + + MutableColumnPtr create_column() const override; + + bool is_summable() const override { return true; } + bool can_be_used_in_bit_operations() const override { return true; } + bool can_be_used_in_boolean_context() const override { return true; } + bool can_be_inside_nullable() const override { return true; } + + void to_pb_column_meta(PColumnMeta* col_meta) const override; + DataTypeSerDeSPtr get_serde() const override { + return std::make_shared(_scale); + }; + TypeIndex get_type_id() const override { return TypeIndex::TimeV2; } + const char* get_family_name() const override { return "timev2"; } + UInt32 get_scale() const override { return _scale; } + +private: + UInt32 _scale; +}; } // namespace doris::vectorized diff --git a/be/src/vec/data_types/serde/data_type_time_serde.cpp b/be/src/vec/data_types/serde/data_type_time_serde.cpp index 46d5989a0b..4fed145c8b 100644 --- a/be/src/vec/data_types/serde/data_type_time_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_time_serde.cpp @@ -43,5 +43,26 @@ Status DataTypeTimeSerDe::write_column_to_mysql(const IColumn& column, return _write_column_to_mysql(column, row_buffer, row_idx, col_const); } +Status DataTypeTimeV2SerDe::write_column_to_mysql(const IColumn& column, + MysqlRowBuffer& row_buffer, int row_idx, + bool col_const) const { + return _write_column_to_mysql(column, row_buffer, row_idx, col_const); +} +Status DataTypeTimeV2SerDe::write_column_to_mysql(const IColumn& column, + MysqlRowBuffer& row_buffer, int row_idx, + bool col_const) const { + return _write_column_to_mysql(column, row_buffer, row_idx, col_const); +} +template +Status DataTypeTimeV2SerDe::_write_column_to_mysql(const IColumn& column, + MysqlRowBuffer& result, + int row_idx, bool col_const) const { + auto& data = assert_cast&>(column).get_data(); + const auto col_index = index_check_const(row_idx, col_const); + if (UNLIKELY(0 != result.push_timev2(data[col_index], scale))) { + return Status::InternalError("pack mysql buffer failed."); + } + return Status::OK(); +} } // namespace vectorized } // namespace doris \ No newline at end of file diff --git a/be/src/vec/data_types/serde/data_type_time_serde.h b/be/src/vec/data_types/serde/data_type_time_serde.h index 3c1c14f977..c96163be55 100644 --- a/be/src/vec/data_types/serde/data_type_time_serde.h +++ b/be/src/vec/data_types/serde/data_type_time_serde.h @@ -41,5 +41,19 @@ private: Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer& result, int row_idx, bool col_const) const; }; +class DataTypeTimeV2SerDe : public DataTypeNumberSerDe { +public: + DataTypeTimeV2SerDe(int scale = -1) : scale(scale) {}; + Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer& row_buffer, + int row_idx, bool col_const) const override; + Status write_column_to_mysql(const IColumn& column, MysqlRowBuffer& row_buffer, + int row_idx, bool col_const) const override; + +private: + template + Status _write_column_to_mysql(const IColumn& column, MysqlRowBuffer& result, + int row_idx, bool col_const) const; + int scale; +}; } // namespace vectorized } // namespace doris \ No newline at end of file diff --git a/be/src/vec/exec/format/parquet/schema_desc.cpp b/be/src/vec/exec/format/parquet/schema_desc.cpp index 6d87425983..f52e3921ad 100644 --- a/be/src/vec/exec/format/parquet/schema_desc.cpp +++ b/be/src/vec/exec/format/parquet/schema_desc.cpp @@ -237,7 +237,7 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::LogicalType logi } } } else if (logicalType.__isset.TIME) { - type = TypeDescriptor(TYPE_TIMEV2); + type = TypeDescriptor(TYPE_TIME); } else if (logicalType.__isset.TIMESTAMP) { type = TypeDescriptor(TYPE_DATETIMEV2); const auto& time_unit = logicalType.TIMESTAMP.unit; diff --git a/be/src/vec/functions/function_date_or_datetime_computation.h b/be/src/vec/functions/function_date_or_datetime_computation.h index e31d8f64c6..1bc5d37c53 100644 --- a/be/src/vec/functions/function_date_or_datetime_computation.h +++ b/be/src/vec/functions/function_date_or_datetime_computation.h @@ -281,8 +281,47 @@ struct SubtractYearsImpl : SubtractIntervalImpl, DateType } \ }; DECLARE_DATE_FUNCTIONS(DateDiffImpl, datediff, DataTypeInt32, (ts0.daynr() - ts1.daynr())); -DECLARE_DATE_FUNCTIONS(TimeDiffImpl, timediff, DataTypeTime, ts0.second_diff(ts1)); +// DECLARE_DATE_FUNCTIONS(TimeDiffImpl, timediff, DataTypeTime, ts0.second_diff(ts1)); +// Expands to +template +struct TimeDiffImpl { + using ArgType1 = std ::conditional_t< + std ::is_same_v, UInt32, + std ::conditional_t, UInt64, Int64>>; + using ArgType2 = std ::conditional_t< + std ::is_same_v, UInt32, + std ::conditional_t, UInt64, Int64>>; + using DateValueType1 = std ::conditional_t< + std ::is_same_v, DateV2Value, + std ::conditional_t, + DateV2Value, VecDateTimeValue>>; + using DateValueType2 = std ::conditional_t< + std ::is_same_v, DateV2Value, + std ::conditional_t, + DateV2Value, VecDateTimeValue>>; + static constexpr bool UsingTimev2 = std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v; + using ReturnType = std ::conditional_t; + static constexpr auto name = "timediff"; + static constexpr auto is_nullable = false; + static inline ReturnType ::FieldType execute(const ArgType1& t0, const ArgType2& t1, + bool& is_null) { + const auto& ts0 = reinterpret_cast(t0); + const auto& ts1 = reinterpret_cast(t1); + is_null = !ts0.is_valid_date() || !ts1.is_valid_date(); + if constexpr (UsingTimev2) { + return ts0.microsecond_diff(ts1); + } else { + return ts0.second_diff(ts1); + } + } + static DataTypes get_variadic_argument_types() { + return {std ::make_shared(), std ::make_shared()}; + } +}; #define TIME_DIFF_FUNCTION_IMPL(CLASS, NAME, UNIT) \ DECLARE_DATE_FUNCTIONS(CLASS, NAME, DataTypeInt64, datetime_diff(ts1, ts0)) diff --git a/be/src/vec/functions/function_running_difference.h b/be/src/vec/functions/function_running_difference.h index 248c29a9a5..73446dd2b2 100644 --- a/be/src/vec/functions/function_running_difference.h +++ b/be/src/vec/functions/function_running_difference.h @@ -82,8 +82,10 @@ public: return_type = std::make_shared(); } else if (which.is_decimal()) { return_type = nested_type; - } else if (which.is_date_time() || which.is_date_time_v2()) { + } else if (which.is_date_time()) { return_type = std::make_shared(); + } else if (which.is_date_time_v2()) { + return_type = std::make_shared(); } else if (which.is_date() || which.is_date_v2()) { return_type = std::make_shared(); } diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index 891d6fb8b1..639d7171cb 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -870,6 +870,10 @@ public: return hour() * SECOND_PER_HOUR + minute() * SECOND_PER_MINUTE + second(); } + int64_t time_part_to_microsecond() const { + return time_part_to_seconds() * 1000 * 1000 + microsecond(); + } + uint16_t year() const { return date_v2_value_.year_; } uint8_t month() const { return date_v2_value_.month_; } int quarter() const { return (date_v2_value_.month_ - 1) / 3 + 1; } @@ -1065,11 +1069,27 @@ public: return time_part_to_seconds() - rhs.time_part_to_seconds(); } + //only calculate the diff of dd:mm:ss.SSSSSS + template + int64_t time_part_diff_microsecond(const RHS& rhs) const { + return time_part_to_microsecond() - rhs.time_part_to_microsecond(); + } + template int64_t second_diff(const RHS& rhs) const { return (daynr() - rhs.daynr()) * SECOND_PER_HOUR * HOUR_PER_DAY + time_part_diff(rhs); } + template + double microsecond_diff(const RHS& rhs) const { + int64_t diff_m = (daynr() - rhs.daynr()) * SECOND_PER_HOUR * HOUR_PER_DAY * 1000 * 1000 + + time_part_diff_microsecond(rhs); + if (diff_m > (int64_t)3020399 * 1000 * 1000) { + diff_m = (int64_t)3020399 * 1000 * 1000; + } + return diff_m; + } + bool can_cast_to_date_without_loss_accuracy() { return this->hour() == 0 && this->minute() == 0 && this->second() == 0 && this->microsecond() == 0; diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp index fd600e0bee..6c42403343 100644 --- a/be/src/vec/sink/vmysql_result_writer.cpp +++ b/be/src/vec/sink/vmysql_result_writer.cpp @@ -368,9 +368,12 @@ Status VMysqlResultWriter::_add_one_column( if constexpr (type == TYPE_DOUBLE) { buf_ret = rows_buffer[i].push_double(data[col_index]); } - if constexpr (type == TYPE_TIME || type == TYPE_TIMEV2) { + if constexpr (type == TYPE_TIME) { buf_ret = rows_buffer[i].push_time(data[col_index]); } + if constexpr (type == TYPE_TIMEV2) { + buf_ret = rows_buffer[i].push_timev2(data[col_index]); + } if constexpr (type == TYPE_DATETIME) { auto time_num = data[col_index]; VecDateTimeValue time_val = binary_cast(time_num); diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp index 45f0c96dbd..b93d66cc4a 100644 --- a/be/test/vec/function/function_time_test.cpp +++ b/be/test/vec/function/function_time_test.cpp @@ -859,7 +859,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) { {{std::string("2019-00-18"), std::string("2019-07-18")}, Null()}, {{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}}; - check_function(func_name, input_types, data_set); + check_function(func_name, input_types, data_set); } { @@ -867,11 +867,12 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) { DataSet data_set = { {{std::string("2019-07-18 00:00:00"), std::string("2019-07-18 00:00:00")}, 0.0}, - {{std::string("2019-07-18 00:00:10"), std::string("2019-07-18 00:00:00")}, 10.0}, + {{std::string("2019-07-18 00:00:10"), std::string("2019-07-18 00:00:00")}, + 10000000.0}, {{std::string("2019-00-18 00:00:00"), std::string("2019-07-18 00:00:00")}, Null()}, {{std::string("2019-07-18 00:00:00"), std::string("2019-07-00 00:00:00")}, Null()}}; - check_function(func_name, input_types, data_set); + check_function(func_name, input_types, data_set); } } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java index dc36a9ab70..41cd59d474 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java @@ -143,6 +143,8 @@ public class ScalarType extends Type { return createDecimalType(precision, scale); case DATETIMEV2: return createDatetimeV2Type(scale); + case TIMEV2: + return createTimeV2Type(scale); default: return createType(type); } @@ -687,6 +689,12 @@ public class ScalarType extends Type { scalarType.setPrecision(precision); break; } + case TIMEV2: { + Preconditions.checkArgument(precision >= scale); + scalarType.setScale(scale); + scalarType.setPrecision(precision); + break; + } default: break; } @@ -867,7 +875,10 @@ public class ScalarType extends Type { return false; } ScalarType other = (ScalarType) o; - if ((this.isDatetimeV2() && other.isDatetimeV2()) || (this.isTimeV2() && other.isTimeV2())) { + if ((this.isDatetimeV2() && other.isDatetimeV2())) { + return this.decimalScale() == other.decimalScale(); + } + if (this.isTimeV2() && other.isTimeV2()) { return this.decimalScale() == other.decimalScale(); } if (type.isDecimalV3Type() && other.isDecimalV3()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index d56c398836..705a45ca37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -2457,7 +2457,7 @@ public abstract class Expr extends TreeNode implements ParseNode, Cloneabl } } - protected Type getActualScalarType(Type originType) { + protected Type getActualScalarType(Type originType) { if (originType.getPrimitiveType() == PrimitiveType.DECIMAL32) { return Type.DECIMAL32; } else if (originType.getPrimitiveType() == PrimitiveType.DECIMAL64) { @@ -2472,6 +2472,8 @@ public abstract class Expr extends TreeNode implements ParseNode, Cloneabl return Type.CHAR; } else if (originType.getPrimitiveType() == PrimitiveType.DECIMALV2) { return Type.MAX_DECIMALV2_TYPE; + } else if (originType.getPrimitiveType() == PrimitiveType.TIMEV2) { + return Type.TIMEV2; } return originType; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 24169eccf8..4e57ee9df0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1559,6 +1559,13 @@ public class FunctionCallExpr extends Expr { } if (fn.getFunctionName().getFunction().equals("timediff")) { fn.getReturnType().getPrimitiveType().setTimeType(); + ScalarType left = (ScalarType) argTypes[0]; + ScalarType right = (ScalarType) argTypes[1]; + if (left.isDatetimeV2() || right.isDatetimeV2() || left.isDateV2() || right.isDateV2()) { + Type ret = ScalarType.createTimeV2Type(Math.max(left.getScalarScale(), right.getScalarScale())); + fn.setReturnType(ret); + fn.getReturnType().getPrimitiveType().setTimeType(); + } } if (fnName.getFunction().equalsIgnoreCase("map")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimeDiff.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimeDiff.java index 8004e254a2..2cff7c8886 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimeDiff.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TimeDiff.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeType; @@ -28,6 +29,7 @@ import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.TimeType; import org.apache.doris.nereids.types.TimeV2Type; +import org.apache.doris.nereids.types.coercion.AbstractDataType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -46,8 +48,7 @@ public class TimeDiff extends ScalarFunction FunctionSignature.ret(TimeType.INSTANCE).args(DateTimeType.INSTANCE, DateTimeType.INSTANCE), FunctionSignature.ret(TimeV2Type.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT, DateV2Type.INSTANCE), FunctionSignature.ret(TimeV2Type.INSTANCE).args(DateV2Type.INSTANCE, DateTimeV2Type.SYSTEM_DEFAULT), - FunctionSignature.ret(TimeV2Type.INSTANCE).args(DateV2Type.INSTANCE, DateV2Type.INSTANCE) - ); + FunctionSignature.ret(TimeV2Type.INSTANCE).args(DateV2Type.INSTANCE, DateV2Type.INSTANCE)); /** * constructor with 2 arguments. @@ -74,4 +75,41 @@ public class TimeDiff extends ScalarFunction public R accept(ExpressionVisitor visitor, C context) { return visitor.visitTimeDiff(this, context); } + + @Override + public FunctionSignature computeSignature(FunctionSignature signature) { + signature = super.computeSignature(signature); + boolean useTimev2 = false; + int scale = 0; + if (getArgument(0).getDataType() instanceof DateTimeV2Type) { + DateTimeV2Type left = (DateTimeV2Type) getArgument(0).getDataType(); + scale = Math.max(scale, left.getScale()); + useTimev2 = true; + } + if (getArgument(1).getDataType() instanceof DateTimeV2Type) { + DateTimeV2Type right = (DateTimeV2Type) getArgument(1).getDataType(); + scale = Math.max(scale, right.getScale()); + useTimev2 = true; + } + if (useTimev2) { + signature = signature.withReturnType(TimeV2Type.of(scale)); + } + return signature; + } + + @Override + public List expectedInputTypes() { + FunctionSignature signature = getSignature(); + if (getArgument(0) instanceof StringLikeLiteral) { + StringLikeLiteral str = (StringLikeLiteral) getArgument(0); + DateTimeV2Type left = DateTimeV2Type.forTypeFromString(str.getStringValue()); + signature = signature.withArgumentType(0, left); + } + if (getArgument(1) instanceof StringLikeLiteral) { + StringLikeLiteral str = (StringLikeLiteral) getArgument(1); + DateTimeV2Type right = DateTimeV2Type.forTypeFromString(str.getStringValue()); + signature = signature.withArgumentType(1, right); + } + return signature.argumentsTypes; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java index f798a9b3f7..ec625d0cd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/TimeV2Type.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.types; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.types.coercion.PrimitiveType; @@ -28,13 +29,26 @@ public class TimeV2Type extends PrimitiveType { public static final TimeV2Type INSTANCE = new TimeV2Type(); private static final int WIDTH = 8; + private final int scale; + + private TimeV2Type(int scale) { + this.scale = scale; + } private TimeV2Type() { + scale = 0; } @Override public Type toCatalogDataType() { - return Type.TIMEV2; + return ScalarType.createTimeV2Type(scale); + } + + /** + * create TimeV2Type from scale + */ + public static TimeV2Type of(int scale) { + return new TimeV2Type(scale); } @Override @@ -46,4 +60,8 @@ public class TimeV2Type extends PrimitiveType { public int width() { return WIDTH; } + + public int getScale() { + return scale; + } } diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto index 7b74f6a804..af0f1fcc1d 100644 --- a/gensrc/proto/types.proto +++ b/gensrc/proto/types.proto @@ -109,6 +109,7 @@ message PGenericType { QUANTILE_STATE = 34; TIME = 35; AGG_STATE = 36; + TIMEV2 = 37; UNKNOWN = 999; } required TypeId id = 2; diff --git a/regression-test/data/correctness/test_time_diff_microseconds.out b/regression-test/data/correctness/test_time_diff_microseconds.out new file mode 100644 index 0000000000..ce4c6c10f5 --- /dev/null +++ b/regression-test/data/correctness/test_time_diff_microseconds.out @@ -0,0 +1,29 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1 -- +67:19:00.0000 +24:00:00.5140 +-437:00:00.7683 + +-- !select2 -- +48:00:00 + +-- !select3 -- +48:00:00.114514 + +-- !select4 -- +48:00:00.11400 + +-- !select5 -- +67:19:00.0000 +24:00:00.5140 +-437:00:00.7683 + +-- !select6 -- +48:00:00 + +-- !select7 -- +48:00:00.114514 + +-- !select8 -- +48:00:00.11400 + diff --git a/regression-test/data/nereids_p0/test_char_implicit_cast.out b/regression-test/data/nereids_p0/test_char_implicit_cast.out index 59f5d47377..599dd6500d 100644 --- a/regression-test/data/nereids_p0/test_char_implicit_cast.out +++ b/regression-test/data/nereids_p0/test_char_implicit_cast.out @@ -6,10 +6,10 @@ 7 -- !test_timediff_varchar -- --24:00:00 +-96:00:00 --- !test_timediff_char -- --24:00:00 +-- !test_timediff_varcharv2 -- +-96:00:00.3337 -- !test_money_format_varchar -- 123,456.00 diff --git a/regression-test/suites/correctness/test_time_diff_microseconds.groovy b/regression-test/suites/correctness/test_time_diff_microseconds.groovy new file mode 100644 index 0000000000..e754515250 --- /dev/null +++ b/regression-test/suites/correctness/test_time_diff_microseconds.groovy @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_time_diff_microseconds") { + sql """ DROP TABLE IF EXISTS tbl_time """ + sql """ + CREATE TABLE IF NOT EXISTS tbl_time ( + `id` int(11) , + `t1` DATETIMEV2(3) , + `t2` DATETIMEV2(4) + ) + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "replication_num" = "1" + ); + """ + sql """ + INSERT INTO tbl_time VALUES(1,'0001-01-03 19:19:00.000000','0001-01-01 00:00:00.0000'); + """ + + sql """ + INSERT INTO tbl_time VALUES(2,'0001-01-02 00:00:00.514514','0001-01-01 00:00:00.000000'); + """ + + sql """ + INSERT INTO tbl_time VALUES(3,'0001-01-03 19:00:00.123123','0001-01-22 00:00:00.891312'); + """ + + sql """set enable_nereids_planner=false""" + + qt_select1 """ + select timediff(t1,t2) from tbl_time order by id + """ + + qt_select2 """ + select timediff( + cast('0001-01-03 11:45:14' as Date ) , + cast('0001-01-01 00:00:00.000000' as Date)); + """ + + qt_select3 """ + select timediff( + cast('0001-01-03 00:00:00.114514' as Datetimev2(6) ) , + cast('0001-01-01 00:00:00.000000' as Datetimev2(6) )); + """ + + + qt_select4 """ + select timediff( + cast('0001-01-03 00:00:00.114514' as Datetimev2(3) ) , + cast('0001-01-01 00:00:00.000000' as Datetimev2(5) )); + """ + + sql """set enable_nereids_planner=true """ + sql """set enable_fallback_to_original_planner=false""" + + qt_select5 """ + select timediff(t1,t2) from tbl_time order by id + """ + + qt_select6 """ + select timediff( + cast('0001-01-03 11:45:14' as Date ) , + cast('0001-01-01 00:00:00.000000' as Date)); + """ + + qt_select7 """ + select timediff( + cast('0001-01-03 00:00:00.114514' as Datetimev2(6) ) , + cast('0001-01-01 00:00:00.000000' as Datetimev2(6) )); + """ + + + qt_select8 """ + select timediff( + cast('0001-01-03 00:00:00.114514' as Datetimev2(3) ) , + cast('0001-01-01 00:00:00.000000' as Datetimev2(5) )); + """ +} diff --git a/regression-test/suites/nereids_p0/test_char_implicit_cast.groovy b/regression-test/suites/nereids_p0/test_char_implicit_cast.groovy index b7e23812e8..415a808365 100644 --- a/regression-test/suites/nereids_p0/test_char_implicit_cast.groovy +++ b/regression-test/suites/nereids_p0/test_char_implicit_cast.groovy @@ -20,8 +20,8 @@ suite("test_char_implicit_cast") { sql "SET enable_fallback_to_original_planner=false" qt_test_dayofweek_varchar 'select dayofweek("2012-12-01");' qt_test_dayofweek_char 'select dayofweek(cast("2012-12-01" as char(16)));' - qt_test_timediff_varchar 'select timediff("2010-01-01 01:00:00", "2010-01-02 01:00:00");' - qt_test_timediff_char 'select timediff("2010-01-01 01:00:00", cast("2010-01-02 01:00:00" as char));' + qt_test_timediff_varchar 'select timediff("2010-01-01 01:00:00", "2010-01-05 01:00:00");' + qt_test_timediff_varcharv2 'select timediff("2010-01-01 01:00:00.123", "2010-01-05 01:00:00.4567");' qt_test_money_format_varchar 'select money_format("123456");' qt_test_money_format_char 'select money_format(cast("123456" as char));' }