database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug](timediff) Fix wrong result for function timediff (#15312)

Browse Source
This commit is contained in:
Gabriel
2022-12-30 00:28:51 +08:00
committed by GitHub
parent 9a517d6a8f
commit edb9a3b58d
13 changed files with 163 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
be/src/util/date_func.cpp
View File

@ -109,4 +109,25 @@ int32_t time_to_buffer_from_double(double time, char* buffer) {
return buffer - begin;
}
std::string time_to_buffer_from_double(double time) {
fmt::memory_buffer buffer;
if (time < 0) {
time = -time;
fmt::format_to(buffer, "-");
}
if (time > 3020399) {
time = 3020399;
}
int64_t hour = (int64_t)(time / 3600);
int32_t minute = ((int32_t)(time / 60)) % 60;
int32_t second = ((int32_t)time) % 60;
if (hour >= 100) {
fmt::format_to(buffer, fmt::format("{}", hour));
} else {
fmt::format_to(buffer, fmt::format("{:02d}", hour));
}
fmt::format_to(buffer, fmt::format(":{:02d}:{:02d}", minute, second));
return fmt::to_string(buffer);
}
} // namespace doris

2
be/src/util/date_func.h
View File

@ -32,4 +32,6 @@ int32_t time_to_buffer_from_double(double time, char* buffer);
uint32_t timestamp_from_date_v2(const std::string& date_str);
uint64_t timestamp_from_datetime_v2(const std::string& date_str);
std::string time_to_buffer_from_double(double time);
} // namespace doris

1
be/src/vec/CMakeLists.txt
View File

@ -89,6 +89,7 @@ set(VEC_FILES
data_types/data_type_date_time.cpp
data_types/data_type_time_v2.cpp
data_types/data_type_jsonb.cpp
data_types/data_type_time.cpp
exec/vaggregation_node.cpp
exec/varrow_scanner.cpp
exec/vsort_node.cpp

4
be/src/vec/data_types/data_type_factory.cpp
View File

@ -20,6 +20,8 @@
#include "vec/data_types/data_type_factory.hpp"
#include "data_type_time.h"
namespace doris::vectorized {
DataTypePtr DataTypeFactory::create_data_type(const doris::Field& col_desc) {
@ -92,6 +94,8 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
break;
case TYPE_TIME:
case TYPE_TIMEV2:
nested = std::make_shared<vectorized::DataTypeTime>();
break;
case TYPE_DOUBLE:
nested = std::make_shared<vectorized::DataTypeFloat64>();
break;

52
be/src/vec/data_types/data_type_time.cpp Normal file
View File

@ -0,0 +1,52 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// This file is copied from
// https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeDateTime.cpp
// and modified by Doris
#include "vec/data_types/data_type_time.h"
#include "util/date_func.h"
#include "vec/columns/columns_number.h"
namespace doris::vectorized {
bool DataTypeTime::equals(const IDataType& rhs) const {
return typeid(rhs) == typeid(*this);
}
std::string DataTypeTime::to_string(const IColumn& column, size_t row_num) const {
Float64 float_val =
assert_cast<const ColumnFloat64&>(*column.convert_to_full_column_if_const().get())
.get_data()[row_num];
return time_to_buffer_from_double(float_val);
}
void DataTypeTime::to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const {
Float64 float_val =
assert_cast<const ColumnFloat64&>(*column.convert_to_full_column_if_const().get())
.get_data()[row_num];
std::string time_val = time_to_buffer_from_double(float_val);
// DateTime to_string the end is /0
ostr.write(time_val.data(), time_val.size());
}
MutableColumnPtr DataTypeTime::create_column() const {
return DataTypeNumberBase<Float64>::create_column();
}
} // namespace doris::vectorized

53
be/src/vec/data_types/data_type_time.h Normal file
View File

@ -0,0 +1,53 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// This file is copied from
// https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeDateTime.h
// and modified by Doris
#pragma once
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_number_base.h"
namespace doris::vectorized {
class DataTypeTime final : public DataTypeNumberBase<Float64> {
public:
DataTypeTime() = default;
bool equals(const IDataType& rhs) const override;
std::string to_string(const IColumn& column, size_t row_num) const override;
void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
MutableColumnPtr create_column() const override;
bool can_be_used_as_version() const override { return true; }
bool is_summable() const override { return true; }
bool can_be_used_in_bit_operations() const override { return true; }
bool can_be_used_in_boolean_context() const override { return true; }
bool can_be_inside_nullable() const override { return true; }
bool can_be_promoted() const override { return true; }
DataTypePtr promote_numeric_type() const override {
using PromotedType = DataTypeNumber<NearestFieldType<Float64>>;
return std::make_shared<PromotedType>();
}
};
} // namespace doris::vectorized

5
be/src/vec/functions/function_date_or_datetime_computation.h
View File

@ -27,6 +27,7 @@
#include "vec/data_types/data_type_date.h"
#include "vec/data_types/data_type_date_time.h"
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_time.h"
#include "vec/functions/function.h"
#include "vec/functions/function_helpers.h"
#include "vec/runtime/vdatetime_value.h"
@ -245,7 +246,7 @@ struct SubtractYearsImpl : SubtractIntervalImpl<AddYearsImpl<DateType>, DateType
} \
};
DECLARE_DATE_FUNCTIONS(DateDiffImpl, datediff, DataTypeInt32, (ts0.daynr() - ts1.daynr()));
DECLARE_DATE_FUNCTIONS(TimeDiffImpl, timediff, DataTypeFloat64, ts0.second_diff(ts1));
DECLARE_DATE_FUNCTIONS(TimeDiffImpl, timediff, DataTypeTime, ts0.second_diff(ts1));
#define TIME_DIFF_FUNCTION_IMPL(CLASS, NAME, UNIT) \
DECLARE_DATE_FUNCTIONS(CLASS, NAME, DataTypeInt64, datetime_diff<TimeUnit::UNIT>(ts1, ts0))
@ -798,7 +799,7 @@ struct CurrentDateImpl {
template <typename FunctionName>
struct CurrentTimeImpl {
using ReturnType = DataTypeFloat64;
using ReturnType = DataTypeTime;
static constexpr auto name = FunctionName::name;
static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
size_t result, size_t input_rows_count) {

3
be/src/vec/functions/function_running_difference.h
View File

@ -31,6 +31,7 @@
#include "vec/data_types/data_type_date_time.h"
#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_time.h"
#include "vec/data_types/data_type_time_v2.h"
#include "vec/data_types/number_traits.h"
#include "vec/functions/function.h"
@ -75,7 +76,7 @@ public:
} else if (which.is_decimal()) {
return_type = nested_type;
} else if (which.is_date_time() || which.is_date_time_v2()) {
return_type = std::make_shared<DataTypeFloat64>();
return_type = std::make_shared<DataTypeTime>();
} else if (which.is_date() || which.is_date_v2()) {
return_type = std::make_shared<DataTypeInt32>();
}

2
be/test/vec/function/function_running_difference_test.cpp
View File

@ -55,7 +55,7 @@ TEST(FunctionRunningDifferenceTest, function_running_difference_test) {
{{std::string("2019-07-18 12:00:06")}, (double)1.0},
{{std::string("2019-07-18 12:00:08")}, (double)2.0},
{{std::string("2019-07-18 12:00:10")}, (double)2.0}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
InputTypeSet input_types = {TypeIndex::Date};

7
be/test/vec/function/function_test_util.h
View File

@ -35,6 +35,7 @@
#include "vec/data_types/data_type_jsonb.h"
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_string.h"
#include "vec/data_types/data_type_time.h"
#include "vec/functions/simple_function_factory.h"
namespace doris::vectorized {
@ -233,7 +234,8 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty
fn_ctx_return.type = doris_udf::FunctionContext::TYPE_BOOLEAN;
} else if constexpr (std::is_same_v<ReturnType, DataTypeInt32>) {
fn_ctx_return.type = doris_udf::FunctionContext::TYPE_INT;
} else if constexpr (std::is_same_v<ReturnType, DataTypeFloat64>) {
} else if constexpr (std::is_same_v<ReturnType, DataTypeFloat64> ||
std::is_same_v<ReturnType, DataTypeTime>) {
fn_ctx_return.type = doris_udf::FunctionContext::TYPE_DOUBLE;
} else if constexpr (std::is_same_v<ReturnType, DateTime>) {
fn_ctx_return.type = doris_udf::FunctionContext::TYPE_DATETIME;
@ -293,7 +295,8 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty
const auto& column_data = field.get<DecimalField<Decimal128>>().get_value();
EXPECT_EQ(expect_data.value, column_data.value) << " at row " << i;
} else if constexpr (std::is_same_v<ReturnType, DataTypeFloat32> ||
std::is_same_v<ReturnType, DataTypeFloat64>) {
std::is_same_v<ReturnType, DataTypeFloat64> ||
std::is_same_v<ReturnType, DataTypeTime>) {
const auto& column_data = field.get<DataTypeFloat64::FieldType>();
EXPECT_DOUBLE_EQ(expect_data, column_data) << " at row " << i;
} else {

22
be/test/vec/function/function_time_test.cpp
View File

@ -186,7 +186,7 @@ TEST(VTimestampFunctionsTest, timediff_test) {
{{std::string("2019-00-18 12:00:00"), std::string("2019-07-18 13:01:02")}, Null()},
{{std::string("2019-07-18 12:00:00"), std::string("2019-07-00 13:01:02")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
TEST(VTimestampFunctionsTest, date_format_test) {
@ -849,7 +849,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18"), std::string("2019-07-18")}, Null()},
{{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
@ -860,7 +860,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18"), std::string("2019-07-18")}, Null()},
{{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
@ -871,7 +871,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18"), std::string("2019-07-18")}, Null()},
{{std::string("2019-07-18"), std::string("2019-07-00")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
@ -883,7 +883,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18 00:00:00"), std::string("2019-07-18")}, Null()},
{{std::string("2019-07-18 00:00:00"), std::string("2019-07-00")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
@ -895,7 +895,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18"), std::string("2019-07-18 00:00:00")}, Null()},
{{std::string("2019-07-18"), std::string("2019-07-00 00:00:00")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
InputTypeSet input_types = {TypeIndex::DateTimeV2, TypeIndex::DateTimeV2};
@ -906,7 +906,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18 00:00:00"), std::string("2019-07-18 00:00:00")}, Null()},
{{std::string("2019-07-18 00:00:00"), std::string("2019-07-00 00:00:00")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
@ -918,7 +918,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18 00:00:00"), std::string("2019-07-18")}, Null()},
{{std::string("2019-07-18 00:00:00"), std::string("2019-07-00")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
@ -930,7 +930,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18"), std::string("2019-07-18 00:00:00")}, Null()},
{{std::string("2019-07-18"), std::string("2019-07-00 00:00:00")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
@ -942,7 +942,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-00-18 00:00:00"), std::string("2019-07-18 00:00:00")}, Null()},
{{std::string("2019-07-18 00:00:00"), std::string("2019-07-00 00:00:00")}, Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
{
@ -957,7 +957,7 @@ TEST(VTimestampFunctionsTest, timediff_v2_test) {
{{std::string("2019-07-18 00:00:00.123"), std::string("2019-07-00 00:00:00")},
Null()}};
check_function<DataTypeFloat64, true>(func_name, input_types, data_set);
check_function<DataTypeTime, true>(func_name, input_types, data_set);
}
}

4
gensrc/script/doris_builtins_functions.py
View File

@ -2424,8 +2424,8 @@ visible_functions = [
[['running_difference'], 'DECIMAL128', ['DECIMAL128'], '', '', '', 'vec', ''],
[['running_difference'], 'INT', ['DATE'], '', '', '', 'vec', ''],
[['running_difference'], 'INT', ['DATEV2'], '', '', '', 'vec', ''],
[['running_difference'], 'DOUBLE', ['DATETIME'], '', '', '', 'vec', ''],
[['running_difference'], 'DOUBLE', ['DATETIMEV2'], '', '', '', 'vec', ''],
[['running_difference'], 'TIME', ['DATETIME'], '', '', '', 'vec', ''],
[['running_difference'], 'TIMEV2', ['DATETIMEV2'], '', '', '', 'vec', ''],
# Longtext function
[['substr', 'substring'], 'STRING', ['STRING', 'INT'],

12
regression-test/data/query_p0/sql_functions/math_functions/test_running_difference.out
View File

@ -33,12 +33,12 @@
2022-11-08 8
-- !test_running_difference_7 --
2022-03-12T10:41 0.0
2022-03-12T10:41:02 2.0
2022-03-12T10:41:03 1.0
2022-03-12T10:41:03 0.0
2022-03-12T10:42:01 58.0
2022-03-12T11:05:04 1383.0
2022-03-12T10:41 00:00:00
2022-03-12T10:41:02 00:00:02
2022-03-12T10:41:03 00:00:01
2022-03-12T10:41:03 00:00:00
2022-03-12T10:42:01 00:00:58
2022-03-12T11:05:04 00:23:03
-- !test_running_difference_8 --
\N \N