From e8d18788f65de8321bc7ba7013a5d2b6da42932f Mon Sep 17 00:00:00 2001 From: daidai Date: Fri, 18 Jul 2025 14:43:11 +0800 Subject: [PATCH] branch-2.1:[fix](function)fix month=0 of from_iso8601_date function. (#53050) (#53438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bp #53050 Related PR: #40695 Problem Summary: pr #40695 introduced the function `from_iso8601_date`, which parses the string to get year、mouth、 day, and sets the date value through the `set_time_unit` function. Since `set_time_unit` lacks some judgment on mouth, it may get an illegal date in the end, which may cause core in debug mode. sql : `select from_iso8601_date('2023-00-01');` ``` F20250709 09:50:14.366984 3587796 vdatetime_value.h:1222] Check failed: date_v2_value_.month_ != 0 *** Check failure stack trace: *** @ 0x559bd7050d96 google::LogMessage::SendToLog() @ 0x559bd704d7e0 google::LogMessage::Flush() @ 0x559bd70515d9 google::LogMessageFatal::~LogMessageFatal() @ 0x559bc725a570 doris::DateV2Value<>::set_time_unit<>() @ 0x559bc7257380 doris::vectorized::FromIso8601DateV2::execute() @ 0x559bc7255a88 doris::vectorized::FunctionOtherTypesToDateType<>::execute_impl() @ 0x559bc09e0781 doris::vectorized::DefaultExecutable::execute_impl() @ 0x559bc423aa20 doris::vectorized::PreparedFunctionImpl::_execute_skipped_constant_deal() @ 0x559bc4234938 doris::vectorized::PreparedFunctionImpl::execute_without_low_cardinality_columns() @ 0x559bc4233f42 doris::vectorized::PreparedFunctionImpl::default_implementation_for_nulls() @ 0x559bc423a773 doris::vectorized::PreparedFunctionImpl::_execute_skipped_constant_deal() @ 0x559bc4234938 doris::vectorized::PreparedFunctionImpl::execute_without_low_cardinality_columns() @ 0x559bc4234a57 doris::vectorized::PreparedFunctionImpl::execute() ``` ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/vec/runtime/vdatetime_value.h | 7 +- be/test/vec/function/function_time_test.cpp | 72 +++++++++++++++++++ .../test_from_iso8601_date.out | 52 ++++++++++++++ .../test_from_iso8601_date.groovy | 58 +++++++++++++++ 4 files changed, 188 insertions(+), 1 deletion(-) diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index 12e961f5fa..11dfb6e796 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -1211,13 +1211,18 @@ public: } date_v2_value_.year_ = val; } else if constexpr (unit == TimeUnit::MONTH) { - if (val > MAX_MONTH) [[unlikely]] { + DCHECK(date_v2_value_.year_ <= MAX_YEAR); + if (val > MAX_MONTH || val == 0) [[unlikely]] { return false; } date_v2_value_.month_ = val; } else if constexpr (unit == TimeUnit::DAY) { + DCHECK(date_v2_value_.year_ <= MAX_YEAR); DCHECK(date_v2_value_.month_ <= MAX_MONTH); DCHECK(date_v2_value_.month_ != 0); + if (val == 0) [[unlikely]] { + return false; + } if (val > S_DAYS_IN_MONTH[date_v2_value_.month_] && !(is_leap(date_v2_value_.year_) && date_v2_value_.month_ == 2 && val == 29)) { return false; diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp index d0e698029a..79af1e0731 100644 --- a/be/test/vec/function/function_time_test.cpp +++ b/be/test/vec/function/function_time_test.cpp @@ -1771,4 +1771,76 @@ TEST(VTimestampFunctionsTest, year_of_week_test) { } } +TEST(VTimestampFunctionsTest, from_iso8601_date) { + std::string func_name = "from_iso8601_date"; + InputTypeSet input_types = {TypeIndex::String}; + + DataSet data_set = { + {{std::string("2020-01-01")}, str_to_date_v2("2020-01-01", "%Y-%m-%d")}, + {{std::string("2020-01-01")}, str_to_date_v2("2020-01-01", "%Y-%m-%d")}, + {{std::string("-1")}, Null()}, + {{std::string("2025-07-11")}, str_to_date_v2("2025-07-11", "%Y-%m-%d")}, + {{std::string("2024-02-29")}, str_to_date_v2("2024-02-29", "%Y-%m-%d")}, + {{std::string("2025-02-29")}, Null()}, + {{std::string("2025-13-01")}, Null()}, + {{std::string("2020-W10")}, str_to_date_v2("2020-03-02", "%Y-%m-%d")}, + {{std::string("2025-W28")}, str_to_date_v2("2025-07-07", "%Y-%m-%d")}, + {{std::string("2025-W53")}, str_to_date_v2("2025-12-29", "%Y-%m-%d")}, + {{std::string("2025-W00")}, Null()}, + {{std::string("2020-123")}, str_to_date_v2("2020-05-02", "%Y-%m-%d")}, + {{std::string("2025-192")}, str_to_date_v2("2025-07-11", "%Y-%m-%d")}, + {{std::string("2024-366")}, str_to_date_v2("2024-12-31", "%Y-%m-%d")}, + {{std::string("2025-366")}, Null()}, + {{std::string("2025-000")}, str_to_date_v2("2024-12-31", "%Y-%m-%d")}, + {{std::string("2025/07/11")}, Null()}, + {{std::string("25-07-11")}, Null()}, + {{std::string("2025-7-11")}, Null()}, + {{std::string("invalid-date")}, Null()}, + {{std::string("2025-07-11T12:34:56")}, Null()}, + {{std::string("-1")}, Null()}, + {{std::string("9999-12-31")}, str_to_date_v2("9999-12-31", "%Y-%m-%d")}, + {{std::string("10000-01-01")}, Null()}, + {{std::string("0001-01-01")}, str_to_date_v2("0001-01-01", "%Y-%m-%d")}, + {{std::string("0000-12-31")}, str_to_date_v2("0000-12-31", "%Y-%m-%d")}, + {{std::string("-0001-01-01")}, Null()}, + {{std::string("2025-01-01")}, str_to_date_v2("2025-01-01", "%Y-%m-%d")}, + {{std::string("2025-12-31")}, str_to_date_v2("2025-12-31", "%Y-%m-%d")}, + {{std::string("2025-00-01")}, Null()}, + {{std::string("2025-13-01")}, Null()}, + {{std::string("2025--01-01")}, Null()}, + {{std::string("2025-01-31")}, str_to_date_v2("2025-01-31", "%Y-%m-%d")}, + {{std::string("2025-04-30")}, str_to_date_v2("2025-04-30", "%Y-%m-%d")}, + {{std::string("2025-02-28")}, str_to_date_v2("2025-02-28", "%Y-%m-%d")}, + {{std::string("2024-02-29")}, str_to_date_v2("2024-02-29", "%Y-%m-%d")}, + {{std::string("2025-01-32")}, Null()}, + {{std::string("2025-04-31")}, Null()}, + {{std::string("2025-02-29")}, Null()}, + {{std::string("2025-02-30")}, Null()}, + {{std::string("2025-01-00")}, Null()}, + {{std::string("2025-01--01")}, Null()}, + {{std::string("2000-02-29")}, str_to_date_v2("2000-02-29", "%Y-%m-%d")}, + {{std::string("2024-02-29")}, str_to_date_v2("2024-02-29", "%Y-%m-%d")}, + {{std::string("1900-02-29")}, Null()}, + {{std::string("2100-02-29")}, Null()}, + {{std::string("2025-02-29")}, Null()}, + {{std::string("-2025-01-01")}, Null()}, + {{std::string("2025--07-01")}, Null()}, + {{std::string("2025-07--01")}, Null()}, + {{std::string("")}, Null()}, + {{std::string("2025")}, str_to_date_v2("2025-01-01", "%Y-%m-%d")}, + {{std::string("2025-07")}, str_to_date_v2("2025-07-01", "%Y-%m-%d")}, + {{std::string("99999-01-01")}, Null()}, + {{std::string("2025-123-01")}, Null()}, + {{std::string("2025-01-123")}, Null()}, + {{std::string("2025/01/01")}, Null()}, + {{std::string("2025.01.01")}, Null()}, + {{std::string("2025-01-01X")}, Null()}, + {{std::string("2025--01--01")}, Null()}, + {{std::string("abcd-01-01")}, Null()}, + {{std::string("2025-ab-01")}, Null()}, + {{std::string("2025-01-ab")}, Null()}, + }; + + static_cast(check_function(func_name, input_types, data_set)); +} } // namespace doris::vectorized diff --git a/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out b/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out index 43a4f0bd49..93b8bda36e 100644 --- a/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out +++ b/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out @@ -242,3 +242,55 @@ \N \N +-- !test_99 -- +1 2023-01-01 +2 2023-12-31 +3 2020-02-29 +4 1970-01-01 +5 9999-12-31 +6 1000-01-01 +7 \N +8 \N +9 \N +10 \N +11 \N +12 2023-02-28 +13 2024-02-29 +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 2023-01-01 +21 \N +22 \N +23 \N +24 2023-01-02 +25 2023-01-01 +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N + diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy index 52c7bb0da3..79d508cae8 100644 --- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy +++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy @@ -143,6 +143,64 @@ suite("test_from_iso8601_date") { qt_test_89 """ select from_iso8601_date(NULL) from tb2 order by k0; """ + sql """ + CREATE TABLE tb3 (id INT, date_str VARCHAR(255)) DISTRIBUTED BY HASH(id) BUCKETS 4 PROPERTIES ("replication_num" = "1"); """ + sql """ + INSERT INTO tb3 (id, date_str) VALUES + (1, '2023-01-01'), + (2, '2023-12-31'), + (3, '2020-02-29'), + (4, '1970-01-01'), + (5, '9999-12-31'), + (6, '1000-01-01'), + (7, '2023-13-01'), + (8, '2023-00-01'), + (9, '2023-01-00'), + (10, '2023-01-32'), + (11, '2023-02-29'), + (12, '2023-02-28'), + (13, '2024-02-29'), + (14, '2024-02-30'), + (15, '2023-01-01T12:00:00'), + (16, '2023-01-01 12:00:00'), + (17, '2023/01/01'), + (18, '01-01-2023'), + (19, '01/01/2023'), + (20, '20230101'), + (21, '2023-01-01Z'), + (22, '2023-01-01+08:00'), + (23, '2023-01-01-08:00'), + (24, '2023-W01-1'), + (25, '2023-001'), + (26, '2023-01-01T12:00:00.000Z'), + (27, '2023-01-01T12:00:00.000+08:00'), + (28, '2023-01-01T12:00:00.000-08:00'), + (29, '2023-01-01T12:00:00.123456Z'), + (30, '2023-01-01T12:00:00.123456+08:00'), + (31, '2023-01-01T12:00:00.123456-08:00'), + (32, '2023-01-01T24:00:00'), + (33, '2023-01-01T00:00:00.000000'), + (34, '2023-01-01T00:00:00.000001'), + (35, '2023-01-01T00:00:00.999999'), + (36, '2023-01-01T23:59:59.999999'), + (37, '2023-01-01T23:59:60'), + (38, '2023-01-01T23:59:59.9999999'), + (39, '2023-01-01T23:59:59.999999999'), + (40, '2023-01-01T23:59:59.999999999Z'), + (41, '2023-01-01T23:59:59.999999999+08:00'), + (42, '2023-01-01T23:59:59.999999999-08:00'), + (43, '2023-01-01T23:59:59.999999999999'), + (44, '2023-01-01T23:59:59.999999999999Z'), + (45, '2023-01-01T23:59:59.999999999999+08:00'), + (46, '2023-01-01T23:59:59.999999999999-08:00'), + (47, '2023-01-01T23:59:59.999999999999999'), + (48, '2023-01-01T23:59:59.999999999999999Z'), + (49, '2023-01-01T23:59:59.999999999999999+08:00'), + (50, '2023-01-01T23:59:59.999999999999999-08:00'); + """ + qt_test_99 """ SELECT id, from_iso8601_date(date_str) AS result FROM tb3 order by id; """ + + sql """ drop table tb2 """