database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug] Fix the bug of cast string to datetime return not null (#6228)

Browse Source
This commit is contained in:
HappenLee
2021-07-16 21:55:08 -05:00
committed by GitHub
parent 66f1ddaa72
commit fae3eff2e6
4 changed files with 169 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
be/src/runtime/datetime_value.cpp
View File

@ -61,21 +61,19 @@ static uint32_t calc_days_in_year(uint32_t year) {
RE2 DateTimeValue::time_zone_offset_format_reg("^[+-]{1}\\d{2}\\:\\d{2}$");
bool DateTimeValue::check_range() const {
return _year > 9999 || _month > 12 || _day > 31 ||
_hour > (_type == TIME_TIME ? TIME_MAX_HOUR : 23) || _minute > 59 || _second > 59 ||
_microsecond > 999999;
bool DateTimeValue::check_range(uint32_t year, uint32_t month, uint32_t day, uint32_t hour,
uint32_t minute, uint32_t second, uint32_t microsecond, uint16_t type) {
bool time = hour > (type == TIME_TIME ? TIME_MAX_HOUR : 23) || minute > 59 || second > 59 ||
microsecond > 999999;
return time || check_date(year, month, day);
}
bool DateTimeValue::check_date() const {
if (_month != 0 && _day > s_days_in_month[_month]) {
bool DateTimeValue::check_date(uint32_t year, uint32_t month, uint32_t day) {
if (month != 0 && month <= 12 && day > s_days_in_month[month]) {
// Feb 29 in leap year is valid.
if (_month == 2 && _day == 29 && is_leap(_year)) {
return false;
}
return true;
if (!(month == 2 && day == 29 && is_leap(year))) return true;
}
return false;
return year > 9999 || month > 12 || day > 31;
}
// The interval format is that with no delimiters
@ -182,30 +180,21 @@ bool DateTimeValue::from_date_str(const char* date_str, int len) {
date_len[field_idx] = 0;
date_val[field_idx] = 0;
}
_year = date_val[0];
_month = date_val[1];
_day = date_val[2];
_hour = date_val[3];
_minute = date_val[4];
_second = date_val[5];
_microsecond = date_val[6];
if (_microsecond && date_len[6] < 6) {
_microsecond *= log_10_int[6 - date_len[6]];
if (date_val[6] && date_len[6] < 6) {
date_val[6] *= log_10_int[6 - date_len[6]];
}
if (year_len == 2) {
if (_year < YY_PART_YEAR) {
_year += 2000;
if (date_val[0] < YY_PART_YEAR) {
date_val[0] += 2000;
} else {
_year += 1900;
date_val[0] += 1900;
}
}
if (num_field < 3 || check_range()) {
return false;
}
if (check_date()) {
return false;
}
return true;
if (num_field < 3) return false;
return check_range_and_set_time(date_val[0], date_val[1], date_val[2],
date_val[3], date_val[4], date_val[5], date_val[6], _type);
}
// [0, 101) invalid
@ -289,20 +278,18 @@ bool DateTimeValue::from_date_int64(int64_t value) {
uint64_t date = value / 1000000;
uint64_t time = value % 1000000;
_year = date / 10000;
auto [year, month, day, hour, minute, second, microsecond] = std::tuple{0,0,0,0,0,0,0};
year = date / 10000;
date %= 10000;
_month = date / 100;
_day = date % 100;
_hour = time / 10000;
month = date / 100;
day = date % 100;
hour = time / 10000;
time %= 10000;
_minute = time / 100;
_second = time % 100;
_microsecond = 0;
minute = time / 100;
second = time % 100;
microsecond = 0;
if (check_range() || check_date()) {
return false;
}
return true;
return check_range_and_set_time(year, month, day, hour, minute, second, microsecond, _type);
}
void DateTimeValue::set_zero(int type) {
@ -478,14 +465,16 @@ bool DateTimeValue::get_date_from_daynr(uint64_t daynr) {
if (daynr <= 0 || daynr > DATE_MAX_DAYNR) {
return false;
}
_year = daynr / 365;
auto [year, month, day] = std::tuple{0, 0, 0};
year = daynr / 365;
uint32_t days_befor_year = 0;
while (daynr < (days_befor_year = calc_daynr(_year, 1, 1))) {
_year--;
while (daynr < (days_befor_year = calc_daynr(year, 1, 1))) {
year--;
}
uint32_t days_of_year = daynr - days_befor_year + 1;
int leap_day = 0;
if (is_leap(_year)) {
if (is_leap(year)) {
if (days_of_year > 31 + 28) {
days_of_year--;
if (days_of_year == 31 + 28) {
@ -493,12 +482,17 @@ bool DateTimeValue::get_date_from_daynr(uint64_t daynr) {
}
}
}
_month = 1;
while (days_of_year > s_days_in_month[_month]) {
days_of_year -= s_days_in_month[_month];
_month++;
month = 1;
while (days_of_year > s_days_in_month[month]) {
days_of_year -= s_days_in_month[month];
month++;
}
_day = days_of_year + leap_day;
day = days_of_year + leap_day;
if (check_range(year, month, day, 0, 0, 0, 0, _type)) {
return false;
}
set_time(year, month, day, _hour, _minute, _second, _microsecond);
return true;
}
@ -1091,6 +1085,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
bool date_part_used = false;
bool time_part_used = false;
bool frac_part_used = false;
bool already_set_time_part = false;
int day_part = 0;
int weekday = -1;
@ -1103,6 +1098,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
int strict_week_number_year = -1;
bool usa_time = false;
auto [year, month, day, hour, minute, second, microsecond] = std::tuple{0,0,0,0,0,0,0};
while (ptr < end && val < val_end) {
// Skip space character
while (val < val_end && isspace(*val)) {
@ -1125,7 +1121,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
return false;
}
int_value += int_value >= 70 ? 1900 : 2000;
_year = int_value;
year = int_value;
val = tmp;
date_part_used = true;
break;
@ -1138,7 +1134,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
if (tmp - val <= 2) {
int_value += int_value >= 70 ? 1900 : 2000;
}
_year = int_value;
year = int_value;
val = tmp;
date_part_used = true;
break;
@ -1149,7 +1145,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
}
_month = int_value;
month = int_value;
val = tmp;
date_part_used = true;
break;
@ -1158,14 +1154,14 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
if (int_value < 0) {
return false;
}
_month = int_value;
month = int_value;
break;
case 'b':
int_value = check_word(s_ab_month_name, val, val_end, &val);
if (int_value < 0) {
return false;
}
_month = int_value;
month = int_value;
break;
// Day
case 'd':
@ -1174,7 +1170,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
}
_day = int_value;
day = int_value;
val = tmp;
date_part_used = true;
break;
@ -1183,7 +1179,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
}
_day = int_value;
day = int_value;
val = tmp + min(2, val_end - tmp);
date_part_used = true;
break;
@ -1199,7 +1195,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
}
_hour = int_value;
hour = int_value;
val = tmp;
time_part_used = true;
break;
@ -1209,7 +1205,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
}
_minute = int_value;
minute = int_value;
val = tmp;
time_part_used = true;
break;
@ -1220,7 +1216,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
}
_second = int_value;
second = int_value;
val = tmp;
time_part_used = true;
break;
@ -1231,7 +1227,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
return false;
}
int_value *= log_10_int[6 - (tmp - val)];
_microsecond = int_value;
microsecond = int_value;
val = tmp;
frac_part_used = true;
break;
@ -1326,12 +1322,14 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
}
val = tmp;
time_part_used = true;
already_set_time_part = true;
break;
case 'T':
if (!from_date_format_str("%H:%i:%S", 8, val, val_end - val, &tmp)) {
return false;
}
time_part_used = true;
already_set_time_part = true;
val = tmp;
break;
case '.':
@ -1397,18 +1395,39 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
}
if (usa_time) {
if (_hour > 12 || _hour < 1) {
if (hour > 12 || hour < 1) {
return false;
}
_hour = (_hour % 12) + day_part;
hour = (hour % 12) + day_part;
}
if (sub_val_end) {
*sub_val_end = val;
return true;
}
// Compute timestamp type
if (frac_part_used) {
if (date_part_used) {
_type = TIME_DATETIME;
} else {
_type = TIME_TIME;
}
} else {
if (date_part_used) {
if (time_part_used) {
_type = TIME_DATETIME;
} else {
_type = TIME_DATE;
}
} else {
_type = TIME_TIME;
}
}
_neg = false;
// Year day
if (yearday > 0) {
uint64_t days = calc_daynr(_year, 1, 1) + yearday - 1;
uint64_t days = calc_daynr(year, 1, 1) + yearday - 1;
if (!get_date_from_daynr(days)) {
return false;
}
@ -1434,31 +1453,17 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
return false;
}
}
// 1. already_set_date_part means _year, _month, _day be set, so we only set time part
// 2. already_set_time_part means _hour, _minute, _second, _microsecond be set,
// so we only neet to set date part
// 3. if both are true, means all part of date_time be set, no need check_range_and_set_time
bool already_set_date_part = yearday > 0 || (week_num >= 0 && weekday > 0);
if (already_set_date_part && already_set_time_part) return true;
if (already_set_date_part) return check_range_and_set_time(_year, _month, _day, hour, minute, second, microsecond, _type);
if (already_set_time_part) return check_range_and_set_time(year, month, day,
_hour, _minute, _second, _microsecond, _type);
// Compute timestamp type
if (frac_part_used) {
if (date_part_used) {
_type = TIME_DATETIME;
} else {
_type = TIME_TIME;
}
} else {
if (date_part_used) {
if (time_part_used) {
_type = TIME_DATETIME;
} else {
_type = TIME_DATE;
}
} else {
_type = TIME_TIME;
}
}
if (check_range() || check_date()) {
return false;
}
_neg = false;
return true;
return check_range_and_set_time(year, month, day, hour, minute, second, microsecond, _type);
}
bool DateTimeValue::date_add_interval(const TimeInterval& interval, TimeUnit unit) {
@ -1615,6 +1620,17 @@ DateTimeValue DateTimeValue::local_time() {
return value;
}
void DateTimeValue::set_time(uint32_t year, uint32_t month, uint32_t day, uint32_t hour,
uint32_t minute, uint32_t second, uint32_t microsecond) {
_year = year;
_month = month;
_day = day;
_hour = hour;
_minute = minute;
_second = second;
_microsecond = microsecond;
}
std::ostream& operator<<(std::ostream& os, const DateTimeValue& value) {
char buf[64];
value.to_string(buf);

64
be/src/runtime/datetime_value.h
View File

@ -155,6 +155,9 @@ public:
DateTimeValue(int64_t t) { from_date_int64(t); }
void set_time(uint32_t year, uint32_t month, uint32_t day, uint32_t hour,
uint32_t minute, uint32_t second, uint32_t microsecond);
// Converted from Olap Date or Datetime
bool from_olap_datetime(uint64_t datetime) {
_neg = 0;
@ -162,20 +165,18 @@ public:
uint64_t date = datetime / 1000000;
uint64_t time = datetime % 1000000;
_year = date / 10000;
auto [year, month, day, hour, minute, second, microsecond] = std::tuple{0,0,0,0,0,0,0};
year = date / 10000;
date %= 10000;
_month = date / 100;
_day = date % 100;
_hour = time / 10000;
month = date / 100;
day = date % 100;
hour = time / 10000;
time %= 10000;
_minute = time / 100;
_second = time % 100;
_microsecond = 0;
minute = time / 100;
second = time % 100;
microsecond = 0;
if (check_range() || check_date()) {
return false;
}
return true;
return check_range_and_set_time(year, month, day, hour, minute, second, microsecond, _type);
}
uint64_t to_olap_datetime() const {
@ -187,22 +188,16 @@ public:
bool from_olap_date(uint64_t date) {
_neg = 0;
_type = TIME_DATE;
_hour = 0;
_minute = 0;
_second = 0;
_microsecond = 0;
_day = date & 0x1f;
auto [year, month, day, hour, minute, second, microsecond] = std::tuple{0,0,0,0,0,0,0};
day = date & 0x1f;
date >>= 5;
_month = date & 0x0f;
month = date & 0x0f;
date >>= 4;
year = date;
_year = date;
if (check_range() || check_date()) {
return false;
}
return true;
return check_range_and_set_time(year, month, day, hour, minute, second, microsecond, _type);
}
uint64_t to_olap_date() const {
@ -254,10 +249,25 @@ public:
// compute the length of data format pattern
static int compute_format_len(const char* format, int len);
// Return true if range or date is invalid
static bool check_range(uint32_t year, uint32_t month, uint32_t day, uint32_t hour,
uint32_t minute, uint32_t second, uint32_t microsecond, uint16_t type);
static bool check_date(uint32_t year, uint32_t month, uint32_t day);
// Convert this value to uint64_t
// Will check its type
int64_t to_int64() const;
bool check_range_and_set_time(uint32_t year, uint32_t month, uint32_t day, uint32_t hour,
uint32_t minute, uint32_t second, uint32_t microsecond, uint16_t type) {
if (check_range(year, month, day, hour, minute, second, microsecond, type)) {
return false;
}
set_time(year, month, day, hour, minute, second, microsecond);
return true;
};
inline uint64_t daynr() const { return calc_daynr(_year, _month, _day); }
// Calculate how many days since 0000-01-01
@ -272,6 +282,7 @@ public:
int hour() const { return _hour; }
int minute() const { return _minute; }
int second() const { return _second; }
int microsecond() const { return _microsecond; }
bool check_loss_accuracy_cast_to_date() {
auto loss_accuracy = _hour != 0 || _minute != 0 || _second != 0 || _microsecond != 0;
@ -455,7 +466,8 @@ public:
int type() const { return _type; }
bool is_valid_date() const { return !check_range() && !check_date() && _month > 0 && _day > 0; }
bool is_valid_date() const { return !check_range(_year, _month, _day,
_hour, _minute, _second, _microsecond, _type) && _month > 0 && _day > 0; }
private:
// Used to make sure sizeof DateTimeValue
@ -497,10 +509,6 @@ private:
return _neg ? -tmp : tmp;
}
// Return true if range or date is invalid
bool check_range() const;
bool check_date() const;
// Used to construct from int value
int64_t standardize_timevalue(int64_t value);

2
be/test/exprs/timestamp_functions_test.cpp
View File

@ -388,7 +388,7 @@ TEST_F(TimestampFunctionsTest, timestampdiff_test) {
ASSERT_DIFF(month, 0, tv2, tv1);
ASSERT_DIFF(month, 0, tv2, tv1);
ASSERT_DIFF(month, 0, 20120924000000, tv1);
ASSERT_DIFF(month, 1, tv1, 20120631000000);
ASSERT_DIFF(month, 1, tv1, 20120630000000);
//WEEK
ASSERT_DIFF(week, 0, tv2, tv1);
//DAY

42
be/test/runtime/datetime_value_test.cpp
View File

@ -191,40 +191,48 @@ TEST_F(DateTimeValueTest, local_time) {
// Test check range
TEST_F(DateTimeValueTest, check_range) {
DateTimeValue value;
value.from_date_int64(19880201123456);
ASSERT_FALSE(value.check_range());
ASSERT_TRUE(value.from_date_int64(19880201123456));
value._year = 10000;
ASSERT_TRUE(value.check_range());
ASSERT_TRUE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._year = 1988;
value._month = 13;
ASSERT_TRUE(value.check_range());
ASSERT_TRUE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._month = 2;
value._day = 32;
ASSERT_TRUE(value.check_range());
ASSERT_TRUE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._day = 1;
value._hour = TIME_MAX_HOUR;
ASSERT_TRUE(value.check_range());
ASSERT_TRUE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._type = TIME_TIME;
ASSERT_FALSE(value.check_range());
ASSERT_FALSE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._hour = TIME_MAX_HOUR + 1;
ASSERT_TRUE(value.check_range());
ASSERT_TRUE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._type = TIME_DATETIME;
value._hour = 12;
value._minute = 60;
ASSERT_TRUE(value.check_range());
ASSERT_TRUE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._minute = 34;
value._second = 60;
ASSERT_TRUE(value.check_range());
ASSERT_TRUE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._second = 56;
value._microsecond = 1000000;
ASSERT_TRUE(value.check_range());
ASSERT_TRUE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._month = 0;
}
@ -233,16 +241,20 @@ TEST_F(DateTimeValueTest, check_date) {
ASSERT_TRUE(value.from_date_int64(19880201));
value._month = 0;
ASSERT_FALSE(value.check_date());
ASSERT_FALSE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._month = 2;
value._day = 0;
ASSERT_FALSE(value.check_date());
ASSERT_FALSE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._year = 1987;
value._day = 29;
ASSERT_TRUE(value.check_date());
ASSERT_TRUE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
value._year = 2000;
ASSERT_FALSE(value.check_date());
ASSERT_FALSE(DateTimeValue::check_range(value.year(), value.month(), value.day(),
value.hour(), value.minute(), value.second(), value.microsecond(), value.type()));
}
// Calculate format