database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](function) Use Dict to opt the function of time_round (#25029)

Browse Source 
Before：

select hour_floor(`@timestamp`, 7) as t, count() as cnt from httplogs_date group by t order by t limit 10;
+---------------------+--------+
| t                   | cnt    |
+---------------------+--------+
| 1998-04-30 21:00:00 |    324 |
| 1998-05-01 04:00:00 | 286156 |
| 1998-05-01 11:00:00 | 266130 |
| 1998-05-01 18:00:00 | 483765 |
| 1998-05-02 01:00:00 | 276706 |
| 1998-05-02 08:00:00 | 169945 |
| 1998-05-02 15:00:00 | 223593 |
| 1998-05-02 22:00:00 | 272616 |
| 1998-05-03 05:00:00 | 188689 |
| 1998-05-03 12:00:00 | 184405 |
+---------------------+--------+
10 rows in set (3.39 sec)
after:

select hour_floor(`@timestamp`, 7) as t, count() as cnt from httplogs_date group by t order by t limit 10;
+---------------------+--------+
| t                   | cnt    |
+---------------------+--------+
| 1998-04-30 21:00:00 |    324 |
| 1998-05-01 04:00:00 | 286156 |
| 1998-05-01 11:00:00 | 266130 |
| 1998-05-01 18:00:00 | 483765 |
| 1998-05-02 01:00:00 | 276706 |
| 1998-05-02 08:00:00 | 169945 |
| 1998-05-02 15:00:00 | 223593 |
| 1998-05-02 22:00:00 | 272616 |
| 1998-05-03 05:00:00 | 188689 |
| 1998-05-03 12:00:00 | 184405 |
+---------------------+--------+
10 rows in set (2.19 sec)
This commit is contained in:
HappenLee
2023-10-04 23:34:24 +08:00
committed by GitHub
parent 4ce5213b1c
commit 93eedaff62
4 changed files with 79 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
be/src/util/time_lut.cpp
View File

@ -18,6 +18,8 @@
#include "util/time_lut.h"
#include "vec/runtime/vdatetime_value.h"
namespace doris {
TimeLUTImpl::TimeLUTImpl() {
init_time_lut();
@ -94,6 +96,12 @@ uint8_t calc_weekday(uint64_t day_nr, bool is_sunday_first_day) {
}
uint32_t calc_daynr(uint16_t year, uint8_t month, uint8_t day) {
// date_day_offet_dict range from [1900-01-01, 2039-10-24]
if (vectorized::date_day_offset_dict::can_speed_up_calc_daynr(year) &&
LIKELY(vectorized::date_day_offset_dict::get_dict_init())) {
return vectorized::date_day_offset_dict::get().daynr(year, month, day);
}
uint32_t delsum = 0;
int y = year;

2
be/src/vec/functions/function_datetime_floor_ceil.cpp
View File

@ -743,7 +743,7 @@ struct TimeRound {
//round down/up inside time period(several time-units)
int64_t count = period;
int64_t delta_inside_period = (diff % count + count) % count;
int64_t delta_inside_period = diff >= 0 ? diff % count : (diff % count + count) % count;
int64_t step = diff - delta_inside_period +
(Impl::Type == FLOOR ? 0
: delta_inside_period == 0 ? 0

76
be/src/vec/runtime/vdatetime_value.cpp
View File

@ -2656,27 +2656,42 @@ typename DateV2Value<T>::underlying_value DateV2Value<T>::to_date_int_val() cons
static std::array<DateV2Value<DateV2ValueType>, date_day_offset_dict::DICT_DAYS>
DATE_DAY_OFFSET_ITEMS;
static std::array<std::array<std::array<int, 31>, 12>, 140> DATE_DAY_OFFSET_DICT;
static bool DATE_DAY_OFFSET_ITEMS_INIT = false;
date_day_offset_dict date_day_offset_dict::instance = date_day_offset_dict();
date_day_offset_dict& date_day_offset_dict::get() {
return instance;
}
bool date_day_offset_dict::get_dict_init() {
return DATE_DAY_OFFSET_ITEMS_INIT;
}
date_day_offset_dict::date_day_offset_dict() {
DateV2Value<DateV2ValueType> d;
d.set_time(1969, 12, 31, 0, 0, 0, 0);
for (int i = 0; i < DAY_AFTER_EPOCH; ++i) {
DATE_DAY_OFFSET_ITEMS[DAY_BEFORE_EPOCH + i] = d;
DATE_DAY_OFFSET_DICT[d.year() - START_YEAR][d.month() - 1][d.day() - 1] =
calc_daynr(d.year(), d.month(), d.day());
d += 1;
}
d.set_time(1969, 12, 31, 0, 0, 0, 0);
for (int i = 0; i <= DAY_BEFORE_EPOCH; ++i) {
DATE_DAY_OFFSET_ITEMS[DAY_BEFORE_EPOCH - i] = d;
DATE_DAY_OFFSET_DICT[d.year() - START_YEAR][d.month() - 1][d.day() - 1] =
calc_daynr(d.year(), d.month(), d.day());
d -= 1;
}
DATE_DAY_OFFSET_ITEMS_INIT = true;
}
DateV2Value<DateV2ValueType> date_day_offset_dict::operator[](int day) {
DateV2Value<DateV2ValueType> date_day_offset_dict::operator[](int day) const {
int index = day + DAY_BEFORE_EPOCH;
if (LIKELY(index >= 0 && index < DICT_DAYS)) {
return DATE_DAY_OFFSET_ITEMS[index];
@ -2686,6 +2701,10 @@ DateV2Value<DateV2ValueType> date_day_offset_dict::operator[](int day) {
}
}
int date_day_offset_dict::daynr(int year, int month, int day) const {
return DATE_DAY_OFFSET_DICT[year - START_YEAR][month - 1][day - 1];
}
template <typename T>
uint32_t DateV2Value<T>::set_date_uint32(uint32_t int_val) {
union DateV2UInt32Union {
@ -2756,34 +2775,43 @@ bool DateV2Value<T>::get_date_from_daynr(uint64_t daynr) {
if (daynr <= 0 || daynr > DATE_MAX_DAYNR) {
return false;
}
auto [year, month, day] = std::tuple {0, 0, 0};
year = daynr / 365;
uint32_t days_befor_year = 0;
while (daynr < (days_befor_year = doris::calc_daynr(year, 1, 1))) {
year--;
}
uint32_t days_of_year = daynr - days_befor_year + 1;
int leap_day = 0;
if (doris::is_leap(year)) {
if (days_of_year > 31 + 28) {
days_of_year--;
if (days_of_year == 31 + 28) {
leap_day = 1;
if (date_day_offset_dict::can_speed_up_daynr_to_date(daynr) &&
LIKELY(date_day_offset_dict::get_dict_init())) {
auto dt = date_day_offset_dict::get()[date_day_offset_dict::get_offset_by_daynr(daynr)];
year = dt.year();
month = dt.month();
day = dt.day();
} else {
year = daynr / 365;
uint32_t days_befor_year = 0;
while (daynr < (days_befor_year = doris::calc_daynr(year, 1, 1))) {
year--;
}
uint32_t days_of_year = daynr - days_befor_year + 1;
int leap_day = 0;
if (doris::is_leap(year)) {
if (days_of_year > 31 + 28) {
days_of_year--;
if (days_of_year == 31 + 28) {
leap_day = 1;
}
}
}
}
month = 1;
while (days_of_year > s_days_in_month[month]) {
days_of_year -= s_days_in_month[month];
month++;
}
day = days_of_year + leap_day;
month = 1;
while (days_of_year > s_days_in_month[month]) {
days_of_year -= s_days_in_month[month];
month++;
}
day = days_of_year + leap_day;
if (is_invalid(year, month, day, this->hour(), this->minute(), this->second(),
this->microsecond())) {
return false;
if (is_invalid(year, month, day, this->hour(), this->minute(), this->second(),
this->microsecond())) {
return false;
}
}
set_time(year, month, day, this->hour(), this->minute(), this->second(), this->microsecond());
return true;
}

19
be/src/vec/runtime/vdatetime_value.h
View File

@ -1524,9 +1524,26 @@ public:
static constexpr int DAY_AFTER_EPOCH = 25500; // 2039-10-24
static constexpr int DICT_DAYS = DAY_BEFORE_EPOCH + DAY_AFTER_EPOCH;
static constexpr int START_YEAR = 1900; // 1900-01-01
static constexpr int END_YEAR = 2039; // 2039-10-24
static constexpr int DAY_OFFSET_CAL_START_POINT_DAYNR = 719527; // 1969-12-31
static bool can_speed_up_calc_daynr(int year) { return year >= START_YEAR && year < END_YEAR; }
static int get_offset_by_daynr(int daynr) { return daynr - DAY_OFFSET_CAL_START_POINT_DAYNR; }
static bool can_speed_up_daynr_to_date(int daynr) {
auto res = get_offset_by_daynr(daynr);
return res >= 0 ? res <= DAY_AFTER_EPOCH : -res <= DAY_BEFORE_EPOCH;
}
static date_day_offset_dict& get();
DateV2Value<DateV2ValueType> operator[](int day);
static bool get_dict_init();
DateV2Value<DateV2ValueType> operator[](int day) const;
int daynr(int year, int month, int day) const;
};
template <typename T>