From a60cb378cd7e20ee474afff2e897a960094e6bc5 Mon Sep 17 00:00:00 2001 From: Tsunaou <895254752@qq.com> Date: Thu, 16 May 2024 06:26:20 +0000 Subject: [PATCH] refactor ObCompactionTimeGuard to process failed event --- .../freeze/ob_major_merge_progress_util.cpp | 30 -- .../freeze/ob_major_merge_progress_util.h | 20 -- .../compaction/ob_compaction_time_guard.cpp | 272 ++++++++++++++++-- .../compaction/ob_compaction_time_guard.h | 157 ++++++++-- .../ob_compaction_schedule_util.cpp | 40 --- .../compaction/ob_compaction_schedule_util.h | 29 -- .../compaction/ob_tablet_merge_task.cpp | 65 ----- src/storage/compaction/ob_tablet_merge_task.h | 29 -- unittest/share/CMakeLists.txt | 1 + unittest/share/compaction/CMakeLists.txt | 1 + .../compaction/test_compaction_time_guard.cpp | 160 +++++++++++ 11 files changed, 546 insertions(+), 258 deletions(-) create mode 100644 unittest/share/compaction/CMakeLists.txt create mode 100644 unittest/share/compaction/test_compaction_time_guard.cpp diff --git a/src/rootserver/freeze/ob_major_merge_progress_util.cpp b/src/rootserver/freeze/ob_major_merge_progress_util.cpp index 904e9d379..be364b986 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_util.cpp +++ b/src/rootserver/freeze/ob_major_merge_progress_util.cpp @@ -82,36 +82,6 @@ int64_t ObMergeProgress::to_string(char *buf, const int64_t buf_len) const } return pos; } -/** - * -------------------------------------------------------------------ObRSCompactionTimeGuard------------------------------------------------------------------- - */ -const char *ObRSCompactionTimeGuard::CompactionEventStr[] = { - "PREPARE_UNFINISH_TABLE_IDS", - "GET_TABLET_LS_PAIRS", - "GET_TABLET_META_TABLE", - "CKM_VERIFICATION" -}; - -const char *ObRSCompactionTimeGuard::get_comp_event_str(enum CompactionEvent event) -{ - STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); - const char *str = ""; - if (event >= COMPACTION_EVENT_MAX || event < PREPARE_UNFINISH_TABLE_IDS) { - str = "invalid_type"; - } else { - str = CompactionEventStr[event]; - } - return str; -} - -int64_t ObRSCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const -{ - int64_t pos = 0; - for (int64_t idx = 0; idx < idx_; ++idx) { - fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str((CompactionEvent)line_array_[idx]), click_poinsts_[idx]); - } - return pos; -} /** * -------------------------------------------------------------------ObTabletLSPairCache------------------------------------------------------------------- diff --git a/src/rootserver/freeze/ob_major_merge_progress_util.h b/src/rootserver/freeze/ob_major_merge_progress_util.h index 59ef668c6..33de0a984 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_util.h +++ b/src/rootserver/freeze/ob_major_merge_progress_util.h @@ -209,26 +209,6 @@ typedef hash::ObHashMap ObTabletStatusMap; typedef common::ObArray ObTabletLSPairArray; typedef hash::ObHashMap ObTableCompactionInfoMap; -struct ObRSCompactionTimeGuard : public ObCompactionTimeGuard -{ -public: - ObRSCompactionTimeGuard() - : ObCompactionTimeGuard(UINT64_MAX, "[RS] ") - {} - virtual ~ObRSCompactionTimeGuard() {} - enum CompactionEvent : uint16_t { - PREPARE_UNFINISH_TABLE_IDS = 0, - GET_TABLET_LS_PAIRS, - GET_TABLET_META_TABLE, - CKM_VERIFICATION, - COMPACTION_EVENT_MAX, - }; - virtual int64_t to_string(char *buf, const int64_t buf_len) const override; -private: - const static char *CompactionEventStr[]; - static const char *get_comp_event_str(enum CompactionEvent event); -}; - struct ObCkmValidatorStatistics { ObCkmValidatorStatistics() { reset(); } diff --git a/src/share/compaction/ob_compaction_time_guard.cpp b/src/share/compaction/ob_compaction_time_guard.cpp index 156e1647b..348ccb310 100644 --- a/src/share/compaction/ob_compaction_time_guard.cpp +++ b/src/share/compaction/ob_compaction_time_guard.cpp @@ -14,45 +14,263 @@ namespace oceanbase namespace compaction { +/** + * -------------------------------------------------------------------ObCompactionTimeGuard------------------------------------------------------------------- + */ +ObCompactionTimeGuard::~ObCompactionTimeGuard() +{ + int64_t total_cost = 0; + for (int64_t idx = 0; idx < size_; ++idx) { + total_cost += event_times_[idx]; + } + total_cost += common::ObTimeUtility::current_time() - last_click_ts_; + if (OB_UNLIKELY(total_cost >= warn_threshold_)) { + ::oceanbase::common::OB_PRINT(log_mod_, OB_LOG_LEVEL_DIRECT_NO_ERRCODE(WARN), OB_SUCCESS, "cost too much time", LOG_KVS(K(*this))); + } +} + +int64_t ObCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + fmt_ts_to_meaningful_str(buf, buf_len, pos, "|threshold", warn_threshold_); + common::databuff_printf(buf, buf_len, pos, "start at %s|", common::ObTime2Str::ob_timestamp_str(add_time_)); + int64_t total_cost = 0; + for (int64_t idx = 0; idx < size_; ++idx) { + const uint64_t ts = event_times_[idx]; + if (ts < 1_ms) { + common::databuff_printf(buf, buf_len, pos, "%ldus|", ts); + } else if (ts < 1_s) { + common::databuff_printf(buf, buf_len, pos, "%.2lfms|", double(ts) / 1_ms); + } else { + common::databuff_printf(buf, buf_len, pos, "%.2lfs|", double(ts) / 1_s); + } + total_cost += event_times_[idx]; + } + total_cost += common::ObTimeUtility::current_time() - last_click_ts_; + fmt_ts_to_meaningful_str(buf, buf_len, pos, "total", total_cost); + if (pos != 0 && pos < buf_len) { + pos -= 1; + } + return pos; +} + +void ObCompactionTimeGuard::reuse() +{ + size_ = 0; + last_click_ts_ = common::ObTimeUtility::current_time(); + add_time_ = common::ObTimeUtility::current_time(); + for (uint16_t i = 0; i < capacity_; ++i) { + event_times_[i] = 0; + } +} + +bool ObCompactionTimeGuard::click(const uint16_t event) +{ + if (OB_LIKELY(event < CAPACITY)) { + if (OB_LIKELY(size_ <= event)) { + size_ = event + 1; + } + const int64_t now = common::ObTimeUtility::current_time(); + event_times_[event] += now - last_click_ts_; + last_click_ts_ = now; + } + return true; +} + +void ObCompactionTimeGuard::fmt_ts_to_meaningful_str( + char *buf, + const int64_t buf_len, + int64_t &pos, + const char *lvalue, + const int64_t ts) const +{ + common::databuff_printf(buf, buf_len, pos, "%s", lvalue); + if (ts < 1_ms) { + common::databuff_printf(buf, buf_len, pos, "=%ldus|", ts); + } else if (ts < 1_s) { + common::databuff_printf(buf, buf_len, pos, "=%.2lfms|", double(ts) / 1_ms); + } else { + common::databuff_printf(buf, buf_len, pos, "=%.2lfs|", double(ts) / 1_s); + } +} void ObCompactionTimeGuard::add_time_guard(const ObCompactionTimeGuard &other) { - // last_click_ts_ is not useflu - ObCompactionTimeGuard time_guard; - int i = 0; - int j = 0; - while (i < idx_ && j < other.idx_) { - if (line_array_[i] == other.line_array_[j]) { - time_guard.line_array_[time_guard.idx_] = line_array_[i]; - time_guard.click_poinsts_[time_guard.idx_++] = click_poinsts_[i++] + other.click_poinsts_[j++]; - } else if (line_array_[i] < other.line_array_[j]) { - time_guard.line_array_[time_guard.idx_] = line_array_[i]; - time_guard.click_poinsts_[time_guard.idx_++] = click_poinsts_[i++]; - } else { - time_guard.line_array_[time_guard.idx_] = other.line_array_[j]; - time_guard.click_poinsts_[time_guard.idx_++] = other.click_poinsts_[j++]; + if (OB_LIKELY(guard_type_ == other.guard_type_ && CAPACITY == other.capacity_)) { + size_ = std::max(size_, other.size_); + for (uint16_t i = 0; i < size_; i++) { + event_times_[i] += other.event_times_[i]; } } - while (i < idx_) { - time_guard.line_array_[time_guard.idx_] = line_array_[i]; - time_guard.click_poinsts_[time_guard.idx_++] = click_poinsts_[i++]; - } - while (j < other.idx_) { - time_guard.line_array_[time_guard.idx_] = other.line_array_[j]; - time_guard.click_poinsts_[time_guard.idx_++] = other.click_poinsts_[j++]; - } - *this = time_guard; } ObCompactionTimeGuard & ObCompactionTimeGuard::operator=(const ObCompactionTimeGuard &other) { + guard_type_ = other.guard_type_; + capacity_ = other.capacity_; + size_ = other.size_; last_click_ts_ = other.last_click_ts_; - idx_ = other.idx_; - for (int i = 0; i < other.idx_; ++i) { - line_array_[i] = other.line_array_[i]; - click_poinsts_[i] = other.click_poinsts_[i]; + add_time_ = other.add_time_; + for (uint16_t i = 0; i < other.size_; ++i) { + event_times_[i] = other.event_times_[i]; } return *this; } +uint16_t ObCompactionTimeGuard::get_max_event_count(const ObCompactionTimeGuardType guard_type) +{ + uint16_t max_event_count = CAPACITY; + if (RS_COMPACT_TIME_GUARD == guard_type) { + max_event_count = ObRSCompactionTimeGuard::COMPACTION_EVENT_MAX; + } else if (SCHEDULE_COMPACT_TIME_GUARD == guard_type) { + max_event_count = ObCompactionScheduleTimeGuard::COMPACTION_EVENT_MAX; + } else if (STORAGE_COMPACT_TIME_GUARD == guard_type) { + max_event_count = ObStorageCompactionTimeGuard::COMPACTION_EVENT_MAX; + } + return max_event_count; +} + +/** + * -------------------------------------------------------------------ObRSCompactionTimeGuard------------------------------------------------------------------- + */ +const char *ObRSCompactionTimeGuard::CompactionEventStr[] = { + "PREPARE_UNFINISH_TABLE_IDS", + "GET_TABLET_LS_PAIRS", + "GET_TABLET_META_TABLE", + "CKM_VERIFICATION" +}; + +const char *ObRSCompactionTimeGuard::get_comp_event_str(enum CompactionEvent event) +{ + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) <= static_cast(CAPACITY), "too many events, need update CAPACITY"); + const char *str = ""; + if (event >= COMPACTION_EVENT_MAX || event < PREPARE_UNFINISH_TABLE_IDS) { + str = "invalid_type"; + } else { + str = CompactionEventStr[event]; + } + return str; +} + +int64_t ObRSCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + for (uint16_t idx = 0; idx < size_; ++idx) { + if (event_times_[idx] > 0) { + fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str(static_cast(idx)), event_times_[idx]); + } + } + return pos; +} + +/** + * ObCompactionScheduleTimeGuard Impl + */ +const char *ObCompactionScheduleTimeGuard::CompactionEventStr[] = { + "GET_TABLET", + "UPDATE_TABLET_REPORT_STATUS", + "READ_MEDIUM_INFO", + "SCHEDULE_NEXT_MEDIUM", + "SCHEDULE_TABLET_MEDIUM", + "FAST_FREEZE", + "SEARCH_META_TABLE", + "CHECK_META_TABLE", + "SEARCH_CHECKSUM", + "CHECK_CHECKSUM", + "SCHEDULER_NEXT_ROUND" +}; + +const char *ObCompactionScheduleTimeGuard::get_comp_event_str(enum CompactionEvent event) +{ + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) <= static_cast(CAPACITY), "too many events, need update CAPACITY"); + const char *str = ""; + if (event >= COMPACTION_EVENT_MAX || event < GET_TABLET) { + str = "invalid_type"; + } else { + str = CompactionEventStr[event]; + } + return str; +} + +int64_t ObCompactionScheduleTimeGuard::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + for (int16_t idx = 0; idx < size_; ++idx) { + if (event_times_[idx] > 0) { + fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str(static_cast(idx)), event_times_[idx]); + } + } + return pos; +} + +/* + * ----------------------------------------------ObCompactionTimeGuard-------------------------------------------------- + */ +constexpr float ObStorageCompactionTimeGuard::COMPACTION_SHOW_PERCENT_THRESHOLD; +const char *ObStorageCompactionTimeGuard::CompactionEventStr[] = { + "WAIT_TO_SCHEDULE", + "COMPACTION_POLICY", + "PRE_PROCESS_TX_TABLE", + "GET_PARALLEL_RANGE", + "EXECUTE", + "CREATE_SSTABLE", + "UPDATE_TABLET", + "RELEASE_MEMTABLE", + "SCHEDULE_OTHER_COMPACTION", + "DAG_FINISH" +}; + +const char *ObStorageCompactionTimeGuard::get_comp_event_str(const enum CompactionEvent event) +{ + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); + STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) <= static_cast(CAPACITY), "too many events, need update CAPACITY"); + const char *str = ""; + if (event >= COMPACTION_EVENT_MAX || event < DAG_WAIT_TO_SCHEDULE) { + str = "invalid_type"; + } else { + str = CompactionEventStr[event]; + } + return str; +} + +int64_t ObStorageCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + int64_t total_cost = 0; + J_KV(K_(add_time)); + common::databuff_printf(buf, buf_len, pos, "|"); + if (size_ > DAG_WAIT_TO_SCHEDULE && event_times_[DAG_WAIT_TO_SCHEDULE] > COMPACTION_SHOW_TIME_THRESHOLD) { + fmt_ts_to_meaningful_str(buf, buf_len, pos, "wait_schedule_time", event_times_[DAG_WAIT_TO_SCHEDULE]); + } + for (int64_t idx = COMPACTION_POLICY; idx < size_; ++idx) { + total_cost += event_times_[idx]; + } + if (total_cost > COMPACTION_SHOW_TIME_THRESHOLD) { + float ratio = 0; + for (int64_t idx = COMPACTION_POLICY; idx < size_; ++idx) { + const uint32_t time_interval = event_times_[idx]; // include the retry time since previous event + ratio = (float)(time_interval)/ total_cost; + if (ratio >= COMPACTION_SHOW_PERCENT_THRESHOLD || time_interval >= COMPACTION_SHOW_TIME_THRESHOLD) { + fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str(static_cast(idx)), event_times_[idx]); + if (ratio > 0.01) { + common::databuff_printf(buf, buf_len, pos, "(%.2f)", ratio); + } + common::databuff_printf(buf, buf_len, pos, "|"); + } + } + } + fmt_ts_to_meaningful_str(buf, buf_len, pos, "total", total_cost); + if (pos != 0 && pos < buf_len) { + buf[pos - 1] = ';'; + } + + if (pos != 0 && pos < buf_len) { + pos -= 1; + } + return pos; +} + + } // namespace compaction } // namespace oceanbase diff --git a/src/share/compaction/ob_compaction_time_guard.h b/src/share/compaction/ob_compaction_time_guard.h index dd17ddbae..c5253404a 100644 --- a/src/share/compaction/ob_compaction_time_guard.h +++ b/src/share/compaction/ob_compaction_time_guard.h @@ -10,48 +10,169 @@ #ifndef OB_SHARE_COMPACTION_COMPACTION_TIME_GUARD_H_ #define OB_SHARE_COMPACTION_COMPACTION_TIME_GUARD_H_ #include "share/ob_occam_time_guard.h" +#include "lib/container/ob_se_array.h" namespace oceanbase { namespace compaction { -class ObCompactionTimeGuard : public common::occam::ObOccamTimeGuard +/* + * ObCompactionTimeGuard refers to the implementation of from ObOccamTimeGuard. + * For example, you have 3 enum events {e0, e1, e2} + * If you want to record the time cost of event e1, you can use click(e1), then event_times_[1] will accumulate the time cost of e1. + * + * ObCompactionTimeGuard + * -- ObRSCompactionTimeGuard + * -- ObScheduleCompactionTimeGuard + * -- ObStorageCompactionTimeGuard + */ +class ObCompactionTimeGuard { public: - const static uint64_t WARN_THRESHOLD = 30L * 1000 * 1000; // 30s - ObCompactionTimeGuard(const uint64_t warn_threshold = WARN_THRESHOLD, const char *mod = "") - : ObOccamTimeGuard(warn_threshold, nullptr, nullptr, mod), - add_time_(0) - {} - virtual ~ObCompactionTimeGuard() {} - virtual int64_t to_string(char *buf, const int64_t buf_len) const + enum ObCompactionTimeGuardType : uint8_t { - UNUSEDx(buf, buf_len); - return 0; + BASE_COMPACT_TIME_GUARD = 0, + RS_COMPACT_TIME_GUARD = 1, + SCHEDULE_COMPACT_TIME_GUARD = 2, + STORAGE_COMPACT_TIME_GUARD = 3, + MAX_COMPACT_TIME_GUARD + }; +public: + const static uint64_t WARN_THRESHOLD = 30L * 1000 * 1000; // 30s + const static uint16_t CAPACITY = 16; + ObCompactionTimeGuard(const ObCompactionTimeGuardType gurad_type = BASE_COMPACT_TIME_GUARD, + const uint64_t warn_threshold = WARN_THRESHOLD, + const char *mod = "") + : guard_type_(gurad_type), + warn_threshold_(warn_threshold), + log_mod_(mod), + capacity_(get_max_event_count(gurad_type)), + size_(0), + last_click_ts_(common::ObTimeUtility::current_time()), + add_time_(common::ObTimeUtility::current_time()) + { + reuse(); } + virtual ~ObCompactionTimeGuard(); + virtual int64_t to_string(char *buf, const int64_t buf_len) const; + void reuse(); void add_time_guard(const ObCompactionTimeGuard &other); ObCompactionTimeGuard & operator=(const ObCompactionTimeGuard &other); - OB_INLINE bool is_empty() const { return 0 == idx_; } + OB_INLINE bool is_empty() const { return 0 == size_; } // set the dag add_time as the first click time OB_INLINE void set_last_click_ts(const int64_t time) { last_click_ts_ = time; add_time_ = time; } - OB_INLINE uint32_t get_specified_cost_time(const int64_t line) const { + OB_INLINE uint32_t get_specified_cost_time(const int64_t event) const { uint32_t ret_val = 0; - for (int64_t idx = 0; idx < idx_; ++idx) { - if (line_array_[idx] == line) { - ret_val = click_poinsts_[idx]; - break; - } + if (OB_LIKELY(event < size_)) { + ret_val = event_times_[event]; } return ret_val; } - + bool click(const uint16_t event); + // copy from ObOccamTimeGuard + void fmt_ts_to_meaningful_str( + char *buf, + const int64_t buf_len, + int64_t &pos, + const char *lvalue, + const int64_t ts) const; +public: + template + static constexpr uint16_t event_idx(E e) { return static_cast(e); } + static uint16_t get_max_event_count(const ObCompactionTimeGuardType guard_type); +public: + ObCompactionTimeGuardType guard_type_; + const uint64_t warn_threshold_; + const char * log_mod_; + uint16_t capacity_; // equal with CAPACITY, used for child class to check the array boundary + uint16_t size_; + int64_t last_click_ts_; int64_t add_time_; + uint64_t event_times_[CAPACITY]; }; +struct ObRSCompactionTimeGuard : public ObCompactionTimeGuard +{ +public: + ObRSCompactionTimeGuard() + : ObCompactionTimeGuard(RS_COMPACT_TIME_GUARD, UINT64_MAX, "[RS] ") + {} + virtual ~ObRSCompactionTimeGuard() {} + enum CompactionEvent : uint16_t { + PREPARE_UNFINISH_TABLE_IDS = 0, + GET_TABLET_LS_PAIRS, + GET_TABLET_META_TABLE, + CKM_VERIFICATION, + COMPACTION_EVENT_MAX, + }; + virtual int64_t to_string(char *buf, const int64_t buf_len) const override; +private: + const static char *CompactionEventStr[]; + static const char *get_comp_event_str(const enum CompactionEvent event); +}; +struct ObCompactionScheduleTimeGuard : public ObCompactionTimeGuard +{ +public: + ObCompactionScheduleTimeGuard() + : ObCompactionTimeGuard(SCHEDULE_COMPACT_TIME_GUARD, UINT64_MAX, "[STORAGE] ") + {} + virtual ~ObCompactionScheduleTimeGuard() {} + enum CompactionEvent : uint16_t { + // medium scheduler + GET_TABLET = 0, + UPDATE_TABLET_REPORT_STATUS, + READ_MEDIUM_INFO, + SCHEDULE_NEXT_MEDIUM, + SCHEDULE_TABLET_MEDIUM, + FAST_FREEZE, + // medium checker + SEARCH_META_TABLE, + CHECK_META_TABLE, + SEARCH_CHECKSUM, + CHECK_CHECKSUM, + SCHEDULER_NEXT_ROUND, + COMPACTION_EVENT_MAX + }; + virtual int64_t to_string(char *buf, const int64_t buf_len) const override; +private: + const static char *CompactionEventStr[]; + static const char *get_comp_event_str(const enum CompactionEvent event); +}; + +struct ObStorageCompactionTimeGuard : public ObCompactionTimeGuard +{ +public: + ObStorageCompactionTimeGuard() + : ObCompactionTimeGuard(STORAGE_COMPACT_TIME_GUARD, COMPACTION_WARN_THRESHOLD_RATIO, "[STORAGE] ") + {} + virtual ~ObStorageCompactionTimeGuard() {} + enum CompactionEvent : uint16_t { + DAG_WAIT_TO_SCHEDULE = 0, + COMPACTION_POLICY, + PRE_PROCESS_TX_TABLE, + GET_PARALLEL_RANGE, + EXECUTE, + CREATE_SSTABLE, + UPDATE_TABLET, + RELEASE_MEMTABLE, + SCHEDULE_OTHER_COMPACTION, + DAG_FINISH, + COMPACTION_EVENT_MAX + }; + virtual int64_t to_string(char *buf, const int64_t buf_len) const override; +private: + const static char *CompactionEventStr[]; + static const char *get_comp_event_str(const enum CompactionEvent event); + static const int64_t COMPACTION_WARN_THRESHOLD_RATIO = 60 * 1000L * 1000L; // 1 min + static constexpr float COMPACTION_SHOW_PERCENT_THRESHOLD = 0.1; + static const int64_t COMPACTION_SHOW_TIME_THRESHOLD = 1 * 1000L * 1000L; // 1s +}; + + } // namespace compaction } // namespace oceanbase diff --git a/src/storage/compaction/ob_compaction_schedule_util.cpp b/src/storage/compaction/ob_compaction_schedule_util.cpp index 099939314..b615f1cca 100644 --- a/src/storage/compaction/ob_compaction_schedule_util.cpp +++ b/src/storage/compaction/ob_compaction_schedule_util.cpp @@ -18,45 +18,5 @@ namespace oceanbase namespace compaction { -/** - * ObCompactionScheduleTimeGuard Impl - */ -const char *ObCompactionScheduleTimeGuard::CompactionEventStr[] = { - "GET_TABLET", - "UPDATE_TABLET_REPORT_STATUS", - "READ_MEDIUM_INFO", - "SCHEDULE_NEXT_MEDIUM", - "SCHEDULE_TABLET_MEDIUM", - "FAST_FREEZE", - "SEARCH_META_TABLE", - "CHECK_META_TABLE", - "SEARCH_CHECKSUM", - "CHECK_CHECKSUM", - "SCHEDULER_NEXT_ROUND" -}; - -const char *ObCompactionScheduleTimeGuard::get_comp_event_str(enum CompactionEvent event) -{ - STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); - const char *str = ""; - if (event >= COMPACTION_EVENT_MAX || event < GET_TABLET) { - str = "invalid_type"; - } else { - str = CompactionEventStr[event]; - } - return str; -} - -int64_t ObCompactionScheduleTimeGuard::to_string(char *buf, const int64_t buf_len) const -{ - int64_t pos = 0; - for (int64_t idx = 0; idx < idx_; ++idx) { - if (0 < click_poinsts_[idx]) { - fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str((CompactionEvent)line_array_[idx]), click_poinsts_[idx]); - } - } - return pos; -} - } // namespace compaction } // namespace oceanbase \ No newline at end of file diff --git a/src/storage/compaction/ob_compaction_schedule_util.h b/src/storage/compaction/ob_compaction_schedule_util.h index 1f6623d11..2201aecef 100644 --- a/src/storage/compaction/ob_compaction_schedule_util.h +++ b/src/storage/compaction/ob_compaction_schedule_util.h @@ -58,35 +58,6 @@ public: int64_t wait_rs_validate_cnt_; }; -struct ObCompactionScheduleTimeGuard : public ObCompactionTimeGuard -{ -public: - ObCompactionScheduleTimeGuard() - : ObCompactionTimeGuard(UINT64_MAX, "[STORAGE] ") - {} - virtual ~ObCompactionScheduleTimeGuard() {} - enum CompactionEvent : uint16_t { - // medium scheduler - GET_TABLET, - UPDATE_TABLET_REPORT_STATUS, - READ_MEDIUM_INFO, - SCHEDULE_NEXT_MEDIUM, - SCHEDULE_TABLET_MEDIUM, - FAST_FREEZE, - // medium checker - SEARCH_META_TABLE, - CHECK_META_TABLE, - SEARCH_CHECKSUM, - CHECK_CHECKSUM, - SCHEDULER_NEXT_ROUND, - COMPACTION_EVENT_MAX - }; - virtual int64_t to_string(char *buf, const int64_t buf_len) const override; -private: - const static char *CompactionEventStr[]; - static const char *get_comp_event_str(enum CompactionEvent event); -}; - } // compaction } // oceanbase diff --git a/src/storage/compaction/ob_tablet_merge_task.cpp b/src/storage/compaction/ob_tablet_merge_task.cpp index 9ac144514..6a3c0e5db 100644 --- a/src/storage/compaction/ob_tablet_merge_task.cpp +++ b/src/storage/compaction/ob_tablet_merge_task.cpp @@ -52,71 +52,6 @@ using namespace blocksstable; namespace compaction { -/* - * ----------------------------------------------ObCompactionTimeGuard-------------------------------------------------- - */ -constexpr float ObStorageCompactionTimeGuard::COMPACTION_SHOW_PERCENT_THRESHOLD; -const char *ObStorageCompactionTimeGuard::CompactionEventStr[] = { - "WAIT_TO_SCHEDULE", - "COMPACTION_POLICY", - "PRE_PROCESS_TX_TABLE", - "GET_PARALLEL_RANGE", - "EXECUTE", - "CREATE_SSTABLE", - "UPDATE_TABLET", - "RELEASE_MEMTABLE", - "SCHEDULE_OTHER_COMPACTION", - "DAG_FINISH" -}; - -const char *ObStorageCompactionTimeGuard::get_comp_event_str(enum CompactionEvent event) -{ - STATIC_ASSERT(static_cast(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch"); - const char *str = ""; - if (event >= COMPACTION_EVENT_MAX || event < DAG_WAIT_TO_SCHEDULE) { - str = "invalid_type"; - } else { - str = CompactionEventStr[event]; - } - return str; -} - -int64_t ObStorageCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const -{ - int64_t pos = 0; - int64_t total_cost = 0; - J_KV(K_(add_time)); - common::databuff_printf(buf, buf_len, pos, "|"); - if (idx_ > DAG_WAIT_TO_SCHEDULE && click_poinsts_[DAG_WAIT_TO_SCHEDULE] > COMPACTION_SHOW_TIME_THRESHOLD) { - fmt_ts_to_meaningful_str(buf, buf_len, pos, "wait_schedule_time", click_poinsts_[DAG_WAIT_TO_SCHEDULE]); - } - for (int64_t idx = COMPACTION_POLICY; idx < idx_; ++idx) { - total_cost += click_poinsts_[idx]; - } - if (total_cost > COMPACTION_SHOW_TIME_THRESHOLD) { - float ratio = 0; - for (int64_t idx = COMPACTION_POLICY; idx < idx_; ++idx) { - const uint32_t time_interval = click_poinsts_[idx]; - ratio = (float)(time_interval)/ total_cost; - if (ratio >= COMPACTION_SHOW_PERCENT_THRESHOLD || time_interval >= COMPACTION_SHOW_TIME_THRESHOLD) { - fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str((CompactionEvent)line_array_[idx]), click_poinsts_[idx]); - if (ratio > 0.01) { - common::databuff_printf(buf, buf_len, pos, "(%.2f)", ratio); - } - common::databuff_printf(buf, buf_len, pos, "|"); - } - } - } - fmt_ts_to_meaningful_str(buf, buf_len, pos, "total", total_cost); - if (pos != 0 && pos < buf_len) { - buf[pos - 1] = ';'; - } - - if (pos != 0 && pos < buf_len) { - pos -= 1; - } - return pos; -} /* * ----------------------------------------------ObMergeParameter-------------------------------------------------- */ diff --git a/src/storage/compaction/ob_tablet_merge_task.h b/src/storage/compaction/ob_tablet_merge_task.h index cc6db28fb..377f625e6 100644 --- a/src/storage/compaction/ob_tablet_merge_task.h +++ b/src/storage/compaction/ob_tablet_merge_task.h @@ -57,35 +57,6 @@ Medium Compaction ObTabletMajorMergeDag ObTabletMajorMergeCtx Tx Table Compaction ObTxTableMergeDag ObTabletMiniMergeCtx/ObTabletExeMergeCtx */ -struct ObStorageCompactionTimeGuard : public ObCompactionTimeGuard -{ -public: - ObStorageCompactionTimeGuard() - : ObCompactionTimeGuard(COMPACTION_WARN_THRESHOLD_RATIO, "[STORAGE] ") - {} - virtual ~ObStorageCompactionTimeGuard() {} - enum CompactionEvent : uint16_t { - DAG_WAIT_TO_SCHEDULE = 0, - COMPACTION_POLICY, - PRE_PROCESS_TX_TABLE, - GET_PARALLEL_RANGE, - EXECUTE, - CREATE_SSTABLE, - UPDATE_TABLET, - RELEASE_MEMTABLE, - SCHEDULE_OTHER_COMPACTION, - DAG_FINISH, - COMPACTION_EVENT_MAX - }; - virtual int64_t to_string(char *buf, const int64_t buf_len) const override; -private: - const static char *CompactionEventStr[]; - static const char *get_comp_event_str(enum CompactionEvent event); - static const int64_t COMPACTION_WARN_THRESHOLD_RATIO = 60 * 1000L * 1000L; // 1 min - static constexpr float COMPACTION_SHOW_PERCENT_THRESHOLD = 0.1; - static const int64_t COMPACTION_SHOW_TIME_THRESHOLD = 1 * 1000L * 1000L; // 1s -}; - struct ObMergeParameter { ObMergeParameter( const ObStaticMergeParam &static_param); diff --git a/unittest/share/CMakeLists.txt b/unittest/share/CMakeLists.txt index b66b070ac..206b885d4 100644 --- a/unittest/share/CMakeLists.txt +++ b/unittest/share/CMakeLists.txt @@ -103,3 +103,4 @@ add_subdirectory(location_cache) add_subdirectory(detect) add_subdirectory(vector) add_subdirectory(index_usage) +add_subdirectory(compaction) diff --git a/unittest/share/compaction/CMakeLists.txt b/unittest/share/compaction/CMakeLists.txt new file mode 100644 index 000000000..a1cd90d17 --- /dev/null +++ b/unittest/share/compaction/CMakeLists.txt @@ -0,0 +1 @@ +storage_unittest(test_compaction_time_guard) \ No newline at end of file diff --git a/unittest/share/compaction/test_compaction_time_guard.cpp b/unittest/share/compaction/test_compaction_time_guard.cpp new file mode 100644 index 000000000..657f07d21 --- /dev/null +++ b/unittest/share/compaction/test_compaction_time_guard.cpp @@ -0,0 +1,160 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#define USING_LOG_PREFIX STORAGE +#define private public +#define protected public + +#include "share/compaction/ob_compaction_time_guard.h" + +namespace oceanbase +{ +namespace unittest +{ + +using namespace compaction; + +class TestCompactionTimeGuard : public ::testing::Test +{ +public: + TestCompactionTimeGuard() {} + virtual ~TestCompactionTimeGuard() {} +}; + +TEST_F(TestCompactionTimeGuard, basic_time_guard) +{ + const uint16_t CAPACITY = ObCompactionTimeGuard::CAPACITY; + const uint64_t current_time = common::ObTimeUtility::current_time(); + ObCompactionTimeGuard time_guard; + // construction + ASSERT_EQ(CAPACITY, time_guard.capacity_); + ASSERT_EQ(0, time_guard.size_); + ASSERT_TRUE(time_guard.is_empty()); + time_guard.set_last_click_ts(current_time); + ASSERT_EQ(current_time, time_guard.add_time_); + ASSERT_EQ(current_time, time_guard.last_click_ts_); + + // click + ASSERT_TRUE(time_guard.click(CAPACITY)); + ASSERT_TRUE(time_guard.is_empty()); + for (uint16_t i = 0; i < CAPACITY; ++i) { + ASSERT_TRUE(time_guard.click(i)); + ASSERT_EQ(time_guard.size_, i + 1); + ASSERT_EQ(time_guard.event_times_[i], time_guard.get_specified_cost_time(i)); + } + ASSERT_TRUE(time_guard.click(CAPACITY + 1)); + ASSERT_EQ(CAPACITY, time_guard.size_); + + // add and assignment + ObCompactionTimeGuard other_guard; + ASSERT_EQ(CAPACITY, other_guard.capacity_); + ASSERT_TRUE(other_guard.is_empty()); + for (uint16_t i = 0; i < CAPACITY; ++i) { + ASSERT_EQ(0, other_guard.event_times_[i]); + } + other_guard = time_guard; + ASSERT_EQ(time_guard.guard_type_, other_guard.guard_type_); + ASSERT_EQ(time_guard.warn_threshold_, other_guard.warn_threshold_); + ASSERT_EQ(time_guard.capacity_, other_guard.capacity_); + ASSERT_EQ(time_guard.size_, other_guard.size_); + ASSERT_EQ(time_guard.last_click_ts_, other_guard.last_click_ts_); + ASSERT_EQ(time_guard.add_time_, other_guard.add_time_); + for (uint16_t i = 0; i < CAPACITY; ++i) { + ASSERT_EQ(time_guard.event_times_[i], other_guard.event_times_[i]); + } + + // add time guard + ObCompactionTimeGuard summary_guard; + for (uint16_t i = 0; i < 2 * CAPACITY; i++) { + summary_guard.add_time_guard(time_guard); + summary_guard.add_time_guard(other_guard); + } + for (uint16_t i = 0; i < CAPACITY; ++i) { + ASSERT_EQ(summary_guard.event_times_[i], time_guard.event_times_[i] * 4 * CAPACITY); + ASSERT_EQ(summary_guard.event_times_[i], other_guard.event_times_[i] * 4 * CAPACITY); + } + + // reuse + ObCompactionTimeGuard* guards[3] = {&time_guard, &other_guard, &summary_guard}; + for (int i = 0; i < 3; i++) { + guards[i]->reuse(); + ASSERT_TRUE(guards[i]->is_empty()); + ASSERT_EQ(CAPACITY, guards[i]->capacity_); + ASSERT_EQ(0, guards[i]->size_); + for (uint16_t j = 0; j < CAPACITY; ++j) { + ASSERT_EQ(0, guards[i]->event_times_[j]); + } + } + // out of order click and add time guard with diffrent size_ + time_guard.click(0); + ASSERT_EQ(1, time_guard.size_); + time_guard.click(2); + ASSERT_EQ(3, time_guard.size_); + time_guard.click(1); + ASSERT_EQ(3, time_guard.size_); + time_guard.click(10); + ASSERT_EQ(11, time_guard.size_); + other_guard.click(6); + ASSERT_EQ(7, other_guard.size_); + summary_guard.add_time_guard(other_guard); + ASSERT_EQ(summary_guard.size_, other_guard.size_); + summary_guard.add_time_guard(time_guard); + ASSERT_EQ(summary_guard.size_, time_guard.size_); +} + +TEST_F(TestCompactionTimeGuard, time_guard_to_string) +{ + ObStorageCompactionTimeGuard storage_guard; + for (uint16_t i = 0; i < ObStorageCompactionTimeGuard::COMPACTION_EVENT_MAX; i++) { + storage_guard.click(i); + storage_guard.event_times_[i] += ObStorageCompactionTimeGuard::COMPACTION_SHOW_TIME_THRESHOLD; + ASSERT_EQ(i + 1, storage_guard.size_); + } + ASSERT_EQ(ObStorageCompactionTimeGuard::COMPACTION_EVENT_MAX, storage_guard.size_); + storage_guard.event_times_[ObStorageCompactionTimeGuard::DAG_WAIT_TO_SCHEDULE] += 2 * ObStorageCompactionTimeGuard::COMPACTION_SHOW_TIME_THRESHOLD; + storage_guard.event_times_[ObStorageCompactionTimeGuard::DAG_FINISH] += 2 * ObCompactionTimeGuard::WARN_THRESHOLD; + STORAGE_LOG(INFO, "storage guard is", K(storage_guard)); + + ObRSCompactionTimeGuard rs_guard; + for (uint16_t i = 0; i < ObRSCompactionTimeGuard::COMPACTION_EVENT_MAX; i++) { + rs_guard.click(i); + rs_guard.event_times_[i] += 10 * i; + ASSERT_EQ(i + 1, rs_guard.size_); + } + ASSERT_EQ(ObRSCompactionTimeGuard::COMPACTION_EVENT_MAX, rs_guard.size_); + rs_guard.event_times_[ObRSCompactionTimeGuard::CKM_VERIFICATION] += 2 * ObCompactionTimeGuard::WARN_THRESHOLD; + STORAGE_LOG(INFO, "rs guard is", K(rs_guard)); + + ObCompactionScheduleTimeGuard schedule_guard; + for (uint16_t i = 0; i < ObCompactionScheduleTimeGuard::COMPACTION_EVENT_MAX; i++) { + schedule_guard.click(i); + schedule_guard.event_times_[i] += 10 * i; + ASSERT_EQ(i + 1, schedule_guard.size_); + } + ASSERT_EQ(ObCompactionScheduleTimeGuard::COMPACTION_EVENT_MAX, schedule_guard.size_); + schedule_guard.event_times_[ObCompactionScheduleTimeGuard::SCHEDULER_NEXT_ROUND] += 2 * ObCompactionTimeGuard::WARN_THRESHOLD; + STORAGE_LOG(INFO, "schedule guard is", K(schedule_guard)); +} + +} // namespace unittest +} // namespace oceanbase + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + system("rm -f test_compaction_time_guard.log*"); + OB_LOGGER.set_file_name("test_compaction_time_guard.log"); + OB_LOGGER.set_log_level("INFO"); + OB_LOGGER.set_max_file_size(256*1024*1024); + return RUN_ALL_TESTS(); +}