hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor ObCompactionTimeGuard to process failed event

Browse Source
This commit is contained in:
Tsunaou 2024-05-16 06:26:20 +00:00 committed by ob-robot
parent d94ee24ba3
commit a60cb378cd
11 changed files with 546 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/rootserver/freeze/ob_major_merge_progress_util.cpp
View File

@ -82,36 +82,6 @@ int64_t ObMergeProgress::to_string(char *buf, const int64_t buf_len) const
}
return pos;
}
/**
* -------------------------------------------------------------------ObRSCompactionTimeGuard-------------------------------------------------------------------
*/
const char *ObRSCompactionTimeGuard::CompactionEventStr[] = {
"PREPARE_UNFINISH_TABLE_IDS",
"GET_TABLET_LS_PAIRS",
"GET_TABLET_META_TABLE",
"CKM_VERIFICATION"
};
const char *ObRSCompactionTimeGuard::get_comp_event_str(enum CompactionEvent event)
{
STATIC_ASSERT(static_cast<int64_t>(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch");
const char *str = "";
if (event >= COMPACTION_EVENT_MAX || event < PREPARE_UNFINISH_TABLE_IDS) {
str = "invalid_type";
} else {
str = CompactionEventStr[event];
}
return str;
}
int64_t ObRSCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
for (int64_t idx = 0; idx < idx_; ++idx) {
fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str((CompactionEvent)line_array_[idx]), click_poinsts_[idx]);
}
return pos;
}
/**
* -------------------------------------------------------------------ObTabletLSPairCache-------------------------------------------------------------------

20
src/rootserver/freeze/ob_major_merge_progress_util.h
View File

@ -209,26 +209,6 @@ typedef hash::ObHashMap<ObTabletID, ObTabletCompactionStatus> ObTabletStatusMap;
typedef common::ObArray<share::ObTabletLSPair> ObTabletLSPairArray;
typedef hash::ObHashMap<uint64_t, ObTableCompactionInfo> ObTableCompactionInfoMap;
struct ObRSCompactionTimeGuard : public ObCompactionTimeGuard
{
public:
ObRSCompactionTimeGuard()
: ObCompactionTimeGuard(UINT64_MAX, "[RS] ")
{}
virtual ~ObRSCompactionTimeGuard() {}
enum CompactionEvent : uint16_t {
PREPARE_UNFINISH_TABLE_IDS = 0,
GET_TABLET_LS_PAIRS,
GET_TABLET_META_TABLE,
CKM_VERIFICATION,
COMPACTION_EVENT_MAX,
};
virtual int64_t to_string(char *buf, const int64_t buf_len) const override;
private:
const static char *CompactionEventStr[];
static const char *get_comp_event_str(enum CompactionEvent event);
};
struct ObCkmValidatorStatistics
{
ObCkmValidatorStatistics() { reset(); }

272
src/share/compaction/ob_compaction_time_guard.cpp
View File

@ -14,45 +14,263 @@ namespace oceanbase
namespace compaction
{
/**
* -------------------------------------------------------------------ObCompactionTimeGuard-------------------------------------------------------------------
*/
ObCompactionTimeGuard::~ObCompactionTimeGuard()
{
int64_t total_cost = 0;
for (int64_t idx = 0; idx < size_; ++idx) {
total_cost += event_times_[idx];
}
total_cost += common::ObTimeUtility::current_time() - last_click_ts_;
if (OB_UNLIKELY(total_cost >= warn_threshold_)) {
::oceanbase::common::OB_PRINT(log_mod_, OB_LOG_LEVEL_DIRECT_NO_ERRCODE(WARN), OB_SUCCESS, "cost too much time", LOG_KVS(K(*this)));
}
}
int64_t ObCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
fmt_ts_to_meaningful_str(buf, buf_len, pos, "|threshold", warn_threshold_);
common::databuff_printf(buf, buf_len, pos, "start at %s|", common::ObTime2Str::ob_timestamp_str(add_time_));
int64_t total_cost = 0;
for (int64_t idx = 0; idx < size_; ++idx) {
const uint64_t ts = event_times_[idx];
if (ts < 1_ms) {
common::databuff_printf(buf, buf_len, pos, "%ldus|", ts);
} else if (ts < 1_s) {
common::databuff_printf(buf, buf_len, pos, "%.2lfms|", double(ts) / 1_ms);
} else {
common::databuff_printf(buf, buf_len, pos, "%.2lfs|", double(ts) / 1_s);
}
total_cost += event_times_[idx];
}
total_cost += common::ObTimeUtility::current_time() - last_click_ts_;
fmt_ts_to_meaningful_str(buf, buf_len, pos, "total", total_cost);
if (pos != 0 && pos < buf_len) {
pos -= 1;
}
return pos;
}
void ObCompactionTimeGuard::reuse()
{
size_ = 0;
last_click_ts_ = common::ObTimeUtility::current_time();
add_time_ = common::ObTimeUtility::current_time();
for (uint16_t i = 0; i < capacity_; ++i) {
event_times_[i] = 0;
}
}
bool ObCompactionTimeGuard::click(const uint16_t event)
{
if (OB_LIKELY(event < CAPACITY)) {
if (OB_LIKELY(size_ <= event)) {
size_ = event + 1;
}
const int64_t now = common::ObTimeUtility::current_time();
event_times_[event] += now - last_click_ts_;
last_click_ts_ = now;
}
return true;
}
void ObCompactionTimeGuard::fmt_ts_to_meaningful_str(
char *buf,
const int64_t buf_len,
int64_t &pos,
const char *lvalue,
const int64_t ts) const
{
common::databuff_printf(buf, buf_len, pos, "%s", lvalue);
if (ts < 1_ms) {
common::databuff_printf(buf, buf_len, pos, "=%ldus|", ts);
} else if (ts < 1_s) {
common::databuff_printf(buf, buf_len, pos, "=%.2lfms|", double(ts) / 1_ms);
} else {
common::databuff_printf(buf, buf_len, pos, "=%.2lfs|", double(ts) / 1_s);
}
}
void ObCompactionTimeGuard::add_time_guard(const ObCompactionTimeGuard &other)
{
// last_click_ts_ is not useflu
ObCompactionTimeGuard time_guard;
int i = 0;
int j = 0;
while (i < idx_ && j < other.idx_) {
if (line_array_[i] == other.line_array_[j]) {
time_guard.line_array_[time_guard.idx_] = line_array_[i];
time_guard.click_poinsts_[time_guard.idx_++] = click_poinsts_[i++] + other.click_poinsts_[j++];
} else if (line_array_[i] < other.line_array_[j]) {
time_guard.line_array_[time_guard.idx_] = line_array_[i];
time_guard.click_poinsts_[time_guard.idx_++] = click_poinsts_[i++];
} else {
time_guard.line_array_[time_guard.idx_] = other.line_array_[j];
time_guard.click_poinsts_[time_guard.idx_++] = other.click_poinsts_[j++];
if (OB_LIKELY(guard_type_ == other.guard_type_ && CAPACITY == other.capacity_)) {
size_ = std::max(size_, other.size_);
for (uint16_t i = 0; i < size_; i++) {
event_times_[i] += other.event_times_[i];
}
}
while (i < idx_) {
time_guard.line_array_[time_guard.idx_] = line_array_[i];
time_guard.click_poinsts_[time_guard.idx_++] = click_poinsts_[i++];
}
while (j < other.idx_) {
time_guard.line_array_[time_guard.idx_] = other.line_array_[j];
time_guard.click_poinsts_[time_guard.idx_++] = other.click_poinsts_[j++];
}
*this = time_guard;
}
ObCompactionTimeGuard & ObCompactionTimeGuard::operator=(const ObCompactionTimeGuard &other)
{
guard_type_ = other.guard_type_;
capacity_ = other.capacity_;
size_ = other.size_;
last_click_ts_ = other.last_click_ts_;
idx_ = other.idx_;
for (int i = 0; i < other.idx_; ++i) {
line_array_[i] = other.line_array_[i];
click_poinsts_[i] = other.click_poinsts_[i];
add_time_ = other.add_time_;
for (uint16_t i = 0; i < other.size_; ++i) {
event_times_[i] = other.event_times_[i];
}
return *this;
}
uint16_t ObCompactionTimeGuard::get_max_event_count(const ObCompactionTimeGuardType guard_type)
{
uint16_t max_event_count = CAPACITY;
if (RS_COMPACT_TIME_GUARD == guard_type) {
max_event_count = ObRSCompactionTimeGuard::COMPACTION_EVENT_MAX;
} else if (SCHEDULE_COMPACT_TIME_GUARD == guard_type) {
max_event_count = ObCompactionScheduleTimeGuard::COMPACTION_EVENT_MAX;
} else if (STORAGE_COMPACT_TIME_GUARD == guard_type) {
max_event_count = ObStorageCompactionTimeGuard::COMPACTION_EVENT_MAX;
}
return max_event_count;
}
/**
* -------------------------------------------------------------------ObRSCompactionTimeGuard-------------------------------------------------------------------
*/
const char *ObRSCompactionTimeGuard::CompactionEventStr[] = {
"PREPARE_UNFINISH_TABLE_IDS",
"GET_TABLET_LS_PAIRS",
"GET_TABLET_META_TABLE",
"CKM_VERIFICATION"
};
const char *ObRSCompactionTimeGuard::get_comp_event_str(enum CompactionEvent event)
{
STATIC_ASSERT(static_cast<int64_t>(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch");
STATIC_ASSERT(static_cast<int64_t>(COMPACTION_EVENT_MAX) <= static_cast<int64_t>(CAPACITY), "too many events, need update CAPACITY");
const char *str = "";
if (event >= COMPACTION_EVENT_MAX || event < PREPARE_UNFINISH_TABLE_IDS) {
str = "invalid_type";
} else {
str = CompactionEventStr[event];
}
return str;
}
int64_t ObRSCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
for (uint16_t idx = 0; idx < size_; ++idx) {
if (event_times_[idx] > 0) {
fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str(static_cast<CompactionEvent>(idx)), event_times_[idx]);
}
}
return pos;
}
/**
* ObCompactionScheduleTimeGuard Impl
*/
const char *ObCompactionScheduleTimeGuard::CompactionEventStr[] = {
"GET_TABLET",
"UPDATE_TABLET_REPORT_STATUS",
"READ_MEDIUM_INFO",
"SCHEDULE_NEXT_MEDIUM",
"SCHEDULE_TABLET_MEDIUM",
"FAST_FREEZE",
"SEARCH_META_TABLE",
"CHECK_META_TABLE",
"SEARCH_CHECKSUM",
"CHECK_CHECKSUM",
"SCHEDULER_NEXT_ROUND"
};
const char *ObCompactionScheduleTimeGuard::get_comp_event_str(enum CompactionEvent event)
{
STATIC_ASSERT(static_cast<int64_t>(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch");
STATIC_ASSERT(static_cast<int64_t>(COMPACTION_EVENT_MAX) <= static_cast<int64_t>(CAPACITY), "too many events, need update CAPACITY");
const char *str = "";
if (event >= COMPACTION_EVENT_MAX || event < GET_TABLET) {
str = "invalid_type";
} else {
str = CompactionEventStr[event];
}
return str;
}
int64_t ObCompactionScheduleTimeGuard::to_string(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
for (int16_t idx = 0; idx < size_; ++idx) {
if (event_times_[idx] > 0) {
fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str(static_cast<CompactionEvent>(idx)), event_times_[idx]);
}
}
return pos;
}
/*
* ----------------------------------------------ObCompactionTimeGuard--------------------------------------------------
*/
constexpr float ObStorageCompactionTimeGuard::COMPACTION_SHOW_PERCENT_THRESHOLD;
const char *ObStorageCompactionTimeGuard::CompactionEventStr[] = {
"WAIT_TO_SCHEDULE",
"COMPACTION_POLICY",
"PRE_PROCESS_TX_TABLE",
"GET_PARALLEL_RANGE",
"EXECUTE",
"CREATE_SSTABLE",
"UPDATE_TABLET",
"RELEASE_MEMTABLE",
"SCHEDULE_OTHER_COMPACTION",
"DAG_FINISH"
};
const char *ObStorageCompactionTimeGuard::get_comp_event_str(const enum CompactionEvent event)
{
STATIC_ASSERT(static_cast<int64_t>(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch");
STATIC_ASSERT(static_cast<int64_t>(COMPACTION_EVENT_MAX) <= static_cast<int64_t>(CAPACITY), "too many events, need update CAPACITY");
const char *str = "";
if (event >= COMPACTION_EVENT_MAX || event < DAG_WAIT_TO_SCHEDULE) {
str = "invalid_type";
} else {
str = CompactionEventStr[event];
}
return str;
}
int64_t ObStorageCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
int64_t total_cost = 0;
J_KV(K_(add_time));
common::databuff_printf(buf, buf_len, pos, "|");
if (size_ > DAG_WAIT_TO_SCHEDULE && event_times_[DAG_WAIT_TO_SCHEDULE] > COMPACTION_SHOW_TIME_THRESHOLD) {
fmt_ts_to_meaningful_str(buf, buf_len, pos, "wait_schedule_time", event_times_[DAG_WAIT_TO_SCHEDULE]);
}
for (int64_t idx = COMPACTION_POLICY; idx < size_; ++idx) {
total_cost += event_times_[idx];
}
if (total_cost > COMPACTION_SHOW_TIME_THRESHOLD) {
float ratio = 0;
for (int64_t idx = COMPACTION_POLICY; idx < size_; ++idx) {
const uint32_t time_interval = event_times_[idx]; // include the retry time since previous event
ratio = (float)(time_interval)/ total_cost;
if (ratio >= COMPACTION_SHOW_PERCENT_THRESHOLD || time_interval >= COMPACTION_SHOW_TIME_THRESHOLD) {
fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str(static_cast<CompactionEvent>(idx)), event_times_[idx]);
if (ratio > 0.01) {
common::databuff_printf(buf, buf_len, pos, "(%.2f)", ratio);
}
common::databuff_printf(buf, buf_len, pos, "|");
}
}
}
fmt_ts_to_meaningful_str(buf, buf_len, pos, "total", total_cost);
if (pos != 0 && pos < buf_len) {
buf[pos - 1] = ';';
}
if (pos != 0 && pos < buf_len) {
pos -= 1;
}
return pos;
}
} // namespace compaction
} // namespace oceanbase

157
src/share/compaction/ob_compaction_time_guard.h
View File

@ -10,48 +10,169 @@
#ifndef OB_SHARE_COMPACTION_COMPACTION_TIME_GUARD_H_
#define OB_SHARE_COMPACTION_COMPACTION_TIME_GUARD_H_
#include "share/ob_occam_time_guard.h"
#include "lib/container/ob_se_array.h"
namespace oceanbase
{
namespace compaction
{
class ObCompactionTimeGuard : public common::occam::ObOccamTimeGuard
/*
* ObCompactionTimeGuard refers to the implementation of from ObOccamTimeGuard.
* For example, you have 3 enum events {e0, e1, e2}
* If you want to record the time cost of event e1, you can use click(e1), then event_times_[1] will accumulate the time cost of e1.
*
* ObCompactionTimeGuard
* -- ObRSCompactionTimeGuard
* -- ObScheduleCompactionTimeGuard
* -- ObStorageCompactionTimeGuard
*/
class ObCompactionTimeGuard
{
public:
const static uint64_t WARN_THRESHOLD = 30L * 1000 * 1000; // 30s
ObCompactionTimeGuard(const uint64_t warn_threshold = WARN_THRESHOLD, const char *mod = "")
: ObOccamTimeGuard(warn_threshold, nullptr, nullptr, mod),
add_time_(0)
{}
virtual ~ObCompactionTimeGuard() {}
virtual int64_t to_string(char *buf, const int64_t buf_len) const
enum ObCompactionTimeGuardType : uint8_t
{
UNUSEDx(buf, buf_len);
return 0;
BASE_COMPACT_TIME_GUARD = 0,
RS_COMPACT_TIME_GUARD = 1,
SCHEDULE_COMPACT_TIME_GUARD = 2,
STORAGE_COMPACT_TIME_GUARD = 3,
MAX_COMPACT_TIME_GUARD
};
public:
const static uint64_t WARN_THRESHOLD = 30L * 1000 * 1000; // 30s
const static uint16_t CAPACITY = 16;
ObCompactionTimeGuard(const ObCompactionTimeGuardType gurad_type = BASE_COMPACT_TIME_GUARD,
const uint64_t warn_threshold = WARN_THRESHOLD,
const char *mod = "")
: guard_type_(gurad_type),
warn_threshold_(warn_threshold),
log_mod_(mod),
capacity_(get_max_event_count(gurad_type)),
size_(0),
last_click_ts_(common::ObTimeUtility::current_time()),
add_time_(common::ObTimeUtility::current_time())
{
reuse();
}
virtual ~ObCompactionTimeGuard();
virtual int64_t to_string(char *buf, const int64_t buf_len) const;
void reuse();
void add_time_guard(const ObCompactionTimeGuard &other);
ObCompactionTimeGuard & operator=(const ObCompactionTimeGuard &other);
OB_INLINE bool is_empty() const { return 0 == idx_; }
OB_INLINE bool is_empty() const { return 0 == size_; }
// set the dag add_time as the first click time
OB_INLINE void set_last_click_ts(const int64_t time)
{
last_click_ts_ = time;
add_time_ = time;
}
OB_INLINE uint32_t get_specified_cost_time(const int64_t line) const {
OB_INLINE uint32_t get_specified_cost_time(const int64_t event) const {
uint32_t ret_val = 0;
for (int64_t idx = 0; idx < idx_; ++idx) {
if (line_array_[idx] == line) {
ret_val = click_poinsts_[idx];
break;
}
if (OB_LIKELY(event < size_)) {
ret_val = event_times_[event];
}
return ret_val;
}
bool click(const uint16_t event);
// copy from ObOccamTimeGuard
void fmt_ts_to_meaningful_str(
char *buf,
const int64_t buf_len,
int64_t &pos,
const char *lvalue,
const int64_t ts) const;
public:
template <typename E>
static constexpr uint16_t event_idx(E e) { return static_cast<uint16_t>(e); }
static uint16_t get_max_event_count(const ObCompactionTimeGuardType guard_type);
public:
ObCompactionTimeGuardType guard_type_;
const uint64_t warn_threshold_;
const char * log_mod_;
uint16_t capacity_; // equal with CAPACITY, used for child class to check the array boundary
uint16_t size_;
int64_t last_click_ts_;
int64_t add_time_;
uint64_t event_times_[CAPACITY];
};
struct ObRSCompactionTimeGuard : public ObCompactionTimeGuard
{
public:
ObRSCompactionTimeGuard()
: ObCompactionTimeGuard(RS_COMPACT_TIME_GUARD, UINT64_MAX, "[RS] ")
{}
virtual ~ObRSCompactionTimeGuard() {}
enum CompactionEvent : uint16_t {
PREPARE_UNFINISH_TABLE_IDS = 0,
GET_TABLET_LS_PAIRS,
GET_TABLET_META_TABLE,
CKM_VERIFICATION,
COMPACTION_EVENT_MAX,
};
virtual int64_t to_string(char *buf, const int64_t buf_len) const override;
private:
const static char *CompactionEventStr[];
static const char *get_comp_event_str(const enum CompactionEvent event);
};
struct ObCompactionScheduleTimeGuard : public ObCompactionTimeGuard
{
public:
ObCompactionScheduleTimeGuard()
: ObCompactionTimeGuard(SCHEDULE_COMPACT_TIME_GUARD, UINT64_MAX, "[STORAGE] ")
{}
virtual ~ObCompactionScheduleTimeGuard() {}
enum CompactionEvent : uint16_t {
// medium scheduler
GET_TABLET = 0,
UPDATE_TABLET_REPORT_STATUS,
READ_MEDIUM_INFO,
SCHEDULE_NEXT_MEDIUM,
SCHEDULE_TABLET_MEDIUM,
FAST_FREEZE,
// medium checker
SEARCH_META_TABLE,
CHECK_META_TABLE,
SEARCH_CHECKSUM,
CHECK_CHECKSUM,
SCHEDULER_NEXT_ROUND,
COMPACTION_EVENT_MAX
};
virtual int64_t to_string(char *buf, const int64_t buf_len) const override;
private:
const static char *CompactionEventStr[];
static const char *get_comp_event_str(const enum CompactionEvent event);
};
struct ObStorageCompactionTimeGuard : public ObCompactionTimeGuard
{
public:
ObStorageCompactionTimeGuard()
: ObCompactionTimeGuard(STORAGE_COMPACT_TIME_GUARD, COMPACTION_WARN_THRESHOLD_RATIO, "[STORAGE] ")
{}
virtual ~ObStorageCompactionTimeGuard() {}
enum CompactionEvent : uint16_t {
DAG_WAIT_TO_SCHEDULE = 0,
COMPACTION_POLICY,
PRE_PROCESS_TX_TABLE,
GET_PARALLEL_RANGE,
EXECUTE,
CREATE_SSTABLE,
UPDATE_TABLET,
RELEASE_MEMTABLE,
SCHEDULE_OTHER_COMPACTION,
DAG_FINISH,
COMPACTION_EVENT_MAX
};
virtual int64_t to_string(char *buf, const int64_t buf_len) const override;
private:
const static char *CompactionEventStr[];
static const char *get_comp_event_str(const enum CompactionEvent event);
static const int64_t COMPACTION_WARN_THRESHOLD_RATIO = 60 * 1000L * 1000L; // 1 min
static constexpr float COMPACTION_SHOW_PERCENT_THRESHOLD = 0.1;
static const int64_t COMPACTION_SHOW_TIME_THRESHOLD = 1 * 1000L * 1000L; // 1s
};
} // namespace compaction
} // namespace oceanbase

40
src/storage/compaction/ob_compaction_schedule_util.cpp
View File

@ -18,45 +18,5 @@ namespace oceanbase
namespace compaction
{
/**
* ObCompactionScheduleTimeGuard Impl
*/
const char *ObCompactionScheduleTimeGuard::CompactionEventStr[] = {
"GET_TABLET",
"UPDATE_TABLET_REPORT_STATUS",
"READ_MEDIUM_INFO",
"SCHEDULE_NEXT_MEDIUM",
"SCHEDULE_TABLET_MEDIUM",
"FAST_FREEZE",
"SEARCH_META_TABLE",
"CHECK_META_TABLE",
"SEARCH_CHECKSUM",
"CHECK_CHECKSUM",
"SCHEDULER_NEXT_ROUND"
};
const char *ObCompactionScheduleTimeGuard::get_comp_event_str(enum CompactionEvent event)
{
STATIC_ASSERT(static_cast<int64_t>(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch");
const char *str = "";
if (event >= COMPACTION_EVENT_MAX || event < GET_TABLET) {
str = "invalid_type";
} else {
str = CompactionEventStr[event];
}
return str;
}
int64_t ObCompactionScheduleTimeGuard::to_string(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
for (int64_t idx = 0; idx < idx_; ++idx) {
if (0 < click_poinsts_[idx]) {
fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str((CompactionEvent)line_array_[idx]), click_poinsts_[idx]);
}
}
return pos;
}
} // namespace compaction
} // namespace oceanbase

29
src/storage/compaction/ob_compaction_schedule_util.h
View File

@ -58,35 +58,6 @@ public:
int64_t wait_rs_validate_cnt_;
};
struct ObCompactionScheduleTimeGuard : public ObCompactionTimeGuard
{
public:
ObCompactionScheduleTimeGuard()
: ObCompactionTimeGuard(UINT64_MAX, "[STORAGE] ")
{}
virtual ~ObCompactionScheduleTimeGuard() {}
enum CompactionEvent : uint16_t {
// medium scheduler
GET_TABLET,
UPDATE_TABLET_REPORT_STATUS,
READ_MEDIUM_INFO,
SCHEDULE_NEXT_MEDIUM,
SCHEDULE_TABLET_MEDIUM,
FAST_FREEZE,
// medium checker
SEARCH_META_TABLE,
CHECK_META_TABLE,
SEARCH_CHECKSUM,
CHECK_CHECKSUM,
SCHEDULER_NEXT_ROUND,
COMPACTION_EVENT_MAX
};
virtual int64_t to_string(char *buf, const int64_t buf_len) const override;
private:
const static char *CompactionEventStr[];
static const char *get_comp_event_str(enum CompactionEvent event);
};
} // compaction
} // oceanbase

65
src/storage/compaction/ob_tablet_merge_task.cpp
View File

@ -52,71 +52,6 @@ using namespace blocksstable;
namespace compaction
{
/*
* ----------------------------------------------ObCompactionTimeGuard--------------------------------------------------
*/
constexpr float ObStorageCompactionTimeGuard::COMPACTION_SHOW_PERCENT_THRESHOLD;
const char *ObStorageCompactionTimeGuard::CompactionEventStr[] = {
"WAIT_TO_SCHEDULE",
"COMPACTION_POLICY",
"PRE_PROCESS_TX_TABLE",
"GET_PARALLEL_RANGE",
"EXECUTE",
"CREATE_SSTABLE",
"UPDATE_TABLET",
"RELEASE_MEMTABLE",
"SCHEDULE_OTHER_COMPACTION",
"DAG_FINISH"
};
const char *ObStorageCompactionTimeGuard::get_comp_event_str(enum CompactionEvent event)
{
STATIC_ASSERT(static_cast<int64_t>(COMPACTION_EVENT_MAX) == ARRAYSIZEOF(CompactionEventStr), "events str len is mismatch");
const char *str = "";
if (event >= COMPACTION_EVENT_MAX || event < DAG_WAIT_TO_SCHEDULE) {
str = "invalid_type";
} else {
str = CompactionEventStr[event];
}
return str;
}
int64_t ObStorageCompactionTimeGuard::to_string(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
int64_t total_cost = 0;
J_KV(K_(add_time));
common::databuff_printf(buf, buf_len, pos, "|");
if (idx_ > DAG_WAIT_TO_SCHEDULE && click_poinsts_[DAG_WAIT_TO_SCHEDULE] > COMPACTION_SHOW_TIME_THRESHOLD) {
fmt_ts_to_meaningful_str(buf, buf_len, pos, "wait_schedule_time", click_poinsts_[DAG_WAIT_TO_SCHEDULE]);
}
for (int64_t idx = COMPACTION_POLICY; idx < idx_; ++idx) {
total_cost += click_poinsts_[idx];
}
if (total_cost > COMPACTION_SHOW_TIME_THRESHOLD) {
float ratio = 0;
for (int64_t idx = COMPACTION_POLICY; idx < idx_; ++idx) {
const uint32_t time_interval = click_poinsts_[idx];
ratio = (float)(time_interval)/ total_cost;
if (ratio >= COMPACTION_SHOW_PERCENT_THRESHOLD || time_interval >= COMPACTION_SHOW_TIME_THRESHOLD) {
fmt_ts_to_meaningful_str(buf, buf_len, pos, get_comp_event_str((CompactionEvent)line_array_[idx]), click_poinsts_[idx]);
if (ratio > 0.01) {
common::databuff_printf(buf, buf_len, pos, "(%.2f)", ratio);
}
common::databuff_printf(buf, buf_len, pos, "|");
}
}
}
fmt_ts_to_meaningful_str(buf, buf_len, pos, "total", total_cost);
if (pos != 0 && pos < buf_len) {
buf[pos - 1] = ';';
}
if (pos != 0 && pos < buf_len) {
pos -= 1;
}
return pos;
}
/*
* ----------------------------------------------ObMergeParameter--------------------------------------------------
*/

29
src/storage/compaction/ob_tablet_merge_task.h
View File

@ -57,35 +57,6 @@ Medium Compaction ObTabletMajorMergeDag ObTabletMajorMergeCtx
Tx Table Compaction ObTxTableMergeDag ObTabletMiniMergeCtx/ObTabletExeMergeCtx
*/
struct ObStorageCompactionTimeGuard : public ObCompactionTimeGuard
{
public:
ObStorageCompactionTimeGuard()
: ObCompactionTimeGuard(COMPACTION_WARN_THRESHOLD_RATIO, "[STORAGE] ")
{}
virtual ~ObStorageCompactionTimeGuard() {}
enum CompactionEvent : uint16_t {
DAG_WAIT_TO_SCHEDULE = 0,
COMPACTION_POLICY,
PRE_PROCESS_TX_TABLE,
GET_PARALLEL_RANGE,
EXECUTE,
CREATE_SSTABLE,
UPDATE_TABLET,
RELEASE_MEMTABLE,
SCHEDULE_OTHER_COMPACTION,
DAG_FINISH,
COMPACTION_EVENT_MAX
};
virtual int64_t to_string(char *buf, const int64_t buf_len) const override;
private:
const static char *CompactionEventStr[];
static const char *get_comp_event_str(enum CompactionEvent event);
static const int64_t COMPACTION_WARN_THRESHOLD_RATIO = 60 * 1000L * 1000L; // 1 min
static constexpr float COMPACTION_SHOW_PERCENT_THRESHOLD = 0.1;
static const int64_t COMPACTION_SHOW_TIME_THRESHOLD = 1 * 1000L * 1000L; // 1s
};
struct ObMergeParameter {
ObMergeParameter(
const ObStaticMergeParam &static_param);

1
unittest/share/CMakeLists.txt
View File

@ -103,3 +103,4 @@ add_subdirectory(location_cache)
add_subdirectory(detect)
add_subdirectory(vector)
add_subdirectory(index_usage)
add_subdirectory(compaction)

1
unittest/share/compaction/CMakeLists.txt Normal file
View File

@ -0,0 +1 @@
storage_unittest(test_compaction_time_guard)

160
unittest/share/compaction/test_compaction_time_guard.cpp Normal file
View File

@ -0,0 +1,160 @@
/**
* Copyright (c) 2024 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define USING_LOG_PREFIX STORAGE
#define private public
#define protected public
#include "share/compaction/ob_compaction_time_guard.h"
namespace oceanbase
{
namespace unittest
{
using namespace compaction;
class TestCompactionTimeGuard : public ::testing::Test
{
public:
TestCompactionTimeGuard() {}
virtual ~TestCompactionTimeGuard() {}
};
TEST_F(TestCompactionTimeGuard, basic_time_guard)
{
const uint16_t CAPACITY = ObCompactionTimeGuard::CAPACITY;
const uint64_t current_time = common::ObTimeUtility::current_time();
ObCompactionTimeGuard time_guard;
// construction
ASSERT_EQ(CAPACITY, time_guard.capacity_);
ASSERT_EQ(0, time_guard.size_);
ASSERT_TRUE(time_guard.is_empty());
time_guard.set_last_click_ts(current_time);
ASSERT_EQ(current_time, time_guard.add_time_);
ASSERT_EQ(current_time, time_guard.last_click_ts_);
// click
ASSERT_TRUE(time_guard.click(CAPACITY));
ASSERT_TRUE(time_guard.is_empty());
for (uint16_t i = 0; i < CAPACITY; ++i) {
ASSERT_TRUE(time_guard.click(i));
ASSERT_EQ(time_guard.size_, i + 1);
ASSERT_EQ(time_guard.event_times_[i], time_guard.get_specified_cost_time(i));
}
ASSERT_TRUE(time_guard.click(CAPACITY + 1));
ASSERT_EQ(CAPACITY, time_guard.size_);
// add and assignment
ObCompactionTimeGuard other_guard;
ASSERT_EQ(CAPACITY, other_guard.capacity_);
ASSERT_TRUE(other_guard.is_empty());
for (uint16_t i = 0; i < CAPACITY; ++i) {
ASSERT_EQ(0, other_guard.event_times_[i]);
}
other_guard = time_guard;
ASSERT_EQ(time_guard.guard_type_, other_guard.guard_type_);
ASSERT_EQ(time_guard.warn_threshold_, other_guard.warn_threshold_);
ASSERT_EQ(time_guard.capacity_, other_guard.capacity_);
ASSERT_EQ(time_guard.size_, other_guard.size_);
ASSERT_EQ(time_guard.last_click_ts_, other_guard.last_click_ts_);
ASSERT_EQ(time_guard.add_time_, other_guard.add_time_);
for (uint16_t i = 0; i < CAPACITY; ++i) {
ASSERT_EQ(time_guard.event_times_[i], other_guard.event_times_[i]);
}
// add time guard
ObCompactionTimeGuard summary_guard;
for (uint16_t i = 0; i < 2 * CAPACITY; i++) {
summary_guard.add_time_guard(time_guard);
summary_guard.add_time_guard(other_guard);
}
for (uint16_t i = 0; i < CAPACITY; ++i) {
ASSERT_EQ(summary_guard.event_times_[i], time_guard.event_times_[i] * 4 * CAPACITY);
ASSERT_EQ(summary_guard.event_times_[i], other_guard.event_times_[i] * 4 * CAPACITY);
}
// reuse
ObCompactionTimeGuard* guards[3] = {&time_guard, &other_guard, &summary_guard};
for (int i = 0; i < 3; i++) {
guards[i]->reuse();
ASSERT_TRUE(guards[i]->is_empty());
ASSERT_EQ(CAPACITY, guards[i]->capacity_);
ASSERT_EQ(0, guards[i]->size_);
for (uint16_t j = 0; j < CAPACITY; ++j) {
ASSERT_EQ(0, guards[i]->event_times_[j]);
}
}
// out of order click and add time guard with diffrent size_
time_guard.click(0);
ASSERT_EQ(1, time_guard.size_);
time_guard.click(2);
ASSERT_EQ(3, time_guard.size_);
time_guard.click(1);
ASSERT_EQ(3, time_guard.size_);
time_guard.click(10);
ASSERT_EQ(11, time_guard.size_);
other_guard.click(6);
ASSERT_EQ(7, other_guard.size_);
summary_guard.add_time_guard(other_guard);
ASSERT_EQ(summary_guard.size_, other_guard.size_);
summary_guard.add_time_guard(time_guard);
ASSERT_EQ(summary_guard.size_, time_guard.size_);
}
TEST_F(TestCompactionTimeGuard, time_guard_to_string)
{
ObStorageCompactionTimeGuard storage_guard;
for (uint16_t i = 0; i < ObStorageCompactionTimeGuard::COMPACTION_EVENT_MAX; i++) {
storage_guard.click(i);
storage_guard.event_times_[i] += ObStorageCompactionTimeGuard::COMPACTION_SHOW_TIME_THRESHOLD;
ASSERT_EQ(i + 1, storage_guard.size_);
}
ASSERT_EQ(ObStorageCompactionTimeGuard::COMPACTION_EVENT_MAX, storage_guard.size_);
storage_guard.event_times_[ObStorageCompactionTimeGuard::DAG_WAIT_TO_SCHEDULE] += 2 * ObStorageCompactionTimeGuard::COMPACTION_SHOW_TIME_THRESHOLD;
storage_guard.event_times_[ObStorageCompactionTimeGuard::DAG_FINISH] += 2 * ObCompactionTimeGuard::WARN_THRESHOLD;
STORAGE_LOG(INFO, "storage guard is", K(storage_guard));
ObRSCompactionTimeGuard rs_guard;
for (uint16_t i = 0; i < ObRSCompactionTimeGuard::COMPACTION_EVENT_MAX; i++) {
rs_guard.click(i);
rs_guard.event_times_[i] += 10 * i;
ASSERT_EQ(i + 1, rs_guard.size_);
}
ASSERT_EQ(ObRSCompactionTimeGuard::COMPACTION_EVENT_MAX, rs_guard.size_);
rs_guard.event_times_[ObRSCompactionTimeGuard::CKM_VERIFICATION] += 2 * ObCompactionTimeGuard::WARN_THRESHOLD;
STORAGE_LOG(INFO, "rs guard is", K(rs_guard));
ObCompactionScheduleTimeGuard schedule_guard;
for (uint16_t i = 0; i < ObCompactionScheduleTimeGuard::COMPACTION_EVENT_MAX; i++) {
schedule_guard.click(i);
schedule_guard.event_times_[i] += 10 * i;
ASSERT_EQ(i + 1, schedule_guard.size_);
}
ASSERT_EQ(ObCompactionScheduleTimeGuard::COMPACTION_EVENT_MAX, schedule_guard.size_);
schedule_guard.event_times_[ObCompactionScheduleTimeGuard::SCHEDULER_NEXT_ROUND] += 2 * ObCompactionTimeGuard::WARN_THRESHOLD;
STORAGE_LOG(INFO, "schedule guard is", K(schedule_guard));
}
} // namespace unittest
} // namespace oceanbase
int main(int argc, char **argv)
{
::testing::InitGoogleTest(&argc, argv);
system("rm -f test_compaction_time_guard.log*");
OB_LOGGER.set_file_name("test_compaction_time_guard.log");
OB_LOGGER.set_log_level("INFO");
OB_LOGGER.set_max_file_size(256*1024*1024);
return RUN_ALL_TESTS();
}