hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

replace min_block_min_ts_ns with min_block_max_ts for block gc

Browse Source
This commit is contained in:
HaHaJeff
2022-11-03 03:36:46 +00:00
committed by wangzelin.wzl
parent 8fc92865eb
commit 5475afde93
7 changed files with 68 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/logservice/palf/log_engine.cpp
View File

@ -38,7 +38,7 @@ namespace palf
// ===================== LogEngine start =======================
LogEngine::LogEngine() :
min_block_ts_ns_(OB_INVALID_TIMESTAMP),
min_block_max_ts_ns_(OB_INVALID_TIMESTAMP),
min_block_id_(LOG_INVALID_BLOCK_ID),
base_lsn_for_block_gc_(PALF_INITIAL_LSN_VAL),
log_meta_lock_(),
@ -158,7 +158,7 @@ void LogEngine::destroy()
log_storage_.destroy();
base_lsn_for_block_gc_.reset();
min_block_id_ = LOG_INVALID_BLOCK_ID;
min_block_ts_ns_ = OB_INVALID_TIMESTAMP;
min_block_max_ts_ns_ = OB_INVALID_TIMESTAMP;
}
}
@ -537,8 +537,7 @@ int LogEngine::truncate_prefix_blocks(const LSN &lsn)
} else if (OB_FAIL(log_storage_.truncate_prefix_blocks(lsn))) {
PALF_LOG(WARN, "truncate_prefix_blocks failed", K(ret), K_(palf_id), K_(is_inited), K(lsn));
} else {
PALF_LOG(INFO, "truncate_prefix_blocks success", K(ret), K_(palf_id), K_(is_inited), K(lsn));
}
PALF_LOG(INFO, "truncate_prefix_blocks success", K(ret), K_(palf_id), K_(is_inited), K(lsn)); }
return ret;
}
@ -551,7 +550,7 @@ int LogEngine::delete_block(const block_id_t &block_id)
{
int ret = OB_SUCCESS;
block_id_t next_block_id = block_id + 1;
int64_t next_block_min_ts_ns = OB_INVALID_TIMESTAMP;
int64_t next_block_max_ts_ns = OB_INVALID_TIMESTAMP;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited));
@ -566,12 +565,12 @@ int LogEngine::delete_block(const block_id_t &block_id)
} else {
PALF_LOG(INFO, "delete success", K(block_id), K_(palf_id), K_(is_inited));
}
if (OB_SUCC(ret) && OB_FAIL(get_block_min_ts_ns(next_block_id , next_block_min_ts_ns))) {
PALF_LOG(WARN, "get the min ts of next block failed", K(ret), K(next_block_id));
int tmp_ret = OB_SUCCESS;
if (OB_SUCC(ret) && OB_SUCCESS != (tmp_ret = get_block_min_ts_ns(next_block_id+1, next_block_max_ts_ns))) {
PALF_LOG(WARN, "get the max ts of next block failed", K(tmp_ret), K(next_block_id));
}
// If 'delete_block' or 'get_block_min_ts_ns' failed, need reset 'min_block_ts_ns_' to be invalid.
reset_min_block_info_guarded_by_lock_(next_block_id, next_block_min_ts_ns);
reset_min_block_info_guarded_by_lock_(next_block_id, next_block_max_ts_ns);
return ret;
}
@ -587,30 +586,50 @@ int LogEngine::get_block_id_range(block_id_t &min_block_id, block_id_t &max_bloc
return log_storage_.get_block_id_range(min_block_id, max_block_id);
}
int LogEngine::get_min_block_id_and_min_ts_ns(block_id_t &block_id, int64_t &ts_ns) const
int LogEngine::get_min_block_info_for_gc(block_id_t &block_id, int64_t &max_ts_ns) const
{
int ret = OB_SUCCESS;
block_id_t max_block_id = LOG_INVALID_BLOCK_ID;
block_id_t min_block_id = LOG_INVALID_BLOCK_ID;
int64_t min_block_ts_ns = OB_INVALID_TIMESTAMP;
int64_t min_block_max_ts_ns = OB_INVALID_TIMESTAMP;
do {
ObSpinLockGuard guard(block_gc_lock_);
min_block_id = min_block_id_;
min_block_ts_ns = min_block_ts_ns_;
min_block_max_ts_ns = min_block_max_ts_ns_;
} while (0);
if (OB_INVALID_TIMESTAMP != min_block_ts_ns) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
PALF_LOG(WARN, "LogEngine is not inited", K(ret), KPC(this));
} else if (OB_INVALID_TIMESTAMP != min_block_max_ts_ns) {
block_id = min_block_id;
ts_ns = min_block_ts_ns;
max_ts_ns = min_block_max_ts_ns;
} else if (OB_FAIL(get_block_id_range(min_block_id, max_block_id))) {
PALF_LOG(WARN, "get_block_id_range failed", K(ret));
} else if (OB_FAIL(get_block_min_ts_ns(min_block_id, min_block_ts_ns))) {
// NB: used next block min_block_ts as the max_ts_ns of current block
} else if (OB_FAIL(get_block_min_ts_ns(min_block_id+1, min_block_max_ts_ns))) {
PALF_LOG(WARN, "get_block_min_ts_ns failed", K(ret));
} else {
reset_min_block_info_guarded_by_lock_(min_block_id, min_block_ts_ns);
reset_min_block_info_guarded_by_lock_(min_block_id, min_block_max_ts_ns);
block_id = min_block_id;
ts_ns = min_block_ts_ns;
max_ts_ns = min_block_max_ts_ns;
}
return ret;
}
int LogEngine::get_min_block_info(block_id_t &min_block_id, int64_t &min_block_ts_ns) const
{
int ret = OB_SUCCESS;
block_id_t max_block_id;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
PALF_LOG(WARN, "LogEngine is not inited", K(ret), KPC(this));
} else if (OB_FAIL(get_block_id_range(min_block_id, max_block_id))) {
PALF_LOG(WARN, "get_block_id_range failed", K(ret), KPC(this));
} else if (OB_FAIL(get_block_min_ts_ns(min_block_id, min_block_ts_ns))) {
PALF_LOG(WARN, "get_block_min_ts_ns failed", K(ret), KPC(this));
} else {
}
return ret;
}
@ -1255,11 +1274,11 @@ bool LogEngine::check_last_block_whether_is_integrity_(const block_id_t expected
|| expected_next_block_id < max_block_id;
}
void LogEngine::reset_min_block_info_guarded_by_lock_(const block_id_t min_block_id, const int64_t min_block_ts_ns) const
void LogEngine::reset_min_block_info_guarded_by_lock_(const block_id_t min_block_id, const int64_t min_block_max_ts_ns) const
{
ObSpinLockGuard guard(block_gc_lock_);
min_block_id_ = min_block_id;
min_block_ts_ns_ = min_block_ts_ns;
min_block_max_ts_ns_ = min_block_max_ts_ns;
}
} // end namespace palf
} // end namespace oceanbase

10
src/logservice/palf/log_engine.h
View File

@ -362,14 +362,15 @@ public:
LogMeta get_log_meta() const;
const LSN &get_base_lsn_used_for_block_gc() const;
// not thread safe
int get_min_block_id_and_min_ts_ns(block_id_t &block_id, int64_t &ts_ns) const;
int get_min_block_info_for_gc(block_id_t &block_id, int64_t &max_ts_ns) const;
int get_min_block_info(block_id_t &block_id, int64_t &min_log_ts) const;
//
// ===================== NetService end ========================
LogStorage *get_log_storage() { return &log_storage_; }
LogStorage *get_log_meta_storage() { return &log_meta_storage_; }
int get_total_used_disk_space(int64_t &total_used_size_byte) const;
virtual int64_t get_palf_epoch() const { return palf_epoch_; }
TO_STRING_KV(K_(palf_id), K_(is_inited), K_(min_block_ts_ns), K_(min_block_id), K_(base_lsn_for_block_gc),
TO_STRING_KV(K_(palf_id), K_(is_inited), K_(min_block_max_ts_ns), K_(min_block_id), K_(base_lsn_for_block_gc),
K_(log_meta), K_(log_meta_storage), K_(log_storage), K_(palf_epoch), KP(this));
private:
int submit_flush_meta_task_(const FlushMetaCbCtx &flush_meta_cb_ctx, const LogMeta &log_meta);
@ -405,7 +406,8 @@ private:
int serialize_log_meta_(const LogMeta &log_meta, char *buf, int64_t buf_len);
void reset_min_block_info_guarded_by_lock_(const block_id_t block_id, const int64_t min_ts_ns) const;
void reset_min_block_info_guarded_by_lock_(const block_id_t min_block_id,
const int64_t min_block_max_ts_ns) const;
private:
DISALLOW_COPY_AND_ASSIGN(LogEngine);
@ -413,7 +415,7 @@ private:
private:
// used for GC
mutable ObSpinLock block_gc_lock_;
mutable int64_t min_block_ts_ns_;
mutable int64_t min_block_max_ts_ns_;
mutable block_id_t min_block_id_;
LSN base_lsn_for_block_gc_;

24
src/logservice/palf/palf_env_impl.cpp
View File

@ -591,7 +591,7 @@ int PalfEnvImpl::try_freeze_log_for_all()
PalfEnvImpl::LogGetRecycableFileCandidate::LogGetRecycableFileCandidate()
: id_(-1),
min_block_id_(LOG_INVALID_BLOCK_ID),
min_block_id_ts_(OB_INVALID_TIMESTAMP),
min_block_max_ts_(OB_INVALID_TIMESTAMP),
min_using_block_id_(LOG_INVALID_BLOCK_ID),
oldest_palf_id_(INVALID_PALF_ID),
oldest_block_ts_(OB_INVALID_TIMESTAMP),
@ -602,7 +602,7 @@ PalfEnvImpl::LogGetRecycableFileCandidate::~LogGetRecycableFileCandidate()
{
ret_code_ = OB_SUCCESS;
min_using_block_id_ = LOG_INVALID_BLOCK_ID;
min_block_id_ts_ = OB_INVALID_TIMESTAMP;
min_block_max_ts_ = OB_INVALID_TIMESTAMP;
min_block_id_ = LOG_INVALID_BLOCK_ID;
oldest_palf_id_ = INVALID_PALF_ID;
oldest_block_ts_ = OB_INVALID_TIMESTAMP;
@ -620,13 +620,13 @@ bool PalfEnvImpl::LogGetRecycableFileCandidate::operator()(const LSKey &palf_id,
const LSN base_lsn = palf_handle_impl->get_base_lsn_used_for_block_gc();
const block_id_t min_using_block_id = lsn_2_block(base_lsn, PALF_BLOCK_SIZE);
block_id_t min_block_id = LOG_INVALID_BLOCK_ID;
int64_t min_block_id_ts = OB_INVALID_TIMESTAMP;
int64_t min_block_max_ts = OB_INVALID_TIMESTAMP;
if (false == base_lsn.is_valid()) {
PALF_LOG(WARN, "base_lsn is invalid", K(base_lsn), KPC(palf_handle_impl));
// OB_ENTRY_EXIST means there is not any block;
// OB_NO_SUCH_FILE_OR_DIRECTORY means there is concurrently with rebuild.
} else if (OB_FAIL(palf_handle_impl->get_min_block_id_min_ts_ns(min_block_id, min_block_id_ts))
} else if (OB_FAIL(palf_handle_impl->get_min_block_info_for_gc(min_block_id, min_block_max_ts))
&& OB_ENTRY_NOT_EXIST != ret
&& OB_NO_SUCH_FILE_OR_DIRECTORY != ret) {
ret_code_ = ret;
@ -644,20 +644,20 @@ bool PalfEnvImpl::LogGetRecycableFileCandidate::operator()(const LSKey &palf_id,
PALF_LOG(TRACE, "can not recycle blocks, need keep at least two blocks or has been concurrently"
" with rebuild, skip it",
K(ret), KPC(palf_handle_impl), K(min_block_id), K(min_using_block_id));
} else if (OB_INVALID_TIMESTAMP != min_block_id_ts_ && min_block_id_ts_ < min_block_id_ts) {
PALF_LOG(TRACE, "current palf_handle_impl is not older than previous, skip it", K(min_block_id_ts),
K(min_block_id_ts_), KPC(palf_handle_impl), K(min_block_id));
} else if (OB_INVALID_TIMESTAMP != min_block_max_ts_ && min_block_max_ts_ < min_block_max_ts) {
PALF_LOG(TRACE, "current palf_handle_impl is not older than previous, skip it", K(min_block_max_ts),
K(min_block_max_ts_), KPC(palf_handle_impl), K(min_block_id));
} else {
id_ = palf_id.id_;
min_block_id_ = min_block_id;
min_block_id_ts_ = min_block_id_ts;
min_block_max_ts_ = min_block_max_ts;
min_using_block_id_ = min_using_block_id;
PALF_LOG(TRACE, "can be recycable palf_handle_impl", K(id_), K(min_block_id_), K(min_using_block_id_),
K(min_block_id_ts_), K(base_lsn));
K(min_block_max_ts_), K(base_lsn));
}
if (OB_INVALID_TIMESTAMP != min_block_id_ts
&& (OB_INVALID_TIMESTAMP == oldest_block_ts_ || oldest_block_ts_ > min_block_id_ts)) {
oldest_block_ts_ = min_block_id_ts;
if (OB_INVALID_TIMESTAMP != min_block_max_ts
&& (OB_INVALID_TIMESTAMP == oldest_block_ts_ || oldest_block_ts_ > min_block_max_ts)) {
oldest_block_ts_ = min_block_max_ts;
oldest_palf_id_ = palf_id.id_;
}
}

4
src/logservice/palf/palf_env_impl.h
View File

@ -252,12 +252,12 @@ private:
bool operator() (const LSKey &palf_id, PalfHandleImpl *palf_handle_impl);
int64_t id_;
block_id_t min_block_id_;
int64_t min_block_id_ts_;
int64_t min_block_max_ts_;
block_id_t min_using_block_id_;
int64_t oldest_palf_id_;
int64_t oldest_block_ts_;
int ret_code_;
TO_STRING_KV(K_(id), K_(min_block_id_ts), K_(min_block_id), K_(min_using_block_id), K_(oldest_palf_id), K_(oldest_block_ts), K_(ret_code));
TO_STRING_KV(K_(id), K_(min_block_max_ts), K_(min_block_id), K_(min_using_block_id), K_(oldest_palf_id), K_(oldest_block_ts), K_(ret_code));
};
struct GetTotalUsedDiskSpace
{

14
src/logservice/palf/palf_handle_impl.cpp
View File

@ -301,8 +301,8 @@ int PalfHandleImpl::get_begin_ts_ns(int64_t &ts) const
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
PALF_LOG(WARN, "PalfHandleImpl not init", K(ret), KPC(this));
} else if (OB_FAIL(log_engine_.get_min_block_id_and_min_ts_ns(unused_block_id, ts))) {
PALF_LOG(WARN, "LogEngine get_min_block_id_and_min_ts_ns failed", K(ret), KPC(this));
} else if (OB_FAIL(log_engine_.get_min_block_info(unused_block_id, ts))) {
PALF_LOG(WARN, "LogEngine get_min_block_info failed", K(ret), KPC(this));
}
return ret;
}
@ -1595,16 +1595,16 @@ int PalfHandleImpl::locate_by_lsn_coarsely(const LSN &lsn, int64_t &result_ts_ns
return ret;
}
int PalfHandleImpl::get_min_block_id_min_ts_ns(block_id_t &block_id, int64_t &ts_ns)
int PalfHandleImpl::get_min_block_info_for_gc(block_id_t &min_block_id, int64_t &max_ts_ns)
{
int ret = OB_SUCCESS;
// if (false == end_lsn.is_valid()) {
// ret = OB_ENTRY_NOT_EXIST;
// }
if (OB_FAIL(log_engine_.get_min_block_id_and_min_ts_ns(block_id, ts_ns))) {
PALF_LOG(WARN, "get_min_block_id_and_min_ts_ns failed", K(ret), KPC(this));
if (OB_FAIL(log_engine_.get_min_block_info_for_gc(min_block_id, max_ts_ns))) {
PALF_LOG(WARN, "get_min_block_info_for_gc failed", K(ret), KPC(this));
} else {
PALF_LOG(TRACE, "get_min_block_id_and_min_ts_ns success", K(ret), KPC(this), K(block_id), K(ts_ns));
PALF_LOG(TRACE, "get_min_block_info_for_gc success", K(ret), KPC(this), K(min_block_id), K(max_ts_ns));
}
return ret;
}
@ -3485,7 +3485,7 @@ int PalfHandleImpl::stat(PalfStat &palf_stat)
palf_stat.allow_vote_ = state_mgr_.is_allow_vote();
palf_stat.replica_type_ = state_mgr_.get_replica_type();
palf_stat.base_lsn_ = log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_;
(void)log_engine_.get_min_block_id_and_min_ts_ns(min_block_id, min_block_min_ts);
(void)log_engine_.get_min_block_info(min_block_id, min_block_min_ts);
palf_stat.begin_lsn_ = LSN(min_block_id * PALF_BLOCK_SIZE);
palf_stat.begin_ts_ns_ = min_block_min_ts;
palf_stat.end_lsn_ = get_end_lsn();

4
src/logservice/palf/palf_handle_impl.h
View File

@ -423,7 +423,7 @@ public:
virtual int get_begin_ts_ns(int64_t &ts) const = 0;
virtual int get_base_info(const LSN &base_lsn, PalfBaseInfo &base_info) = 0;
virtual int get_min_block_id_min_ts_ns(block_id_t &block_id, int64_t &ts_ns) = 0;
virtual int get_min_block_info_for_gc(block_id_t &min_block_id, int64_t &max_ts_ns) = 0;
//begin lsn base lsn end lsn
// │ │ │
// │ │ │
@ -716,7 +716,7 @@ public:
int get_begin_lsn(LSN &lsn) const;
int get_begin_ts_ns(int64_t &ts) const;
int get_base_info(const LSN &base_lsn, PalfBaseInfo &base_info);
int get_min_block_id_min_ts_ns(block_id_t &block_id, int64_t &ts_ns) override final;
int get_min_block_info_for_gc(block_id_t &min_block_id, int64_t &max_ts_ns) override final;
// return the block length which the previous data was committed
const LSN get_end_lsn() const override final
{

2
unittest/logservice/mock_logservice_container/mock_log_engine.h
View File

@ -538,7 +538,7 @@ public:
{
return base_lsn_for_block_gc_;
}
int get_min_block_id_and_min_ts_ns(block_id_t &block_id, int64_t &ts_ns)
int get_min_block_info_for_gc(block_id_t &block_id, int64_t &ts_ns)
{
int ret = OB_SUCCESS;
UNUSED(block_id);