hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[scn] mv definition of SCN to dir of share

Browse Source
This commit is contained in:
obdev 2022-11-28 02:50:55 +00:00 committed by ob-robot
parent 68151eb87a
commit 99d4f56fec
716 changed files with 7746 additions and 6117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -28,13 +28,13 @@ See [Quick start](https://open.oceanbase.com/quickStart) to try out OceanBase Da
## System architecture
## ![image.png](https://cdn.nlark.com/yuque/0/2022/png/106206/1660643534074-2649c2e4-473a-4d07-8021-d8d1a2b2da49.png#clientId=u5d9acd3a-25ad-4&crop=0&crop=0&crop=1&crop=1&from=paste&height=1101&id=u46f6dd09&margin=%5Bobject%20Object%5D&name=image.png&originHeight=1101&originWidth=1746&originalType=binary&ratio=1&rotation=0&showTitle=false&size=130476&status=done&style=none&taskId=ud87a5d0f-5140-45f2-9274-c8375b3c7c0&title=&width=1746)
![image.png](https://cdn.nlark.com/yuque/0/2022/png/25820454/1667369873624-c1707034-471a-4f79-980f-6d1760dac8eb.png)
## Roadmap
![image.png](https://cdn.nlark.com/yuque/0/2022/png/106206/1660643534293-a5f53258-a9ac-462c-b9fd-9832901853c2.png#clientId=u5d9acd3a-25ad-4&crop=0&crop=0&crop=1&crop=1&from=paste&height=352&id=u0f71535e&margin=%5Bobject%20Object%5D&name=image.png&originHeight=528&originWidth=1683&originalType=binary&ratio=1&rotation=0&showTitle=false&size=719961&status=done&style=none&taskId=u676e97dd-1309-42b9-b380-a423c27199c&title=&width=1122)
![image.png](https://cdn.nlark.com/yuque/0/2022/png/25820454/1667369873613-44957682-76fe-42c2-b4c7-9356ed5b35f0.png)
Link: [3.1.5 function list](https://github.com/oceanbase/oceanbase/milestone/6)
Link: [4.0.0 function list](https://github.com/oceanbase/oceanbase/milestone/3)
## Case study
For our success stories, see [Success stories](https://www.oceanbase.com/en/customer/home).

2
deps/init/oceanbase.el7.aarch64.deps vendored
View File

@ -40,7 +40,7 @@ obdevtools-gcc9-9.3.0-52022092914.el7.aarch64.rpm
obdevtools-llvm-11.0.1-312022092921.el7.aarch64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-32022100420.el7.aarch64.rpm
devdeps-oblogmsg-1.0-42022101823.el7.aarch64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el7.aarch64.rpm
[test-utils]

2
deps/init/oceanbase.el7.x86_64.deps vendored
View File

@ -36,7 +36,7 @@ obdevtools-gcc9-9.3.0-52022092914.el7.x86_64.rpm
obdevtools-llvm-11.0.1-312022092921.el7.x86_64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-32022100420.el7.x86_64.rpm
devdeps-oblogmsg-1.0-42022101823.el7.x86_64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el7.x86_64.rpm
[test-utils]

2
deps/init/oceanbase.el8.aarch64.deps vendored
View File

@ -35,7 +35,7 @@ obdevtools-gcc9-9.3.0-52022092914.el8.aarch64.rpm
obdevtools-llvm-11.0.1-312022092921.el8.aarch64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-32022100420.el8.aarch64.rpm
devdeps-oblogmsg-1.0-42022101823.el8.aarch64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el8.aarch64.rpm
[test-utils]

2
deps/init/oceanbase.el8.x86_64.deps vendored
View File

@ -36,7 +36,7 @@ obdevtools-gcc9-9.3.0-52022092914.el8.x86_64.rpm
obdevtools-llvm-11.0.1-312022092921.el8.x86_64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-32022100420.el8.x86_64.rpm
devdeps-oblogmsg-1.0-42022101823.el8.x86_64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el8.x86_64.rpm
[test-utils]

2
deps/oblib/src/CMakeLists.txt vendored
View File

@ -45,7 +45,7 @@ if (OB_USE_CLANG)
if (ENABLE_SANITY)
target_compile_options(oblib_base_base
INTERFACE
"$<$<COMPILE_LANGUAGE:CXX>:SHELL:-Xclang -load -Xclang ${CMAKE_CURRENT_SOURCE_DIR}/lib/alloc/libsanity.so>"
"$<$<COMPILE_LANGUAGE:CXX>:SHELL:-Xclang -load -Xclang ${CMAKE_CURRENT_SOURCE_DIR}/lib/alloc/libsanity_${ARCHITECTURE}.so>"
-fno-builtin-memset
-fno-builtin-bzero
-fno-builtin-memcpy

BIN
deps/oblib/src/lib/alloc/libsanity_aarch64.so vendored Executable file
View File

Binary file not shown.

0
deps/oblib/src/lib/alloc/libsanity.so → deps/oblib/src/lib/alloc/libsanity_x86_64.so vendored
View File

17
deps/oblib/src/lib/allocator/ob_slice_alloc.h vendored
View File

@ -288,18 +288,21 @@ public:
blk_ref_[ObBlockSlicer::hash((uint64_t)old_blk) % MAX_REF_NUM].sync();
if (old_blk->release()) {
blk_list_.add(&old_blk->dlink_);
if (old_blk->recycle()) {
destroy_block(old_blk);
} else {
_LIB_LOG(ERROR, "there was memory leak, stock=%d, remain=%d", old_blk->stock(), old_blk->remain());
}
}
}
}
ObDLink* dlink = nullptr;
if (OB_NOT_NULL(dlink = blk_list_.top())) {
dlink = blk_list_.top();
while (OB_NOT_NULL(dlink)) {
Block* blk = CONTAINER_OF(dlink, Block, dlink_);
_LIB_LOG(ERROR, "there was memory leak, stock=%d, remain=%d", blk->stock(), blk->remain());
if (blk->recycle()) {
destroy_block(blk);
dlink = blk_list_.top();
} else {
_LIB_LOG(ERROR, "there was memory leak, stock=%d, total=%d, remain=%d"
, blk->stock(), blk->total(), blk->remain());
dlink = nullptr; // break
}
}
tmallocator_ = NULL;
bsize_ = 0;

2
deps/oblib/src/lib/ash/ob_active_session_guard.h vendored
View File

@ -31,6 +31,7 @@ public:
tenant_id_(0),
user_id_(0),
session_id_(0),
plan_id_(0),
sample_time_(0),
event_no_(0),
wait_time_(0),
@ -75,6 +76,7 @@ public:
uint64_t tenant_id_;
uint64_t user_id_;
uint64_t session_id_;
uint64_t plan_id_;
int64_t sample_time_; // sample time
int64_t event_no_; // wait event no
uint64_t wait_time_; // how long the event waited in total

7
deps/oblib/src/lib/resource/achunk_mgr.cpp vendored
View File

@ -130,7 +130,7 @@ void *AChunkMgr::low_alloc(const uint64_t size, const bool can_use_huge_page, bo
void *ptr = nullptr;
huge_page_used = false;
const int prot = PROT_READ | PROT_WRITE;
const int flags = MAP_PRIVATE | MAP_ANONYMOUS | (SANITY_BOOL_EXPR(alloc_shadow) ? MAP_FIXED : 0);
int flags = MAP_PRIVATE | MAP_ANONYMOUS | (SANITY_BOOL_EXPR(alloc_shadow) ? MAP_FIXED : 0);
int huge_flags = flags;
#ifdef MAP_HUGETLB
if (OB_LIKELY(can_use_huge_page)) {
@ -143,8 +143,11 @@ void *AChunkMgr::low_alloc(const uint64_t size, const bool can_use_huge_page, bo
if (SANITY_BOOL_EXPR(alloc_shadow)) {
int64_t new_addr = ATOMIC_FAA(&global_canonical_addr, size);
if (!SANITY_ADDR_IN_RANGE((void*)new_addr)) {
LOG_ERROR("sanity address exhausted", K(errno), KP(new_addr));
LOG_WARN("sanity address exhausted", K(errno), KP(new_addr));
ATOMIC_FAA(&global_canonical_addr, -size);
ptr = NULL; // let it goon, it means no shadow, same as out of checker!
// in aarch64, mmap will return EPERM error when NULL address and MAP_FIXED are privided at the same time
flags &= ~MAP_FIXED;
} else {
ptr = (void*)new_addr;
}

1
deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h vendored
View File

@ -444,6 +444,7 @@ PCODE_DEF(OB_CANCEL_DDL_TASK, 0x4A3)
PCODE_DEF(OB_HA_FETCH_SSTABLE_MACRO_INFO, 0x4A4)
PCODE_DEF(OB_HA_FETCH_LS_MEMBER_LIST, 0x4A5)
PCODE_DEF(OB_HA_FETCH_LS_META_INFO, 0x4A6)
PCODE_DEF(OB_NOTIFY_CREATE_TENANT_USER_LS, 0x4A7)
// sql, including executor
// sql

1
src/logservice/CMakeLists.txt
View File

@ -130,7 +130,6 @@ ob_set_subtarget(ob_logservice palf
palf/palf_handle_impl.cpp
palf/palf_handle_impl_guard.cpp
palf/log_block_pool_interface.cpp
palf/scn.cpp
palf/palf_options.cpp
palf/log_mode_mgr.cpp
)

8
src/logservice/applyservice/ob_log_apply_service.cpp
View File

@ -14,7 +14,7 @@
#include "logservice/ob_log_handler.h"
#include "logservice/ob_ls_adapter.h"
#include "logservice/palf/palf_env.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
#include "share/rc/ob_tenant_base.h"
#include "share/ob_thread_mgr.h"
#include "storage/tx_storage/ob_ls_service.h"
@ -391,7 +391,7 @@ int ObApplyStatus::push_append_cb(AppendCb *cb)
LSN palf_committed_end_lsn;
const LSN cb_lsn = cb->__get_lsn();
const SCN cb_scn = cb->__get_scn();
const uint64_t cb_sign = cb_scn.get_val_for_lsn_allocator();
const uint64_t cb_sign = cb_scn.get_val_for_logservice();
int64_t thread_index = cb_sign & (APPLY_TASK_QUEUE_SIZE - 1);
ObLink *link = AppendCb::__get_member_address(cb);
cb_queues_[thread_index].inc_total_submit_cb_cnt();
@ -676,7 +676,7 @@ int ObApplyStatus::get_min_unapplied_scn(SCN &scn)
} else if (FOLLOWER == role_) {
palf::LSN palf_end_lsn;
palf::LSN apply_end_lsn;
palf::SCN palf_end_scn;
SCN palf_end_scn;
bool is_done = false;
if (OB_FAIL(is_apply_done(is_done, apply_end_lsn))) {
CLOG_LOG(WARN, "check is_apply_done failed", K(ret), KPC(this));
@ -720,7 +720,7 @@ int ObApplyStatus::get_min_unapplied_scn(SCN &scn)
CLOG_LOG(ERROR, "max_applied_cb_scn_ larger than last_check_scn_, unexpected", K(ret), KPC(this));
}
if (OB_SUCC(ret) && max_applied_cb_scn_.is_valid()) {
scn = palf::SCN::plus(max_applied_cb_scn_, 1);
scn = SCN::plus(max_applied_cb_scn_, 1);
}
CLOG_LOG(TRACE, "get_min_unapplied_scn finish", K(ret), KPC(this), K(scn));
if (palf_reach_time_interval(5 * 1000 * 1000, get_info_debug_time_)) {

19
src/logservice/applyservice/ob_log_apply_service.h
View File

@ -18,15 +18,18 @@
#include "lib/thread/ob_thread_lease.h"
#include "logservice/palf/palf_callback.h"
#include "logservice/palf/palf_handle.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
#include "share/ob_ls_id.h"
namespace oceanbase
{
namespace share
{
class SCN;
}
namespace palf
{
class PalfEnv;
class SCN;
}
namespace logservice
{
@ -163,7 +166,7 @@ public:
int unregister_file_size_cb();
void close_palf_handle();
//最大连续回调位点
int get_min_unapplied_scn(palf::SCN &scn);
int get_min_unapplied_scn(share::SCN &scn);
int stat(LSApplyStat &stat) const;
int handle_drop_cb();
TO_STRING_KV(K(ls_id_),
@ -174,7 +177,7 @@ public:
K(max_applied_cb_scn_));
private:
int submit_task_to_apply_service_(ObApplyServiceTask &task);
int check_and_update_max_applied_scn_(const palf::SCN scn);
int check_and_update_max_applied_scn_(const share::SCN scn);
int update_last_check_scn_();
int handle_drop_cb_queue_(ObApplyServiceQueueTask &cb_queue);
int switch_to_follower_();
@ -185,7 +188,7 @@ private:
int64_t &cb_first_handle_time,
int64_t &cb_start_time);
void statistics_cb_cost_(const palf::LSN &lsn,
const palf::SCN &scn,
const share::SCN &scn,
const int64_t append_start_time,
const int64_t append_finish_time,
const int64_t cb_first_handle_time,
@ -209,8 +212,8 @@ private:
palf::LSN palf_committed_end_lsn_;
//LSN standy_committed_end_lsn_;
//palf::LSN min_committed_end_lsn_;
palf::SCN last_check_scn_; //当前待确认的最大连续回调位点
palf::SCN max_applied_cb_scn_; //该位点前的cb保证都已经回调完成
share::SCN last_check_scn_; //当前待确认的最大连续回调位点
share::SCN max_applied_cb_scn_; //该位点前的cb保证都已经回调完成
ObApplyServiceSubmitTask submit_task_;
ObApplyServiceQueueTask cb_queues_[APPLY_TASK_QUEUE_SIZE];
palf::PalfEnv *palf_env_;
@ -249,7 +252,7 @@ public:
palf::LSN &end_lsn);
int switch_to_leader(const share::ObLSID &id, const int64_t proposal_id);
int switch_to_follower(const share::ObLSID &id);
int get_min_unapplied_scn(const share::ObLSID &id, palf::SCN &scn);
int get_min_unapplied_scn(const share::ObLSID &id, share::SCN &scn);
int push_task(ObApplyServiceTask *task);
int wait_append_sync(const share::ObLSID &ls_id);
int stat_for_each(const common::ObFunction<int (const ObApplyStatus &)> &func);

1
src/logservice/archiveservice/ob_archive_define.cpp
View File

@ -21,6 +21,7 @@
namespace oceanbase
{
using namespace palf;
using namespace share;
namespace archive
{
// =================================== ObArchiveLease ================================= //

10
src/logservice/archiveservice/ob_archive_define.h
View File

@ -19,11 +19,11 @@
#include "share/backup/ob_archive_piece.h" // ObArchivePiece
#include "share/backup/ob_backup_struct.h" // ObBackupPathString
#include "logservice/palf/lsn.h" // LSN
#include "logservice/palf/scn.h" // SCN
#include "share/scn.h" // share::SCN
namespace oceanbase
{
namespace palf
namespace share
{
class SCN;
}
@ -88,7 +88,7 @@ class LogFileTuple
{
public:
LogFileTuple();
LogFileTuple(const LSN &lsn, const palf::SCN &scn, const ObArchivePiece &piece);
LogFileTuple(const LSN &lsn, const share::SCN &scn, const ObArchivePiece &piece);
~LogFileTuple();
public:
@ -98,14 +98,14 @@ public:
LogFileTuple &operator=(const LogFileTuple &other);
const LSN &get_lsn() const { return offset_; }
const palf::SCN &get_scn() const { return scn_; }
const share::SCN &get_scn() const { return scn_; }
const ObArchivePiece &get_piece() const { return piece_; };
void compensate_piece();
TO_STRING_KV(K_(offset), K_(scn), K_(piece));
private:
LSN offset_; // 文件内偏移
palf::SCN scn_; // 最大log scn
share::SCN scn_; // 最大log scn
ObArchivePiece piece_; // 所属piece
};

19
src/logservice/archiveservice/ob_archive_fetcher.h
View File

@ -20,7 +20,7 @@
#include "share/ob_ls_id.h" // ObLSID
#include "share/backup/ob_archive_piece.h" // ObArchivePiece
#include "logservice/palf/lsn.h" // LSN
#include "logservice/palf/scn.h" // SCN
#include "share/scn.h" // SCN
#include "logservice/palf/palf_iterator.h" // PalfGroupBufferIterator
#include "ob_archive_define.h"
@ -31,10 +31,13 @@ namespace logservice
class ObLogService;
}
namespace share
{
class SCN;
}
namespace palf
{
struct LSN;
class SCN;
class PalfHandleGuard;
class LogGroupEntry;
}
@ -82,7 +85,7 @@ public:
// 开启关闭归档接口
int set_archive_info(const int64_t piece_interval_us,
const palf::SCN &genesis_scn,
const share::SCN &genesis_scn,
const int64_t base_piece_id,
const int64_t unit_size,
const bool need_compress,
@ -133,7 +136,7 @@ private:
const LSN &end_offset, bool &data_enough, bool &data_full);
// 1.1.2 检查日志流落后程度是否需要触发归档
bool check_scn_enough_(const palf::SCN &fetch_scn, const palf::SCN &end_scn) const;
bool check_scn_enough_(const share::SCN &fetch_scn, const share::SCN &end_scn) const;
// 1.2 初始化TmpMemoryHelper
int init_helper_(ObArchiveLogFetchTask &task, TmpMemoryHelper &helper);
@ -231,7 +234,7 @@ private:
const LSN &get_start_offset() const { return start_offset_; }
const LSN &get_end_offset() const { return end_offset_; }
const LSN &get_cur_offset() const { return cur_offset_; }
const palf::SCN &get_unitized_scn() const { return unitized_scn_; }
const share::SCN &get_unitized_scn() const { return unitized_scn_; }
int64_t get_capaicity() const { return end_offset_ - cur_offset_; }
bool original_buffer_enough(const int64_t size);
int get_original_buf(char *&buf, int64_t &buf_size);
@ -281,14 +284,14 @@ private:
int64_t origin_buf_pos_;
// 当前处理原数日志最大offset/scn
LSN cur_offset_;
palf::SCN cur_scn_;
share::SCN cur_scn_;
// 中间缺少加密用buffer
char *ec_buf_;
int64_t ec_buf_size_; // 需要根据数据量计算
int64_t ec_buf_pos_;
// 做了单元化处理的日志最大offset/scn
LSN unitized_offset_;
palf::SCN unitized_scn_;
share::SCN unitized_scn_;
// 当前正在处理piece, 单个helper包含从start_offset到当前piece的全部数据
ObArchivePiece cur_piece_;
// 读取日志过程中, 遇到更大piece说明当前piece已经结束
@ -303,7 +306,7 @@ private:
// 压缩加密处理单元
int64_t unit_size_;
int64_t piece_interval_;
palf::SCN genesis_scn_;
share::SCN genesis_scn_;
int64_t base_piece_id_;
bool need_compress_;
// 压缩算法

4
src/logservice/archiveservice/ob_archive_persist_mgr.h
View File

@ -35,6 +35,7 @@ namespace share
class ObTenantArchiveRoundAttr;
class ObLSID;
class ObLSArchivePersistInfo;
class SCN;
}
namespace storage
@ -45,7 +46,6 @@ class ObLSService;
namespace palf
{
struct LSN;
class SCN;
}
namespace archive
@ -114,7 +114,7 @@ public:
void persist_and_load();
// 获取日志流创建时间戳
int get_ls_create_scn(const share::ObLSID &id, palf::SCN &scn);
int get_ls_create_scn(const share::ObLSID &id, share::SCN &scn);
private:
// 1. 持久化当前server做归档日志流归档进度

2
src/logservice/archiveservice/ob_archive_round_mgr.h
View File

@ -68,7 +68,7 @@ private:
private:
ArchiveKey key_;
palf::SCN round_start_scn_;
share::SCN round_start_scn_;
bool compatible_; // 该轮次兼容性处理
ObArchiveRoundState log_archive_state_;
share::ObBackupDest backup_dest_;

2
src/logservice/archiveservice/ob_archive_sender.cpp
View File

@ -39,7 +39,7 @@
#include "ob_archive_io.h" // ObArchiveIO
#include "share/backup/ob_backup_path.h" // ObBackupPath
#include "share/backup/ob_archive_path.h" // ObArchivePathUtil
#include "logservice/palf/scn.h" // ObArchivePathUtil
#include "share/scn.h" // ObArchivePathUtil
namespace oceanbase
{

8
src/logservice/archiveservice/ob_archive_sequencer.h
View File

@ -19,7 +19,7 @@
#include "ob_archive_define.h" // DEFAULT_THREAD_RUN_INTERVAL
#include "share/ob_ls_id.h" // ObLSID
#include "logservice/palf/lsn.h" // LSN
#include "logservice/palf/scn.h" // SCN
#include "share/scn.h" // SCN
#include <cstdint>
namespace oceanbase
@ -100,7 +100,7 @@ private:
ObArchiveLSMgr *ls_mgr_;
ObArchiveRoundMgr *round_mgr_;
// 全部日志流最小定序任务scn
palf::SCN min_scn_;
share::SCN min_scn_;
// 添加开启归档/日志流归档任务/以及消费LogFetchTask唤醒sequencer
common::ObCond seq_cond_;
@ -121,7 +121,7 @@ public:
min_scn_(),
id_() {}
bool operator()(const ObLSID &id, ObLSArchiveTask *ls_archive_task);
void get_min_log_info(ObLSID &id, palf::SCN &min_scn);
void get_min_log_info(ObLSID &id, share::SCN &min_scn);
TO_STRING_KV(K_(incarnation), K_(round), K_(succ_count), K_(total_count), K_(min_scn), K_(id));
private:
@ -129,7 +129,7 @@ private:
int64_t round_;
int64_t succ_count_;
int64_t total_count_;
palf::SCN min_scn_;
share::SCN min_scn_;
//MaxLogFileInfo min_log_info_;
ObLSID id_;
};

2
src/logservice/archiveservice/ob_archive_service.cpp
View File

@ -19,7 +19,7 @@
#include "share/rc/ob_tenant_base.h" // MTL_*
#include "observer/ob_server_struct.h" // GCTX
#include "logservice/palf/lsn.h" // LSN
#include "logservice/palf/scn.h" // LSN
#include "share/scn.h" // LSN
#include "share/backup/ob_backup_connectivity.h"
#include "share/ob_debug_sync.h"

1
src/logservice/archiveservice/ob_archive_task.cpp
View File

@ -18,6 +18,7 @@
namespace oceanbase
{
using namespace palf;
using namespace share;
namespace archive
{

14
src/logservice/archiveservice/ob_archive_task.h
View File

@ -18,7 +18,7 @@
#include "ob_archive_define.h" // ArchiveWorkStation
#include "share/backup/ob_archive_piece.h" // ObArchivePiece
#include "logservice/palf/lsn.h" // LSN
#include "logservice/palf/scn.h" // SCN
#include "share/scn.h" // SCN
#include <cstdint>
namespace oceanbase
@ -77,7 +77,7 @@ public:
const LSN &get_start_offset() const { return start_offset_; }
const LSN &get_cur_offset() const { return cur_offset_; }
const LSN &get_end_offset() const { return end_offset_; }
const palf::SCN &get_max_scn() const { return max_scn_; }
const share::SCN &get_max_scn() const { return max_scn_; }
int64_t get_log_fetch_size() const { return (int64_t)(end_offset_ - cur_offset_); }
const ObArchivePiece &get_piece() const { return cur_piece_; }
int set_next_piece(const ObArchivePiece &piece);
@ -89,7 +89,7 @@ public:
int back_fill(const ObArchivePiece &cur_piece,
const LSN &start_offset,
const LSN &end_offset,
const palf::SCN &max_scn,
const share::SCN &max_scn,
ObArchiveSendTask *send_task);
bool has_fetch_log() const { return NULL != send_task_; }
bool is_continuous_with(const LSN &lsn) const;
@ -114,7 +114,7 @@ private:
LSN start_offset_; // 起始拉日志文件起始offset
LSN end_offset_; // 拉取日志文件结束offset(如果是拉取完整文件任务，该值为infoblock起点)
LSN cur_offset_;
palf::SCN max_scn_; // 任务包含最大日志log scn
share::SCN max_scn_; // 任务包含最大日志log scn
ObArchiveSendTask *send_task_;
};
@ -159,7 +159,7 @@ public:
const ObArchivePiece &piece,
const LSN &start_offset,
const LSN &end_offset,
const palf::SCN &max_scn,
const share::SCN &max_scn,
char *data,
const int64_t data_len);
bool is_valid() const;
@ -171,7 +171,7 @@ public:
const LSN &get_end_lsn() const { return end_offset_; };
int get_buffer(char *&data, int64_t &data_len) const;
int64_t get_buf_size() const { return data_len_;}
const palf::SCN &get_max_scn() const { return max_scn_; }
const share::SCN &get_max_scn() const { return max_scn_; }
int set_buffer(char *buf, const int64_t buf_size);
bool is_continuous_with(const ObArchiveSendTask &pre_task) const;
TO_STRING_KV(K_(tenant_id),
@ -190,7 +190,7 @@ private:
ObArchivePiece piece_;
LSN start_offset_; // 归档数据在文件起始offset
LSN end_offset_; // 归档数据在文件终止offset
palf::SCN max_scn_; // 该task包含数据最大log scn
share::SCN max_scn_; // 该task包含数据最大log scn
char *data_; // 发送数据
int64_t data_len_; // 发送数据长度
};

12
src/logservice/archiveservice/ob_ls_mgr.h
View File

@ -20,7 +20,7 @@
#include "common/ob_queue_thread.h" // ObCond
#include "ob_ls_task.h" // ObLSArchiveTask
#include "ob_start_archive_helper.h" // StartArchiveHelper
#include "logservice/palf/scn.h" // SCn
#include "share/scn.h" // SCn
#include <cstdint>
namespace oceanbase
@ -79,14 +79,14 @@ public:
void stop();
void wait();
void destroy();
int set_archive_info(const palf::SCN &round_start_scn,
const int64_t piece_interval, const palf::SCN &genesis_scn, const int64_t base_piece_id);
int set_archive_info(const share::SCN &round_start_scn,
const int64_t piece_interval, const share::SCN &genesis_scn, const int64_t base_piece_id);
void clear_archive_info();
void notify_start();
void notify_stop();
int revert_ls_task(ObLSArchiveTask *task);
int get_ls_guard(const ObLSID &id, ObArchiveLSGuard &guard);
int authorize_ls_archive_task(const ObLSID &id, const int64_t epoch, const palf::SCN &start_scn);
int authorize_ls_archive_task(const ObLSID &id, const int64_t epoch, const share::SCN &start_scn);
void reset_task();
int64_t get_ls_task_count() const { return ls_map_.count(); }
int mark_fata_error(const ObLSID &id, const ArchiveKey &key, const ObArchiveInterruptReason &reason);
@ -107,9 +107,9 @@ private:
private:
bool inited_;
uint64_t tenant_id_;
palf::SCN round_start_scn_;
share::SCN round_start_scn_;
int64_t piece_interval_;
palf::SCN genesis_scn_;
share::SCN genesis_scn_;
int64_t base_piece_id_;
LSArchiveMap ls_map_;

20
src/logservice/archiveservice/ob_ls_task.h
View File

@ -20,7 +20,7 @@
#include "ob_archive_define.h" // ArchiveWorkStation LogFileTuple
#include "share/ob_ls_id.h" // ObLSID
#include "logservice/palf/lsn.h" // LSN
#include "logservice/palf/scn.h" // SCN
#include "share/scn.h" // SCN
#include "lib/utility/ob_print_utils.h" // print
#include <cstdint>
@ -97,7 +97,7 @@ public:
// 获取fetch进度
int get_fetcher_progress(const ArchiveWorkStation &station,
palf::LSN &offset,
palf::SCN &scn);
share::SCN &scn);
int compensate_piece(const ArchiveWorkStation &station,
const int64_t next_compensate_piece_id);
@ -147,21 +147,21 @@ private:
public:
void init(const LSN &piece_min_lsn, const LSN &lsn, const int64_t file_id,
const int64_t file_offset, const share::ObArchivePiece &piece,
const palf::SCN &max_archived_scn, const bool is_log_gap_exist,
const share::SCN &max_archived_scn, const bool is_log_gap_exist,
ObArchiveAllocator *allocator);
void destroy();
void get_sequencer_progress(LSN &offset) const;
int update_sequencer_progress(const int64_t size, const LSN &offset);
void get_fetcher_progress(LogFileTuple &tuple) const;
int update_fetcher_progress(const palf::SCN &round_start_scn, const LogFileTuple &tuple);
int update_fetcher_progress(const share::SCN &round_start_scn, const LogFileTuple &tuple);
int push_fetch_log(ObArchiveLogFetchTask &task);
int push_send_task(ObArchiveSendTask &task, ObArchiveWorker &worker);
int get_top_fetch_log(ObArchiveLogFetchTask *&task);
int pop_fetch_log(ObArchiveLogFetchTask *&task);
int compensate_piece(const int64_t piece_id);
void get_max_archive_progress(LSN &piece_min_lsn, LSN &lsn, palf::SCN &scn, ObArchivePiece &piece,
void get_max_archive_progress(LSN &piece_min_lsn, LSN &lsn, share::SCN &scn, ObArchivePiece &piece,
int64_t &file_id, int64_t &file_offset, bool &error_exist);
int update_archive_progress(const palf::SCN &round_start_scn, const int64_t file_id, const int64_t file_offset, const LogFileTuple &tuple);
int update_archive_progress(const share::SCN &round_start_scn, const int64_t file_id, const int64_t file_offset, const LogFileTuple &tuple);
void get_archive_progress(int64_t &file_id, int64_t &file_offset, LogFileTuple &tuple);
void get_archive_send_arg(ObArchiveSendDestArg &arg);
void mark_error();
@ -203,7 +203,7 @@ private:
ObLSID id_;
uint64_t tenant_id_;
ArchiveWorkStation station_;
palf::SCN round_start_scn_;
share::SCN round_start_scn_;
ArchiveDest dest_;
ObArchiveAllocator *allocator_;
mutable RWLock rwlock_;
@ -218,17 +218,17 @@ struct LSArchiveStat
int64_t round_;
// dest_type dest_value
int64_t lease_id_;
palf::SCN round_start_scn_;
share::SCN round_start_scn_;
int64_t max_issued_log_lsn_;
int64_t issued_task_count_;
int64_t issued_task_size_;
int64_t max_prepared_piece_id_;
int64_t max_prepared_lsn_;
palf::SCN max_prepared_scn_;
share::SCN max_prepared_scn_;
int64_t wait_send_task_count_;
int64_t archive_piece_id_;
int64_t archive_lsn_;
palf::SCN archive_scn_;
share::SCN archive_scn_;
int64_t archive_file_id_;
int64_t archive_file_offset_;
TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(dest_id), K_(incarnation), K_(round),

3
src/logservice/archiveservice/ob_start_archive_helper.cpp
View File

@ -30,6 +30,7 @@ namespace archive
{
using namespace oceanbase::logservice;
using namespace oceanbase::palf;
using namespace oceanbase::share;
StartArchiveHelper::StartArchiveHelper(const ObLSID &id,
const uint64_t tenant_id,
const ArchiveWorkStation &station,
@ -239,7 +240,7 @@ int StartArchiveHelper::get_local_start_scn_(SCN &scn)
if (OB_FAIL(persist_mgr_->get_ls_create_scn(id_, create_scn))) {
ARCHIVE_LOG(WARN, "get ls create scn failed", K(ret), K(id_));
} else {
SCN last_scn = palf::SCN::minus(min_scn_, 1);
SCN last_scn = SCN::minus(min_scn_, 1);
scn = create_scn > last_scn ? create_scn : last_scn;
if (!scn.is_valid()) {
ret = OB_ERR_UNEXPECTED;

18
src/logservice/archiveservice/ob_start_archive_helper.h
View File

@ -17,7 +17,7 @@
#include "share/ob_ls_id.h" // ObLSID
#include "share/backup/ob_backup_struct.h" // ObBackupPathString
#include "logservice/palf/lsn.h" // LSN
#include "logservice/palf/scn.h" // SCN
#include "share/scn.h" // SCN
#include "ob_archive_define.h" // ArchiveWorkStation
#include "ob_archive_persist_mgr.h" // ObArchivePersistMgr ObLSArchivePersistInfo
@ -48,9 +48,9 @@ public:
explicit StartArchiveHelper(const ObLSID &id,
const uint64_t tenant_id,
const ArchiveWorkStation &station,
const palf::SCN &min_scn,
const share::SCN &min_scn,
const int64_t piece_interval,
const palf::SCN &genesis_scn,
const share::SCN &genesis_scn,
const int64_t base_piece_id,
ObArchivePersistMgr *persist_mgr);
@ -66,8 +66,8 @@ public:
const LSN &get_offset() const { return start_offset_; }
int64_t get_file_id() const { return archive_file_id_; }
int64_t get_file_offset() const { return archive_file_offset_; }
const palf::SCN &get_round_start_scn() const { return min_scn_; }
const palf::SCN &get_max_archived_scn() const { return max_archived_scn_; }
const share::SCN &get_round_start_scn() const { return min_scn_; }
const share::SCN &get_max_archived_scn() const { return max_archived_scn_; }
const share::ObArchivePiece &get_piece() const { return piece_; }
bool is_log_gap_exist() const { return log_gap_exist_; }
TO_STRING_KV(K_(id),
@ -90,22 +90,22 @@ private:
int locate_round_start_archive_point_();
int cal_archive_file_id_offset_(const LSN &lsn, const int64_t archive_file_id, const int64_t archive_file_offset);
int get_local_base_lsn_(palf::LSN &lsn, bool &log_gap);
int get_local_start_scn_(palf::SCN &scn);
int get_local_start_scn_(share::SCN &scn);
private:
ObLSID id_;
uint64_t tenant_id_;
ArchiveWorkStation station_;
bool log_gap_exist_;
palf::SCN min_scn_;
share::SCN min_scn_;
int64_t piece_interval_;
palf::SCN genesis_scn_;
share::SCN genesis_scn_;
int64_t base_piece_id_;
LSN piece_min_lsn_;
LSN start_offset_;
int64_t archive_file_id_;
int64_t archive_file_offset_;
palf::SCN max_archived_scn_;
share::SCN max_archived_scn_;
share::ObArchivePiece piece_;
ObArchivePersistMgr *persist_mgr_;

2
src/logservice/cdcservice/ob_cdc_fetcher.cpp
View File

@ -344,7 +344,7 @@ void ObCdcFetcher::check_next_group_entry_(const LSN &next_lsn,
bool &reach_upper_limit)
{
//TODO(scn)
int64_t submit_ts = next_log_group_entry.get_scn().get_val_for_lsn_allocator();
int64_t submit_ts = next_log_group_entry.get_scn().get_val_for_logservice();
int64_t entry_size = next_log_group_entry.get_serialize_size();
bool is_buf_full = (! resp.has_enough_buffer(entry_size));

2
src/logservice/cdcservice/ob_cdc_start_lsn_locator.cpp
View File

@ -180,7 +180,7 @@ int ObCdcStartLsnLocator::do_locate_ls_(const ObLocateLSNByTsReq::LocateParam &l
} else if (OB_FAIL(group_iter.get_entry(log_group_entry, result_lsn))) {
LOG_WARN("group_iter get_entry fail", KR(ret), K_(tenant_id), K(ls_id));
} else {
result_ts_ns = log_group_entry.get_scn().get_val_for_lsn_allocator();
result_ts_ns = log_group_entry.get_scn().get_val_for_logservice();
}
// Unconditional setting ret code

6
src/logservice/leader_coordinator/election_priority_impl/election_priority_impl.h
View File

@ -20,7 +20,7 @@
#include "lib/guard/ob_unique_guard.h"
#include "lib/utility/ob_unify_serialize.h"
#include "logservice/palf/election/interface/election_priority.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
#include "share/ob_ls_id.h"
#include "share/ob_cluster_version.h"
#include <type_traits>
@ -111,7 +111,7 @@ public:
protected:
// 刷新优先级的方法
virtual int refresh_(const share::ObLSID &ls_id) override;
int get_scn_(const share::ObLSID &ls_id, palf::SCN &scn);
int get_scn_(const share::ObLSID &ls_id, share::SCN &scn);
private:
int compare_observer_stopped_(int &ret, const PriorityV1&) const;
int compare_server_stopped_flag_(int &ret, const PriorityV1&) const;
@ -130,7 +130,7 @@ private:
common::ObSArray<FailureEvent> fatal_failures_;// negative infos
bool is_primary_region_;
common::ObSArray<FailureEvent> serious_failures_;// negative infos
palf::SCN scn_;
share::SCN scn_;
bool is_in_blacklist_;
common::ObStringHolder in_blacklist_reason_;
bool is_manual_leader_;

11
src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp
View File

@ -28,6 +28,7 @@
namespace oceanbase
{
using namespace share;
namespace logservice
{
namespace coordinator
@ -79,7 +80,7 @@ int PriorityV1::compare(const AbstractPriority &rhs, int &result, ObStringHolder
#undef PRINT_WRAPPER
}
int PriorityV1::get_scn_(const share::ObLSID &ls_id, palf::SCN &scn)
int PriorityV1::get_scn_(const share::ObLSID &ls_id, SCN &scn)
{
#define PRINT_WRAPPER KR(ret), K(MTL_ID()), K(ls_id), K(*this)
int ret = OB_SUCCESS;
@ -94,7 +95,7 @@ int PriorityV1::get_scn_(const share::ObLSID &ls_id, palf::SCN &scn)
} else {
common::ObRole role;
int64_t unused_pid = -1;
palf::SCN min_unreplay_scn;
SCN min_unreplay_scn;
if (OB_FAIL(palf_handle_guard.get_role(role, unused_pid))) {
COORDINATOR_LOG_(WARN, "get_role failed");
} else if (OB_FAIL(palf_handle_guard.get_max_scn(scn))) {
@ -105,9 +106,9 @@ int PriorityV1::get_scn_(const share::ObLSID &ls_id, palf::SCN &scn)
ret = OB_SUCCESS;
} else if (!min_unreplay_scn.is_valid_and_not_min()) {
scn.set_min();
} else if (palf::SCN::minus(min_unreplay_scn, 1) < scn) {
} else if (SCN::minus(min_unreplay_scn, 1) < scn) {
// For restore case, min_unreplay_scn may be larger than max_ts.
scn = palf::SCN::minus(min_unreplay_scn, 1) ;
scn = SCN::minus(min_unreplay_scn, 1) ;
} else {}
} else {}
COORDINATOR_LOG_(TRACE, "get_scn_ finished", K(role), K(min_unreplay_scn), K(scn));
@ -127,7 +128,7 @@ int PriorityV1::refresh_(const share::ObLSID &ls_id)
ObLeaderCoordinator* coordinator = MTL(ObLeaderCoordinator*);
ObFailureDetector* detector = MTL(ObFailureDetector*);
LsElectionReferenceInfo election_reference_info;
palf::SCN scn = palf::SCN::min_scn();
SCN scn = SCN::min_scn();
if (OB_ISNULL(coordinator) || OB_ISNULL(detector)) {
ret = OB_ERR_UNEXPECTED;
COORDINATOR_LOG_(ERROR, "unexpected nullptr");

2
src/logservice/libobcdc/src/ob_cdc_part_trans_resolver.cpp
View File

@ -728,7 +728,7 @@ int ObCDCPartTransResolver::handle_commit_(
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("invalid trans_type", KR(ret), K_(tls_id), K(tx_id), K(commit_log), K(lsn));
} else if (OB_UNLIKELY(OB_INVALID_VERSION ==
(trans_commit_version = get_trans_commit_version_(submit_ts, commit_log.get_commit_version().get_val_for_lsn_allocator())))) {
(trans_commit_version = get_trans_commit_version_(submit_ts, commit_log.get_commit_version().get_val_for_logservice())))) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("invalid trans_commit_version", KR(ret), K_(tls_id), K(tx_id), K(lsn), K(submit_ts), K(commit_log));
} else if (!serve_info.is_served(trans_commit_version)) {

6
src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp
View File

@ -257,7 +257,7 @@ int LSFetchCtx::read_log(
int ret = OB_SUCCESS;
const char *buf = log_entry.get_data_buf();
const int64_t buf_len = log_entry.get_data_len();
const int64_t submit_ts = log_entry.get_scn().get_val_for_lsn_allocator();
const int64_t submit_ts = log_entry.get_scn().get_val_for_logservice();
int64_t pos = 0;
logservice::ObLogBaseHeader log_base_header;
@ -318,7 +318,7 @@ int LSFetchCtx::read_miss_tx_log(
int ret = OB_SUCCESS;
const char *buf = log_entry.get_data_buf();
const int64_t buf_len = log_entry.get_data_len();
const int64_t submit_ts = log_entry.get_scn().get_val_for_lsn_allocator();
const int64_t submit_ts = log_entry.get_scn().get_val_for_logservice();
int64_t pos = 0;
logservice::ObLogBaseHeader log_base_header;
@ -354,7 +354,7 @@ int LSFetchCtx::update_progress(
const palf::LSN &group_entry_lsn)
{
int ret = OB_SUCCESS;
const int64_t submit_ts = group_entry.get_scn().get_val_for_lsn_allocator();
const int64_t submit_ts = group_entry.get_scn().get_val_for_logservice();
const int64_t group_entry_serialize_size = group_entry.get_serialize_size();
// Verifying log continuity

17
src/logservice/libobcdc/src/ob_log_meta_manager.cpp
View File

@ -768,15 +768,28 @@ int ObLogMetaManager::set_column_meta_(IColMeta *col_meta,
col_meta->setValuesOfEnumSet(extended_type_info_vec);
//mysql treat it as MYSQL_TYPE_STRING, it is not suitable for libobcdc
if (ObEnumType == column_schema.get_data_type()) {
if (ObEnumType == col_type) {
mysql_type = obmysql::MYSQL_TYPE_ENUM;
} else if (ObSetType == column_schema.get_data_type()) {
} else if (ObSetType == col_type) {
mysql_type = obmysql::MYSQL_TYPE_SET;
}
} else if (ObNumberType == col_type || ObUNumberType == col_type) {
col_meta->setScale(column_schema.get_data_scale());
col_meta->setPrecision(column_schema.get_data_precision());
}
bool signed_flag = ((type_flag & UNSIGNED_FLAG) == 0);
if (ObBitType == col_type) {
// the length of BIT type is required,
// the "length" of the BIT type is store in precision
col_meta->setLength(column_schema.get_data_precision());
} else {
// for types with valid length(string\enumset\rowid\json\raw\lob\geo),
// get_data_length returns the valid length, returns 0 for other types.
col_meta->setLength(column_schema.get_data_length());
}
col_meta->setName(column_schema.get_column_name());
col_meta->setType(static_cast<int>(mysql_type));
col_meta->setSigned(signed_flag);

44
src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp
View File

@ -520,6 +520,9 @@ int ObBinlogRecordPrinter::output_data_file_column_data(IBinlogRecord *br,
const char *is_not_null = col_meta ? (col_meta->isNotNull() ? "true" : "false") : "NULL";
// const char *default_val = col_meta ? col_meta->getDefault() : "NULL";
const char *is_signed = col_meta ? (col_meta->isSigned() ? "true" : "false") : "NULL";
const long scale = col_meta ? col_meta->getScale(): 0;
const long precision = col_meta ? col_meta->getPrecision(): 0;
const long col_data_length = col_meta ? col_meta->getLength(): 0;
bool is_generated_column = col_meta ? col_meta->isGenerated() : false;
bool is_hidden_row_key_column = col_meta ? col_meta->isHiddenRowKey() : false;
bool is_lob = is_lob_type(ctype);
@ -533,23 +536,40 @@ int ObBinlogRecordPrinter::output_data_file_column_data(IBinlogRecord *br,
ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_signed:%s", column_index, is_signed);
ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_encoding:%s", column_index, encoding);
ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_not_null:%s", column_index, is_not_null);
if (enable_print_detail && is_hidden_row_key_column) {
ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_hidden_rowkey:%d", column_index, is_hidden_row_key_column);
if (enable_print_detail) {
if (is_hidden_row_key_column) {
ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_hidden_rowkey:%d", column_index, is_hidden_row_key_column);
}
// print the length of varchar only in print detail mode,
// because there have been many test cases with varchar type before the varchar length info is added into column meta
if (oceanbase::obmysql::MYSQL_TYPE_VAR_STRING == ctype || oceanbase::obmysql::MYSQL_TYPE_BIT == ctype) {
ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_define_length:%ld", column_index, col_data_length);
}
else if ((oceanbase::obmysql::MYSQL_TYPE_ENUM == ctype) || (oceanbase::obmysql::MYSQL_TYPE_SET == ctype)) {
const std::string delim = ",";
for (int i = 0; i < values_of_enum_set->size(); i++) {
enum_set_values_str += (*values_of_enum_set)[i];
if (i != values_of_enum_set->size() - 1) {
enum_set_values_str += delim;
}
}
ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_extend_info:%s", column_index, enum_set_values_str.c_str());
}
// print precision & scale only in print detail mode, becacuse INT in oracle mode is also a kind of NUMBER(DECIMAL)
// whose precision is 38 and scale is 0, more importantly, the default precision(-1, PRECISION_UNKNOWN_YET)
// and scale(-85, ORA_NUMBER_SCALE_UNKNOWN_YET) of NUMBER in oracle mode is confusing, so we decide not to
// modify test results for oracle mode temporarily for convenience and efficiency.
// TODO
else if ((oceanbase::obmysql::MYSQL_TYPE_DECIMAL == ctype) || (oceanbase::obmysql::MYSQL_TYPE_NEWDECIMAL == ctype)) {
// Not sure if MYSQL_TYPE_DECIMAL is deprecated, DECIMAL in mysql & oracle mode should be MYSQL_TYPE_NEWDECIMAL
ROW_PRINTF(ptr, size, pos , ri, "[C%ld] column_precision:%ld", column_index, precision);
ROW_PRINTF(ptr, size, pos , ri, "[C%ld] column_scale:%ld", column_index, scale);
} else { }
}
if (is_generated_column) {
ROW_PRINTF(ptr, size, pos, ri, "[C%ld] is_generated_column:%d", column_index, is_generated_column);
}
if ((oceanbase::obmysql::MYSQL_TYPE_ENUM == ctype) || (oceanbase::obmysql::MYSQL_TYPE_SET == ctype)) {
const std::string delim = ",";
for (int i = 0; i < values_of_enum_set->size(); i++) {
enum_set_values_str += (*values_of_enum_set)[i];
if (i != values_of_enum_set->size() - 1) {
enum_set_values_str += delim;
}
}
ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_extend_info:%s", column_index, enum_set_values_str.c_str());
}
// FIXME: does not check the value of the field until the length of the default value can be obtained
// ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_default_value:%s", column_index, default_val);

3
src/logservice/logrpc/ob_log_rpc_req.cpp
View File

@ -14,6 +14,7 @@
namespace oceanbase
{
using namespace share;
namespace logservice
{
@ -167,7 +168,7 @@ OB_SERIALIZE_MEMBER(LogGetPalfStatReq, src_, palf_id_);
// ============= LogGetPalfStatResp begin ===========
LogGetPalfStatResp::LogGetPalfStatResp(
const palf::SCN &max_scn)
const SCN &max_scn)
: max_scn_(max_scn) { }
LogGetPalfStatResp::~LogGetPalfStatResp()

6
src/logservice/logrpc/ob_log_rpc_req.h
View File

@ -16,7 +16,7 @@
#include "lib/utility/ob_unify_serialize.h" // OB_UNIS_VERSION
#include "lib/utility/ob_print_utils.h" // TO_STRING_KV
#include "common/ob_member_list.h" // ObMemberList
#include "logservice/palf/scn.h"
#include "share/scn.h"
namespace oceanbase
{
@ -131,12 +131,12 @@ struct LogGetPalfStatResp {
OB_UNIS_VERSION(1);
public:
LogGetPalfStatResp() {};
LogGetPalfStatResp(const palf::SCN &max_scn);
LogGetPalfStatResp(const share::SCN &max_scn);
~LogGetPalfStatResp();
bool is_valid() const;
void reset();
TO_STRING_KV(K_(max_scn));
palf::SCN max_scn_;
share::SCN max_scn_;
};
} // end namespace logservice

8
src/logservice/ob_append_callback.h
View File

@ -13,8 +13,8 @@
#ifndef OCEANBASE_LOGSERVICE_OB_APPEND_CALLBACK_
#define OCEANBASE_LOGSERVICE_OB_APPEND_CALLBACK_
#include "palf/lsn.h"
#include "palf/scn.h"
#include "lib/utility/utility.h"
#include "share/scn.h"
namespace oceanbase
{
@ -35,15 +35,15 @@ public:
void __reset() { __start_lsn_.reset(); __next_ = NULL; __scn_.reset();}
const palf::LSN &__get_lsn() const { return __start_lsn_; }
void __set_lsn(const palf::LSN &lsn) { __start_lsn_ = lsn; }
const palf::SCN& __get_scn() const { return __scn_; }
void __set_scn(const palf::SCN& scn) { __scn_ = scn; }
const share::SCN& __get_scn() const { return __scn_; }
void __set_scn(const share::SCN& scn) { __scn_ = scn; }
static AppendCb* __get_class_address(ObLink *ptr);
static ObLink* __get_member_address(AppendCb *ptr);
ObLink *__next_;
VIRTUAL_TO_STRING_KV(KP(__next_), K(__start_lsn_), K(__scn_));
private:
palf::LSN __start_lsn_;
palf::SCN __scn_;
share::SCN __scn_;
};
class AppendCb : public AppendCbBase

6
src/logservice/ob_garbage_collector.cpp
View File

@ -16,7 +16,7 @@
#include "ob_log_service.h"
#include "ob_switch_leader_adapter.h"
#include "archiveservice/ob_archive_service.h"
#include "palf/scn.h"
#include "share/scn.h"
#include "rpc/obrpc/ob_rpc_net_handler.h"
#include "storage/high_availability/ob_storage_ha_struct.h"
#include "storage/ls/ob_ls.h"
@ -430,9 +430,9 @@ int ObGCHandler::replay(const void *buffer,
return ret;
}
palf::SCN ObGCHandler::get_rec_scn()
SCN ObGCHandler::get_rec_scn()
{
return palf::SCN::max_scn();
return SCN::max_scn();
}
int ObGCHandler::flush(SCN &scn)

16
src/logservice/ob_garbage_collector.h
View File

@ -43,7 +43,7 @@ namespace storage
class ObLSService;
class ObLS;
}
namespace palf
namespace share
{
class SCN;
}
@ -198,7 +198,7 @@ public:
virtual int replay(const void *buffer,
const int64_t nbytes,
const palf::LSN &lsn,
const palf::SCN &scn) override;
const share::SCN &scn) override;
// for role change
virtual void switch_to_follower_forcedly() override;
@ -207,8 +207,8 @@ public:
virtual int resume_leader() override;
// for checkpoint
virtual palf::SCN get_rec_scn() override;
virtual int flush(palf::SCN &scn) override;
virtual share::SCN get_rec_scn() override;
virtual int flush(share::SCN &scn) override;
TO_STRING_KV(K(is_inited_),
K(gc_seq_invalid_member_));
@ -249,7 +249,7 @@ private:
private:
const int64_t LS_CLOG_ALIVE_TIMEOUT_US = 100 * 1000; //100ms
const int64_t GET_GTS_TIMEOUT_US = 10L * 1000 * 1000; //10s
int get_gts_(const int64_t timeout_us, palf::SCN &gts_scn);
int get_gts_(const int64_t timeout_us, share::SCN &gts_scn);
bool is_ls_blocked_state_(const LSGCState &state);
bool is_ls_offline_state_(const LSGCState &state);
bool is_ls_wait_gc_state_(const LSGCState &state);
@ -260,9 +260,9 @@ private:
void try_check_and_set_wait_gc_(ObGarbageCollector::LSStatus &ls_status);
void submit_log_(const ObGCLSLOGType log_type);
void update_ls_gc_state_after_submit_log_(const ObGCLSLOGType log_type,
const palf::SCN &scn);
void block_ls_transfer_in_(const palf::SCN &block_scn);
void offline_ls_(const palf::SCN &offline_scn);
const share::SCN &scn);
void block_ls_transfer_in_(const share::SCN &block_scn);
void offline_ls_(const share::SCN &offline_scn);
int get_palf_role_(common::ObRole &role);
void handle_gc_ls_dropping_(const ObGarbageCollector::LSStatus &ls_status);
void handle_gc_ls_offline_(ObGarbageCollector::LSStatus &ls_status);

8
src/logservice/ob_log_base_type.h
View File

@ -17,7 +17,7 @@
namespace oceanbase
{
namespace palf
namespace share
{
class SCN;
}
@ -125,7 +125,7 @@ public:
virtual int replay(const void *buffer,
const int64_t nbytes,
const palf::LSN &lsn,
const palf::SCN &scn) = 0;
const share::SCN &scn) = 0;
};
class ObIRoleChangeSubHandler
@ -158,8 +158,8 @@ public:
class ObICheckpointSubHandler
{
public:
virtual palf::SCN get_rec_scn() = 0;
virtual int flush(palf::SCN &scn) = 0;
virtual share::SCN get_rec_scn() = 0;
virtual int flush(share::SCN &scn) = 0;
};
#define REGISTER_TO_LOGSERVICE(type, subhandler) \

13
src/logservice/ob_log_handler.cpp
View File

@ -23,7 +23,7 @@
#include "logservice/logrpc/ob_log_rpc_req.h"
#include "logservice/palf/log_define.h"
#include "logservice/palf/lsn.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
#include "logservice/palf/palf_env.h"
#include "logservice/palf/log_group_entry.h"
#include "logservice/palf/palf_options.h"
@ -31,6 +31,7 @@
namespace oceanbase
{
using namespace share;
namespace logservice
{
using namespace palf;
@ -295,7 +296,7 @@ int ObLogHandler::get_access_mode(int64_t &mode_version, palf::AccessMode &acces
int ObLogHandler::change_access_mode(const int64_t mode_version,
const palf::AccessMode &access_mode,
const palf::SCN &ref_scn)
const SCN &ref_scn)
{
int ret = OB_SUCCESS;
RLockGuard guard(lock_);
@ -367,7 +368,7 @@ int ObLogHandler::locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn)
return palf_handle_.locate_by_scn_coarsely(scn, result_lsn);
}
int ObLogHandler::locate_by_lsn_coarsely(const LSN &lsn, palf::SCN &result_scn)
int ObLogHandler::locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn)
{
RLockGuard guard(lock_);
return palf_handle_.locate_by_lsn_coarsely(lsn, result_scn);
@ -391,13 +392,13 @@ int ObLogHandler::get_max_lsn(LSN &lsn) const
return palf_handle_.get_max_lsn(lsn);
}
int ObLogHandler::get_max_scn(palf::SCN &scn) const
int ObLogHandler::get_max_scn(SCN &scn) const
{
RLockGuard guard(lock_);
return palf_handle_.get_max_scn(scn);
}
int ObLogHandler::get_end_scn(palf::SCN &scn) const
int ObLogHandler::get_end_scn(SCN &scn) const
{
RLockGuard guard(lock_);
return palf_handle_.get_end_scn(scn);
@ -1116,7 +1117,7 @@ void ObLogHandler::wait_append_sync() {
}
int ObLogHandler::enable_replay(const palf::LSN &lsn,
const palf::SCN &scn)
const SCN &scn)
{
int ret = OB_SUCCESS;
RLockGuard guard(lock_);

47
src/logservice/ob_log_handler.h
View File

@ -30,6 +30,10 @@ namespace common
{
class ObAddr;
}
namespace share
{
class SCN;
}
namespace transaction
{
class ObTsMgr;
@ -38,7 +42,6 @@ namespace palf
{
class PalfEnv;
class LSN;
class SCN;
}
namespace logservice
{
@ -53,21 +56,21 @@ public:
virtual bool is_valid() const = 0;
virtual int append(const void *buffer,
const int64_t nbytes,
const palf::SCN &ref_scn,
const share::SCN &ref_scn,
const bool need_nonblock,
AppendCb *cb,
palf::LSN &lsn,
palf::SCN &scn) = 0;
share::SCN &scn) = 0;
virtual int get_role(common::ObRole &role, int64_t &proposal_id) const = 0;
virtual int change_access_mode(const int64_t mode_version,
const palf::AccessMode &access_mode,
const palf::SCN &ref_scn) = 0;
const share::SCN &ref_scn) = 0;
virtual int get_access_mode(int64_t &mode_version, palf::AccessMode &access_mode) const = 0;
virtual int seek(const palf::LSN &lsn, palf::PalfBufferIterator &iter) = 0;
virtual int seek(const palf::LSN &lsn, palf::PalfGroupBufferIterator &iter) = 0;
virtual int seek(const palf::SCN &scn, palf::PalfGroupBufferIterator &iter) = 0;
virtual int seek(const share::SCN &scn, palf::PalfGroupBufferIterator &iter) = 0;
virtual int set_initial_member_list(const common::ObMemberList &member_list,
const int64_t paxos_replica_num) = 0;
virtual int set_initial_member_list(const common::ObMemberList &member_list,
@ -77,14 +80,14 @@ public:
virtual int set_election_priority(palf::election::ElectionPriority *priority) = 0;
virtual int reset_election_priority() = 0;
virtual int locate_by_scn_coarsely(const palf::SCN &scn, palf::LSN &result_lsn) = 0;
virtual int locate_by_lsn_coarsely(const palf::LSN &lsn, palf::SCN &result_scn) = 0;
virtual int locate_by_scn_coarsely(const share::SCN &scn, palf::LSN &result_lsn) = 0;
virtual int locate_by_lsn_coarsely(const palf::LSN &lsn, share::SCN &result_scn) = 0;
virtual int advance_base_lsn(const palf::LSN &lsn) = 0;
virtual int get_end_lsn(palf::LSN &lsn) const = 0;
virtual int get_max_lsn(palf::LSN &lsn) const = 0;
virtual int get_max_scn(palf::SCN &scn) const = 0;
virtual int get_end_scn(palf::SCN &scn) const = 0;
virtual int get_max_scn(share::SCN &scn) const = 0;
virtual int get_end_scn(share::SCN &scn) const = 0;
virtual int get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const = 0;
virtual int get_global_learner_list(common::GlobalLearnerList &learner_list) const = 0;
virtual int change_replica_num(const common::ObMemberList &member_list,
@ -123,9 +126,9 @@ public:
virtual int advance_base_info(const palf::PalfBaseInfo &palf_base_info, const bool is_rebuild) = 0;
virtual int is_valid_member(const common::ObAddr &addr, bool &is_valid) const = 0;
virtual void wait_append_sync() = 0;
virtual int enable_replay(const palf::LSN &initial_lsn, const palf::SCN &initial_scn) = 0;
virtual int enable_replay(const palf::LSN &initial_lsn, const share::SCN &initial_scn) = 0;
virtual int disable_replay() = 0;
virtual int get_max_decided_scn(palf::SCN &scn) = 0;
virtual int get_max_decided_scn(share::SCN &scn) = 0;
virtual int pend_submit_replay_log() = 0;
virtual int restore_submit_replay_log() = 0;
virtual bool is_replay_enabled() const = 0;
@ -171,11 +174,11 @@ public:
// NB: only support for primary(AccessMode::APPEND)
int append(const void *buffer,
const int64_t nbytes,
const palf::SCN &ref_scn,
const share::SCN &ref_scn,
const bool need_nonblock,
AppendCb *cb,
palf::LSN &lsn,
palf::SCN &scn) override final;
share::SCN &scn) override final;
// @brief switch log_handle role, to LEADER or FOLLOWER
// @param[in], role, LEADER or FOLLOWER
@ -214,7 +217,7 @@ public:
// get 'mode_version' and pass it to 'change_access_mode'
int change_access_mode(const int64_t mode_mode_version,
const palf::AccessMode &access_mode,
const palf::SCN &ref_scn) override final;
const share::SCN &ref_scn) override final;
// @brief alloc PalfBufferIterator, this iterator will get something append by caller.
// @param[in] const LSN &, the start lsn of iterator.
@ -229,7 +232,7 @@ public:
// @brief alloc PalfBufferIterator, this iterator will get something append by caller.
// @param[in] const int64_t, the start ts of iterator.
// @param[out] PalfBufferIterator &, scn of first log in this iterator must be higher than/equal to start_scn.
int seek(const palf::SCN &scn,
int seek(const share::SCN &scn,
palf::PalfGroupBufferIterator &iter) override final;
// @brief set the initial member list of paxos group
// @param[in] ObMemberList, the initial member list
@ -264,7 +267,7 @@ public:
// - OB_ENTRY_NOT_EXIST: there is no log in disk
// - OB_ERR_OUT_OF_LOWER_BOUND: scn is too old, log files may have been recycled
// - others: bug
int locate_by_scn_coarsely(const palf::SCN &scn, palf::LSN &result_lsn) override final;
int locate_by_scn_coarsely(const share::SCN &scn, palf::LSN &result_lsn) override final;
// @desc: query coarse ts by lsn, that means there is a log in disk,
// its lsn and scn are result_lsn and result_scn, and result_lsn <= lsn.
@ -275,7 +278,7 @@ public:
// - OB_INVALID_ARGUMENT
// - OB_ERR_OUT_OF_LOWER_BOUND: lsn is too small, log files may have been recycled
// - others: bug
int locate_by_lsn_coarsely(const palf::LSN &lsn, palf::SCN &result_scn) override final;
int locate_by_lsn_coarsely(const palf::LSN &lsn, share::SCN &result_scn) override final;
// @brief, set the recycable lsn, palf will ensure that the data before recycable lsn readable.
// @param[in] const LSN&, recycable lsn.
int advance_base_lsn(const palf::LSN &lsn) override final;
@ -285,10 +288,10 @@ public:
int get_max_lsn(palf::LSN &lsn) const override final;
// @brief, get max log ts.
// @param[out] int64_t&, max log ts.
int get_max_scn(palf::SCN &scn) const override final;
int get_max_scn(share::SCN &scn) const override final;
// @brief, get timestamp of end lsn.
// @param[out] int64_t, timestamp.
int get_end_scn(palf::SCN &scn) const override final;
int get_end_scn(share::SCN &scn) const override final;
// @brief, get paxos member list of this paxos group
// @param[out] common::ObMemberList&
// @param[out] int64_t&
@ -484,7 +487,7 @@ public:
// @param[in] const palf::LSN &initial_lsn: replay new start lsn.
// @param[in] const int64_t &initial_scn: replay new start ts.
int enable_replay(const palf::LSN &initial_lsn,
const palf::SCN &initial_scn) override final;
const share::SCN &initial_scn) override final;
// @brief, disable replay for current ls.
int disable_replay() override final;
// @brief, pending sumbit replay log
@ -496,7 +499,7 @@ public:
bool is_replay_enabled() const override final;
// @brief, get max decided log ts considering both apply and replay.
// @param[out] int64_t&, max decided log ts ns.
int get_max_decided_scn(palf::SCN &scn) override final;
int get_max_decided_scn(share::SCN &scn) override final;
// @brief: store a persistent flag which means this paxos replica
// can not reply ack when receiving logs.
// By default, paxos replica can reply ack.
@ -510,7 +513,7 @@ public:
TO_STRING_KV(K_(role), K_(proposal_id), KP(palf_env_), K(is_in_stop_state_), K(is_inited_));
private:
int submit_config_change_cmd_(const LogConfigChangeCmd &req);
int get_leader_max_scn_(palf::SCN &max_scn) const;
int get_leader_max_scn_(share::SCN &max_scn) const;
DISALLOW_COPY_AND_ASSIGN(ObLogHandler);
private:
common::ObAddr self_;

6
src/logservice/ob_log_service.h
View File

@ -51,6 +51,7 @@ namespace share
{
class ObLSID;
class ObLocationService;
class SCN;
}
namespace palf
@ -59,7 +60,6 @@ class PalfHandleGuard;
class PalfRoleChangeCb;
class PalfDiskOptions;
class PalfEnv;
class SCN;
}
namespace storage
{
@ -101,7 +101,7 @@ public:
int create_ls(const share::ObLSID &id,
const common::ObReplicaType &replica_type,
const share::ObTenantRole &tenant_role,
const palf::SCN &create_scn,
const share::SCN &create_scn,
const bool allow_log_sync,
ObLogHandler &log_handler,
ObLogRestoreHandler &restore_handler);
@ -150,7 +150,7 @@ public:
common::ObRole &role,
int64_t &proposal_id);
int update_replayable_point(const palf::SCN &replayable_point);
int update_replayable_point(const share::SCN &replayable_point);
int get_palf_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte);
int update_palf_disk_options(const palf::PalfDiskOptions &disk_options);
// why we need update 'log_disk_size_' and 'log_disk_util_threshold' separately.

1
src/logservice/palf/log_block_header.cpp
View File

@ -19,6 +19,7 @@
namespace oceanbase
{
using namespace common;
using namespace share;
namespace palf
{
LogBlockHeader::LogBlockHeader()

14
src/logservice/palf/log_block_header.h
View File

@ -17,7 +17,7 @@
#include "lib/ob_define.h"
#include "lib/utility/ob_print_utils.h"
#include "logservice/palf/lsn.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
namespace oceanbase
{
@ -31,11 +31,11 @@ public:
bool is_valid() const;
void reset();
// NB: not thread safe
void update_lsn_and_scn(const LSN &lsn, const SCN &scn);
void update_lsn_and_scn(const LSN &lsn, const share::SCN &scn);
void update_palf_id_and_curr_block_id(const int64_t palf_id, const block_id_t curr_block_id);
void mark_block_can_be_reused(const SCN &max_scn);
void mark_block_can_be_reused(const share::SCN &max_scn);
block_id_t get_curr_block_id() const;
const SCN &get_min_scn() const;
const share::SCN &get_min_scn() const;
LSN get_min_lsn() const;
// This interface is just used for PalfIterator
@ -45,7 +45,7 @@ public:
// an empty block.
//
// if this block is a reused block, return 'max_scn_', otherwise return 'min_scn_'.
const SCN get_scn_used_for_iterator() const;
const share::SCN get_scn_used_for_iterator() const;
void calc_checksum();
bool check_integrity() const;
NEED_SERIALIZE_AND_DESERIALIZE;
@ -64,8 +64,8 @@ private:
// The min lsn of curr block
LSN min_lsn_;
// The min timstamp of curr block
SCN min_scn_;
SCN max_scn_;
share::SCN min_scn_;
share::SCN max_scn_;
// Assume that, multiple PALF instances share a directory, and each block names with
// 'block_%d'(the number is monotonic, and it may be not same with 'curr_block_id_').

6
src/logservice/palf/log_config_mgr.h
View File

@ -18,11 +18,11 @@
#include "lib/random/ob_random.h" // ObRandom
#include "lib/hash/ob_array_hash_map.h" // ObArrayHashMap
#include "lib/function/ob_function.h" // ObFunction
#include "share/scn.h" //SCN
#include "log_define.h" // utils
#include "log_meta_info.h" // LogMembershipMeta
#include "log_req.h" // LogLearnerReqType
#include "log_simple_member_list.h" // LogSimpleMemberList
#include "scn.h" //SCN
#include "log_state_mgr.h" // LogStateMgr
#include "palf_callback.h" // PalfLocationCacheCb
@ -134,7 +134,7 @@ public:
type_(INVALID_LOG_CONFIG_CHANGE_TYPE) { }
LogConfigChangeArgs(const LogConfigVersion &config_version,
const SCN &ref_scn,
const share::SCN &ref_scn,
const LogConfigChangeType type)
: server_(), curr_member_list_(), curr_replica_num_(0), new_replica_num_(0),
config_version_(config_version), ref_scn_(ref_scn), type_(type) { }
@ -191,7 +191,7 @@ public:
int64_t curr_replica_num_;
int64_t new_replica_num_;
LogConfigVersion config_version_;
SCN ref_scn_;
share::SCN ref_scn_;
LogConfigChangeType type_;
};

8
src/logservice/palf/log_define.h
View File

@ -26,6 +26,11 @@ namespace common
{
class ObIAllocator;
}
namespace share
{
class SCN;
}
namespace palf
{
#define TMP_SUFFIX ".tmp"
@ -37,7 +42,6 @@ typedef uint64_t block_id_t ;
typedef uint64_t offset_t;
constexpr int64_t INVALID_PALF_ID = -1;
class LSN;
class SCN;
class LogWriteBuf;
// ==================== palf env start =============================
@ -60,7 +64,7 @@ constexpr offset_t LOG_DIO_ALIGN_SIZE = 4 * 1024;
constexpr offset_t LOG_DIO_ALIGNED_BUF_SIZE = MAX_LOG_BUFFER_SIZE + LOG_DIO_ALIGN_SIZE;
constexpr block_id_t LOG_MAX_BLOCK_ID = UINT64_MAX/PALF_BLOCK_SIZE - 1;
constexpr block_id_t LOG_INVALID_BLOCK_ID = LOG_MAX_BLOCK_ID + 1;
typedef common::ObFixedArray<SCN, ObIAllocator> SCNArray;
typedef common::ObFixedArray<share::SCN, ObIAllocator> SCNArray;
typedef common::ObFixedArray<LSN, ObIAllocator> LSNArray;
typedef common::ObFixedArray<LogWriteBuf *, ObIAllocator> LogWriteBufArray;
// ==================== block and log end ===========================

58
src/logservice/palf/log_engine.cpp
View File

@ -38,7 +38,7 @@ namespace palf
// ===================== LogEngine start =======================
LogEngine::LogEngine() :
min_block_scn_(),
min_block_max_scn_(),
min_block_id_(LOG_INVALID_BLOCK_ID),
base_lsn_for_block_gc_(PALF_INITIAL_LSN_VAL),
log_meta_lock_(),
@ -158,7 +158,7 @@ void LogEngine::destroy()
log_storage_.destroy();
base_lsn_for_block_gc_.reset();
min_block_id_ = LOG_INVALID_BLOCK_ID;
min_block_scn_.reset();
min_block_max_scn_.reset();
}
}
@ -537,8 +537,7 @@ int LogEngine::truncate_prefix_blocks(const LSN &lsn)
} else if (OB_FAIL(log_storage_.truncate_prefix_blocks(lsn))) {
PALF_LOG(WARN, "truncate_prefix_blocks failed", K(ret), K_(palf_id), K_(is_inited), K(lsn));
} else {
PALF_LOG(INFO, "truncate_prefix_blocks success", K(ret), K_(palf_id), K_(is_inited), K(lsn));
}
PALF_LOG(INFO, "truncate_prefix_blocks success", K(ret), K_(palf_id), K_(is_inited), K(lsn)); }
return ret;
}
@ -551,7 +550,7 @@ int LogEngine::delete_block(const block_id_t &block_id)
{
int ret = OB_SUCCESS;
block_id_t next_block_id = block_id + 1;
SCN next_block_min_scn;
SCN next_block_max_scn;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited));
@ -567,11 +566,12 @@ int LogEngine::delete_block(const block_id_t &block_id)
PALF_LOG(INFO, "delete success", K(block_id), K_(palf_id), K_(is_inited));
}
if (OB_SUCC(ret) && OB_FAIL(get_block_min_scn(next_block_id , next_block_min_scn))) {
PALF_LOG(WARN, "get the min scn of next block failed", K(ret), K(next_block_id));
int tmp_ret = OB_SUCCESS;
if (OB_SUCC(ret) && OB_SUCCESS != (tmp_ret = get_block_min_scn(next_block_id + 1, next_block_max_scn))) {
PALF_LOG(WARN, "get the max ts of next block failed", K(tmp_ret), K(next_block_id));
}
// If 'delete_block' or 'get_block_min_scn' failed, need reset 'min_block_scn_' to be invalid.
reset_min_block_info_guarded_by_lock_(next_block_id, next_block_min_scn);
reset_min_block_info_guarded_by_lock_(next_block_id, next_block_max_scn);
return ret;
}
@ -587,30 +587,50 @@ int LogEngine::get_block_id_range(block_id_t &min_block_id, block_id_t &max_bloc
return log_storage_.get_block_id_range(min_block_id, max_block_id);
}
int LogEngine::get_min_block_id_and_min_scn(block_id_t &block_id, SCN &scn)
int LogEngine::get_min_block_info_for_gc(block_id_t &block_id, SCN &max_scn)
{
int ret = OB_SUCCESS;
block_id_t max_block_id = LOG_INVALID_BLOCK_ID;
block_id_t min_block_id = LOG_INVALID_BLOCK_ID;
SCN min_block_scn;
SCN min_block_max_scn;
do {
ObSpinLockGuard guard(block_gc_lock_);
min_block_id = min_block_id_;
min_block_scn = min_block_scn_;
min_block_max_scn = min_block_max_scn_;
} while (0);
if (min_block_scn.is_valid()) {
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
PALF_LOG(WARN, "LogEngine is not inited", K(ret), KPC(this));
} else if (min_block_max_scn.is_valid()) {
block_id = min_block_id;
scn = min_block_scn;
max_scn = min_block_max_scn;
} else if (OB_FAIL(get_block_id_range(min_block_id, max_block_id))) {
PALF_LOG(WARN, "get_block_id_range failed", K(ret));
} else if (OB_FAIL(get_block_min_scn(min_block_id, min_block_scn))) {
PALF_LOG(WARN, "get_block_min_scn failed", K(ret));
// NB: used next block min_block_ts as the max_scn of current block
} else if (OB_FAIL(get_block_min_scn(min_block_id+1, min_block_max_scn))) {
PALF_LOG(TRACE, "get_block_min_scn failed", K(ret));
} else {
reset_min_block_info_guarded_by_lock_(min_block_id, min_block_scn);
reset_min_block_info_guarded_by_lock_(min_block_id, min_block_max_scn);
block_id = min_block_id;
scn = min_block_scn;
max_scn = min_block_max_scn;
}
return ret;
}
int LogEngine::get_min_block_info(block_id_t &min_block_id, SCN &min_block_scn) const
{
int ret = OB_SUCCESS;
block_id_t max_block_id;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
PALF_LOG(WARN, "LogEngine is not inited", K(ret), KPC(this));
} else if (OB_FAIL(get_block_id_range(min_block_id, max_block_id))) {
PALF_LOG(WARN, "get_block_id_range failed", K(ret), KPC(this));
} else if (OB_FAIL(get_block_min_scn(min_block_id, min_block_scn))) {
PALF_LOG(WARN, "get_block_min_scn failed", K(ret), KPC(this));
} else {
}
return ret;
}
@ -1255,11 +1275,11 @@ bool LogEngine::check_last_block_whether_is_integrity_(const block_id_t expected
|| expected_next_block_id < max_block_id;
}
void LogEngine::reset_min_block_info_guarded_by_lock_(const block_id_t min_block_id, const SCN &min_block_scn)
void LogEngine::reset_min_block_info_guarded_by_lock_(const block_id_t min_block_id, const SCN &min_block_max_scn)
{
ObSpinLockGuard guard(block_gc_lock_);
min_block_id_ = min_block_id;
min_block_scn_ = min_block_scn;
min_block_max_scn_ = min_block_max_scn;
}
} // end namespace palf
} // end namespace oceanbase

14
src/logservice/palf/log_engine.h
View File

@ -130,7 +130,7 @@ public:
// ==================== Submit aysnc task end ==================
// ====================== LogStorage start =====================
int append_log(const LSN &lsn, const LogWriteBuf &write_buf, const SCN &scn);
int append_log(const LSN &lsn, const LogWriteBuf &write_buf, const share::SCN &scn);
int append_log(const LSNArray &lsn, const LogWriteBufArray &write_buf, const SCNArray &scn_array);
int read_log(const LSN &lsn,
const int64_t in_read_size,
@ -143,7 +143,7 @@ public:
const LSN get_begin_lsn() const;
int get_block_id_range(block_id_t &min_block_id, block_id_t &max_block_id) const;
int get_block_min_scn(const block_id_t &block_id, SCN &scn) const;
int get_block_min_scn(const block_id_t &block_id, share::SCN &scn) const;
//
// ====================== LogStorage end =======================
@ -362,14 +362,15 @@ public:
LogMeta get_log_meta() const;
const LSN &get_base_lsn_used_for_block_gc() const;
// not thread safe
int get_min_block_id_and_min_scn(block_id_t &block_id, SCN &SCN);
int get_min_block_info_for_gc(block_id_t &block_id, share::SCN &max_scn);
int get_min_block_info(block_id_t &block_id, share::SCN &min_scn) const;
//
// ===================== NetService end ========================
LogStorage *get_log_storage() { return &log_storage_; }
LogStorage *get_log_meta_storage() { return &log_meta_storage_; }
int get_total_used_disk_space(int64_t &total_used_size_byte) const;
virtual int64_t get_palf_epoch() const { return palf_epoch_; }
TO_STRING_KV(K_(palf_id), K_(is_inited), K_(min_block_scn), K_(min_block_id), K_(base_lsn_for_block_gc),
TO_STRING_KV(K_(palf_id), K_(is_inited), K_(min_block_max_scn), K_(min_block_id), K_(base_lsn_for_block_gc),
K_(log_meta), K_(log_meta_storage), K_(log_storage), K_(palf_epoch), KP(this));
private:
int submit_flush_meta_task_(const FlushMetaCbCtx &flush_meta_cb_ctx, const LogMeta &log_meta);
@ -405,7 +406,8 @@ private:
int serialize_log_meta_(const LogMeta &log_meta, char *buf, int64_t buf_len);
void reset_min_block_info_guarded_by_lock_(const block_id_t block_id, const SCN &min_scn);
void reset_min_block_info_guarded_by_lock_(const block_id_t min_block_id,
const share::SCN &min_block_max_scn);
private:
DISALLOW_COPY_AND_ASSIGN(LogEngine);
@ -413,7 +415,7 @@ private:
private:
// used for GC
mutable ObSpinLock block_gc_lock_;
SCN min_block_scn_;
share::SCN min_block_max_scn_;
mutable block_id_t min_block_id_;
LSN base_lsn_for_block_gc_;

4
src/logservice/palf/log_entry.h
View File

@ -17,8 +17,8 @@
#include "lib/ob_define.h" // Serialization
#include "lib/utility/ob_print_utils.h" // Print*
#include "lib/utility/ob_macro_utils.h" // DISALLOW_COPY_AND_ASSIGN
#include "share/scn.h"
#include "log_define.h"
#include "scn.h"
namespace oceanbase
{
@ -39,7 +39,7 @@ public:
int64_t get_header_size() const { return header_.get_serialize_size(); }
int64_t get_payload_offset() const { return header_.get_serialize_size(); }
int64_t get_data_len() const { return header_.get_data_len(); }
const SCN get_scn() const { return header_.get_scn(); }
const share::SCN get_scn() const { return header_.get_scn(); }
const char *get_data_buf() const { return buf_; }
const LogEntryHeader &get_header() const { return header_; }

3
src/logservice/palf/log_entry_header.cpp
View File

@ -17,6 +17,7 @@
namespace oceanbase
{
using namespace share;
namespace palf
{
@ -67,7 +68,7 @@ bool LogEntryHeader::get_header_parity_check_res_() const
bool bool_ret = parity_check(reinterpret_cast<const uint16_t &>(magic_));
bool_ret ^= parity_check(reinterpret_cast<const uint16_t &>(version_));
bool_ret ^= parity_check(reinterpret_cast<const uint32_t &>(log_size_));
bool_ret ^= parity_check((scn_.get_val_for_lsn_allocator()));
bool_ret ^= parity_check((scn_.get_val_for_logservice()));
bool_ret ^= parity_check(reinterpret_cast<const uint64_t &>(data_checksum_));
int64_t tmp_flag = (flag_ & ~(0x1));
bool_ret ^= parity_check(reinterpret_cast<const uint64_t &>(tmp_flag));

8
src/logservice/palf/log_entry_header.h
View File

@ -15,7 +15,7 @@
#include "lib/ob_define.h"
#include "lib/utility/ob_print_utils.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
namespace oceanbase
{
@ -29,13 +29,13 @@ public:
public:
int generate_header(const char *log_data,
const int64_t data_len,
const SCN &scn);
const share::SCN &scn);
LogEntryHeader& operator=(const LogEntryHeader &header);
void reset();
bool is_valid() const;
bool check_integrity(const char *buf, const int64_t buf_len) const;
int32_t get_data_len() const { return log_size_; }
const SCN get_scn() const { return scn_; }
const share::SCN get_scn() const { return scn_; }
int64_t get_data_checksum() const { return data_checksum_; }
bool check_header_integrity() const;
NEED_SERIALIZE_AND_DESERIALIZE;
@ -58,7 +58,7 @@ private:
int16_t magic_;
int16_t version_;
int32_t log_size_;
SCN scn_;
share::SCN scn_;
int64_t data_checksum_;
// The lowest bit is used for parity check.
int64_t flag_;

2
src/logservice/palf/log_group_entry.cpp
View File

@ -17,9 +17,9 @@
namespace oceanbase
{
using namespace share;
namespace palf
{
using namespace common;
LogGroupEntry::LogGroupEntry() : header_(), buf_(NULL)
{
}

4
src/logservice/palf/log_group_entry.h
View File

@ -48,14 +48,14 @@ public:
// used for fetch_log, ignore the data len of padding entry
int64_t get_group_size_without_padding_data() const { return header_.get_serialize_size() +
(header_.is_padding_log() ? 0 : header_.get_data_len()); }
const SCN get_scn() const { return header_.get_max_scn(); }
const share::SCN get_scn() const { return header_.get_max_scn(); }
LSN get_committed_end_lsn() const { return header_.get_committed_end_lsn(); }
const LogGroupEntryHeader &get_header() const { return header_; }
const char *get_data_buf() const { return buf_; }
// @brief truncate log group entry the upper_limit_scn, only log entries with scn not bigger than which can reserve
// param[in] upper_limit_scn the upper bound to determain which log entries can reserve
// param[in] pre_accum_checksum, the accum_checksum of the pre log
int truncate(const SCN &upper_limit_scn, const int64_t pre_accum_checksum);
int truncate(const share::SCN &upper_limit_scn, const int64_t pre_accum_checksum);
TO_STRING_KV("LogGroupEntryHeader", header_);
NEED_SERIALIZE_AND_DESERIALIZE;

4
src/logservice/palf/log_group_entry_header.cpp
View File

@ -18,9 +18,9 @@
#include "lib/utility/utility.h" // !FALSE_IT
#include "lib/oblog/ob_log_module.h" // LOG*
#include "share/rc/ob_tenant_base.h" // mtl_malloc
#include "share/scn.h" // SCN
#include "log_define.h" // is_valid_log_id...
#include "log_writer_utils.h" // LogWriteBuf
#include "scn.h" // SCN
namespace oceanbase
{
@ -179,7 +179,7 @@ bool LogGroupEntryHeader::get_header_parity_check_res_() const
bool_ret ^= parity_check(reinterpret_cast<const uint32_t &>(group_size_));
bool_ret ^= parity_check(reinterpret_cast<const uint64_t &>(proposal_id_));
bool_ret ^= parity_check(committed_end_lsn_.val_);
bool_ret ^= parity_check(max_scn_.get_val_for_lsn_allocator());
bool_ret ^= parity_check(max_scn_.get_val_for_logservice());
bool_ret ^= parity_check(reinterpret_cast<const uint64_t &>(accumulated_checksum_));
bool_ret ^= parity_check(reinterpret_cast<const uint64_t &>(log_id_));
int64_t tmp_flag = (flag_ & ~(0x1));

10
src/logservice/palf/log_group_entry_header.h
View File

@ -16,8 +16,8 @@
#include "lib/ob_define.h" // Serialization
#include "lib/ob_name_def.h"
#include "lib/utility/ob_print_utils.h" // Print*
#include "share/scn.h" // SCN
#include "lsn.h" // LSN
#include "scn.h" // SCN
namespace oceanbase
{
@ -45,7 +45,7 @@ public:
const bool is_padding_log,
const LogWriteBuf &log_write_buf,
const int64_t data_len,
const SCN &max_scn,
const share::SCN &max_scn,
const int64_t log_id,
const LSN &committed_end_lsn,
const int64_t &log_proposal_id,
@ -55,7 +55,7 @@ public:
LogGroupEntryHeader& operator=(const LogGroupEntryHeader &header);
int32_t get_data_len() const { return group_size_; }
int64_t get_accum_checksum() const { return accumulated_checksum_; }
const SCN &get_max_scn() const { return max_scn_; }
const share::SCN &get_max_scn() const { return max_scn_; }
int64_t get_log_id() const { return log_id_; }
const int64_t &get_log_proposal_id() const { return proposal_id_; }
const LSN &get_committed_end_lsn() const { return committed_end_lsn_; }
@ -86,7 +86,7 @@ public:
int truncate(const char *buf,
const int64_t data_len,
const SCN &cut_scn,
const share::SCN &cut_scn,
const int64_t pre_accum_checksum);
NEED_SERIALIZE_AND_DESERIALIZE;
@ -133,7 +133,7 @@ private:
// The max committed log offset before this log
LSN committed_end_lsn_;
// The max scn of this log
SCN max_scn_;
share::SCN max_scn_;
// The accumulated checksum before this log, including this log,
// not including log header
int64_t accumulated_checksum_;

1
src/logservice/palf/log_io_task_cb_utils.cpp
View File

@ -16,6 +16,7 @@
namespace oceanbase
{
using namespace share;
namespace palf
{
FlushLogCbCtx::FlushLogCbCtx()

6
src/logservice/palf/log_io_task_cb_utils.h
View File

@ -15,8 +15,8 @@
#include "lib/oblog/ob_log_print_kv.h"
#include "lib/utility/ob_macro_utils.h"
#include "lib/utility/ob_print_utils.h" // TO_STRING_KV
#include "share/scn.h"
#include "lsn.h"
#include "scn.h"
#include "palf_base_info.h"
#include "log_meta_info.h"
@ -27,7 +27,7 @@ namespace palf
struct FlushLogCbCtx
{
FlushLogCbCtx();
FlushLogCbCtx(const int64_t log_id, const SCN &scn, const LSN &lsn,
FlushLogCbCtx(const int64_t log_id, const share::SCN &scn, const LSN &lsn,
const int64_t &log_proposal_id, const int64_t total_len,
const int64_t &curr_log_proposal_id, const int64_t begine_ts);
~FlushLogCbCtx();
@ -36,7 +36,7 @@ struct FlushLogCbCtx
FlushLogCbCtx &operator=(const FlushLogCbCtx &flush_log_cb_ctx);
TO_STRING_KV(K_(log_id), K_(scn), K_(lsn), K_(log_proposal_id), K_(total_len), K_(curr_proposal_id), K_(begin_ts));
int64_t log_id_;
SCN scn_;
share::SCN scn_;
LSN lsn_;
int64_t log_proposal_id_;
int64_t total_len_;

1
src/logservice/palf/log_meta.cpp
View File

@ -19,6 +19,7 @@ namespace oceanbase
namespace palf
{
using namespace common;
using namespace share;
LogMeta::LogMeta() : version_(-1),
log_prepare_meta_(),
log_config_meta_(),

1
src/logservice/palf/log_meta_info.cpp
View File

@ -20,6 +20,7 @@ namespace oceanbase
namespace palf
{
using namespace common;
using namespace share;
LogVotedFor::LogVotedFor()
{

6
src/logservice/palf/log_meta_info.h
View File

@ -17,8 +17,8 @@
#include "lib/utility/ob_print_utils.h" // Print*
#include "common/ob_learner_list.h" // common::GlobalLearnerList
#include "common/ob_member_list.h" // ObMemberList
#include "share/scn.h" // SCN
#include "lsn.h" // LSN
#include "scn.h" // SCN
#include "palf_base_info.h" // LogInfo
#include "palf_options.h" // AccessMode
#include "log_define.h" // ReplicaType
@ -177,7 +177,7 @@ public:
const int64_t proposal_id,
const int64_t mode_version,
const AccessMode &access_mode,
const SCN &ref_scn);
const share::SCN &ref_scn);
bool is_valid() const;
void reset();
void operator=(const LogModeMeta &mode_meta);
@ -192,7 +192,7 @@ public:
AccessMode access_mode_;
// scn lower bound
// after switching over, scn of all submitted log should be bigger than ref_scn_
SCN ref_scn_;
share::SCN ref_scn_;
static constexpr int64_t LOG_MODE_META_VERSION = 1;
};

1
src/logservice/palf/log_mode_mgr.cpp
View File

@ -22,6 +22,7 @@
namespace oceanbase
{
using namespace common;
using namespace share;
namespace palf
{

6
src/logservice/palf/log_mode_mgr.h
View File

@ -57,7 +57,7 @@ public:
virtual void reset_status();
virtual int get_access_mode(AccessMode &access_mode) const;
virtual int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const;
virtual int get_ref_scn(int64_t &mode_version, SCN &ref_scn) const;
virtual int get_ref_scn(int64_t &mode_version, share::SCN &ref_scn) const;
bool can_append() const;
bool can_raw_write() const;
bool is_in_pending_state() const;
@ -68,7 +68,7 @@ public:
virtual int reconfirm_mode_meta();
virtual int change_access_mode(const int64_t mode_version,
const AccessMode &access_mode,
const SCN &ref_scn);
const share::SCN &ref_scn);
virtual int handle_prepare_response(const common::ObAddr &server,
const int64_t msg_proposal_id,
const int64_t accept_log_proposal_id,
@ -95,7 +95,7 @@ private:
bool can_finish_change_mode_() const;
bool is_need_retry_() const;
int switch_state_(const AccessMode &access_mode,
const SCN &ref_scn,
const share::SCN &ref_scn,
const bool is_reconfirm);
int submit_prepare_req_(const bool need_inc_pid, const bool need_send_and_handle_prepare);
int submit_accept_req_(const int64_t proposal_id, const LogModeMeta &mode_meta);

1
src/logservice/palf/log_sliding_window.cpp
View File

@ -31,6 +31,7 @@
#include "log_mode_mgr.h"
namespace oceanbase
{
using namespace share;
namespace palf
{
LogSlidingWindow::LogSlidingWindow()

28
src/logservice/palf/log_sliding_window.h
View File

@ -17,12 +17,12 @@
#include "lib/hash/ob_linear_hash_map.h"
#include "lib/lock/ob_spin_lock.h"
#include "lib/thread/ob_thread_lease.h"
#include "share/scn.h"
#include "log_group_entry.h"
#include "log_group_buffer.h"
#include "log_checksum.h"
#include "log_req.h"
#include "lsn.h"
#include "scn.h"
#include "lsn_allocator.h"
#include "log_task.h"
#include "fixed_sliding_window.h"
@ -110,7 +110,7 @@ public:
const PalfBaseInfo &palf_base_info);
virtual int sliding_cb(const int64_t sn, const FixedSlidingWindowSlot *data);
virtual int64_t get_max_log_id() const;
virtual const SCN get_max_scn() const;
virtual const share::SCN get_max_scn() const;
virtual LSN get_max_lsn() const;
virtual int64_t get_start_id() const;
virtual int get_committed_end_lsn(LSN &committed_end_lsn) const;
@ -128,9 +128,9 @@ public:
// ================= log sync part begin
virtual int submit_log(const char *buf,
const int64_t buf_len,
const SCN &ref_scn,
const share::SCN &ref_scn,
LSN &lsn,
SCN &scn);
share::SCN &scn);
virtual int submit_group_log(const LSN &lsn,
const char *buf,
const int64_t buf_len);
@ -184,9 +184,9 @@ public:
virtual int64_t get_last_submit_log_id_() const;
virtual int get_last_submit_log_info(LSN &last_submit_lsn, int64_t &log_id, int64_t &log_proposal_id) const;
virtual int get_last_slide_end_lsn(LSN &out_end_lsn) const;
virtual const SCN get_last_slide_scn() const;
virtual const share::SCN get_last_slide_scn() const;
virtual int try_freeze_last_log();
virtual int inc_update_scn_base(const SCN &scn);
virtual int inc_update_scn_base(const share::SCN &scn);
// location cache will be removed TODO by yunlong
virtual int set_location_cache_cb(PalfLocationCacheCb *lc_cb);
virtual int reset_location_cache_cb();
@ -218,7 +218,7 @@ private:
int get_prev_log_info_(const int64_t log_id,
LSN &prev_lsn,
LSN &prev_end_lsn,
SCN &prev_log_iscn,
share::SCN &prev_log_iscn,
int64_t &prev_log_pid,
int64_t &prev_log_accum_checksum);
int inc_update_max_flushed_log_info_(const LSN &lsn,
@ -230,13 +230,13 @@ private:
void get_last_slide_end_lsn_(LSN &out_end_lsn) const;
int64_t get_last_slide_log_id_() const;
void get_last_slide_log_info_(int64_t &log_id,
SCN &scn,
share::SCN &scn,
LSN &lsn,
LSN &end_lsn,
int64_t &log_proposal_id,
int64_t &accum_checksum) const;
int try_update_last_slide_log_info_(const int64_t log_id,
const SCN &scn,
const share::SCN &scn,
const LSN &lsn,
const LSN &end_lsn,
const int64_t &proposal_id,
@ -254,7 +254,7 @@ private:
int try_freeze_last_log_(const int64_t expected_log_id, const LSN &expected_end_lsn, bool &is_need_handle);
int generate_new_group_log_(const LSN &lsn,
const int64_t log_id,
const SCN &scn,
const share::SCN &scn,
const int64_t log_body_size,
const LogType &log_type,
const char *log_data,
@ -262,7 +262,7 @@ private:
bool &is_need_handle);
int append_to_group_log_(const LSN &lsn,
const int64_t log_id,
const SCN &scn,
const share::SCN &scn,
const int64_t log_entry_size,
const char *log_data,
const int64_t data_len,
@ -288,7 +288,7 @@ private:
int wait_group_buffer_ready_(const LSN &lsn, const int64_t data_len);
int append_disk_log_to_sw_(const LSN &lsn, const LogGroupEntry &group_entry);
int try_update_max_lsn_(const LSN &lsn, const LogGroupEntryHeader &header);
int truncate_lsn_allocator_(const LSN &last_lsn, const int64_t last_log_id, const SCN &last_scn);
int truncate_lsn_allocator_(const LSN &last_lsn, const int64_t last_log_id, const share::SCN &last_scn);
bool is_all_committed_log_slided_out_(LSN &prev_lsn, int64_t &prev_log_id, LSN &committed_end_lsn) const;
void get_last_fetch_info_(LSN &last_fetch_end_lsn,
LSN &last_committed_end_lsn,
@ -307,7 +307,7 @@ private:
int get_min_scn_from_buf_(const LogGroupEntryHeader &group_entry_header,
const char *buf,
const int64_t buf_len,
SCN &min_scn);
share::SCN &min_scn);
int leader_get_committed_log_info_(const LSN &committed_end_lsn,
int64_t &log_id,
int64_t &log_proposal_id);
@ -385,7 +385,7 @@ private:
// last_slide_log_pid_: it is used for forward checking when receive log.
mutable common::ObSpinLock last_slide_info_lock_;
int64_t last_slide_log_id_; // used by clean log
SCN last_slide_scn_;
share::SCN last_slide_scn_;
LSN last_slide_lsn_;
LSN last_slide_end_lsn_;
int64_t last_slide_log_pid_;

2
src/logservice/palf/log_storage.cpp
View File

@ -15,7 +15,7 @@
#include "lib/ob_errno.h" // OB_INVALID_ARGUMENT
#include "share/rc/ob_tenant_base.h" // mtl_malloc
#include "log_reader_utils.h" // ReadBuf
#include "logservice/palf/scn.h"
#include "share/scn.h"
namespace oceanbase
{

13
src/logservice/palf/log_storage.h
View File

@ -29,9 +29,12 @@ namespace common
{
class ObILogAllocator;
}
namespace palf
namespace share
{
class SCN;
}
namespace palf
{
class ReadBuf;
class LogStorage : public ILogStorage
{
@ -62,7 +65,7 @@ public:
void destroy();
int writev(const LSNArray &lsn_array, const LogWriteBufArray &write_buf_array, const SCNArray &scn_array);
int writev(const LSN &lsn, const LogWriteBuf &write_buf, const SCN &scn);
int writev(const LSN &lsn, const LogWriteBuf &write_buf, const share::SCN &scn);
int append_meta(const char *buf, const int64_t buf_len);
@ -92,7 +95,7 @@ public:
// OB_ERR_OUT_OF_UPPER_BOUND, 'block_id' is the active block, and there is no data in this
// block. OB_ERR_UNEXPECTED, file maybe deleted by human. OB_INVALID_DATA, data has been
// corrupted
int get_block_min_scn(const block_id_t &block_id, SCN &min_scn) const;
int get_block_min_scn(const block_id_t &block_id, share::SCN &min_scn) const;
const LSN get_begin_lsn() const;
const LSN get_end_lsn() const;
@ -115,8 +118,8 @@ private:
bool check_read_out_of_lower_bound_(const block_id_t &block_id) const;
int inner_switch_block_();
int append_block_header_used_for_meta_storage_();
int append_block_header_(const LSN &block_min_lsn, const SCN &block_min_scn);
int update_block_header_(const block_id_t block_id, const LSN &block_min_lsn, const SCN &block_min_scn);
int append_block_header_(const LSN &block_min_lsn, const share::SCN &block_min_scn);
int update_block_header_(const block_id_t block_id, const LSN &block_min_lsn, const share::SCN &block_min_scn);
bool need_switch_block_() const;
// 1. 使用group_entry_iterator迭代最后一个文件, 重建block_header;
// 2. 从iterator中获取终点信息；

1
src/logservice/palf/log_task.cpp
View File

@ -21,6 +21,7 @@
namespace oceanbase
{
using namespace share;
namespace palf
{
LogTaskHeaderInfo& LogTaskHeaderInfo::operator=(const LogTaskHeaderInfo &rval)

14
src/logservice/palf/log_task.h
View File

@ -14,10 +14,10 @@
#define OCEABASE_LOGSERVICE_LOG_TASK_
#include "lib/ob_define.h" // Serialization
#include "lib/utility/ob_print_utils.h" // Print*
#include "share/scn.h"
#include "fixed_sliding_window.h"
#include "log_define.h" // block_id_t
#include "lsn.h"
#include "scn.h"
namespace oceanbase
{
@ -31,8 +31,8 @@ struct LogTaskHeaderInfo
LSN begin_lsn_;
LSN end_lsn_;
int64_t log_id_;
SCN min_scn_;
SCN max_scn_;
share::SCN min_scn_;
share::SCN max_scn_;
int64_t data_len_; // total len without log_group_entry_header
int64_t proposal_id_; // leader's proposal_id when generate this log
LSN prev_lsn_;
@ -95,7 +95,7 @@ public:
void unlock() const {
lock_.unlock();
}
void inc_update_max_scn(const SCN &scn);
void inc_update_max_scn(const share::SCN &scn);
void update_data_len(const int64_t data_len);
void set_end_lsn(const LSN &end_lsn);
int try_freeze(const LSN &end_lsn);
@ -120,7 +120,7 @@ public:
int64_t get_log_cnt() const { return ATOMIC_LOAD(&log_cnt_); }
int set_initial_header_info(const LogTaskHeaderInfo &header_info);
int update_header_info(const LSN &committed_end_lsn, const int64_t accum_checksum);
int set_group_header(const LSN &lsn, const SCN &scn, const LogGroupEntryHeader &group_entry_header);
int set_group_header(const LSN &lsn, const share::SCN &scn, const LogGroupEntryHeader &group_entry_header);
// update group log data_checksum
void set_group_log_checksum(const int64_t data_checksum);
void set_prev_lsn(const LSN &prev_lsn);
@ -132,8 +132,8 @@ public:
bool is_raw_write() const { return header_.is_raw_write_; }
int64_t get_data_len() const { return ATOMIC_LOAD(&(header_.data_len_)); }
int64_t get_log_id() const { return header_.log_id_; }
const SCN get_min_scn() const {return header_.min_scn_; }
const SCN get_max_scn() const { return header_.max_scn_; }
const share::SCN get_min_scn() const {return header_.min_scn_; }
const share::SCN get_max_scn() const { return header_.max_scn_; }
int64_t get_proposal_id() const { return header_.proposal_id_; }
LSN get_begin_lsn() const { return header_.begin_lsn_; }
LSN get_end_lsn() const { return header_.end_lsn_; }

19
src/logservice/palf/lsn_allocator.cpp
View File

@ -15,6 +15,7 @@
namespace oceanbase
{
using namespace share;
namespace palf
{
const int64_t LSNAllocator::LOG_ID_DELTA_UPPER_BOUND;
@ -54,7 +55,7 @@ int LSNAllocator::init(const int64_t log_id,
PALF_LOG(WARN, "invalid arguments", K(ret), K(log_id), K(scn), K(start_lsn));
} else {
log_id_base_ = log_id;
scn_base_ = scn.get_val_for_lsn_allocator();
scn_base_ = scn.get_val_for_logservice();
lsn_ts_meta_.v128_.lo = 0;
lsn_ts_meta_.lsn_val_ = start_lsn.val_;
lsn_ts_meta_.is_need_cut_ = 1;
@ -88,7 +89,7 @@ int LSNAllocator::truncate(const LSN &lsn, const int64_t log_id, const SCN &scn)
next.is_need_cut_ = 1;
if (CAS128(&lsn_ts_meta_, last, next)) {
log_id_base_ = log_id;
scn_base_ = scn.get_val_for_lsn_allocator();
scn_base_ = scn.get_val_for_logservice();
PALF_LOG(INFO, "truncate success", K(lsn), K(log_id), K(scn));
break;
} else {
@ -126,7 +127,7 @@ int LSNAllocator::inc_update_last_log_info(const LSN &lsn, const int64_t log_id,
break;
} else if (CAS128(&lsn_ts_meta_, last, next)) {
log_id_base_ = log_id;
scn_base_ = scn.get_val_for_lsn_allocator();
scn_base_ = scn.get_val_for_logservice();
PALF_LOG(TRACE, "inc_update_last_log_info success", K(lsn), K(scn), K(log_id));
break;
} else {
@ -148,7 +149,7 @@ int LSNAllocator::inc_update_scn_base(const SCN &ref_scn)
} else {
LSNTsMeta last;
LSNTsMeta next;
const uint64_t scn = ref_scn.get_val_for_lsn_allocator();
const uint64_t scn = ref_scn.get_val_for_logservice();
while (true) {
WLockGuard guard(lock_);
LOAD128(last, &lsn_ts_meta_);
@ -194,8 +195,8 @@ SCN LSNAllocator::get_max_scn() const
LOAD128(last, &lsn_ts_meta_);
max_scn = scn_base_ + last.scn_delta_;
int ret = OB_SUCCESS;
if (OB_FAIL(result.convert_for_lsn_allocator(max_scn))) {
PALF_LOG(ERROR, "failed to convert_for_lsn_allocator", K(max_scn),
if (OB_FAIL(result.convert_for_logservice(max_scn))) {
PALF_LOG(ERROR, "failed to convert_for_logservice", K(max_scn),
K(scn_base_), K(last.scn_delta_));
}
}
@ -313,7 +314,7 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn,
LOAD128(last, &lsn_ts_meta_);
const int64_t last_log_id = log_id_base_ + last.log_id_delta_;
const uint64_t last_scn = scn_base_ + last.scn_delta_;
const uint64_t new_scn = std::max(base_scn.get_val_for_lsn_allocator(), last_scn);
const uint64_t new_scn = std::max(base_scn.get_val_for_logservice(), last_scn);
log_id_base_ = last_log_id;
scn_base_ = new_scn;
@ -339,7 +340,7 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn,
LOAD128(last, &lsn_ts_meta_);
const int64_t last_log_id = log_id_base_ + last.log_id_delta_;
const uint64_t last_scn = scn_base_ + last.scn_delta_;
const uint64_t tmp_next_scn = std::max(base_scn.get_val_for_lsn_allocator(), last_scn + 1);
const uint64_t tmp_next_scn = std::max(base_scn.get_val_for_logservice(), last_scn + 1);
if ((tmp_next_scn + 1) - scn_base_ >= LOG_TS_DELTA_UPPER_BOUND) {
// 对于可能生成的padding log, 也会占用一个scn
@ -456,7 +457,7 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn,
}
uint64_t scn_val = scn_base_ + output_next_scn_delta;
if (OB_FAIL(scn.convert_for_lsn_allocator(scn_val))) {
if (OB_FAIL(scn.convert_for_logservice(scn_val))) {
PALF_LOG(ERROR, "failed to convert scn", K(ret), K(base_scn), K(scn));
}

16
src/logservice/palf/lsn_allocator.h
View File

@ -16,8 +16,8 @@
#include "lib/atomic/atomic128.h"
#include "lib/lock/ob_tc_rwlock.h"
#include "lib/utility/ob_macro_utils.h"
#include "share/scn.h"
#include "lsn.h"
#include "scn.h"
namespace oceanbase
{
@ -29,7 +29,7 @@ public:
LSNAllocator();
~LSNAllocator();
public:
int init(const int64_t log_id, const SCN &scn, const LSN &start_lsn);
int init(const int64_t log_id, const share::SCN &scn, const LSN &start_lsn);
void reset();
int get_log_block_size(const uint64_t block_id, int64_t &block_size) const
{
@ -40,7 +40,7 @@ public:
return OB_SUCCESS;
}
int64_t get_max_log_id() const;
SCN get_max_scn() const;
share::SCN get_max_scn() const;
int get_curr_end_lsn(LSN &curr_end_lsn) const;
int try_freeze_by_time(LSN &last_lsn, int64_t &last_log_id);
int try_freeze(LSN &last_lsn, int64_t &last_log_id);
@ -55,20 +55,20 @@ public:
// @param [out] need_gen_padding_entry: 是否需要在本条日志之前生成padding_entry
// @param [out] padding_len: padding部分的总长度
//
int alloc_lsn_scn(const SCN &base_scn,
int alloc_lsn_scn(const share::SCN &base_scn,
const int64_t size,
LSN &lsn,
int64_t &log_id,
SCN &scn,
share::SCN &scn,
bool &is_new_log,
bool &need_gen_padding_entry,
int64_t &padding_len);
// 更新last_lsn和log_timestamp
// receive_log/append_disk_log 时调用
int inc_update_last_log_info(const LSN &lsn, const int64_t log_id, const SCN &scn);
int inc_update_last_log_info(const LSN &lsn, const int64_t log_id, const share::SCN &scn);
// inc update scn base, called by change access mode and to leader active
int inc_update_scn_base(const SCN &scn);
int truncate(const LSN &lsn, const int64_t log_id, const SCN &scn);
int inc_update_scn_base(const share::SCN &scn);
int truncate(const LSN &lsn, const int64_t log_id, const share::SCN &scn);
// 获取last_lsn和log_timestamp
TO_STRING_KV("max_log_id", get_max_log_id(), "max_lsn", lsn_ts_meta_.lsn_val_,
"max_scn", get_max_scn());

4
src/logservice/palf/palf_base_info.h
View File

@ -14,8 +14,8 @@
#define OCEANBASE_LOGSERVICE_PALF_BASE_INFO_
#include "lib/utility/ob_unify_serialize.h" // OB_UNIS_VERSION
#include "share/scn.h"
#include "lsn.h"
#include "scn.h"
namespace oceanbase
{
@ -40,7 +40,7 @@ public:
int64_t version_;
int64_t log_id_;
LSN lsn_;
SCN scn_;
share::SCN scn_;
int64_t log_proposal_id_;
int64_t accum_checksum_;
};

38
src/logservice/palf/palf_env_impl.cpp
View File

@ -31,6 +31,7 @@
namespace oceanbase
{
using namespace common;
using namespace share;
namespace palf
{
PalfHandleImpl *PalfHandleImplFactory::alloc()
@ -591,7 +592,7 @@ int PalfEnvImpl::try_freeze_log_for_all()
PalfEnvImpl::LogGetRecycableFileCandidate::LogGetRecycableFileCandidate()
: id_(-1),
min_block_id_(LOG_INVALID_BLOCK_ID),
min_block_id_scn_(),
min_block_max_scn_(),
min_using_block_id_(LOG_INVALID_BLOCK_ID),
oldest_palf_id_(INVALID_PALF_ID),
oldest_block_scn_(),
@ -602,7 +603,7 @@ PalfEnvImpl::LogGetRecycableFileCandidate::~LogGetRecycableFileCandidate()
{
ret_code_ = OB_SUCCESS;
min_using_block_id_ = LOG_INVALID_BLOCK_ID;
min_block_id_scn_.reset();
min_block_max_scn_.reset();
min_block_id_ = LOG_INVALID_BLOCK_ID;
oldest_palf_id_ = INVALID_PALF_ID;
oldest_block_scn_.reset();
@ -620,14 +621,18 @@ bool PalfEnvImpl::LogGetRecycableFileCandidate::operator()(const LSKey &palf_id,
const LSN base_lsn = palf_handle_impl->get_base_lsn_used_for_block_gc();
const block_id_t min_using_block_id = lsn_2_block(base_lsn, PALF_BLOCK_SIZE);
block_id_t min_block_id = LOG_INVALID_BLOCK_ID;
SCN min_block_id_scn;
SCN min_block_max_scn;
// OB_ENTRY_EXIST means there is not any block;
// OB_NO_SUCH_FILE_OR_DIRECTORY means there is concurrently with rebuild.
// OB_ERR_OUT_OF_UPPER_BOUND means there is one block
auto need_skip_by_ret = [](const int ret ){
return OB_ENTRY_EXIST == ret || OB_NO_SUCH_FILE_OR_DIRECTORY == ret
|| OB_ERR_OUT_OF_UPPER_BOUND == ret;
};
if (false == base_lsn.is_valid()) {
PALF_LOG(WARN, "base_lsn is invalid", K(base_lsn), KPC(palf_handle_impl));
// OB_ENTRY_EXIST means there is not any block;
// OB_NO_SUCH_FILE_OR_DIRECTORY means there is concurrently with rebuild.
} else if (OB_FAIL(palf_handle_impl->get_min_block_id_min_scn(min_block_id, min_block_id_scn))
&& OB_ENTRY_NOT_EXIST != ret
&& OB_NO_SUCH_FILE_OR_DIRECTORY != ret) {
} else if (OB_FAIL(palf_handle_impl->get_min_block_info_for_gc(min_block_id, min_block_max_scn))
&& !need_skip_by_ret(ret)) {
ret_code_ = ret;
bool_ret = false;
PALF_LOG(WARN, "LogGetRecycableFileCandidate get_min_block_id_min_scn failed", K(ret), K(palf_id));
@ -636,26 +641,25 @@ bool PalfEnvImpl::LogGetRecycableFileCandidate::operator()(const LSKey &palf_id,
// 2. current palf_handle_impl must have older blocks(at least two blocks).
// Always keep there are at least two blocks in range [begin_lsn, base_lsn], because for restart, we will read
// first uncommitted log before base_lsn.
} else if (OB_ENTRY_NOT_EXIST == ret
|| OB_NO_SUCH_FILE_OR_DIRECTORY == ret
} else if (need_skip_by_ret(ret)
|| min_using_block_id < min_block_id
|| min_using_block_id - min_block_id < 2) {
PALF_LOG(TRACE, "can not recycle blocks, need keep at least two blocks or has been concurrently"
" with rebuild, skip it",
K(ret), KPC(palf_handle_impl), K(min_block_id), K(min_using_block_id));
} else if (min_block_id_scn_.is_valid() && min_block_id_scn_ < min_block_id_scn) {
PALF_LOG(TRACE, "current palf_handle_impl is not older than previous, skip it", K(min_block_id_scn),
K(min_block_id_scn_), KPC(palf_handle_impl), K(min_block_id));
} else if (min_block_max_scn_.is_valid() && min_block_max_scn_ < min_block_max_scn) {
PALF_LOG(TRACE, "current palf_handle_impl is not older than previous, skip it", K(min_block_max_scn),
K(min_block_max_scn_), KPC(palf_handle_impl), K(min_block_id));
} else {
id_ = palf_id.id_;
min_block_id_ = min_block_id;
min_block_id_scn_ = min_block_id_scn;
min_block_max_scn_ = min_block_max_scn;
min_using_block_id_ = min_using_block_id;
PALF_LOG(TRACE, "can be recycable palf_handle_impl", K(id_), K(min_block_id_), K(min_using_block_id_),
K(min_block_id_scn_), K(base_lsn));
K(min_block_max_scn_), K(base_lsn));
}
if (min_block_id_scn.is_valid() && (!oldest_block_scn_.is_valid() || oldest_block_scn_ > min_block_id_scn)) {
oldest_block_scn_ = min_block_id_scn;
if (min_block_max_scn.is_valid() && (!oldest_block_scn_.is_valid() || oldest_block_scn_ > min_block_max_scn)) {
oldest_block_scn_ = min_block_max_scn;
oldest_palf_id_ = palf_id.id_;
}
}

10
src/logservice/palf/palf_env_impl.h
View File

@ -22,6 +22,7 @@
#include "lib/utility/ob_macro_utils.h"
#include "lib/utility/utility.h"
#include "share/ob_occam_timer.h"
#include "share/scn.h"
#include "fetch_log_engine.h"
#include "log_loop_thread.h"
#include "log_define.h"
@ -32,7 +33,6 @@
#include "palf_handle_impl.h"
#include "log_io_worker.h"
#include "block_gc_timer_task.h"
#include "scn.h"
namespace oceanbase
{
namespace common
@ -253,12 +253,12 @@ private:
bool operator() (const LSKey &palf_id, PalfHandleImpl *palf_handle_impl);
int64_t id_;
block_id_t min_block_id_;
SCN min_block_id_scn_;
share::SCN min_block_max_scn_;
block_id_t min_using_block_id_;
int64_t oldest_palf_id_;
SCN oldest_block_scn_;
share::SCN oldest_block_scn_;
int ret_code_;
TO_STRING_KV(K_(id), K_(min_block_id_scn), K_(min_block_id), K_(min_using_block_id), K_(oldest_palf_id), K_(oldest_block_scn), K_(ret_code));
TO_STRING_KV(K_(id), K_(min_block_max_scn), K_(min_block_id), K_(min_using_block_id), K_(oldest_palf_id), K_(oldest_block_scn), K_(ret_code));
};
struct GetTotalUsedDiskSpace
{
@ -283,7 +283,7 @@ private:
int get_disk_usage_(int64_t &used_size_byte,
int64_t &palf_id,
int64_t &maximum_used_size);
int recycle_blocks_(bool &has_recycled, int64_t &oldest_palf_id, SCN &oldest_scn);
int recycle_blocks_(bool &has_recycled, int64_t &oldest_palf_id, share::SCN &oldest_scn);
int wait_until_reference_count_to_zero_(const int64_t palf_id);
// check the diskspace whether is enough to hold a new palf instance.
bool check_can_create_palf_handle_impl_() const;

1
src/logservice/palf/palf_handle.cpp
View File

@ -20,6 +20,7 @@
namespace oceanbase
{
using namespace share;
namespace palf
{
#define CHECK_VALID if (NULL == palf_handle_impl_) { return OB_NOT_INIT; }

23
src/logservice/palf/palf_handle.h
View File

@ -21,13 +21,16 @@
#include "palf_iterator.h"
namespace oceanbase
{
namespace share
{
class SCN;
}
namespace palf
{
class PalfAppendOptions;
class PalfFSCb;
class PalfRoleChangeCb;
class PalfLocationCacheCb;
class SCN;
class PalfHandle
{
public:
@ -79,9 +82,9 @@ public:
int append(const PalfAppendOptions &opts,
const void *buffer,
const int64_t nbytes,
const SCN &ref_scn,
const share::SCN &ref_scn,
LSN &lsn,
SCN &scn);
share::SCN &scn);
int raw_write(const PalfAppendOptions &opts,
const LSN &lsn,
@ -117,7 +120,7 @@ public:
// - OB_ENTRY_NOT_EXIST: there is no log's scn is higher than scn
// - OB_ERR_OUT_OF_LOWER_BOUND: scn is too old, log files may have been recycled
// - others: bug
int seek(const SCN &scn, PalfGroupBufferIterator &iter);
int seek(const share::SCN &scn, PalfGroupBufferIterator &iter);
// @desc: query coarse lsn by scn, that means there is a LogGroupEntry in disk,
// its lsn and scn are result_lsn and result_scn, and result_scn <= scn.
@ -131,7 +134,7 @@ public:
// - OB_ENTRY_NOT_EXIST: there is no log in disk
// - OB_ERR_OUT_OF_LOWER_BOUND: scn is too small, log files may have been recycled
// - others: bug
virtual int locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn);
virtual int locate_by_scn_coarsely(const share::SCN &scn, LSN &result_lsn);
// @desc: query coarse scn by lsn, that means there is a log in disk,
// its lsn and scn are result_lsn and result_scn, and result_lsn <= lsn.
@ -142,7 +145,7 @@ public:
// - OB_INVALID_ARGUMENT
// - OB_ERR_OUT_OF_LOWER_BOUND: lsn is too small, log files may have been recycled
// - others: bug
virtual int locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn);
virtual int locate_by_lsn_coarsely(const LSN &lsn, share::SCN &result_scn);
// 开启日志同步
virtual int enable_sync();
@ -156,7 +159,7 @@ public:
// 返回文件中可读的最早日志的位置信息
int get_begin_lsn(LSN &lsn) const;
int get_begin_scn(SCN &scn) const;
int get_begin_scn(share::SCN &scn) const;
// PalfBaseInfo include the 'base_lsn' and the 'prev_log_info' of sliding window.
// @param[in] const LSN&, base_lsn of ls.
@ -167,9 +170,9 @@ public:
// 返回最后一条已确认日志的下一位置
// 在没有新的写入的场景下，返回的end_lsn不可读
virtual int get_end_lsn(LSN &lsn) const;
int get_end_scn(SCN &scn) const;
int get_end_scn(share::SCN &scn) const;
int get_max_lsn(LSN &lsn) const;
int get_max_scn(SCN &scn) const;
int get_max_scn(share::SCN &scn) const;
int get_last_rebuild_lsn(LSN &last_rebuild_lsn) const;
//================= 分布式相关接口 =========================
@ -369,7 +372,7 @@ public:
int change_access_mode(const int64_t proposal_id,
const int64_t mode_version,
const AccessMode &access_mode,
const SCN &ref_scn);
const share::SCN &ref_scn);
// @brief: query the access_mode of palf and it's corresponding mode_version
// @param[out] palf::AccessMode &access_mode: current access_mode
// @param[out] int64_t &mode_version: mode_version corresponding to AccessMode

21
src/logservice/palf/palf_handle_impl.cpp
View File

@ -28,6 +28,7 @@
namespace oceanbase
{
using namespace common;
using namespace share;
using namespace palf::election;
namespace palf
{
@ -301,14 +302,13 @@ int PalfHandleImpl::get_begin_scn(SCN &scn)
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
PALF_LOG(WARN, "PalfHandleImpl not init", K(ret), KPC(this));
} else if (OB_FAIL(log_engine_.get_min_block_id_and_min_scn(unused_block_id, scn))) {
PALF_LOG(WARN, "LogEngine get_min_block_id_and_min_scn failed", K(ret), KPC(this));
} else if (OB_FAIL(log_engine_.get_min_block_info(unused_block_id, scn))) {
PALF_LOG(WARN, "LogEngine get_min_block_info failed", K(ret), KPC(this));
}
return ret;
}
int PalfHandleImpl::get_base_info(const LSN &base_lsn,
PalfBaseInfo &base_info)
int PalfHandleImpl::get_base_info(const LSN &base_lsn, PalfBaseInfo &base_info)
{
int ret = OB_SUCCESS;
LSN curr_end_lsn = get_end_lsn();
@ -1595,13 +1595,16 @@ int PalfHandleImpl::locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn)
return ret;
}
int PalfHandleImpl::get_min_block_id_min_scn(block_id_t &block_id, SCN &scn)
int PalfHandleImpl::get_min_block_info_for_gc(block_id_t &min_block_id, SCN &max_scn)
{
int ret = OB_SUCCESS;
if (OB_FAIL(log_engine_.get_min_block_id_and_min_scn(block_id, scn))) {
PALF_LOG(WARN, "get_min_block_id_and_min_scn failed", K(ret), KPC(this));
// if (false == end_lsn.is_valid()) {
// ret = OB_ENTRY_NOT_EXIST;
// }
if (OB_FAIL(log_engine_.get_min_block_info_for_gc(min_block_id, max_scn))) {
PALF_LOG(WARN, "get_min_block_info_for_gc failed", K(ret), KPC(this));
} else {
PALF_LOG(TRACE, "get_min_block_id_and_min_scn success", K(ret), KPC(this), K(block_id), K(scn));
PALF_LOG(TRACE, "get_min_block_info_for_gc success", K(ret), KPC(this), K(min_block_id), K(max_scn));
}
return ret;
}
@ -3482,7 +3485,7 @@ int PalfHandleImpl::stat(PalfStat &palf_stat)
palf_stat.allow_vote_ = state_mgr_.is_allow_vote();
palf_stat.replica_type_ = state_mgr_.get_replica_type();
palf_stat.base_lsn_ = log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_;
(void)log_engine_.get_min_block_id_and_min_scn(min_block_id, min_block_min_scn);
(void)log_engine_.get_min_block_info(min_block_id, min_block_min_scn);
palf_stat.begin_lsn_ = LSN(min_block_id * PALF_BLOCK_SIZE);
palf_stat.begin_scn_ = min_block_min_scn;
palf_stat.end_lsn_ = get_end_lsn();

58
src/logservice/palf/palf_handle_impl.h
View File

@ -23,11 +23,11 @@
#include "election/interface/election_msg_handler.h"
#include "election/message/election_message.h"
#include "election/algorithm/election_impl.h"
#include "share/scn.h"
#include "palf_callback_wrapper.h"
#include "log_engine.h" // LogEngine
#include "log_meta.h"
#include "lsn.h"
#include "scn.h"
#include "log_config_mgr.h"
#include "log_mode_mgr.h"
#include "log_reconfirm.h"
@ -74,12 +74,12 @@ struct PalfStat {
bool allow_vote_;
LogReplicaType replica_type_;
LSN begin_lsn_;
SCN begin_scn_;
share::SCN begin_scn_;
LSN base_lsn_;
LSN end_lsn_;
SCN end_scn_;
share::SCN end_scn_;
LSN max_lsn_;
SCN max_scn_;
share::SCN max_scn_;
TO_STRING_KV(K_(self), K_(palf_id), K_(role), K_(log_proposal_id), K_(config_version),
K_(access_mode), K_(paxos_member_list), K_(paxos_replica_num), K_(allow_vote),
K_(replica_type), K_(base_lsn), K_(end_lsn), K_(end_scn), K_(max_lsn));
@ -182,9 +182,9 @@ public:
virtual int submit_log(const PalfAppendOptions &opts,
const char *buf,
const int64_t buf_len,
const SCN &ref_scn,
const share::SCN &ref_scn,
LSN &lsn,
SCN &scn) = 0;
share::SCN &scn) = 0;
// 提交group_log到palf
// 使用场景：备库leader处理从主库收到的日志
// @param [in] opts, 提交日志的一些可选项参数，具体参见PalfAppendOptions的定义
@ -403,7 +403,7 @@ public:
// - OB_ENTRY_NOT_EXIST: there is no log in disk
// - OB_ERR_OUT_OF_LOWER_BOUND: scn is too old, log files may have been recycled
// - others: bug
virtual int locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn) = 0;
virtual int locate_by_scn_coarsely(const share::SCN &scn, LSN &result_lsn) = 0;
// @desc: query coarse scn by lsn, that means there is a LogGroupEntry in disk,
// its lsn and scn are result_lsn and result_scn, and result_lsn <= lsn.
@ -419,12 +419,12 @@ public:
// - OB_INVALID_ARGUMENT
// - OB_ERR_OUT_OF_LOWER_BOUND: lsn is too small, log files may have been recycled
// - others: bug
virtual int locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) = 0;
virtual int locate_by_lsn_coarsely(const LSN &lsn, share::SCN &result_scn) = 0;
virtual int get_begin_lsn(LSN &lsn) const = 0;
virtual int get_begin_scn(SCN &scn) = 0;
virtual int get_begin_scn(share::SCN &scn) = 0;
virtual int get_base_info(const LSN &base_lsn, PalfBaseInfo &base_info) = 0;
virtual int get_min_block_id_min_scn(block_id_t &block_id, SCN &scn) = 0;
virtual int get_min_block_info_for_gc(block_id_t &min_block_id, share::SCN &max_scn) = 0;
//begin lsn base lsn end lsn
// │ │ │
// │ │ │
@ -440,8 +440,8 @@ public:
// return the block length which the previous data was committed
virtual const LSN get_end_lsn() const = 0;
virtual LSN get_max_lsn() const = 0;
virtual const SCN get_max_scn() const = 0;
virtual const SCN get_end_scn() const = 0;
virtual const share::SCN get_max_scn() const = 0;
virtual const share::SCN get_end_scn() const = 0;
virtual int get_last_rebuild_lsn(LSN &last_rebuild_lsn) const = 0;
virtual int64_t get_total_used_disk_space() const = 0;
virtual const LSN &get_base_lsn_used_for_block_gc() const = 0;
@ -458,7 +458,7 @@ public:
virtual int advance_reuse_lsn(const LSN &flush_log_end_lsn) = 0;
virtual int inner_append_log(const LSN &lsn,
const LogWriteBuf &write_buf,
const SCN &scn) = 0;
const share::SCN &scn) = 0;
virtual int inner_append_log(const LSNArray &lsn_array,
const LogWriteBufArray &write_buf_array,
const SCNArray &scn_array) = 0;
@ -536,7 +536,7 @@ public:
virtual int change_access_mode(const int64_t proposal_id,
const int64_t mode_version,
const AccessMode &access_mode,
const SCN &ref_scn) = 0;
const share::SCN &ref_scn) = 0;
virtual int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const = 0;
virtual int get_access_mode(AccessMode &access_mode) const = 0;
virtual int handle_committed_info(const common::ObAddr &server,
@ -558,7 +558,7 @@ public:
PalfBufferIterator &iterator) = 0;
virtual int alloc_palf_group_buffer_iterator(const LSN &offset,
PalfGroupBufferIterator &iterator) = 0;
virtual int alloc_palf_group_buffer_iterator(const SCN &scn,
virtual int alloc_palf_group_buffer_iterator(const share::SCN &scn,
PalfGroupBufferIterator &iterator) = 0;
// ===================== Iterator end =======================
@ -626,9 +626,9 @@ public:
int submit_log(const PalfAppendOptions &opts,
const char *buf,
const int64_t buf_len,
const SCN &ref_scn,
const share::SCN &ref_scn,
LSN &lsn,
SCN &scn) override final;
share::SCN &scn) override final;
int submit_group_log(const PalfAppendOptions &opts,
const LSN &lsn,
@ -677,8 +677,8 @@ public:
int disable_sync();
bool is_sync_enabled() const;
int advance_base_info(const PalfBaseInfo &palf_base_info, const bool is_rebuild) override final;
int locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn) override final;
int locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) override final;
int locate_by_scn_coarsely(const share::SCN &scn, LSN &result_lsn) override final;
int locate_by_lsn_coarsely(const LSN &lsn, share::SCN &result_scn) override final;
void set_deleted();
int disable_vote() override final;
int enable_vote() override final;
@ -692,13 +692,13 @@ public:
int change_access_mode(const int64_t proposal_id,
const int64_t mode_version,
const AccessMode &access_mode,
const SCN &ref_scn) override final;
const share::SCN &ref_scn) override final;
int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const override final;
int get_access_mode(AccessMode &access_mode) const override final;
// =========================== Iterator start ============================
int alloc_palf_buffer_iterator(const LSN &offset, PalfBufferIterator &iterator) override final;
int alloc_palf_group_buffer_iterator(const LSN &offset, PalfGroupBufferIterator &iterator) override final;
int alloc_palf_group_buffer_iterator(const SCN &scn, PalfGroupBufferIterator &iterator) override final;
int alloc_palf_group_buffer_iterator(const share::SCN &scn, PalfGroupBufferIterator &iterator) override final;
// =========================== Iterator end ============================
// ==================== Callback start ======================
@ -715,9 +715,9 @@ public:
// ==================== Callback end ========================
public:
int get_begin_lsn(LSN &lsn) const;
int get_begin_scn(SCN &scn);
int get_begin_scn(share::SCN &scn);
int get_base_info(const LSN &base_lsn, PalfBaseInfo &base_info);
int get_min_block_id_min_scn(block_id_t &block_id, SCN &scn) override final;
int get_min_block_info_for_gc(block_id_t &min_block_id, share::SCN &max_scn) override final;
// return the block length which the previous data was committed
const LSN get_end_lsn() const override final
{
@ -731,12 +731,12 @@ public:
return sw_.get_max_lsn();
}
const SCN get_max_scn() const override final
const share::SCN get_max_scn() const override final
{
return sw_.get_max_scn();
}
const SCN get_end_scn() const override final
const share::SCN get_end_scn() const override final
{
// 基于实现复杂度考虑，直接用last_slide_scn作为end_scn
// 否则需要在match_lsn_map中额外维护scn
@ -757,7 +757,7 @@ public:
int advance_reuse_lsn(const LSN &flush_log_end_lsn);
int inner_append_log(const LSN &lsn,
const LogWriteBuf &write_buf,
const SCN &scn) override final;
const share::SCN &scn) override final;
int inner_append_log(const LSNArray &lsn_array,
const LogWriteBufArray &write_buf_array,
const SCNArray &scn_array);
@ -890,11 +890,11 @@ private:
const int64_t accepted_mode_pid);
int try_update_proposal_id_(const common::ObAddr &server,
const int64_t &proposal_id);
int get_binary_search_range_(const SCN &scn,
int get_binary_search_range_(const share::SCN &scn,
block_id_t &min_block_id,
block_id_t &max_block_id,
block_id_t &result_block_id);
void inc_update_last_locate_block_scn_(const block_id_t &block_id, const SCN &scn);
void inc_update_last_locate_block_scn_(const block_id_t &block_id, const share::SCN &scn);
int can_change_config_(const LogConfigChangeArgs &args, int64_t &proposal_id);
int check_args_and_generate_config_(const LogConfigChangeArgs &args,
bool &is_already_finished,
@ -993,7 +993,7 @@ private:
PalfLocationCacheCb *lc_cb_;
// ======optimization for locate_by_scn_coarsely=========
mutable SpinLock last_locate_lock_;
SCN last_locate_scn_;
share::SCN last_locate_scn_;
block_id_t last_locate_block_;
// ======optimization for locate_by_scn_coarsely=========
int64_t cannot_recv_log_warn_time_;

9
src/logservice/palf/palf_iterator.h
View File

@ -16,9 +16,12 @@
#include "log_iterator_storage.h" // LogIteratorStorage
namespace oceanbase
{
namespace palf
namespace share
{
class SCN;
}
namespace palf
{
template <class PalfIteratorStorage, class LogEntryType>
class PalfIterator
{
@ -88,7 +91,7 @@ public:
}
return ret;
}
int get_entry(const char *&buffer, int64_t &nbytes, SCN &scn, LSN &lsn, bool &is_raw_write)
int get_entry(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn, bool &is_raw_write)
{
int ret = OB_SUCCESS;
LogEntryType entry;
@ -105,7 +108,7 @@ public:
}
return ret;
}
int get_entry(const char *&buffer, int64_t &nbytes, SCN &scn, LSN &lsn)
int get_entry(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn)
{
int ret = OB_SUCCESS;
LogEntryType entry;

2
src/logservice/palf_handle_guard.h
View File

@ -91,7 +91,7 @@ public:
return palf_handle_.seek(start_lsn, iter);
}
int seek(const palf::SCN &scn,
int seek(const share::SCN &scn,
PalfGroupBufferIterator &iter)
{
return palf_handle_.seek(scn, iter);

17
src/logservice/replayservice/ob_log_replay_service.cpp
View File

@ -15,7 +15,7 @@
#include "logservice/ob_log_base_header.h"
#include "logservice/ob_ls_adapter.h"
#include "logservice/palf/palf_env.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
#include "share/ob_thread_mgr.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/tx_storage/ob_tenant_freezer.h"
@ -572,7 +572,7 @@ int ObLogReplayService::is_replay_done(const share::ObLSID &id,
return ret;
}
int ObLogReplayService::get_min_unreplayed_scn(const share::ObLSID &id, palf::SCN &scn)
int ObLogReplayService::get_min_unreplayed_scn(const share::ObLSID &id, SCN &scn)
{
int ret = OB_SUCCESS;
ObReplayStatus *replay_status = NULL;
@ -728,7 +728,8 @@ bool ObLogReplayService::is_tenant_out_of_memory_() const
total_memstore_used,
memstore_freeze_trigger,
memstore_limit,
freeze_cnt)))) {
freeze_cnt,
false)))) {
CLOG_LOG(WARN, "get_tenant_memstore_cond failed", K(ret));
} else {
// Estimate the size of memstore based on 16 times expansion
@ -907,13 +908,9 @@ int ObLogReplayService::check_can_submit_log_replay_task_(ObLogReplayTask *repla
} else {
is_wait_barrier = true;
}
} else if (replay_status->need_check_memstore(replay_task->lsn_)) {
if (OB_UNLIKELY(is_tenant_out_of_memory_())) {
ret = OB_EAGAIN;
is_tenant_out_of_mem = true;
} else {
replay_status->set_last_check_memstore_lsn(replay_task->lsn_);
}
} else if (OB_UNLIKELY(is_tenant_out_of_memory_())) {
ret = OB_EAGAIN;
is_tenant_out_of_mem = true;
}
if (OB_EAGAIN == ret && REACH_TIME_INTERVAL(5 * 1000 * 1000)) {
CLOG_LOG(INFO, "submit replay task need retry", K(ret), KPC(replay_status), KPC(replay_task), K(current_replayable_point),

19
src/logservice/replayservice/ob_log_replay_service.h
View File

@ -17,14 +17,17 @@
#include "logservice/ob_log_base_header.h"
#include "lib/thread/thread_mgr_interface.h"
#include "lib/hash/ob_linear_hash_map.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
namespace oceanbase
{
namespace share
{
class SCN;
}
namespace palf
{
class PalfEnv;
class SCN;
}
namespace logservice
{
@ -122,7 +125,7 @@ public:
int remove_ls(const share::ObLSID &id);
int enable(const share::ObLSID &id,
const palf::LSN &base_lsn,
const palf::SCN &base_scn);
const share::SCN &base_scn);
int disable(const share::ObLSID &id);
int is_enabled(const share::ObLSID &id, bool &is_enabled);
int set_submit_log_pending(const share::ObLSID &id);
@ -134,9 +137,9 @@ public:
const palf::LSN &end_lsn,
bool &is_done);
int get_min_unreplayed_scn(const share::ObLSID &id,
palf::SCN &scn);
share::SCN &scn);
int submit_task(ObReplayServiceTask *task);
int update_replayable_point(const palf::SCN &replayable_scn);
int update_replayable_point(const share::SCN &replayable_scn);
int stat_for_each(const common::ObFunction<int (const ObReplayStatus &)> &func);
int stat_all_ls_replay_process(int64_t &replayed_log_size, int64_t &unreplayed_log_size);
void inc_pending_task_size(const int64_t log_size);
@ -159,7 +162,7 @@ private:
int fetch_and_submit_single_log_(ObReplayStatus &replay_status,
ObReplayServiceSubmitTask *submit_task,
palf::LSN &cur_lsn,
palf::SCN &cur_log_submit_scn,
share::SCN &cur_log_submit_scn,
int64_t &log_size);
int fetch_pre_barrier_log_(ObReplayStatus &replay_status,
ObReplayServiceSubmitTask *submit_task,
@ -167,7 +170,7 @@ private:
const ObLogBaseHeader &header,
const char *log_buf,
const palf::LSN &cur_lsn,
const palf::SCN &cur_log_submit_scn,
const share::SCN &cur_log_submit_scn,
const int64_t log_size,
const bool is_raw_write);
bool is_tenant_out_of_memory_() const;
@ -213,7 +216,7 @@ private:
ObLSAdapter *ls_adapter_;
palf::PalfEnv *palf_env_;
ObILogAllocator *allocator_;
palf::SCN replayable_point_;
share::SCN replayable_point_;
// 考虑到迁出迁入场景, 不能只通过map管理replay status的生命周期
common::ObLinearHashMap<share::ObLSID, ObReplayStatus*> replay_status_map_;
int64_t pending_replay_log_size_;

1
src/logservice/replayservice/ob_replay_handler.cpp
View File

@ -17,6 +17,7 @@
namespace oceanbase
{
using namespace palf;
using namespace share;
namespace logservice
{
ObReplayHandler::ObReplayHandler(storage::ObLS *ls)

4
src/logservice/replayservice/ob_replay_handler.h
View File

@ -24,7 +24,7 @@ namespace storage
{
class ObLS;
}
namespace palf
namespace share
{
class SCN;
}
@ -46,7 +46,7 @@ public:
const void *buffer,
const int64_t nbytes,
const palf::LSN &lsn,
const palf::SCN &scn);
const share::SCN &scn);
private:
typedef common::RWLock::WLockGuard WLockGuard;
typedef common::RWLock::RLockGuard RLockGuard;

3
src/logservice/replayservice/ob_replay_status.cpp
View File

@ -21,6 +21,7 @@ namespace oceanbase
{
using namespace common;
using namespace palf;
using namespace share;
namespace logservice
{
//---------------ObReplayServiceTask---------------//
@ -94,7 +95,7 @@ bool ObReplayServiceTask::need_replay_immediately() const
//---------------ObReplayServiceSubmitTask---------------//
int ObReplayServiceSubmitTask::init(const palf::LSN &base_lsn,
const palf::SCN &base_scn,
const SCN &base_scn,
PalfHandle *palf_handle,
ObReplayStatus *replay_status)
{

48
src/logservice/replayservice/ob_replay_status.h
View File

@ -17,7 +17,7 @@
#include "logservice/ob_log_base_header.h"
#include "logservice/ob_log_base_type.h"
#include "logservice/palf/lsn.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
#include "logservice/palf/palf_callback.h"
#include "logservice/palf/palf_iterator.h"
#include "logservice/palf/palf_handle.h"
@ -64,7 +64,7 @@ struct LSReplayStat
palf::LSN end_lsn_;
bool enabled_;
palf::LSN unsubmitted_lsn_;
palf::SCN unsubmitted_scn_;
share::SCN unsubmitted_scn_;
int64_t pending_cnt_;
TO_STRING_KV(K(ls_id_),
@ -112,7 +112,7 @@ public:
int init(const share::ObLSID &ls_id,
const ObLogBaseHeader &header,
const palf::LSN &lsn,
const palf::SCN &scn,
const share::SCN &scn,
const int64_t log_size,
const bool is_raw_write,
void *log_buf);
@ -123,7 +123,7 @@ public:
share::ObLSID ls_id_;
ObLogBaseType log_type_;
palf::LSN lsn_;
palf::SCN scn_;
share::SCN scn_;
bool is_pre_barrier_;
bool is_post_barrier_;
int64_t log_size_;
@ -253,7 +253,7 @@ public:
destroy();
}
int init(const palf::LSN &base_lsn,
const palf::SCN &base_scn,
const share::SCN &base_scn,
palf::PalfHandle *palf_handle,
ObReplayStatus *replay_status);
void reset() override;
@ -263,15 +263,15 @@ public:
// 迭代器是否迭代到终点
bool has_remained_submit_log();
// 不允许回退
int update_next_to_submit_log_info(const palf::LSN &lsn, const palf::SCN &scn);
int update_next_to_submit_log_info(const palf::LSN &lsn, const share::SCN &scn);
int update_next_to_submit_lsn(const palf::LSN &lsn);
int update_next_to_submit_scn_allow_equal(const palf::SCN &scn);
int update_next_to_submit_scn_allow_equal(const share::SCN &scn);
int update_committed_end_offset(const palf::LSN &lsn);
int get_next_to_submit_log_info(palf::LSN &lsn, palf::SCN &scn) const;
int get_next_to_submit_log_info(palf::LSN &lsn, share::SCN &scn) const;
int get_committed_end_lsn(palf::LSN &lsn) const;
int get_base_lsn(palf::LSN &lsn) const;
int get_base_scn(palf::SCN &scn) const;
int need_skip(const palf::SCN &scn, bool &need_skip);
int get_base_scn(share::SCN &scn) const;
int need_skip(const share::SCN &scn, bool &need_skip);
bool is_cached_replay_task_exist() const
{
return NULL != cache_replay_task_;
@ -292,7 +292,7 @@ public:
//释放内存,在废弃cache_replay_task时调用
void revert_cached_replay_task();
int get_log(const char *&buffer, int64_t &nbytes, palf::SCN &scn, palf::LSN &offset, bool &is_raw_write);
int get_log(const char *&buffer, int64_t &nbytes, share::SCN &scn, palf::LSN &offset, bool &is_raw_write);
int next_log();
// 以当前的终点作为新起点重置迭代器
int reset_iterator(palf::PalfHandle &palf_handle,
@ -306,24 +306,24 @@ public:
K(base_scn_));
private:
int update_next_to_submit_lsn_(const palf::LSN &lsn);
int update_next_to_submit_scn_(const palf::SCN &scn);
int update_next_to_submit_scn_allow_equal_(const palf::SCN &scn);
int update_next_to_submit_scn_(const share::SCN &scn);
int update_next_to_submit_scn_allow_equal_(const share::SCN &scn);
int update_committed_end_lsn_(const palf::LSN &lsn);
void set_next_to_submit_log_info_(const palf::LSN &lsn, const palf::SCN &scn);
int get_next_to_submit_log_info_(palf::LSN &lsn, palf::SCN &scn) const;
void set_next_to_submit_log_info_(const palf::LSN &lsn, const share::SCN &scn);
int get_next_to_submit_log_info_(palf::LSN &lsn, share::SCN &scn) const;
int get_base_lsn_(palf::LSN &lsn) const;
int get_base_scn_(palf::SCN &scn) const;
int get_base_scn_(share::SCN &scn) const;
private:
//location of next log after the last log that has already been submit to replay, consider as left side of iterator
palf::LSN next_to_submit_lsn_;
//location of the last log that need submit to replay, consider as right side of iterator
palf::LSN committed_end_lsn_;
palf::SCN next_to_submit_scn_;
share::SCN next_to_submit_scn_;
//initial log lsn when enable replay, for stat replay process
palf::LSN base_lsn_;
//initial log scn when enable replay, logs which scn small than this value should skip replay
palf::SCN base_scn_;
share::SCN base_scn_;
//for unittest, should be a member not pointer
palf::PalfBufferIterator iterator_;
//缓存需要重试的log replay task
@ -365,7 +365,7 @@ public:
queue_.push(p);
}
int get_min_unreplayed_log_info(palf::LSN &lsn,
palf::SCN &scn,
share::SCN &scn,
bool &is_queue_empty);
private:
Link *pop_()
@ -439,7 +439,7 @@ public:
void destroy();
public:
int enable(const palf::LSN &base_lsn,
const palf::SCN &base_scn);
const share::SCN &base_scn);
int disable();
// if is_enabled_ is false,
// means log stream will bedestructed and no logs need to replayed any more.
@ -493,9 +493,9 @@ public:
int get_ls_id(share::ObLSID &id);
int get_min_unreplayed_lsn(palf::LSN &lsn);
int get_min_unreplayed_scn(palf::SCN &scn);
int get_min_unreplayed_scn(share::SCN &scn);
int get_min_unreplayed_log_info(palf::LSN &lsn,
palf::SCN &scn);
share::SCN &scn);
int get_replay_process(int64_t &replayed_log_size, int64_t &unreplayed_log_size);
//提交日志检查barrier状态
int check_submit_barrier();
@ -547,11 +547,11 @@ public:
K(submit_log_task_));
private:
void set_next_to_submit_log_info_(const palf::LSN &lsn, const palf::SCN &scn);
void set_next_to_submit_log_info_(const palf::LSN &lsn, const share::SCN &scn);
int submit_task_to_replay_service_(ObReplayServiceTask &task);
// 注册回调并提交当前初始化的submit_log_task
int enable_(const palf::LSN &base_lsn,
const palf::SCN &base_scn);
const share::SCN &base_scn);
// 注销回调并清空任务
int disable_();

1
src/logservice/restoreservice/ob_fetch_log_task.cpp
View File

@ -20,6 +20,7 @@
namespace oceanbase
{
using namespace palf;
using namespace share;
namespace logservice
{
ObFetchLogTask::ObFetchLogTask(const share::ObLSID &id,

12
src/logservice/restoreservice/ob_fetch_log_task.h
View File

@ -16,7 +16,7 @@
#include "lib/utility/ob_print_utils.h"
#include "share/ob_ls_id.h" // ObLSID
#include "logservice/palf/lsn.h" // LSN
#include "logservice/palf/scn.h" // SCN
#include "share/scn.h" // SCN
#include <cstdint>
namespace oceanbase
{
@ -40,7 +40,7 @@ public:
public:
ObFetchLogTask(const share::ObLSID &id,
const palf::SCN &pre_scn,
const share::SCN &pre_scn,
const palf::LSN &lsn,
const int64_t size,
const int64_t proposal_id);
@ -49,7 +49,7 @@ public:
bool is_finish() const { return Status::FINISH == status_; }
bool is_to_end() const { return Status::TO_END == status_; }
void mark_to_end() { status_ = Status::TO_END; }
int update_cur_lsn_scn(const palf::LSN &lsn, const palf::SCN &max_submit_scn, const palf::SCN &max_fetch_scn);
int update_cur_lsn_scn(const palf::LSN &lsn, const share::SCN &max_submit_scn, const share::SCN &max_fetch_scn);
void set_to_end() { status_ = Status::TO_END; }
void set_stale() { status_ = Status::STALE; }
TO_STRING_KV(K_(id), K_(proposal_id), K_(pre_scn), K_(start_lsn), K_(cur_lsn), K_(end_lsn),
@ -59,12 +59,12 @@ public:
share::ObLSID id_;
// to distinguish stale tasks which is generated in previous leader
int64_t proposal_id_;
palf::SCN pre_scn_; // heuristic log scn to locate piece, may be imprecise one
share::SCN pre_scn_; // heuristic log scn to locate piece, may be imprecise one
palf::LSN start_lsn_;
palf::LSN cur_lsn_;
palf::LSN end_lsn_;
palf::SCN max_fetch_scn_; // 拉取日志最大log scn
palf::SCN max_submit_scn_; // 提交日志最大log cn
share::SCN max_fetch_scn_; // 拉取日志最大log scn
share::SCN max_submit_scn_; // 提交日志最大log cn
Status status_;
};

18
src/logservice/restoreservice/ob_log_archive_piece_mgr.cpp
View File

@ -51,7 +51,7 @@ void ObLogArchivePieceContext::RoundContext::reset()
bool ObLogArchivePieceContext::RoundContext::is_valid() const
{
return ((RoundContext::State::ACTIVE == state_ && end_scn_ == palf::SCN::max_scn())
return ((RoundContext::State::ACTIVE == state_ && end_scn_ == SCN::max_scn())
|| (RoundContext::State::STOP == state_ && end_scn_ > start_scn_))
&& round_id_ > 0
&& start_scn_.is_valid()
@ -195,7 +195,7 @@ int ObLogArchivePieceContext::init(const share::ObLSID &id,
return ret;
}
int ObLogArchivePieceContext::get_piece(const palf::SCN &pre_scn,
int ObLogArchivePieceContext::get_piece(const SCN &pre_scn,
const palf::LSN &start_lsn,
int64_t &dest_id,
int64_t &round_id,
@ -266,7 +266,7 @@ int ObLogArchivePieceContext::update_file_info(const int64_t dest_id,
return ret;
}
int ObLogArchivePieceContext::get_round_(const palf::SCN &start_scn)
int ObLogArchivePieceContext::get_round_(const SCN &start_scn)
{
int ret = OB_SUCCESS;
int64_t round_id = 0;
@ -385,7 +385,7 @@ int ObLogArchivePieceContext::load_round_(const int64_t round_id, RoundContext &
return ret;
}
int ObLogArchivePieceContext::get_piece_(const palf::SCN &scn,
int ObLogArchivePieceContext::get_piece_(const SCN &scn,
const palf::LSN &lsn,
const int64_t file_id,
int64_t &dest_id,
@ -417,7 +417,7 @@ int ObLogArchivePieceContext::get_piece_(const palf::SCN &scn,
return ret;
}
int ObLogArchivePieceContext::switch_round_if_need_(const palf::SCN &scn, const palf::LSN &lsn)
int ObLogArchivePieceContext::switch_round_if_need_(const SCN &scn, const palf::LSN &lsn)
{
int ret = OB_SUCCESS;
RoundOp op = RoundOp::NONE;
@ -662,7 +662,7 @@ int ObLogArchivePieceContext::check_round_exist_(const int64_t round_id, bool &e
return ret;
}
int ObLogArchivePieceContext::switch_piece_if_need_(const int64_t file_id, const palf::SCN &scn, const palf::LSN &lsn)
int ObLogArchivePieceContext::switch_piece_if_need_(const int64_t file_id, const SCN &scn, const palf::LSN &lsn)
{
int ret = OB_SUCCESS;
PieceOp op = PieceOp::NONE;
@ -760,7 +760,7 @@ void ObLogArchivePieceContext::check_if_switch_piece_(const int64_t file_id,
// 1. inner_piece_context.round_id == round_context.round_id 说明依然消费当前round, 如果当前piece_id有效, 并且该piece依然active, 则继续刷新该piece
// 2. inner_piece_context.round_id == round_context.round_id 并且piece状态为FROZEN或者EMPTY, 非预期错误
// 3. inner_piece_context.round_id != round_context.round_id, 需要由scn以及round piece范围, 决定piece id
int ObLogArchivePieceContext::get_cur_piece_info_(const palf::SCN &scn, const palf::LSN &lsn)
int ObLogArchivePieceContext::get_cur_piece_info_(const SCN &scn, const palf::LSN &lsn)
{
int ret = OB_SUCCESS;
int64_t piece_id = 0;
@ -781,7 +781,7 @@ int ObLogArchivePieceContext::get_cur_piece_info_(const palf::SCN &scn, const pa
return ret;
}
int ObLogArchivePieceContext::cal_load_piece_id_(const palf::SCN &scn, int64_t &piece_id)
int ObLogArchivePieceContext::cal_load_piece_id_(const SCN &scn, int64_t &piece_id)
{
int ret = OB_SUCCESS;
const int64_t base_piece_id = cal_piece_id_(scn);
@ -955,7 +955,7 @@ int ObLogArchivePieceContext::backward_piece_()
return ret;
}
int64_t ObLogArchivePieceContext::cal_piece_id_(const palf::SCN &scn) const
int64_t ObLogArchivePieceContext::cal_piece_id_(const SCN &scn) const
{
return share::ObArchivePiece(scn, round_context_.piece_switch_interval_,
round_context_.base_piece_scn_, round_context_.base_piece_id_)

24
src/logservice/restoreservice/ob_log_archive_piece_mgr.h
View File

@ -17,7 +17,7 @@
#include "lib/ob_errno.h"
#include "lib/utility/ob_print_utils.h"
#include "logservice/palf/lsn.h"
#include "logservice/palf/scn.h"
#include "share/scn.h"
#include "share/backup/ob_backup_struct.h"
#include "share/ob_ls_id.h"
#include "ob_log_restore_define.h"
@ -47,7 +47,7 @@ public:
void reset();
int get_piece(const palf::SCN &pre_scn,
int get_piece(const share::SCN &pre_scn,
const palf::LSN &start_lsn,
int64_t &dest_id,
int64_t &round_id,
@ -101,14 +101,14 @@ private:
State state_;
int64_t round_id_;
palf::SCN start_scn_; // 该轮次最小日志SCN
palf::SCN end_scn_; // 该轮次最大日志SCN, 非STOP状态该时间为SCN_MAX
share::SCN start_scn_; // 该轮次最小日志SCN
share::SCN end_scn_; // 该轮次最大日志SCN, 非STOP状态该时间为SCN_MAX
int64_t min_piece_id_; // 该round最小piece, 随着归档数据回收, 该值可能增大
int64_t max_piece_id_; // 该round最大piece, ACTIVE round该值可能增大
int64_t base_piece_id_;
int64_t piece_switch_interval_;//us
palf::SCN base_piece_scn_;
share::SCN base_piece_scn_;
RoundContext() { reset(); }
~RoundContext() { reset(); }
@ -169,7 +169,7 @@ private:
};
private:
int get_piece_(const palf::SCN &scn,
int get_piece_(const share::SCN &scn,
const palf::LSN &lsn,
const int64_t file_id,
int64_t &dest_id,
@ -185,10 +185,10 @@ private:
virtual int get_round_range_();
// 根据时间戳定位round
virtual int get_round_(const palf::SCN &scn);
virtual int get_round_(const share::SCN &scn);
// 如果round不满足数据需求, 支持切round
int switch_round_if_need_(const palf::SCN &scn, const palf::LSN &lsn);
int switch_round_if_need_(const share::SCN &scn, const palf::LSN &lsn);
void check_if_switch_round_(const palf::LSN &lsn, RoundOp &op);
bool is_max_round_done_(const palf::LSN &lsn) const;
@ -210,11 +210,11 @@ private:
int backward_round_();
// 当piece不匹配, 需要切piece
int switch_piece_if_need_(const int64_t file_id, const palf::SCN &scn, const palf::LSN &lsn);
int switch_piece_if_need_(const int64_t file_id, const share::SCN &scn, const palf::LSN &lsn);
void check_if_switch_piece_(const int64_t file_id, const palf::LSN &lsn, PieceOp &op);
// load当前piece信息, 包括piece内文件范围, LSN范围
int get_cur_piece_info_(const palf::SCN &scn, const palf::LSN &lsn);
int get_cur_piece_info_(const share::SCN &scn, const palf::LSN &lsn);
virtual int get_piece_meta_info_(const int64_t piece_id);
int get_ls_inner_piece_info_(const share::ObLSID &id, const int64_t dest_id, const int64_t round_id,
const int64_t piece_id, palf::LSN &min_lsn, palf::LSN &max_lsn, bool &exist);
@ -222,9 +222,9 @@ private:
int forward_piece_();
int backward_piece_();
int cal_load_piece_id_(const palf::SCN &scn, int64_t &piece_id);
int cal_load_piece_id_(const share::SCN &scn, int64_t &piece_id);
int64_t cal_piece_id_(const palf::SCN &scn) const;
int64_t cal_piece_id_(const share::SCN &scn) const;
virtual int get_min_lsn_in_piece_();
int64_t cal_archive_file_id_(const palf::LSN &lsn) const;

20
src/logservice/restoreservice/ob_log_restore_handler.h
View File

@ -71,15 +71,15 @@ public:
// @param[in], proposal_id, global monotonically increasing id
virtual void switch_role(const common::ObRole &role, const int64_t proposal_id) override;
// interface only for restore_and_standby branch, TODO should delete after no cut log by shuning.tsn
int get_upper_limit_scn(palf::SCN &scn) const;
int get_upper_limit_scn(share::SCN &scn) const;
// get the log ts of the max restored log
int get_max_restore_scn(palf::SCN &scn) const;
int get_max_restore_scn(share::SCN &scn) const;
// @brief add log fetch source of phyiscal backups
int add_source(share::DirArray &array, const palf::SCN &end_scn);
int add_source(share::DirArray &array, const share::SCN &end_scn);
// for stale TODO delete after log archive source ready
int add_source(logservice::DirArray &array, const palf::SCN &end_scn);
int add_source(share::ObBackupDest &dest, const palf::SCN &end_scn);
int add_source(const ObAddr &addr, const palf::SCN &end_scn);
int add_source(logservice::DirArray &array, const share::SCN &end_scn);
int add_source(share::ObBackupDest &dest, const share::SCN &end_scn);
int add_source(const ObAddr &addr, const share::SCN &end_scn);
// @brief As restore handler maybe destroyed, log source should be copied out
void deep_copy_source(ObRemoteSourceGuard &guard);
// @brief raw write logs to the pointed offset of palf
@ -99,7 +99,7 @@ public:
// @param[in] is_to_end, log fetch is to end, only for physical restore
// @param[out] is_stale, fetch log task is stale due to cur_proposal_id is changed
int update_fetch_log_progress(const int64_t id, const int64_t proposal_id, const LSN &max_fetch_lsn,
const palf::SCN &max_submit_scn, const bool is_finish, const bool is_to_end, bool &is_stale);
const share::SCN &max_submit_scn, const bool is_finish, const bool is_to_end, bool &is_stale);
int update_location_info(ObRemoteLogParent *source);
// @brief check if restore finish
// return true only if in restore state and all replicas have restore and replay finish
@ -108,7 +108,7 @@ public:
// param[in], const share::ObLSID, the identification of LS
// param[in], int64_t, the commit ts of LS
// @param[out], int64_t, the restore upper ts if restore to the end
int get_restore_sync_scn(const share::ObLSID &id, palf::SCN &scn);
int get_restore_sync_scn(const share::ObLSID &id, share::SCN &scn);
void mark_error(share::ObTaskId &trace_id, const int ret_code);
int get_restore_error(share::ObTaskId &trace_id, int &ret_code, bool &error_exist);
TO_STRING_KV(K_(is_inited), K_(is_in_stop_state), K_(id), K_(proposal_id), K_(role), KP_(parent), K_(context));
@ -116,8 +116,8 @@ public:
private:
bool is_valid() const;
void alloc_source(const share::ObLogArchiveSourceType &type);
int check_replay_done_(const palf::SCN &scn, bool &done);
int check_replica_replay_done_(const palf::SCN &scn, common::ObMemberList &member_list, bool &done);
int check_replay_done_(const share::SCN &scn, bool &done);
int check_replica_replay_done_(const share::SCN &scn, common::ObMemberList &member_list, bool &done);
int check_member_list_change_(common::ObMemberList &member_list, bool &member_list_change);
private:

16
src/logservice/restoreservice/ob_remote_data_generator.h
View File

@ -26,7 +26,7 @@
#include "logservice/palf/log_iterator_storage.h" // MemoryStorage
#include "logservice/palf/palf_iterator.h" // MemPalfGroupBufferIterator
#include "logservice/palf/lsn.h" // LSN
#include "logservice/palf/scn.h" // SCN
#include "share/scn.h" // SCN
#include "ob_remote_log_source.h" // Parent
#include "ob_log_restore_rpc_define.h" // ObRemoteFetchLogResponse
@ -73,7 +73,7 @@ public:
const ObLSID &id,
const LSN &start_lsn,
const LSN &end_lsn,
const palf::SCN &end_scn);
const share::SCN &end_scn);
virtual ~RemoteDataGenerator();
public:
@ -92,7 +92,7 @@ protected:
ObLSID id_;
LSN start_lsn_;
LSN next_fetch_lsn_;
palf::SCN end_scn_;
share::SCN end_scn_;
LSN end_lsn_;
bool to_end_;
LSN max_consumed_lsn_;
@ -108,7 +108,7 @@ public:
const ObLSID &id,
const LSN &start_lsn,
const LSN &end_lsn,
const palf::SCN &end_scn,
const share::SCN &end_scn,
const ObAddr &server);
virtual ~ServiceDataGenerator();
@ -134,11 +134,11 @@ class LocationDataGenerator : public RemoteDataGenerator
static const int64_t MAX_DATA_BUF_LEN = 128 * 1024 * 1024L; // 128M
public:
LocationDataGenerator(const uint64_t tenant_id,
const palf::SCN &pre_scn,
const share::SCN &pre_scn,
const ObLSID &id,
const LSN &start_lsn,
const LSN &end_lsn,
const palf::SCN &end_scn,
const share::SCN &end_scn,
share::ObBackupDest *dest,
ObLogArchivePieceContext *piece_context);
~LocationDataGenerator();
@ -157,7 +157,7 @@ private:
palf::LSN &lsn,
share::ObBackupPath &piece_path);
private:
palf::SCN pre_scn_;
share::SCN pre_scn_;
palf::LSN base_lsn_;
int64_t data_len_;
char data_[MAX_DATA_BUF_LEN];
@ -185,7 +185,7 @@ public:
const LSN &start_lsn,
const LSN &end_lsn,
const DirArray &array,
const palf::SCN &end_scn,
const share::SCN &end_scn,
const int64_t piece_index,
const int64_t min_file_id,
const int64_t max_file_id);

8
src/logservice/restoreservice/ob_remote_fetch_log.h
View File

@ -21,6 +21,7 @@ namespace oceanbase
namespace share
{
class ObLSID;
class SCN;
}
namespace storage
@ -32,7 +33,6 @@ class ObLSService;
namespace palf
{
struct LSN;
class SCN;
}
namespace logservice
@ -61,9 +61,9 @@ private:
int check_need_schedule_(ObLS &ls, bool &need_schedule, int64_t &proposal_id, LSN &lsn, int64_t &last_fetch_ts);
int get_fetch_log_max_lsn_(ObLS &ls, palf::LSN &max_lsn);
int get_fetch_log_base_lsn_(ObLS &ls, const LSN &max_fetch_lsn, const int64_t last_fetch_ts,
palf::SCN &scn, LSN &lsn, int64_t &size);
int get_palf_base_lsn_scn_(ObLS &ls, LSN &lsn, palf::SCN &scn);
int submit_fetch_log_task_(ObLS &ls, const palf::SCN &scn, const LSN &lsn, const int64_t size, const int64_t proposal_id);
share::SCN &scn, LSN &lsn, int64_t &size);
int get_palf_base_lsn_scn_(ObLS &ls, LSN &lsn, share::SCN &scn);
int submit_fetch_log_task_(ObLS &ls, const share::SCN &scn, const LSN &lsn, const int64_t size, const int64_t proposal_id);
private:
bool inited_;

2
src/logservice/restoreservice/ob_remote_fetch_log_worker.cpp
View File

@ -27,7 +27,7 @@
#include "logservice/palf/log_group_entry.h" // LogGroupEntry
#include "logservice/palf/lsn.h" // LSN
#include "ob_log_restore_service.h" // ObLogRestoreService
#include "logservice/palf/scn.h" // SCN
#include "share/scn.h" // SCN
#include "ob_fetch_log_task.h" // ObFetchLogTask
#include "ob_remote_log_source.h" // ObRemoteLogParent
#include "ob_remote_log_iterator.h" // ObRemoteLogIterator

14
src/logservice/restoreservice/ob_remote_fetch_log_worker.h
View File

@ -25,11 +25,15 @@ namespace storage
class ObLSService;
}
namespace share
{
class SCN;
}
namespace palf
{
struct LSN;
class LogGroupEntry;
class SCN;
}
namespace logservice
@ -65,12 +69,12 @@ private:
void run1();
void do_thread_task_();
int handle(ObFetchLogTask &task);
int get_upper_limit_scn_(const ObLSID &id, palf::SCN &scn);
int submit_entries_(const ObLSID &id, const palf::SCN &upper_limit_scn, ObRemoteLogIterator &iter, palf::SCN &max_submit_scn);
int cut_group_log_(const ObLSID &id, const LSN &lsn, const palf::SCN &cut_scn, palf::LogGroupEntry &entry);
int get_upper_limit_scn_(const ObLSID &id, share::SCN &scn);
int submit_entries_(const ObLSID &id, const share::SCN &upper_limit_scn, ObRemoteLogIterator &iter, share::SCN &max_submit_scn);
int cut_group_log_(const ObLSID &id, const LSN &lsn, const share::SCN &cut_scn, palf::LogGroupEntry &entry);
int get_pre_accum_checksum_(const ObLSID &id, const LSN &lsn, int64_t &pre_accum_checksum);
int submit_log_(const ObLSID &id, const LSN &lsn, char *buf, const int64_t buf_size);
void mark_if_to_end_(ObFetchLogTask &task, const palf::SCN &upper_limit_scn, const palf::SCN &scn);
void mark_if_to_end_(ObFetchLogTask &task, const share::SCN &upper_limit_scn, const share::SCN &scn);
int try_retire_(ObFetchLogTask *&task);
void try_update_location_info_(const ObFetchLogTask &task, ObRemoteLogIterator &iter);

2
src/logservice/restoreservice/ob_remote_location_adaptor.cpp
View File

@ -18,7 +18,7 @@
#include "share/ob_ls_id.h" // ObLSID
#include "common/ob_role.h" // ObRole
#include "logservice/palf_handle_guard.h" // PalfHandleGuard
#include "logservice/palf/scn.h" // palf::SCN
#include "share/scn.h" // SCN
#include "share/restore/ob_ls_restore_status.h" // ObLSRestoreStatus
#include "share/restore/ob_log_archive_source.h" // ObLogArchiveSourceItem
#include "share/restore/ob_log_archive_source_mgr.h" // ObLogArchiveSourceMgr

2
src/logservice/restoreservice/ob_remote_log_iterator.cpp
View File

@ -75,7 +75,7 @@ ObRemoteLogIterator::~ObRemoteLogIterator()
int ObRemoteLogIterator::init(const uint64_t tenant_id,
const ObLSID &id,
const palf::SCN &pre_scn,
const SCN &pre_scn,
const LSN &start_lsn,
const LSN &end_lsn)
{

Some files were not shown because too many files have changed in this diff Show More